Artifactspublic-v0

curl

master
ply4 git calls214 ms294 KiB2 cachedprotocol v2
737 lines35 KiBRaw
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
21# SPDX-License-Identifier: curl
23###########################################################################
25import difflib
26import filecmp
27import logging
28import os
29import re
30import sys
32import pytest
33from testenv import CurlClient, Env, LocalClient
35log = logging.getLogger(__name__)
38class TestDownload:
40 @pytest.fixture(autouse=True, scope='class')
41 @classmethod
42 def _class_scope(cls, env, httpd):
43 indir = httpd.docs_dir
44 env.make_data_file(indir=indir, fname="data-0k", fsize=0)
45 env.make_data_file(indir=indir, fname="data-10k", fsize=10 * 1024)
46 env.make_data_file(indir=indir, fname="data-100k", fsize=100 * 1024)
47 env.make_data_file(indir=indir, fname="data-1m", fsize=1024 * 1024)
48 env.make_data_file(indir=indir, fname="data-10m", fsize=10 * 1024 * 1024)
49 env.make_data_file(indir=indir, fname="data-50m", fsize=50 * 1024 * 1024)
50 env.make_data_gzipbomb(indir=indir, fname="bomb-100m.txt", fsize=100 * 1024 * 1024)
52 # download 1 file
53 @pytest.mark.parametrize("proto", Env.http_protos())
54 @pytest.mark.parametrize("docname", ['data.json', 'data-0k', 'data-10k', 'data-100k'])
55 def test_02_01_download_1(self, env: Env, httpd, nghttpx, proto, docname):
56 curl = CurlClient(env=env)
57 url = f'https://{env.authority_for(env.domain1, proto)}/{docname}'
58 r = curl.http_download(urls=[url], alpn_proto=proto)
59 r.check_response(http_status=200)
61 # download 2 files
62 @pytest.mark.parametrize("proto", Env.http_protos())
63 def test_02_02_download_2(self, env: Env, httpd, nghttpx, proto):
64 curl = CurlClient(env=env)
65 url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]'
66 r = curl.http_download(urls=[url], alpn_proto=proto)
67 r.check_response(http_status=200, count=2)
69 # download 100 files sequentially
70 @pytest.mark.parametrize("proto", Env.http_protos())
71 def test_02_03_download_sequential(self, env: Env, httpd, nghttpx, proto):
72 if (proto == 'http/1.1' or proto == 'h2') and env.curl_uses_lib('mbedtls') and \
73 sys.platform.startswith('darwin') and env.ci_run:
74 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
75 count = 10
76 curl = CurlClient(env=env)
77 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
78 r = curl.http_download(urls=[urln], alpn_proto=proto)
79 r.check_response(http_status=200, count=count, connect_count=1)
81 # download 100 files parallel
82 @pytest.mark.parametrize("proto", Env.http_protos())
83 def test_02_04_download_parallel(self, env: Env, httpd, nghttpx, proto):
84 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
85 sys.platform.startswith('darwin') and env.ci_run:
86 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
87 count = 10
88 max_parallel = 5
89 curl = CurlClient(env=env)
90 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
91 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
92 '--parallel', '--parallel-max', f'{max_parallel}'
93 ])
94 r.check_response(http_status=200, count=count)
95 if proto == 'http/1.1':
96 # http/1.1 parallel transfers will open multiple connections
97 assert r.total_connects > 1, r.dump_logs()
98 else:
99 # http2 parallel transfers will use one connection (common limit is 100)
100 assert r.total_connects == 1, r.dump_logs()
102 # download 500 files sequential
103 @pytest.mark.parametrize("proto", Env.http_protos())
104 def test_02_05_download_many_sequential(self, env: Env, httpd, nghttpx, proto):
105 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
106 sys.platform.startswith('darwin') and env.ci_run:
107 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
108 count = 200
109 curl = CurlClient(env=env)
110 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
111 r = curl.http_download(urls=[urln], alpn_proto=proto)
112 r.check_response(http_status=200, count=count)
113 if proto == 'http/1.1':
114 # http/1.1 parallel transfers will open multiple connections
115 assert r.total_connects > 1, r.dump_logs()
116 else:
117 # http2 parallel transfers will use one connection (common limit is 100)
118 assert r.total_connects == 1, r.dump_logs()
120 # download 500 files parallel
121 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
122 def test_02_06_download_many_parallel(self, env: Env, httpd, nghttpx, proto):
123 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
124 sys.platform.startswith('darwin') and env.ci_run:
125 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
126 count = 200
127 max_parallel = 50
128 curl = CurlClient(env=env)
129 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[000-{count-1}]'
130 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
131 '--parallel', '--parallel-max', f'{max_parallel}'
132 ])
133 r.check_response(http_status=200, count=count, connect_count=1)
135 # download files parallel, check connection reuse/multiplex
136 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
137 def test_02_07_download_reuse(self, env: Env, httpd, nghttpx, proto):
138 count = 200
139 curl = CurlClient(env=env)
140 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
141 r = curl.http_download(urls=[urln], alpn_proto=proto,
142 with_stats=True, extra_args=[
143 '--parallel', '--parallel-max', '200'
144 ])
145 r.check_response(http_status=200, count=count)
146 # should have used at most 2 connections only (test servers allow 100 req/conn)
147 # it may be 1 on slow systems where request are answered faster than
148 # curl can exhaust the capacity or if curl runs with address-sanitizer speed
149 assert r.total_connects <= 2, "h2 should use fewer connections here"
151 # download files parallel with http/1.1, check connection not reused
152 @pytest.mark.parametrize("proto", ['http/1.1'])
153 def test_02_07b_download_reuse(self, env: Env, httpd, nghttpx, proto):
154 count = 6
155 curl = CurlClient(env=env)
156 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
157 r = curl.http_download(urls=[urln], alpn_proto=proto,
158 with_stats=True, extra_args=[
159 '--parallel'
160 ])
161 r.check_response(count=count, http_status=200)
162 # http/1.1 should have used count connections
163 assert r.total_connects == count, "http/1.1 should use this many connections"
165 @pytest.mark.parametrize("proto", Env.http_protos())
166 def test_02_08_1MB_serial(self, env: Env, httpd, nghttpx, proto):
167 count = 5
168 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]'
169 curl = CurlClient(env=env)
170 r = curl.http_download(urls=[urln], alpn_proto=proto)
171 r.check_response(count=count, http_status=200)
173 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
174 def test_02_09_1MB_parallel(self, env: Env, httpd, nghttpx, proto):
175 count = 5
176 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]'
177 curl = CurlClient(env=env)
178 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
179 '--parallel'
180 ])
181 r.check_response(count=count, http_status=200)
183 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests")
184 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs")
185 @pytest.mark.parametrize("proto", Env.http_protos())
186 def test_02_10_10MB_serial(self, env: Env, httpd, nghttpx, proto):
187 count = 3
188 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]'
189 curl = CurlClient(env=env)
190 r = curl.http_download(urls=[urln], alpn_proto=proto)
191 r.check_response(count=count, http_status=200)
193 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests")
194 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs")
195 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
196 def test_02_11_10MB_parallel(self, env: Env, httpd, nghttpx, proto):
197 count = 3
198 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]'
199 curl = CurlClient(env=env)
200 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
201 '--parallel'
202 ])
203 r.check_response(count=count, http_status=200)
205 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
206 def test_02_12_head_serial_https(self, env: Env, httpd, nghttpx, proto):
207 count = 5
208 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]'
209 curl = CurlClient(env=env)
210 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
211 '--head'
212 ])
213 r.check_response(count=count, http_status=200)
215 @pytest.mark.parametrize("proto", ['h2'])
216 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
217 def test_02_13_head_serial_h2c(self, env: Env, httpd, nghttpx, proto):
218 count = 5
219 urln = f'http://{env.domain1}:{env.http_port}/data-10m?[0-{count-1}]'
220 curl = CurlClient(env=env)
221 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
222 '--head', '--http2-prior-knowledge', '--fail-early'
223 ])
224 r.check_response(count=count, http_status=200)
226 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
227 def test_02_14_not_found(self, env: Env, httpd, nghttpx, proto):
228 count = 5
229 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]'
230 curl = CurlClient(env=env)
231 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
232 '--parallel'
233 ])
234 r.check_stats(count=count, http_status=404, exitcode=0,
235 remote_port=env.port_for(alpn_proto=proto),
236 remote_ip='127.0.0.1')
238 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
239 def test_02_15_fail_not_found(self, env: Env, httpd, nghttpx, proto):
240 count = 5
241 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]'
242 curl = CurlClient(env=env)
243 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
244 '--fail'
245 ])
246 r.check_stats(count=count, http_status=404, exitcode=22,
247 remote_port=env.port_for(alpn_proto=proto),
248 remote_ip='127.0.0.1')
250 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests")
251 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
252 def test_02_20_h2_small_frames(self, env: Env, httpd, configures_httpd):
253 # Test case to reproduce content corruption as observed in
254 # https://github.com/curl/curl/issues/10525
255 # To reliably reproduce, we need an Apache httpd that supports
256 # setting smaller frame sizes. This is not released yet, we
257 # test if it works and back out if not.
258 httpd.set_extra_config(env.domain1, lines=[
259 'H2MaxDataFrameLen 1024',
260 ])
261 if not httpd.reload_if_config_changed():
262 pytest.skip('H2MaxDataFrameLen not supported')
263 # ok, make 100 downloads with 2 parallel running and they
264 # are expected to stumble into the issue when using `lib/http2.c`
265 # from curl 7.88.0
266 count = 5
267 urln = f'https://{env.authority_for(env.domain1, "h2")}/data-1m?[0-{count-1}]'
268 curl = CurlClient(env=env)
269 r = curl.http_download(urls=[urln], alpn_proto="h2", extra_args=[
270 '--parallel', '--parallel-max', '2'
271 ])
272 r.check_response(count=count, http_status=200)
273 srcfile = os.path.join(httpd.docs_dir, 'data-1m')
274 self.check_downloads(curl, srcfile, count)
276 # download serial via lib client, pause/resume at different offsets
277 @pytest.mark.parametrize("pause_offset", [0, 10 * 1024, 100 * 1023, 640000])
278 @pytest.mark.parametrize("proto", Env.http_protos())
279 def test_02_21_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset):
280 count = 2
281 docname = 'data-1m'
282 url = f'https://localhost:{env.https_port}/{docname}'
283 client = LocalClient(name='cli_hx_download', env=env)
284 if not client.exists():
285 pytest.skip(f'example client not built: {client.name}')
286 r = client.run(args=[
287 '-n', f'{count}', '-P', f'{pause_offset}',
288 '-C', env.ca.cert_file, '-V', proto, url
289 ])
290 r.check_exit_code(0)
291 srcfile = os.path.join(httpd.docs_dir, docname)
292 self.check_downloads(client, srcfile, count)
294 # download via lib client, several at a time, pause/resume
295 @pytest.mark.parametrize("pause_offset", [100 * 1023])
296 @pytest.mark.parametrize("proto", Env.http_protos())
297 def test_02_22_lib_parallel_resume(self, env: Env, httpd, nghttpx, proto, pause_offset):
298 count = 2
299 max_parallel = 5
300 docname = 'data-10m'
301 url = f'https://localhost:{env.https_port}/{docname}'
302 client = LocalClient(name='cli_hx_download', env=env)
303 if not client.exists():
304 pytest.skip(f'example client not built: {client.name}')
305 r = client.run(args=[
306 '-n', f'{count}', '-m', f'{max_parallel}',
307 '-P', f'{pause_offset}', '-C', env.ca.cert_file,
308 '-V', proto, url
309 ])
310 r.check_exit_code(0)
311 srcfile = os.path.join(httpd.docs_dir, docname)
312 self.check_downloads(client, srcfile, count)
314 # download, several at a time, pause and abort paused
315 @pytest.mark.parametrize("proto", Env.http_protos())
316 def test_02_23a_lib_abort_paused(self, env: Env, httpd, nghttpx, proto):
317 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'):
318 pytest.skip("fails in CI, but works locally for unknown reasons")
319 count = 10
320 max_parallel = 5
321 pause_offset = 1024 * (64 if proto in Env.http_mplx_protos() else 12)
322 docname = 'data-1m'
323 url = f'https://localhost:{env.https_port}/{docname}'
324 client = LocalClient(name='cli_hx_download', env=env)
325 if not client.exists():
326 pytest.skip(f'example client not built: {client.name}')
327 r = client.run(args=[
328 '-n', f'{count}', '-m', f'{max_parallel}', '-a',
329 '-C', env.ca.cert_file,
330 '-P', f'{pause_offset}', '-V', proto, url
331 ])
332 r.check_exit_code(0)
333 srcfile = os.path.join(httpd.docs_dir, docname)
334 # downloads should be there, but not necessarily complete
335 self.check_downloads(client, srcfile, count, complete=False)
337 # download, several at a time, abort after n bytes
338 @pytest.mark.parametrize("proto", Env.http_protos())
339 def test_02_23b_lib_abort_offset(self, env: Env, httpd, nghttpx, proto):
340 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'):
341 pytest.skip("fails in CI, but works locally for unknown reasons")
342 count = 10
343 max_parallel = 5
344 abort_offset = 1024 * (64 if proto in Env.http_mplx_protos() else 12)
345 docname = 'data-1m'
346 url = f'https://localhost:{env.https_port}/{docname}'
347 client = LocalClient(name='cli_hx_download', env=env)
348 if not client.exists():
349 pytest.skip(f'example client not built: {client.name}')
350 r = client.run(args=[
351 '-n', f'{count}', '-m', f'{max_parallel}', '-a',
352 '-C', env.ca.cert_file,
353 '-A', f'{abort_offset}', '-V', proto, url
354 ])
355 r.check_exit_code(42) # CURLE_ABORTED_BY_CALLBACK
356 srcfile = os.path.join(httpd.docs_dir, docname)
357 # downloads should be there, but not necessarily complete
358 self.check_downloads(client, srcfile, count, complete=False)
360 # download, several at a time, abort after n bytes
361 @pytest.mark.parametrize("proto", Env.http_protos())
362 def test_02_23c_lib_fail_offset(self, env: Env, httpd, nghttpx, proto):
363 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'):
364 pytest.skip("fails in CI, but works locally for unknown reasons")
365 count = 10
366 max_parallel = 5
367 fail_offset = 1024 * (64 if proto in Env.http_mplx_protos() else 12)
368 docname = 'data-1m'
369 url = f'https://localhost:{env.https_port}/{docname}'
370 client = LocalClient(name='cli_hx_download', env=env)
371 if not client.exists():
372 pytest.skip(f'example client not built: {client.name}')
373 r = client.run(args=[
374 '-n', f'{count}', '-m', f'{max_parallel}', '-a',
375 '-C', env.ca.cert_file,
376 '-F', f'{fail_offset}', '-V', proto, url
377 ])
378 r.check_exit_code(23) # CURLE_WRITE_ERROR
379 srcfile = os.path.join(httpd.docs_dir, docname)
380 # downloads should be there, but not necessarily complete
381 self.check_downloads(client, srcfile, count, complete=False)
383 # speed limited download
384 @pytest.mark.parametrize("proto", Env.http_protos())
385 def test_02_24_speed_limit(self, env: Env, httpd, nghttpx, proto):
386 if proto == 'h3' and not env.curl_uses_lib('ngtcp2'):
387 pytest.skip("precise h3 rate limits only with ngtcp2")
388 count = 1
389 url = f'https://{env.authority_for(env.domain1, proto)}/data-1m'
390 curl = CurlClient(env=env)
391 speed_limit = 512 * 1024
392 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
393 '--limit-rate', f'{speed_limit}'
394 ])
395 r.check_response(count=count, http_status=200)
396 dl_speed = r.stats[0]['speed_download']
397 assert dl_speed <= (speed_limit * 1.1), f'{r.stats[0]}'
399 # make extreme parallel h2 upgrades, check invalid conn reuse
400 # before protocol switch has happened
401 def test_02_25_h2_upgrade_x(self, env: Env, httpd):
402 url = f'http://localhost:{env.http_port}/data-100k'
403 client = LocalClient(name='cli_h2_upgrade_extreme', env=env, timeout=15)
404 if not client.exists():
405 pytest.skip(f'example client not built: {client.name}')
406 r = client.run(args=[url])
407 assert r.exit_code == 0, f'{client.dump_logs()}'
409 # Special client that tests TLS session reuse in parallel transfers
410 # TODO: uses a single connection for h2/h3. Not sure how to prevent that
411 @pytest.mark.parametrize("proto", Env.http_protos())
412 def test_02_26_session_shared_reuse(self, env: Env, proto, httpd, nghttpx):
413 url = f'https://{env.authority_for(env.domain1, proto)}/data-100k'
414 client = LocalClient(name='cli_tls_session_reuse', env=env)
415 if not client.exists():
416 pytest.skip(f'example client not built: {client.name}')
417 r = client.run(args=[url, proto])
418 r.check_exit_code(0)
420 # test on paused transfers, based on issue #11982
421 @pytest.mark.parametrize("proto", Env.http_protos())
422 def test_02_27a_paused_no_cl(self, env: Env, httpd, nghttpx, proto):
423 url = f'https://{env.authority_for(env.domain1, proto)}' \
424 '/curltest/tweak/?&chunks=6&chunk_size=8000'
425 client = LocalClient(env=env, name='cli_h2_pausing')
426 r = client.run(args=['-V', proto, url])
427 r.check_exit_code(0)
429 # test on paused transfers, based on issue #11982
430 @pytest.mark.parametrize("proto", Env.http_protos())
431 def test_02_27b_paused_no_cl(self, env: Env, httpd, nghttpx, proto):
432 url = f'https://{env.authority_for(env.domain1, proto)}' \
433 '/curltest/tweak/?error=502'
434 client = LocalClient(env=env, name='cli_h2_pausing')
435 r = client.run(args=['-V', proto, url])
436 r.check_exit_code(0)
438 # test on paused transfers, based on issue #11982
439 @pytest.mark.parametrize("proto", Env.http_protos())
440 def test_02_27c_paused_no_cl(self, env: Env, httpd, nghttpx, proto):
441 url = f'https://{env.authority_for(env.domain1, proto)}' \
442 '/curltest/tweak/?status=200&chunks=1&chunk_size=100'
443 client = LocalClient(env=env, name='cli_h2_pausing')
444 r = client.run(args=['-V', proto, url])
445 r.check_exit_code(0)
447 @pytest.mark.parametrize("proto", Env.http_protos())
448 def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, proto):
449 if not env.have_compressed_curl():
450 pytest.skip("--compressed not supported")
451 count = 1
452 urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
453 curl = CurlClient(env=env)
454 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
455 '--compressed'
456 ])
457 r.check_exit_code(code=0)
458 r.check_response(count=count, http_status=200)
460 def check_downloads(self, client, srcfile: str, count: int,
461 complete: bool = True):
462 for i in range(count):
463 dfile = client.download_file(i)
464 assert os.path.exists(dfile)
465 if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
466 with open(srcfile) as fa, open(dfile) as fb:
467 a = fa.readlines()
468 b = fb.readlines()
469 diff = "".join(difflib.unified_diff(a=a, b=b,
470 fromfile=srcfile,
471 tofile=dfile,
472 n=1))
473 assert False, f'download {dfile} differs:\n{diff}'
475 # download via lib client, 1 at a time, pause/resume at different offsets
476 @pytest.mark.parametrize("pause_offset", [0, 10 * 1024, 100 * 1023, 640000])
477 @pytest.mark.parametrize("proto", Env.http_protos())
478 def test_02_29_h2_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset):
479 count = 2
480 docname = 'data-10m'
481 url = f'https://localhost:{env.https_port}/{docname}'
482 client = LocalClient(name='cli_hx_download', env=env)
483 if not client.exists():
484 pytest.skip(f'example client not built: {client.name}')
485 r = client.run(args=[
486 '-n', f'{count}', '-P', f'{pause_offset}',
487 '-C', env.ca.cert_file, '-V', proto, url
488 ])
489 r.check_exit_code(0)
490 srcfile = os.path.join(httpd.docs_dir, docname)
491 self.check_downloads(client, srcfile, count)
493 # download parallel with prior knowledge
494 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
495 def test_02_30_parallel_prior_knowledge(self, env: Env, httpd):
496 count = 3
497 curl = CurlClient(env=env)
498 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]'
499 r = curl.http_download(urls=[urln], extra_args=[
500 '--parallel', '--http2-prior-knowledge'
501 ])
502 r.check_response(http_status=200, count=count)
503 assert r.total_connects == 1, r.dump_logs()
505 # download parallel with h2 "Upgrade:"
506 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
507 def test_02_31_parallel_upgrade(self, env: Env, httpd, nghttpx):
508 count = 3
509 curl = CurlClient(env=env)
510 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]'
511 r = curl.http_download(urls=[urln], extra_args=[
512 '--parallel', '--http2'
513 ])
514 r.check_response(http_status=200, count=count)
515 # we see up to 3 connections, because Apache wants to serve only a single
516 # request via Upgrade: and then closes the connection. But if a new
517 # request comes in time, it might still get served.
518 assert r.total_connects <= 3, r.dump_logs()
520 # nghttpx is the only server we have that supports TLS early data
521 @pytest.mark.skipif(condition=not Env.have_h3_server(), reason="no nghttpx with QUIC")
522 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
523 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
524 @pytest.mark.parametrize("proto", Env.http_protos())
525 def test_02_32_earlydata(self, env: Env, httpd, nghttpx, proto):
526 if not env.curl_can_early_data():
527 pytest.skip('TLS earlydata not implemented')
528 if proto == 'h3' and not env.curl_can_h3_early_data():
529 pytest.skip("h3 early data not supported")
530 count = 2
531 docname = 'data-10k'
532 # we want this test to always connect to nghttpx, since it is
533 # the only server we have that supports TLS earlydata
534 port = env.port_for(proto)
535 if proto != 'h3':
536 port = env.nghttpx_https_port
537 url = f'https://{env.domain1}:{port}/{docname}'
538 client = LocalClient(name='cli_hx_download', env=env)
539 if not client.exists():
540 pytest.skip(f'example client not built: {client.name}')
541 r = client.run(args=[
542 '-n', f'{count}',
543 '-C', env.ca.cert_file,
544 '-e', # use TLS earlydata
545 '-f', # forbid reuse of connections
546 '-r', f'{env.domain1}:{port}:127.0.0.1',
547 '-V', proto, url
548 ])
549 r.check_exit_code(0)
550 srcfile = os.path.join(httpd.docs_dir, docname)
551 self.check_downloads(client, srcfile, count)
552 # check that TLS earlydata worked as expected
553 earlydata = {}
554 reused_session = False
555 for line in r.trace_lines:
556 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line)
557 if m:
558 earlydata[int(m.group(1))] = int(m.group(2))
559 continue
560 if re.match(r'\[1-1] \* (\[1-1] )?SSL reusing session.*', line):
561 reused_session = True
562 assert reused_session, 'session was not reused for 2nd transfer'
563 assert earlydata[0] == 0, f'{earlydata}'
564 if proto == 'http/1.1':
565 assert earlydata[1] == 111, f'{earlydata}'
566 elif proto == 'h2':
567 assert earlydata[1] == 127, f'{earlydata}'
568 elif proto == 'h3':
569 assert earlydata[1] == 109, f'{earlydata}'
571 @pytest.mark.parametrize("proto", Env.http_h1_h2_protos())
572 @pytest.mark.parametrize("max_host_conns", [0, 1, 5])
573 @pytest.mark.parametrize("share_connect", [False, True])
574 def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns, share_connect):
575 if not env.curl_is_debug():
576 pytest.skip('only works for curl debug builds')
577 if not env.curl_is_verbose():
578 pytest.skip('only works for curl with verbose strings')
579 count = 50
580 max_parallel = 50
581 docname = 'data-10k'
582 port = env.port_for(proto)
583 url = f'https://{env.domain1}:{port}/{docname}'
584 run_env = os.environ.copy()
585 run_env['CURL_DEBUG'] = 'multi'
586 client = LocalClient(name='cli_hx_download', env=env, run_env=run_env)
587 if not client.exists():
588 pytest.skip(f'example client not built: {client.name}')
589 extra_args = ['-S'] if share_connect else [] # share connections
590 r = client.run(args=[
591 '-n', f'{count}',
592 '-m', f'{max_parallel}',
593 '-C', env.ca.cert_file,
594 '-x', # always use a fresh connection
595 '-M', str(max_host_conns), # limit conns per host
596 '-r', f'{env.domain1}:{port}:127.0.0.1',
597 ] + extra_args + ['-V', proto, url])
598 r.check_exit_code(0)
599 srcfile = os.path.join(httpd.docs_dir, docname)
600 self.check_downloads(client, srcfile, count)
601 if max_host_conns > 0:
602 matched_lines = 0
603 for line in r.trace_lines:
604 m = re.match(r'.*The cache now contains (\d+) members.*', line)
605 if m:
606 matched_lines += 1
607 n = int(m.group(1))
608 assert n <= max_host_conns
609 assert matched_lines > 0
611 @pytest.mark.parametrize("proto", Env.http_h1_h2_protos())
612 @pytest.mark.parametrize("max_total_conns", [0, 1, 5])
613 @pytest.mark.parametrize("share_connect", [False, True])
614 def test_02_34_max_total_conns(self, env: Env, httpd, nghttpx, proto, max_total_conns, share_connect):
615 if not env.curl_is_debug():
616 pytest.skip('only works for curl debug builds')
617 if not env.curl_is_verbose():
618 pytest.skip('only works for curl with verbose strings')
619 count = 50
620 max_parallel = 50
621 docname = 'data-10k'
622 port = env.port_for(proto)
623 url = f'https://{env.domain1}:{port}/{docname}'
624 run_env = os.environ.copy()
625 run_env['CURL_DEBUG'] = 'multi'
626 client = LocalClient(name='cli_hx_download', env=env, run_env=run_env)
627 if not client.exists():
628 pytest.skip(f'example client not built: {client.name}')
629 extra_args = ['-S'] if share_connect else [] # share connections
630 r = client.run(args=[
631 '-n', f'{count}',
632 '-m', f'{max_parallel}',
633 '-C', env.ca.cert_file,
634 '-x', # always use a fresh connection
635 '-T', str(max_total_conns), # limit total connections
636 '-r', f'{env.domain1}:{port}:127.0.0.1',
637 ] + extra_args + ['-V', proto, url])
638 r.check_exit_code(0)
639 srcfile = os.path.join(httpd.docs_dir, docname)
640 self.check_downloads(client, srcfile, count)
641 if max_total_conns > 0:
642 matched_lines = 0
643 for line in r.trace_lines:
644 m = re.match(r'.*The cache now contains (\d+) members.*', line)
645 if m:
646 matched_lines += 1
647 n = int(m.group(1))
648 assert n <= max_total_conns
649 assert matched_lines > 0
651 # 2 parallel transfers, pause and resume. Load a 100 MB zip bomb from
652 # the server with "Content-Encoding: gzip" that gets exploded during
653 # response writing to the client. Client pauses after 1MB unzipped data
654 # and causes buffers to fill while the server sends more response
655 # data.
656 # * http/1.1: not much buffering is done as curl does no longer
657 # serve the connections that are paused
658 # * h2/h3: server continues sending what the stream window allows and
659 # since the one connection involved unpaused transfers, data continues
660 # to be received, requiring buffering.
661 @pytest.mark.parametrize("proto", Env.http_protos())
662 def test_02_35_pause_bomb(self, env: Env, httpd, nghttpx, proto):
663 count = 2
664 pause_offset = 1024 * 1024
665 docname = 'bomb-100m.txt.var'
666 url = f'https://localhost:{env.https_port}/{docname}'
667 client = LocalClient(name='cli_hx_download', env=env)
668 if not client.exists():
669 pytest.skip(f'example client not built: {client.name}')
670 r = client.run(args=[
671 '-n', f'{count}', '-m', f'{count}',
672 '-P', f'{pause_offset}', '-C', env.ca.cert_file,
673 '-V', proto, url
674 ])
675 r.check_exit_code(0)
677 # download with looong urls
678 @pytest.mark.parametrize("proto", Env.http_protos())
679 @pytest.mark.parametrize("url_junk", [1024, 16 * 1024, 32 * 1024, 64 * 1024, 80 * 1024, 96 * 1024])
680 def test_02_36_looong_urls(self, env: Env, httpd, nghttpx, proto, url_junk):
681 if proto == 'h3' and env.curl_uses_lib('quiche'):
682 pytest.skip("quiche fails from 16k onwards")
683 curl = CurlClient(env=env)
684 # 'url' is longer than 'url_len'
685 url = f'https://{env.authority_for(env.domain1, proto)}/data.json?{"x"*(url_junk)}'
686 r = curl.http_download(urls=[url], alpn_proto=proto)
687 if url_junk <= 1024:
688 r.check_exit_code(0)
689 r.check_response(http_status=200)
690 elif url_junk <= 32 * 1024:
691 r.check_exit_code(0)
692 # server replies with 414, Request URL too long
693 r.check_response(http_status=414)
694 else:
695 # with urls larger than 64k, behaviour differs
696 if proto == 'http/1.1':
697 r.check_exit_code(0)
698 r.check_response(http_status=414)
699 elif proto == 'h2':
700 # h2 is unable to send such large headers (frame limits)
701 r.check_exit_code(55)
702 elif proto == 'h3':
703 # nghttpx reports 431 Request Header Field too Large
704 # or destroys the connection with internal error
705 # ERR_QPACK_HEADER_TOO_LARGE,
706 # depending on nghttp3 version and payload size
707 assert r.exit_code in [0, 56], f'expected exit code 0 or 56, '\
708 f'got {r.exit_code}\n{r.dump_logs()}'
709 if r.exit_code == 0:
710 r.check_response(http_status=431)
712 # download 2 http/1.1 resources, limited to a single connection max
713 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
714 def test_02_37_pending_timeout(self, env: Env, httpd):
715 # Run 2 http/1.1 transfers in parallel with a connection limit of 1
716 # - xfer 0 has a delay response, making xfer 1 pending during its run
717 # - xfer 1 will timeout while pending, waiting on the connection
718 proto = 'http/1.1'
719 run_env = os.environ.copy()
720 run_env['CURL_DBG_MAX_TOTAL_CONNECTIONS'] = '1'
721 curl = CurlClient(env=env, run_env=run_env)
722 url1 = f'https://{env.authority_for(env.domain1, proto)}' \
723 '/curltest/tweak/?&delay=3s'
724 url2 = f'https://{env.authority_for(env.domain1, proto)}/data.json'
725 r = curl.http_download(urls=[url1, url2], alpn_proto=proto, extra_args=[
726 '--http1.1', '--parallel'],
727 url_options={
728 url1: ['--max-time', '10'],
729 url2: ['--max-time', '1']
730 })
731 r.check_exit_code(28)
732 xfers = {stat['xfer_id']: stat for stat in r.stats}
733 assert xfers[0]['http_code'] == 200, f'{r.stats[0]}' # succeeded
734 assert xfers[1]['exitcode'] == 28, f'{r.stats[1]}' # timed out
735 m = re.search(r'timed out after (\d+) milliseconds', xfers[1]['errormsg'])
736 assert m, f'{r.stats[1]}'
737 assert int(m.group(1)) < 3000, f'{r.stats[1]}'
Diagnostics4 git calls · 214 ms
CallStatusSizeTime0214 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=209
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2006.9 KiB209 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=7076
git.response.cache=MISS
git.response.duration_ms=209
git.fetch.wants=106094fd46c82b237a2b3acfbe1e50ae8731b174
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=7076
git.pkt.payload.max=7055
git.sideband.band1.bytes=7054
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e30730de7115db
Request
7deed70e…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format