master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import logging
26import os
27import re
28from datetime import timedelta
30import pytest
31from testenv import CurlClient, Env, LocalClient
33log = logging.getLogger(__name__)
36@pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
37@pytest.mark.skipif(condition=not Env.curl_has_feature('AsynchDNS'), reason="needs AsynchDNS")
38class TestResolve:
40 @pytest.fixture(autouse=True, scope='class')
41 @classmethod
42 def _class_scope(cls, env, httpd):
43 indir = httpd.docs_dir
44 env.make_data_file(indir=indir, fname="data-0k", fsize=0)
46 # use .invalid hostname that should never resolv
47 def test_21_01_resolv_invalid_one(self, env: Env, httpd, nghttpx):
48 count = 1
49 run_env = os.environ.copy()
50 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = '5'
51 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
52 url = f'https://test-{count}.http.curl.invalid/'
53 r = curl.http_download(urls=[url], with_stats=True)
54 r.check_exit_code(6)
55 r.check_stats(count=count, http_status=0, exitcode=6)
57 # use .invalid hostname, one after the other
58 @pytest.mark.parametrize("delay_ms", [1, 50])
59 def test_21_02_resolv_invalid_serial(self, env: Env, delay_ms, httpd, nghttpx):
60 count = 10
61 run_env = os.environ.copy()
62 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
63 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
64 urls = [f'https://test-{i}.http.curl.invalid/' for i in range(count)]
65 r = curl.http_download(urls=urls, with_stats=True)
66 r.check_exit_code(6)
67 r.check_stats(count=count, http_status=0, exitcode=6)
69 # use .invalid hostname, parallel
70 @pytest.mark.parametrize("delay_ms", [1, 50])
71 def test_21_03_resolv_invalid_parallel(self, env: Env, delay_ms, httpd, nghttpx):
72 count = 20
73 run_env = os.environ.copy()
74 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
75 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
76 urls = [f'https://test-{i}.http.curl.invalid/' for i in range(count)]
77 r = curl.http_download(urls=urls, with_stats=True, extra_args=[
78 '--parallel'
79 ])
80 r.check_exit_code(6)
81 r.check_stats(count=count, http_status=0, exitcode=6)
83 # resolve first URL with IPv6 only and fail that, resolve second
84 # with ipv*, should succeed.
85 def test_21_04_resolv_inv_v6(self, env: Env, httpd):
86 count = 2
87 run_env = os.environ.copy()
88 run_env['CURL_DBG_RESOLV_FAIL_IPV6'] = '1'
89 url = f'https://localhost:{env.https_port}/'
90 client = LocalClient(name='cli_hx_download', env=env, run_env=run_env)
91 if not client.exists():
92 pytest.skip(f'example client not built: {client.name}')
93 dfiles = [client.download_file(i) for i in range(count)]
94 self._clean_files(dfiles)
95 # let the first URL resolve via IPv6 only, which we force to fail
96 r = client.run(args=[
97 '-n', f'{count}', '-6', '-C', env.ca.cert_file, url
98 ])
99 r.check_exit_code(6)
100 assert not os.path.exists(dfiles[0])
101 assert os.path.exists(dfiles[1])
103 # use .invalid hostname, parallel, single resolve thread
104 @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
105 def test_21_05_resolv_single_thread(self, env: Env, httpd, nghttpx):
106 count = 10
107 delay_ms = 50
108 run_env = os.environ.copy()
109 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
110 run_env['CURL_DBG_RESOLV_MAX_THREADS'] = '1'
111 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
112 urls = [f'https://test-{i}.http.curl.invalid/' for i in range(count)]
113 r = curl.http_download(urls=urls, with_stats=True, extra_args=[
114 '--parallel', '-6'
115 ])
116 r.check_exit_code(6)
117 r.check_stats(count=count, http_status=0, exitcode=6)
118 assert r.duration > timedelta(milliseconds=count * delay_ms), f'{r}'
120 def dns_settings(self, dns_method, dnsd, path="/"):
121 xargs = []
122 run_env = os.environ.copy()
123 run_env['CURL_DEBUG'] = 'all'
124 if dns_method == 'DoH':
125 if not Env.curl_can_doh():
126 pytest.skip(reason="curl built without DoH")
127 xargs = ['--doh-insecure', '--doh-url', f'http://127.0.0.1:{dnsd.port}{path}']
128 else:
129 if not Env.curl_override_dns():
130 pytest.skip(reason="no DNS override")
131 run_env['CURL_DNS_SERVER'] = f'127.0.0.1:{dnsd.port}'
132 run_env['CURL_QUICK_EXIT'] = '1'
133 return run_env, xargs
135 # dnsd with no answers
136 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
137 def test_21_06_dnsd_empty(self, env: Env, httpd, dnsd, dns_method):
138 dnsd.set_answers()
139 run_env, xargs = self.dns_settings(dns_method, dnsd)
140 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
141 url = 'https://test-dnsd.http.curl.invalid/'
142 r = curl.http_download(urls=[url], with_stats=True, extra_args=xargs)
143 r.check_exit_code(6) # could not resolve host
144 r.check_stats(count=1, http_status=0, exitcode=6)
146 # dnsd with one answer for A
147 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
148 def test_21_07_dnsd_a(self, env: Env, httpd, dnsd, dns_method):
149 dnsd.set_answers(addr_a=['127.0.0.1'])
150 run_env, xargs = self.dns_settings(dns_method, dnsd)
151 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
152 url = f'https://{env.authority_for(env.domain1, "http/1.1")}/data.json'
153 r = curl.http_download(urls=[url], with_stats=True, extra_args=xargs)
154 r.check_exit_code(0)
155 r.check_stats(count=1, http_status=200, exitcode=0)
156 assert r.stats[0]['remote_ip'] == '127.0.0.1'
158 # dnsd with one answer for AAAA
159 @pytest.mark.skipif(condition=not Env.curl_has_feature('IPv6'), reason="no IPv6")
160 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
161 def test_21_08_dnsd_aaaa(self, env: Env, httpd, dnsd, dns_method):
162 dnsd.set_answers(addr_aaaa=['[::1]'])
163 run_env, xargs = self.dns_settings(dns_method, dnsd)
164 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
165 url = f'https://{env.authority_for(env.domain1, "http/1.1")}/data.json'
166 r = curl.http_download(urls=[url], with_stats=True, extra_args=xargs)
167 r.check_exit_code(0)
168 r.check_stats(count=1, http_status=200, exitcode=0)
169 assert r.stats[0]['remote_ip'] == '::1'
171 # dnsd with one answer for A, delayed one for AAAA
172 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
173 def test_21_09_dnsd_a_delay(self, env: Env, httpd, dnsd, dns_method):
174 dnsd.set_answers(addr_a=['127.0.0.1'], addr_aaaa=['[::1]'],
175 delay_aaaa_ms=env.test_timeout * 1000)
176 run_env, xargs = self.dns_settings(dns_method, dnsd)
177 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
178 url = f'https://{env.authority_for(env.domain1, "http/1.1")}/data.json'
179 r = curl.http_download(urls=[url], with_stats=True, extra_args=xargs)
180 r.check_exit_code(0)
181 r.check_stats(count=1, http_status=200, exitcode=0)
182 assert r.stats[0]['remote_ip'] == '127.0.0.1'
184 # dnsd with one answer for AAAA, delayed one for A
185 @pytest.mark.skipif(condition=not Env.curl_has_feature('IPv6'), reason="no IPv6")
186 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
187 def test_21_10_dnsd_aaaa_delay(self, env: Env, httpd, dnsd, dns_method):
188 dnsd.set_answers(addr_a=['127.0.0.1'], addr_aaaa=['[::1]'],
189 delay_a_ms=env.test_timeout * 1000)
190 run_env, xargs = self.dns_settings(dns_method, dnsd)
191 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
192 url = f'https://{env.authority_for(env.domain1, "http/1.1")}/data.json'
193 r = curl.http_download(urls=[url], with_stats=True, extra_args=xargs)
194 r.check_exit_code(0)
195 r.check_stats(count=1, http_status=200, exitcode=0)
196 assert r.stats[0]['remote_ip'] == '::1'
198 # transient resolve failures must not be cached as negative
199 # entries: a second lookup of the same name tries again
200 @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
201 def test_21_11_resolv_transient_uncached(self, env: Env, httpd, nghttpx):
202 count = 2
203 delay_ms = 250
204 run_env = os.environ.copy()
205 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
206 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
207 urls = [f'https://test-again.http.curl.invalid/?id={i}' for i in range(count)]
208 r = curl.http_download(urls=urls, with_stats=True)
209 r.check_exit_code(6)
210 r.check_stats(count=count, http_status=0, exitcode=6)
211 # not cached as negative: the second transfer resolved again
212 if env.curl_is_verbose():
213 assert not [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
214 assert r.stats[1]['time_total'] > (delay_ms / 2) / 1000.0, f'{r.stats[1]}'
216 # a negative resolve answer is cached: a second lookup of the same
217 # name fails right away from the cache
218 @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
219 def test_21_12_resolv_negative_cached(self, env: Env, httpd, nghttpx):
220 count = 2
221 delay_ms = 250
222 run_env = os.environ.copy()
223 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
224 run_env['CURL_DBG_RESOLV_FAIL_NEGATIVE'] = '1'
225 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
226 urls = [f'https://test-nxdomain.http.curl.invalid/?id={i}' for i in range(count)]
227 r = curl.http_download(urls=urls, with_stats=True)
228 r.check_exit_code(6)
229 r.check_stats(count=count, http_status=0, exitcode=6)
230 # the second transfer failed right away from the cache entry
231 if env.curl_is_verbose():
232 assert [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
233 assert r.stats[1]['time_total'] < (delay_ms / 2) / 1000.0, f'{r.stats[1]}'
235 # dnsd giving NXDOMAIN for all families: the negative answer is
236 # cached and a second lookup of the same name uses the cache
237 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
238 def test_21_13_dnsd_nxdomain_cached(self, env: Env, httpd, dnsd, dns_method):
239 count = 2
240 dnsd.set_answers(rcode_a=3, rcode_aaaa=3)
241 run_env, xargs = self.dns_settings(dns_method, dnsd)
242 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
243 urls = [f'https://test-nx.http.curl.invalid/?id={i}' for i in range(count)]
244 r = curl.http_download(urls=urls, with_stats=True, extra_args=xargs)
245 r.check_exit_code(6)
246 r.check_stats(count=count, http_status=0, exitcode=6)
247 if env.curl_is_verbose():
248 assert [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
250 # dnsd failing one family with SERVFAIL: not an authoritative
251 # negative answer, a second lookup of the same name tries again
252 @pytest.mark.parametrize("dns_method", ["DNS", "DoH"])
253 def test_21_14_dnsd_servfail_uncached(self, env: Env, httpd, dnsd, dns_method):
254 count = 2
255 dnsd.set_answers(rcode_a=2, rcode_aaaa=3)
256 run_env, xargs = self.dns_settings(dns_method, dnsd)
257 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
258 urls = [f'https://test-sf.http.curl.invalid/?id={i}' for i in range(count)]
259 r = curl.http_download(urls=urls, with_stats=True, extra_args=xargs)
260 r.check_exit_code(6)
261 r.check_stats(count=count, http_status=0, exitcode=6)
262 if env.curl_is_verbose():
263 assert not [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
265 # a resolve gets processed even when the first resolver thread
266 # starts fail, e.g. when the system temporarily refuses to spawn
267 # threads. The transfer must fail on the (debug-forced) lookup
268 # failure well before the resolve timeout.
269 @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
270 def test_21_15_resolv_thread_start_fails(self, env: Env, httpd, nghttpx):
271 run_env = os.environ.copy()
272 run_env['CURL_DBG_THRDPOOL_FAIL_STARTS'] = '3'
273 run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = '10'
274 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
275 url = 'https://test-1.http.curl.invalid/'
276 r = curl.http_download(urls=[url], with_stats=True, extra_args=[
277 '--connect-timeout', '20'
278 ])
279 r.check_exit_code(6)
280 r.check_stats(count=1, http_status=0, exitcode=6)
281 assert r.duration < timedelta(seconds=20), f'{r}'
283 # dnsd with one answer for AAAA, delayed one for A
284 @pytest.mark.skipif(condition=not Env.curl_override_dns(), reason="no DNS override")
285 @pytest.mark.skipif(condition=not Env.curl_has_feature('IPv6'), reason="no IPv6")
286 @pytest.mark.skipif(condition=not Env.curl_resolv_threaded(), reason="no threaded resolver")
287 def test_21_16_dnsd_link_local(self, env: Env, httpd, dnsd):
288 dnsd.set_answers(addr_aaaa=['[fe80::1]'])
289 run_env = os.environ.copy()
290 run_env['CURL_DNS_SERVER'] = f'127.0.0.1:{dnsd.port}'
291 run_env['CURL_QUICK_EXIT'] = '1'
292 run_env['CURL_DEBUG'] = 'dns'
293 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
294 url = f'https://{env.authority_for(env.domain1, "http/1.1")}/data.json'
295 r = curl.http_download(urls=[url], with_stats=True, extra_args=[
296 '--connect-timeout', '1'
297 ])
298 # should fail with CURLE_OPERATION_TIMEOUT or COULDNT_CONNECT
299 assert r.exit_code in [7, 28], f'{r.dump_logs()}'
300 af_unspec_resolves = [
301 line for line in r.trace_lines
302 if re.match(r'.* \[DNS] re-queueing query .+ for AF_UNSPEC resolve', line)
303 ]
304 assert len(af_unspec_resolves) == 1, f'{r.dump_logs()}'
305 aaaa_resolves = [line for line in r.trace_lines if
306 re.match(r'.* \* IPv6: fe80::1', line)]
307 assert len(aaaa_resolves) == 1, f'{r.dump_logs()}'
309 # dnsd+DoH, handling HTTP response failure
310 def test_21_17_dnsd_http_fails(self, env: Env, httpd, dnsd):
311 count = 2
312 dnsd.set_answers(rcode_a=2, rcode_aaaa=3)
313 run_env, xargs = self.dns_settings('DoH', dnsd, path='/notfound')
314 curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
315 urls = [f'https://test-sf.http.curl.invalid/?id={i}' for i in range(count)]
316 r = curl.http_download(urls=urls, with_stats=True, extra_args=xargs)
317 r.check_exit_code(6)
318 r.check_stats(count=count, http_status=0, exitcode=6)
319 if env.curl_is_verbose():
320 assert not [t for t in r.trace_lines if 'Negative DNS entry' in t], f'{r}'
322 def _clean_files(self, files):
323 for file in files:
324 if os.path.exists(file):
325 os.remove(file)
Diagnostics4 git calls · 217 ms
CallStatusSizeTime0217 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=211 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.5 KiB211 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3536 git.response.cache=MISS git.response.duration_ms=211 git.fetch.wants=9e9ea4bfd185dd44e3b7da0d41f512cac2965485 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3536 git.pkt.payload.max=3515 git.sideband.band1.bytes=3514 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2e0501a882308
- Request
- 5acf2f37…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format