master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import contextlib
26import logging
27import os
28import socket
29import threading
30import time
31from datetime import datetime, timedelta, timezone
32from typing import Dict, Generator
34import pytest
35from testenv import CurlClient, Env
36from testenv.ports import alloc_ports_and_do
38log = logging.getLogger(__name__)
41def pr_process(self):
42 log.info("pr accepting")
43 try:
44 while self._socket and not self._done:
45 conn, _ = self._socket.accept()
46 with conn:
47 conn.sendall(HttpPureResp.RESPONSE)
48 log.info("response sent")
49 except ConnectionAbortedError:
50 return
53class HttpPureResp:
55 RESPONSE = (b'HTTP/1.1 200 OK\r\n'
56 b'Content-Length: 3\r\n'
57 b'Connection: close\r\n'
58 b'\r\n'
59 b'ok\n')
61 def __init__(self, env):
62 self.name = 'httppureresp'
63 self.env = env
64 self.run_dir = os.path.join(env.gen_dir, self.name)
65 self._thread = None
66 self._socket = None
67 self._done = False
68 self.port = 0
70 def start(self):
71 log.info("pr start")
73 def startup(ports: Dict[str, int]) -> bool:
74 self.port = ports[self.name]
75 log.info(f"pr port={self.port}")
76 self._socket = socket.create_server(('127.0.0.1', self.port))
77 log.info("pr socket open")
78 self._thread = threading.Thread(target=pr_process, daemon=True, args=[self])
79 self._thread.start()
80 self.env.update_ports(ports)
81 return True
83 port_spec = {
84 self.name: socket.SOCK_STREAM
85 }
86 assert alloc_ports_and_do(port_spec, startup,
87 self.env.gen_root, max_tries=3)
88 assert self._socket
89 return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
91 def stop(self):
92 self._done = True
93 if self._socket:
94 self._socket.close()
95 self._socket = None
96 log.info("pr stopped")
98 def wait_live(self, timeout: timedelta):
99 curl = CurlClient(env=self.env, run_dir=self.run_dir,
100 timeout=timeout.total_seconds())
101 try_until = datetime.now(timezone.utc) + timeout
102 while datetime.now(timezone.utc) < try_until:
103 r = curl.http_get(url=f'http://{self.env.domain1}:{self.port}/')
104 if r.exit_code == 0:
105 return True
106 time.sleep(.1)
107 log.error(f"Server still not responding after {timeout}")
108 return False
111@pytest.mark.skipif(condition=not Env.httpd_is_at_least('2.4.55'),
112 reason=f"httpd version too old for this: {Env.httpd_version()}")
113class TestErrors:
115 # download 1 file, check that we get CURLE_PARTIAL_FILE
116 @pytest.mark.parametrize("proto", Env.http_protos())
117 def test_05_01_partial_1(self, env: Env, httpd, nghttpx, proto):
118 if proto == 'h3' and env.curl_uses_lib('quiche') and \
119 not env.curl_lib_version_at_least('quiche', '0.29.0'):
120 pytest.skip("quiche issue #2277 not fixed")
121 count = 1
122 curl = CurlClient(env=env)
123 urln = f'https://{env.authority_for(env.domain1, proto)}' \
124 f'/curltest/tweak?id=[0-{count - 1}]'\
125 '&chunks=3&chunk_size=16000&body_error=reset'
126 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
127 '--retry', '0'
128 ])
129 r.check_exit_code(False)
130 invalid_stats = []
131 for idx, s in enumerate(r.stats):
132 if 'exitcode' not in s or s['exitcode'] not in [18, 56, 92, 95]:
133 invalid_stats.append(f'request {idx} exit with {s["exitcode"]}')
134 assert len(invalid_stats) == 0, f'failed: {invalid_stats}'
136 # download files, check that we get CURLE_PARTIAL_FILE for all
137 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
138 def test_05_02_partial_20(self, env: Env, httpd, nghttpx, proto):
139 if proto == 'h3' and env.curl_uses_lib('quiche') and \
140 not env.curl_lib_version_at_least('quiche', '0.29.0'):
141 pytest.skip("quiche issue #2277 not fixed")
142 count = 20
143 curl = CurlClient(env=env)
144 urln = f'https://{env.authority_for(env.domain1, proto)}' \
145 f'/curltest/tweak?id=[0-{count - 1}]'\
146 '&chunks=5&chunk_size=16000&body_error=reset'
147 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
148 '--retry', '0', '--parallel',
149 ])
150 r.check_exit_code(False)
151 assert len(r.stats) == count, f'did not get all stats: {r}'
152 invalid_stats = []
153 for idx, s in enumerate(r.stats):
154 if 'exitcode' not in s or s['exitcode'] not in [18, 55, 56, 92, 95]:
155 invalid_stats.append(f'request {idx} exit with {s["exitcode"]}\n{r.dump_logs()}')
156 assert len(invalid_stats) == 0, f'failed: {invalid_stats}\n{r.dump_logs()}'
158 # access a resource that, on h2, RST the stream with HTTP_1_1_REQUIRED
159 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
160 def test_05_03_required(self, env: Env, httpd, nghttpx):
161 curl = CurlClient(env=env)
162 proto = 'http/1.1'
163 urln = f'https://{env.authority_for(env.domain1, proto)}/curltest/1_1'
164 r = curl.http_download(urls=[urln], alpn_proto=proto)
165 r.check_exit_code(0)
166 r.check_response(http_status=200, count=1)
167 proto = 'h2'
168 urln = f'https://{env.authority_for(env.domain1, proto)}/curltest/1_1'
169 r = curl.http_download(urls=[urln], alpn_proto=proto)
170 r.check_exit_code(0)
171 r.check_response(http_status=200, count=1)
172 # check that we did a downgrade
173 assert r.stats[0]['http_version'] == '1.1', r.dump_logs()
175 # On the URL used here, Apache is doing an "unclean" TLS shutdown,
176 # meaning it sends no shutdown notice and closes TCP.
177 # The HTTP response delivers a body without Content-Length. We expect:
178 # - http/1.0 to fail since it relies on a clean connection close to
179 # detect the end of the body
180 # - http/1.1 to work since it will used "chunked" transfer encoding
181 # and stop receiving when that signals the end
182 # - h2 to work since it will signal the end of the response before
183 # and not see the "unclean" close either
184 @pytest.mark.parametrize("proto", ['http/1.0', 'http/1.1', 'h2'])
185 def test_05_04_unclean_tls_shutdown(self, env: Env, httpd, nghttpx, proto):
186 if proto == 'h2' and not env.have_h2_curl():
187 pytest.skip("h2 not supported")
188 if proto == 'h3' and not env.have_h3():
189 pytest.skip("h3 not supported")
190 count = 10 if proto == 'h2' else 1
191 curl = CurlClient(env=env)
192 url = f'https://{env.authority_for(env.domain1, proto)}'\
193 f'/curltest/shutdown_unclean?id=[0-{count-1}]&chunks=4'
194 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
195 '--parallel', '--trace-config', 'ssl'
196 ])
197 if proto == 'http/1.0':
198 # we are inconsistent if we fail or not in missing TLS shutdown
199 # openssl code ignore such errors intentionally in non-debug builds
200 r.check_exit_code(56)
201 else:
202 r.check_exit_code(0)
203 r.check_response(http_status=200, count=count)
205 # Make a connect with a fail for '::1' and wrong certificate for '127.0.0.1'
206 # The error message reported needs to be from the certificate.
207 # verifies issue #20608
208 @pytest.mark.parametrize("proto", Env.http_h1_h2_protos())
209 def test_05_05_failed_peer(self, env: Env, proto, httpd, nghttpx):
210 domain = f'invalid.{env.tld}'
211 url = f'https://{domain}:{env.port_for(proto)}/'
212 run_env = os.environ.copy()
213 run_env['CURL_DBG_SOCK_FAIL_IPV6'] = '1'
214 curl = CurlClient(env=env, run_env=run_env)
215 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
216 '--resolve', f'{domain}:{env.port_for(proto)}:::1,127.0.0.1',
217 ])
218 assert r.exit_code == 60, f'{r}'
219 assert r.stats[0]['errormsg'] != 'CURL_DBG_SOCK_FAIL_IPV6: failed to open socket'
221 # Get, retry on 502
222 def test_05_06_retry_502(self, env: Env, httpd, nghttpx):
223 proto = 'http/1.1'
224 curl = CurlClient(env=env)
225 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=502'
226 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
227 '--retry', '2', '--retry-all-errors', '--retry-delay', '1',
228 ])
229 r.check_response(http_status=502)
230 assert r.stats[0]['num_retries'] == 2, f'{r}'
231 # curious, since curl does the retries, it finds the previous
232 # connection in the cache and reports that no connects were done
233 assert r.stats[0]['num_connects'] == 0, f'{r}'
235 # Get, retry on 502 in parallel mode
236 def test_05_07_retry_502_parallel(self, env: Env, httpd, nghttpx):
237 proto = 'http/1.1'
238 curl = CurlClient(env=env)
239 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=502'
240 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
241 '--retry', '2', '--retry-all-errors', '--retry-delay', '1', '--parallel'
242 ])
243 r.check_response(http_status=502)
244 assert r.stats[0]['num_retries'] == 2, f'{r}'
246 # Get, retry on 401, not happening
247 def test_05_08_retry_401(self, env: Env, httpd, nghttpx):
248 proto = 'http/1.1'
249 curl = CurlClient(env=env)
250 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=401'
251 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
252 '--retry', '2', '--retry-all-errors', '--retry-delay', '1'
253 ])
254 r.check_response(http_status=401)
255 # No retries on a 401
256 assert r.stats[0]['num_retries'] == 0, f'{r}'
258 # Server closes the connection immediately after accept,
259 def test_05_09_handshake_eof(self, env: Env):
260 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
261 server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
262 server.bind(('127.0.0.1', 0))
263 server.listen(1)
264 port = server.getsockname()[1]
266 # accept one connection and immediately close it
267 def accept_and_close():
268 # ignore expected socket error
269 with contextlib.suppress(OSError):
270 conn, _ = server.accept()
271 conn.recv(1) # wait for ClientHello
272 conn.close()
274 t = threading.Thread(target=accept_and_close)
275 t.start()
277 run_env = os.environ.copy()
278 run_env['CURL_DEBUG'] = 'all'
279 curl = CurlClient(env=env, timeout=env.test_timeout, run_env=run_env)
280 url = f'https://127.0.0.1:{port}/'
281 r = curl.run_direct(args=[url, '--insecure', '-v'])
283 t.join(timeout=10)
285 # We expect an error code, not success (0) and not timeout (-1)
286 # Expected error codes are:
287 # - CURLE_SSL_CONNECT_ERROR (35) - common for handshake failures
288 # - CURLE_RECV_ERROR (56) - some TLS backends fail with that
289 assert r.exit_code in [35, 56], \
290 f'unexpected error {r.exit_code}\n{r.dump_logs()}'
292 # Get a resource many times with limited requests
293 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason='curl without HTTP/2')
294 def test_05_10_limits(self, env: Env, httpd):
295 proto = 'h2'
296 count = 6
297 curl = CurlClient(env=env)
298 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/limit?id=[0-{count-1}]'
299 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
300 '--parallel', '--retry', '5'
301 ])
302 r.check_stats(count=count, http_status=200, exitcode=0)
304 @pytest.fixture(scope='class')
305 @classmethod
306 def pr_server(cls, env: Env) -> Generator[HttpPureResp, None, None]:
307 server = HttpPureResp(env=env)
308 server.start()
309 yield server
310 server.stop()
312 def test_05_11_pure_resp(self, env: Env, pr_server):
313 proto = 'http/1.1'
314 run_env = os.environ.copy()
315 run_env['CURL_DBG_SOCK_CONNECT_WAIT_MS'] = '1000'
316 curl = CurlClient(env=env, run_env=run_env)
317 url = f'http://{env.domain1}:{pr_server.port}/xxx'
318 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
319 '--connect-timeout', '5', '--max-time', '5'
320 ])
321 r.check_exit_code(0)
322 r.check_stats(count=1, http_status=200, exitcode=0)
Diagnostics4 git calls · 27 ms
CallStatusSizeTime027 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=22 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.9 KiB · hit22 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=4042 git.response.cache=HIT git.response.duration_ms=22 git.fetch.wants=793230490d31b72c1d7cc6ccd4d0404c288f7a9f git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4042 git.pkt.payload.max=4021 git.sideband.band1.bytes=4020 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e205154a54c69c
- Request
- fad6c6db…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format