Artifactspublic-v0

curl

master
ply4 git calls27 ms291 KiB3 cachedprotocol v2
322 lines13 KiBRaw
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
21# SPDX-License-Identifier: curl
23###########################################################################
25import contextlib
26import logging
27import os
28import socket
29import threading
30import time
31from datetime import datetime, timedelta, timezone
32from typing import Dict, Generator
34import pytest
35from testenv import CurlClient, Env
36from testenv.ports import alloc_ports_and_do
38log = logging.getLogger(__name__)
41def pr_process(self):
42 log.info("pr accepting")
43 try:
44 while self._socket and not self._done:
45 conn, _ = self._socket.accept()
46 with conn:
47 conn.sendall(HttpPureResp.RESPONSE)
48 log.info("response sent")
49 except ConnectionAbortedError:
50 return
53class HttpPureResp:
55 RESPONSE = (b'HTTP/1.1 200 OK\r\n'
56 b'Content-Length: 3\r\n'
57 b'Connection: close\r\n'
58 b'\r\n'
59 b'ok\n')
61 def __init__(self, env):
62 self.name = 'httppureresp'
63 self.env = env
64 self.run_dir = os.path.join(env.gen_dir, self.name)
65 self._thread = None
66 self._socket = None
67 self._done = False
68 self.port = 0
70 def start(self):
71 log.info("pr start")
73 def startup(ports: Dict[str, int]) -> bool:
74 self.port = ports[self.name]
75 log.info(f"pr port={self.port}")
76 self._socket = socket.create_server(('127.0.0.1', self.port))
77 log.info("pr socket open")
78 self._thread = threading.Thread(target=pr_process, daemon=True, args=[self])
79 self._thread.start()
80 self.env.update_ports(ports)
81 return True
83 port_spec = {
84 self.name: socket.SOCK_STREAM
85 }
86 assert alloc_ports_and_do(port_spec, startup,
87 self.env.gen_root, max_tries=3)
88 assert self._socket
89 return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
91 def stop(self):
92 self._done = True
93 if self._socket:
94 self._socket.close()
95 self._socket = None
96 log.info("pr stopped")
98 def wait_live(self, timeout: timedelta):
99 curl = CurlClient(env=self.env, run_dir=self.run_dir,
100 timeout=timeout.total_seconds())
101 try_until = datetime.now(timezone.utc) + timeout
102 while datetime.now(timezone.utc) < try_until:
103 r = curl.http_get(url=f'http://{self.env.domain1}:{self.port}/')
104 if r.exit_code == 0:
105 return True
106 time.sleep(.1)
107 log.error(f"Server still not responding after {timeout}")
108 return False
111@pytest.mark.skipif(condition=not Env.httpd_is_at_least('2.4.55'),
112 reason=f"httpd version too old for this: {Env.httpd_version()}")
113class TestErrors:
115 # download 1 file, check that we get CURLE_PARTIAL_FILE
116 @pytest.mark.parametrize("proto", Env.http_protos())
117 def test_05_01_partial_1(self, env: Env, httpd, nghttpx, proto):
118 if proto == 'h3' and env.curl_uses_lib('quiche') and \
119 not env.curl_lib_version_at_least('quiche', '0.29.0'):
120 pytest.skip("quiche issue #2277 not fixed")
121 count = 1
122 curl = CurlClient(env=env)
123 urln = f'https://{env.authority_for(env.domain1, proto)}' \
124 f'/curltest/tweak?id=[0-{count - 1}]'\
125 '&chunks=3&chunk_size=16000&body_error=reset'
126 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
127 '--retry', '0'
128 ])
129 r.check_exit_code(False)
130 invalid_stats = []
131 for idx, s in enumerate(r.stats):
132 if 'exitcode' not in s or s['exitcode'] not in [18, 56, 92, 95]:
133 invalid_stats.append(f'request {idx} exit with {s["exitcode"]}')
134 assert len(invalid_stats) == 0, f'failed: {invalid_stats}'
136 # download files, check that we get CURLE_PARTIAL_FILE for all
137 @pytest.mark.parametrize("proto", Env.http_mplx_protos())
138 def test_05_02_partial_20(self, env: Env, httpd, nghttpx, proto):
139 if proto == 'h3' and env.curl_uses_lib('quiche') and \
140 not env.curl_lib_version_at_least('quiche', '0.29.0'):
141 pytest.skip("quiche issue #2277 not fixed")
142 count = 20
143 curl = CurlClient(env=env)
144 urln = f'https://{env.authority_for(env.domain1, proto)}' \
145 f'/curltest/tweak?id=[0-{count - 1}]'\
146 '&chunks=5&chunk_size=16000&body_error=reset'
147 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
148 '--retry', '0', '--parallel',
149 ])
150 r.check_exit_code(False)
151 assert len(r.stats) == count, f'did not get all stats: {r}'
152 invalid_stats = []
153 for idx, s in enumerate(r.stats):
154 if 'exitcode' not in s or s['exitcode'] not in [18, 55, 56, 92, 95]:
155 invalid_stats.append(f'request {idx} exit with {s["exitcode"]}\n{r.dump_logs()}')
156 assert len(invalid_stats) == 0, f'failed: {invalid_stats}\n{r.dump_logs()}'
158 # access a resource that, on h2, RST the stream with HTTP_1_1_REQUIRED
159 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason="curl without h2")
160 def test_05_03_required(self, env: Env, httpd, nghttpx):
161 curl = CurlClient(env=env)
162 proto = 'http/1.1'
163 urln = f'https://{env.authority_for(env.domain1, proto)}/curltest/1_1'
164 r = curl.http_download(urls=[urln], alpn_proto=proto)
165 r.check_exit_code(0)
166 r.check_response(http_status=200, count=1)
167 proto = 'h2'
168 urln = f'https://{env.authority_for(env.domain1, proto)}/curltest/1_1'
169 r = curl.http_download(urls=[urln], alpn_proto=proto)
170 r.check_exit_code(0)
171 r.check_response(http_status=200, count=1)
172 # check that we did a downgrade
173 assert r.stats[0]['http_version'] == '1.1', r.dump_logs()
175 # On the URL used here, Apache is doing an "unclean" TLS shutdown,
176 # meaning it sends no shutdown notice and closes TCP.
177 # The HTTP response delivers a body without Content-Length. We expect:
178 # - http/1.0 to fail since it relies on a clean connection close to
179 # detect the end of the body
180 # - http/1.1 to work since it will used "chunked" transfer encoding
181 # and stop receiving when that signals the end
182 # - h2 to work since it will signal the end of the response before
183 # and not see the "unclean" close either
184 @pytest.mark.parametrize("proto", ['http/1.0', 'http/1.1', 'h2'])
185 def test_05_04_unclean_tls_shutdown(self, env: Env, httpd, nghttpx, proto):
186 if proto == 'h2' and not env.have_h2_curl():
187 pytest.skip("h2 not supported")
188 if proto == 'h3' and not env.have_h3():
189 pytest.skip("h3 not supported")
190 count = 10 if proto == 'h2' else 1
191 curl = CurlClient(env=env)
192 url = f'https://{env.authority_for(env.domain1, proto)}'\
193 f'/curltest/shutdown_unclean?id=[0-{count-1}]&chunks=4'
194 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
195 '--parallel', '--trace-config', 'ssl'
196 ])
197 if proto == 'http/1.0':
198 # we are inconsistent if we fail or not in missing TLS shutdown
199 # openssl code ignore such errors intentionally in non-debug builds
200 r.check_exit_code(56)
201 else:
202 r.check_exit_code(0)
203 r.check_response(http_status=200, count=count)
205 # Make a connect with a fail for '::1' and wrong certificate for '127.0.0.1'
206 # The error message reported needs to be from the certificate.
207 # verifies issue #20608
208 @pytest.mark.parametrize("proto", Env.http_h1_h2_protos())
209 def test_05_05_failed_peer(self, env: Env, proto, httpd, nghttpx):
210 domain = f'invalid.{env.tld}'
211 url = f'https://{domain}:{env.port_for(proto)}/'
212 run_env = os.environ.copy()
213 run_env['CURL_DBG_SOCK_FAIL_IPV6'] = '1'
214 curl = CurlClient(env=env, run_env=run_env)
215 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
216 '--resolve', f'{domain}:{env.port_for(proto)}:::1,127.0.0.1',
217 ])
218 assert r.exit_code == 60, f'{r}'
219 assert r.stats[0]['errormsg'] != 'CURL_DBG_SOCK_FAIL_IPV6: failed to open socket'
221 # Get, retry on 502
222 def test_05_06_retry_502(self, env: Env, httpd, nghttpx):
223 proto = 'http/1.1'
224 curl = CurlClient(env=env)
225 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=502'
226 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
227 '--retry', '2', '--retry-all-errors', '--retry-delay', '1',
228 ])
229 r.check_response(http_status=502)
230 assert r.stats[0]['num_retries'] == 2, f'{r}'
231 # curious, since curl does the retries, it finds the previous
232 # connection in the cache and reports that no connects were done
233 assert r.stats[0]['num_connects'] == 0, f'{r}'
235 # Get, retry on 502 in parallel mode
236 def test_05_07_retry_502_parallel(self, env: Env, httpd, nghttpx):
237 proto = 'http/1.1'
238 curl = CurlClient(env=env)
239 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=502'
240 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
241 '--retry', '2', '--retry-all-errors', '--retry-delay', '1', '--parallel'
242 ])
243 r.check_response(http_status=502)
244 assert r.stats[0]['num_retries'] == 2, f'{r}'
246 # Get, retry on 401, not happening
247 def test_05_08_retry_401(self, env: Env, httpd, nghttpx):
248 proto = 'http/1.1'
249 curl = CurlClient(env=env)
250 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak?status=401'
251 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
252 '--retry', '2', '--retry-all-errors', '--retry-delay', '1'
253 ])
254 r.check_response(http_status=401)
255 # No retries on a 401
256 assert r.stats[0]['num_retries'] == 0, f'{r}'
258 # Server closes the connection immediately after accept,
259 def test_05_09_handshake_eof(self, env: Env):
260 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
261 server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
262 server.bind(('127.0.0.1', 0))
263 server.listen(1)
264 port = server.getsockname()[1]
266 # accept one connection and immediately close it
267 def accept_and_close():
268 # ignore expected socket error
269 with contextlib.suppress(OSError):
270 conn, _ = server.accept()
271 conn.recv(1) # wait for ClientHello
272 conn.close()
274 t = threading.Thread(target=accept_and_close)
275 t.start()
277 run_env = os.environ.copy()
278 run_env['CURL_DEBUG'] = 'all'
279 curl = CurlClient(env=env, timeout=env.test_timeout, run_env=run_env)
280 url = f'https://127.0.0.1:{port}/'
281 r = curl.run_direct(args=[url, '--insecure', '-v'])
283 t.join(timeout=10)
285 # We expect an error code, not success (0) and not timeout (-1)
286 # Expected error codes are:
287 # - CURLE_SSL_CONNECT_ERROR (35) - common for handshake failures
288 # - CURLE_RECV_ERROR (56) - some TLS backends fail with that
289 assert r.exit_code in [35, 56], \
290 f'unexpected error {r.exit_code}\n{r.dump_logs()}'
292 # Get a resource many times with limited requests
293 @pytest.mark.skipif(condition=not Env.have_h2_curl(), reason='curl without HTTP/2')
294 def test_05_10_limits(self, env: Env, httpd):
295 proto = 'h2'
296 count = 6
297 curl = CurlClient(env=env)
298 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/limit?id=[0-{count-1}]'
299 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
300 '--parallel', '--retry', '5'
301 ])
302 r.check_stats(count=count, http_status=200, exitcode=0)
304 @pytest.fixture(scope='class')
305 @classmethod
306 def pr_server(cls, env: Env) -> Generator[HttpPureResp, None, None]:
307 server = HttpPureResp(env=env)
308 server.start()
309 yield server
310 server.stop()
312 def test_05_11_pure_resp(self, env: Env, pr_server):
313 proto = 'http/1.1'
314 run_env = os.environ.copy()
315 run_env['CURL_DBG_SOCK_CONNECT_WAIT_MS'] = '1000'
316 curl = CurlClient(env=env, run_env=run_env)
317 url = f'http://{env.domain1}:{pr_server.port}/xxx'
318 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
319 '--connect-timeout', '5', '--max-time', '5'
320 ])
321 r.check_exit_code(0)
322 r.check_stats(count=1, http_status=200, exitcode=0)
Diagnostics4 git calls · 27 ms
CallStatusSizeTime027 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=22
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2003.9 KiB · hit22 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4042
git.response.cache=HIT
git.response.duration_ms=22
git.fetch.wants=793230490d31b72c1d7cc6ccd4d0404c288f7a9f
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4042
git.pkt.payload.max=4021
git.sideband.band1.bytes=4020
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e205154a54c69c
Request
fad6c6db…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format