master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import logging
26import os
27import signal
28import socket
29import subprocess
30import textwrap
31import time
32from datetime import datetime, timedelta, timezone
33from typing import ClassVar, Dict, Optional
35from .curl import CurlClient
36from .env import Env, NghttpxUtil
37from .ports import alloc_ports_and_do
39log = logging.getLogger(__name__)
42class Nghttpx:
44 def __init__(self, env: Env, name: str, domain: str, cred_name: str):
45 self.env = env
46 self._name = name
47 self._domain = domain
48 self._port = 0
49 self._port_is_quic = False
50 self._cmd = env.nghttpx
51 self._run_dir = os.path.join(env.gen_dir, name)
52 self._pid_file = os.path.join(self._run_dir, 'nghttpx.pid')
53 self._conf_file = os.path.join(self._run_dir, 'nghttpx.conf')
54 self._error_log = os.path.join(self._run_dir, 'nghttpx.log')
55 self._stderr = os.path.join(self._run_dir, 'nghttpx.stderr')
56 self._tmp_dir = os.path.join(self._run_dir, 'tmp')
57 self._error_fd = None
58 self._process: Optional[subprocess.Popen] = None
59 self._cred_name = self._def_cred_name = cred_name
60 self._loaded_cred_name = ''
61 self._version = NghttpxUtil.version(self._cmd)
62 self._uses_ech = False
64 def supports_h3(self):
65 return NghttpxUtil.version_with_h3(self._version)
67 def set_cred_name(self, name: str):
68 self._cred_name = name
70 def reset_config(self):
71 self._cred_name = self._def_cred_name
73 def reload_if_config_changed(self):
74 if self._process and self._port > 0 and \
75 self._loaded_cred_name == self._cred_name:
76 return True
77 return self.reload()
79 @property
80 def port(self):
81 return self._port
83 @property
84 def port_is_quic(self):
85 return self._port_is_quic
87 @property
88 def uses_ech(self):
89 return self._uses_ech
91 def exists(self):
92 return self._cmd and os.path.exists(self._cmd)
94 def close_log(self):
95 if self._error_fd:
96 self._error_fd.close()
97 self._error_fd = None
99 def clear_logs(self):
100 self._rmf(self._error_log)
101 self._rmf(self._stderr)
103 def is_running(self):
104 if self._process:
105 self._process.poll()
106 return self._process.returncode is None
107 return False
109 def start_if_needed(self):
110 if not self.is_running():
111 return self.start()
112 return True
114 def initial_start(self):
115 self._rmf(self._pid_file)
116 self._rmf(self._error_log)
117 self._mkpath(self._run_dir)
118 self._write_config()
120 def start(self, wait_live=True):
121 pass
123 def stop(self, wait_dead=True):
124 self._mkpath(self._tmp_dir)
125 if self._process:
126 self._process.terminate()
127 self._process.wait(timeout=2)
128 self._process = None
129 self.close_log()
130 return not wait_dead or self.wait_dead(timeout=timedelta(seconds=5))
131 self.close_log()
132 return True
134 def restart(self):
135 self.stop()
136 return self.start()
138 def reload(self, timeout: timedelta = timedelta(seconds=Env.SERVER_TIMEOUT)):
139 if self._process:
140 running = self._process
141 self._process = None
142 os.kill(running.pid, signal.SIGQUIT)
143 end_wait = datetime.now(timezone.utc) + timedelta(seconds=5)
144 if not self.start(wait_live=False):
145 self._process = running
146 return False
147 while datetime.now(timezone.utc) < end_wait:
148 try:
149 log.debug(f'waiting for nghttpx({running.pid}) to exit.')
150 running.wait(1)
151 log.debug(f'nghttpx({running.pid}) terminated -> {running.returncode}')
152 running = None
153 break
154 except subprocess.TimeoutExpired:
155 log.warning(f'nghttpx({running.pid}), not shut down yet.')
156 os.kill(running.pid, signal.SIGQUIT)
157 if running and datetime.now(timezone.utc) >= end_wait:
158 log.error(f'nghttpx({running.pid}), terminate forcefully.')
159 os.kill(running.pid, signal.SIGKILL)
160 running.terminate()
161 running.wait(1)
162 return self.wait_live(timeout=timeout)
163 return False
165 def wait_dead(self, timeout: timedelta):
166 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
167 try_until = datetime.now(timezone.utc) + timeout
168 while datetime.now(timezone.utc) < try_until:
169 xargs = [
170 '--trace', 'curl.trace', '--trace-time',
171 '--connect-timeout', '1'
172 ]
173 if self.port_is_quic:
174 xargs.extend(['--http3-only'])
175 check_url = f'https://{self._domain}:{self.port}/'
176 r = curl.http_get(url=check_url, extra_args=xargs)
177 if r.exit_code != 0:
178 return True
179 log.debug(f'waiting for nghttpx to stop responding: {r}')
180 time.sleep(.1)
181 log.debug(f"Server still responding after {timeout}")
182 return False
184 def wait_live(self, timeout: timedelta):
185 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
186 try_until = datetime.now(timezone.utc) + timeout
187 while datetime.now(timezone.utc) < try_until:
188 xargs = [
189 '--trace', 'curl.trace', '--trace-time',
190 '--connect-timeout', '1', '--insecure'
191 ]
192 if self.port_is_quic:
193 xargs.extend(['--http3-only'])
194 check_url = f'https://{self._domain}:{self.port}/'
195 r = curl.http_get(url=check_url, extra_args=xargs)
196 if r.exit_code == 0:
197 return True
198 time.sleep(.1)
199 log.error(f"Server still not responding after {timeout}")
200 return False
202 def _rmf(self, path):
203 if os.path.exists(path):
204 os.remove(path)
206 def _mkpath(self, path):
207 if not os.path.exists(path):
208 os.makedirs(path)
210 def _write_config(self):
211 with open(self._conf_file, 'w') as fd:
212 fd.write(textwrap.dedent("""\
213 # nghttpx test config
214 # do we need something here?
215 """))
218class NghttpxQuic(Nghttpx):
220 PORT_SPECS: ClassVar[Dict[str, int]] = {
221 'nghttpx_https': socket.SOCK_STREAM,
222 }
224 def __init__(self, env: Env):
225 super().__init__(env=env, name='nghttpx-quic',
226 domain=env.domain1, cred_name=env.domain1)
227 self._https_port = 0
229 @property
230 def https_port(self):
231 return self._https_port
233 def initial_start(self):
234 super().initial_start()
236 def startup(ports: Dict[str, int]) -> bool:
237 self._https_port = ports['nghttpx_https']
238 if self.supports_h3():
239 self._port = self.env.h3_port
240 self._port_is_quic = True
241 else:
242 self._port = self._https_port
243 if self.start():
244 self.env.update_ports(ports)
245 return True
246 self.stop()
247 self._port = 0
248 return False
250 return alloc_ports_and_do(NghttpxQuic.PORT_SPECS, startup,
251 self.env.gen_root, max_tries=3)
253 def start(self, wait_live=True):
254 self._mkpath(self._tmp_dir)
255 if self._process:
256 self.stop()
257 creds = self.env.get_credentials(self._cred_name)
258 assert creds # convince pytype this is not None
259 self._loaded_cred_name = self._cred_name
260 args = [self._cmd, f'--frontend=*,{self._https_port};tls']
261 if self.supports_h3():
262 args.extend([
263 f'--frontend=*,{self._port};quic',
264 '--frontend-quic-early-data',
265 ])
266 args.extend([
267 f'--backend=127.0.0.1,{self.env.http_port}',
268 '--log-level=ERROR',
269 f'--pid-file={self._pid_file}',
270 f'--errorlog-file={self._error_log}',
271 f'--conf={self._conf_file}',
272 f'--cacert={self.env.ca.cert_file}',
273 creds.pkey_file,
274 creds.cert_file,
275 '--frontend-http3-window-size=1M',
276 '--frontend-http3-max-window-size=10M',
277 '--frontend-http3-connection-window-size=10M',
278 '--frontend-http3-max-connection-window-size=100M',
279 # f'--frontend-quic-debug-log',
280 ])
281 if self.env.have_nghttpx_ech():
282 ech_conf = self.env.get_echconfig_file(self._cred_name)
283 if ech_conf:
284 args.extend([
285 '--ech-config-file', ech_conf
286 ])
287 self._uses_ech = True
288 self._error_fd = open(self._stderr, 'a') # noqa: SIM115
289 self._process = subprocess.Popen(args=args, stderr=self._error_fd)
290 if self._process.returncode is not None:
291 return False
292 return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
295class NghttpxTcp(Nghttpx):
297 PORT_SPECS: ClassVar[Dict[str, int]] = {
298 'nghttpx_tcp': socket.SOCK_STREAM,
299 }
301 def __init__(self, env: Env):
302 super().__init__(env=env, name='nghttpx-tcp',
303 domain=env.domain1, cred_name=env.domain1)
305 def initial_start(self):
306 super().initial_start()
308 def startup(ports: Dict[str, int]) -> bool:
309 self._port = ports['nghttpx_tcp']
310 if self.start():
311 self.env.update_ports(ports)
312 return True
313 self.stop()
314 self._port = 0
315 return False
317 return alloc_ports_and_do(NghttpxTcp.PORT_SPECS, startup,
318 self.env.gen_root, max_tries=3)
320 def start(self, wait_live=True):
321 self._mkpath(self._tmp_dir)
322 if self._process:
323 self.stop()
324 creds = self.env.get_credentials(self._cred_name)
325 assert creds # convince pytype this is not None
326 self._loaded_cred_name = self._cred_name
327 args = [self._cmd, f'--frontend=*,{self._port};tls']
328 args.extend([
329 f'--backend=127.0.0.1,{self.env.http_port}',
330 '--log-level=ERROR',
331 f'--pid-file={self._pid_file}',
332 f'--errorlog-file={self._error_log}',
333 f'--conf={self._conf_file}',
334 f'--cacert={self.env.ca.cert_file}',
335 creds.pkey_file,
336 creds.cert_file,
337 '--frontend-http3-window-size=1M',
338 '--frontend-http3-max-window-size=10M',
339 '--frontend-http3-connection-window-size=10M',
340 '--frontend-http3-max-connection-window-size=100M',
341 ])
342 if self.env.have_nghttpx_ech():
343 ech_conf = self.env.get_echconfig_file(self._cred_name)
344 if ech_conf:
345 args.extend([
346 '--ech-config-file', ech_conf
347 ])
348 self._uses_ech = True
349 self._error_fd = open(self._stderr, 'a') # noqa: SIM115
350 self._process = subprocess.Popen(args=args, stderr=self._error_fd)
351 if self._process.returncode is not None:
352 return False
353 return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
356class NghttpxFwd(Nghttpx):
358 def __init__(self, env: Env):
359 super().__init__(env=env, name='nghttpx-fwd',
360 domain=env.proxy_domain,
361 cred_name=env.proxy_domain)
363 def initial_start(self):
364 super().initial_start()
366 def startup(ports: Dict[str, int]) -> bool:
367 self._port = ports['h2proxys']
368 if self.start():
369 self.env.update_ports(ports)
370 return True
371 self.stop()
372 self._port = 0
373 return False
375 return alloc_ports_and_do({'h2proxys': socket.SOCK_STREAM},
376 startup, self.env.gen_root, max_tries=3)
378 def start(self, wait_live=True):
379 assert self._port > 0
380 self._mkpath(self._tmp_dir)
381 if self._process:
382 self.stop()
383 creds = self.env.get_credentials(self._cred_name)
384 assert creds # convince pytype this is not None
385 self._loaded_cred_name = self._cred_name
386 args = [
387 self._cmd,
388 '--http2-proxy',
389 f'--frontend=*,{self._port}',
390 f'--backend=127.0.0.1,{self.env.proxy_port}',
391 '--log-level=ERROR',
392 f'--pid-file={self._pid_file}',
393 f'--errorlog-file={self._error_log}',
394 f'--conf={self._conf_file}',
395 f'--cacert={self.env.ca.cert_file}',
396 creds.pkey_file,
397 creds.cert_file,
398 ]
399 self._error_fd = open(self._stderr, 'a') # noqa: SIM115
400 self._process = subprocess.Popen(args=args, stderr=self._error_fd)
401 if self._process.returncode is not None:
402 return False
403 return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
405 def wait_dead(self, timeout: timedelta):
406 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
407 try_until = datetime.now(timezone.utc) + timeout
408 while datetime.now(timezone.utc) < try_until:
409 check_url = f'https://{self.env.proxy_domain}:{self._port}/'
410 r = curl.http_get(url=check_url, extra_args=['--insecure'])
411 if r.exit_code != 0:
412 return True
413 log.debug(f'waiting for nghttpx-fwd to stop responding: {r}')
414 time.sleep(.1)
415 log.debug(f"Server still responding after {timeout}")
416 return False
418 def wait_live(self, timeout: timedelta):
419 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
420 try_until = datetime.now(timezone.utc) + timeout
421 while datetime.now(timezone.utc) < try_until:
422 check_url = f'https://{self.env.proxy_domain}:{self._port}/'
423 r = curl.http_get(url=check_url, extra_args=[
424 '--trace', 'curl.trace', '--trace-time'
425 ])
426 if r.exit_code == 0:
427 return True
428 time.sleep(.1)
429 log.error(f"Server still not responding after {timeout}")
430 return False
Diagnostics4 git calls · 43 ms
CallStatusSizeTime043 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.3 KiB · hit38 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=38 git.response.bytes=3369 git.response.cache=HIT git.response.duration_ms=38 git.fetch.wants=87ba7ec760d3f27c7ed5ebc4951e293059fe2b06 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3369 git.pkt.payload.max=3348 git.sideband.band1.bytes=3347 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e30e4009758a31
- Request
- eb3fe0e5…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format