master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import logging
26import os
27import signal
28import socket
29import subprocess
30import time
31from datetime import datetime, timedelta, timezone
32from typing import ClassVar, Dict, Optional
34from .curl import CurlClient
35from .env import Env
36from .ports import alloc_ports_and_do
38log = logging.getLogger(__name__)
41class H2o:
42 def __init__(self, env: Env, name: str, domain: str, cred_name: str):
43 self.env = env
44 self._name = name
45 self._domain = domain
46 self._port = 0 # defaults to h3_port
47 self._cred_name = cred_name
48 self._loaded_cred_name = None
49 self._error_fd = None
50 self._process = None
51 self._tmp_dir = os.path.join(self.env.gen_dir, self._name)
52 self._run_dir = os.path.join(self._tmp_dir, "run")
53 self._conf_file = os.path.join(self._run_dir, "h2o.conf")
54 self._error_log = os.path.join(self._run_dir, "h2o.log")
55 self._pid_file = os.path.join(self._run_dir, "h2o.pid")
56 self._stderr = os.path.join(self._run_dir, "h2o.stderr")
57 self._cmd = env.CONFIG.h2o
58 # For proxy subclasses
59 self._h1_port = None
60 self._h2_port = None
62 @property
63 def port(self) -> int:
64 return self._port
66 @property
67 def h1_port(self) -> Optional[int]:
68 return getattr(self, "_h1_port", None)
70 @property
71 def h2_port(self) -> Optional[int]:
72 return getattr(self, "_h2_port", None)
74 def close_log(self):
75 if self._error_fd:
76 self._error_fd.close()
77 self._error_fd = None
79 def clear_logs(self):
80 self._rmf(self._error_log)
81 self._rmf(self._stderr)
83 def dump_logs(self):
84 lines = []
85 lines.extend([f"stderr of {self._name}",
86 "-------------------------------------------"])
87 self._dump_file(self._stderr, lines)
88 lines.extend(["",
89 f"errorlog of {self._name}",
90 "-------------------------------------------"])
91 self._dump_file(self._error_log, lines)
92 lines.append("")
93 return lines
95 def _rmf(self, path):
96 if os.path.isfile(path):
97 os.remove(path)
99 def _dump_file(self, path, lines):
100 if os.path.isfile(path):
101 with open(path) as fd:
102 for line in fd:
103 lines.append(line.rstrip())
105 def _mkpath(self, path):
106 if not os.path.exists(path):
107 os.makedirs(path)
109 def _log(self, level, msg):
110 getattr(log, level)(f"[{self._name}] {msg}")
112 def is_running(self):
113 if self._process:
114 self._process.poll()
115 return self._process.returncode is None
116 return False
118 def initial_start(self):
119 self._rmf(self._pid_file)
120 self._rmf(self._error_log)
121 self._mkpath(self._run_dir)
122 self.write_config()
124 def start(self, wait_live=True):
125 self._mkpath(self._tmp_dir)
126 self._mkpath(self._run_dir)
127 if self._process:
128 self.stop()
129 self._loaded_cred_name = self._cred_name
130 self.write_config()
131 args = [self._cmd, "-c", self._conf_file]
132 self._error_fd = open(self._stderr, "a") # noqa: SIM115
133 self._process = subprocess.Popen(args=args, stderr=self._error_fd)
134 if self._process.returncode is not None:
135 return False
136 if wait_live:
137 time.sleep(1)
138 # fail fast if h2o rejected the config and already exited
139 self._process.poll()
140 if self._process.returncode is not None:
141 self._log("error",
142 f"h2o exited early (rc={self._process.returncode})"
143 f" - check {self._stderr} for details")
144 self._process = None
145 return False
146 return not wait_live or self.wait_for_state(
147 live=True, timeout=timedelta(seconds=Env.SERVER_TIMEOUT)
148 )
150 def stop(self, wait_dead=True):
151 self._mkpath(self._tmp_dir)
152 if self._process:
153 self._process.terminate()
154 try:
155 self._process.wait(timeout=5)
156 except subprocess.TimeoutExpired:
157 self._process.kill()
158 self._process.wait(timeout=2)
159 self._process = None
160 self.close_log()
161 return not wait_dead or self.wait_for_state(
162 live=False, timeout=timedelta(seconds=5)
163 )
164 self.close_log()
165 return True
167 def kill(self, wait_dead=True):
168 if self._process:
169 self._process.kill()
170 self.close_log()
171 return True
172 self.close_log()
173 return False
175 def restart(self):
176 self.stop()
177 return self.start()
179 def reload(self, timeout: timedelta = timedelta(seconds=Env.SERVER_TIMEOUT)):
180 if self._process:
181 running = self._process
182 self._process = None
183 os.kill(running.pid, signal.SIGQUIT)
184 end_wait = datetime.now(timezone.utc) + timedelta(seconds=5)
185 exited = False
186 if not self.start(wait_live=False):
187 self._process = running
188 return False
189 while datetime.now(timezone.utc) < end_wait:
190 try:
191 self._log("debug", f"waiting for h2o({running.pid}) to exit.")
192 running.wait(1)
193 self._log(
194 "debug",
195 f"h2o({running.pid}) terminated -> {running.returncode}",
196 )
197 exited = True
198 break
199 except subprocess.TimeoutExpired:
200 self._log("warning", f"h2o({running.pid}), not shut down yet.")
201 os.kill(running.pid, signal.SIGQUIT)
202 if not exited and datetime.now(timezone.utc) >= end_wait:
203 self._log("error", f"h2o({running.pid}), terminate forcefully.")
204 os.kill(running.pid, signal.SIGKILL)
205 running.terminate()
206 running.wait(1)
207 return self.wait_for_state(live=True, timeout=timeout)
208 return False
210 def wait_for_state(
211 self,
212 live: bool,
213 timeout: timedelta,
214 url: Optional[str] = None,
215 log_prefix: str = "h2o",
216 ):
217 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
218 try_until = datetime.now(timezone.utc) + timeout
219 if url is None:
220 url = f"https://{self._domain}:{self._port}/"
221 while datetime.now(timezone.utc) < try_until:
222 if live:
223 r = curl.http_get(url=url, extra_args=[
224 '--trace', 'curl.trace', '--trace-time', '--insecure'
225 ])
226 if r.exit_code == 0:
227 return True
228 else:
229 r = curl.http_get(url=url, extra_args=['--insecure'])
230 if r.exit_code != 0:
231 return True
232 time.sleep(0.1)
233 if live:
234 self._log("error", f"Server still not responding after {timeout}")
235 else:
236 self._log("debug", f"Server still responding after {timeout}")
237 return False
239 def write_config(self):
240 # To be overridden by subclasses
241 with open(self._conf_file, "w") as fd:
242 fd.write("# h2o test config\n")
245class H2oServer(H2o):
246 """h2o HTTP/3 server for testing."""
248 PORT_SPECS: ClassVar[Dict[str, int]] = {
249 "h2o_https": socket.SOCK_STREAM,
250 }
252 def __init__(self, env: Env):
253 super().__init__(
254 env=env, name="h2o-server", domain=env.domain1, cred_name=env.domain1
255 )
256 self._docs_dir = os.path.join(self.env.gen_dir, "docs")
258 @property
259 def docs_dir(self):
260 return self._docs_dir
262 def initial_start(self):
263 super().initial_start()
265 def startup(ports: Dict[str, int]) -> bool:
266 self._port = ports["h2o_https"]
267 if self.start():
268 self.env.update_ports(ports)
269 return True
270 self.stop()
271 self._port = 0
272 return False
274 return alloc_ports_and_do(
275 H2oServer.PORT_SPECS, startup, self.env.gen_root, max_tries=3
276 )
278 def write_config(self):
279 creds = self.env.get_credentials(self._cred_name)
280 assert creds # convince pytype this is not None
281 self._mkpath(self._docs_dir)
282 self._mkpath(self._run_dir)
283 # Create a simple test file
284 with open(os.path.join(self._docs_dir, "data.json"), "w") as f:
285 f.write('{"message": "Hello from h2o HTTP/3 server"}\n')
286 with open(self._conf_file, "w") as fd:
287 fd.write(f"""# h2o HTTP/3 server configuration
288server-name: "h2o-test-server"
289num-threads: 1
291listen: &ssl_listen
292 port: {self._port}
293 ssl:
294 certificate-file: {creds.cert_file}
295 key-file: {creds.pkey_file}
296 neverbleed: OFF
297 minimum-version: TLSv1.2
298 ocsp-update-interval: 0
300listen:
301 <<: *ssl_listen
302 type: quic
304hosts:
305 "{self._domain}":
306 paths:
307 "/":
308 file.dir: {self._docs_dir}
310http2-reprioritize-blocking-assets: ON
312access-log: {self._run_dir}/access.log
313error-log: {self._error_log}
314""")
317class H2oProxy(H2o):
318 """h2o MASQUE proxy for testing."""
320 def __init__(self, env: Env):
321 super().__init__(
322 env=env,
323 name="h2o-proxy",
324 domain=env.proxy_domain,
325 cred_name=env.proxy_domain,
326 )
328 def initial_start(self):
329 super().initial_start()
331 def startup(ports: Dict[str, int]) -> bool:
332 self._port = ports["h2o_h3proxys"]
333 self._h2_port = ports["h2o_h2proxys"]
334 self._h1_port = ports["h2o_proxys"]
335 if self.start():
336 self.env.update_ports(ports)
337 return True
338 self.stop()
339 self._port = 0
340 self._h2_port = 0
341 self._h1_port = 0
342 return False
344 return alloc_ports_and_do(
345 {
346 "h2o_h3proxys": socket.SOCK_DGRAM,
347 "h2o_h2proxys": socket.SOCK_STREAM,
348 "h2o_proxys": socket.SOCK_STREAM,
349 },
350 startup,
351 self.env.gen_root,
352 max_tries=3,
353 )
355 def write_config(self):
356 creds = self.env.get_credentials(self._cred_name)
357 assert creds # convince pytype this is not None
358 self._mkpath(self._run_dir)
359 with open(self._conf_file, "w") as fd:
360 fd.write(f"""# h2o MASQUE proxy configuration
361server-name: "h2o-test-proxy"
362num-threads: 1
364proxy.tunnel: ON
366# HTTP/1.1 proxy listener
367listen: &h1_listen
368 port: {getattr(self, "_h1_port", self._port)}
369 ssl:
370 certificate-file: {creds.cert_file}
371 key-file: {creds.pkey_file}
372 neverbleed: OFF
373 minimum-version: TLSv1.2
374 ocsp-update-interval: 0
376# HTTP/2 proxy listener
377listen: &h2_listen
378 port: {getattr(self, "_h2_port", self._port)}
379 ssl:
380 certificate-file: {creds.cert_file}
381 key-file: {creds.pkey_file}
382 neverbleed: OFF
383 minimum-version: TLSv1.2
384 ocsp-update-interval: 0
386# HTTP/3 proxy listener (main port)
387listen: &h3_listen
388 port: {self._port}
389 ssl:
390 certificate-file: {creds.cert_file}
391 key-file: {creds.pkey_file}
392 neverbleed: OFF
393 minimum-version: TLSv1.2
394 ocsp-update-interval: 0
396# QUIC listener for HTTP/3
397listen:
398 <<: *h3_listen
399 type: quic
401hosts:
402 "{self._domain}":
403 paths:
404 "/":
405 proxy.connect: [+*]
406 proxy.ssl.verify-peer: OFF
407 "/.well-known/masque/udp":
408 proxy.connect-udp: [+*]
409 proxy.ssl.verify-peer: OFF
411http2-reprioritize-blocking-assets: ON
413access-log: {self._run_dir}/access.log
414error-log: {self._error_log}
415""")
417 def wait_for_state(
418 self,
419 live: bool,
420 timeout: timedelta,
421 url: Optional[str] = None,
422 log_prefix: str = "h2o",
423 ):
424 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
425 try_until = datetime.now(timezone.utc) + timeout
426 if url is None:
427 url = f"https://{self.env.proxy_domain}:{self._port}/"
428 while datetime.now(timezone.utc) < try_until:
429 if live:
430 r = curl.http_get(
431 url=url, extra_args=["--trace", "curl.trace", "--trace-time"]
432 )
433 if r.exit_code == 0:
434 return True
435 else:
436 r = curl.http_get(url=url)
437 if r.exit_code != 0:
438 return True
439 time.sleep(0.1)
440 if live:
441 self._log("error", f"Proxy still not responding after {timeout}")
442 else:
443 self._log("debug", f"Proxy still responding after {timeout}")
444 return False
Diagnostics4 git calls · 333 ms
CallStatusSizeTime0333 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=327 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.6 KiB327 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3725 git.response.cache=MISS git.response.duration_ms=327 git.fetch.wants=64c24eba2779da47f0284531b2ad0d784e1dd85f git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3725 git.pkt.payload.max=3704 git.sideband.band1.bytes=3703 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2eb5d3d044bc2
- Request
- b5370de8…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format