Artifactspublic-v0

curl

master
ply4 git calls185 ms296 KiB2 cachedprotocol v2
1117 lines44 KiBexecutableRaw
1#!/usr/bin/env python3
2#***************************************************************************
3# _ _ ____ _
4# Project ___| | | | _ \| |
5# / __| | | | |_) | |
6# | (__| |_| | _ <| |___
7# \___|\___/|_| \_\_____|
8#
9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
11# This software is licensed as described in the file COPYING, which
12# you should have received as part of this distribution. The terms
13# are also available at https://curl.se/docs/copyright.html.
15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
16# copies of the Software, and permit persons to whom the Software is
17# furnished to do so, under the terms of the COPYING file.
19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20# KIND, either express or implied.
22# SPDX-License-Identifier: curl
24###########################################################################
26import argparse
27import datetime
28import json
29import logging
30import os
31import re
32import sys
33from statistics import mean
34from typing import Any, Dict, List, Optional
36from testenv import (
37 Caddy,
38 CurlClient,
39 Dante,
40 Env,
41 ExecResult,
42 H2oServer,
43 Httpd,
44 LocalClient,
45 NghttpxQuic,
46 RunProfile,
47 WsServer,
50log = logging.getLogger(__name__)
53class ScoreCardError(Exception):
54 pass
57class Card:
58 @classmethod
59 def fmt_ms(cls, tval):
60 return f'{int(tval * 1000)} ms' if tval >= 0 else '--'
62 @classmethod
63 def fmt_size(cls, val):
64 if val >= (1024 * 1024 * 1024):
65 return f'{val / (1024 * 1024 * 1024):0.000f}GB'
66 if val >= (1024 * 1024):
67 return f'{val / (1024 * 1024):0.000f}MB'
68 if val >= 1024:
69 return f'{val / 1024:0.000f}KB'
70 return f'{val:0.000f}B'
72 @classmethod
73 def fmt_mbs(cls, val):
74 if val is None or val < 0:
75 return '--'
76 if val >= (1024 * 1024 * 1024):
77 return f'{val / (1024 * 1024 * 1024):.3g} GB/s'
78 if val >= (1024 * 1024):
79 return f'{val / (1024 * 1024):.3g} MB/s'
80 if val >= 1024:
81 return f'{val / 1024:.3g} KB/s'
82 return f'{val:.3g} B/s'
84 @classmethod
85 def fmt_speed(cls, val):
86 if val is None or val < 0:
87 return '--'
88 if val >= (10 * 1024 * 1024):
89 return f'{(val / (1024 * 1024)):.3f} MB/s'
90 if val >= (10 * 1024):
91 return f'{val / 1024:.3f} KB/s'
92 return f'{val:.3f} B/s'
94 @classmethod
95 def fmt_speed_result(cls, val, limit):
96 if val is None or val < 0:
97 return '--'
98 pct = ((val / limit) * 100) - 100
99 if val >= (10 * 1024 * 1024):
100 return f'{(val / (1024 * 1024)):.3f} MB/s, {pct:+.1f}%'
101 if val >= (10 * 1024):
102 return f'{val / 1024:.3f} KB/s, {pct:+.1f}%'
103 return f'{val:.3f} B/s, {pct:+.1f}%'
105 @classmethod
106 def fmt_reqs(cls, val):
107 return f'{val:0.000f} r/s' if val >= 0 else '--'
109 @classmethod
110 def mk_mbs_cell(cls, samples, profiles, errors):
111 val = mean(samples) if samples else -1
112 cell = {
113 'val': val,
114 'sval': Card.fmt_mbs(val) if val >= 0 else '--',
115 }
116 if profiles:
117 cell['stats'] = RunProfile.AverageStats(profiles)
118 if errors:
119 cell['errors'] = errors
120 return cell
122 @classmethod
123 def mk_speed_cell(cls, samples, profiles, errors, limit):
124 val = mean(samples) if samples else -1
125 cell = {
126 'val': val,
127 'sval': Card.fmt_speed_result(val, limit) if val >= 0 else '--',
128 }
129 if profiles:
130 cell['stats'] = RunProfile.AverageStats(profiles)
131 if errors:
132 cell['errors'] = errors
133 return cell
135 @classmethod
136 def mk_reqs_cell(cls, samples, profiles, errors):
137 val = mean(samples) if samples else -1
138 cell = {
139 'val': val,
140 'sval': Card.fmt_reqs(val) if val >= 0 else '--',
141 }
142 if profiles:
143 cell['stats'] = RunProfile.AverageStats(profiles)
144 if errors:
145 cell['errors'] = errors
146 return cell
148 @classmethod
149 def parse_size(cls, s):
150 m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE)
151 if m is None:
152 raise ScoreCardError(f'unrecognized size: {s}')
153 size = int(m.group(1))
154 if not m.group(2):
155 pass
156 elif m.group(2).lower() == 'kb':
157 size *= 1024
158 elif m.group(2).lower() == 'mb':
159 size *= 1024 * 1024
160 elif m.group(2).lower() == 'gb':
161 size *= 1024 * 1024 * 1024
162 return size
164 @classmethod
165 def print_score(cls, score):
166 print(f'Scorecard curl, protocol {score["meta"]["protocol"]} '
167 f'via {score["meta"]["implementation"]}/'
168 f'{score["meta"]["implementation_version"]}')
169 print(f'Date: {score["meta"]["date"]}')
170 if 'curl_V' in score["meta"]:
171 print(f'Version: {score["meta"]["curl_V"]}')
172 if 'curl_features' in score["meta"]:
173 print(f'Features: {score["meta"]["curl_features"]}')
174 if 'limit-rate' in score['meta']:
175 print(f'--limit-rate: {score["meta"]["limit-rate"]}')
176 print(f'Samples Size: {score["meta"]["samples"]}')
177 if 'handshakes' in score:
178 print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}')
179 print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} '
180 f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}')
181 for key, val in score["handshakes"].items():
182 print(f' {key:<17} {Card.fmt_ms(val["ipv4-connect"]):>12} '
183 f'{Card.fmt_ms(val["ipv4-handshake"]):>12} '
184 f'{Card.fmt_ms(val["ipv6-connect"]):>12} '
185 f'{Card.fmt_ms(val["ipv6-handshake"]):>12} '
186 f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}'
187 )
188 for name in ['downloads', 'uploads', 'requests']:
189 if name in score:
190 Card.print_score_table(score[name])
192 @classmethod
193 def print_score_table(cls, score):
194 cols = score['cols']
195 rows = score['rows']
196 colw = []
197 statw = 13
198 errors = []
199 col_has_stats = []
200 for idx, col in enumerate(cols):
201 cellw = max(len(r[idx]["sval"]) for r in rows)
202 colw.append(max(cellw, len(col)))
203 col_has_stats.append(False)
204 for row in rows:
205 if 'stats' in row[idx]:
206 col_has_stats[idx] = True
207 break
208 if 'title' in score['meta']:
209 print(score['meta']['title'])
210 for idx, col in enumerate(cols):
211 if col_has_stats[idx]:
212 print(f' {col:>{colw[idx]}} {"[cpu/rss]":<{statw}}', end='')
213 else:
214 print(f' {col:>{colw[idx]}}', end='')
215 print()
216 for row in rows:
217 for idx, cell in enumerate(row):
218 print(f' {cell["sval"]:>{colw[idx]}}', end='')
219 if col_has_stats[idx]:
220 if 'stats' in cell:
221 s = f'[{cell["stats"]["cpu"]:>.1f}%' \
222 f'/{Card.fmt_size(cell["stats"]["rss-max"])}]'
223 else:
224 s = ''
225 print(f' {s:<{statw}}', end='')
226 if 'errors' in cell:
227 errors.extend(cell['errors'])
228 print()
229 if errors:
230 print(f'Errors: {errors}')
233class ScoreRunner:
235 def __init__(self, env: Env,
236 protocol: str,
237 server_descr: str,
238 server_port: int,
239 verbose: int,
240 curl_verbose: int,
241 download_parallel: int = 0,
242 upload_parallel: int = 0,
243 server_addr: Optional[str] = None,
244 with_flame: bool = False,
245 socks_args: Optional[List[str]] = None,
246 limit_rate: Optional[str] = None,
247 http_plain: bool = False,
248 suppress_cl: bool = False):
249 self.verbose = verbose
250 self.env = env
251 self.protocol = protocol
252 self.server_descr = server_descr
253 self.server_addr = server_addr
254 self.server_port = server_port
255 self._silent_curl = not curl_verbose
256 self._download_parallel = download_parallel
257 self._upload_parallel = upload_parallel
258 self._with_flame = with_flame
259 self._socks_args = socks_args
260 self._limit_rate_num = 0
261 self._limit_rate = limit_rate
262 self._http_plain = http_plain
263 if self.protocol in ['http/1.1', 'h2', 'h3']:
264 self._scheme = 'http' if http_plain else 'https'
265 self._alpn_proto = self.protocol
266 elif self.protocol == 'ws':
267 self._scheme = 'ws'
268 self._alpn_proto = 'http/1.1'
269 else:
270 raise ScoreCardError(f'unknown protocol {self.protocol}')
271 if self._limit_rate:
272 m = re.match(r'(\d+(\.\d+)?)([gmkb])?', self._limit_rate.lower())
273 if not m:
274 raise ScoreCardError(f'unrecognised limit-rate: {self._limit_rate}')
275 self._limit_rate_num = float(m.group(1))
276 if m.group(3) == 'g':
277 self._limit_rate_num *= 1024 * 1024 * 1024
278 elif m.group(3) == 'm':
279 self._limit_rate_num *= 1024 * 1024
280 elif m.group(3) == 'k':
281 self._limit_rate_num *= 1024
282 elif m.group(3) == 'b':
283 pass
284 else:
285 raise ScoreCardError(f'unrecognised limit-rate: {self._limit_rate}')
286 self.suppress_cl = suppress_cl
288 def info(self, msg):
289 if self.verbose > 0:
290 sys.stderr.write(msg)
291 sys.stderr.flush()
293 def mk_curl_client(self):
294 return CurlClient(env=self.env, silent=self._silent_curl,
295 server_addr=self.server_addr,
296 with_flame=self._with_flame,
297 socks_args=self._socks_args)
299 def handshakes(self) -> Dict[str, Any]:
300 props = {}
301 sample_size = 5
302 self.info('TLS Handshake\n')
303 for authority in [
304 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org'
305 ]:
306 self.info(f' {authority}...')
307 props[authority] = {}
308 for ipv in ['ipv4', 'ipv6']:
309 self.info(f'{ipv}...')
310 c_samples = []
311 hs_samples = []
312 errors = []
313 for _ in range(sample_size):
314 curl = self.mk_curl_client()
315 args = [
316 '--http3-only' if self.protocol == 'h3' else '--http2',
317 f'--{ipv}', f'{self._scheme}://{authority}/'
318 ]
319 r = curl.run_direct(args=args, with_stats=True)
320 if r.exit_code == 0 and len(r.stats) == 1:
321 c_samples.append(r.stats[0]['time_connect'])
322 hs_samples.append(r.stats[0]['time_appconnect'])
323 else:
324 errors.append(f'exit={r.exit_code}')
325 props[authority][f'{ipv}-connect'] = mean(c_samples) \
326 if c_samples else -1
327 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \
328 if hs_samples else -1
329 props[authority][f'{ipv}-errors'] = errors
330 self.info('ok.\n')
331 return props
333 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int):
334 fpath = os.path.join(docs_dir, fname)
335 data1k = 1024*'x'
336 flen = 0
337 with open(fpath, 'w') as fd:
338 while flen < fsize:
339 fd.write(data1k)
340 flen += len(data1k)
341 return fpath
343 def setup_resources(self, server_docs: str,
344 downloads: Optional[List[int]] = None):
345 if downloads is not None:
346 for fsize in downloads:
347 label = Card.fmt_size(fsize)
348 fname = f'score{label}.data'
349 self._make_docs_file(docs_dir=server_docs,
350 fname=fname, fsize=fsize)
351 self._make_docs_file(docs_dir=server_docs,
352 fname='reqs10.data', fsize=10 * 1024)
354 def _check_downloads(self, r: ExecResult, count: int):
355 error = ''
356 if r.exit_code != 0:
357 error += f'exit={r.exit_code} '
358 if r.exit_code != 0 or len(r.stats) != count:
359 error += f'stats={len(r.stats)}/{count} '
360 exp_resp = 101 if self.protocol == 'ws' else 200
361 fails = [s for s in r.stats if s['response_code'] != exp_resp]
362 if len(fails) > 0:
363 error += f'{len(fails)} failed, {fails}'
364 return error if len(error) > 0 else None
366 def dl_single(self, url: str, nsamples: int = 1):
367 count = 1
368 samples = []
369 errors = []
370 profiles = []
371 self.info('single...')
372 for _ in range(nsamples):
373 curl = self.mk_curl_client()
374 r = curl.http_download(urls=[url], alpn_proto=self._alpn_proto,
375 no_save=True, with_headers=False,
376 with_profile=True,
377 limit_rate=self._limit_rate)
378 err = self._check_downloads(r, count)
379 if err:
380 errors.append(err)
381 elif self._limit_rate:
382 total_speed = sum(s['speed_download'] for s in r.stats)
383 samples.append(total_speed / len(r.stats))
384 profiles.append(r.profile)
385 else:
386 total_size = sum(s['size_download'] for s in r.stats)
387 samples.append(total_size / r.duration.total_seconds())
388 profiles.append(r.profile)
389 if self._limit_rate:
390 return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
391 return Card.mk_mbs_cell(samples, profiles, errors)
393 def dl_serial(self, url: str, count: int, nsamples: int = 1):
394 samples = []
395 errors = []
396 profiles = []
397 url = f'{url}?[0-{count - 1}]'
398 self.info('serial...')
399 for _ in range(nsamples):
400 curl = self.mk_curl_client()
401 r = curl.http_download(urls=[url], alpn_proto=self._alpn_proto,
402 no_save=True,
403 with_headers=False,
404 with_profile=True,
405 limit_rate=self._limit_rate)
406 err = self._check_downloads(r, count)
407 if err:
408 errors.append(err)
409 elif self._limit_rate:
410 total_speed = sum(s['speed_download'] for s in r.stats)
411 samples.append(total_speed / len(r.stats))
412 profiles.append(r.profile)
413 else:
414 total_size = sum(s['size_download'] for s in r.stats)
415 samples.append(total_size / r.duration.total_seconds())
416 profiles.append(r.profile)
417 if self._limit_rate:
418 return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
419 return Card.mk_mbs_cell(samples, profiles, errors)
421 def dl_parallel(self, url: str, count: int, nsamples: int = 1):
422 samples = []
423 errors = []
424 profiles = []
425 max_parallel = self._download_parallel if self._download_parallel > 0 else count
426 url = f'{url}?[0-{count - 1}]'
427 self.info('parallel...')
428 for _ in range(nsamples):
429 curl = self.mk_curl_client()
430 r = curl.http_download(urls=[url], alpn_proto=self._alpn_proto,
431 no_save=True,
432 with_headers=False,
433 with_profile=True,
434 limit_rate=self._limit_rate,
435 extra_args=[
436 '--parallel',
437 '--parallel-max', str(max_parallel)
438 ])
439 err = self._check_downloads(r, count)
440 if err:
441 errors.append(err)
442 elif self._limit_rate:
443 total_speed = sum(s['speed_download'] for s in r.stats)
444 samples.append(total_speed / len(r.stats))
445 profiles.append(r.profile)
446 else:
447 total_size = sum(s['size_download'] for s in r.stats)
448 samples.append(total_size / r.duration.total_seconds())
449 profiles.append(r.profile)
450 if self._limit_rate:
451 return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
452 return Card.mk_mbs_cell(samples, profiles, errors)
454 def downloads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
455 nsamples = meta['samples']
456 max_parallel = self._download_parallel if self._download_parallel > 0 else count
457 cols = ['size']
458 if not self._download_parallel or count == 1:
459 cols.append('single')
460 if count > 1:
461 cols.append(f'serial({count})')
462 if count > 1:
463 if max_parallel == 1:
464 cols.append(f'serial({count})')
465 else:
466 cols.append(f'parallel({count}x{max_parallel})')
467 rows = []
468 for fsize in fsizes:
469 row = [{
470 'val': fsize,
471 'sval': Card.fmt_size(fsize)
472 }]
473 self.info(f'{row[0]["sval"]} downloads...')
474 if self.protocol == 'ws':
475 url = f'{self._scheme}://{self.env.domain1}:{self.server_port}/{row[0]["sval"]}'
476 else:
477 url = f'{self._scheme}://{self.env.domain1}:{self.server_port}/score{row[0]["sval"]}.data'
478 if 'single' in cols:
479 row.append(self.dl_single(url=url, nsamples=nsamples))
480 if count > 1:
481 if 'single' in cols:
482 row.append(self.dl_serial(url=url, count=count, nsamples=nsamples))
483 row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples))
484 rows.append(row)
485 self.info('done.\n')
486 if self._limit_rate:
487 title = f'Download Speed ({self.protocol}), limit={Card.fmt_speed(self._limit_rate_num)}, from {meta["server"]}'
488 else:
489 title = f'Downloads ({self.protocol}) from {meta["server"]}'
490 if self._socks_args:
491 title += f' via {self._socks_args}'
492 return {
493 'meta': {
494 'title': title,
495 'count': count,
496 'max-parallel': max_parallel,
497 },
498 'cols': cols,
499 'rows': rows,
500 }
502 def _check_uploads(self, r: ExecResult, count: int):
503 error = ''
504 if r.exit_code != 0:
505 error += f'exit={r.exit_code} '
506 if r.exit_code != 0 or len(r.stats) != count:
507 error += f'stats={len(r.stats)}/{count} '
508 fails = [s for s in r.stats if s['response_code'] != 200]
509 if len(fails) > 0:
510 error += f'{len(fails)} failed'
511 for f in fails:
512 error += f'[{f["response_code"]}]'
513 return error if len(error) > 0 else None
515 def ul_run(self, url, count, fsize, fpath, xargs=None):
516 if self.protocol == 'ws':
517 frame_size = 16384
518 nframes = max(fsize / frame_size, 1)
519 url = f'ws://localhost:{self.server_port}/'
520 client = LocalClient(env=self.env, name='cli_ws_data')
521 if not client.exists():
522 raise ScoreCardError(f'example client not built: {client.name}')
523 r = client.run(args=['-1', '-c', str(nframes), '-m', str(frame_size), url],
524 with_profile=True)
525 if r.exit_code == 0:
526 r.set_stats([{
527 'size_upload': fsize,
528 'response_code': 200,
529 }])
530 return r
532 curl = self.mk_curl_client()
533 return curl.http_put(urls=[url], fdata=fpath, alpn_proto=self._alpn_proto,
534 with_headers=False, with_profile=True,
535 suppress_cl=self.suppress_cl,
536 extra_args=xargs)
538 def ul_single(self, url: str, fsize: int, fpath: str, nsamples: int = 1):
539 samples = []
540 errors = []
541 profiles = []
542 self.info('single...')
543 for _ in range(nsamples):
544 r = self.ul_run(url, 1, fsize, fpath)
545 err = self._check_uploads(r, 1)
546 if err:
547 errors.append(err)
548 else:
549 total_size = sum(s['size_upload'] for s in r.stats)
550 samples.append(total_size / r.duration.total_seconds())
551 profiles.append(r.profile)
552 return Card.mk_mbs_cell(samples, profiles, errors)
554 def ul_serial(self, url: str, fsize: int, fpath: str, count: int, nsamples: int = 1):
555 samples = []
556 errors = []
557 profiles = []
558 url = f'{url}?id=[0-{count - 1}]'
559 self.info('serial...')
560 for _ in range(nsamples):
561 r = self.ul_run(url, count, fsize, fpath)
562 err = self._check_uploads(r, count)
563 if err:
564 errors.append(err)
565 else:
566 total_size = sum(s['size_upload'] for s in r.stats)
567 samples.append(total_size / r.duration.total_seconds())
568 profiles.append(r.profile)
569 return Card.mk_mbs_cell(samples, profiles, errors)
571 def ul_parallel(self, url: str, fsize: int, fpath: str, count: int, nsamples: int = 1):
572 samples = []
573 errors = []
574 profiles = []
575 max_parallel = self._upload_parallel if self._upload_parallel > 0 else count
576 url = f'{url}?id=[0-{count - 1}]'
577 self.info('parallel...')
578 for _ in range(nsamples):
579 r = self.ul_run(url, count, fsize, fpath, xargs=[
580 '--parallel', '--parallel-max', str(max_parallel),
581 ])
582 err = self._check_uploads(r, count)
583 if err:
584 errors.append(err)
585 else:
586 total_size = sum(s['size_upload'] for s in r.stats)
587 samples.append(total_size / r.duration.total_seconds())
588 profiles.append(r.profile)
589 return Card.mk_mbs_cell(samples, profiles, errors)
591 def uploads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
592 nsamples = meta['samples']
593 max_parallel = self._upload_parallel if self._upload_parallel > 0 else count
594 cols = ['size']
595 run_single = not self._upload_parallel
596 run_serial = not self._upload_parallel and count > 1
597 run_parallel = self._upload_parallel or count > 1
598 if run_single:
599 cols.append('single')
600 if run_serial:
601 cols.append(f'serial({count})')
602 if run_parallel:
603 cols.append(f'parallel({count}x{max_parallel})')
604 rows = []
605 for fsize in fsizes:
606 row = [{
607 'val': fsize,
608 'sval': Card.fmt_size(fsize)
609 }]
610 self.info(f'{row[0]["sval"]} uploads...')
611 url = f'{self._scheme}://{self.env.domain1}:{self.server_port}/curltest/put'
612 fname = f'upload{row[0]["sval"]}.data'
613 fpath = self._make_docs_file(docs_dir=self.env.gen_dir,
614 fname=fname, fsize=fsize)
615 if run_single:
616 row.append(self.ul_single(url=url, fsize=fsize, fpath=fpath, nsamples=nsamples))
617 if run_serial:
618 row.append(self.ul_serial(url=url, fsize=fsize, fpath=fpath, count=count, nsamples=nsamples))
619 if run_parallel:
620 row.append(self.ul_parallel(url=url, fsize=fsize, fpath=fpath, count=count, nsamples=nsamples))
621 rows.append(row)
622 self.info('done.\n')
623 title = f'Uploads to {meta["server"]}'
624 if self._socks_args:
625 title += f' via {self._socks_args}'
626 return {
627 'meta': {
628 'title': title,
629 'count': count,
630 'max-parallel': max_parallel,
631 },
632 'cols': cols,
633 'rows': rows,
634 }
636 def do_requests(self, url: str, count: int, max_parallel: int = 1, nsamples: int = 1):
637 samples = []
638 errors = []
639 profiles = []
640 url = f'{url}?[0-{count - 1}]'
641 extra_args = [
642 '-w', '%{response_code},\\n',
643 ]
644 if max_parallel > 1:
645 extra_args.extend([
646 '--parallel', '--parallel-max', str(max_parallel)
647 ])
648 self.info(f'{max_parallel}...')
649 for _ in range(nsamples):
650 curl = self.mk_curl_client()
651 r = curl.http_download(urls=[url], alpn_proto=self._alpn_proto, no_save=True,
652 with_headers=False, with_profile=True,
653 with_stats=False, extra_args=extra_args)
654 if r.exit_code != 0:
655 errors.append(f'exit={r.exit_code}')
656 else:
657 samples.append(count / r.duration.total_seconds())
658 non_200s = 0
659 for line in r.stdout.splitlines():
660 if not line.startswith('200,'):
661 non_200s += 1
662 if non_200s > 0:
663 errors.append(f'responses != 200: {non_200s}')
664 profiles.append(r.profile)
665 return Card.mk_reqs_cell(samples, profiles, errors)
667 def requests(self, count: int, meta: Dict[str, Any]) -> Dict[str, Any]:
668 url = f'{self._scheme}://{self.env.domain1}:{self.server_port}/reqs10.data'
669 fsize = 10 * 1024
670 cols = ['size', 'total']
671 rows = []
672 mparallel = meta['request_parallels']
673 cols.extend([f'{mp} max' for mp in mparallel])
674 row = [
675 {
676 'val': fsize,
677 'sval': Card.fmt_size(fsize)
678 }, {
679 'val': count,
680 'sval': f'{count}',
681 }
682 ]
683 self.info('requests, max parallel...')
684 row.extend([self.do_requests(url=url, count=count,
685 max_parallel=mp, nsamples=meta["samples"])
686 for mp in mparallel])
687 rows.append(row)
688 self.info('done.\n')
689 title = f'Requests in parallel to {meta["server"]}'
690 if self._socks_args:
691 title += f' via {self._socks_args}'
692 return {
693 'meta': {
694 'title': title,
695 'count': count,
696 },
697 'cols': cols,
698 'rows': rows,
699 }
701 def score(self,
702 handshakes: bool = True,
703 downloads: Optional[List[int]] = None,
704 download_count: int = 50,
705 uploads: Optional[List[int]] = None,
706 upload_count: int = 50,
707 req_count=5000,
708 request_parallels=None,
709 nsamples: int = 1,
710 requests: bool = True):
711 self.info(f"scoring {self.protocol} against {self.server_descr}\n")
713 score = {
714 'meta': {
715 'curl_version': self.env.curl_version(),
716 'curl_V': self.env.curl_fullname(),
717 'curl_features': self.env.curl_features_string(),
718 'os': self.env.curl_os(),
719 'server': self.server_descr,
720 'samples': nsamples,
721 'date': f'{datetime.datetime.now(datetime.timezone.utc).isoformat()}',
722 }
723 }
724 if self._limit_rate:
725 score['meta']['limit-rate'] = self._limit_rate
727 if self.protocol == 'h3':
728 score['meta']['protocol'] = 'h3'
729 if not self.env.have_h3_curl():
730 raise ScoreCardError('curl does not support HTTP/3')
731 for lib in ['ngtcp2', 'quiche', 'nghttp3']:
732 if self.env.curl_uses_lib(lib):
733 score['meta']['implementation'] = lib
734 break
735 elif self.protocol == 'h2':
736 score['meta']['protocol'] = 'h2'
737 if not self.env.have_h2_curl():
738 raise ScoreCardError('curl does not support HTTP/2')
739 for lib in ['nghttp2']:
740 if self.env.curl_uses_lib(lib):
741 score['meta']['implementation'] = lib
742 break
743 elif self.protocol == 'h1' or self.protocol == 'http/1.1':
744 score['meta']['protocol'] = 'http/1.1'
745 score['meta']['implementation'] = 'native'
746 elif self.protocol == 'ws':
747 score['meta']['protocol'] = 'ws'
748 score['meta']['implementation'] = 'native'
749 else:
750 raise ScoreCardError(f"unknown protocol: {self.protocol}")
752 if 'implementation' not in score['meta']:
753 raise ScoreCardError('did not recognized protocol lib')
754 score['meta']['implementation_version'] = Env.curl_lib_version(score['meta']['implementation'])
756 if handshakes:
757 score['handshakes'] = self.handshakes()
758 if downloads and len(downloads) > 0:
759 score['downloads'] = self.downloads(count=download_count,
760 fsizes=downloads,
761 meta=score['meta'])
762 if uploads and len(uploads) > 0:
763 score['uploads'] = self.uploads(count=upload_count,
764 fsizes=uploads,
765 meta=score['meta'])
766 if requests:
767 if request_parallels is None:
768 request_parallels = [1, 6, 25, 50, 100, 300]
769 score['meta']['request_parallels'] = request_parallels
770 score['requests'] = self.requests(count=req_count, meta=score['meta'])
771 return score
774def run_score(args, protocol):
775 if protocol not in ['http/1.1', 'h1', 'h2', 'h3', 'ws']:
776 sys.stderr.write(f'ERROR: protocol "{protocol}" not known to scorecard\n')
777 sys.exit(1)
778 if protocol == 'h1':
779 protocol = 'http/1.1'
781 handshakes = protocol in ['http/1.1', 'h1', 'h2', 'h3']
782 downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
783 if args.download_sizes is not None:
784 downloads = []
785 for x in args.download_sizes:
786 downloads.extend([Card.parse_size(s) for s in x.split(',')])
788 uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
789 if args.upload_sizes is not None:
790 uploads = []
791 for x in args.upload_sizes:
792 uploads.extend([Card.parse_size(s) for s in x.split(',')])
794 requests = True
795 request_parallels = None
796 if args.request_parallels:
797 request_parallels = []
798 for x in args.request_parallels:
799 request_parallels.extend([int(s) for s in x.split(',')])
801 if args.downloads or args.uploads or args.requests or args.handshakes:
802 handshakes = args.handshakes
803 if not args.downloads:
804 downloads = None
805 if not args.uploads:
806 uploads = None
807 requests = args.requests
809 rv = 0
810 env = Env()
811 env.setup()
812 env.test_timeout = None
814 test_httpd = protocol in ['http/1.1', 'h2']
815 test_h2o = protocol == 'h3' and env.have_h2o()
816 test_caddy = protocol == 'h3' and not test_h2o
817 if args.caddy or args.httpd or args.h2o:
818 test_caddy = args.caddy
819 test_httpd = args.httpd
820 test_h2o = args.h2o
821 test_ws = protocol == 'ws'
823 sockd = None
824 socks_args = None
825 if args.socks4 and args.socks5:
826 raise ScoreCardError('unable to run --socks4 and --socks5 together')
827 if args.socks4 or args.socks5:
828 sockd = Dante(env=env)
829 if sockd:
830 assert sockd.initial_start()
831 socks_args = [
832 '--socks4' if args.socks4 else '--socks5',
833 f'127.0.0.1:{sockd.port}',
834 ]
836 httpd = None
837 nghttpx = None
838 caddy = None
839 h2o = None
840 wsserver = None
841 try:
842 cards = []
844 if args.remote:
845 m = re.match(r'^(.+):(\d+)$', args.remote)
846 if m is None:
847 raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}')
848 test_httpd = False
849 test_caddy = False
850 remote_addr = m.group(1)
851 remote_port = int(m.group(2))
852 card = ScoreRunner(env=env,
853 protocol=protocol,
854 server_descr=f'Server at {args.remote}',
855 server_addr=remote_addr,
856 server_port=remote_port,
857 verbose=args.verbose,
858 curl_verbose=args.curl_verbose,
859 download_parallel=args.download_parallel,
860 upload_parallel=args.upload_parallel,
861 with_flame=args.flame,
862 socks_args=socks_args,
863 limit_rate=args.limit_rate,
864 suppress_cl=args.upload_no_cl)
865 cards.append(card)
867 if test_httpd:
868 httpd = Httpd(env=env)
869 assert httpd.exists(), \
870 f'httpd not found: {env.httpd}'
871 httpd.clear_logs()
872 server_docs = httpd.docs_dir
873 assert httpd.initial_start()
874 if protocol == 'h3':
875 nghttpx = NghttpxQuic(env=env)
876 nghttpx.clear_logs()
877 assert nghttpx.initial_start()
878 server_descr = f'nghttpx: https:{env.h3_port} [backend httpd/{env.httpd_version()}]'
879 server_port = env.h3_port
880 else:
881 server_descr = f'httpd/{env.httpd_version()}'
882 server_port = env.http_port if args.http_plain else env.https_port
883 card = ScoreRunner(env=env,
884 protocol=protocol,
885 server_descr=server_descr,
886 server_port=server_port,
887 verbose=args.verbose, curl_verbose=args.curl_verbose,
888 download_parallel=args.download_parallel,
889 upload_parallel=args.upload_parallel,
890 with_flame=args.flame,
891 socks_args=socks_args,
892 limit_rate=args.limit_rate,
893 http_plain=args.http_plain)
894 card.setup_resources(server_docs, downloads)
895 cards.append(card)
897 if test_h2o:
898 h2o = H2oServer(env=env)
899 h2o.clear_logs()
900 assert h2o.initial_start()
901 server_descr = f'H2o/{env.h2o_version()}'
902 server_port = h2o.port
903 server_docs = h2o.docs_dir
904 card = ScoreRunner(env=env,
905 protocol=protocol,
906 server_descr=server_descr,
907 server_port=server_port,
908 verbose=args.verbose, curl_verbose=args.curl_verbose,
909 download_parallel=args.download_parallel,
910 upload_parallel=args.upload_parallel,
911 with_flame=args.flame,
912 socks_args=socks_args,
913 limit_rate=args.limit_rate)
914 card.setup_resources(server_docs, downloads)
915 cards.append(card)
917 if test_caddy and env.caddy:
918 backend = ''
919 if uploads and httpd is None:
920 backend = f' [backend httpd: {env.httpd_version()}]'
921 httpd = Httpd(env=env)
922 assert httpd.exists(), \
923 f'httpd not found: {env.httpd}'
924 httpd.clear_logs()
925 assert httpd.initial_start()
926 caddy = Caddy(env=env)
927 caddy.clear_logs()
928 assert caddy.initial_start()
929 server_descr = f'Caddy/{env.caddy_version()} {backend}'
930 server_port = caddy.port
931 server_docs = caddy.docs_dir
932 card = ScoreRunner(env=env,
933 protocol=protocol,
934 server_descr=server_descr,
935 server_port=server_port,
936 verbose=args.verbose, curl_verbose=args.curl_verbose,
937 download_parallel=args.download_parallel,
938 upload_parallel=args.upload_parallel,
939 with_flame=args.flame,
940 socks_args=socks_args,
941 limit_rate=args.limit_rate)
942 card.setup_resources(server_docs, downloads)
943 cards.append(card)
945 if test_ws:
946 wsname = 'ws_echo' if args.uploads else 'ws_dl'
947 wsserver = WsServer(env=env, name=wsname)
948 wsserver.startup()
949 server_descr = 'WsServer/echo'
950 server_port = wsserver.port
951 card = ScoreRunner(env=env,
952 protocol=protocol,
953 server_descr=server_descr,
954 server_port=server_port,
955 verbose=args.verbose,
956 curl_verbose=args.curl_verbose,
957 download_parallel=args.download_parallel,
958 upload_parallel=args.upload_parallel,
959 with_flame=args.flame,
960 socks_args=socks_args,
961 limit_rate=args.limit_rate)
962 cards.append(card)
964 if not len(cards):
965 raise ScoreCardError(f"no cards selected for {protocol}")
967 if args.start_only:
968 print('started servers:')
969 for card in cards:
970 print(f'{card.server_descr}')
971 sys.stderr.write('press [RETURN] to finish')
972 sys.stderr.flush()
973 sys.stdin.readline()
974 else:
975 for card in cards:
976 score = card.score(handshakes=handshakes,
977 downloads=downloads,
978 download_count=args.download_count,
979 uploads=uploads,
980 upload_count=args.upload_count,
981 req_count=args.request_count,
982 requests=requests,
983 request_parallels=request_parallels,
984 nsamples=args.samples)
985 if args.json:
986 print(json.JSONEncoder(indent=2).encode(score))
987 else:
988 Card.print_score(score)
990 except ScoreCardError as ex:
991 sys.stderr.write(f"ERROR: {ex}\n")
992 rv = 1
993 except KeyboardInterrupt:
994 log.warning("aborted")
995 rv = 1
996 finally:
997 if caddy:
998 caddy.stop()
999 if nghttpx:
1000 nghttpx.stop(wait_dead=False)
1001 if h2o:
1002 h2o.stop()
1003 if httpd:
1004 httpd.stop()
1005 if sockd:
1006 sockd.stop()
1007 if wsserver:
1008 wsserver.shutdown()
1009 return rv
1012def print_file(filename):
1013 if not os.path.exists(filename):
1014 sys.stderr.write(f"ERROR: file does not exist {filename}\n")
1015 return 1
1016 with open(filename) as file:
1017 data = json.load(file)
1018 Card.print_score(data)
1019 return 0
1022def main():
1023 parser = argparse.ArgumentParser(prog='scorecard', description="""
1024 Run a range of tests to give a scorecard for an HTTP protocol
1025 'h3' or 'h2' implementation in curl.
1026 """)
1027 parser.add_argument("-v", "--verbose", action='count', default=1,
1028 help="log more output on stderr")
1029 parser.add_argument("-j", "--json", action='store_true',
1030 default=False, help="print json instead of text")
1031 parser.add_argument("--samples", action='store', type=int, metavar='number',
1032 default=1, help="how many sample runs to make")
1033 parser.add_argument("--httpd", action='store_true', default=False,
1034 help="evaluate httpd server only")
1035 parser.add_argument("--h2o", action='store_true', default=False,
1036 help="evaluate h2o server only")
1037 parser.add_argument("--caddy", action='store_true', default=False,
1038 help="evaluate caddy server only")
1039 parser.add_argument("--curl-verbose", action='store_true',
1040 default=False, help="run curl with `-v`")
1041 parser.add_argument("--print", type=str, default=None, metavar='filename',
1042 help="print the results from a JSON file")
1043 parser.add_argument("protocol", default=None, nargs='?',
1044 help="Name of protocol to score")
1045 parser.add_argument("--start-only", action='store_true', default=False,
1046 help="only start the servers")
1047 parser.add_argument("--remote", action='store', type=str,
1048 default=None, help="score against the remote server at <ip>:<port>")
1049 parser.add_argument("--flame", action='store_true',
1050 default=False, help="produce a flame graph on curl")
1051 parser.add_argument("--limit-rate", action='store', type=str,
1052 default=None, help="use curl's --limit-rate")
1053 parser.add_argument("--http-plain", action='store_true',
1054 default=False, help="run http: test instead of https:")
1056 parser.add_argument("-H", "--handshakes", action='store_true',
1057 default=False, help="evaluate handshakes only")
1059 parser.add_argument("-d", "--downloads", action='store_true',
1060 default=False, help="evaluate downloads")
1061 parser.add_argument("--download-sizes", action='append', type=str,
1062 metavar='numberlist',
1063 default=None, help="evaluate download size")
1064 parser.add_argument("--download-count", action='store', type=int,
1065 metavar='number',
1066 default=50, help="perform that many downloads")
1067 parser.add_argument("--download-parallel", action='store', type=int,
1068 metavar='number', default=0,
1069 help="perform that many downloads in parallel (default all)")
1071 parser.add_argument("-u", "--uploads", action='store_true',
1072 default=False, help="evaluate uploads")
1073 parser.add_argument("--upload-sizes", action='append', type=str,
1074 metavar='numberlist',
1075 default=None, help="evaluate upload size")
1076 parser.add_argument("--upload-count", action='store', type=int,
1077 metavar='number', default=50,
1078 help="perform that many uploads")
1079 parser.add_argument("--upload-parallel", action='store', type=int,
1080 metavar='number', default=0,
1081 help="perform that many uploads in parallel (default all)")
1082 parser.add_argument("--upload-no-cl", action='store_true',
1083 default=False, help="suppress content-length on upload")
1085 parser.add_argument("-r", "--requests", action='store_true',
1086 default=False, help="evaluate requests")
1087 parser.add_argument("--request-count", action='store', type=int,
1088 metavar='number',
1089 default=5000, help="perform that many requests")
1090 parser.add_argument("--request-parallels", action='append', type=str,
1091 metavar='numberlist',
1092 default=None, help="evaluate request with these max-parallel numbers")
1093 parser.add_argument("--socks4", action='store_true',
1094 default=False, help="test with SOCKS4 proxy")
1095 parser.add_argument("--socks5", action='store_true',
1096 default=False, help="test with SOCKS5 proxy")
1097 args = parser.parse_args()
1099 if args.verbose > 0:
1100 console = logging.StreamHandler()
1101 console.setLevel(logging.INFO)
1102 console.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
1103 logging.getLogger('').addHandler(console)
1105 if args.print:
1106 rv = print_file(args.print)
1107 elif not args.protocol:
1108 parser.print_usage()
1109 rv = 1
1110 else:
1111 rv = run_score(args, args.protocol)
1113 sys.exit(rv)
1116if __name__ == "__main__":
1117 main()
Diagnostics4 git calls · 185 ms
CallStatusSizeTime0185 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit26 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=26
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2009.5 KiB156 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=155
git.response.bytes=9679
git.response.cache=MISS
git.response.duration_ms=156
git.fetch.wants=56463c2cc86c14b85f1b1a5fa00da9c8f8837140
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=9679
git.pkt.payload.max=9658
git.sideband.band1.bytes=9657
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2d7aab9be14c2
Request
66fea805…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format