Artifactspublic-v0

curl

master
ply4 git calls346 ms290 KiB2 cachedprotocol v2
287 lines10 KiBRaw
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
21# SPDX-License-Identifier: curl
23###########################################################################
25import logging
26import os
27import socket
28import stat
29import subprocess
30import time
31from datetime import datetime, timedelta, timezone
32from typing import Dict
34from . import CurlClient
35from .env import Env
36from .ports import alloc_ports_and_do
38log = logging.getLogger(__name__)
41class Sshd:
43 def __init__(self, env: Env):
44 self.env = env
45 self._cmd = Env.CONFIG.sshd
46 self._sftpd = Env.CONFIG.sftpd
47 self._keygen = 'ssh-keygen'
48 self._port = 0
49 self.name = 'sshd'
50 self._port_skey = 'sshd'
51 self._port_specs = {
52 'sshd': socket.SOCK_STREAM,
53 }
54 self._sshd_dir = os.path.join(env.gen_dir, self.name)
55 self._home_dir = os.path.join(self._sshd_dir, 'home')
56 self._run_dir = os.path.join(self._sshd_dir, 'run')
57 self._tmp_dir = os.path.join(self._sshd_dir, 'tmp')
58 self._conf_file = os.path.join(self._sshd_dir, 'test.conf')
59 self._auth_keys = os.path.join(self._sshd_dir, 'authorized_keys')
60 self._known_hosts = os.path.join(self._sshd_dir, 'known_hosts')
61 self._unknown_hosts = os.path.join(self._sshd_dir, 'unknown_hosts')
62 self._sshd_log = os.path.join(self._sshd_dir, 'sshd.log')
63 self._pid_file = os.path.join(self._sshd_dir, 'sshd.pid')
64 self._key_algs = [
65 'rsa', 'ecdsa', 'ed25519',
66 ]
67 self._host_key_files = []
68 self._host_pub_files = []
69 self._users = [
70 'user1',
71 'user2',
72 ]
73 self._user_key_files = []
74 self._user_pub_files = []
75 self._error_fd = None
76 self._process = None
78 self.clear_logs()
79 self._mkpath(self._home_dir)
80 env.make_data_file(indir=self._home_dir, fname="data", fsize=1024)
81 self._mkpath(self._tmp_dir)
83 @property
84 def port(self) -> int:
85 return self._port
87 @property
88 def home_dir(self):
89 return self._home_dir
91 @property
92 def known_hosts(self):
93 return self._known_hosts
95 @property
96 def unknown_hosts(self):
97 return self._unknown_hosts
99 @property
100 def user1_pubkey_file(self):
101 return self._user_pub_files[0]
103 @property
104 def user1_privkey_file(self):
105 return self._user_key_files[0]
107 @property
108 def user2_pubkey_file(self):
109 return self._user_pub_files[1]
111 @property
112 def user2_privkey_file(self):
113 return self._user_key_files[1]
115 def mk_host_keys(self):
116 self._host_key_files = []
117 self._host_pub_files = []
118 # a known_host file that knows all our test host pubkeys
119 with open(self._unknown_hosts, 'w') as fd_unknown, \
120 open(self._known_hosts, 'w') as fd_known:
121 os.chmod(self._unknown_hosts, stat.S_IRUSR | stat.S_IWUSR)
122 os.chmod(self._known_hosts, stat.S_IRUSR | stat.S_IWUSR)
123 for alg in self._key_algs:
124 key_file = os.path.join(self._sshd_dir, f'ssh_host_{alg}_key')
125 if not os.path.exists(key_file):
126 subprocess.run(args=[
127 self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
128 ], capture_output=True, text=True, check=True)
129 self._host_key_files.append(key_file)
130 pub_file = f'{key_file}.pub'
131 self._host_pub_files.append(pub_file)
132 with open(pub_file) as fp:
133 pubkey = fp.read()
134 # fd_known.write(f'[127.0.0.1]:{self.port} {pubkey}')
135 fd_known.write(f'[{self.env.domain1.lower()}]:{self.port} {pubkey}')
136 fd_unknown.write(f'dummy.invalid {pubkey}')
137 # hash the known_hosts file, libssh requires it
138 subprocess.run(args=[
139 self._keygen, '-H', '-f', self._known_hosts
140 ], capture_output=True, text=True, check=True)
142 def mk_user_keys(self):
143 self._user_key_files = []
144 self._user_pub_files = []
145 alg = 'rsa'
146 for user in self._users:
147 key_file = os.path.join(self._sshd_dir, f'id_{user}_user_{alg}_key')
148 if not os.path.exists(key_file):
149 subprocess.run(args=[
150 self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
151 ], capture_output=True, text=True, check=True)
152 self._user_key_files.append(key_file)
153 self._user_pub_files.append(f'{key_file}.pub')
154 with open(self._auth_keys, 'w') as fd:
155 os.chmod(self._auth_keys, stat.S_IRUSR | stat.S_IWUSR)
156 with open(self._user_pub_files[0]) as fp:
157 pubkey = fp.read()
158 fd.write(pubkey)
160 def close_log(self):
161 if self._error_fd:
162 self._error_fd.close()
163 self._error_fd = None
165 def clear_logs(self):
166 self._rmf(self._sshd_log)
168 def dump_log(self):
169 lines = ['>>--sshd log ----------------------------------------------\n']
170 with open(self._sshd_log) as fd:
171 lines.extend(fd.readlines())
172 lines.extend(['>>--curl log ----------------------------------------------\n'])
173 with open(os.path.join(self._tmp_dir, 'curl.stderr')) as fd:
174 lines.extend(fd.readlines())
175 lines.append('<<-------------------------------------------------------\n')
176 return ''.join(lines)
178 def exists(self):
179 return os.path.exists(self._cmd)
181 def is_running(self):
182 if self._process:
183 self._process.poll()
184 return self._process.returncode is None
185 return False
187 def start_if_needed(self):
188 if not self.is_running():
189 return self.start()
190 return True
192 def stop(self, wait_dead=True):
193 self._mkpath(self._tmp_dir)
194 if self._process:
195 self._process.terminate()
196 self._process.wait(timeout=2)
197 self._process = None
198 self.close_log()
199 return True
201 def restart(self):
202 self.stop()
203 return self.start()
205 def initial_start(self):
207 def startup(ports: Dict[str, int]) -> bool:
208 self._port = ports[self._port_skey]
209 self.mk_host_keys()
210 self.mk_user_keys()
211 if self.start():
212 self.env.update_ports(ports)
213 return True
214 self.stop()
215 self._port = 0
216 return False
218 return alloc_ports_and_do(self._port_specs, startup,
219 self.env.gen_root, max_tries=3)
221 def start(self, wait_live=True):
222 assert self._port > 0
223 self._mkpath(self._tmp_dir)
224 if self._process:
225 self.stop()
226 self._write_config()
227 args = [
228 self._cmd,
229 '-D',
230 '-f', f'{self._conf_file}',
231 '-E', f'{self._sshd_log}',
232 ]
233 run_env = os.environ.copy()
234 # does not have any effect, sadly
235 # run_env['HOME'] = f'{self._home_dir}'
236 self._error_fd = open(self._sshd_log, 'a') # noqa: SIM115
237 self._process = subprocess.Popen(args=args, stderr=self._error_fd, env=run_env)
238 if self._process.returncode is not None:
239 return False
240 return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
242 def wait_live(self, timeout: timedelta):
243 curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
244 timeout=timeout.total_seconds())
245 try_until = datetime.now(timezone.utc) + timeout
246 while datetime.now(timezone.utc) < try_until:
247 r = curl.http_get(url=f'scp://{self.env.domain1}:{self._port}/{self.home_dir}/data',
248 extra_args=[
249 '--insecure',
250 '--pubkey', self.user1_pubkey_file,
251 '--key', self.user1_privkey_file,
252 '--user', f'{os.environ["USER"]}:',
253 ])
254 if r.exit_code == 0:
255 return True
256 time.sleep(.1)
257 log.error(f"sshd still not responding after {timeout}")
258 return False
260 def _rmf(self, path):
261 if os.path.exists(path):
262 os.remove(path)
264 def _mkpath(self, path):
265 if not os.path.exists(path):
266 os.makedirs(path)
268 def _write_config(self):
269 conf = [
270 f'ListenAddress 127.0.0.1:{self._port}',
271 'AllowTcpForwarding yes',
272 'AuthenticationMethods publickey',
273 'PasswordAuthentication no',
274 # in CI, we might run as root, allow this
275 'PermitRootLogin yes',
276 'LogLevel VERBOSE',
277 f'AuthorizedKeysFile {self._auth_keys}',
278 f'PidFile {self._pid_file}',
279 ]
280 if Env.CONFIG.versiontuple(self.env.sshd_version()) >= Env.CONFIG.versiontuple('9.8'):
281 conf.append('PerSourcePenalties no')
282 conf.extend([f'HostKey {key_file}' for key_file in self._host_key_files])
283 if self._sftpd:
284 conf.append(f'Subsystem sftp {self._sftpd}')
285 conf.append('\n')
286 with open(self._conf_file, 'w') as fd:
287 fd.write("\n".join(conf))
Diagnostics4 git calls · 346 ms
CallStatusSizeTime0346 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=341
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2003.1 KiB341 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3164
git.response.cache=MISS
git.response.duration_ms=341
git.fetch.wants=24922bef694fe21d7714cafc7d4c1e74c9c52e00
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3164
git.pkt.payload.max=3143
git.sideband.band1.bytes=3142
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2df2dc90a2308
Request
80d8bbf3…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format