master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import logging
26import os
27import re
28import socket
29import subprocess
30import time
31from datetime import datetime, timedelta, timezone
32from typing import Dict, List, Tuple
34from .curl import CurlClient, ExecResult
35from .env import Env
36from .ports import alloc_ports_and_do
38log = logging.getLogger(__name__)
41class VsFTPD:
43 def __init__(self, env: Env, with_ssl=False, ssl_implicit=False):
44 self.env = env
45 self._cmd = env.vsftpd
46 self._port = 0
47 self._with_ssl = with_ssl
48 self._ssl_implicit = ssl_implicit and with_ssl
49 self._scheme = 'ftps' if self._ssl_implicit else 'ftp'
50 if self._with_ssl:
51 self.name = 'vsftpds'
52 self._port_skey = 'ftps'
53 self._port_specs = {
54 'ftps': socket.SOCK_STREAM,
55 }
56 else:
57 self.name = 'vsftpd'
58 self._port_skey = 'ftp'
59 self._port_specs = {
60 'ftp': socket.SOCK_STREAM,
61 }
62 self._vsftpd_dir = os.path.join(env.gen_dir, self.name)
63 self._run_dir = os.path.join(self._vsftpd_dir, 'run')
64 self._docs_dir = os.path.join(self._vsftpd_dir, 'docs')
65 self._tmp_dir = os.path.join(self._vsftpd_dir, 'tmp')
66 self._conf_file = os.path.join(self._vsftpd_dir, 'test.conf')
67 self._pid_file = os.path.join(self._vsftpd_dir, 'vsftpd.pid')
68 self._error_log = os.path.join(self._vsftpd_dir, 'vsftpd.log')
69 self._error_fd = None
70 self._process = None
72 self.clear_logs()
74 @property
75 def domain(self):
76 return self.env.ftp_domain
78 @property
79 def docs_dir(self):
80 return self._docs_dir
82 @property
83 def port(self) -> int:
84 return self._port
86 def close_log(self):
87 if self._error_fd:
88 self._error_fd.close()
89 self._error_fd = None
91 def clear_logs(self):
92 self._rmf(self._error_log)
94 def exists(self):
95 return os.path.exists(self._cmd)
97 def is_running(self):
98 if self._process:
99 self._process.poll()
100 return self._process.returncode is None
101 return False
103 def start_if_needed(self):
104 if not self.is_running():
105 return self.start()
106 return True
108 def stop(self, wait_dead=True):
109 self._mkpath(self._tmp_dir)
110 if self._process:
111 self._process.terminate()
112 self._process.wait(timeout=2)
113 self._process = None
114 self.close_log()
115 return not wait_dead or self.wait_dead(timeout=timedelta(seconds=5))
116 self.close_log()
117 return True
119 def restart(self):
120 self.stop()
121 return self.start()
123 def initial_start(self):
125 def startup(ports: Dict[str, int]) -> bool:
126 self._port = ports[self._port_skey]
127 if self.start():
128 self.env.update_ports(ports)
129 return True
130 self.stop()
131 self._port = 0
132 return False
134 return alloc_ports_and_do(self._port_specs, startup,
135 self.env.gen_root, max_tries=3)
137 def start(self, wait_live=True):
138 assert self._port > 0
139 self._mkpath(self._tmp_dir)
140 if self._process:
141 self.stop()
142 self._write_config()
143 args = [
144 self._cmd,
145 f'{self._conf_file}',
146 ]
147 self._error_fd = open(self._error_log, 'a') # noqa: SIM115
148 self._process = subprocess.Popen(args=args, stderr=self._error_fd)
149 if self._process.returncode is not None:
150 return False
151 return not wait_live or self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
153 def wait_dead(self, timeout: timedelta):
154 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
155 try_until = datetime.now(timezone.utc) + timeout
156 while datetime.now(timezone.utc) < try_until:
157 check_url = f'{self._scheme}://{self.domain}:{self.port}/'
158 r = curl.ftp_get(urls=[check_url], extra_args=['-v'])
159 if r.exit_code != 0:
160 return True
161 log.debug(f'waiting for vsftpd to stop responding: {r}')
162 time.sleep(.1)
163 log.debug(f"Server still responding after {timeout}")
164 return False
166 def wait_live(self, timeout: timedelta):
167 curl = CurlClient(env=self.env, run_dir=self._tmp_dir)
168 try_until = datetime.now(timezone.utc) + timeout
169 while datetime.now(timezone.utc) < try_until:
170 check_url = f'{self._scheme}://{self.domain}:{self.port}/'
171 r = curl.ftp_get(urls=[check_url], extra_args=[
172 '--trace', 'curl-start.trace', '--trace-time'
173 ])
174 if r.exit_code == 0:
175 return True
176 time.sleep(.1)
177 log.error(f"Server still not responding after {timeout}")
178 return False
180 def _rmf(self, path):
181 if os.path.exists(path):
182 os.remove(path)
184 def _mkpath(self, path):
185 if not os.path.exists(path):
186 os.makedirs(path)
188 def _write_config(self):
189 self._mkpath(self._docs_dir)
190 self._mkpath(self._tmp_dir)
191 conf = [ # base server config
192 'listen=YES',
193 'run_as_launching_user=YES',
194 '#listen_address=127.0.0.1',
195 f'listen_port={self.port}',
196 'local_enable=NO',
197 'anonymous_enable=YES',
198 f'anon_root={self._docs_dir}',
199 'dirmessage_enable=YES',
200 'write_enable=YES',
201 'anon_upload_enable=YES',
202 'log_ftp_protocol=YES',
203 'xferlog_enable=YES',
204 'xferlog_std_format=NO',
205 f'vsftpd_log_file={self._error_log}',
206 '\n',
207 ]
208 if self._with_ssl:
209 creds = self.env.get_credentials(self.domain)
210 assert creds # convince pytype this is not None
211 conf.extend([
212 'ssl_enable=YES',
213 'debug_ssl=YES',
214 'allow_anon_ssl=YES',
215 f'rsa_cert_file={creds.cert_file}',
216 f'rsa_private_key_file={creds.pkey_file}',
217 # require_ssl_reuse=YES means ctrl and data connection need to use the same session
218 'require_ssl_reuse=YES',
219 ])
220 if self._ssl_implicit:
221 conf.extend([
222 'implicit_ssl=YES',
223 ])
224 with open(self._conf_file, 'w') as fd:
225 fd.write("\n".join(conf))
227 def get_data_ports(self, r: ExecResult) -> List[Tuple[int, int]]:
228 return [(int(m.group(1)), int(m.group(2))) for line in r.trace_lines if
229 (m := re.match(r'.*Established 2nd connection to .* '
230 r'\(\S+ port (\d+)\) from \S+ port (\d+)', line))]
Diagnostics4 git calls · 183 ms
CallStatusSizeTime0183 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2002.5 KiB178 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=178 git.response.bytes=2589 git.response.cache=MISS git.response.duration_ms=178 git.fetch.wants=cda57b1822089f09825084b185e68a4055c3cd5c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2589 git.pkt.payload.max=2568 git.sideband.band1.bytes=2567 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2ce0f5d27efad
- Request
- 11f47a69…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format