master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import difflib
26import filecmp
27import logging
28import os
29import shutil
31import pytest
32from testenv import CurlClient, Env, LocalClient, VsFTPD
34log = logging.getLogger(__name__)
37@pytest.mark.skipif(condition=not Env.has_vsftpd(), reason="missing vsftpd")
38@pytest.mark.skipif(condition=Env.curl_uses_lib('rustls-ffi'),
39 reason="rustls does not support TLS session reuse")
40@pytest.mark.skipif(condition=Env.curl_uses_lib('libressl'),
41 reason="libressl fails on TLS session reuse")
42class TestVsFTPD:
44 SUPPORTS_SSL = True
46 @pytest.fixture(autouse=True, scope='class')
47 @classmethod
48 def vsftpds(cls, env):
49 if not TestVsFTPD.SUPPORTS_SSL:
50 pytest.skip('vsftpd does not seem to support SSL')
51 vsftpds = VsFTPD(env=env, with_ssl=True)
52 if not vsftpds.initial_start():
53 vsftpds.stop()
54 TestVsFTPD.SUPPORTS_SSL = False
55 pytest.skip('vsftpd does not seem to support SSL')
56 yield vsftpds
57 vsftpds.stop()
59 @classmethod
60 def _make_docs_file(cls, docs_dir: str, fname: str, fsize: int):
61 fpath = os.path.join(docs_dir, fname)
62 data1k = 1024*'x'
63 flen = 0
64 with open(fpath, 'w') as fd:
65 while flen < fsize:
66 fd.write(data1k)
67 flen += len(data1k)
68 return flen
70 @pytest.fixture(autouse=True, scope='class')
71 @classmethod
72 def _class_scope(cls, env, vsftpds):
73 if os.path.exists(vsftpds.docs_dir):
74 shutil.rmtree(vsftpds.docs_dir)
75 if not os.path.exists(vsftpds.docs_dir):
76 os.makedirs(vsftpds.docs_dir)
77 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-1k', fsize=1024)
78 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-10k', fsize=10 * 1024)
79 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-1m', fsize=1024 * 1024)
80 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-10m', fsize=10 * 1024 * 1024)
81 env.make_data_file(indir=env.gen_dir, fname="upload-1k", fsize=1024)
82 env.make_data_file(indir=env.gen_dir, fname="upload-100k", fsize=100 * 1024)
83 env.make_data_file(indir=env.gen_dir, fname="upload-1m", fsize=1024 * 1024)
85 def test_31_01_list_dir(self, env: Env, vsftpds: VsFTPD):
86 curl = CurlClient(env=env)
87 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
88 r = curl.ftp_ssl_get(urls=[url], with_stats=True)
89 r.check_stats(count=1, http_status=226)
90 with open(os.path.join(curl.run_dir, 'download_#1.data')) as fd:
91 lines = fd.readlines()
92 assert len(lines) == 4, f'list: {lines}'
93 r.check_stats_timelines()
95 # download 1 file, no SSL
96 @pytest.mark.parametrize("docname", [
97 'data-1k', 'data-1m', 'data-10m'
98 ])
99 def test_31_02_download_1(self, env: Env, vsftpds: VsFTPD, docname):
100 curl = CurlClient(env=env)
101 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
102 count = 1
103 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
104 r = curl.ftp_ssl_get(urls=[url], with_stats=True)
105 r.check_stats(count=count, http_status=226)
106 self.check_downloads(curl, srcfile, count)
107 r.check_stats_timelines()
109 @pytest.mark.parametrize("docname,count,secure", [
110 ['data-1k', 10, True],
111 ['data-1m', 5, True],
112 ['data-1m', 5, False],
113 ['data-10m', 2, True]
114 ])
115 def test_31_03_download_10_serial(self, env: Env, vsftpds: VsFTPD, docname, count, secure):
116 curl = CurlClient(env=env)
117 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
118 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
119 xargs = []
120 if not secure:
121 xargs.append('--insecure')
122 r = curl.ftp_ssl_get(urls=[url], with_stats=True, extra_args=xargs)
123 r.check_stats(count=count, http_status=226)
124 self.check_downloads(curl, srcfile, count)
125 assert r.total_connects == count + 1, 'should reuse the control conn'
126 r.check_stats_timelines()
128 @pytest.mark.parametrize("docname,count,secure", [
129 ['data-1k', 10, True],
130 ['data-1m', 5, True],
131 ['data-1m', 5, False],
132 ['data-10m', 2, True]
133 ])
134 def test_31_04_download_10_parallel(self, env: Env, vsftpds: VsFTPD, docname, count, secure):
135 curl = CurlClient(env=env)
136 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
137 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
138 xargs = ['--parallel']
139 if not secure:
140 xargs.append('--insecure')
141 r = curl.ftp_ssl_get(urls=[url], with_stats=True, extra_args=xargs)
142 r.check_stats(count=count, http_status=226)
143 self.check_downloads(curl, srcfile, count)
144 assert r.total_connects > count + 1, 'should have used several control conns'
145 r.check_stats_timelines()
147 @pytest.mark.parametrize("docname", [
148 'upload-1k', 'upload-100k', 'upload-1m'
149 ])
150 def test_31_05_upload_1(self, env: Env, vsftpds: VsFTPD, docname):
151 curl = CurlClient(env=env)
152 srcfile = os.path.join(env.gen_dir, docname)
153 dstfile = os.path.join(vsftpds.docs_dir, docname)
154 self._rmf(dstfile)
155 count = 1
156 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
157 r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True)
158 r.check_stats(count=count, http_status=226)
159 self.check_upload(env, vsftpds, docname=docname)
160 r.check_stats_timelines()
162 def _rmf(self, path):
163 if os.path.exists(path):
164 os.remove(path)
166 # check with `tcpdump` if curl causes any TCP RST packets
167 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
168 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
169 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
170 def test_31_06_shutdown_download(self, env: Env, vsftpds: VsFTPD):
171 docname = 'data-1k'
172 curl = CurlClient(env=env)
173 count = 1
174 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
175 r = curl.ftp_ssl_get(urls=[url], with_stats=True, with_tcpdump=True)
176 r.check_stats(count=count, http_status=226)
177 # vsftp closes control connection without niceties,
178 # look only at ports from DATA connection.
179 data_ports = vsftpds.get_data_ports(r)
180 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
181 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
183 # check with `tcpdump` if curl causes any TCP RST packets
184 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
185 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
186 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
187 def test_31_07_shutdown_upload(self, env: Env, vsftpds: VsFTPD):
188 docname = 'upload-1k'
189 curl = CurlClient(env=env)
190 srcfile = os.path.join(env.gen_dir, docname)
191 dstfile = os.path.join(vsftpds.docs_dir, docname)
192 self._rmf(dstfile)
193 count = 1
194 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
195 r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True)
196 r.check_stats(count=count, http_status=226)
197 # vsftp closes control connection without niceties,
198 # look only at ports from DATA connection.
199 data_ports = vsftpds.get_data_ports(r)
200 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
201 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
203 def test_31_08_upload_ascii(self, env: Env, vsftpds: VsFTPD):
204 docname = 'upload-ascii'
205 line_length = 21
206 srcfile = os.path.join(env.gen_dir, docname)
207 dstfile = os.path.join(vsftpds.docs_dir, docname)
208 env.make_data_file(indir=env.gen_dir, fname=docname, fsize=100 * 1024,
209 line_length=line_length)
210 srcsize = os.path.getsize(srcfile)
211 self._rmf(dstfile)
212 count = 1
213 curl = CurlClient(env=env)
214 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
215 r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True,
216 extra_args=['--use-ascii'])
217 r.check_stats(count=count, http_status=226)
218 # expect the uploaded file to be number of converted newlines larger
219 dstsize = os.path.getsize(dstfile)
220 with open(srcfile) as fd:
221 newlines = len(fd.readlines())
222 assert (srcsize + newlines) == dstsize, \
223 f'expected source with {newlines} lines to be that much larger,'\
224 f'instead srcsize={srcsize}, upload size={dstsize}, diff={dstsize-srcsize}'
226 def test_31_08_active_download(self, env: Env, vsftpds: VsFTPD):
227 docname = 'data-10k'
228 curl = CurlClient(env=env)
229 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
230 count = 1
231 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
232 r = curl.ftp_ssl_get(urls=[url], with_stats=True, extra_args=[
233 '--ftp-port', '127.0.0.1'
234 ])
235 r.check_stats(count=count, http_status=226)
236 self.check_downloads(curl, srcfile, count)
238 def test_31_09_active_upload(self, env: Env, vsftpds: VsFTPD):
239 docname = 'upload-1k'
240 curl = CurlClient(env=env)
241 srcfile = os.path.join(env.gen_dir, docname)
242 dstfile = os.path.join(vsftpds.docs_dir, docname)
243 self._rmf(dstfile)
244 count = 1
245 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/'
246 r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, extra_args=[
247 '--ftp-port', '127.0.0.1'
248 ])
249 r.check_stats(count=count, http_status=226)
250 self.check_upload(env, vsftpds, docname=docname)
252 @pytest.mark.parametrize("indata", [
253 pytest.param('1234567890', id='10-bytes'),
254 pytest.param('', id='0-bytes'),
255 ])
256 def test_31_10_upload_stdin(self, env: Env, vsftpds: VsFTPD, indata):
257 curl = CurlClient(env=env)
258 docname = "upload_31_10"
259 dstfile = os.path.join(vsftpds.docs_dir, docname)
260 self._rmf(dstfile)
261 count = 1
262 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}'
263 r = curl.ftp_ssl_upload(urls=[url], updata=indata, with_stats=True)
264 r.check_stats(count=count, http_status=226)
265 assert os.path.exists(dstfile)
266 with open(dstfile) as fd:
267 destdata = fd.readlines()
268 expdata = [indata] if indata else []
269 assert expdata == destdata, f'expected: {expdata}, got: {destdata}'
271 def test_31_11_download_non_existing(self, env: Env, vsftpds: VsFTPD):
272 curl = CurlClient(env=env)
273 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/does-not-exist'
274 r = curl.ftp_ssl_get(urls=[url], with_stats=True)
275 r.check_exit_code(78)
276 r.check_stats(count=1, exitcode=78)
278 def test_31_12_upload_eprt(self, env: Env, vsftpds: VsFTPD):
279 docname = 'test_31_12'
280 client = LocalClient(name='cli_ftp_upload', env=env)
281 if not client.exists():
282 pytest.skip(f'example client not built: {client.name}')
283 url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}'
284 r = client.run(args=['-r', f'{env.ftp_domain}:{vsftpds.port}:127.0.0.1', url])
285 r.check_exit_code(0)
286 dstfile = os.path.join(vsftpds.docs_dir, docname)
287 assert os.path.exists(dstfile), f'{r.dump_logs()}'
289 # connection reuse with STARTTLS required
290 # 1st download without STARTTLS, 2nd with --ssl-reqd
291 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
292 def test_31_13_starttls_reuse(self, env: Env, vsftpds: VsFTPD):
293 run_env = os.environ.copy()
294 run_env['CURL_DBG_NO_USE_SSL_ON_FIRST'] = '1'
295 curl = CurlClient(env=env, run_env=run_env)
296 url1 = f'ftp://{env.ftp_domain}:{vsftpds.port}/data-1k'
297 url2 = f'ftp://{env.ftp_domain}:{vsftpds.port}/data-10k'
298 r = curl.run_direct(with_stats=True, args=[
299 '-svv', '--resolve', f'{env.ftp_domain}:{vsftpds.port}:127.0.0.1',
300 '--cacert', env.ca.cert_file,
301 url1, '--out-null',
302 url2, '--out-null', '--ssl-reqd'
303 ])
304 r.check_exit_code(0)
305 r.check_stats(count=2, http_status=226)
306 # expect 4 connections to have been made:
307 # 1. 1st CONTROL without STARTTLS
308 # 2. 1st DATA for download
309 # 3. 2nd CONTROL with STARTTLS (not reuse of 1)
310 # 4. 2nd DATA for download
311 assert r.total_connects == 4, f'{r.dump_logs()}'
313 def check_downloads(self, client, srcfile: str, count: int,
314 complete: bool = True):
315 for i in range(count):
316 dfile = client.download_file(i)
317 assert os.path.exists(dfile)
318 if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
319 with open(srcfile) as fa, open(dfile) as fb:
320 a = fa.readlines()
321 b = fb.readlines()
322 diff = "".join(difflib.unified_diff(a=a, b=b,
323 fromfile=srcfile,
324 tofile=dfile,
325 n=1))
326 assert False, f'download {dfile} differs:\n{diff}'
328 def check_upload(self, env, vsftpd: VsFTPD, docname):
329 srcfile = os.path.join(env.gen_dir, docname)
330 dstfile = os.path.join(vsftpd.docs_dir, docname)
331 assert os.path.exists(srcfile)
332 assert os.path.exists(dstfile)
333 if not filecmp.cmp(srcfile, dstfile, shallow=False):
334 with open(srcfile) as fa, open(dstfile) as fb:
335 a = fa.readlines()
336 b = fb.readlines()
337 diff = "".join(difflib.unified_diff(a=a, b=b,
338 fromfile=srcfile,
339 tofile=dstfile,
340 n=1))
341 assert False, f'upload {dstfile} differs:\n{diff}'
Diagnostics4 git calls · 234 ms
CallStatusSizeTime0234 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit24 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=24 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.6 KiB208 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=208 git.response.bytes=3667 git.response.cache=MISS git.response.duration_ms=208 git.fetch.wants=c3032e4924c3f93e6046ba9fbe96272b4dc1baf4 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3667 git.pkt.payload.max=3646 git.sideband.band1.bytes=3645 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2eef81b1b77be
- Request
- 2ddf787f…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format