master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import difflib
26import filecmp
27import logging
28import os
29import shutil
31import pytest
32from testenv import CurlClient, Env, LocalClient, VsFTPD
34log = logging.getLogger(__name__)
37@pytest.mark.skipif(condition=not Env.has_vsftpd(), reason="missing vsftpd")
38@pytest.mark.skipif(condition=Env.curl_uses_lib('rustls-ffi'),
39 reason="rustls does not support TLS session reuse")
40@pytest.mark.skipif(condition=Env.curl_uses_lib('libressl'),
41 reason="libressl fails on TLS session reuse")
42class TestFtpsVsFTPD:
44 SUPPORTS_SSL = True
46 @pytest.fixture(autouse=True, scope='class')
47 @classmethod
48 def vsftpds(cls, env):
49 if not TestFtpsVsFTPD.SUPPORTS_SSL:
50 pytest.skip('vsftpd does not seem to support SSL')
51 vsftpds = VsFTPD(env=env, with_ssl=True, ssl_implicit=True)
52 if not vsftpds.initial_start():
53 vsftpds.stop()
54 TestFtpsVsFTPD.SUPPORTS_SSL = False
55 pytest.skip('vsftpd does not seem to support SSL')
56 yield vsftpds
57 vsftpds.stop()
59 @classmethod
60 def _make_docs_file(cls, docs_dir: str, fname: str, fsize: int):
61 fpath = os.path.join(docs_dir, fname)
62 data1k = 1024*'x'
63 flen = 0
64 with open(fpath, 'w') as fd:
65 while flen < fsize:
66 fd.write(data1k)
67 flen += len(data1k)
68 return flen
70 @pytest.fixture(autouse=True, scope='class')
71 @classmethod
72 def _class_scope(cls, env, vsftpds):
73 if os.path.exists(vsftpds.docs_dir):
74 shutil.rmtree(vsftpds.docs_dir)
75 if not os.path.exists(vsftpds.docs_dir):
76 os.makedirs(vsftpds.docs_dir)
77 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-1k', fsize=1024)
78 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-10k', fsize=10 * 1024)
79 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-1m', fsize=1024 * 1024)
80 cls._make_docs_file(docs_dir=vsftpds.docs_dir, fname='data-10m', fsize=10 * 1024 * 1024)
81 env.make_data_file(indir=env.gen_dir, fname="upload-1k", fsize=1024)
82 env.make_data_file(indir=env.gen_dir, fname="upload-100k", fsize=100 * 1024)
83 env.make_data_file(indir=env.gen_dir, fname="upload-1m", fsize=1024 * 1024)
85 def test_32_01_list_dir(self, env: Env, vsftpds: VsFTPD):
86 curl = CurlClient(env=env)
87 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/'
88 r = curl.ftp_get(urls=[url], with_stats=True)
89 r.check_stats(count=1, http_status=226)
90 with open(os.path.join(curl.run_dir, 'download_#1.data')) as fd:
91 lines = fd.readlines()
92 assert len(lines) == 4, f'list: {lines}'
93 r.check_stats_timelines()
95 # download 1 file, no SSL
96 @pytest.mark.parametrize("docname", [
97 'data-1k', 'data-1m', 'data-10m'
98 ])
99 def test_32_02_download_1(self, env: Env, vsftpds: VsFTPD, docname):
100 curl = CurlClient(env=env)
101 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
102 count = 1
103 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
104 r = curl.ftp_get(urls=[url], with_stats=True)
105 r.check_stats(count=count, http_status=226)
106 self.check_downloads(curl, srcfile, count)
107 r.check_stats_timelines()
109 @pytest.mark.parametrize("docname,count,secure", [
110 ['data-1k', 10, True],
111 ['data-1m', 5, True],
112 ['data-1m', 5, False],
113 ['data-10m', 2, True]
114 ])
115 def test_32_03_download_10_serial(self, env: Env, vsftpds: VsFTPD, docname, count, secure):
116 curl = CurlClient(env=env)
117 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
118 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
119 xargs = []
120 if not secure:
121 xargs.append('--insecure')
122 r = curl.ftp_get(urls=[url], with_stats=True, extra_args=xargs)
123 r.check_stats(count=count, http_status=226)
124 self.check_downloads(curl, srcfile, count)
125 assert r.total_connects == count + 1, 'should reuse the control conn'
126 r.check_stats_timelines()
128 # 2 serial transfers, first with 'ftps://' and second with 'ftp://'
129 # we want connection reuse in this case
130 def test_32_03b_ftp_compat_ftps(self, env: Env, vsftpds: VsFTPD):
131 curl = CurlClient(env=env)
132 docname = 'data-1k'
133 count = 2
134 url1 = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}'
135 url2 = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}'
136 r = curl.ftp_get(urls=[url1, url2], with_stats=True)
137 r.check_stats(count=count, http_status=226)
138 assert r.total_connects == count + 1, 'should reuse the control conn'
139 r.check_stats_timelines()
141 @pytest.mark.parametrize("docname", [
142 'data-1k', 'data-1m', 'data-10m'
143 ])
144 def test_32_04_download_10_parallel(self, env: Env, vsftpds: VsFTPD, docname):
145 curl = CurlClient(env=env)
146 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
147 count = 10
148 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
149 r = curl.ftp_get(urls=[url], with_stats=True, extra_args=[
150 '--parallel'
151 ])
152 r.check_stats(count=count, http_status=226)
153 self.check_downloads(curl, srcfile, count)
154 assert r.total_connects > count + 1, 'should have used several control conns'
155 r.check_stats_timelines()
157 @pytest.mark.parametrize("docname", [
158 'upload-1k', 'upload-100k', 'upload-1m'
159 ])
160 def test_32_05_upload_1(self, env: Env, vsftpds: VsFTPD, docname):
161 curl = CurlClient(env=env)
162 srcfile = os.path.join(env.gen_dir, docname)
163 dstfile = os.path.join(vsftpds.docs_dir, docname)
164 self._rmf(dstfile)
165 count = 1
166 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/'
167 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True)
168 r.check_stats(count=count, http_status=226)
169 self.check_upload(env, vsftpds, docname=docname)
170 r.check_stats_timelines()
172 def _rmf(self, path):
173 if os.path.exists(path):
174 os.remove(path)
176 # check with `tcpdump` if curl causes any TCP RST packets
177 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
178 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
179 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
180 def test_32_06_shutdown_download(self, env: Env, vsftpds: VsFTPD):
181 docname = 'data-1k'
182 curl = CurlClient(env=env)
183 count = 1
184 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
185 r = curl.ftp_get(urls=[url], with_stats=True, with_tcpdump=True)
186 r.check_stats(count=count, http_status=226)
187 # vsftp closes control connection without niceties,
188 # look only at ports from DATA connection.
189 data_ports = vsftpds.get_data_ports(r)
190 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
191 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
193 # check with `tcpdump` if curl causes any TCP RST packets
194 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
195 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
196 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
197 def test_32_07_shutdown_upload(self, env: Env, vsftpds: VsFTPD):
198 docname = 'upload-1k'
199 curl = CurlClient(env=env)
200 srcfile = os.path.join(env.gen_dir, docname)
201 dstfile = os.path.join(vsftpds.docs_dir, docname)
202 self._rmf(dstfile)
203 count = 1
204 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/'
205 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True)
206 r.check_stats(count=count, http_status=226)
207 # vsftp closes control connection without niceties,
208 # look only at ports from DATA connection.
209 data_ports = vsftpds.get_data_ports(r)
210 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
211 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
213 def test_32_08_upload_ascii(self, env: Env, vsftpds: VsFTPD):
214 docname = 'upload-ascii'
215 line_length = 21
216 srcfile = os.path.join(env.gen_dir, docname)
217 dstfile = os.path.join(vsftpds.docs_dir, docname)
218 env.make_data_file(indir=env.gen_dir, fname=docname, fsize=100 * 1024,
219 line_length=line_length)
220 srcsize = os.path.getsize(srcfile)
221 self._rmf(dstfile)
222 count = 1
223 curl = CurlClient(env=env)
224 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/'
225 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True,
226 extra_args=['--use-ascii'])
227 r.check_stats(count=count, http_status=226)
228 # expect the uploaded file to be number of converted newlines larger
229 dstsize = os.path.getsize(dstfile)
230 with open(srcfile) as fd:
231 newlines = len(fd.readlines())
232 assert (srcsize + newlines) == dstsize, \
233 f'expected source with {newlines} lines to be that much larger,'\
234 f'instead srcsize={srcsize}, upload size={dstsize}, diff={dstsize-srcsize}'
236 def test_32_08_active_download(self, env: Env, vsftpds: VsFTPD):
237 docname = 'data-10k'
238 curl = CurlClient(env=env)
239 srcfile = os.path.join(vsftpds.docs_dir, f'{docname}')
240 count = 1
241 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]'
242 r = curl.ftp_get(urls=[url], with_stats=True, extra_args=[
243 '--ftp-port', '127.0.0.1'
244 ])
245 r.check_stats(count=count, http_status=226)
246 self.check_downloads(curl, srcfile, count)
248 def test_32_09_active_upload(self, env: Env, vsftpds: VsFTPD):
249 docname = 'upload-1k'
250 curl = CurlClient(env=env)
251 srcfile = os.path.join(env.gen_dir, docname)
252 dstfile = os.path.join(vsftpds.docs_dir, docname)
253 self._rmf(dstfile)
254 count = 1
255 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/'
256 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, extra_args=[
257 '--ftp-port', '127.0.0.1'
258 ])
259 r.check_stats(count=count, http_status=226)
260 self.check_upload(env, vsftpds, docname=docname)
262 @pytest.mark.parametrize("indata", [
263 pytest.param('1234567890', id='10-bytes'),
264 pytest.param('', id='0-bytes'),
265 ])
266 def test_32_10_upload_stdin(self, env: Env, vsftpds: VsFTPD, indata):
267 curl = CurlClient(env=env)
268 docname = "upload_31_10"
269 dstfile = os.path.join(vsftpds.docs_dir, docname)
270 self._rmf(dstfile)
271 count = 1
272 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}'
273 r = curl.ftp_upload(urls=[url], updata=indata, with_stats=True)
274 r.check_stats(count=count, http_status=226)
275 assert os.path.exists(dstfile)
276 with open(dstfile) as fd:
277 destdata = fd.readlines()
278 expdata = [indata] if indata else []
279 assert expdata == destdata, f'expected: {expdata}, got: {destdata}'
281 def test_32_11_download_non_existing(self, env: Env, vsftpds: VsFTPD):
282 curl = CurlClient(env=env)
283 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/does-not-exist'
284 r = curl.ftp_get(urls=[url], with_stats=True)
285 r.check_exit_code(78)
286 r.check_stats(count=1, exitcode=78)
288 def test_32_12_upload_eprt(self, env: Env, vsftpds: VsFTPD):
289 docname = 'test_32_12'
290 client = LocalClient(name='cli_ftp_upload', env=env)
291 if not client.exists():
292 pytest.skip(f'example client not built: {client.name}')
293 url = f'ftps://{env.ftp_domain}:{vsftpds.port}/{docname}'
294 r = client.run(args=['-r', f'{env.ftp_domain}:{vsftpds.port}:127.0.0.1', url])
295 r.check_exit_code(0)
296 dstfile = os.path.join(vsftpds.docs_dir, docname)
297 assert os.path.exists(dstfile), f'{r.dump_logs()}'
299 def check_downloads(self, client, srcfile: str, count: int,
300 complete: bool = True):
301 for i in range(count):
302 dfile = client.download_file(i)
303 assert os.path.exists(dfile)
304 if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
305 with open(srcfile) as fa, open(dfile) as fb:
306 a = fa.readlines()
307 b = fb.readlines()
308 diff = "".join(difflib.unified_diff(a=a, b=b,
309 fromfile=srcfile,
310 tofile=dfile,
311 n=1))
312 assert False, f'download {dfile} differs:\n{diff}'
314 def check_upload(self, env, vsftpd: VsFTPD, docname):
315 srcfile = os.path.join(env.gen_dir, docname)
316 dstfile = os.path.join(vsftpd.docs_dir, docname)
317 assert os.path.exists(srcfile)
318 assert os.path.exists(dstfile)
319 if not filecmp.cmp(srcfile, dstfile, shallow=False):
320 with open(srcfile) as fa, open(dstfile) as fb:
321 a = fa.readlines()
322 b = fb.readlines()
323 diff = "".join(difflib.unified_diff(a=a, b=b,
324 fromfile=srcfile,
325 tofile=dstfile,
326 n=1))
327 assert False, f'upload {dstfile} differs:\n{diff}'
Diagnostics4 git calls · 139 ms
CallStatusSizeTime0139 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=133 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2003.4 KiB133 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3468 git.response.cache=MISS git.response.duration_ms=133 git.fetch.wants=9301b5265ade688abbd519275b1ebec06793882c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3468 git.pkt.payload.max=3447 git.sideband.band1.bytes=3446 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2ef402a6977be
- Request
- 1e84c2ae…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format