Artifactspublic-v0

curl

master
ply4 git calls143 ms290 KiB2 cachedprotocol v2
300 lines13 KiBRaw
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
21# SPDX-License-Identifier: curl
23###########################################################################
25import difflib
26import filecmp
27import logging
28import os
29import shutil
31import pytest
32from testenv import CurlClient, Env, LocalClient, VsFTPD
34log = logging.getLogger(__name__)
37@pytest.mark.skipif(condition=not Env.has_vsftpd(), reason="missing vsftpd")
38class TestVsFTPD:
40 @pytest.fixture(autouse=True, scope='class')
41 @classmethod
42 def vsftpd(cls, env):
43 vsftpd = VsFTPD(env=env)
44 assert vsftpd.initial_start()
45 yield vsftpd
46 vsftpd.stop()
48 @classmethod
49 def _make_docs_file(cls, docs_dir: str, fname: str, fsize: int):
50 fpath = os.path.join(docs_dir, fname)
51 data1k = 1024*'x'
52 flen = 0
53 with open(fpath, 'w') as fd:
54 fd.write(f'{fname}--->\n')
55 while flen < fsize:
56 fd.write(data1k)
57 flen += len(data1k)
58 fd.write(f'<---{fname}\n')
59 return flen
61 @pytest.fixture(autouse=True, scope='class')
62 @classmethod
63 def _class_scope(cls, env, vsftpd):
64 if os.path.exists(vsftpd.docs_dir):
65 shutil.rmtree(vsftpd.docs_dir)
66 if not os.path.exists(vsftpd.docs_dir):
67 os.makedirs(vsftpd.docs_dir)
68 cls._make_docs_file(docs_dir=vsftpd.docs_dir, fname='data-0k', fsize=0)
69 cls._make_docs_file(docs_dir=vsftpd.docs_dir, fname='data-1k', fsize=1024)
70 cls._make_docs_file(docs_dir=vsftpd.docs_dir, fname='data-10k', fsize=10 * 1024)
71 cls._make_docs_file(docs_dir=vsftpd.docs_dir, fname='data-1m', fsize=1024 * 1024)
72 cls._make_docs_file(docs_dir=vsftpd.docs_dir, fname='data-10m', fsize=10 * 1024 * 1024)
73 env.make_data_file(indir=env.gen_dir, fname="upload-0k", fsize=0)
74 env.make_data_file(indir=env.gen_dir, fname="upload-1k", fsize=1024)
75 env.make_data_file(indir=env.gen_dir, fname="upload-100k", fsize=100 * 1024)
76 env.make_data_file(indir=env.gen_dir, fname="upload-1m", fsize=1024 * 1024)
78 def test_30_01_list_dir(self, env: Env, vsftpd: VsFTPD):
79 curl = CurlClient(env=env)
80 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
81 r = curl.ftp_get(urls=[url], with_stats=True)
82 r.check_stats(count=1, http_status=226)
83 with open(os.path.join(curl.run_dir, 'download_#1.data')) as fd:
84 lines = fd.readlines()
85 assert len(lines) == 5, f'list: {lines}'
86 r.check_stats_timelines()
88 # download 1 file, no SSL
89 @pytest.mark.parametrize("docname", [
90 'data-0k', 'data-1k', 'data-1m', 'data-10m'
91 ])
92 def test_30_02_download_1(self, env: Env, vsftpd: VsFTPD, docname):
93 curl = CurlClient(env=env)
94 srcfile = os.path.join(vsftpd.docs_dir, f'{docname}')
95 count = 1
96 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
97 r = curl.ftp_get(urls=[url], with_stats=True)
98 r.check_stats(count=count, http_status=226)
99 self.check_downloads(curl, srcfile, count)
100 r.check_stats_timelines()
102 @pytest.mark.parametrize("docname", [
103 'data-0k', 'data-1k', 'data-1m', 'data-10m'
104 ])
105 def test_30_03_download_10_serial(self, env: Env, vsftpd: VsFTPD, docname):
106 curl = CurlClient(env=env)
107 srcfile = os.path.join(vsftpd.docs_dir, f'{docname}')
108 count = 10
109 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
110 r = curl.ftp_get(urls=[url], with_stats=True)
111 r.check_stats(count=count, http_status=226)
112 self.check_downloads(curl, srcfile, count)
113 assert r.total_connects == count + 1, 'should reuse the control conn'
114 r.check_stats_timelines()
116 @pytest.mark.parametrize("docname", [
117 'data-0k', 'data-1k', 'data-1m', 'data-10m'
118 ])
119 def test_30_04_download_10_parallel(self, env: Env, vsftpd: VsFTPD, docname):
120 curl = CurlClient(env=env)
121 srcfile = os.path.join(vsftpd.docs_dir, f'{docname}')
122 count = 10
123 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
124 r = curl.ftp_get(urls=[url], with_stats=True, extra_args=[
125 '--parallel'
126 ])
127 r.check_stats(count=count, http_status=226)
128 self.check_downloads(curl, srcfile, count)
129 assert r.total_connects > count + 1, 'should have used several control conns'
130 r.check_stats_timelines()
132 @pytest.mark.parametrize("docname", [
133 'upload-0k', 'upload-1k', 'upload-100k', 'upload-1m'
134 ])
135 def test_30_05_upload_1(self, env: Env, vsftpd: VsFTPD, docname):
136 curl = CurlClient(env=env)
137 srcfile = os.path.join(env.gen_dir, docname)
138 dstfile = os.path.join(vsftpd.docs_dir, docname)
139 self._rmf(dstfile)
140 count = 1
141 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
142 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True)
143 r.check_stats(count=count, http_status=226)
144 self.check_upload(env, vsftpd, docname=docname)
145 r.check_stats_timelines()
147 def _rmf(self, path):
148 if os.path.exists(path):
149 os.remove(path)
151 # check with `tcpdump` if curl causes any TCP RST packets
152 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
153 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
154 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
155 def test_30_06_shutdown_download(self, env: Env, vsftpd: VsFTPD):
156 docname = 'data-1k'
157 curl = CurlClient(env=env)
158 count = 1
159 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
160 r = curl.ftp_get(urls=[url], with_stats=True, with_tcpdump=True)
161 r.check_stats(count=count, http_status=226)
162 assert r.tcpdump
163 # vsftp closes control connection without niceties,
164 # look only at ports from DATA connection.
165 data_ports = vsftpd.get_data_ports(r)
166 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
167 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
169 # check with `tcpdump` if curl causes any TCP RST packets
170 @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available")
171 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
172 @pytest.mark.skipif(condition=not Env.curl_is_verbose(), reason="needs curl verbose strings")
173 def test_30_07_shutdown_upload(self, env: Env, vsftpd: VsFTPD):
174 docname = 'upload-1k'
175 curl = CurlClient(env=env)
176 srcfile = os.path.join(env.gen_dir, docname)
177 dstfile = os.path.join(vsftpd.docs_dir, docname)
178 self._rmf(dstfile)
179 count = 1
180 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
181 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True)
182 r.check_stats(count=count, http_status=226)
183 assert r.tcpdump
184 # vsftp closes control connection without niceties,
185 # look only at ports from DATA connection.
186 data_ports = vsftpd.get_data_ports(r)
187 assert len(data_ports), f'unable to find FTP data port connected to\n{r.dump_logs()}'
188 assert len(r.tcpdump.get_rsts(port_pairs=data_ports)) == 0, 'Unexpected TCP RST packets'
190 def test_30_08_active_download(self, env: Env, vsftpd: VsFTPD):
191 docname = 'data-10k'
192 curl = CurlClient(env=env)
193 srcfile = os.path.join(vsftpd.docs_dir, f'{docname}')
194 count = 1
195 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]'
196 r = curl.ftp_get(urls=[url], with_stats=True, extra_args=[
197 '--ftp-port', '127.0.0.1'
198 ])
199 r.check_stats(count=count, http_status=226)
200 self.check_downloads(curl, srcfile, count)
201 r.check_stats_timelines()
203 def test_30_09_active_up_file(self, env: Env, vsftpd: VsFTPD):
204 docname = 'upload-1k'
205 curl = CurlClient(env=env)
206 srcfile = os.path.join(env.gen_dir, docname)
207 dstfile = os.path.join(vsftpd.docs_dir, docname)
208 self._rmf(dstfile)
209 count = 1
210 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
211 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, extra_args=[
212 '--ftp-port', '127.0.0.1'
213 ])
214 r.check_stats(count=count, http_status=226)
215 self.check_upload(env, vsftpd, docname=docname)
216 r.check_stats_timelines()
218 def test_30_10_active_up_ascii(self, env: Env, vsftpd: VsFTPD):
219 docname = 'upload-1k'
220 curl = CurlClient(env=env)
221 srcfile = os.path.join(env.gen_dir, docname)
222 dstfile = os.path.join(vsftpd.docs_dir, docname)
223 self._rmf(dstfile)
224 count = 1
225 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/'
226 r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, extra_args=[
227 '--ftp-port', '127.0.0.1', '--use-ascii'
228 ])
229 r.check_stats(count=count, http_status=226)
230 self.check_upload(env, vsftpd, docname=docname, binary=False)
231 r.check_stats_timelines()
233 def test_30_11_download_non_existing(self, env: Env, vsftpd: VsFTPD):
234 curl = CurlClient(env=env)
235 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/does-not-exist'
236 r = curl.ftp_get(urls=[url], with_stats=True)
237 r.check_exit_code(78)
238 r.check_stats(count=1, exitcode=78)
239 r.check_stats_timelines()
241 def test_30_12_upload_eprt(self, env: Env, vsftpd: VsFTPD):
242 docname = 'test_30_12'
243 client = LocalClient(name='cli_ftp_upload', env=env)
244 if not client.exists():
245 pytest.skip(f'example client not built: {client.name}')
246 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}'
247 r = client.run(args=['-r', f'{env.ftp_domain}:{vsftpd.port}:127.0.0.1', url])
248 r.check_exit_code(0)
249 dstfile = os.path.join(vsftpd.docs_dir, docname)
250 assert os.path.exists(dstfile), f'{r.dump_logs()}'
252 @pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
253 def test_30_13_wildcard(self, env: Env, vsftpd: VsFTPD):
254 run_env = os.environ.copy()
255 run_env['CURL_DBG_FTP_WILDCARD'] = '1'
256 curl = CurlClient(env=env, run_env=run_env)
257 url = f'ftp://{env.ftp_domain}:{vsftpd.port}/data*0k'
258 r = curl.ftp_get(urls=[url], with_stats=True)
259 r.check_stats(count=1, http_status=0)
260 reffile = os.path.join(env.gen_dir, 'test-30-13.reference')
261 with open(reffile, "w") as fd:
262 with open(os.path.join(vsftpd.docs_dir, 'data-0k')) as fd2:
263 fd.writelines(fd2.readlines())
264 with open(os.path.join(vsftpd.docs_dir, 'data-10k')) as fd2:
265 fd.writelines(fd2.readlines())
266 dfile = os.path.join(curl.run_dir, 'download_#1.data')
267 self.check_download(reffile, dfile)
269 def check_downloads(self, client, srcfile: str, count: int,
270 complete: bool = True):
271 for i in range(count):
272 dfile = client.download_file(i)
273 self.check_download(srcfile, dfile, complete)
275 def check_download(self, reffile: str, dfile: str, complete: bool = True):
276 assert os.path.exists(dfile)
277 if complete and not filecmp.cmp(reffile, dfile, shallow=False):
278 with open(reffile) as fa, open(dfile) as fb:
279 a = fa.readlines()
280 b = fb.readlines()
281 diff = "".join(difflib.unified_diff(a=a, b=b,
282 fromfile=reffile,
283 tofile=dfile,
284 n=1))
285 assert False, f'download {dfile} differs:\n{diff}'
287 def check_upload(self, env, vsftpd: VsFTPD, docname, binary=True):
288 srcfile = os.path.join(env.gen_dir, docname)
289 dstfile = os.path.join(vsftpd.docs_dir, docname)
290 assert os.path.exists(srcfile)
291 assert os.path.exists(dstfile)
292 if not filecmp.cmp(srcfile, dstfile, shallow=False):
293 with open(srcfile) as fa, open(dstfile) as fb:
294 a = fa.readlines()
295 b = fb.readlines()
296 diff = "".join(difflib.unified_diff(a=a, b=b,
297 fromfile=srcfile,
298 tofile=dstfile,
299 n=1))
300 assert not binary and len(diff) == 0, f'upload {dstfile} differs:\n{diff}'
Diagnostics4 git calls · 143 ms
CallStatusSizeTime0143 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=139
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2002.9 KiB139 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3004
git.response.cache=MISS
git.response.duration_ms=139
git.fetch.wants=756457fbca84f8e6419e18d0d333f6188acb7224
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3004
git.pkt.payload.max=2983
git.sideband.band1.bytes=2982
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2dff278992308
Request
6c92b504…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format