master
1#***************************************************************************
2# _ _ ____ _
3# Project ___| | | | _ \| |
4# / __| | | | |_) | |
5# | (__| |_| | _ <| |___
6# \___|\___/|_| \_\_____|
7#
8# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://curl.se/docs/copyright.html.
13#
14# You may opt to use, copy, modify, merge, publish, distribute and/or sell
15# copies of the Software, and permit persons to whom the Software is
16# furnished to do so, under the terms of the COPYING file.
17#
18# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19# KIND, either express or implied.
20#
21# SPDX-License-Identifier: curl
22#
23###########################################################################
24#
25import difflib
26import filecmp
27import logging
28import os
30import pytest
31from testenv import CurlClient, Env, Sshd
33log = logging.getLogger(__name__)
36@pytest.mark.skipif(condition=not Env.curl_has_protocol('sftp'), reason="curl built without sfto:")
37@pytest.mark.skipif(condition=not Env.has_sftpd(), reason="missing sftp server")
38class TestSftp:
40 @pytest.fixture(autouse=True, scope='class')
41 @classmethod
42 def _class_scope(cls, env, sshd):
43 env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10 * 1024)
44 env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10 * 1024 * 1024)
45 env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10 * 1024)
46 env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10 * 1024 * 1024)
48 def test_51_01_insecure(self, env: Env, sshd: Sshd):
49 curl = CurlClient(env=env)
50 doc_file = os.path.join(sshd.home_dir, 'data')
51 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
52 r = curl.ssh_download(urls=[url], extra_args=[
53 '--insecure',
54 '--pubkey', sshd.user1_pubkey_file,
55 '--key', sshd.user1_privkey_file,
56 '--user', f'{os.environ["USER"]}:',
57 ])
58 r.check_exit_code(0)
60 def test_51_02_unknown_hosts(self, env: Env, sshd: Sshd):
61 curl = CurlClient(env=env)
62 doc_file = os.path.join(sshd.home_dir, 'data')
63 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
64 r = curl.ssh_download(urls=[url], extra_args=[
65 '--knownhosts', sshd.unknown_hosts,
66 '--pubkey', sshd.user1_pubkey_file,
67 '--key', sshd.user1_privkey_file,
68 '--user', f'{os.environ["USER"]}:',
69 ])
70 r.check_exit_code(60) # CURLE_PEER_FAILED_VERIFICATION
72 def test_51_03_known_hosts(self, env: Env, sshd: Sshd):
73 curl = CurlClient(env=env)
74 doc_file = os.path.join(sshd.home_dir, 'data')
75 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
76 r = curl.ssh_download(urls=[url], extra_args=[
77 '--knownhosts', sshd.known_hosts,
78 '--pubkey', sshd.user1_pubkey_file,
79 '--key', sshd.user1_privkey_file,
80 '--user', f'{os.environ["USER"]}:',
81 ])
82 r.check_exit_code(0)
84 # use key not in authorized_keys file
85 def test_51_04_unauth_user(self, env: Env, sshd: Sshd):
86 curl = CurlClient(env=env)
87 doc_file = os.path.join(sshd.home_dir, 'data')
88 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
89 r = curl.ssh_download(urls=[url], extra_args=[
90 '--knownhosts', sshd.known_hosts,
91 '--pubkey', sshd.user2_pubkey_file,
92 '--key', sshd.user2_privkey_file,
93 '--user', f'{os.environ["USER"]}:',
94 ])
95 r.check_exit_code(67) # CURLE_LOGIN_DENIED
97 def test_51_10_dl_single(self, env: Env, sshd: Sshd):
98 count = 1
99 curl = CurlClient(env=env)
100 doc_file = os.path.join(sshd.home_dir, 'data-10k')
101 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
102 r = curl.ssh_download(urls=[url], extra_args=[
103 '--knownhosts', sshd.known_hosts,
104 '--pubkey', sshd.user1_pubkey_file,
105 '--key', sshd.user1_privkey_file,
106 '--user', f'{os.environ["USER"]}:',
107 ])
108 r.check_exit_code(0)
109 self.check_downloads(curl, doc_file, count)
111 def test_51_11_dl_serial(self, env: Env, sshd: Sshd):
112 count = 5
113 curl = CurlClient(env=env)
114 doc_file = os.path.join(sshd.home_dir, 'data-10k')
115 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
116 r = curl.ssh_download(urls=[url], extra_args=[
117 '--knownhosts', sshd.known_hosts,
118 '--pubkey', sshd.user1_pubkey_file,
119 '--key', sshd.user1_privkey_file,
120 '--user', f'{os.environ["USER"]}:',
121 ])
122 r.check_exit_code(0)
123 self.check_downloads(curl, doc_file, count)
125 def test_51_12_dl_parallel(self, env: Env, sshd: Sshd):
126 count = 5
127 curl = CurlClient(env=env)
128 doc_file = os.path.join(sshd.home_dir, 'data-10k')
129 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
130 r = curl.http_download(urls=[url], extra_args=[
131 '--parallel',
132 '--knownhosts', sshd.known_hosts,
133 '--pubkey', sshd.user1_pubkey_file,
134 '--key', sshd.user1_privkey_file,
135 '--user', f'{os.environ["USER"]}:',
136 ])
137 r.check_exit_code(0)
138 self.check_downloads(curl, doc_file, count)
140 def test_51_13_dl_reuse(self, env: Env, sshd: Sshd):
141 count = 3
142 curl = CurlClient(env=env)
143 doc_file = os.path.join(sshd.home_dir, 'data-10k')
144 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
145 r = curl.ssh_download(urls=[url], extra_args=[
146 '--knownhosts', sshd.known_hosts,
147 '--pubkey', sshd.user1_pubkey_file,
148 '--key', sshd.user1_privkey_file,
149 '--user', f'{os.environ["USER"]}:',
150 ])
151 r.check_exit_code(0)
152 assert len(r.stats) == count, r.dump_logs()
153 assert r.total_connects == 1, r.dump_logs()
155 def test_51_14_dl_no_reuse(self, env: Env, sshd: Sshd):
156 if not env.curl_is_debug():
157 pytest.skip('only works for curl debug builds')
158 count = 3
159 run_env = os.environ.copy()
160 run_env['CURL_FORBID_REUSE'] = '1'
161 curl = CurlClient(env=env, run_env=run_env)
162 doc_file = os.path.join(sshd.home_dir, 'data-10k')
163 url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
164 r = curl.ssh_download(urls=[url], extra_args=[
165 '--knownhosts', sshd.known_hosts,
166 '--pubkey', sshd.user1_pubkey_file,
167 '--key', sshd.user1_privkey_file,
168 '--user', f'{os.environ["USER"]}:',
169 ])
170 r.check_exit_code(0)
171 assert len(r.stats) == count, r.dump_logs()
172 assert r.total_connects == count, r.dump_logs()
174 def test_51_20_ul_single(self, env: Env, sshd: Sshd):
175 srcfile = os.path.join(env.gen_dir, 'data-10k')
176 destfile = os.path.join(sshd.home_dir, 'upload_20.data')
177 curl = CurlClient(env=env)
178 url = f'sftp://{env.domain1}:{sshd.port}/{destfile}'
179 r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
180 '--knownhosts', sshd.known_hosts,
181 '--pubkey', sshd.user1_pubkey_file,
182 '--key', sshd.user1_privkey_file,
183 '--user', f'{os.environ["USER"]}:',
184 ])
185 r.check_exit_code(0)
186 self.check_upload(sshd, srcfile, destfile)
188 def test_51_21_ul_serial(self, env: Env, sshd: Sshd):
189 count = 5
190 srcfile = os.path.join(env.gen_dir, 'data-10k')
191 destfile = os.path.join(sshd.home_dir, 'upload_21.data')
192 curl = CurlClient(env=env)
193 url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
194 r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
195 '--knownhosts', sshd.known_hosts,
196 '--pubkey', sshd.user1_pubkey_file,
197 '--key', sshd.user1_privkey_file,
198 '--user', f'{os.environ["USER"]}:',
199 ])
200 r.check_exit_code(0)
201 for i in range(count):
202 self.check_upload(sshd, srcfile, f'{destfile}.{i}')
204 def test_51_22_ul_parallel(self, env: Env, sshd: Sshd):
205 count = 5
206 srcfile = os.path.join(env.gen_dir, 'data-10k')
207 destfile = os.path.join(sshd.home_dir, 'upload_22.data')
208 curl = CurlClient(env=env)
209 url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
210 r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
211 '--parallel',
212 '--knownhosts', sshd.known_hosts,
213 '--pubkey', sshd.user1_pubkey_file,
214 '--key', sshd.user1_privkey_file,
215 '--user', f'{os.environ["USER"]}:',
216 ])
217 r.check_exit_code(0)
218 for i in range(count):
219 self.check_upload(sshd, srcfile, f'{destfile}.{i}')
221 def check_downloads(self, client, srcfile: str, count: int,
222 complete: bool = True):
223 for i in range(count):
224 dfile = client.download_file(i)
225 assert os.path.exists(dfile)
226 if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
227 with open(srcfile) as fa, open(dfile) as fb:
228 a = fa.readlines()
229 b = fb.readlines()
230 diff = "".join(difflib.unified_diff(a=a, b=b,
231 fromfile=srcfile,
232 tofile=dfile,
233 n=1))
234 assert False, f'download {dfile} differs:\n{diff}'
236 def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True):
237 assert os.path.exists(srcfile)
238 assert os.path.exists(destfile)
239 if not filecmp.cmp(srcfile, destfile, shallow=False):
240 with open(srcfile) as fa, open(destfile) as fb:
241 a = fa.readlines()
242 b = fb.readlines()
243 diff = "".join(difflib.unified_diff(a=a, b=b,
244 fromfile=srcfile,
245 tofile=destfile,
246 n=1))
247 assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}'
Diagnostics4 git calls · 32 ms
CallStatusSizeTime032 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2002.0 KiB · hit27 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=27 git.response.bytes=2089 git.response.cache=HIT git.response.duration_ms=27 git.fetch.wants=878b2e434531479dacd78f354150bd177d4fa297 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2089 git.pkt.payload.max=2068 git.sideband.band1.bytes=2067 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2f9289b6d90c0
- Request
- b303ae9d…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format