Artifactspublic-v0

curl

master
ply4 git calls257 ms291 KiB2 cachedprotocol v2
506 lines15 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "curl_setup.h"
26#include "urldata.h"
27#include "cfilters.h"
28#include "multiif.h"
29#include "sendf.h"
30#include "curl_trc.h"
31#include "transfer.h"
32#include "cw-out.h"
33#include "cw-pause.h"
34#include "progress.h"
36/**
37 * OVERALL DESIGN of this client writer
38 *
39 * The 'cw-out' writer is supposed to be the last writer in a transfer's
40 * stack. It is always added when that stack is initialized. Its purpose
41 * is to pass BODY and HEADER bytes to the client-installed callback
42 * functions.
43 *
44 * These callback may return `CURL_WRITEFUNC_PAUSE` to indicate that the
45 * data had not been written and the whole transfer should stop receiving
46 * new data. Or at least, stop calling the functions. When the transfer
47 * is "unpaused" by the client, the previous data shall be passed as
48 * if nothing happened.
49 *
50 * The `cw-out` writer therefore manages buffers for bytes that could
51 * not be written. Data that was already in flight from the server also
52 * needs buffering on paused transfer when it arrives.
53 *
54 * In addition, the writer allows buffering of "small" body writes,
55 * so client functions are called less often. That is only enabled on a
56 * number of conditions.
57 *
58 * HEADER and BODY data may arrive in any order. For paused transfers,
59 * a list of `struct cw_out_buf` is kept for `cw_out_type` types. The
60 * list may be: [BODY]->[HEADER]->[BODY]->[HEADER]....
61 * When unpausing, this list is "played back" to the client callbacks.
62 *
63 * The amount of bytes being buffered is limited by `DYN_PAUSE_BUFFER`
64 * and when that is exceeded `CURLE_TOO_LARGE` is returned as error.
65 */
66typedef enum {
67 CW_OUT_NONE,
68 CW_OUT_BODY,
69 CW_OUT_BODY_0LEN,
70 CW_OUT_HDS
71} cw_out_type;
73struct cw_out_buf {
74 struct cw_out_buf *next;
75 struct dynbuf b;
76 cw_out_type type;
77};
79static struct cw_out_buf *cw_out_buf_create(cw_out_type otype)
81 struct cw_out_buf *cwbuf = curlx_calloc(1, sizeof(*cwbuf));
82 if(cwbuf) {
83 cwbuf->type = otype;
84 curlx_dyn_init(&cwbuf->b, DYN_PAUSE_BUFFER);
85 }
86 return cwbuf;
89static void cw_out_buf_free(struct cw_out_buf *cwbuf)
91 if(cwbuf) {
92 curlx_dyn_free(&cwbuf->b);
93 curlx_free(cwbuf);
94 }
97struct cw_out_ctx {
98 struct Curl_cwriter super;
99 struct cw_out_buf *buf;
100 struct cw_out_buf *tail;
101 size_t buffered_len;
102 BIT(errored);
103};
105static CURLcode cw_out_init(struct Curl_easy *data,
106 struct Curl_cwriter *writer)
108 struct cw_out_ctx *ctx = writer->ctx;
109 (void)data;
110 ctx->buf = NULL;
111 ctx->tail = NULL;
112 ctx->buffered_len = 0;
113 return CURLE_OK;
116static void cw_out_bufs_free(struct cw_out_ctx *ctx)
118 while(ctx->buf) {
119 struct cw_out_buf *next = ctx->buf->next;
120 cw_out_buf_free(ctx->buf);
121 ctx->buf = next;
122 }
123 ctx->tail = NULL;
124 ctx->buffered_len = 0;
127static void cw_out_close(struct Curl_easy *data, struct Curl_cwriter *writer)
129 struct cw_out_ctx *ctx = writer->ctx;
131 (void)data;
132 cw_out_bufs_free(ctx);
135/**
136 * Return the current curl_write_callback and user_data for the buf type
137 */
138static void cw_get_writefunc(struct Curl_easy *data, cw_out_type otype,
139 curl_write_callback *pwcb, void **pwcb_data,
140 size_t *pmax_write, size_t *pmin_write)
142 switch(otype) {
143 case CW_OUT_BODY:
144 case CW_OUT_BODY_0LEN:
145 if(Curl_ftp_listmode(data)) {
146 *pwcb = Curl_ftp_parselist;
147 *pwcb_data = data;
148 }
149 else {
150 *pwcb = data->set.fwrite_func;
151 *pwcb_data = data->set.out;
152 }
153 *pmax_write = CURL_MAX_WRITE_SIZE;
154 /* if we ever want buffering of BODY output, we can set `min_write`
155 * the preferred size. The default should always be to pass data
156 * to the client as it comes without delay */
157 *pmin_write = 0;
158 break;
159 case CW_OUT_HDS:
160 *pwcb = data->set.fwrite_header ? data->set.fwrite_header :
161 (data->set.writeheader ? data->set.fwrite_func : NULL);
162 *pwcb_data = data->set.writeheader;
163 *pmax_write = 0; /* do not chunk-write headers, write them as they are */
164 *pmin_write = 0;
165 break;
166 default:
167 *pwcb = NULL;
168 *pwcb_data = NULL;
169 *pmax_write = CURL_MAX_WRITE_SIZE;
170 *pmin_write = 0;
171 }
174static CURLcode cw_out_cb_write(struct Curl_easy *data,
175 curl_write_callback wcb,
176 void *wcb_data,
177 cw_out_type otype,
178 const char *buf, size_t blen,
179 size_t *pnwritten)
181 size_t nwritten;
182 CURLcode result;
184 NOVERBOSE((void)otype);
186 DEBUGASSERT(data->conn);
187 *pnwritten = 0;
188 {
189 struct Curl_mapi_guard guard;
190 CURL_CBAPI_START(&guard, data, easy_cw_out_cb);
191 nwritten = wcb((char *)CURL_UNCONST(buf), 1, blen, wcb_data);
192 CURL_CBAPI_END(&guard);
193 }
194 CURL_TRC_WRITE(data, "[OUT] wrote %zu %s bytes, type=%x -> %zu",
195 blen, (otype == CW_OUT_HDS) ? "header" : "body",
196 (unsigned int)otype, nwritten);
197 if(nwritten == CURL_WRITEFUNC_PAUSE) {
198 if(data->conn->scheme->flags & PROTOPT_NONETWORK) {
199 /* Protocols that work without network cannot be paused. This is
200 actually only file:// now, and it cannot pause since the transfer is
201 not done using the "normal" procedure. */
202 failf(data, "Write callback asked for PAUSE when not supported");
203 return CURLE_WRITE_ERROR;
204 }
205 data->req.writer.paused = TRUE;
206 CURL_TRC_WRITE(data, "[OUT] PAUSE requested by client");
207 result = Curl_xfer_pause_recv(data, TRUE);
208 return result ? result : CURLE_AGAIN;
209 }
210 else if(nwritten == CURL_WRITEFUNC_ERROR) {
211 failf(data, "client returned ERROR on write of %zu bytes", blen);
212 return CURLE_WRITE_ERROR;
213 }
214 else if(nwritten != blen) {
215 failf(data, "Failure writing output to destination, "
216 "passed %zu returned %zu", blen, nwritten);
217 return CURLE_WRITE_ERROR;
218 }
219 *pnwritten = nwritten;
220 return CURLE_OK;
223static CURLcode cw_out_ptr_flush(struct cw_out_ctx *ctx,
224 struct Curl_easy *data,
225 cw_out_type otype,
226 bool flush_all,
227 const char *buf, size_t blen,
228 size_t *pconsumed)
230 curl_write_callback wcb = NULL;
231 void *wcb_data;
232 size_t max_write, min_write;
233 size_t wlen, nwritten = 0;
234 CURLcode result = CURLE_OK;
236 /* If we errored once, we do not invoke the client callback again */
237 if(ctx->errored)
238 return CURLE_WRITE_ERROR;
240 /* write callbacks may get NULLed by the client between calls. */
241 cw_get_writefunc(data, otype, &wcb, &wcb_data, &max_write, &min_write);
242 if(!wcb) {
243 *pconsumed = blen;
244 return CURLE_OK;
245 }
247 *pconsumed = 0;
248 if(otype == CW_OUT_BODY_0LEN) {
249 DEBUGASSERT(!blen);
250 return cw_out_cb_write(data, wcb, wcb_data, otype,
251 buf, blen, &nwritten);
252 }
253 else {
254 while(blen && !data->req.writer.paused) {
255 if(!flush_all && blen < min_write)
256 break;
257 wlen = max_write ? CURLMIN(blen, max_write) : blen;
258 if(otype == CW_OUT_BODY)
259 result = Curl_pgrs_deliver_check(data, wlen);
260 if(!result)
261 result = cw_out_cb_write(data, wcb, wcb_data, otype,
262 buf, wlen, &nwritten);
263 if(result)
264 return result;
265 if(otype == CW_OUT_BODY)
266 Curl_pgrs_deliver_inc(data, nwritten);
267 *pconsumed += nwritten;
268 blen -= nwritten;
269 buf += nwritten;
270 /* write functions/data may have been re-setopt'ed. sigh. */
271 cw_get_writefunc(data, otype, &wcb, &wcb_data, &max_write, &min_write);
272 if(!wcb) {
273 *pconsumed += blen;
274 break;
275 }
276 }
277 }
278 return CURLE_OK;
281static CURLcode cw_out_buf_flush(struct cw_out_ctx *ctx,
282 struct Curl_easy *data,
283 struct cw_out_buf *cwbuf,
284 bool flush_all)
286 CURLcode result = CURLE_OK;
288 if(curlx_dyn_len(&cwbuf->b) || (cwbuf->type == CW_OUT_BODY_0LEN)) {
289 size_t consumed;
291 result = cw_out_ptr_flush(ctx, data, cwbuf->type, flush_all,
292 curlx_dyn_ptr(&cwbuf->b),
293 curlx_dyn_len(&cwbuf->b),
294 &consumed);
295 if(result && (result != CURLE_AGAIN))
296 return result;
297 result = CURLE_OK;
299 if(consumed) {
300 if(consumed == curlx_dyn_len(&cwbuf->b)) {
301 curlx_dyn_free(&cwbuf->b);
302 }
303 else {
304 DEBUGASSERT(consumed < curlx_dyn_len(&cwbuf->b));
305 result = curlx_dyn_tail(&cwbuf->b,
306 curlx_dyn_len(&cwbuf->b) - consumed);
307 if(result)
308 return result;
309 }
310 }
311 }
312 return result;
315static CURLcode cw_out_flush_chain(struct cw_out_ctx *ctx,
316 struct Curl_easy *data,
317 bool flush_all)
319 CURLcode result;
321 /* write the chain oldest first until it blocks or gets empty */
322 while(ctx->buf) {
323 struct cw_out_buf *cwbuf = ctx->buf;
324 size_t blen_before;
326 if(data->req.writer.paused)
327 return CURLE_OK;
329 blen_before = curlx_dyn_len(&cwbuf->b);
330 result = cw_out_buf_flush(ctx, data, cwbuf, flush_all);
331 if(result)
332 return result;
333 ctx->buffered_len -= blen_before - curlx_dyn_len(&cwbuf->b);
335 if(curlx_dyn_len(&cwbuf->b)) {
336 /* could not write it all, paused again? */
337 DEBUGASSERT(data->req.writer.paused);
338 return CURLE_OK;
339 }
341 ctx->buf = cwbuf->next;
342 if(!ctx->buf)
343 ctx->tail = NULL;
344 cw_out_buf_free(cwbuf);
345 }
346 return CURLE_OK;
349static CURLcode cw_out_append(struct cw_out_ctx *ctx,
350 struct Curl_easy *data,
351 cw_out_type otype,
352 const char *buf, size_t blen)
354 CURLcode result;
356 CURL_TRC_WRITE(data, "[OUT] paused, buffering %zu more bytes (%zu/%d)",
357 blen, ctx->buffered_len, DYN_PAUSE_BUFFER);
358 if(ctx->buffered_len + blen > DYN_PAUSE_BUFFER) {
359 failf(data, "pause buffer not large enough -> CURLE_TOO_LARGE");
360 return CURLE_TOO_LARGE;
361 }
363 /* if we do not have a buffer, or it is of another type, make a new one
364 * and append it at the tail. For CW_OUT_HDS always make a new one, so
365 * we "replay" headers exactly as they came in */
366 if(!ctx->tail || (ctx->tail->type != otype) || (otype == CW_OUT_HDS)) {
367 struct cw_out_buf *cwbuf = cw_out_buf_create(otype);
368 if(!cwbuf)
369 return CURLE_OUT_OF_MEMORY;
370 if(ctx->tail)
371 ctx->tail->next = cwbuf;
372 else
373 ctx->buf = cwbuf;
374 ctx->tail = cwbuf;
375 }
376 DEBUGASSERT(ctx->tail && (ctx->tail->type == otype));
377 result = curlx_dyn_addn(&ctx->tail->b, buf, blen);
378 if(!result)
379 ctx->buffered_len += blen;
380 return result;
383static CURLcode cw_out_do_write(struct cw_out_ctx *ctx,
384 struct Curl_easy *data,
385 cw_out_type otype,
386 bool flush_all,
387 const char *buf, size_t blen)
389 CURLcode result = CURLE_OK;
391 /* if we have buffered data and the last chunk buffered is of a
392 * different type than what we are writing now, try to flush all */
393 if(ctx->tail && ctx->tail->type != otype) {
394 result = cw_out_flush_chain(ctx, data, TRUE);
395 if(result)
396 goto out;
397 }
399 if(ctx->buf) {
400 /* still have buffered data, append and flush */
401 result = cw_out_append(ctx, data, otype, buf, blen);
402 if(result)
403 goto out;
404 result = cw_out_flush_chain(ctx, data, flush_all);
405 if(result)
406 goto out;
407 }
408 else {
409 /* nothing buffered, try direct write */
410 size_t consumed;
411 result = cw_out_ptr_flush(ctx, data, otype, flush_all,
412 buf, blen, &consumed);
413 if(result && (result != CURLE_AGAIN))
414 return result;
415 result = CURLE_OK;
416 if(consumed < blen) {
417 /* did not write all, append the rest */
418 result = cw_out_append(ctx, data, otype,
419 buf + consumed, blen - consumed);
420 if(result)
421 goto out;
422 }
423 }
425out:
426 if(result) {
427 /* We do not want to invoke client callbacks a second time after
428 * encountering an error. See issue #13337 */
429 ctx->errored = TRUE;
430 cw_out_bufs_free(ctx);
431 }
432 return result;
435static CURLcode cw_out_write(struct Curl_easy *data,
436 struct Curl_cwriter *writer, int type,
437 const char *buf, size_t blen)
439 struct cw_out_ctx *ctx = writer->ctx;
440 CURLcode result;
441 bool flush_all = !!(type & CLIENTWRITE_EOS);
443 if((type & CLIENTWRITE_BODY) ||
444 ((type & CLIENTWRITE_HEADER) && data->set.include_header)) {
445 cw_out_type otype = (!blen && (type & CLIENTWRITE_0LEN)) ?
446 CW_OUT_BODY_0LEN : CW_OUT_BODY;
447 result = cw_out_do_write(ctx, data, otype, flush_all, buf, blen);
448 if(result)
449 return result;
450 }
452 if(type & (CLIENTWRITE_HEADER | CLIENTWRITE_INFO)) {
453 result = cw_out_do_write(ctx, data, CW_OUT_HDS, flush_all, buf, blen);
454 if(result)
455 return result;
456 }
458 return CURLE_OK;
461static CURLcode cw_out_do_flush(struct Curl_easy *data,
462 struct Curl_cwriter *cw_out,
463 bool flush_all)
465 struct cw_out_ctx *ctx = (struct cw_out_ctx *)cw_out;
467 if(ctx->errored)
468 return CURLE_WRITE_ERROR;
470 if(!data->req.writer.paused && ctx->buf) {
471 CURLcode result;
473 CURL_TRC_WRITE(data, "[OUT] flush");
474 result = cw_out_flush_chain(ctx, data, flush_all);
475 if(result) {
476 ctx->errored = TRUE;
477 cw_out_bufs_free(ctx);
478 return result;
479 }
480 }
481 return CURLE_OK;
484const struct Curl_cwtype Curl_cwt_out = {
485 "cw-out",
486 NULL,
487 0,
488 cw_out_init,
489 cw_out_write,
490 cw_out_do_flush,
491 cw_out_close,
492 sizeof(struct cw_out_ctx)
493};
495CURLcode Curl_cw_out_done(struct Curl_easy *data)
497 struct Curl_cwriter *cw_out;
498 CURLcode result = CURLE_OK;
500 cw_out = Curl_cwriter_get_by_type(data, &Curl_cwt_out);
501 if(cw_out) {
502 CURL_TRC_WRITE(data, "[OUT] done");
503 result = Curl_client_flush(data, TRUE);
504 }
505 return result;
Diagnostics4 git calls · 257 ms
CallStatusSizeTime0257 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=252
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2004.3 KiB252 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4363
git.response.cache=MISS
git.response.duration_ms=252
git.fetch.wants=e880a38761ef46bf12a8762ce2306eab5541b943
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4363
git.pkt.payload.max=4342
git.sideband.band1.bytes=4341
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e1ceadfcdec6f8
Request
d0cad9bf…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format