master
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "curl_setup.h"
26#include "bufq.h"
28static bool chunk_is_empty(const struct buf_chunk *chunk)
29{
30 return chunk->r_offset >= chunk->w_offset;
31}
33static bool chunk_is_full(const struct buf_chunk *chunk)
34{
35 return chunk->w_offset >= chunk->dlen;
36}
38static size_t chunk_len(const struct buf_chunk *chunk)
39{
40 return chunk->w_offset - chunk->r_offset;
41}
43static void chunk_reset(struct buf_chunk *chunk)
44{
45 chunk->next = NULL;
46 chunk->r_offset = chunk->w_offset = 0;
47}
49static size_t chunk_append(struct buf_chunk *chunk,
50 const uint8_t *buf, size_t len)
51{
52 uint8_t *p = &chunk->x.data[chunk->w_offset];
53 size_t n = chunk->dlen - chunk->w_offset;
54 DEBUGASSERT(chunk->dlen >= chunk->w_offset);
55 if(n) {
56 n = CURLMIN(n, len);
57 memcpy(p, buf, n);
58 chunk->w_offset += n;
59 }
60 return n;
61}
63static size_t chunk_read(struct buf_chunk *chunk,
64 uint8_t *buf, size_t len)
65{
66 uint8_t *p = &chunk->x.data[chunk->r_offset];
67 size_t n = chunk->w_offset - chunk->r_offset;
68 DEBUGASSERT(chunk->w_offset >= chunk->r_offset);
69 if(!n) {
70 return 0;
71 }
72 else if(n <= len) {
73 memcpy(buf, p, n);
74 chunk->r_offset = chunk->w_offset = 0;
75 return n;
76 }
77 else {
78 memcpy(buf, p, len);
79 chunk->r_offset += len;
80 return len;
81 }
82}
84static CURLcode chunk_slurpn(struct buf_chunk *chunk, size_t max_len,
85 Curl_bufq_reader *reader,
86 void *reader_ctx, size_t *pnread)
87{
88 uint8_t *p = &chunk->x.data[chunk->w_offset];
89 size_t n = chunk->dlen - chunk->w_offset; /* free amount */
90 CURLcode result;
92 *pnread = 0;
93 DEBUGASSERT(chunk->dlen >= chunk->w_offset);
94 if(!n)
95 return CURLE_AGAIN;
96 if(max_len && n > max_len)
97 n = max_len;
98 result = reader(reader_ctx, p, n, pnread);
99 if(!result) {
100 DEBUGASSERT(*pnread <= n);
101 chunk->w_offset += *pnread;
102 }
103 return result;
104}
106static void chunk_peek(const struct buf_chunk *chunk,
107 const uint8_t **pbuf, size_t *plen)
108{
109 DEBUGASSERT(chunk->w_offset >= chunk->r_offset);
110 *pbuf = &chunk->x.data[chunk->r_offset];
111 *plen = chunk->w_offset - chunk->r_offset;
112}
114static void chunk_peek_at(const struct buf_chunk *chunk, size_t offset,
115 const uint8_t **pbuf, size_t *plen)
116{
117 offset += chunk->r_offset;
118 DEBUGASSERT(chunk->w_offset >= offset);
119 *pbuf = &chunk->x.data[offset];
120 *plen = chunk->w_offset - offset;
121}
123static size_t chunk_skip(struct buf_chunk *chunk, size_t amount)
124{
125 size_t n = chunk->w_offset - chunk->r_offset;
126 DEBUGASSERT(chunk->w_offset >= chunk->r_offset);
127 if(n) {
128 n = CURLMIN(n, amount);
129 chunk->r_offset += n;
130 if(chunk->r_offset == chunk->w_offset)
131 chunk->r_offset = chunk->w_offset = 0;
132 }
133 return n;
134}
136static void chunk_list_free(struct buf_chunk **anchor)
137{
138 struct buf_chunk *chunk;
139 while(*anchor) {
140 chunk = *anchor;
141 *anchor = chunk->next;
142 curlx_free(chunk);
143 }
144}
146void Curl_bufcp_init(struct bufc_pool *pool,
147 size_t chunk_size, size_t spare_max)
148{
149 DEBUGASSERT(chunk_size > 0);
150 DEBUGASSERT(spare_max > 0);
151 memset(pool, 0, sizeof(*pool));
152 pool->chunk_size = chunk_size;
153 pool->spare_max = spare_max;
154}
156static CURLcode bufcp_take(struct bufc_pool *pool,
157 struct buf_chunk **pchunk)
158{
159 struct buf_chunk *chunk = NULL;
161 if(pool->spare) {
162 chunk = pool->spare;
163 pool->spare = chunk->next;
164 --pool->spare_count;
165 chunk_reset(chunk);
166 *pchunk = chunk;
167 return CURLE_OK;
168 }
170 /* Check for integer overflow before allocation */
171 if(pool->chunk_size > SIZE_MAX - sizeof(*chunk)) {
172 *pchunk = NULL;
173 return CURLE_OUT_OF_MEMORY;
174 }
176 chunk = curlx_calloc(1, sizeof(*chunk) + pool->chunk_size);
177 if(!chunk) {
178 *pchunk = NULL;
179 return CURLE_OUT_OF_MEMORY;
180 }
181 chunk->dlen = pool->chunk_size;
182 *pchunk = chunk;
183 return CURLE_OK;
184}
186static void bufcp_put(struct bufc_pool *pool,
187 struct buf_chunk *chunk)
188{
189 if(pool->spare_count >= pool->spare_max) {
190 curlx_free(chunk);
191 }
192 else {
193 chunk_reset(chunk);
194 chunk->next = pool->spare;
195 pool->spare = chunk;
196 ++pool->spare_count;
197 }
198}
200void Curl_bufcp_free(struct bufc_pool *pool)
201{
202 chunk_list_free(&pool->spare);
203 pool->spare_count = 0;
204}
206static void bufq_init(struct bufq *q, struct bufc_pool *pool,
207 size_t chunk_size, size_t max_chunks, int opts)
208{
209 DEBUGASSERT(chunk_size > 0);
210 DEBUGASSERT(max_chunks > 0);
211 memset(q, 0, sizeof(*q));
212 q->chunk_size = chunk_size;
213 q->max_chunks = max_chunks;
214 q->pool = pool;
215 q->opts = opts;
216}
218void Curl_bufq_init2(struct bufq *q, size_t chunk_size, size_t max_chunks,
219 int opts)
220{
221 bufq_init(q, NULL, chunk_size, max_chunks, opts);
222}
224void Curl_bufq_init(struct bufq *q, size_t chunk_size, size_t max_chunks)
225{
226 bufq_init(q, NULL, chunk_size, max_chunks, BUFQ_OPT_NONE);
227}
229void Curl_bufq_initp(struct bufq *q, struct bufc_pool *pool,
230 size_t max_chunks, int opts)
231{
232 bufq_init(q, pool, pool->chunk_size, max_chunks, opts);
233}
235void Curl_bufq_free(struct bufq *q)
236{
237 chunk_list_free(&q->head);
238 chunk_list_free(&q->spare);
239 q->tail = NULL;
240 q->chunk_count = 0;
241}
243void Curl_bufq_reset(struct bufq *q)
244{
245 struct buf_chunk *chunk;
246 while(q->head) {
247 chunk = q->head;
248 q->head = chunk->next;
249 chunk->next = q->spare;
250 q->spare = chunk;
251 }
252 q->tail = NULL;
253}
255size_t Curl_bufq_len(const struct bufq *q)
256{
257 const struct buf_chunk *chunk = q->head;
258 size_t len = 0;
259 while(chunk) {
260 len += chunk_len(chunk);
261 chunk = chunk->next;
262 }
263 return len;
264}
266bool Curl_bufq_is_empty(const struct bufq *q)
267{
268 return !q->head || chunk_is_empty(q->head);
269}
271bool Curl_bufq_is_full(const struct bufq *q)
272{
273 if(!q->tail || q->spare)
274 return FALSE;
275 if(q->chunk_count < q->max_chunks)
276 return FALSE;
277 if(q->chunk_count > q->max_chunks)
278 return TRUE;
279 /* we have no spares and cannot make more, is the tail full? */
280 return chunk_is_full(q->tail);
281}
283static struct buf_chunk *get_spare(struct bufq *q)
284{
285 struct buf_chunk *chunk = NULL;
287 if(q->spare) {
288 chunk = q->spare;
289 q->spare = chunk->next;
290 chunk_reset(chunk);
291 return chunk;
292 }
294 if(q->chunk_count >= q->max_chunks && (!(q->opts & BUFQ_OPT_SOFT_LIMIT)))
295 return NULL;
297 if(q->pool) {
298 if(bufcp_take(q->pool, &chunk))
299 return NULL;
300 ++q->chunk_count;
301 return chunk;
302 }
303 else {
304 /* Check for integer overflow before allocation */
305 if(q->chunk_size > SIZE_MAX - sizeof(*chunk)) {
306 return NULL;
307 }
309 chunk = curlx_calloc(1, sizeof(*chunk) + q->chunk_size);
310 if(!chunk)
311 return NULL;
312 chunk->dlen = q->chunk_size;
313 ++q->chunk_count;
314 return chunk;
315 }
316}
318static void prune_head(struct bufq *q)
319{
320 struct buf_chunk *chunk;
322 while(q->head && chunk_is_empty(q->head)) {
323 chunk = q->head;
324 q->head = chunk->next;
325 if(q->tail == chunk)
326 q->tail = q->head;
327 if(q->pool) {
328 bufcp_put(q->pool, chunk);
329 --q->chunk_count;
330 }
331 else if((q->chunk_count > q->max_chunks) ||
332 (q->opts & BUFQ_OPT_NO_SPARES)) {
333 /* SOFT_LIMIT allowed us more than max. free spares until
334 * we are at max again. Or free them if we are configured
335 * to not use spares. */
336 curlx_free(chunk);
337 --q->chunk_count;
338 }
339 else {
340 chunk->next = q->spare;
341 q->spare = chunk;
342 }
343 }
344}
346static struct buf_chunk *get_non_full_tail(struct bufq *q)
347{
348 struct buf_chunk *chunk;
350 if(q->tail && !chunk_is_full(q->tail))
351 return q->tail;
352 chunk = get_spare(q);
353 if(chunk) {
354 /* new tail, and possibly new head */
355 if(q->tail) {
356 q->tail->next = chunk;
357 q->tail = chunk;
358 }
359 else {
360 DEBUGASSERT(!q->head);
361 q->head = q->tail = chunk;
362 }
363 }
364 return chunk;
365}
367CURLcode Curl_bufq_write(struct bufq *q,
368 const uint8_t *buf, size_t len,
369 size_t *pnwritten)
370{
371 struct buf_chunk *tail;
372 size_t n;
374 DEBUGASSERT(q->max_chunks > 0);
375 *pnwritten = 0;
376 while(len) {
377 tail = get_non_full_tail(q);
378 if(!tail) {
379 if((q->chunk_count < q->max_chunks) || (q->opts & BUFQ_OPT_SOFT_LIMIT))
380 /* should have gotten a tail, but did not */
381 return CURLE_OUT_OF_MEMORY;
382 break;
383 }
384 n = chunk_append(tail, buf, len);
385 if(!n)
386 break;
387 *pnwritten += n;
388 buf += n;
389 len -= n;
390 }
391 return (!*pnwritten && len) ? CURLE_AGAIN : CURLE_OK;
392}
394CURLcode Curl_bufq_cwrite(struct bufq *q,
395 const char *buf, size_t len,
396 size_t *pnwritten)
397{
398 return Curl_bufq_write(q, (const uint8_t *)buf, len, pnwritten);
399}
401CURLcode Curl_bufq_read(struct bufq *q, uint8_t *buf, size_t len,
402 size_t *pnread)
403{
404 *pnread = 0;
405 while(len && q->head) {
406 size_t n = chunk_read(q->head, buf, len);
407 if(n) {
408 *pnread += n;
409 buf += n;
410 len -= n;
411 }
412 prune_head(q);
413 }
414 return (!*pnread) ? CURLE_AGAIN : CURLE_OK;
415}
417CURLcode Curl_bufq_cread(struct bufq *q, char *buf, size_t len,
418 size_t *pnread)
419{
420 return Curl_bufq_read(q, (uint8_t *)buf, len, pnread);
421}
423bool Curl_bufq_peek(struct bufq *q,
424 const uint8_t **pbuf, size_t *plen)
425{
426 if(q->head && chunk_is_empty(q->head)) {
427 prune_head(q);
428 }
429 if(q->head && !chunk_is_empty(q->head)) {
430 chunk_peek(q->head, pbuf, plen);
431 return TRUE;
432 }
433 *pbuf = NULL;
434 *plen = 0;
435 return FALSE;
436}
438bool Curl_bufq_peek_at(struct bufq *q, size_t offset,
439 const uint8_t **pbuf, size_t *plen)
440{
441 struct buf_chunk *c = q->head;
442 size_t clen;
444 while(c) {
445 clen = chunk_len(c);
446 if(!clen)
447 break;
448 if(offset >= clen) {
449 offset -= clen;
450 c = c->next;
451 continue;
452 }
453 chunk_peek_at(c, offset, pbuf, plen);
454 return TRUE;
455 }
456 *pbuf = NULL;
457 *plen = 0;
458 return FALSE;
459}
461void Curl_bufq_skip(struct bufq *q, size_t amount)
462{
463 size_t n;
465 while(amount && q->head) {
466 n = chunk_skip(q->head, amount);
467 amount -= n;
468 prune_head(q);
469 }
470}
472CURLcode Curl_bufq_pass(struct bufq *q, Curl_bufq_writer *writer,
473 void *writer_ctx, size_t *pwritten)
474{
475 const uint8_t *buf;
476 size_t blen;
477 CURLcode result = CURLE_OK;
479 *pwritten = 0;
480 while(Curl_bufq_peek(q, &buf, &blen)) {
481 size_t chunk_written;
483 result = writer(writer_ctx, buf, blen, &chunk_written);
484 if(result) {
485 if((result == CURLE_AGAIN) && *pwritten) {
486 /* blocked on subsequent write, report success */
487 result = CURLE_OK;
488 }
489 break;
490 }
491 if(!chunk_written) {
492 if(!*pwritten) {
493 /* treat as blocked */
494 result = CURLE_AGAIN;
495 }
496 break;
497 }
498 *pwritten += chunk_written;
499 Curl_bufq_skip(q, chunk_written);
500 }
501 return result;
502}
504CURLcode Curl_bufq_write_pass(struct bufq *q,
505 const uint8_t *buf, size_t len,
506 Curl_bufq_writer *writer, void *writer_ctx,
507 size_t *pwritten)
508{
509 CURLcode result = CURLE_OK;
510 size_t n;
512 *pwritten = 0;
513 while(len) {
514 if(Curl_bufq_is_full(q)) {
515 /* try to make room in case we are full */
516 result = Curl_bufq_pass(q, writer, writer_ctx, &n);
517 if(result) {
518 if(result != CURLE_AGAIN) {
519 /* real error, fail */
520 return result;
521 }
522 /* would block, bufq is full, give up */
523 break;
524 }
525 }
527 /* Add to bufq as much as there is room for */
528 result = Curl_bufq_write(q, buf, len, &n);
529 if(result) {
530 if(result != CURLE_AGAIN)
531 /* real error, fail */
532 return result;
533 /* result == CURLE_AGAIN */
534 if(*pwritten)
535 /* we did write successfully before */
536 result = CURLE_OK;
537 return result;
538 }
539 else if(n == 0)
540 /* edge case of writer returning 0 (and len is >0)
541 * break or we might enter an infinite loop here */
542 break;
544 /* Track what we added to bufq */
545 buf += n;
546 len -= n;
547 *pwritten += n;
548 }
550 return (!*pwritten && len) ? CURLE_AGAIN : CURLE_OK;
551}
553CURLcode Curl_bufq_sipn(struct bufq *q, size_t max_len,
554 Curl_bufq_reader *reader, void *reader_ctx,
555 size_t *pnread)
556{
557 struct buf_chunk *tail = NULL;
559 *pnread = 0;
560 tail = get_non_full_tail(q);
561 if(!tail) {
562 if(q->chunk_count < q->max_chunks)
563 return CURLE_OUT_OF_MEMORY;
564 /* full, blocked */
565 return CURLE_AGAIN;
566 }
568 return chunk_slurpn(tail, max_len, reader, reader_ctx, pnread);
569}
571/**
572 * Read up to `max_len` bytes and append it to the end of the buffer queue.
573 * if `max_len` is 0, no limit is imposed and the call behaves exactly
574 * the same as `Curl_bufq_slurp()`.
575 * Returns the total amount of buf read (may be 0) in `pnread` or error
576 * Note that even in case of an error chunks may have been read and
577 * the buffer queue will have different length than before.
578 */
579static CURLcode bufq_slurpn(struct bufq *q, size_t max_len,
580 Curl_bufq_reader *reader, void *reader_ctx,
581 size_t *pnread)
582{
583 CURLcode result;
585 *pnread = 0;
586 while(1) {
587 size_t n;
588 result = Curl_bufq_sipn(q, max_len, reader, reader_ctx, &n);
589 if(result) {
590 if(!*pnread || result != CURLE_AGAIN) {
591 /* blocked on first read or real error, fail */
592 return result;
593 }
594 result = CURLE_OK;
595 break;
596 }
597 else if(n == 0) {
598 /* eof, result remains CURLE_OK */
599 break;
600 }
601 *pnread += n;
602 if(max_len) {
603 DEBUGASSERT(n <= max_len);
604 max_len -= n;
605 if(!max_len)
606 break;
607 }
608 /* give up slurping when we get less bytes than we asked for */
609 if(q->tail && !chunk_is_full(q->tail))
610 break;
611 }
612 return result;
613}
615CURLcode Curl_bufq_slurp(struct bufq *q, Curl_bufq_reader *reader,
616 void *reader_ctx, size_t *pnread)
617{
618 return bufq_slurpn(q, 0, reader, reader_ctx, pnread);
619}
Diagnostics4 git calls · 40 ms
CallStatusSizeTime040 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2004.1 KiB · hit34 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=33 git.response.bytes=4173 git.response.cache=HIT git.response.duration_ms=34 git.fetch.wants=750d86888f381586bfabd8ef60d9900ac120a190 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4173 git.pkt.payload.max=4152 git.sideband.band1.bytes=4151 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e2a81b18061293
- Request
- 31f5ba51…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format