Artifactspublic-v0

curl

master
ply4 git calls350 ms294 KiB1 cachedprotocol v2
891 lines26 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "curl_setup.h"
26#include "urldata.h"
27#include "curlx/dynbuf.h"
29#ifdef HAVE_LIBZ
30#include <zlib.h>
31#endif
33#ifdef HAVE_BROTLI
34#ifdef CURL_HAVE_DIAG
35/* Ignore -Wvla warnings in brotli headers */
36#pragma GCC diagnostic push
37#pragma GCC diagnostic ignored "-Wvla"
38#endif
39#include <brotli/decode.h>
40#ifdef CURL_HAVE_DIAG
41#pragma GCC diagnostic pop
42#endif
43#endif
45#ifdef HAVE_ZSTD
46#include <zstd.h>
47#endif
49#include "connect.h"
50#include "sendf.h"
51#include "curl_trc.h"
52#include "content_encoding.h"
54#define CONTENT_ENCODING_DEFAULT "identity"
56#ifndef CURL_DISABLE_HTTP
58/* allow no more than 5 "chained" compression steps */
59#define MAX_ENCODE_STACK 5
61#if defined(HAVE_LIBZ) || defined(HAVE_BROTLI) || defined(HAVE_ZSTD)
62#define DECOMPRESS_BUFFER_SIZE 16384 /* buffer size for decompressed data */
63#endif
65#ifdef HAVE_LIBZ
67#if !defined(ZLIB_VERNUM) || (ZLIB_VERNUM < 0x1252)
68#error "zlib 1.2.5.2 or greater required"
69#endif
71typedef enum {
72 ZLIB_UNINIT, /* uninitialized */
73 ZLIB_INIT, /* initialized */
74 ZLIB_INFLATING, /* inflating started. */
75 ZLIB_EXTERNAL_TRAILER, /* reading external trailer */
76 ZLIB_INIT_GZIP /* initialized in transparent gzip mode */
77} zlibInitState;
79/* Deflate and gzip writer. */
80struct zlib_writer {
81 struct Curl_cwriter super;
82 zlibInitState zlib_init; /* zlib init state */
83 char buffer[DECOMPRESS_BUFFER_SIZE]; /* Put the decompressed data here. */
84 uInt trailerlen; /* Remaining trailer byte count. */
85 z_stream z; /* State structure for zlib. */
86};
88static voidpf zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
90 (void)opaque;
91 /* not a typo, keep it curlx_calloc() */
92 return curlx_calloc(items, size);
95static void zfree_cb(voidpf opaque, voidpf ptr)
97 (void)opaque;
98 curlx_free(ptr);
101static CURLcode process_zlib_error(struct Curl_easy *data, z_stream *z)
103 if(z->msg)
104 failf(data, "Error while processing content unencoding: %s", z->msg);
105 else
106 failf(data, "Error while processing content unencoding: "
107 "Unknown failure within decompression software.");
109 return CURLE_BAD_CONTENT_ENCODING;
112static CURLcode exit_zlib(struct Curl_easy *data, z_stream *z,
113 zlibInitState *zlib_init, CURLcode result)
115 if(*zlib_init != ZLIB_UNINIT) {
116 if(inflateEnd(z) != Z_OK && result == CURLE_OK)
117 result = process_zlib_error(data, z);
118 *zlib_init = ZLIB_UNINIT;
119 }
121 return result;
124static CURLcode process_trailer(struct Curl_easy *data, struct zlib_writer *zp)
126 z_stream *z = &zp->z;
127 CURLcode result = CURLE_OK;
128 uInt len = z->avail_in < zp->trailerlen ? z->avail_in : zp->trailerlen;
130 /* Consume expected trailer bytes. Terminate stream if exhausted.
131 Issue an error if unexpected bytes follow. */
133 zp->trailerlen -= len;
134 z->avail_in -= len;
135 z->next_in += len;
136 if(z->avail_in)
137 result = CURLE_WRITE_ERROR;
138 if(result || !zp->trailerlen)
139 result = exit_zlib(data, z, &zp->zlib_init, result);
140 else {
141 /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
142 zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
143 }
144 return result;
147static CURLcode inflate_stream(struct Curl_easy *data,
148 struct Curl_cwriter *writer, int type,
149 zlibInitState started)
151 struct zlib_writer *zp = (struct zlib_writer *)writer;
152 z_stream *z = &zp->z; /* zlib state structure */
153 uInt nread = z->avail_in;
154 z_const Bytef *orig_in = z->next_in;
155 bool done = FALSE;
156 CURLcode result = CURLE_OK; /* Curl_client_write status */
157 int i = 0;
159 /* Check state. */
160 if(zp->zlib_init != ZLIB_INIT &&
161 zp->zlib_init != ZLIB_INFLATING &&
162 zp->zlib_init != ZLIB_INIT_GZIP)
163 return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
165 /* because the buffer size is fixed, iteratively decompress and transfer to
166 the client via next_write function. */
167 while(!done) {
168 int status; /* zlib status */
169 done = TRUE;
171 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
172 /* check every MB of output if we are not exceeding time limit */
173 i = 0;
174 if(Curl_timeleft_ms(data) < 0) {
175 failf(data, "Operation timed out while decoding payload");
176 return exit_zlib(data, z, &zp->zlib_init, CURLE_OPERATION_TIMEDOUT);
177 }
178 }
180 /* (re)set buffer for decompressed output for every iteration */
181 z->next_out = (Bytef *)zp->buffer;
182 z->avail_out = DECOMPRESS_BUFFER_SIZE;
184 status = inflate(z, Z_BLOCK);
186 /* Flush output data if some. */
187 if(z->avail_out != DECOMPRESS_BUFFER_SIZE) {
188 if(status == Z_OK || status == Z_STREAM_END) {
189 zp->zlib_init = started; /* Data started. */
190 result = Curl_cwriter_write(data, writer->next, type, zp->buffer,
191 DECOMPRESS_BUFFER_SIZE - z->avail_out);
192 if(result) {
193 exit_zlib(data, z, &zp->zlib_init, result);
194 break;
195 }
196 }
197 }
199 /* Dispatch by inflate() status. */
200 switch(status) {
201 case Z_OK:
202 /* Always loop: there may be unflushed latched data in zlib state. */
203 done = FALSE;
204 break;
205 case Z_BUF_ERROR:
206 /* No more data to flush: exit loop. */
207 break;
208 case Z_STREAM_END:
209 if((started == ZLIB_INIT_GZIP) && (z->avail_in >= 2) &&
210 (z->next_in[0] == 0x1f) && (z->next_in[1] == 0x8b)) {
211 /* a second gzip member follows; curl does not support
212 multi-member gzip responses */
213 failf(data, "Multi-member gzip response not supported");
214 result = exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
215 break;
216 }
217 result = process_trailer(data, zp);
218 break;
219 case Z_DATA_ERROR:
220 /* some servers seem to not generate zlib headers, so this is an attempt
221 to fix and continue anyway */
222 if(zp->zlib_init == ZLIB_INIT) {
223 if(inflateReset2(z, -MAX_WBITS) == Z_OK) {
224 z->next_in = orig_in;
225 z->avail_in = nread;
226 zp->zlib_init = ZLIB_INFLATING;
227 zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
228 done = FALSE;
229 break;
230 }
231 zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */
232 }
233 result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
234 break;
235 default:
236 result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
237 break;
238 }
239 }
241 /* We are about to leave this call so the `nread' data bytes will not be seen
242 again. If we are in a state that would wrongly allow restart in raw mode
243 at the next call, assume output has already started. */
244 if(nread && zp->zlib_init == ZLIB_INIT)
245 zp->zlib_init = started; /* Cannot restart anymore. */
247 return result;
250/* Deflate handler. */
251static CURLcode deflate_do_init(struct Curl_easy *data,
252 struct Curl_cwriter *writer)
254 struct zlib_writer *zp = (struct zlib_writer *)writer;
255 z_stream *z = &zp->z; /* zlib state structure */
257 /* Initialize zlib */
258 z->zalloc = (alloc_func)zalloc_cb;
259 z->zfree = (free_func)zfree_cb;
261 if(inflateInit(z) != Z_OK)
262 return process_zlib_error(data, z);
263 zp->zlib_init = ZLIB_INIT;
264 return CURLE_OK;
267static CURLcode deflate_do_write(struct Curl_easy *data,
268 struct Curl_cwriter *writer, int type,
269 const char *buf, size_t nbytes)
271 struct zlib_writer *zp = (struct zlib_writer *)writer;
272 z_stream *z = &zp->z; /* zlib state structure */
274 if(!(type & CLIENTWRITE_BODY) || !nbytes)
275 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
277 /* Set the compressed input when this function is called */
278 z->next_in = (z_const Bytef *)buf;
279 z->avail_in = (uInt)nbytes;
281 if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
282 return process_trailer(data, zp);
284 /* Now uncompress the data */
285 return inflate_stream(data, writer, type, ZLIB_INFLATING);
288static void deflate_do_close(struct Curl_easy *data,
289 struct Curl_cwriter *writer)
291 struct zlib_writer *zp = (struct zlib_writer *)writer;
292 z_stream *z = &zp->z; /* zlib state structure */
294 exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
297static const struct Curl_cwtype deflate_encoding = {
298 "deflate",
299 NULL,
300 CURL_CW_FLAG_BLOWUP,
301 deflate_do_init,
302 deflate_do_write,
303 Curl_cwriter_def_flush,
304 deflate_do_close,
305 sizeof(struct zlib_writer)
306};
308/*
309 * Gzip handler.
310 */
312static CURLcode gzip_do_init(struct Curl_easy *data,
313 struct Curl_cwriter *writer)
315 struct zlib_writer *zp = (struct zlib_writer *)writer;
316 z_stream *z = &zp->z; /* zlib state structure */
318 /* Initialize zlib */
319 z->zalloc = (alloc_func)zalloc_cb;
320 z->zfree = (free_func)zfree_cb;
322 if(inflateInit2(z, MAX_WBITS + 32) != Z_OK)
323 return process_zlib_error(data, z);
325 zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
326 return CURLE_OK;
329static CURLcode gzip_do_write(struct Curl_easy *data,
330 struct Curl_cwriter *writer, int type,
331 const char *buf, size_t nbytes)
333 struct zlib_writer *zp = (struct zlib_writer *)writer;
334 z_stream *z = &zp->z; /* zlib state structure */
336 if(!(type & CLIENTWRITE_BODY) || !nbytes)
337 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
339 if(zp->zlib_init == ZLIB_INIT_GZIP) {
340 /* Let zlib handle the gzip decompression entirely */
341 z->next_in = (z_const Bytef *)buf;
342 z->avail_in = (uInt)nbytes;
343 /* Now uncompress the data */
344 return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
345 }
347 /* We are running with an old version: return error. */
348 return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
351static void gzip_do_close(struct Curl_easy *data,
352 struct Curl_cwriter *writer)
354 struct zlib_writer *zp = (struct zlib_writer *)writer;
355 z_stream *z = &zp->z; /* zlib state structure */
357 exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
360static const struct Curl_cwtype gzip_encoding = {
361 "gzip",
362 "x-gzip",
363 CURL_CW_FLAG_BLOWUP,
364 gzip_do_init,
365 gzip_do_write,
366 Curl_cwriter_def_flush,
367 gzip_do_close,
368 sizeof(struct zlib_writer)
369};
371#endif /* HAVE_LIBZ */
373#ifdef HAVE_BROTLI
374/* Brotli writer. */
375struct brotli_writer {
376 struct Curl_cwriter super;
377 char buffer[DECOMPRESS_BUFFER_SIZE];
378 BrotliDecoderState *br; /* State structure for brotli. */
379};
381static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
383 switch(be) {
384 case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
385 case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
386 case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
387 case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
388 case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
389 case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
390 case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
391 case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
392 case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
393 case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
394 case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
395 case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
396 case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
397 case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
398#ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY /* brotli v1.1.0+ */
399 case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
400#endif
401 case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
402 case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
403 return CURLE_BAD_CONTENT_ENCODING;
404 case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
405 case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
406 case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
407 case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
408 case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
409 case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
410 return CURLE_OUT_OF_MEMORY;
411 default:
412 break;
413 }
414 return CURLE_WRITE_ERROR;
417static CURLcode brotli_do_init(struct Curl_easy *data,
418 struct Curl_cwriter *writer)
420 struct brotli_writer *bp = (struct brotli_writer *)writer;
421 (void)data;
423 bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
424 return bp->br ? CURLE_OK : CURLE_OUT_OF_MEMORY;
427static CURLcode brotli_do_write(struct Curl_easy *data,
428 struct Curl_cwriter *writer, int type,
429 const char *buf, size_t nbytes)
431 struct brotli_writer *bp = (struct brotli_writer *)writer;
432 const uint8_t *src = (const uint8_t *)buf;
433 uint8_t *dst;
434 size_t dstleft;
435 CURLcode result = CURLE_OK;
436 BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
437 int i = 0;
439 if(!(type & CLIENTWRITE_BODY) || !nbytes)
440 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
442 if(!bp->br)
443 return CURLE_WRITE_ERROR; /* Stream already ended. */
445 while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
446 result == CURLE_OK) {
448 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
449 /* check every MB of output if we are not exceeding time limit */
450 i = 0;
451 if(Curl_timeleft_ms(data) < 0) {
452 failf(data, "Operation timed out while decoding payload");
453 return CURLE_OPERATION_TIMEDOUT;
454 }
455 }
457 dst = (uint8_t *)bp->buffer;
458 dstleft = DECOMPRESS_BUFFER_SIZE;
459 r = BrotliDecoderDecompressStream(bp->br,
460 &nbytes, &src, &dstleft, &dst, NULL);
461 result = Curl_cwriter_write(data, writer->next, type,
462 bp->buffer, DECOMPRESS_BUFFER_SIZE - dstleft);
463 if(result)
464 break;
465 switch(r) {
466 case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
467 case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
468 break;
469 case BROTLI_DECODER_RESULT_SUCCESS:
470 BrotliDecoderDestroyInstance(bp->br);
471 bp->br = NULL;
472 if(nbytes)
473 result = CURLE_WRITE_ERROR;
474 break;
475 default:
476 result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
477 break;
478 }
479 }
480 return result;
483static CURLcode brotli_do_flush(struct Curl_easy *data,
484 struct Curl_cwriter *writer, bool eos)
486 struct brotli_writer *bp = (struct brotli_writer *)writer;
488 /* A decoder that consumed input must reach SUCCESS before the transfer
489 * ends. An unused decoder is fine for responses without a body. */
490 if(eos && bp->br && BrotliDecoderIsUsed(bp->br)) {
491 failf(data, "Brotli stream ended before completion");
492 return CURLE_BAD_CONTENT_ENCODING;
493 }
494 return Curl_cwriter_flush(data, writer->next, eos);
497static void brotli_do_close(struct Curl_easy *data,
498 struct Curl_cwriter *writer)
500 struct brotli_writer *bp = (struct brotli_writer *)writer;
501 (void)data;
503 if(bp->br) {
504 BrotliDecoderDestroyInstance(bp->br);
505 bp->br = NULL;
506 }
509static const struct Curl_cwtype brotli_encoding = {
510 "br",
511 NULL,
512 CURL_CW_FLAG_BLOWUP,
513 brotli_do_init,
514 brotli_do_write,
515 brotli_do_flush,
516 brotli_do_close,
517 sizeof(struct brotli_writer)
518};
519#endif
521#ifdef HAVE_ZSTD
522/* Zstd writer. */
523struct zstd_writer {
524 struct Curl_cwriter super;
525 ZSTD_DStream *zds; /* State structure for zstd. */
526 char buffer[DECOMPRESS_BUFFER_SIZE];
527};
529#ifdef ZSTD_STATIC_LINKING_ONLY
530static void *Curl_zstd_alloc(void *opaque, size_t size)
532 (void)opaque;
533 return Curl_cmalloc(size);
536static void Curl_zstd_free(void *opaque, void *address)
538 (void)opaque;
539 Curl_cfree(address);
541#endif
543static CURLcode zstd_do_init(struct Curl_easy *data,
544 struct Curl_cwriter *writer)
546 struct zstd_writer *zp = (struct zstd_writer *)writer;
548 (void)data;
550#ifdef ZSTD_STATIC_LINKING_ONLY
551 zp->zds = ZSTD_createDStream_advanced((ZSTD_customMem) {
552 .customAlloc = Curl_zstd_alloc,
553 .customFree = Curl_zstd_free,
554 .opaque = NULL
555 });
556#else
557 zp->zds = ZSTD_createDStream();
558#endif
560 return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
563static CURLcode zstd_do_write(struct Curl_easy *data,
564 struct Curl_cwriter *writer, int type,
565 const char *buf, size_t nbytes)
567 CURLcode result = CURLE_OK;
568 struct zstd_writer *zp = (struct zstd_writer *)writer;
569 ZSTD_inBuffer in;
570 ZSTD_outBuffer out;
571 size_t errorCode;
572 int i = 0;
574 if(!(type & CLIENTWRITE_BODY) || !nbytes)
575 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
577 in.pos = 0;
578 in.src = buf;
579 in.size = nbytes;
581 for(;;) {
582 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
583 /* check every MB of output if we are not exceeding time limit */
584 i = 0;
585 if(Curl_timeleft_ms(data) < 0) {
586 failf(data, "Operation timed out while decoding payload");
587 return CURLE_OPERATION_TIMEDOUT;
588 }
589 }
591 out.pos = 0;
592 out.dst = zp->buffer;
593 out.size = DECOMPRESS_BUFFER_SIZE;
595 errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
596 if(ZSTD_isError(errorCode)) {
597 return CURLE_BAD_CONTENT_ENCODING;
598 }
599 if(out.pos > 0) {
600 result = Curl_cwriter_write(data, writer->next, type,
601 zp->buffer, out.pos);
602 if(result)
603 break;
604 }
605 if((in.pos == nbytes) && (out.pos < out.size))
606 break;
607 }
609 return result;
612static void zstd_do_close(struct Curl_easy *data,
613 struct Curl_cwriter *writer)
615 struct zstd_writer *zp = (struct zstd_writer *)writer;
616 (void)data;
618 if(zp->zds) {
619 ZSTD_freeDStream(zp->zds);
620 zp->zds = NULL;
621 }
624static const struct Curl_cwtype zstd_encoding = {
625 "zstd",
626 NULL,
627 CURL_CW_FLAG_BLOWUP,
628 zstd_do_init,
629 zstd_do_write,
630 Curl_cwriter_def_flush,
631 zstd_do_close,
632 sizeof(struct zstd_writer)
633};
634#endif
636/* Identity handler. */
637static const struct Curl_cwtype identity_encoding = {
638 "identity",
639 "none",
640 0,
641 Curl_cwriter_def_init,
642 Curl_cwriter_def_write,
643 Curl_cwriter_def_flush,
644 Curl_cwriter_def_close,
645 sizeof(struct Curl_cwriter)
646};
648/* supported general content decoders. */
649static const struct Curl_cwtype * const general_unencoders[] = {
650 &identity_encoding,
651#ifdef HAVE_LIBZ
652 &deflate_encoding,
653 &gzip_encoding,
654#endif
655#ifdef HAVE_BROTLI
656 &brotli_encoding,
657#endif
658#ifdef HAVE_ZSTD
659 &zstd_encoding,
660#endif
661 NULL
662};
664/* supported content decoders only for transfer encodings */
665static const struct Curl_cwtype * const transfer_unencoders[] = {
666 &Curl_httpchunk_unencoder,
667 NULL
668};
670/* Return the list of comma-separated names of supported encodings.
671 */
672char *Curl_get_content_encodings(void)
674 struct dynbuf enc;
675 const struct Curl_cwtype * const *cep;
676 CURLcode result = CURLE_OK;
677 curlx_dyn_init(&enc, 255);
679 for(cep = general_unencoders; *cep && !result; cep++) {
680 const struct Curl_cwtype *ce = *cep;
681 if(!curl_strequal(ce->name, CONTENT_ENCODING_DEFAULT)) {
682 if(curlx_dyn_len(&enc))
683 result = curlx_dyn_addn(&enc, ", ", 2);
684 if(!result)
685 result = curlx_dyn_add(&enc, ce->name);
686 }
687 }
688 if(!result && !curlx_dyn_len(&enc))
689 result = curlx_dyn_add(&enc, CONTENT_ENCODING_DEFAULT);
691 if(!result)
692 return curlx_dyn_ptr(&enc);
693 return NULL;
696/* Deferred error dummy writer. */
697static CURLcode error_do_init(struct Curl_easy *data,
698 struct Curl_cwriter *writer)
700 (void)data;
701 (void)writer;
702 return CURLE_OK;
705static CURLcode error_do_write(struct Curl_easy *data,
706 struct Curl_cwriter *writer, int type,
707 const char *buf, size_t nbytes)
709 (void)writer;
710 (void)buf;
711 (void)nbytes;
713 if(!(type & CLIENTWRITE_BODY) || !nbytes)
714 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
715 failf(data, "Unrecognized content encoding type");
716 return CURLE_BAD_CONTENT_ENCODING;
719static void error_do_close(struct Curl_easy *data,
720 struct Curl_cwriter *writer)
722 (void)data;
723 (void)writer;
726static const struct Curl_cwtype error_writer = {
727 "ce-error",
728 NULL,
729 0,
730 error_do_init,
731 error_do_write,
732 Curl_cwriter_def_flush,
733 error_do_close,
734 sizeof(struct Curl_cwriter)
735};
737/* Find the content encoding by name. */
738static const struct Curl_cwtype *find_unencode_writer(const char *name,
739 size_t len,
740 Curl_cwriter_phase phase)
742 const struct Curl_cwtype * const *cep;
744 if(phase == CURL_CW_TRANSFER_DECODE) {
745 for(cep = transfer_unencoders; *cep; cep++) {
746 const struct Curl_cwtype *ce = *cep;
747 if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
748 (ce->alias && curl_strnequal(name, ce->alias, len) &&
749 !ce->alias[len]))
750 return ce;
751 }
752 }
753 /* look among the general decoders */
754 for(cep = general_unencoders; *cep; cep++) {
755 const struct Curl_cwtype *ce = *cep;
756 if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
757 (ce->alias && curl_strnequal(name, ce->alias, len) && !ce->alias[len]))
758 return ce;
759 }
760 return NULL;
763/* Setup the unencoding stack from the Content-Encoding header value.
764 * See RFC 7231 section 3.1.2.2. */
765CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
766 const char *enclist, int is_transfer)
768 Curl_cwriter_phase phase = is_transfer ?
769 CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
770 CURLcode result;
771 bool has_chunked = FALSE;
773 do {
774 const char *name;
775 size_t namelen;
776 bool is_chunked = FALSE;
778 /* Parse a single encoding name. */
779 while(ISBLANK(*enclist) || *enclist == ',')
780 enclist++;
782 name = enclist;
784 for(namelen = 0; *enclist && *enclist != ','; enclist++)
785 if(*enclist > ' ')
786 namelen = enclist - name + 1;
788 if(namelen) {
789 const struct Curl_cwtype *cwt;
790 struct Curl_cwriter *writer;
792 CURL_TRC_WRITE(data, "looking for %s decoder: %.*s",
793 is_transfer ? "transfer" : "content", (int)namelen, name);
794 is_chunked = (is_transfer && (namelen == 7) &&
795 curl_strnequal(name, "chunked", 7));
796 /* if we skip the decoding in this phase, do not look further.
797 * Exception is "chunked" transfer-encoding which always must happen */
798 if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
799 (!is_transfer && data->set.http_ce_skip)) {
800 bool is_identity = (namelen == 8) &&
801 curl_strnequal(name, "identity", 8);
802 /* not requested, ignore */
803 CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
804 (int)namelen, name);
805 if(is_transfer && !data->set.http_te_skip) {
806 if(has_chunked)
807 failf(data, "A Transfer-Encoding (%.*s) was listed after chunked",
808 (int)namelen, name);
809 else if(is_identity)
810 continue;
811 else
812 failf(data, "Unsolicited Transfer-Encoding (%.*s) found",
813 (int)namelen, name);
814 return CURLE_BAD_CONTENT_ENCODING;
815 }
816 return CURLE_OK;
817 }
819 if(Curl_cwriter_count(data, phase) >= MAX_ENCODE_STACK) {
820 failf(data, "Reject response exceeding limit of %d %s encodings",
821 MAX_ENCODE_STACK,
822 is_transfer ? "transfer" : "content");
823 return CURLE_BAD_CONTENT_ENCODING;
824 }
826 cwt = find_unencode_writer(name, namelen, phase);
827 if(is_transfer && !is_chunked &&
828 Curl_cwriter_get_by_name(data, "chunked")) {
829 /* RFC 9112, ch. 6.1:
830 * "If any transfer coding other than chunked is applied to a
831 * response's content, the sender MUST either apply chunked as the
832 * final transfer coding or terminate the message by closing the
833 * connection."
834 * "chunked" must be the last added to be the first in its phase,
835 * reject this.
836 */
837 failf(data, "Reject response due to 'chunked' not being the last "
838 "Transfer-Encoding");
839 return CURLE_BAD_CONTENT_ENCODING;
840 }
841 if(cwt && is_chunked && Curl_cwriter_get_by_type(data, cwt)) {
842 /* A 'chunked' transfer encoding has already been added.
843 * Ignore duplicates. See #13451.
844 * Also RFC 9112, ch. 6.1:
845 * "A sender MUST NOT apply the chunked transfer coding more than
846 * once to a message body."
847 */
848 CURL_TRC_WRITE(data, "ignoring duplicate 'chunked' decoder");
849 }
850 else {
851 if(!cwt)
852 cwt = &error_writer; /* Defer error at use. */
854 result = Curl_cwriter_create(&writer, data, cwt, phase);
855 CURL_TRC_WRITE(data, "added %s decoder %s -> %d",
856 is_transfer ? "transfer" : "content", cwt->name,
857 (int)result);
858 if(result)
859 return result;
861 result = Curl_cwriter_add(data, writer);
862 if(result) {
863 Curl_cwriter_free(data, writer);
864 return result;
865 }
866 }
867 if(is_chunked)
868 has_chunked = TRUE;
869 }
870 } while(*enclist);
872 return CURLE_OK;
875#else
876/* Stubs for builds without HTTP. */
877CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
878 const char *enclist, int is_transfer)
880 (void)data;
881 (void)enclist;
882 (void)is_transfer;
883 return CURLE_NOT_BUILT_IN;
886char *Curl_get_content_encodings(void)
888 return curlx_strdup(CONTENT_ENCODING_DEFAULT);
891#endif /* CURL_DISABLE_HTTP */
Diagnostics4 git calls · 350 ms
CallStatusSizeTime0350 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=149
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2007.3 KiB345 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=345
git.response.bytes=7477
git.response.cache=MISS
git.response.duration_ms=345
git.fetch.wants=ead180fa32b00b514f0545aa56b50ba50791d637
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=7477
git.pkt.payload.max=7456
git.sideband.band1.bytes=7455
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e1f4cdfe58efe9
Request
ab54773b…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format