master
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "curl_setup.h"
26#include "urldata.h"
27#include "curlx/dynbuf.h"
29#ifdef HAVE_LIBZ
30#include <zlib.h>
31#endif
33#ifdef HAVE_BROTLI
34#ifdef CURL_HAVE_DIAG
35/* Ignore -Wvla warnings in brotli headers */
36#pragma GCC diagnostic push
37#pragma GCC diagnostic ignored "-Wvla"
38#endif
39#include <brotli/decode.h>
40#ifdef CURL_HAVE_DIAG
41#pragma GCC diagnostic pop
42#endif
43#endif
45#ifdef HAVE_ZSTD
46#include <zstd.h>
47#endif
49#include "connect.h"
50#include "sendf.h"
51#include "curl_trc.h"
52#include "content_encoding.h"
54#define CONTENT_ENCODING_DEFAULT "identity"
56#ifndef CURL_DISABLE_HTTP
58/* allow no more than 5 "chained" compression steps */
59#define MAX_ENCODE_STACK 5
61#if defined(HAVE_LIBZ) || defined(HAVE_BROTLI) || defined(HAVE_ZSTD)
62#define DECOMPRESS_BUFFER_SIZE 16384 /* buffer size for decompressed data */
63#endif
65#ifdef HAVE_LIBZ
67#if !defined(ZLIB_VERNUM) || (ZLIB_VERNUM < 0x1252)
68#error "zlib 1.2.5.2 or greater required"
69#endif
71typedef enum {
72 ZLIB_UNINIT, /* uninitialized */
73 ZLIB_INIT, /* initialized */
74 ZLIB_INFLATING, /* inflating started. */
75 ZLIB_EXTERNAL_TRAILER, /* reading external trailer */
76 ZLIB_INIT_GZIP /* initialized in transparent gzip mode */
77} zlibInitState;
79/* Deflate and gzip writer. */
80struct zlib_writer {
81 struct Curl_cwriter super;
82 zlibInitState zlib_init; /* zlib init state */
83 char buffer[DECOMPRESS_BUFFER_SIZE]; /* Put the decompressed data here. */
84 uInt trailerlen; /* Remaining trailer byte count. */
85 z_stream z; /* State structure for zlib. */
86};
88static voidpf zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
89{
90 (void)opaque;
91 /* not a typo, keep it curlx_calloc() */
92 return curlx_calloc(items, size);
93}
95static void zfree_cb(voidpf opaque, voidpf ptr)
96{
97 (void)opaque;
98 curlx_free(ptr);
99}
101static CURLcode process_zlib_error(struct Curl_easy *data, z_stream *z)
102{
103 if(z->msg)
104 failf(data, "Error while processing content unencoding: %s", z->msg);
105 else
106 failf(data, "Error while processing content unencoding: "
107 "Unknown failure within decompression software.");
109 return CURLE_BAD_CONTENT_ENCODING;
110}
112static CURLcode exit_zlib(struct Curl_easy *data, z_stream *z,
113 zlibInitState *zlib_init, CURLcode result)
114{
115 if(*zlib_init != ZLIB_UNINIT) {
116 if(inflateEnd(z) != Z_OK && result == CURLE_OK)
117 result = process_zlib_error(data, z);
118 *zlib_init = ZLIB_UNINIT;
119 }
121 return result;
122}
124static CURLcode process_trailer(struct Curl_easy *data, struct zlib_writer *zp)
125{
126 z_stream *z = &zp->z;
127 CURLcode result = CURLE_OK;
128 uInt len = z->avail_in < zp->trailerlen ? z->avail_in : zp->trailerlen;
130 /* Consume expected trailer bytes. Terminate stream if exhausted.
131 Issue an error if unexpected bytes follow. */
133 zp->trailerlen -= len;
134 z->avail_in -= len;
135 z->next_in += len;
136 if(z->avail_in)
137 result = CURLE_WRITE_ERROR;
138 if(result || !zp->trailerlen)
139 result = exit_zlib(data, z, &zp->zlib_init, result);
140 else {
141 /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
142 zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
143 }
144 return result;
145}
147static CURLcode inflate_stream(struct Curl_easy *data,
148 struct Curl_cwriter *writer, int type,
149 zlibInitState started)
150{
151 struct zlib_writer *zp = (struct zlib_writer *)writer;
152 z_stream *z = &zp->z; /* zlib state structure */
153 uInt nread = z->avail_in;
154 z_const Bytef *orig_in = z->next_in;
155 bool done = FALSE;
156 CURLcode result = CURLE_OK; /* Curl_client_write status */
157 int i = 0;
159 /* Check state. */
160 if(zp->zlib_init != ZLIB_INIT &&
161 zp->zlib_init != ZLIB_INFLATING &&
162 zp->zlib_init != ZLIB_INIT_GZIP)
163 return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
165 /* because the buffer size is fixed, iteratively decompress and transfer to
166 the client via next_write function. */
167 while(!done) {
168 int status; /* zlib status */
169 done = TRUE;
171 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
172 /* check every MB of output if we are not exceeding time limit */
173 i = 0;
174 if(Curl_timeleft_ms(data) < 0) {
175 failf(data, "Operation timed out while decoding payload");
176 return exit_zlib(data, z, &zp->zlib_init, CURLE_OPERATION_TIMEDOUT);
177 }
178 }
180 /* (re)set buffer for decompressed output for every iteration */
181 z->next_out = (Bytef *)zp->buffer;
182 z->avail_out = DECOMPRESS_BUFFER_SIZE;
184 status = inflate(z, Z_BLOCK);
186 /* Flush output data if some. */
187 if(z->avail_out != DECOMPRESS_BUFFER_SIZE) {
188 if(status == Z_OK || status == Z_STREAM_END) {
189 zp->zlib_init = started; /* Data started. */
190 result = Curl_cwriter_write(data, writer->next, type, zp->buffer,
191 DECOMPRESS_BUFFER_SIZE - z->avail_out);
192 if(result) {
193 exit_zlib(data, z, &zp->zlib_init, result);
194 break;
195 }
196 }
197 }
199 /* Dispatch by inflate() status. */
200 switch(status) {
201 case Z_OK:
202 /* Always loop: there may be unflushed latched data in zlib state. */
203 done = FALSE;
204 break;
205 case Z_BUF_ERROR:
206 /* No more data to flush: exit loop. */
207 break;
208 case Z_STREAM_END:
209 if((started == ZLIB_INIT_GZIP) && (z->avail_in >= 2) &&
210 (z->next_in[0] == 0x1f) && (z->next_in[1] == 0x8b)) {
211 /* a second gzip member follows; curl does not support
212 multi-member gzip responses */
213 failf(data, "Multi-member gzip response not supported");
214 result = exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
215 break;
216 }
217 result = process_trailer(data, zp);
218 break;
219 case Z_DATA_ERROR:
220 /* some servers seem to not generate zlib headers, so this is an attempt
221 to fix and continue anyway */
222 if(zp->zlib_init == ZLIB_INIT) {
223 if(inflateReset2(z, -MAX_WBITS) == Z_OK) {
224 z->next_in = orig_in;
225 z->avail_in = nread;
226 zp->zlib_init = ZLIB_INFLATING;
227 zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
228 done = FALSE;
229 break;
230 }
231 zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */
232 }
233 result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
234 break;
235 default:
236 result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
237 break;
238 }
239 }
241 /* We are about to leave this call so the `nread' data bytes will not be seen
242 again. If we are in a state that would wrongly allow restart in raw mode
243 at the next call, assume output has already started. */
244 if(nread && zp->zlib_init == ZLIB_INIT)
245 zp->zlib_init = started; /* Cannot restart anymore. */
247 return result;
248}
250/* Deflate handler. */
251static CURLcode deflate_do_init(struct Curl_easy *data,
252 struct Curl_cwriter *writer)
253{
254 struct zlib_writer *zp = (struct zlib_writer *)writer;
255 z_stream *z = &zp->z; /* zlib state structure */
257 /* Initialize zlib */
258 z->zalloc = (alloc_func)zalloc_cb;
259 z->zfree = (free_func)zfree_cb;
261 if(inflateInit(z) != Z_OK)
262 return process_zlib_error(data, z);
263 zp->zlib_init = ZLIB_INIT;
264 return CURLE_OK;
265}
267static CURLcode deflate_do_write(struct Curl_easy *data,
268 struct Curl_cwriter *writer, int type,
269 const char *buf, size_t nbytes)
270{
271 struct zlib_writer *zp = (struct zlib_writer *)writer;
272 z_stream *z = &zp->z; /* zlib state structure */
274 if(!(type & CLIENTWRITE_BODY) || !nbytes)
275 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
277 /* Set the compressed input when this function is called */
278 z->next_in = (z_const Bytef *)buf;
279 z->avail_in = (uInt)nbytes;
281 if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
282 return process_trailer(data, zp);
284 /* Now uncompress the data */
285 return inflate_stream(data, writer, type, ZLIB_INFLATING);
286}
288static void deflate_do_close(struct Curl_easy *data,
289 struct Curl_cwriter *writer)
290{
291 struct zlib_writer *zp = (struct zlib_writer *)writer;
292 z_stream *z = &zp->z; /* zlib state structure */
294 exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
295}
297static const struct Curl_cwtype deflate_encoding = {
298 "deflate",
299 NULL,
300 CURL_CW_FLAG_BLOWUP,
301 deflate_do_init,
302 deflate_do_write,
303 Curl_cwriter_def_flush,
304 deflate_do_close,
305 sizeof(struct zlib_writer)
306};
308/*
309 * Gzip handler.
310 */
312static CURLcode gzip_do_init(struct Curl_easy *data,
313 struct Curl_cwriter *writer)
314{
315 struct zlib_writer *zp = (struct zlib_writer *)writer;
316 z_stream *z = &zp->z; /* zlib state structure */
318 /* Initialize zlib */
319 z->zalloc = (alloc_func)zalloc_cb;
320 z->zfree = (free_func)zfree_cb;
322 if(inflateInit2(z, MAX_WBITS + 32) != Z_OK)
323 return process_zlib_error(data, z);
325 zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
326 return CURLE_OK;
327}
329static CURLcode gzip_do_write(struct Curl_easy *data,
330 struct Curl_cwriter *writer, int type,
331 const char *buf, size_t nbytes)
332{
333 struct zlib_writer *zp = (struct zlib_writer *)writer;
334 z_stream *z = &zp->z; /* zlib state structure */
336 if(!(type & CLIENTWRITE_BODY) || !nbytes)
337 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
339 if(zp->zlib_init == ZLIB_INIT_GZIP) {
340 /* Let zlib handle the gzip decompression entirely */
341 z->next_in = (z_const Bytef *)buf;
342 z->avail_in = (uInt)nbytes;
343 /* Now uncompress the data */
344 return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
345 }
347 /* We are running with an old version: return error. */
348 return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
349}
351static void gzip_do_close(struct Curl_easy *data,
352 struct Curl_cwriter *writer)
353{
354 struct zlib_writer *zp = (struct zlib_writer *)writer;
355 z_stream *z = &zp->z; /* zlib state structure */
357 exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
358}
360static const struct Curl_cwtype gzip_encoding = {
361 "gzip",
362 "x-gzip",
363 CURL_CW_FLAG_BLOWUP,
364 gzip_do_init,
365 gzip_do_write,
366 Curl_cwriter_def_flush,
367 gzip_do_close,
368 sizeof(struct zlib_writer)
369};
371#endif /* HAVE_LIBZ */
373#ifdef HAVE_BROTLI
374/* Brotli writer. */
375struct brotli_writer {
376 struct Curl_cwriter super;
377 char buffer[DECOMPRESS_BUFFER_SIZE];
378 BrotliDecoderState *br; /* State structure for brotli. */
379};
381static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
382{
383 switch(be) {
384 case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
385 case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
386 case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
387 case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
388 case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
389 case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
390 case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
391 case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
392 case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
393 case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
394 case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
395 case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
396 case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
397 case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
398#ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY /* brotli v1.1.0+ */
399 case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
400#endif
401 case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
402 case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
403 return CURLE_BAD_CONTENT_ENCODING;
404 case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
405 case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
406 case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
407 case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
408 case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
409 case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
410 return CURLE_OUT_OF_MEMORY;
411 default:
412 break;
413 }
414 return CURLE_WRITE_ERROR;
415}
417static CURLcode brotli_do_init(struct Curl_easy *data,
418 struct Curl_cwriter *writer)
419{
420 struct brotli_writer *bp = (struct brotli_writer *)writer;
421 (void)data;
423 bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
424 return bp->br ? CURLE_OK : CURLE_OUT_OF_MEMORY;
425}
427static CURLcode brotli_do_write(struct Curl_easy *data,
428 struct Curl_cwriter *writer, int type,
429 const char *buf, size_t nbytes)
430{
431 struct brotli_writer *bp = (struct brotli_writer *)writer;
432 const uint8_t *src = (const uint8_t *)buf;
433 uint8_t *dst;
434 size_t dstleft;
435 CURLcode result = CURLE_OK;
436 BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
437 int i = 0;
439 if(!(type & CLIENTWRITE_BODY) || !nbytes)
440 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
442 if(!bp->br)
443 return CURLE_WRITE_ERROR; /* Stream already ended. */
445 while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
446 result == CURLE_OK) {
448 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
449 /* check every MB of output if we are not exceeding time limit */
450 i = 0;
451 if(Curl_timeleft_ms(data) < 0) {
452 failf(data, "Operation timed out while decoding payload");
453 return CURLE_OPERATION_TIMEDOUT;
454 }
455 }
457 dst = (uint8_t *)bp->buffer;
458 dstleft = DECOMPRESS_BUFFER_SIZE;
459 r = BrotliDecoderDecompressStream(bp->br,
460 &nbytes, &src, &dstleft, &dst, NULL);
461 result = Curl_cwriter_write(data, writer->next, type,
462 bp->buffer, DECOMPRESS_BUFFER_SIZE - dstleft);
463 if(result)
464 break;
465 switch(r) {
466 case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
467 case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
468 break;
469 case BROTLI_DECODER_RESULT_SUCCESS:
470 BrotliDecoderDestroyInstance(bp->br);
471 bp->br = NULL;
472 if(nbytes)
473 result = CURLE_WRITE_ERROR;
474 break;
475 default:
476 result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
477 break;
478 }
479 }
480 return result;
481}
483static CURLcode brotli_do_flush(struct Curl_easy *data,
484 struct Curl_cwriter *writer, bool eos)
485{
486 struct brotli_writer *bp = (struct brotli_writer *)writer;
488 /* A decoder that consumed input must reach SUCCESS before the transfer
489 * ends. An unused decoder is fine for responses without a body. */
490 if(eos && bp->br && BrotliDecoderIsUsed(bp->br)) {
491 failf(data, "Brotli stream ended before completion");
492 return CURLE_BAD_CONTENT_ENCODING;
493 }
494 return Curl_cwriter_flush(data, writer->next, eos);
495}
497static void brotli_do_close(struct Curl_easy *data,
498 struct Curl_cwriter *writer)
499{
500 struct brotli_writer *bp = (struct brotli_writer *)writer;
501 (void)data;
503 if(bp->br) {
504 BrotliDecoderDestroyInstance(bp->br);
505 bp->br = NULL;
506 }
507}
509static const struct Curl_cwtype brotli_encoding = {
510 "br",
511 NULL,
512 CURL_CW_FLAG_BLOWUP,
513 brotli_do_init,
514 brotli_do_write,
515 brotli_do_flush,
516 brotli_do_close,
517 sizeof(struct brotli_writer)
518};
519#endif
521#ifdef HAVE_ZSTD
522/* Zstd writer. */
523struct zstd_writer {
524 struct Curl_cwriter super;
525 ZSTD_DStream *zds; /* State structure for zstd. */
526 char buffer[DECOMPRESS_BUFFER_SIZE];
527};
529#ifdef ZSTD_STATIC_LINKING_ONLY
530static void *Curl_zstd_alloc(void *opaque, size_t size)
531{
532 (void)opaque;
533 return Curl_cmalloc(size);
534}
536static void Curl_zstd_free(void *opaque, void *address)
537{
538 (void)opaque;
539 Curl_cfree(address);
540}
541#endif
543static CURLcode zstd_do_init(struct Curl_easy *data,
544 struct Curl_cwriter *writer)
545{
546 struct zstd_writer *zp = (struct zstd_writer *)writer;
548 (void)data;
550#ifdef ZSTD_STATIC_LINKING_ONLY
551 zp->zds = ZSTD_createDStream_advanced((ZSTD_customMem) {
552 .customAlloc = Curl_zstd_alloc,
553 .customFree = Curl_zstd_free,
554 .opaque = NULL
555 });
556#else
557 zp->zds = ZSTD_createDStream();
558#endif
560 return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
561}
563static CURLcode zstd_do_write(struct Curl_easy *data,
564 struct Curl_cwriter *writer, int type,
565 const char *buf, size_t nbytes)
566{
567 CURLcode result = CURLE_OK;
568 struct zstd_writer *zp = (struct zstd_writer *)writer;
569 ZSTD_inBuffer in;
570 ZSTD_outBuffer out;
571 size_t errorCode;
572 int i = 0;
574 if(!(type & CLIENTWRITE_BODY) || !nbytes)
575 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
577 in.pos = 0;
578 in.src = buf;
579 in.size = nbytes;
581 for(;;) {
582 if(++i > (1024 * 1024 / DECOMPRESS_BUFFER_SIZE)) {
583 /* check every MB of output if we are not exceeding time limit */
584 i = 0;
585 if(Curl_timeleft_ms(data) < 0) {
586 failf(data, "Operation timed out while decoding payload");
587 return CURLE_OPERATION_TIMEDOUT;
588 }
589 }
591 out.pos = 0;
592 out.dst = zp->buffer;
593 out.size = DECOMPRESS_BUFFER_SIZE;
595 errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
596 if(ZSTD_isError(errorCode)) {
597 return CURLE_BAD_CONTENT_ENCODING;
598 }
599 if(out.pos > 0) {
600 result = Curl_cwriter_write(data, writer->next, type,
601 zp->buffer, out.pos);
602 if(result)
603 break;
604 }
605 if((in.pos == nbytes) && (out.pos < out.size))
606 break;
607 }
609 return result;
610}
612static void zstd_do_close(struct Curl_easy *data,
613 struct Curl_cwriter *writer)
614{
615 struct zstd_writer *zp = (struct zstd_writer *)writer;
616 (void)data;
618 if(zp->zds) {
619 ZSTD_freeDStream(zp->zds);
620 zp->zds = NULL;
621 }
622}
624static const struct Curl_cwtype zstd_encoding = {
625 "zstd",
626 NULL,
627 CURL_CW_FLAG_BLOWUP,
628 zstd_do_init,
629 zstd_do_write,
630 Curl_cwriter_def_flush,
631 zstd_do_close,
632 sizeof(struct zstd_writer)
633};
634#endif
636/* Identity handler. */
637static const struct Curl_cwtype identity_encoding = {
638 "identity",
639 "none",
640 0,
641 Curl_cwriter_def_init,
642 Curl_cwriter_def_write,
643 Curl_cwriter_def_flush,
644 Curl_cwriter_def_close,
645 sizeof(struct Curl_cwriter)
646};
648/* supported general content decoders. */
649static const struct Curl_cwtype * const general_unencoders[] = {
650 &identity_encoding,
651#ifdef HAVE_LIBZ
652 &deflate_encoding,
653 &gzip_encoding,
654#endif
655#ifdef HAVE_BROTLI
656 &brotli_encoding,
657#endif
658#ifdef HAVE_ZSTD
659 &zstd_encoding,
660#endif
661 NULL
662};
664/* supported content decoders only for transfer encodings */
665static const struct Curl_cwtype * const transfer_unencoders[] = {
666 &Curl_httpchunk_unencoder,
667 NULL
668};
670/* Return the list of comma-separated names of supported encodings.
671 */
672char *Curl_get_content_encodings(void)
673{
674 struct dynbuf enc;
675 const struct Curl_cwtype * const *cep;
676 CURLcode result = CURLE_OK;
677 curlx_dyn_init(&enc, 255);
679 for(cep = general_unencoders; *cep && !result; cep++) {
680 const struct Curl_cwtype *ce = *cep;
681 if(!curl_strequal(ce->name, CONTENT_ENCODING_DEFAULT)) {
682 if(curlx_dyn_len(&enc))
683 result = curlx_dyn_addn(&enc, ", ", 2);
684 if(!result)
685 result = curlx_dyn_add(&enc, ce->name);
686 }
687 }
688 if(!result && !curlx_dyn_len(&enc))
689 result = curlx_dyn_add(&enc, CONTENT_ENCODING_DEFAULT);
691 if(!result)
692 return curlx_dyn_ptr(&enc);
693 return NULL;
694}
696/* Deferred error dummy writer. */
697static CURLcode error_do_init(struct Curl_easy *data,
698 struct Curl_cwriter *writer)
699{
700 (void)data;
701 (void)writer;
702 return CURLE_OK;
703}
705static CURLcode error_do_write(struct Curl_easy *data,
706 struct Curl_cwriter *writer, int type,
707 const char *buf, size_t nbytes)
708{
709 (void)writer;
710 (void)buf;
711 (void)nbytes;
713 if(!(type & CLIENTWRITE_BODY) || !nbytes)
714 return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
715 failf(data, "Unrecognized content encoding type");
716 return CURLE_BAD_CONTENT_ENCODING;
717}
719static void error_do_close(struct Curl_easy *data,
720 struct Curl_cwriter *writer)
721{
722 (void)data;
723 (void)writer;
724}
726static const struct Curl_cwtype error_writer = {
727 "ce-error",
728 NULL,
729 0,
730 error_do_init,
731 error_do_write,
732 Curl_cwriter_def_flush,
733 error_do_close,
734 sizeof(struct Curl_cwriter)
735};
737/* Find the content encoding by name. */
738static const struct Curl_cwtype *find_unencode_writer(const char *name,
739 size_t len,
740 Curl_cwriter_phase phase)
741{
742 const struct Curl_cwtype * const *cep;
744 if(phase == CURL_CW_TRANSFER_DECODE) {
745 for(cep = transfer_unencoders; *cep; cep++) {
746 const struct Curl_cwtype *ce = *cep;
747 if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
748 (ce->alias && curl_strnequal(name, ce->alias, len) &&
749 !ce->alias[len]))
750 return ce;
751 }
752 }
753 /* look among the general decoders */
754 for(cep = general_unencoders; *cep; cep++) {
755 const struct Curl_cwtype *ce = *cep;
756 if((curl_strnequal(name, ce->name, len) && !ce->name[len]) ||
757 (ce->alias && curl_strnequal(name, ce->alias, len) && !ce->alias[len]))
758 return ce;
759 }
760 return NULL;
761}
763/* Setup the unencoding stack from the Content-Encoding header value.
764 * See RFC 7231 section 3.1.2.2. */
765CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
766 const char *enclist, int is_transfer)
767{
768 Curl_cwriter_phase phase = is_transfer ?
769 CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
770 CURLcode result;
771 bool has_chunked = FALSE;
773 do {
774 const char *name;
775 size_t namelen;
776 bool is_chunked = FALSE;
778 /* Parse a single encoding name. */
779 while(ISBLANK(*enclist) || *enclist == ',')
780 enclist++;
782 name = enclist;
784 for(namelen = 0; *enclist && *enclist != ','; enclist++)
785 if(*enclist > ' ')
786 namelen = enclist - name + 1;
788 if(namelen) {
789 const struct Curl_cwtype *cwt;
790 struct Curl_cwriter *writer;
792 CURL_TRC_WRITE(data, "looking for %s decoder: %.*s",
793 is_transfer ? "transfer" : "content", (int)namelen, name);
794 is_chunked = (is_transfer && (namelen == 7) &&
795 curl_strnequal(name, "chunked", 7));
796 /* if we skip the decoding in this phase, do not look further.
797 * Exception is "chunked" transfer-encoding which always must happen */
798 if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
799 (!is_transfer && data->set.http_ce_skip)) {
800 bool is_identity = (namelen == 8) &&
801 curl_strnequal(name, "identity", 8);
802 /* not requested, ignore */
803 CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
804 (int)namelen, name);
805 if(is_transfer && !data->set.http_te_skip) {
806 if(has_chunked)
807 failf(data, "A Transfer-Encoding (%.*s) was listed after chunked",
808 (int)namelen, name);
809 else if(is_identity)
810 continue;
811 else
812 failf(data, "Unsolicited Transfer-Encoding (%.*s) found",
813 (int)namelen, name);
814 return CURLE_BAD_CONTENT_ENCODING;
815 }
816 return CURLE_OK;
817 }
819 if(Curl_cwriter_count(data, phase) >= MAX_ENCODE_STACK) {
820 failf(data, "Reject response exceeding limit of %d %s encodings",
821 MAX_ENCODE_STACK,
822 is_transfer ? "transfer" : "content");
823 return CURLE_BAD_CONTENT_ENCODING;
824 }
826 cwt = find_unencode_writer(name, namelen, phase);
827 if(is_transfer && !is_chunked &&
828 Curl_cwriter_get_by_name(data, "chunked")) {
829 /* RFC 9112, ch. 6.1:
830 * "If any transfer coding other than chunked is applied to a
831 * response's content, the sender MUST either apply chunked as the
832 * final transfer coding or terminate the message by closing the
833 * connection."
834 * "chunked" must be the last added to be the first in its phase,
835 * reject this.
836 */
837 failf(data, "Reject response due to 'chunked' not being the last "
838 "Transfer-Encoding");
839 return CURLE_BAD_CONTENT_ENCODING;
840 }
841 if(cwt && is_chunked && Curl_cwriter_get_by_type(data, cwt)) {
842 /* A 'chunked' transfer encoding has already been added.
843 * Ignore duplicates. See #13451.
844 * Also RFC 9112, ch. 6.1:
845 * "A sender MUST NOT apply the chunked transfer coding more than
846 * once to a message body."
847 */
848 CURL_TRC_WRITE(data, "ignoring duplicate 'chunked' decoder");
849 }
850 else {
851 if(!cwt)
852 cwt = &error_writer; /* Defer error at use. */
854 result = Curl_cwriter_create(&writer, data, cwt, phase);
855 CURL_TRC_WRITE(data, "added %s decoder %s -> %d",
856 is_transfer ? "transfer" : "content", cwt->name,
857 (int)result);
858 if(result)
859 return result;
861 result = Curl_cwriter_add(data, writer);
862 if(result) {
863 Curl_cwriter_free(data, writer);
864 return result;
865 }
866 }
867 if(is_chunked)
868 has_chunked = TRUE;
869 }
870 } while(*enclist);
872 return CURLE_OK;
873}
875#else
876/* Stubs for builds without HTTP. */
877CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
878 const char *enclist, int is_transfer)
879{
880 (void)data;
881 (void)enclist;
882 (void)is_transfer;
883 return CURLE_NOT_BUILT_IN;
884}
886char *Curl_get_content_encodings(void)
887{
888 return curlx_strdup(CONTENT_ENCODING_DEFAULT);
889}
891#endif /* CURL_DISABLE_HTTP */
Diagnostics4 git calls · 350 ms
CallStatusSizeTime0350 ms
discover200152 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=149 git.response.cache=UPDATING git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2007.3 KiB345 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=345 git.response.bytes=7477 git.response.cache=MISS git.response.duration_ms=345 git.fetch.wants=ead180fa32b00b514f0545aa56b50ba50791d637 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=7477 git.pkt.payload.max=7456 git.sideband.band1.bytes=7455 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e1f4cdfe58efe9
- Request
- ab54773b…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format