Artifactspublic-v0

curl

master
ply4 git calls29 ms298 KiB3 cachedprotocol v2
1270 lines39 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "curl_setup.h"
26#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_AWS)
28#include "urldata.h"
29#include "strcase.h"
30#include "curlx/strdup.h"
31#include "http_aws_sigv4.h"
32#include "curl_sha256.h"
33#include "transfer.h"
34#include "curl_trc.h"
35#include "escape.h"
36#include "curlx/strparse.h"
37#include "slist.h"
39#include <time.h>
41#define HMAC_SHA256(k, kl, d, dl, o) \
42 do { \
43 result = Curl_hmacit(&Curl_HMAC_SHA256, \
44 (const unsigned char *)(k), \
45 kl, \
46 (const unsigned char *)(d), \
47 dl, o); \
48 if(result) { \
49 goto fail; \
50 } \
51 } while(0)
53#define TIMESTAMP_SIZE 17
55/* hex-encoded with null-terminator */
56#define SHA256_HEX_LENGTH ((2 * CURL_SHA256_DIGEST_LENGTH) + 1)
58#define MAX_QUERY_COMPONENTS 128
60struct pair {
61 struct dynbuf key;
62 struct dynbuf value;
63};
65static void sha256_to_hex(char *dst, unsigned char *sha)
67 Curl_hexencode(sha, CURL_SHA256_DIGEST_LENGTH,
68 (unsigned char *)dst, SHA256_HEX_LENGTH);
71static char *find_date_hdr(struct Curl_easy *data, const char *sig_hdr)
73 char *tmp = Curl_checkheaders(data, sig_hdr, strlen(sig_hdr));
75 if(tmp)
76 return tmp;
77 return Curl_checkheaders(data, STRCONST("Date"));
80/* remove whitespace, and lowercase all headers */
81static void trim_headers(struct curl_slist *head)
83 struct curl_slist *l;
84 for(l = head; l; l = l->next) {
85 const char *value; /* to read from */
86 char *store;
87 size_t colon = strcspn(l->data, ":");
88 Curl_strntolower(l->data, l->data, colon);
90 value = &l->data[colon];
91 if(!*value)
92 continue;
93 ++value;
94 store = (char *)CURL_UNCONST(value);
96 /* skip leading whitespace */
97 curlx_str_passblanks(&value);
99 while(*value) {
100 int space = 0;
101 while(ISBLANK(*value)) {
102 value++;
103 space++;
104 }
105 if(space) {
106 /* replace any number of consecutive whitespace with a single space,
107 unless at the end of the string, then nothing */
108 if(*value)
109 *store++ = ' ';
110 }
111 else
112 *store++ = *value++;
113 }
114 *store = 0; /* null-terminate */
115 }
118/*
119 * Frees all allocated strings in a dynbuf pair array, and the dynbuf itself
120 */
121static void pair_array_free(struct pair *pair_array, size_t num_elements)
123 size_t index;
125 for(index = 0; index != num_elements; index++) {
126 curlx_dyn_free(&pair_array[index].key);
127 curlx_dyn_free(&pair_array[index].value);
128 }
131/*
132 * Frees all allocated strings in a split dynbuf, and the dynbuf itself
133 */
134static void dyn_array_free(struct dynbuf *db, size_t num_elements)
136 size_t index;
138 for(index = 0; index < num_elements; index++)
139 curlx_dyn_free((&db[index]));
142/*
143 * Splits source string by SPLIT_BY, and creates an array of dynbuf in db.
144 * db is initialized by this function.
145 * Caller is responsible for freeing the array elements with dyn_array_free
146 */
148#define SPLIT_BY '&'
150static CURLcode split_to_dyn_array(const char *source,
151 struct dynbuf db[MAX_QUERY_COMPONENTS],
152 size_t *num_splits_out)
154 CURLcode result = CURLE_OK;
155 size_t len = strlen(source);
156 size_t pos; /* Position in result buffer */
157 size_t start = 0; /* Start of current segment */
158 size_t segment_length = 0;
159 size_t index = 0;
160 size_t num_splits = 0;
162 /* Split source_ptr on SPLIT_BY and store the segment offsets and length in
163 * array */
164 for(pos = 0; pos < len; pos++) {
165 if(source[pos] == SPLIT_BY) {
166 if(segment_length) {
167 curlx_dyn_init(&db[index], segment_length + 1);
168 result = curlx_dyn_addn(&db[index], &source[start], segment_length);
169 if(result)
170 goto fail;
172 segment_length = 0;
173 index++;
174 if(++num_splits == MAX_QUERY_COMPONENTS) {
175 result = CURLE_TOO_LARGE;
176 goto fail;
177 }
178 }
179 start = pos + 1;
180 }
181 else {
182 segment_length++;
183 }
184 }
186 if(segment_length) {
187 curlx_dyn_init(&db[index], segment_length + 1);
188 result = curlx_dyn_addn(&db[index], &source[start], segment_length);
189 if(!result && ++num_splits == MAX_QUERY_COMPONENTS)
190 result = CURLE_TOO_LARGE;
191 }
192fail:
193 *num_splits_out = num_splits;
194 return result;
197static bool is_reserved_char(const char c)
199 return (ISALNUM(c) || ISURLPUNTCS(c));
202static CURLcode uri_encode_path(struct Curl_str *original_path,
203 struct dynbuf *new_path)
205 const char *p = curlx_str(original_path);
206 size_t i;
208 for(i = 0; i < curlx_strlen(original_path); i++) {
209 /* Do not encode slashes or unreserved chars from RFC 3986 */
210 CURLcode result = CURLE_OK;
211 unsigned char c = p[i];
212 if(is_reserved_char(c) || c == '/')
213 result = curlx_dyn_addn(new_path, &c, 1);
214 else
215 result = curlx_dyn_addf(new_path, "%%%02X", c);
216 if(result)
217 return result;
218 }
220 return CURLE_OK;
223/* Normalize the query part. Make sure %2B is left percent encoded, and not
224 decoded to plus, then encoded to space. */
225static CURLcode normalize_query(const char *string, size_t len,
226 struct dynbuf *db)
228 CURLcode result = CURLE_OK;
230 while(len && !result) {
231 unsigned char in = (unsigned char)*string;
232 if(('%' == in) && (len > 2) &&
233 ISXDIGIT(string[1]) && ISXDIGIT(string[2])) {
234 /* this is two hexadecimal digits following a '%' */
235 in = (unsigned char)((curlx_hexval(string[1]) << 4) |
236 curlx_hexval(string[2]));
237 string += 3;
238 len -= 3;
239 if(in == '+') {
240 /* decodes to plus, so leave this encoded */
241 result = curlx_dyn_addn(db, "%2B", 3);
242 continue;
243 }
244 }
245 else {
246 string++;
247 len--;
248 }
250 if(is_reserved_char(in))
251 /* Escape unreserved chars from RFC 3986 */
252 result = curlx_dyn_addn(db, &in, 1);
253 else if(in == '+')
254 /* Encode '+' as space */
255 result = curlx_dyn_add(db, "%20");
256 else
257 result = curlx_dyn_addf(db, "%%%02X", in);
258 }
260 return result;
263static bool should_urlencode(struct Curl_str *service_name)
265 /*
266 * These services require unmodified (not additionally URL-encoded) URL
267 * paths. They are the S3 signing names, which have disableDoubleEncoding
268 * set in the AWS S3 endpoint rules. Urls are already normalized by the curl
269 * URL parser
270 */
271 if(curlx_str_cmp(service_name, "s3") ||
272 curlx_str_cmp(service_name, "s3express") ||
273 curlx_str_cmp(service_name, "s3-outposts") ||
274 curlx_str_cmp(service_name, "s3-object-lambda")) {
275 return FALSE;
276 }
277 return TRUE;
280/* maximum length for the aws sivg4 parts */
281#define MAX_SIGV4_LEN 64
282#define DATE_HDR_KEY_LEN (MAX_SIGV4_LEN + sizeof("X--Date"))
284/* string been x-PROVIDER-date:TIMESTAMP, I need +1 for ':' */
285#define DATE_FULL_HDR_LEN (DATE_HDR_KEY_LEN + TIMESTAMP_SIZE + 1)
287/* alphabetically compare two headers by their name, expecting
288 headers to use ':' at this point */
289static int compare_header_names(const char *a, const char *b)
291 const char *colon_a;
292 const char *colon_b;
293 size_t len_a;
294 size_t len_b;
295 size_t min_len;
296 int cmp;
298 colon_a = strchr(a, ':');
299 colon_b = strchr(b, ':');
301 DEBUGASSERT(colon_a);
302 DEBUGASSERT(colon_b);
304 len_a = colon_a ? (size_t)(colon_a - a) : strlen(a);
305 len_b = colon_b ? (size_t)(colon_b - b) : strlen(b);
307 min_len = (len_a < len_b) ? len_a : len_b;
309 cmp = strncmp(a, b, min_len);
311 /* return the shorter of the two if one is shorter */
312 if(!cmp)
313 return (int)(len_a - len_b);
315 return cmp;
318/* Entry used when sorting headers; carries the original index to keep equal
319 names at their insertion order. */
320struct sort_hdr {
321 int idx;
322 char *data;
323};
325/* qsort comparator: sort by header name, break ties by original index */
326static int qsort_compare_header_names(const void *a, const void *b)
328 const struct sort_hdr *ha = (const struct sort_hdr *)a;
329 const struct sort_hdr *hb = (const struct sort_hdr *)b;
330 int r = compare_header_names(ha->data, hb->data);
331 if(r)
332 return r;
333 /* preserve insertion order for duplicate names (stable sort) */
334 return ha->idx - hb->idx;
337/* Merge duplicate header definitions by comma delimiting their values
338 in the order defined the headers are defined, expecting headers to
339 be alpha-sorted and use ':' at this point */
340static CURLcode merge_duplicate_headers(struct curl_slist *head)
342 struct curl_slist *curr = head;
343 CURLcode result = CURLE_OK;
345 while(curr) {
346 struct curl_slist *next = curr->next;
347 if(!next)
348 break;
350 if(compare_header_names(curr->data, next->data) == 0) {
351 struct dynbuf buf;
352 const char *colon_next;
353 const char *val_next;
355 curlx_dyn_init(&buf, CURL_MAX_HTTP_HEADER);
357 result = curlx_dyn_add(&buf, curr->data);
358 if(result)
359 return result;
361 colon_next = strchr(next->data, ':');
362 DEBUGASSERT(colon_next);
363 val_next = colon_next + 1;
365 result = curlx_dyn_addn(&buf, ",", 1);
366 if(result)
367 return result;
369 result = curlx_dyn_add(&buf, val_next);
370 if(result)
371 return result;
373 curlx_free(curr->data);
374 curr->data = curlx_dyn_ptr(&buf);
376 curr->next = next->next;
377 curlx_free(next->data);
378 curlx_free(next);
379 }
380 else {
381 curr = curr->next;
382 }
383 }
385 return CURLE_OK;
388/* alpha-sort by header name in a case sensitive manner */
389#define SORT_HEADERS_STACK_MAX 32
391static CURLcode sort_headers(struct curl_slist *head)
393 size_t n = 0;
394 struct sort_hdr *arr;
395 struct sort_hdr stackbuf[SORT_HEADERS_STACK_MAX];
396 struct curl_slist *l;
397 int i;
398 for(l = head; l; l = l->next)
399 ++n;
400 if(n > 1) {
401 if(n <= SORT_HEADERS_STACK_MAX)
402 arr = stackbuf;
403 else {
404 arr = curlx_malloc(n * sizeof(*arr));
405 if(!arr)
406 return CURLE_OUT_OF_MEMORY;
407 }
408 for(i = 0, l = head; l; l = l->next, ++i) {
409 arr[i].idx = i;
410 arr[i].data = l->data;
411 }
412 qsort(arr, n, sizeof(*arr), qsort_compare_header_names);
413 for(i = 0, l = head; l; l = l->next, ++i)
414 l->data = arr[i].data;
415 if(arr != stackbuf)
416 curlx_free(arr);
417 }
418 return CURLE_OK;
421/* timestamp should point to a buffer of at last TIMESTAMP_SIZE bytes */
422static CURLcode make_headers(struct Curl_easy *data,
423 char *timestamp,
424 const char *provider1,
425 size_t plen, /* length of provider1 */
426 char **date_header,
427 char *content_sha256_header,
428 struct dynbuf *canonical_headers,
429 struct dynbuf *signed_headers)
431 char date_hdr_key[DATE_HDR_KEY_LEN];
432 char date_full_hdr[DATE_FULL_HDR_LEN];
433 struct curl_slist *head = NULL;
434 struct curl_slist *tmp_head = NULL;
435 CURLcode result = CURLE_OUT_OF_MEMORY;
436 struct curl_slist *l;
438 curl_msnprintf(date_hdr_key, DATE_HDR_KEY_LEN, "X-%.*s-Date",
439 (int)plen, provider1);
440 /* provider1 ucfirst */
441 Curl_strntolower(&date_hdr_key[2], provider1, plen);
442 date_hdr_key[2] = Curl_raw_toupper(provider1[0]);
444 curl_msnprintf(date_full_hdr, DATE_FULL_HDR_LEN,
445 "x-%.*s-date:%s", (int)plen, provider1, timestamp);
446 /* provider1 lowercase */
447 Curl_strntolower(&date_full_hdr[2], provider1, plen);
449 if(!Curl_checkheaders(data, STRCONST("Host")) &&
450 data->state.http_host) {
451 /* Host: [host]:[port] */
452 char *fullhost = curlx_strdup(data->state.http_host);
454 if(fullhost)
455 head = Curl_slist_append_nodup(NULL, fullhost);
456 if(!head) {
457 curlx_free(fullhost);
458 goto fail;
459 }
460 }
462 if(*content_sha256_header) {
463 tmp_head = curl_slist_append(head, content_sha256_header);
464 if(!tmp_head)
465 goto fail;
466 head = tmp_head;
467 }
469 /* copy user headers to our header list. the logic is based on how http.c
470 handles user headers.
472 user headers in format 'name:' with no value are used to signal that an
473 internal header of that name should be removed. those user headers are not
474 added to this list.
476 user headers in format 'name;' with no value are used to signal that a
477 header of that name with no value should be sent. those user headers are
478 added to this list but in the format that they will be sent, ie the
479 semi-colon is changed to a colon for format 'name:'.
481 user headers with a value of whitespace only, or without a colon or
482 semi-colon, are not added to this list.
483 */
484 for(l = data->set.headers; l; l = l->next) {
485 char *dupdata;
486 const char *ptr;
487 const char *sep = strchr(l->data, ':');
488 if(!sep)
489 sep = strchr(l->data, ';');
490 if(!sep || (*sep == ':' && !*(sep + 1)))
491 continue;
492 for(ptr = sep + 1; ISBLANK(*ptr); ++ptr)
493 ;
494 if(!*ptr && ptr != sep + 1) /* a value of whitespace only */
495 continue;
496 dupdata = curlx_strdup(l->data);
497 if(!dupdata)
498 goto fail;
499 dupdata[sep - l->data] = ':';
500 tmp_head = Curl_slist_append_nodup(head, dupdata);
501 if(!tmp_head) {
502 curlx_free(dupdata);
503 goto fail;
504 }
505 head = tmp_head;
506 }
508 trim_headers(head);
510 *date_header = find_date_hdr(data, date_hdr_key);
511 if(!*date_header) {
512 tmp_head = curl_slist_append(head, date_full_hdr);
513 if(!tmp_head)
514 goto fail;
515 head = tmp_head;
516 *date_header = curl_maprintf("%s: %s\r\n", date_hdr_key, timestamp);
517 if(!*date_header)
518 goto fail;
519 }
520 else {
521 const char *value;
522 const char *endp;
523 value = strchr(*date_header, ':');
524 if(!value) {
525 *date_header = NULL;
526 goto fail;
527 }
528 ++value;
529 curlx_str_passblanks(&value);
530 endp = value;
531 while(*endp && ISALNUM(*endp))
532 ++endp;
533 /* 16 bytes => "19700101T000000Z" */
534 if((endp - value) == TIMESTAMP_SIZE - 1) {
535 memcpy(timestamp, value, TIMESTAMP_SIZE - 1);
536 timestamp[TIMESTAMP_SIZE - 1] = 0;
537 }
538 else
539 /* bad timestamp length */
540 timestamp[0] = 0;
541 *date_header = NULL;
542 }
544 result = sort_headers(head);
545 if(!result)
546 result = merge_duplicate_headers(head);
547 if(result)
548 goto fail;
550 for(l = head; l; l = l->next) {
551 char *tmp;
553 if(curlx_dyn_add(canonical_headers, l->data))
554 goto fail;
555 if(curlx_dyn_add(canonical_headers, "\n"))
556 goto fail;
558 tmp = strchr(l->data, ':');
559 if(tmp)
560 *tmp = 0;
562 if(l != head && curlx_dyn_add(signed_headers, ";"))
563 goto fail;
564 if(curlx_dyn_add(signed_headers, l->data))
565 goto fail;
566 }
568 result = CURLE_OK;
569fail:
570 curl_slist_free_all(head);
572 return result;
575#define CONTENT_SHA256_KEY_LEN (MAX_SIGV4_LEN + sizeof("X--Content-Sha256"))
576/* add 2 for ": " between header name and value */
577#define CONTENT_SHA256_HDR_LEN (CONTENT_SHA256_KEY_LEN + 2 + SHA256_HEX_LENGTH)
579/* try to parse a payload hash from the content-sha256 header */
580static const char *parse_content_sha_hdr(struct Curl_easy *data,
581 const char *provider1,
582 size_t plen,
583 size_t *value_len)
585 char key[CONTENT_SHA256_KEY_LEN];
586 size_t key_len;
587 const char *value;
588 size_t len;
590 key_len = curl_msnprintf(key, sizeof(key), "x-%.*s-content-sha256",
591 (int)plen, provider1);
593 value = Curl_checkheaders(data, key, key_len);
594 if(!value)
595 return NULL;
597 value = strchr(value, ':');
598 if(!value)
599 return NULL;
600 ++value;
602 curlx_str_passblanks(&value);
604 len = strlen(value);
605 while(len > 0 && ISBLANK(value[len - 1]))
606 --len;
608 *value_len = len;
609 return value;
612static CURLcode calc_payload_hash(struct Curl_easy *data,
613 unsigned char *sha_hash, char *sha_hex)
615 const char *post_data = data->set.postfields;
616 size_t post_data_len = 0;
617 CURLcode result;
619 if(post_data) {
620 if(data->set.postfieldsize < 0)
621 post_data_len = strlen(post_data);
622 else
623 post_data_len = (size_t)data->set.postfieldsize;
624 }
625 result = Curl_sha256it(sha_hash, (const unsigned char *)post_data,
626 post_data_len);
627 if(!result)
628 sha256_to_hex(sha_hex, sha_hash);
629 return result;
632#define S3_UNSIGNED_PAYLOAD "UNSIGNED-PAYLOAD"
634static CURLcode calc_s3_payload_hash(struct Curl_easy *data,
635 Curl_HttpReq httpreq,
636 const char *provider1,
637 size_t plen,
638 unsigned char *sha_hash,
639 char *sha_hex, char *header)
641 bool empty_method = (httpreq == HTTPREQ_GET || httpreq == HTTPREQ_HEAD);
642 /* The request method or filesize indicate no request payload */
643 bool empty_payload = (empty_method || data->set.filesize == 0);
644 /* The POST payload is in memory */
645 bool post_payload = (httpreq == HTTPREQ_POST && data->set.postfields);
646 CURLcode result = CURLE_OUT_OF_MEMORY;
648 if(empty_payload || post_payload) {
649 /* Calculate a real hash when we know the request payload */
650 result = calc_payload_hash(data, sha_hash, sha_hex);
651 if(result)
652 goto fail;
653 }
654 else {
655 /* Fall back to s3's UNSIGNED-PAYLOAD */
656 size_t len = CURL_CSTRLEN(S3_UNSIGNED_PAYLOAD);
657 DEBUGASSERT(len < SHA256_HEX_LENGTH); /* 16 < 65 */
658 memcpy(sha_hex, S3_UNSIGNED_PAYLOAD, len);
659 sha_hex[len] = 0;
660 }
662 /* format the required content-sha256 header */
663 curl_msnprintf(header, CONTENT_SHA256_HDR_LEN,
664 "x-%.*s-content-sha256: %s", (int)plen, provider1, sha_hex);
666 result = CURLE_OK;
667fail:
668 return result;
671static int compare_func(const void *a, const void *b)
673 const struct pair *aa = a;
674 const struct pair *bb = b;
675 const size_t aa_key_len = curlx_dyn_len(&aa->key);
676 const size_t bb_key_len = curlx_dyn_len(&bb->key);
677 const size_t aa_value_len = curlx_dyn_len(&aa->value);
678 const size_t bb_value_len = curlx_dyn_len(&bb->value);
679 int compare;
681 /* If one element is empty, the other is always sorted higher */
683 /* Compare keys */
684 if((aa_key_len == 0) && (bb_key_len == 0))
685 return 0;
686 if(aa_key_len == 0)
687 return -1;
688 if(bb_key_len == 0)
689 return 1;
690 compare = strcmp(curlx_dyn_ptr(&aa->key), curlx_dyn_ptr(&bb->key));
691 if(compare) {
692 return compare;
693 }
695 /* Compare values */
696 if((aa_value_len == 0) && (bb_value_len == 0))
697 return 0;
698 if(aa_value_len == 0)
699 return -1;
700 if(bb_value_len == 0)
701 return 1;
702 compare = strcmp(curlx_dyn_ptr(&aa->value), curlx_dyn_ptr(&bb->value));
704 return compare;
707/* @unittest 1979 */
708UNITTEST CURLcode canon_path(const char *q, size_t len,
709 struct dynbuf *new_path,
710 bool do_uri_encode);
711UNITTEST CURLcode canon_path(const char *q, size_t len,
712 struct dynbuf *new_path,
713 bool do_uri_encode)
715 CURLcode result = CURLE_OK;
717 struct Curl_str original_path;
719 curlx_str_assign(&original_path, q, len);
721 /* Normalized path will be either the same or shorter than the original
722 * path, plus trailing slash */
724 if(do_uri_encode)
725 result = uri_encode_path(&original_path, new_path);
726 else
727 result = curlx_dyn_addn(new_path, q, len);
729 if(!result && curlx_dyn_len(new_path) == 0)
730 result = curlx_dyn_add(new_path, "/");
732 return result;
735/* @unittest 1980 */
736UNITTEST CURLcode canon_query(const char *query, struct dynbuf *dq);
737UNITTEST CURLcode canon_query(const char *query, struct dynbuf *dq)
739 CURLcode result = CURLE_OK;
741 struct dynbuf query_array[MAX_QUERY_COMPONENTS];
742 struct pair encoded_query_array[MAX_QUERY_COMPONENTS];
743 size_t num_query_components;
744 size_t counted_query_components = 0;
745 size_t index;
747 if(!query)
748 return result;
750 result = split_to_dyn_array(query, &query_array[0], &num_query_components);
751 if(result) {
752 goto fail;
753 }
755 /* Create list of pairs, each pair containing an encoded query
756 * component */
758 for(index = 0; index < num_query_components; index++) {
759 const char *in_key;
760 size_t in_key_len;
761 const char *offset;
762 size_t query_part_len = curlx_dyn_len(&query_array[index]);
763 const char *query_part = curlx_dyn_ptr(&query_array[index]);
765 in_key = query_part;
767 offset = strchr(query_part, '=');
768 /* If there is no equals, this key has no value */
769 if(!offset) {
770 in_key_len = strlen(in_key);
771 }
772 else {
773 in_key_len = offset - in_key;
774 }
776 curlx_dyn_init(&encoded_query_array[index].key,
777 (query_part_len * 3) + 1);
778 curlx_dyn_init(&encoded_query_array[index].value,
779 (query_part_len * 3) + 1);
780 counted_query_components++;
782 /* Decode/encode the key */
783 result = normalize_query(in_key, in_key_len,
784 &encoded_query_array[index].key);
785 if(result) {
786 goto fail;
787 }
789 /* Decode/encode the value if it exists */
790 if(offset && offset != (query_part + query_part_len - 1)) {
791 size_t in_value_len;
792 const char *in_value = offset + 1;
793 in_value_len = query_part + query_part_len - (offset + 1);
794 result = normalize_query(in_value, in_value_len,
795 &encoded_query_array[index].value);
796 if(result) {
797 goto fail;
798 }
799 }
800 else {
801 /* If there is no value, the value is an empty string */
802 curlx_dyn_init(&encoded_query_array[index].value, 2);
803 result = curlx_dyn_addn(&encoded_query_array[index].value, "", 1);
804 }
806 if(result) {
807 goto fail;
808 }
809 }
811 /* Sort the encoded query components by key and value */
812 qsort(&encoded_query_array, num_query_components,
813 sizeof(struct pair), compare_func);
815 /* Append the query components together to make a full query string */
816 for(index = 0; index < num_query_components; index++) {
818 if(index)
819 result = curlx_dyn_addn(dq, "&", 1);
820 if(!result) {
821 const char *key_ptr = curlx_dyn_ptr(&encoded_query_array[index].key);
822 const char *value_ptr = curlx_dyn_ptr(&encoded_query_array[index].value);
823 size_t vlen = curlx_dyn_len(&encoded_query_array[index].value);
824 if(value_ptr && vlen) {
825 result = curlx_dyn_addf(dq, "%s=%s", key_ptr, value_ptr);
826 }
827 else {
828 /* Empty value is always encoded to key= */
829 result = curlx_dyn_addf(dq, "%s=", key_ptr);
830 }
831 }
832 if(result)
833 break;
834 }
836fail:
837 if(counted_query_components)
838 /* the encoded_query_array might not be initialized yet */
839 pair_array_free(&encoded_query_array[0], counted_query_components);
840 dyn_array_free(&query_array[0], num_query_components);
841 return result;
844static CURLcode parse_sigv4_params(struct Curl_easy *data,
845 const char *hostname,
846 struct Curl_str *provider0,
847 struct Curl_str *provider1,
848 struct Curl_str *region,
849 struct Curl_str *service)
851 const char *line = CURL_EASY_STR(data, STRING_AWS_SIGV4);
852 if(!line || !*line)
853 line = "aws:amz";
855 /* provider0[:provider1[:region[:service]]]
857 No string can be longer than N bytes of non-whitespace */
858 if(curlx_str_until(&line, provider0, MAX_SIGV4_LEN, ':')) {
859 failf(data, "first aws-sigv4 provider cannot be empty");
860 return CURLE_BAD_FUNCTION_ARGUMENT;
861 }
862 if(curlx_str_single(&line, ':') ||
863 curlx_str_until(&line, provider1, MAX_SIGV4_LEN, ':')) {
864 *provider1 = *provider0;
865 }
866 else if(curlx_str_single(&line, ':') ||
867 curlx_str_until(&line, region, MAX_SIGV4_LEN, ':') ||
868 curlx_str_single(&line, ':') ||
869 curlx_str_until(&line, service, MAX_SIGV4_LEN, ':')) {
870 /* nothing to do */
871 }
873 if(!curlx_strlen(service)) {
874 const char *p = hostname;
875 if(curlx_str_until(&p, service, MAX_SIGV4_LEN, '.') ||
876 curlx_str_single(&p, '.')) {
877 failf(data, "aws-sigv4: service missing in parameters and hostname");
878 return CURLE_URL_MALFORMAT;
879 }
881 infof(data, "aws_sigv4: picked service %.*s from host",
882 (int)curlx_strlen(service), curlx_str(service));
884 if(!curlx_strlen(region)) {
885 if(curlx_str_until(&p, region, MAX_SIGV4_LEN, '.') ||
886 curlx_str_single(&p, '.')) {
887 failf(data, "aws-sigv4: region missing in parameters and hostname");
888 return CURLE_URL_MALFORMAT;
889 }
890 infof(data, "aws_sigv4: picked region %.*s from host",
891 (int)curlx_strlen(region), curlx_str(region));
892 }
893 }
895 return CURLE_OK;
898static CURLcode get_payload_hash(struct Curl_easy *data,
899 Curl_HttpReq httpreq,
900 struct Curl_str *provider0,
901 struct Curl_str *provider1,
902 struct Curl_str *service,
903 unsigned char *sha_hash,
904 char *sha_hex,
905 char *content_sha256_hdr,
906 const char **payload_hash_out,
907 size_t *payload_hash_len_out)
909 *payload_hash_out =
910 parse_content_sha_hdr(data, curlx_str(provider1),
911 curlx_strlen(provider1), payload_hash_len_out);
913 if(!*payload_hash_out) {
914 CURLcode result;
915 /* AWS S3 requires a x-amz-content-sha256 header, and supports special
916 * values like UNSIGNED-PAYLOAD */
917 bool sign_as_s3 = curlx_str_casecompare(provider0, "aws") &&
918 curlx_str_casecompare(service, "s3");
920 if(sign_as_s3)
921 result = calc_s3_payload_hash(data, httpreq, curlx_str(provider1),
922 curlx_strlen(provider1), sha_hash,
923 sha_hex, content_sha256_hdr);
924 else
925 result = calc_payload_hash(data, sha_hash, sha_hex);
926 if(result)
927 return result;
929 *payload_hash_out = sha_hex;
930 /* may be shorter than SHA256_HEX_LENGTH, like S3_UNSIGNED_PAYLOAD */
931 *payload_hash_len_out = strlen(sha_hex);
932 }
933 return CURLE_OK;
936static CURLcode get_timestamp(char *timestamp, size_t stampsize)
938 time_t clock;
939 struct tm tm;
940 CURLcode result;
942#ifdef DEBUGBUILD
943 {
944 char *force_timestamp = getenv("CURL_FORCETIME");
945 if(force_timestamp)
946 clock = 0;
947 else
948 clock = time(NULL);
949 }
950#else
951 clock = time(NULL);
952#endif
953 result = curlx_gmtime(clock, &tm);
954 if(result)
955 return result;
957 if(!strftime(timestamp, stampsize, "%Y%m%dT%H%M%SZ", &tm))
958 return CURLE_OUT_OF_MEMORY;
960 return CURLE_OK;
963static CURLcode make_canonical_request(struct Curl_easy *data,
964 char *timestamp,
965 struct Curl_str *provider1,
966 struct Curl_str *service,
967 const char *method,
968 const char *payload_hash,
969 size_t payload_hash_len,
970 char **date_header_out,
971 char *content_sha256_hdr,
972 struct dynbuf *canonical_headers,
973 struct dynbuf *signed_headers,
974 char **canonical_request_out)
976 struct dynbuf canonical_query;
977 struct dynbuf canonical_path;
978 CURLcode result;
980 curlx_dyn_init(&canonical_query, CURL_MAX_HTTP_HEADER);
981 curlx_dyn_init(&canonical_path, CURL_MAX_HTTP_HEADER);
983 result = make_headers(data, timestamp,
984 curlx_str(provider1), curlx_strlen(provider1),
985 date_header_out, content_sha256_hdr,
986 canonical_headers, signed_headers);
987 if(result)
988 goto fail;
990 result = canon_query(data->state.up.query, &canonical_query);
991 if(result)
992 goto fail;
994 result = canon_path(data->state.up.path, strlen(data->state.up.path),
995 &canonical_path,
996 should_urlencode(service));
997 if(result)
998 goto fail;
1000 *canonical_request_out =
1001 curl_maprintf("%s\n" /* HTTPRequestMethod */
1002 "%s\n" /* CanonicalURI */
1003 "%s\n" /* CanonicalQueryString */
1004 "%s\n" /* CanonicalHeaders */
1005 "%s\n" /* SignedHeaders */
1006 "%.*s", /* HashedRequestPayload in hex */
1007 method,
1008 curlx_dyn_ptr(&canonical_path),
1009 curlx_dyn_ptr(&canonical_query) ?
1010 curlx_dyn_ptr(&canonical_query) : "",
1011 curlx_dyn_ptr(canonical_headers),
1012 curlx_dyn_ptr(signed_headers),
1013 (int)payload_hash_len, payload_hash);
1014 if(!*canonical_request_out) {
1015 result = CURLE_OUT_OF_MEMORY;
1016 goto fail;
1019 result = CURLE_OK;
1020fail:
1021 curlx_dyn_free(&canonical_query);
1022 curlx_dyn_free(&canonical_path);
1023 return result;
1026static CURLcode make_string_to_sign(struct Curl_easy *data,
1027 struct Curl_str *provider0,
1028 struct Curl_str *region,
1029 struct Curl_str *service,
1030 const char *date,
1031 const char *timestamp,
1032 const char *canonical_request,
1033 char **request_type_out,
1034 char **credential_scope_out,
1035 char **str_to_sign_out)
1037 char *request_type;
1038 char *credential_scope;
1039 char *str_to_sign;
1040 unsigned char sha_hash[CURL_SHA256_DIGEST_LENGTH];
1041 char sha_hex[SHA256_HEX_LENGTH];
1043 request_type = curl_maprintf("%.*s4_request",
1044 (int)curlx_strlen(provider0),
1045 curlx_str(provider0));
1046 if(!request_type)
1047 return CURLE_OUT_OF_MEMORY;
1049 /* provider0 is lowercased *after* curl_maprintf() so that the buffer
1050 can be written to */
1051 Curl_strntolower(request_type, request_type, curlx_strlen(provider0));
1053 credential_scope = curl_maprintf("%s/%.*s/%.*s/%s", date,
1054 (int)curlx_strlen(region),
1055 curlx_str(region),
1056 (int)curlx_strlen(service),
1057 curlx_str(service),
1058 request_type);
1059 if(!credential_scope) {
1060 curlx_free(request_type);
1061 return CURLE_OUT_OF_MEMORY;
1064 if(Curl_sha256it(sha_hash, (const unsigned char *)canonical_request,
1065 strlen(canonical_request))) {
1066 curlx_free(request_type);
1067 curlx_free(credential_scope);
1068 return CURLE_OUT_OF_MEMORY;
1071 sha256_to_hex(sha_hex, sha_hash);
1073 /*
1074 * Google allows using RSA key instead of HMAC, so this code might change
1075 * in the future. For now we only support HMAC.
1076 */
1077 str_to_sign = curl_maprintf("%.*s4-HMAC-SHA256\n" /* Algorithm */
1078 "%s\n" /* RequestDateTime */
1079 "%s\n" /* CredentialScope */
1080 "%s", /* HashedCanonicalRequest in hex */
1081 (int)curlx_strlen(provider0),
1082 curlx_str(provider0),
1083 timestamp,
1084 credential_scope,
1085 sha_hex);
1086 if(!str_to_sign) {
1087 curlx_free(request_type);
1088 curlx_free(credential_scope);
1089 return CURLE_OUT_OF_MEMORY;
1092 /* make provider0 part done uppercase */
1093 Curl_strntoupper(str_to_sign, curlx_str(provider0),
1094 curlx_strlen(provider0));
1096 infof(data, "aws_sigv4: String to sign (enclosed in []) - [%s]",
1097 str_to_sign);
1099 *request_type_out = request_type;
1100 *credential_scope_out = credential_scope;
1101 *str_to_sign_out = str_to_sign;
1102 return CURLE_OK;
1105static CURLcode sign_and_set_auth_headers(struct Curl_easy *data,
1106 struct Curl_str *provider0,
1107 struct Curl_str *region,
1108 struct Curl_str *service,
1109 const char *request_type,
1110 const char *credential_scope,
1111 const char *date,
1112 const char *str_to_sign,
1113 const char *date_header,
1114 const char *content_sha256_hdr,
1115 struct dynbuf *signed_headers)
1117 CURLcode result = CURLE_OUT_OF_MEMORY;
1118 const char *passwd = Curl_creds_passwd(data->state.creds);
1119 char *secret = NULL;
1120 unsigned char sign0[CURL_SHA256_DIGEST_LENGTH] = { 0 };
1121 unsigned char sign1[CURL_SHA256_DIGEST_LENGTH] = { 0 };
1122 char sha_hex[SHA256_HEX_LENGTH];
1123 char *auth_headers = NULL;
1124 char *user = curl_escape(Curl_creds_user(data->state.creds), 0);
1125 if(!user)
1126 return CURLE_OUT_OF_MEMORY;
1128 secret = curl_maprintf("%.*s4%s", (int)curlx_strlen(provider0),
1129 curlx_str(provider0), passwd);
1130 if(!secret)
1131 goto fail;
1132 /* make provider0 part done uppercase */
1133 Curl_strntoupper(secret, curlx_str(provider0), curlx_strlen(provider0));
1135 HMAC_SHA256(secret, strlen(secret), date, strlen(date), sign0);
1136 HMAC_SHA256(sign0, sizeof(sign0),
1137 curlx_str(region), curlx_strlen(region), sign1);
1138 HMAC_SHA256(sign1, sizeof(sign1),
1139 curlx_str(service), curlx_strlen(service), sign0);
1140 HMAC_SHA256(sign0, sizeof(sign0),
1141 request_type, strlen(request_type), sign1);
1142 HMAC_SHA256(sign1, sizeof(sign1),
1143 str_to_sign, strlen(str_to_sign), sign0);
1145 sha256_to_hex(sha_hex, sign0);
1147 infof(data, "aws_sigv4: Signature - %s", sha_hex);
1149 auth_headers = curl_maprintf("Authorization: %.*s4-HMAC-SHA256 "
1150 "Credential=%s/%s, "
1151 "SignedHeaders=%s, "
1152 "Signature=%s\r\n"
1153 "%s"
1154 "%s%s",
1155 (int)curlx_strlen(provider0),
1156 curlx_str(provider0),
1157 user,
1158 credential_scope,
1159 curlx_dyn_ptr(signed_headers),
1160 sha_hex,
1161 /*
1162 * date_header is added here, only if it was not
1163 * user-specified (using CURLOPT_HTTPHEADER).
1164 * date_header includes \r\n
1165 */
1166 date_header ? date_header : "",
1167 content_sha256_hdr,
1168 content_sha256_hdr[0] ? "\r\n": "");
1169 if(!auth_headers)
1170 goto fail;
1172 /* provider 0 uppercase */
1173 Curl_strntoupper(&auth_headers[CURL_CSTRLEN("Authorization: ")],
1174 curlx_str(provider0), curlx_strlen(provider0));
1176 curlx_free(data->req.hd_auth);
1177 data->req.hd_auth = auth_headers;
1178 data->state.authhost.done = TRUE;
1179 result = CURLE_OK;
1181fail:
1182 curlx_free(user);
1183 curlx_free(secret);
1184 return result;
1187CURLcode Curl_output_aws_sigv4(struct Curl_easy *data)
1189 CURLcode result = CURLE_OUT_OF_MEMORY;
1190 struct Curl_str provider0 = { NULL, 0 };
1191 struct Curl_str provider1 = { NULL, 0 };
1192 struct Curl_str region = { NULL, 0 };
1193 struct Curl_str service = { NULL, 0 };
1194 const char *hostname = data->state.origin->hostname;
1195 char timestamp[TIMESTAMP_SIZE];
1196 char date[9];
1197 struct dynbuf canonical_headers;
1198 struct dynbuf signed_headers;
1199 char *date_header = NULL;
1200 Curl_HttpReq httpreq;
1201 const char *method = NULL;
1202 const char *payload_hash = NULL;
1203 size_t payload_hash_len = 0;
1204 unsigned char sha_hash[CURL_SHA256_DIGEST_LENGTH];
1205 char sha_hex[SHA256_HEX_LENGTH];
1206 char content_sha256_hdr[CONTENT_SHA256_HDR_LEN + 2] = ""; /* add \r\n */
1207 char *canonical_request = NULL;
1208 char *request_type = NULL;
1209 char *credential_scope = NULL;
1210 char *str_to_sign = NULL;
1212 if(data->set.path_as_is) {
1213 failf(data, "Cannot use sigv4 authentication with path-as-is flag");
1214 return CURLE_BAD_FUNCTION_ARGUMENT;
1217 if(Curl_checkheaders(data, STRCONST("Authorization")))
1218 /* Authorization already present, Bailing out */
1219 return CURLE_OK;
1221 /* we init those buffers here, so goto fail will free initialized dynbuf */
1222 curlx_dyn_init(&canonical_headers, CURL_MAX_HTTP_HEADER);
1223 curlx_dyn_init(&signed_headers, CURL_MAX_HTTP_HEADER);
1225 result = parse_sigv4_params(data, hostname, &provider0, &provider1,
1226 &region, &service);
1227 if(!result) {
1228 Curl_http_method(data, &method, &httpreq);
1229 result = get_payload_hash(data, httpreq, &provider0, &provider1, &service,
1230 sha_hash, sha_hex, content_sha256_hdr,
1231 &payload_hash, &payload_hash_len);
1234 if(!result)
1235 result = get_timestamp(timestamp, sizeof(timestamp));
1237 if(!result)
1238 result = make_canonical_request(data, timestamp,
1239 &provider1, &service,
1240 method, payload_hash, payload_hash_len,
1241 &date_header, content_sha256_hdr,
1242 &canonical_headers, &signed_headers,
1243 &canonical_request);
1244 if(!result) {
1245 /* the timestamp might have been updated in make_canonical_request */
1246 memcpy(date, timestamp, sizeof(date) - 1);
1247 date[sizeof(date) - 1] = 0;
1249 result = make_string_to_sign(data, &provider0, &region, &service,
1250 date, timestamp, canonical_request,
1251 &request_type, &credential_scope,
1252 &str_to_sign);
1254 if(!result)
1255 result = sign_and_set_auth_headers(data, &provider0, &region, &service,
1256 request_type, credential_scope,
1257 date, str_to_sign, date_header,
1258 content_sha256_hdr, &signed_headers);
1260 curlx_dyn_free(&canonical_headers);
1261 curlx_dyn_free(&signed_headers);
1262 curlx_free(canonical_request);
1263 curlx_free(request_type);
1264 curlx_free(credential_scope);
1265 curlx_free(str_to_sign);
1266 curlx_free(date_header);
1267 return result;
1270#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_AWS */
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=23
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch20011 KiB · hit23 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=11170
git.response.cache=HIT
git.response.duration_ms=23
git.fetch.wants=5fe5dfcb12d85c0cc108daa20df3a2f95909c6d9
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11170
git.pkt.payload.max=11149
git.sideband.band1.bytes=11148
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e1a712ed755e44
Request
f8483c9f…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format