Artifactspublic-v0

curl

master
ply4 git calls42 ms310 KiB3 cachedprotocol v2
2571 lines75 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24#include "tool_setup.h"
26#include "curlx/nonblock.h" /* for curlx_nonblock() */
28#ifdef HAVE_LOCALE_H
29# include <locale.h>
30#endif
32#ifdef HAVE_SYS_SELECT_H
33# include <sys/select.h>
34#elif defined(HAVE_UNISTD_H)
35# include <unistd.h>
36#endif
38#ifdef __VMS
39# include <fabdef.h>
40#endif
42#ifdef __AMIGA__
43# include <proto/dos.h>
44#endif
46#ifdef HAVE_NETINET_IN_H
47# include <netinet/in.h>
48#endif
50#ifdef HAVE_UV_H
51/* this is for libuv-enabled debug builds only */
52#include <uv.h>
53#endif
55#include "tool_cfgable.h"
56#include "tool_cb_hdr.h"
57#include "tool_cb_prg.h"
58#include "tool_cb_wrt.h"
59#include "tool_dirhie.h"
60#include "tool_doswin.h"
61#include "tool_easysrc.h"
62#include "tool_filetime.h"
63#include "tool_getparam.h"
64#include "tool_helpers.h"
65#include "tool_libinfo.h"
66#include "tool_main.h"
67#include "tool_msgs.h"
68#include "tool_operate.h"
69#include "tool_operhlp.h"
70#include "tool_paramhlp.h"
71#include "tool_parsecfg.h"
72#include "tool_setopt.h"
73#include "tool_ssls.h"
74#include "tool_urlglob.h"
75#include "tool_util.h"
76#include "tool_writeout.h"
77#include "tool_xattr.h"
78#include "tool_vms.h"
79#include "tool_help.h"
80#include "tool_hugehelp.h"
81#include "tool_progress.h"
82#include "tool_ipfs.h"
83#include "config2setopts.h"
84#include "var.h"
86#ifdef DEBUGBUILD
87/* libcurl's debug-only curl_easy_perform_ev() */
88CURL_EXTERN CURLcode curl_easy_perform_ev(CURL *easy);
89#endif
91#define CURL_CA_CERT_ERRORMSG \
92 "More details here: https://curl.se/docs/sslcerts.html\n\n" \
93 "curl failed to verify the legitimacy of the server and therefore " \
94 "could not\nestablish a secure connection to it. To learn more about " \
95 "this situation and\nhow to fix it, please visit the webpage mentioned " \
96 "above.\n"
98static CURLcode create_transfer(CURLSH *share,
99 bool *added,
100 bool *skipped);
102static bool is_fatal_error(CURLcode code)
104 switch(code) {
105 case CURLE_FAILED_INIT:
106 case CURLE_OUT_OF_MEMORY:
107 case CURLE_UNKNOWN_OPTION:
108 case CURLE_BAD_FUNCTION_ARGUMENT:
109 /* critical error */
110 return TRUE;
111 default:
112 break;
113 }
115 /* no error or not critical */
116 return FALSE;
119/*
120 * Check if a given string is a PKCS#11 URI
121 */
122static bool is_pkcs11_uri(const char *string)
124 if(curl_strnequal(string, "pkcs11:", 7)) {
125 return TRUE;
126 }
127 else {
128 return FALSE;
129 }
132#ifdef __VMS
133/*
134 * get_vms_file_size does what it takes to get the real size of the file
135 *
136 * For fixed files, find out the size of the EOF block and adjust.
137 *
138 * For all others, have to read the entire file in, discarding the contents.
139 * Most posted text files are small, and binary files like zlib archives
140 * and CD/DVD images should be either a STREAM_LF format or a fixed format.
141 *
142 */
143static curl_off_t vms_realfilesize(const char *name,
144 const curlx_struct_stat *stat_buf)
146 char buffer[8192];
147 curl_off_t count;
148 int ret_stat;
149 FILE *file;
151 /* !checksrc! disable FOPENMODE 1 */
152 file = curlx_fopen(name, "r"); /* VMS */
153 if(!file) {
154 return 0;
155 }
156 count = 0;
157 ret_stat = 1;
158 while(ret_stat > 0) {
159 ret_stat = fread(buffer, 1, sizeof(buffer), file);
160 if(ret_stat)
161 count += ret_stat;
162 }
163 curlx_fclose(file);
165 return count;
168/*
169 *
170 * VmsSpecialSize checks to see if the stat st_size can be trusted and
171 * if not to call a routine to get the correct size.
172 *
173 */
174static curl_off_t VmsSpecialSize(const char *name,
175 const curlx_struct_stat *stat_buf)
177 switch(stat_buf->st_fab_rfm) {
178 case FAB$C_VAR:
179 case FAB$C_VFC:
180 return vms_realfilesize(name, stat_buf);
181 default:
182 return stat_buf->st_size;
183 }
185#endif /* __VMS */
187struct per_transfer *transfers; /* first node */
188static struct per_transfer *transfersl; /* last node */
190/* add_per_transfer creates a new 'per_transfer' node in the linked
191 list of transfers */
192static CURLcode add_per_transfer(struct per_transfer **per)
194 struct per_transfer *p;
195 p = curlx_calloc(1, sizeof(struct per_transfer));
196 if(!p)
197 return CURLE_OUT_OF_MEMORY;
198 if(!transfers)
199 /* first entry */
200 transfersl = transfers = p;
201 else {
202 /* make the last node point to the new node */
203 transfersl->next = p;
204 /* make the new node point back to the formerly last node */
205 p->prev = transfersl;
206 /* move the last node pointer to the new entry */
207 transfersl = p;
208 }
209 *per = p;
211 return CURLE_OK;
214/* Remove the specified transfer from the list (and free it), return the next
215 in line */
216static struct per_transfer *del_per_transfer(struct per_transfer *per)
218 struct per_transfer *n;
219 struct per_transfer *p;
220 DEBUGASSERT(transfers);
221 DEBUGASSERT(transfersl);
222 DEBUGASSERT(per);
224 n = per->next;
225 p = per->prev;
227 if(p)
228 p->next = n;
229 else
230 transfers = n;
232 if(n)
233 n->prev = p;
234 else
235 transfersl = p;
237 curlx_free(per->uploadfile);
238 curlx_free(per->outfile);
239 curlx_free(per->url);
240 curl_easy_cleanup(per->curl);
241 curlx_free(per);
243 return n;
246static CURLcode pre_transfer(struct per_transfer *per)
248 curl_off_t uploadfilesize = -1;
249 curlx_struct_stat fileinfo;
250 CURLcode result = CURLE_OK;
252 if(per->uploadfile && !stdin_upload(per->uploadfile)) {
253 /* VMS Note:
254 *
255 * Reading binary from files can be a problem... Only FIXED, VAR
256 * etc WITHOUT implied CC do work. Others need a \n appended to
257 * a line
258 *
259 * - Stat gives a size but this is UNRELIABLE in VMS. E.g.
260 * a fixed file with implied CC needs to have a byte added for every
261 * record processed, this can be derived from Filesize & recordsize
262 * for VARiable record files the records need to be counted! for
263 * every record add 1 for linefeed and subtract 2 for the record
264 * header for VARIABLE header files only the bare record data needs
265 * to be considered with one appended if implied CC
266 */
267#ifdef __VMS
268 /* Calculate the real upload size for VMS */
269 per->infd = -1;
270 if(curlx_stat(per->uploadfile, &fileinfo) == 0) {
271 fileinfo.st_size = VmsSpecialSize(per->uploadfile, &fileinfo);
272 switch(fileinfo.st_fab_rfm) {
273 case FAB$C_VAR:
274 case FAB$C_VFC:
275 case FAB$C_STMCR:
276 per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY);
277 break;
278 default:
279 per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY,
280 "rfm=stmlf", "ctx=stm");
281 }
282 }
283 if(per->infd == -1)
284#else
285 per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY);
286 if((per->infd == -1) || curlx_fstat(per->infd, &fileinfo))
287#endif
288 {
289 helpf("cannot open '%s'", per->uploadfile);
290 if(per->infd != -1) {
291 curlx_close(per->infd);
292 per->infd = STDIN_FILENO;
293 }
294 return CURLE_READ_ERROR;
295 }
296 per->infdopen = TRUE;
298 /* we ignore file size for char/block devices, sockets, etc. */
299 if(S_ISREG(fileinfo.st_mode))
300 uploadfilesize = fileinfo.st_size;
302#ifdef DEBUGBUILD
303 /* allow dedicated test cases to override */
304 {
305 const char *ev = getenv("CURL_UPLOAD_SIZE");
306 if(ev) {
307 curl_off_t sz;
308 curlx_str_number(&ev, &sz, CURL_OFF_T_MAX);
309 uploadfilesize = sz;
310 }
311 }
312#endif
314 if(uploadfilesize != -1)
315 my_setopt_offt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
316 }
317 per->uploadfilesize = uploadfilesize;
318 per->start = curlx_now();
319 return result;
322void single_transfer_cleanup(void)
324 struct State *state = &global->state;
325 /* Free list of remaining URLs */
326 glob_cleanup(&state->urlglob);
327 curlx_safefree(state->uploadfile);
328 /* Free list of globbed upload files */
329 glob_cleanup(&state->inglob);
332/* Helper function for retrycheck.
333 *
334 * This function is a prerequisite check used to determine whether or not some
335 * already downloaded data (ie out->bytes written) can be safely resumed in a
336 * subsequent transfer. The conditions are somewhat pedantic to avoid any risk
337 * of data corruption.
338 *
339 * Specific HTTP limitations (scheme, method, response code, etc) are checked
340 * in retrycheck if this prerequisite check is met.
341 */
342static bool is_outfile_auto_resumable(struct OperationConfig *config,
343 struct per_transfer *per,
344 CURLcode result)
346 struct OutStruct *outs = &per->outs;
347 return config->use_resume && config->resume_from_current &&
348 config->resume_from >= 0 && outs->init == config->resume_from &&
349 outs->bytes > 0 && outs->filename && outs->regular_file && outs->fopened &&
350 outs->stream && !ferror(outs->stream) &&
351 !config->customrequest && !per->uploadfile &&
352 (config->httpreq == TOOL_HTTPREQ_UNSPEC ||
353 config->httpreq == TOOL_HTTPREQ_GET) &&
354 /* CURLE_WRITE_ERROR could mean outs->bytes is not accurate */
355 result != CURLE_WRITE_ERROR && result != CURLE_RANGE_ERROR;
358enum retryreason {
359 RETRY_NO,
360 RETRY_ALL_ERRORS,
361 RETRY_TIMEOUT,
362 RETRY_CONNREFUSED,
363 RETRY_HTTP,
364 RETRY_FTP
365};
367/* figure out how long to wait until retry */
368static uint32_t retry_sleep(struct OperationConfig *config,
369 struct per_transfer *per,
370 enum retryreason retry,
371 bool *retryp)
373 static const char * const m[] = {
374 "(retrying all errors)",
375 ": timeout",
376 ": connection refused",
377 ": HTTP error",
378 ": FTP error"
379 };
380 CURL *curl = per->curl;
381 uint32_t sleeptime = 0;
382 if(RETRY_HTTP == retry) {
383 curl_off_t retry_after = 0;
384 curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);
385 if(retry_after) {
386 /* make sure it does not overflow */
387 if(retry_after > (curl_off_t)(UINT32_MAX / 1000))
388 sleeptime = UINT32_MAX;
389 else
390 sleeptime = (uint32_t)retry_after * 1000U; /* milliseconds */
392 /* if adding retry_after seconds to the process would exceed the
393 maximum time allowed for retrying, then exit the retries right
394 away */
395 if(config->retry_maxtime_ms) {
396 timediff_t ms = curlx_timediff_ms(curlx_now(), per->retrystart);
398 if((CURL_OFF_T_MAX - sleeptime < ms) ||
399 (ms + sleeptime > config->retry_maxtime_ms)) {
400 warnf("The Retry-After: time would "
401 "make this command line exceed the maximum allowed time "
402 "for retries.");
403 *retryp = FALSE;
404 return 0; /* no retry */
405 }
406 }
407 }
408 }
409 if(!sleeptime && !config->retry_delay_ms) {
410 if(!per->retry_sleep)
411 per->retry_sleep = RETRY_SLEEP_DEFAULT;
412 else
413 per->retry_sleep *= 2;
414 if(per->retry_sleep > RETRY_SLEEP_MAX)
415 per->retry_sleep = RETRY_SLEEP_MAX;
416 }
417 if(!sleeptime)
418 sleeptime = per->retry_sleep;
420 warnf("Problem %s. Retrying in %u%s%.*u second%s. "
421 "%ld retr%s left.",
422 m[retry - 1], sleeptime / 1000,
423 (sleeptime % 1000 ? "." : ""),
424 (sleeptime % 1000 ? 3 : 0),
425 sleeptime % 1000,
426 (sleeptime == 1000 ? "" : "s"),
427 per->retry_remaining,
428 (per->retry_remaining > 1 ? "ies" : "y"));
430 return sleeptime;
433static CURLcode retrycheck(struct OperationConfig *config,
434 struct per_transfer *per,
435 CURLcode result,
436 bool *retryp,
437 uint32_t *delayms)
439 CURL *curl = per->curl;
440 struct OutStruct *outs = &per->outs;
441 enum retryreason reason = RETRY_NO;
442 if((result == CURLE_OPERATION_TIMEDOUT) ||
443 (result == CURLE_COULDNT_RESOLVE_HOST) ||
444 (result == CURLE_COULDNT_RESOLVE_PROXY) ||
445 (result == CURLE_FTP_ACCEPT_TIMEOUT))
446 /* retry timeout always */
447 reason = RETRY_TIMEOUT;
448 else if(config->retry_connrefused &&
449 (result == CURLE_COULDNT_CONNECT)) {
450 long oserrno = 0;
451 curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno);
452 if(SOCKECONNREFUSED == oserrno)
453 reason = RETRY_CONNREFUSED;
454 }
455 else if((result == CURLE_OK) ||
456 (config->fail && (result == CURLE_HTTP_RETURNED_ERROR))) {
457 /* If it returned OK. _or_ failonerror was enabled and it
458 returned due to such an error, check for HTTP transient
459 errors to retry on. */
460 const char *scheme;
461 curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
462 scheme = proto_token(scheme);
463 if(scheme == proto_http || scheme == proto_https) {
464 /* This was HTTP(S) */
465 long response = 0;
466 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
468 switch(response) {
469 case 408: /* Request Timeout */
470 case 429: /* Too Many Requests (RFC6585) */
471 case 500: /* Internal Server Error */
472 case 502: /* Bad Gateway */
473 case 503: /* Service Unavailable */
474 case 504: /* Gateway Timeout */
475 case 522: /* Connection Timed Out (Cloudflare) */
476 case 524: /* Proxy Read Timeout (Cloudflare) */
477 reason = RETRY_HTTP;
478 /*
479 * At this point, we have already written data to the output
480 * file (or terminal). If we write to a file, we must rewind
481 * or close/re-open the file so that the next attempt starts
482 * over from the beginning.
483 *
484 * For the upload case, we might need to start over reading from a
485 * previous point if we have uploaded something when this was
486 * returned.
487 */
488 break;
489 }
490 }
491 } /* if CURLE_OK */
492 else if(result) {
493 const char *scheme;
494 long response = 0;
496 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
497 curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
498 scheme = proto_token(scheme);
500 if((scheme == proto_ftp || scheme == proto_ftps) && response / 100 == 4)
501 /*
502 * This is typically when the FTP server only allows a certain
503 * amount of users and we are not one of them. All 4xx codes
504 * are transient.
505 */
506 reason = RETRY_FTP;
507 }
509 if(result && !reason && config->retry_all_errors)
510 reason = RETRY_ALL_ERRORS;
512 if(reason) {
513 bool truncate = TRUE; /* truncate output file */
515 /* how long to wait until next attempt */
516 uint32_t sleeptime;
518 *retryp = TRUE;
520 sleeptime = retry_sleep(config, per, reason, retryp);
521 if(!*retryp)
522 /* no retry */
523 return CURLE_OK;
524 per->retry_remaining--;
526 /* Skip truncation of outfile if auto-resume is enabled for download and
527 the partially received data is good. Only for HTTP GET requests in
528 limited circumstances. */
529 if(is_outfile_auto_resumable(config, per, result)) {
530 long response = 0;
531 struct curl_header *header = NULL;
532 const char *method = NULL, *scheme = NULL;
534 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_METHOD, &method);
535 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
536 curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
537 scheme = proto_token(scheme);
539 if((scheme == proto_http || scheme == proto_https) &&
540 method && !strcmp(method, "GET") &&
541 ((response == 206 && config->resume_from) ||
542 (response == 200 &&
543 !curl_easy_header(curl, "Accept-Ranges", 0,
544 CURLH_HEADER, -1, &header) &&
545 !strcmp(header->value, "bytes")))) {
547 notef("Keeping %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes);
548 if(fflush(outs->stream)) {
549 errorf("Failed to flush output file stream");
550 *retryp = FALSE;
551 return CURLE_WRITE_ERROR;
552 }
553 if(outs->bytes >= CURL_OFF_T_MAX - outs->init) {
554 errorf("Exceeded maximum supported file size ("
555 "%" CURL_FORMAT_CURL_OFF_T " + "
556 "%" CURL_FORMAT_CURL_OFF_T ")",
557 outs->init, outs->bytes);
558 *retryp = FALSE;
559 return CURLE_WRITE_ERROR;
560 }
561 truncate = FALSE;
562 outs->init += outs->bytes;
563 per->dlretry += outs->bytes;
564 outs->bytes = 0;
565 config->resume_from = outs->init;
566 curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from);
567 }
568 }
570 if(truncate && outs->bytes && outs->filename && outs->stream) {
571 curlx_struct_stat fileinfo;
573 /* The output can be a named pipe or a character device etc that
574 cannot be truncated. Only truncate regular files. */
575 if(!curlx_fstat(fileno(outs->stream), &fileinfo) &&
576 S_ISREG(fileinfo.st_mode)) {
577 int rc;
578 /* We have written data to an output file, we truncate file */
579 fflush(outs->stream);
580 notef("Throwing away %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes);
582 /* truncate file at the position where we started appending */
583 if(
584#ifdef DEBUGBUILD
585 getenv("CURL_DBG_TRUNCATE_FAIL") ||
586#endif
587 toolx_ftruncate(fileno(outs->stream), outs->init)) {
588 /* when truncate fails, we cannot append as then we
589 create something strange, bail out */
590 errorf("Failed to truncate file");
591 *retryp = FALSE;
592 return CURLE_WRITE_ERROR;
593 }
594 /* now seek to the end of the file, the position where we
595 truncated the file in a large file-safe way */
596 rc = fseek(outs->stream, 0, SEEK_END);
598 if(rc) {
599 errorf("Failed seeking to end of file");
600 *retryp = FALSE;
601 return CURLE_WRITE_ERROR;
602 }
603 outs->bytes = 0; /* clear for next round */
604 }
605 }
606 per->num_retries++;
607 *delayms = sleeptime;
608 result = CURLE_OK;
609 }
610 return result;
613static CURLcode post_check_result(struct per_transfer *per, CURLcode result)
615 struct OperationConfig *config = per->config;
617#ifdef __VMS
618 if(is_vms_shell()) {
619 /* VMS DCL shell behavior */
620 if(global->silent && !global->showerror)
621 vms_show = VMSSTS_HIDE;
622 }
623#endif
624 if(!config->synthetic_error && result &&
625 (!global->silent || global->showerror)) {
626 const char *msg = per->errorbuffer;
627 curl_mfprintf(tool_stderr, "curl: (%d) %s\n", (int)result,
628 msg[0] ? msg : curl_easy_strerror(result));
629 if(result == CURLE_PEER_FAILED_VERIFICATION)
630 fputs(CURL_CA_CERT_ERRORMSG, tool_stderr);
631 }
632 else if(config->fail == FAIL_WITH_BODY) {
633 /* if HTTP response >= 400, return error */
634 long code = 0;
635 curl_easy_getinfo(per->curl, CURLINFO_RESPONSE_CODE, &code);
636 if(code >= 400) {
637 if(!global->silent || global->showerror)
638 curl_mfprintf(tool_stderr,
639 "curl: (%d) The requested URL returned error: %ld\n",
640 CURLE_HTTP_RETURNED_ERROR, code);
641 return CURLE_HTTP_RETURNED_ERROR;
642 }
643 }
644 return result;
647static CURLcode post_output_handling(struct per_transfer *per,
648 CURLcode result)
650 struct OutStruct *outs = &per->outs;
651 struct OperationConfig *config = per->config;
652 CURL *curl = per->curl;
653 int rc;
655#ifdef _WIN32
656 /* Discard incomplete UTF-8 sequence buffered from body */
657 memset(outs->utf8seq, 0, sizeof(outs->utf8seq));
658#endif
660 /* Set file extended attributes */
661 if(!result && config->xattr && outs->fopened && outs->stream) {
662 rc = fwrite_xattr(curl, per->url, fileno(outs->stream), outs->filename);
663 if(rc) {
664 char errbuf[STRERROR_LEN];
665 warnf("Error setting extended attributes on '%s': %s", outs->filename,
666 curlx_strerror(errno, errbuf, sizeof(errbuf)));
667 }
668 }
670 if(!result && !outs->stream && !outs->bytes) {
671 /* we have received no data despite the transfer was successful
672 ==> force creation of an empty output file (if an output file
673 was specified) */
674 long cond_unmet = 0L;
675 /* do not create (or even overwrite) the file in case we get no
676 data because of unmet condition */
677 curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet);
678 if(!cond_unmet && !tool_create_output_file(outs, config))
679 return CURLE_WRITE_ERROR;
680 /* write headers buffered while waiting for a -J filename */
681 if(outs->stream && per->hdrcbdata.headlist &&
682 tool_write_headers(&per->hdrcbdata, outs->stream))
683 return CURLE_WRITE_ERROR;
684 }
686 if(!outs->regular_file && outs->stream && !outs->out_null) {
687 /* Dump standard stream buffered data */
688 rc = fflush(outs->stream);
689 if(!result && rc) {
690 /* something went wrong in the writing process */
691 errorf("Failed writing body");
692 return CURLE_WRITE_ERROR;
693 }
694 }
696 return result;
699static CURLcode post_close_output(struct per_transfer *per,
700 CURLcode result)
702 struct OutStruct *outs = &per->outs;
703 struct OperationConfig *config = per->config;
704 int rc;
706 /* Close the outs file */
707 if(outs->fopened && outs->stream) {
708 rc = curlx_fclose(outs->stream);
709 outs->stream = NULL;
710 outs->fopened = FALSE;
711 if(!result && rc) {
712 /* something went wrong in the writing process */
713 result = CURLE_WRITE_ERROR;
714 errorf("curl: (%d) Failed writing body", (int)result);
715 }
716 if(result && config->rm_partial) {
717 curlx_struct_stat st;
718 if(!curlx_stat(outs->filename, &st) && S_ISREG(st.st_mode)) {
719 if(!unlink(outs->filename))
720 notef("Removed output file: %s", outs->filename);
721 else
722 warnf("Failed removing: %s", outs->filename);
723 }
724 else
725 warnf("Skipping removal; not a regular file: %s", outs->filename);
726 }
727 }
729 /* File time can only be set _after_ the file has been closed */
730 if(!result && config->remote_time && outs->regular_file && outs->filename) {
731 /* Ask libcurl if we got a remote file time */
732 curl_off_t filetime = -1;
733 curl_easy_getinfo(per->curl, CURLINFO_FILETIME_T, &filetime);
734 if(filetime != -1)
735 setfiletime(filetime, outs->filename);
736 }
737 return result;
740/*
741 * Call this after a transfer has completed.
742 */
743static CURLcode post_per_transfer(struct per_transfer *per,
744 CURLcode result,
745 bool *retryp,
746 uint32_t *delay) /* milliseconds! */
748 struct OutStruct *outs = &per->outs;
749 struct OperationConfig *config = per->config;
751 *retryp = FALSE;
752 *delay = 0; /* for no retry, keep it zero */
754 if(!per->curl || !config)
755 return result;
757#ifdef _WIN32
758 if(per->uploadfile) {
759 if(!strcmp(per->uploadfile, ".") && per->infd > 0) {
760#ifndef CURL_WINDOWS_UWP
761 sclose(per->infd);
762#else
763 warnf("Closing per->infd != 0: FD == %d. "
764 "This behavior is only supported on desktop Windows", per->infd);
765#endif
766 }
767 }
768 else
769#endif
770 if(per->infdopen)
771 curlx_close(per->infd);
773 if(!per->skip) {
774 result = post_check_result(per, result);
775 result = post_output_handling(per, result);
777 /* if retry-max-time is non-zero, make sure we have not exceeded the
778 time */
779 if(per->retry_remaining &&
780 (!config->retry_maxtime_ms ||
781 (curlx_timediff_ms(curlx_now(), per->retrystart) <
782 config->retry_maxtime_ms))) {
783 result = retrycheck(config, per, result, retryp, delay);
784 if(!result && *retryp)
785 return CURLE_OK; /* retry! */
786 }
788 if((global->progressmode == CURL_PROGRESS_BAR) &&
789 per->progressbar.calls)
790 /* if the custom progress bar has been displayed, we output a
791 newline here */
792 fputs("\n", per->progressbar.out);
794 result = post_close_output(per, result);
795 }
796 /* Write the --write-out data before cleanup but after result is final */
797 if(config->writeout)
798 ourWriteOut(config, per, result);
800 /* Close function-local opened file descriptors */
801 if(per->heads.fopened && per->heads.stream)
802 curlx_fclose(per->heads.stream);
804 if(per->heads.alloc_filename)
805 curlx_safefree(per->heads.filename);
807 if(per->etag_save.fopened && per->etag_save.stream)
808 curlx_fclose(per->etag_save.stream);
810 if(per->etag_save.alloc_filename)
811 curlx_safefree(per->etag_save.filename);
813 if(outs->alloc_filename)
814 curlx_safefree(outs->filename);
815 slist_wc_free_all(per->hdrcbdata.headlist);
816 per->hdrcbdata.headlist = NULL;
817 return result;
820static CURLcode set_cert_types(struct OperationConfig *config)
822 if(feature_ssl) {
823 /* Check if config->cert is a PKCS#11 URI and set the config->cert_type if
824 * necessary */
825 if(config->cert && !config->cert_type && is_pkcs11_uri(config->cert)) {
826 config->cert_type = curlx_strdup("ENG");
827 if(!config->cert_type)
828 return CURLE_OUT_OF_MEMORY;
829 }
831 /* Check if config->key is a PKCS#11 URI and set the config->key_type if
832 * necessary */
833 if(config->key && !config->key_type && is_pkcs11_uri(config->key)) {
834 config->key_type = curlx_strdup("ENG");
835 if(!config->key_type)
836 return CURLE_OUT_OF_MEMORY;
837 }
839 /* Check if config->proxy_cert is a PKCS#11 URI and set the
840 * config->proxy_type if necessary */
841 if(config->proxy_cert && !config->proxy_cert_type &&
842 is_pkcs11_uri(config->proxy_cert)) {
843 config->proxy_cert_type = curlx_strdup("ENG");
844 if(!config->proxy_cert_type)
845 return CURLE_OUT_OF_MEMORY;
846 }
848 /* Check if config->proxy_key is a PKCS#11 URI and set the
849 * config->proxy_key_type if necessary */
850 if(config->proxy_key && !config->proxy_key_type &&
851 is_pkcs11_uri(config->proxy_key)) {
852 config->proxy_key_type = curlx_strdup("ENG");
853 if(!config->proxy_key_type)
854 return CURLE_OUT_OF_MEMORY;
855 }
856 }
857 return CURLE_OK;
860static CURLcode append2query(struct OperationConfig *config,
861 struct per_transfer *per,
862 const char *q)
864 CURLcode result = CURLE_OK;
865 if(q) {
866 CURLU *uh = curl_url();
867 if(uh) {
868 CURLUcode uerr;
869 uerr = curl_url_set(uh, CURLUPART_URL, per->url, CURLU_GUESS_SCHEME);
870 if(uerr) {
871 result = urlerr_cvt(uerr);
872 errorf("(%d) Could not parse the URL, "
873 "failed to set query", (int)result);
874 config->synthetic_error = TRUE;
875 }
876 else {
877 char *updated = NULL;
878 uerr = curl_url_set(uh, CURLUPART_QUERY, q, CURLU_APPENDQUERY);
879 if(!uerr)
880 uerr = curl_url_get(uh, CURLUPART_URL, &updated,
881 CURLU_NO_GUESS_SCHEME);
882 if(uerr)
883 result = urlerr_cvt(uerr);
884 else {
885 /* use our new URL instead! */
886 curlx_free(per->url); /* free previous URL */
887 /* make it allocated by tool memory */
888 per->url = curlx_strdup(updated);
889 curl_free(updated);
890 if(!per->url)
891 result = CURLE_OUT_OF_MEMORY;
892 }
893 }
894 curl_url_cleanup(uh);
895 }
896 else
897 result = CURLE_OUT_OF_MEMORY;
898 }
899 return result;
902static CURLcode etag_compare(struct OperationConfig *config)
904 CURLcode result = CURLE_OK;
905 char *etag_from_file = NULL;
906 char *header = NULL;
907 ParameterError pe;
909 /* open file for reading: */
910 FILE *file = curlx_fopen(config->etag_compare_file, FOPEN_READTEXT);
911 if(!file) {
912 char errbuf[STRERROR_LEN];
913 warnf("Failed to open %s: %s", config->etag_compare_file,
914 curlx_strerror(errno, errbuf, sizeof(errbuf)));
915 }
917 if((PARAM_OK == file2string(&etag_from_file, file)) &&
918 etag_from_file) {
919 char *h = curl_maprintf("If-None-Match: %s", etag_from_file);
920 curlx_safefree(etag_from_file);
921 if(h) {
922 /* move it to the right memory */
923 header = curlx_strdup(h);
924 curl_free(h);
925 }
926 }
927 else
928 header = curlx_strdup("If-None-Match: \"\"");
930 if(!header) {
931 if(file)
932 curlx_fclose(file);
933 errorf("Failed to allocate memory for custom etag header");
934 return CURLE_OUT_OF_MEMORY;
935 }
937 /* add Etag from file to list of custom headers */
938 pe = add2list(&config->headers, header);
939 curlx_free(header);
941 if(file)
942 curlx_fclose(file);
943 if(pe != PARAM_OK)
944 result = CURLE_OUT_OF_MEMORY;
945 return result;
948static CURLcode etag_store(struct OperationConfig *config,
949 struct OutStruct *etag_save,
950 bool *skip)
952 if(config->create_dirs) {
953 CURLcode result = create_dir_hierarchy(config->etag_save_file);
954 if(result)
955 return result;
956 }
958 /* open file for output: */
959 if(strcmp(config->etag_save_file, "-")) {
960 FILE *newfile = curlx_fopen(config->etag_save_file, "ab");
961 if(!newfile) {
962 warnf("Failed creating file for saving etags: \"%s\". "
963 "Skip this transfer", config->etag_save_file);
964 *skip = TRUE;
965 return CURLE_OK;
966 }
967 else {
968 etag_save->filename = config->etag_save_file;
969 etag_save->regular_file = TRUE;
970 etag_save->fopened = TRUE;
971 etag_save->stream = newfile;
972 }
973 }
974 else {
975 /* always use binary mode for protocol header output */
976 CURL_BINMODE(etag_save->stream);
977 }
978 return CURLE_OK;
981static CURLcode setup_headerfile(struct OperationConfig *config,
982 struct per_transfer *per,
983 struct OutStruct *heads)
985 if(config->headerfile) {
986 /* open file for output: */
987 if(!strcmp(config->headerfile, "%")) {
988 heads->stream = stderr;
989 /* use binary mode for protocol header output */
990 CURL_BINMODE(heads->stream);
991 }
992 else if(strcmp(config->headerfile, "-")) {
993 FILE *newfile;
995 /*
996 * Since every transfer has its own file handle for dumping
997 * the headers, we need to open it in append mode, since transfers
998 * might finish in any order.
999 * The first transfer clears the file.
1001 * Consider placing the file handle inside the OperationConfig, so
1002 * that it does not need to be opened/closed for every transfer.
1003 */
1004 if(config->create_dirs) {
1005 CURLcode result = create_dir_hierarchy(config->headerfile);
1006 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
1007 if(result)
1008 return result;
1010 if(!per->prev || per->prev->config != config) {
1011 newfile = curlx_fopen(config->headerfile, "wb");
1012 if(newfile)
1013 curlx_fclose(newfile);
1015 newfile = curlx_fopen(config->headerfile, "ab");
1017 if(!newfile) {
1018 errorf("Failed to open %s", config->headerfile);
1019 return CURLE_WRITE_ERROR;
1021 else {
1022 heads->filename = config->headerfile;
1023 heads->regular_file = TRUE;
1024 heads->fopened = TRUE;
1025 heads->stream = newfile;
1028 else {
1029 /* always use binary mode for protocol header output */
1030 CURL_BINMODE(heads->stream);
1033 return CURLE_OK;
1036/* figure out and populate per->outfile */
1037static CURLcode setup_outfile(struct OperationConfig *config,
1038 struct per_transfer *per,
1039 struct getout *u,
1040 struct OutStruct *outs,
1041 bool *skipped)
1043 struct State *state = &global->state;
1045 outs->out_null = u->out_null;
1046 /* if --out-null or output to stdout, return */
1047 if(outs->out_null ||
1048 (!u->useremote && (!u->outfile || !strcmp("-", u->outfile)))) {
1049 per->outfile = u->outfile;
1050 u->outfile = NULL;
1051 return CURLE_OK;
1054 if(!u->outfile) {
1055 SANITIZEcode sc;
1056 /* extract the filename from the URL */
1057 CURLcode result = get_url_file_name(&per->outfile, per->url, &sc);
1058 if(sc) {
1059 if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
1060 return CURLE_OUT_OF_MEMORY;
1061 if(sc == SANITIZE_ERR_INVALID_PATH)
1062 errorf("bad output filename");
1063 return result;
1065 else if(result) {
1066 errorf("Failed to extract a filename from the URL to use for storage");
1067 return result;
1070 else if(glob_inuse(&state->urlglob) || glob_inuse(&state->inglob)) {
1071 /* expand '#1' ... '#9' references from URL pattern and named references
1072 from the upload file glob */
1073 SANITIZEcode sc;
1074 CURLcode result =
1075 glob_match_url(&per->outfile, u->outfile, &state->urlglob,
1076 glob_inuse(&state->inglob) ? &state->inglob : NULL, &sc);
1078 if(sc) {
1079 if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
1080 return CURLE_OUT_OF_MEMORY;
1081 if(sc == SANITIZE_ERR_INVALID_PATH)
1082 warnf("bad output filename");
1083 return result;
1085 else if(result) {
1086 /* bad globbing */
1087 if(state->urlglob.error) {
1088 glob_show_error(&state->urlglob, u->outfile, tool_stderr, result);
1089 config->synthetic_error = TRUE;
1091 else
1092 warnf("bad output glob");
1093 return result;
1095 if(!*per->outfile) {
1096 warnf("output glob produces empty string");
1097 return CURLE_WRITE_ERROR;
1100 else {
1101 per->outfile = curlx_strdup(u->outfile);
1102 if(!per->outfile)
1103 return CURLE_OUT_OF_MEMORY;
1106 DEBUGASSERT(per->outfile);
1108 if(config->output_dir && *config->output_dir) {
1109 char *d = curl_maprintf("%s/%s", config->output_dir, per->outfile);
1110 curlx_safefree(per->outfile);
1111 if(d) {
1112 per->outfile = curlx_strdup(d); /* move to right memory */
1113 curl_free(d);
1115 if(!d || !per->outfile)
1116 return CURLE_WRITE_ERROR;
1118 /* Create the directory hierarchy, if not pre-existent to a multiple
1119 file output call */
1121 if(config->create_dirs) {
1122 CURLcode result = create_dir_hierarchy(per->outfile);
1123 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
1124 if(result)
1125 return result;
1128 if(config->skip_existing) {
1129 curlx_struct_stat fileinfo;
1130 if(!curlx_stat(per->outfile, &fileinfo)) {
1131 /* file is present */
1132 notef("skips transfer, \"%s\" exists locally", per->outfile);
1133 per->skip = TRUE;
1134 *skipped = TRUE;
1138 if(config->resume_from_current) {
1139 /* We are told to continue from where we are now. Get the size
1140 of the file as it is now and open it for append instead */
1141 curlx_struct_stat fileinfo;
1142 /* VMS -- Danger, the filesize is only valid for stream files */
1143 if(curlx_stat(per->outfile, &fileinfo) == 0)
1144 /* set offset to current file size: */
1145 config->resume_from = fileinfo.st_size;
1146 else
1147 /* let offset be 0 */
1148 config->resume_from = 0;
1151 if(config->resume_from) {
1152#ifdef __VMS
1153 /* open file for output, forcing VMS output format into stream
1154 mode which is needed for stat() call above to always work. */
1155 FILE *file = curlx_fopen(per->outfile, "ab",
1156 "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0");
1157#else
1158 /* open file for output: */
1159 FILE *file = curlx_fopen(per->outfile, "ab");
1160#endif
1161 if(!file) {
1162 errorf("cannot open '%s'", per->outfile);
1163 return CURLE_WRITE_ERROR;
1165 outs->fopened = TRUE;
1166 outs->stream = file;
1167 outs->init = config->resume_from;
1169 else {
1170 outs->stream = NULL; /* open when needed */
1172 outs->filename = per->outfile;
1173 outs->regular_file = TRUE;
1174 return CURLE_OK;
1177static void check_stdin_upload(struct OperationConfig *config,
1178 struct per_transfer *per)
1180 /* count to see if there are more than one auth bit set
1181 in the authtype field */
1182 int authbits = 0;
1183 int bitcheck = 0;
1184 while(bitcheck < 32) {
1185 if(config->authtype & (1UL << bitcheck++)) {
1186 authbits++;
1187 if(authbits > 1) {
1188 /* more than one, we are done! */
1189 break;
1194 /*
1195 * If the user has also selected --anyauth or --proxy-anyauth
1196 * we should warn them.
1197 */
1198 if(config->proxyanyauth || (authbits > 1)) {
1199 warnf("Using --anyauth or --proxy-anyauth with upload from stdin"
1200 " involves a big risk of it not working. Use a temporary"
1201 " file or a fixed auth type instead");
1204 DEBUGASSERT(per->infdopen == FALSE);
1205 DEBUGASSERT(per->infd == STDIN_FILENO);
1207 CURL_BINMODE(stdin);
1208 if(!strcmp(per->uploadfile, ".")) {
1209#if defined(USE_WINSOCK) && !defined(CURL_WINDOWS_UWP)
1210 /* non-blocking stdin behavior on Windows is challenging
1211 Spawn a new thread that reads from stdin and write
1212 out to a socket */
1213 curl_socket_t f = win32_stdin_read_thread();
1215 if(f == CURL_SOCKET_BAD)
1216 warnf("win32_stdin_read_thread returned INVALID_SOCKET "
1217 "falling back to blocking mode");
1218 else if(f > INT_MAX) {
1219 warnf("win32_stdin_read_thread returned identifier "
1220 "larger than INT_MAX. This should not happen unless "
1221 "the upper 32 bits of a Windows socket have started "
1222 "being used for something... falling back to blocking "
1223 "mode");
1224 sclose(f);
1226 else
1227 per->infd = (int)f;
1228#endif
1229 if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0) {
1230 char errbuf[STRERROR_LEN];
1231 warnf("fcntl failed on fd=%d: %s", per->infd,
1232 curlx_strerror(errno, errbuf, sizeof(errbuf)));
1237static void setup_header_cb(struct OperationConfig *config,
1238 struct HdrCbData *hdrcbdata,
1239 struct getout *u,
1240 struct OutStruct *outs,
1241 struct OutStruct *heads,
1242 struct OutStruct *etag)
1244 hdrcbdata->honor_cd_filename = (config->content_disposition && u->useremote);
1245 hdrcbdata->outs = outs;
1246 hdrcbdata->heads = heads;
1247 hdrcbdata->etag_save = etag;
1248 hdrcbdata->config = config;
1251static CURLcode setup_input_file(struct OperationConfig *config,
1252 struct State *state, struct getout *u,
1253 FILE *err)
1255 CURLcode result = CURLE_OK;
1256 if(u->infile) {
1257 if(!config->globoff && !glob_inuse(&state->inglob))
1258 result = glob_url(&state->inglob, u->infile, &state->upnum, err);
1259 if(!result && !state->uploadfile) {
1260 if(glob_inuse(&state->inglob) &&
1261 !glob_is_literal(&state->inglob))
1262 result = glob_next_url(&state->uploadfile, &state->inglob);
1263 else if(!state->upidx) {
1264 /* take ownership of the allocated string */
1265 state->uploadfile = u->infile;
1266 u->infile = NULL;
1270 return result;
1273static CURLcode setup_url_pattern(struct OperationConfig *config,
1274 struct State *state, struct getout *u,
1275 FILE *err)
1277 CURLcode result = CURLE_OK;
1278 if(!state->urlnum) {
1279 if(!config->globoff && !u->noglob) {
1280 /* Unless explicitly shut off, we expand '{...}' and '[...]'
1281 expressions and return total number of URLs in pattern set */
1282 result = glob_url(&state->urlglob, u->url, &state->urlnum, err);
1284 else
1285 state->urlnum = 1; /* without globbing, this is a single URL */
1287 return result;
1290static CURLcode setup_etag_files(struct OperationConfig *config,
1291 struct OutStruct *etag_save,
1292 bool *break_loop)
1294 CURLcode result = CURLE_OK;
1295 *break_loop = FALSE;
1297 /* --etag-save */
1298 memset(etag_save, 0, sizeof(*etag_save));
1299 etag_save->stream = stdout;
1301 /* --etag-compare */
1302 if(config->etag_compare_file) {
1303 result = etag_compare(config);
1304 if(result)
1305 return result;
1308 if(config->etag_save_file) {
1309 bool badetag = FALSE;
1310 result = etag_store(config, etag_save, &badetag);
1311 if(result || badetag) {
1312 *break_loop = TRUE;
1313 return result;
1316 return result;
1319static CURLcode select_next_url(struct State *state,
1320 struct getout *u,
1321 char **url)
1323 CURLcode result = CURLE_OK;
1324 if(glob_inuse(&state->urlglob) &&
1325 !glob_is_literal(&state->urlglob))
1326 result = glob_next_url(url, &state->urlglob);
1327 else if(!state->urlidx) {
1328 *url = curlx_strdup(u->url);
1329 if(!*url)
1330 result = CURLE_OUT_OF_MEMORY;
1332 else
1333 *url = NULL;
1334 return result;
1337static CURLcode setup_transfer_upload(struct OperationConfig *config,
1338 struct per_transfer *per)
1340 CURLcode result = CURLE_OK;
1341 if(per->uploadfile) {
1342 if(stdin_upload(per->uploadfile))
1343 check_stdin_upload(config, per);
1344 else {
1345 /* We have specified a file to upload and it is not "-" */
1346 result = add_file_name_to_url(per->curl, &per->url, per->uploadfile);
1347 if(result)
1348 return result;
1351 if(config->resume_from_current)
1352 config->resume_from = -1; /* -1 then forces get-it-yourself */
1354 return result;
1357/* returns TRUE if the given stream is a terminal */
1358static bool stream_isatty(FILE *stream)
1360#ifdef DEBUGBUILD
1361 /* the test suite sets CURL_ISATTY to make curl act as if the output goes
1362 to a terminal */
1363 if(getenv("CURL_ISATTY"))
1364 return TRUE;
1365#endif
1366 return !!isatty(fileno(stream));
1369/* create a transfer */
1370static CURLcode create_single(struct OperationConfig *config,
1371 CURLSH *share, struct State *state,
1372 bool *added, bool *skipped)
1374 const bool orig_isatty = (bool)global->isatty;
1375 const bool orig_noprogress = (bool)global->noprogress;
1376 CURLcode result = CURLE_OK;
1377 while(state->urlnode) {
1378 struct per_transfer *per = NULL;
1379 struct OutStruct *outs;
1380 struct OutStruct *heads;
1381 struct OutStruct etag_first;
1382 CURL *curl;
1383 struct getout *u = state->urlnode;
1384 FILE *err = (!global->silent || global->showerror) ? tool_stderr : NULL;
1385 bool break_loop = FALSE;
1387 if(!u->url) {
1388 /* This node has no URL. End of the road. */
1389 warnf("Got more output options than URLs");
1390 break;
1393 result = setup_input_file(config, state, u, err);
1394 if(result)
1395 return result;
1397 if(state->upidx >= state->upnum) {
1398 state->urlnum = 0;
1399 curlx_safefree(state->uploadfile);
1400 glob_cleanup(&state->inglob);
1401 state->upidx = 0;
1402 state->urlnode = u->next; /* next node */
1403 state->upnum = 1;
1404 continue;
1407 result = setup_url_pattern(config, state, u, err);
1408 if(result)
1409 return result;
1411 result = setup_etag_files(config, &etag_first, &break_loop);
1412 if(result || break_loop)
1413 return result;
1415 curl = curl_easy_init();
1416 if(curl)
1417 result = add_per_transfer(&per);
1418 else
1419 result = CURLE_OUT_OF_MEMORY;
1420 if(result) {
1421 curl_easy_cleanup(curl);
1422 if(etag_first.fopened)
1423 curlx_fclose(etag_first.stream);
1424 return result;
1426 per->etag_save = etag_first; /* copy the whole struct */
1427 if(state->uploadfile) {
1428 per->uploadfile = curlx_strdup(state->uploadfile);
1429 if(!per->uploadfile ||
1430 SetHTTPrequest(TOOL_HTTPREQ_PUT, &config->httpreq)) {
1431 curlx_safefree(per->uploadfile);
1432 curl_easy_cleanup(curl);
1433 return CURLE_FAILED_INIT;
1436 per->config = config;
1437 per->curl = curl;
1438 per->urlnum = u->num;
1439 per->out_scheme = NULL;
1441 /* default headers output stream is stdout */
1442 heads = &per->heads;
1443 heads->stream = stdout;
1445 /* Single header file for all URLs */
1446 result = setup_headerfile(config, per, heads);
1447 if(result)
1448 return result;
1449 outs = &per->outs;
1451 per->infdopen = FALSE;
1452 per->infd = STDIN_FILENO;
1454 /* default output stream is stdout */
1455 outs->stream = stdout;
1457 result = select_next_url(state, u, &per->url);
1458 if(result)
1459 return result;
1460 if(!per->url)
1461 break;
1463 result = setup_outfile(config, per, u, outs, skipped);
1464 if(result)
1465 return result;
1467 result = setup_transfer_upload(config, per);
1468 if(result)
1469 return result;
1471 if(!outs->out_null && output_expected(per->url, per->uploadfile) &&
1472 outs->stream && stream_isatty(outs->stream))
1473 /* we send the output to a tty, therefore we switch off the progress
1474 meter */
1475 per->noprogress = global->noprogress = global->isatty = TRUE;
1476 else {
1477 /* progress meter is per download, so restore config values */
1478 per->noprogress = global->noprogress = orig_noprogress;
1479 global->isatty = orig_isatty;
1482 result = append2query(config, per, config->httpgetfields);
1483 if(result)
1484 return result;
1486 if((!per->outfile || !strcmp(per->outfile, "-")) &&
1487 !config->use_ascii) {
1488 /* We get the output to stdout and we have not got the ASCII/text flag,
1489 then set stdout to be binary */
1490 CURL_BINMODE(stdout);
1493 /* explicitly passed to stdout means okaying binary gunk */
1494 config->terminal_binary_ok = (per->outfile && !strcmp(per->outfile, "-"));
1496 setup_header_cb(config, &per->hdrcbdata, u, outs, heads, &etag_first);
1498 result = config2setopts(config, per, curl, share);
1499 if(result)
1500 return result;
1502 /* initialize retry vars for loop below */
1503 per->retry_sleep_default = config->retry_delay_ms;
1504 per->retry_remaining = config->req_retry;
1505 per->retry_sleep = per->retry_sleep_default; /* ms */
1506 per->retrystart = curlx_now();
1508 /* Here's looping around each globbed URL */
1509 if(++state->urlidx >= state->urlnum) {
1510 state->urlidx = state->urlnum = 0;
1511 glob_cleanup(&state->urlglob);
1512 state->upidx++;
1513 curlx_safefree(state->uploadfile); /* clear it to get the next */
1515 *added = TRUE;
1516 break;
1518 return result;
1521/* create the next (singular) transfer */
1522static CURLcode single_transfer(struct OperationConfig *config,
1523 CURLSH *share, bool *added, bool *skipped)
1525 CURLcode result = CURLE_OK;
1526 struct State *state = &global->state;
1527 char *httpgetfields = config->httpgetfields;
1529 *skipped = *added = FALSE; /* not yet */
1531 if(config->postfields) {
1532 if(config->use_httpget) {
1533 if(!httpgetfields) {
1534 /* Use the postfields data for an HTTP get */
1535 httpgetfields = config->httpgetfields = config->postfields;
1536 config->postfields = NULL;
1537 if(SetHTTPrequest((config->no_body ? TOOL_HTTPREQ_HEAD :
1538 TOOL_HTTPREQ_GET), &config->httpreq))
1539 return CURLE_FAILED_INIT;
1542 else if(SetHTTPrequest(TOOL_HTTPREQ_SIMPLEPOST, &config->httpreq))
1543 return CURLE_FAILED_INIT;
1545 if(!httpgetfields)
1546 config->httpgetfields = config->query;
1548 result = set_cert_types(config);
1549 if(result)
1550 return result;
1552 if(!state->urlnode) {
1553 /* first time caller, setup things */
1554 state->urlnode = config->url_list;
1555 state->upnum = 1;
1558 return create_single(config, share, state, added, skipped);
1561static long all_added; /* number of easy handles currently added */
1564 * add_parallel_transfers() sets 'morep' to TRUE if there are more transfers
1565 * to add even after this call returns. sets 'addedp' to TRUE if one or more
1566 * transfers were added.
1567 */
1568static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share,
1569 bool *morep, bool *addedp)
1571 struct per_transfer *per;
1572 CURLcode result = CURLE_OK;
1573 CURLMcode mcode;
1574 bool sleeping = FALSE;
1575 curl_off_t nxfers;
1577 *addedp = FALSE;
1578 *morep = FALSE;
1579 mcode = curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT, &nxfers);
1580 if(mcode) {
1581 DEBUGASSERT(0);
1582 return CURLE_UNKNOWN_OPTION;
1585 if(all_added >= global->parallel_max) {
1586 /* we are at max parallelism, no need to create more transfers */
1587 *morep = TRUE; /* pretend there are, we have not checked */
1588 return CURLE_OK;
1590 /* if the list is empty, create one to kickstart the loop */
1591 if(!transfers) {
1592 bool skipped = FALSE;
1593 do {
1594 result = create_transfer(share, addedp, &skipped);
1595 if(result)
1596 return result;
1597 } while(skipped);
1599 /* add transfers until max parallelism is achieved again */
1600 for(per = transfers; per && (all_added < global->parallel_max);
1601 per = per->next) {
1602 if(per->added || per->skip)
1603 /* already added or to be skipped */
1604 continue;
1605 if(per->startat && (time(NULL) < per->startat)) {
1606 /* this is still delaying */
1607 sleeping = TRUE;
1608 continue;
1610 per->added = TRUE;
1612 result = pre_transfer(per);
1613 if(result)
1614 return result;
1616 /* parallel connect means that we do not set PIPEWAIT since pipewait
1617 makes libcurl prefer multiplexing */
1618 (void)curl_easy_setopt(per->curl, CURLOPT_PIPEWAIT,
1619 global->parallel_connect ? 0L : 1L);
1620 (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per);
1621 /* curl does not use signals, switching this on saves some system calls */
1622 (void)curl_easy_setopt(per->curl, CURLOPT_NOSIGNAL, 1L);
1623 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
1624 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per);
1625 (void)curl_easy_setopt(per->curl, CURLOPT_NOPROGRESS, 0L);
1626 (void)curl_easy_setopt(per->curl, CURLOPT_ERRORBUFFER, per->errorbuffer);
1627#ifdef DEBUGBUILD
1628 if(getenv("CURL_FORBID_REUSE"))
1629 (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
1630#endif
1632 mcode = curl_multi_add_handle(multi, per->curl);
1633 if(mcode) {
1634 DEBUGASSERT(mcode == CURLM_OUT_OF_MEMORY);
1635 result = CURLE_OUT_OF_MEMORY;
1638 if(!result) {
1639 bool getadded = FALSE;
1640 bool skipped = FALSE;
1641 do {
1642 result = create_transfer(share, &getadded, &skipped);
1643 if(result)
1644 break;
1645 } while(skipped);
1647 if(result)
1648 return result;
1650 per->errorbuffer[0] = 0;
1651 per->added = TRUE;
1652 all_added++;
1653 *addedp = TRUE;
1655 *morep = (per || sleeping);
1656 return CURLE_OK;
1659struct parastate {
1660 CURLM *multi;
1661 CURLSH *share;
1662 CURLMcode mcode;
1663 CURLcode result;
1664 int still_running;
1665 struct curltime start;
1666 bool more_transfers;
1667 bool added_transfers;
1668 /* wrapitup is set TRUE after a critical error occurs to end all transfers */
1669 bool wrapitup;
1670 /* wrapitup_processed is set TRUE after the per transfer abort flag is set */
1671 bool wrapitup_processed;
1672 time_t tick;
1675#if defined(DEBUGBUILD) && defined(USE_LIBUV)
1677#define DEBUG_UV 0
1679/* object to pass to the callbacks */
1680struct datauv {
1681 uv_timer_t timeout;
1682 uv_loop_t *loop;
1683 struct parastate *s;
1686struct contextuv {
1687 uv_poll_t poll_handle;
1688 curl_socket_t sockfd;
1689 struct datauv *uv;
1692static void mnotify(CURLM *multi, unsigned int notification,
1693 CURL *easy, void *user_data);
1695/* callback from libuv on socket activity */
1696static void on_uv_socket(uv_poll_t *req, int status, int events)
1698 int flags = 0;
1699 struct contextuv *c = (struct contextuv *)req->data;
1700 (void)status;
1701 if(events & UV_READABLE)
1702 flags |= CURL_CSELECT_IN;
1703 if(events & UV_WRITABLE)
1704 flags |= CURL_CSELECT_OUT;
1706 curl_multi_socket_action(c->uv->s->multi, c->sockfd, flags,
1707 &c->uv->s->still_running);
1708 progress_meter(c->uv->s->multi, &c->uv->s->start, FALSE);
1711/* callback from libuv when timeout expires */
1712static void on_uv_timeout(uv_timer_t *req)
1714 struct datauv *uv = (struct datauv *)req->data;
1715#if DEBUG_UV
1716 curl_mfprintf(tool_stderr, "parallel_event: on_uv_timeout\n");
1717#endif
1718 if(uv && uv->s) {
1719 curl_multi_socket_action(uv->s->multi, CURL_SOCKET_TIMEOUT, 0,
1720 &uv->s->still_running);
1721 progress_meter(uv->s->multi, &uv->s->start, FALSE);
1725/* callback from libcurl to update the timeout expiry */
1726static int cb_timeout(CURLM *multi, long timeout_ms, void *userp)
1728 struct datauv *uv = userp;
1729 (void)multi;
1730#if DEBUG_UV
1731 curl_mfprintf(tool_stderr, "parallel_event: cb_timeout=%ld\n", timeout_ms);
1732#endif
1733 if(timeout_ms < 0)
1734 uv_timer_stop(&uv->timeout);
1735 else {
1736 if(timeout_ms == 0)
1737 timeout_ms = 1; /* 0 means call curl_multi_socket_action asap but NOT
1738 within the callback itself */
1739 uv_timer_start(&uv->timeout, on_uv_timeout, timeout_ms,
1740 0); /* do not repeat */
1742 return 0;
1745static struct contextuv *create_context(curl_socket_t sockfd,
1746 struct datauv *uv)
1748 struct contextuv *c = curlx_malloc(sizeof(*c));
1750 c->sockfd = sockfd;
1751 c->uv = uv;
1753 uv_poll_init_socket(uv->loop, &c->poll_handle, sockfd);
1754 c->poll_handle.data = c;
1756 return c;
1759static void close_cb(uv_handle_t *handle)
1761 struct contextuv *c = (struct contextuv *)handle->data;
1762 curlx_free(c);
1765static void destroy_context(struct contextuv *c)
1767 uv_close((uv_handle_t *)&c->poll_handle, close_cb);
1770/* callback from libcurl to update socket activity to wait for */
1771static int cb_socket(CURL *easy, curl_socket_t s, int action,
1772 void *userp, void *socketp)
1774 struct contextuv *c;
1775 int events = 0;
1776 struct datauv *uv = userp;
1777 (void)easy;
1779#if DEBUG_UV
1780 curl_mfprintf(tool_stderr, "parallel_event: cb_socket, "
1781 "fd=%" FMT_SOCKET_T ", action=%x, p=%p\n", s, action, socketp);
1782#endif
1783 switch(action) {
1784 case CURL_POLL_IN:
1785 case CURL_POLL_OUT:
1786 case CURL_POLL_INOUT:
1787 c = socketp ? (struct contextuv *)socketp : create_context(s, uv);
1789 curl_multi_assign(uv->s->multi, s, c);
1791 if(action != CURL_POLL_IN)
1792 events |= UV_WRITABLE;
1793 if(action != CURL_POLL_OUT)
1794 events |= UV_READABLE;
1796 uv_poll_start(&c->poll_handle, events, on_uv_socket);
1797 break;
1798 case CURL_POLL_REMOVE:
1799 if(socketp) {
1800 c = (struct contextuv *)socketp;
1801 uv_poll_stop(&c->poll_handle);
1802 destroy_context(c);
1803 curl_multi_assign(uv->s->multi, s, NULL);
1805 break;
1806 default:
1807 DEBUGASSERT(0);
1808 return -1;
1811 return 0;
1814static CURLcode parallel_event(struct parastate *s)
1816 CURLcode result = CURLE_OK;
1817 struct datauv uv = { 0 };
1819 s->result = CURLE_OK;
1820 uv.s = s;
1821 uv.loop = uv_default_loop();
1822 uv_timer_init(uv.loop, &uv.timeout);
1823 uv.timeout.data = &uv;
1825 /* setup event callbacks */
1826 curl_multi_setopt(s->multi, CURLMOPT_SOCKETFUNCTION, cb_socket);
1827 curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv);
1828 curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout);
1829 curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv);
1830 curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS, (long)
1831 global->parallel_host);
1833 /* kickstart the thing */
1834 curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0,
1835 &s->still_running);
1837 while(!s->mcode && (s->still_running || s->more_transfers)) {
1839 if(!s->still_running && s->more_transfers) {
1840 curl_off_t nxfers;
1841 /* None are running, but there may be more ready to run. Try
1842 * to add more when possible. */
1843 s->result = add_parallel_transfers(s->multi, s->share,
1844 &s->more_transfers,
1845 &s->added_transfers);
1846 if(s->result)
1847 break;
1848 if(s->more_transfers) {
1849 /* There are still more transfers, so not all were added. If we
1850 * did not add *any*, it means there is nothing to do until time
1851 * passes and a transfer becomes ready to add.
1852 * Avoid busy looping by taking a small break. */
1853 CURLMcode mcode = curl_multi_get_offt(
1854 s->multi, CURLMINFO_XFERS_CURRENT, &nxfers);
1855 if(!mcode && !nxfers) {
1856 /* None have been added, must be waiting on a timeout */
1857 curlx_wait_ms(500);
1861#if DEBUG_UV
1862 curl_mfprintf(tool_stderr, "parallel_event: uv_run(), "
1863 "mcode=%d, %d running, %d more\n",
1864 s->mcode, uv.s->still_running, s->more_transfers);
1865#endif
1866 uv_run(uv.loop, UV_RUN_DEFAULT);
1867#if DEBUG_UV
1868 curl_mfprintf(tool_stderr, "parallel_event: uv_run() returned\n");
1869#endif
1871 /* early exit called */
1872 if(s->wrapitup) {
1873 if(s->still_running && !s->wrapitup_processed) {
1874 struct per_transfer *per;
1875 for(per = transfers; per; per = per->next) {
1876 if(per->added)
1877 per->abort = TRUE;
1879 s->wrapitup_processed = TRUE;
1881 break;
1885 result = s->result;
1887 /* Make sure to return some kind of error if there was a multi problem */
1888 if(s->mcode) {
1889 result = (s->mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
1890 /* The other multi errors should never happen, so return
1891 something suitably generic */
1892 CURLE_BAD_FUNCTION_ARGUMENT;
1895 /* We need to cleanup the multi here, since the uv context lives on the
1896 * stack and about to be gone. multi_cleanup can trigger events! */
1897 (void)progress_meter(s->multi, &s->start, TRUE);
1898 curl_multi_cleanup(s->multi);
1900#if DEBUG_UV
1901 curl_mfprintf(tool_stderr, "DONE parallel_event -> %d, mcode=%d, "
1902 "%d running, %d more\n",
1903 result, s->mcode, uv.s->still_running, s->more_transfers);
1904#endif
1905 return result;
1908#endif
1910static CURLcode check_finished(struct parastate *s)
1912 CURLcode result = CURLE_OK;
1913 int rc;
1914 CURLMsg *msg;
1915 bool checkmore = FALSE;
1916 do {
1917 msg = curl_multi_info_read(s->multi, &rc);
1918 if(msg) {
1919 bool retry;
1920 uint32_t delay;
1921 struct per_transfer *ended;
1922 CURL *easy = msg->easy_handle;
1923 CURLcode tres = msg->data.result;
1924 curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ended);
1925 curl_multi_remove_handle(s->multi, easy);
1927 if(ended->abort && (tres == CURLE_ABORTED_BY_CALLBACK)) {
1928 curl_msnprintf(ended->errorbuffer, CURL_ERROR_SIZE,
1929 "Transfer aborted due to critical error "
1930 "in another transfer");
1932 tres = post_per_transfer(ended, tres, &retry, &delay);
1933 all_added--; /* one fewer added */
1934 checkmore = TRUE;
1935 if(retry) {
1936 progress_reset_retry(ended);
1937 ended->added = FALSE; /* add it again */
1938 /* we delay retries in full integer seconds only */
1939 ended->startat = delay ? time(NULL) + (delay / 1000) : 0;
1941 else {
1942 progress_finalize(ended); /* before it goes away */
1943 /* result receives this transfer's error unless the transfer was
1944 marked for abort due to a critical error in another transfer */
1945 if(tres && (!ended->abort || !result))
1946 result = tres;
1947 if(is_fatal_error(result) || (result && global->fail_early))
1948 s->wrapitup = TRUE;
1949 (void)del_per_transfer(ended);
1952 } while(msg);
1953 if(!s->wrapitup) {
1954 if(!checkmore) {
1955 time_t tock = time(NULL);
1956 if(s->tick != tock) {
1957 checkmore = TRUE;
1958 s->tick = tock;
1961 if(checkmore) {
1962 /* one or more transfers completed, add more! */
1963 CURLcode tres = add_parallel_transfers(s->multi, s->share,
1964 &s->more_transfers,
1965 &s->added_transfers);
1966 if(tres)
1967 result = tres;
1968 if(s->added_transfers)
1969 /* we added new ones, make sure the loop does not exit yet */
1970 s->still_running = 1;
1972 if(is_fatal_error(result) || (result && global->fail_early))
1973 s->wrapitup = TRUE;
1975 return result;
1978static void mnotify(CURLM *multi, unsigned int notification,
1979 CURL *easy, void *user_data)
1981 struct parastate *s = user_data;
1982 CURLcode result;
1984 (void)multi;
1985 (void)easy;
1987 switch(notification) {
1988 case CURLMNOTIFY_INFO_READ:
1989 result = check_finished(s);
1990 /* remember first failure */
1991 if(result && !s->result)
1992 s->result = result;
1993 break;
1994 default:
1995 break;
1999static CURLcode parallel_transfers(CURLSH *share)
2001 CURLcode result;
2002 struct parastate p;
2003 struct parastate *s = &p;
2004 s->share = share;
2005 s->mcode = CURLM_OK;
2006 s->result = CURLE_OK;
2007 s->still_running = 1;
2008 s->start = curlx_now();
2009 s->wrapitup = FALSE;
2010 s->wrapitup_processed = FALSE;
2011 s->tick = time(NULL);
2012 s->multi = curl_multi_init();
2013 if(!s->multi)
2014 return CURLE_OUT_OF_MEMORY;
2016 if(TRUE
2017#ifdef DEBUGBUILD
2018 && getenv("CURL_QUICK_EXIT")
2019#endif
2020 ) {
2021 /* QUICK_EXIT allows for running threads to be detached and not
2022 * joined. Preferably in non-debug runs. */
2023 (void)curl_multi_setopt(s->multi, CURLMOPT_QUICK_EXIT, 1L);
2025 (void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYFUNCTION, mnotify);
2026 (void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYDATA, s);
2027 (void)curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS, (long)
2028 global->parallel_host);
2029 (void)curl_multi_notify_enable(s->multi, CURLMNOTIFY_INFO_READ);
2031 result = add_parallel_transfers(s->multi, s->share,
2032 &s->more_transfers, &s->added_transfers);
2033 if(result) {
2034 curl_multi_cleanup(s->multi);
2035 return result;
2038#ifdef DEBUGBUILD
2039 if(global->test_event_based)
2040#ifdef USE_LIBUV
2041 return parallel_event(s);
2042#else
2043 errorf("Testing --parallel event-based requires libuv");
2044#endif
2045 else
2046#endif
2048 if(all_added) {
2049 while(!s->mcode && (s->still_running || s->more_transfers)) {
2050 /* If stopping prematurely (eg due to a --fail-early condition) then
2051 signal that any transfers in the multi should abort (via progress
2052 callback). */
2053 if(s->wrapitup) {
2054 if(!s->still_running)
2055 break;
2056 if(!s->wrapitup_processed) {
2057 struct per_transfer *per;
2058 for(per = transfers; per; per = per->next) {
2059 if(per->added)
2060 per->abort = TRUE;
2062 s->wrapitup_processed = TRUE;
2065 else if(s->more_transfers) {
2066 s->result = add_parallel_transfers(s->multi, s->share,
2067 &s->more_transfers,
2068 &s->added_transfers);
2069 if(s->result)
2070 break;
2073 s->mcode = curl_multi_poll(s->multi, NULL, 0, 1000, NULL);
2074 if(!s->mcode)
2075 s->mcode = curl_multi_perform(s->multi, &s->still_running);
2077 progress_meter(s->multi, &s->start, FALSE);
2080 (void)progress_meter(s->multi, &s->start, TRUE);
2083 /* Result is the first failed transfer - if there was one. */
2084 result = s->result;
2086 /* Make sure to return some kind of error if there was a multi problem */
2087 if(s->mcode) {
2088 result = (s->mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
2089 /* The other multi errors should never happen, so return
2090 something suitably generic */
2091 CURLE_BAD_FUNCTION_ARGUMENT;
2094 curl_multi_cleanup(s->multi);
2096 return result;
2099static CURLcode serial_transfers(CURLSH *share)
2101 CURLcode returncode = CURLE_OK;
2102 CURLcode result = CURLE_OK;
2103 struct per_transfer *per;
2104 bool added = FALSE;
2105 bool skipped = FALSE;
2107 result = create_transfer(share, &added, &skipped);
2108 if(result)
2109 return result;
2110 if(!added) {
2111 errorf("no transfer performed");
2112 return CURLE_READ_ERROR;
2114 for(per = transfers; per;) {
2115 bool retry;
2116 uint32_t delay_ms;
2117 bool bailout = FALSE;
2118 struct curltime start;
2120 start = curlx_now();
2121 if(!per->skip) {
2122 result = pre_transfer(per);
2123 if(result)
2124 break;
2126 if(global->libcurl) {
2127 result = easysrc_perform();
2128 if(result)
2129 break;
2132#ifdef DEBUGBUILD
2133 if(getenv("CURL_FORBID_REUSE"))
2134 (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
2136 if(global->test_duphandle) {
2137 CURL *dup = curl_easy_duphandle(per->curl);
2138 curl_easy_cleanup(per->curl);
2139 per->curl = dup;
2140 if(!dup) {
2141 result = CURLE_OUT_OF_MEMORY;
2142 break;
2144 /* a duplicate needs the share re-added */
2145 (void)curl_easy_setopt(per->curl, CURLOPT_SHARE, share);
2147 if(global->test_event_based)
2148 result = curl_easy_perform_ev(per->curl);
2149 else
2150#endif
2151 result = curl_easy_perform(per->curl);
2154 returncode = post_per_transfer(per, result, &retry, &delay_ms);
2155 if(retry) {
2156 curlx_wait_ms(delay_ms);
2157 continue;
2160 /* Bail out upon critical errors or --fail-early */
2161 if(is_fatal_error(returncode) || (returncode && global->fail_early))
2162 bailout = TRUE;
2163 else {
2164 do {
2165 /* setup the next one before we delete this */
2166 result = create_transfer(share, &added, &skipped);
2167 if(result) {
2168 returncode = result;
2169 bailout = TRUE;
2170 break;
2172 } while(skipped);
2175 per = del_per_transfer(per);
2177 if(bailout)
2178 break;
2180 if(per && global->ms_per_transfer) {
2181 /* how long time did the most recent transfer take in number of
2182 milliseconds */
2183 timediff_t milli = curlx_timediff_ms(curlx_now(), start);
2184 if(milli < global->ms_per_transfer) {
2185 notef("Transfer took %" CURL_FORMAT_CURL_OFF_T " ms, "
2186 "waits %ldms as set by --rate",
2187 milli, (long)(global->ms_per_transfer - milli));
2188 /* The transfer took less time than wanted. Wait a little. */
2189 curlx_wait_ms((long)(global->ms_per_transfer - milli));
2193 if(returncode)
2194 result = returncode; /* returncode errors have priority */
2196 if(result)
2197 single_transfer_cleanup();
2199 return result;
2202#ifdef _WIN32
2203/* returns TRUE if using Schannel or if there is an error, passes back result
2204 in 'resultp' */
2205static bool win32_using_schannel(CURLcode *resultp)
2207 static int using_schannel = -1; /* -1 = not checked
2208 0 = nope
2209 1 = yes */
2210 *resultp = CURLE_OK;
2211 if(using_schannel == -1) {
2212 CURL *curl = curl_easy_init();
2213 /* The TLS backend remains, so keep the info */
2214 const struct curl_tlssessioninfo *tls_backend_info = NULL;
2216 if(!curl)
2217 *resultp = CURLE_OUT_OF_MEMORY;
2218 else {
2219 *resultp = curl_easy_getinfo(curl, CURLINFO_TLS_SSL_PTR,
2220 &tls_backend_info);
2221 if(!*resultp)
2222 using_schannel =
2223 (tls_backend_info->backend == CURLSSLBACKEND_SCHANNEL);
2225 curl_easy_cleanup(curl);
2226 if(*resultp)
2227 return TRUE;
2229 return using_schannel == 1;
2232static CURLcode win32_setup_certs(struct OperationConfig *config)
2234 if(!config->capath && !config->cacert) {
2235#ifdef CURL_CA_SEARCH_SAFE
2236 char *cacert = NULL;
2237 FILE *cafile = tool_execpath("curl-ca-bundle.crt", &cacert);
2238 if(cafile) {
2239 curlx_fclose(cafile);
2240 config->cacert = curlx_strdup(cacert);
2241 if(!config->cacert)
2242 return CURLE_OUT_OF_MEMORY;
2244#elif !defined(CURL_WINDOWS_UWP) && !defined(CURL_DISABLE_CA_SEARCH)
2245 CURLcode result = FindWin32CACert(config, TEXT("curl-ca-bundle.crt"));
2246 if(result)
2247 return result;
2248#endif
2250 return CURLE_OK;
2252#else
2253#define win32_setup_certs(x) CURLE_OK
2254#define win32_using_schannel(x) FALSE
2255#endif
2257/* Set the CA cert locations specified in the environment. For Windows if no
2258 * environment-specified filename is found then check for CA bundle default
2259 * filename curl-ca-bundle.crt in the user's PATH.
2261 * If the user has set a CA cert/path or disabled peer verification (including
2262 * for DoH, so completely disabled) then these locations are ignored.
2264 * If Schannel is the selected SSL backend then these locations are ignored.
2265 * We allow setting CA location for Schannel only when explicitly specified by
2266 * the user via CURLOPT_CAINFO / --cacert.
2267 */
2268static CURLcode cacertpaths(struct OperationConfig *config)
2270 char *env;
2271 CURLcode result = CURLE_OK;
2273 if(!feature_ssl || config->cacert || config->capath ||
2274 (config->insecure_ok && (!config->doh_url || config->doh_insecure_ok)))
2275 return CURLE_OK;
2277 if(win32_using_schannel(&result))
2278 return result;
2280 env = curl_getenv("CURL_CA_BUNDLE");
2281 if(env) {
2282 config->cacert = curlx_strdup(env);
2283 curl_free(env);
2284 if(!config->cacert)
2285 result = CURLE_OUT_OF_MEMORY;
2287 else {
2288 env = curl_getenv("SSL_CERT_DIR");
2289 if(env) {
2290 config->capath = curlx_strdup(env);
2291 curl_free(env);
2292 if(!config->capath)
2293 result = CURLE_OUT_OF_MEMORY;
2295 if(!result) {
2296 env = curl_getenv("SSL_CERT_FILE");
2297 if(env) {
2298 config->cacert = curlx_strdup(env);
2299 curl_free(env);
2300 if(!config->cacert)
2301 result = CURLE_OUT_OF_MEMORY;
2306 if(!result)
2307 result = win32_setup_certs(config);
2308 if(result) {
2309 curlx_safefree(config->capath);
2310 curlx_safefree(config->cacert);
2312 return result;
2315/* setup a transfer for the given config */
2316static CURLcode transfer_per_config(struct OperationConfig *config,
2317 CURLSH *share,
2318 bool *added,
2319 bool *skipped)
2321 CURLcode result;
2322 *added = FALSE;
2324 /* Check we have a URL */
2325 if(!config->url_list || !config->url_list->url) {
2326 helpf("(%d) no URL specified", CURLE_FAILED_INIT);
2327 result = CURLE_FAILED_INIT;
2329 else {
2330 result = cacertpaths(config);
2331 if(!result) {
2332 result = single_transfer(config, share, added, skipped);
2333 if(!*added || result)
2334 single_transfer_cleanup();
2338 return result;
2342 * 'create_transfer' gets the details and sets up a new transfer if 'added'
2343 * returns TRUE.
2344 */
2345static CURLcode create_transfer(CURLSH *share,
2346 bool *added,
2347 bool *skipped)
2349 CURLcode result = CURLE_OK;
2350 *added = FALSE;
2351 while(global->current) {
2352 result = transfer_per_config(global->current, share, added, skipped);
2353 if(!result && !*added) {
2354 /* when one set is drained, continue to next */
2355 global->current = global->current->next;
2356 continue;
2358 break;
2360 return result;
2363static CURLcode run_all_transfers(CURLSH *share,
2364 CURLcode result)
2366 /* Save the values of noprogress and isatty to restore them later on */
2367 bool orig_noprogress = (bool)global->noprogress;
2368 bool orig_isatty = (bool)global->isatty;
2369 struct per_transfer *per;
2371 /* Time to actually do the transfers */
2372 if(!result) {
2373 if(global->parallel)
2374 result = parallel_transfers(share);
2375 else
2376 result = serial_transfers(share);
2379 /* cleanup if there are any left */
2380 for(per = transfers; per;) {
2381 bool retry;
2382 uint32_t delay;
2383 CURLcode result2;
2384 /* No further retries can be performed during final cleanup. */
2385 per->retry_remaining = 0;
2386 result2 = post_per_transfer(per, result, &retry, &delay);
2387 if(!result)
2388 /* do not overwrite the original error */
2389 result = result2;
2391 /* Free list of given URLs */
2392 clean_getout(per->config);
2394 per = del_per_transfer(per);
2397 /* Reset the global config variables */
2398 global->noprogress = orig_noprogress;
2399 global->isatty = orig_isatty;
2401 return result;
2404static CURLcode share_setopt(CURLSH *share, int opt)
2406 CURLSHcode shres = curl_share_setopt(share, CURLSHOPT_SHARE, opt);
2407 if(!shres || (shres == CURLSHE_NOT_BUILT_IN))
2408 return CURLE_OK;
2409 return CURLE_FAILED_INIT;
2412static CURLcode share_setup(CURLSH *share)
2414 CURLcode result = CURLE_OK;
2415 int i;
2416 static int options[7] = {
2417 CURL_LOCK_DATA_COOKIE,
2418 CURL_LOCK_DATA_DNS,
2419 CURL_LOCK_DATA_SSL_SESSION,
2420 CURL_LOCK_DATA_PSL,
2421 CURL_LOCK_DATA_HSTS,
2422 0, /* 5 might be set below */
2424 };
2425 /* Running parallel, use the multi connection cache */
2426 if(!global->parallel)
2427 options[5] = CURL_LOCK_DATA_CONNECT;
2428 for(i = 0; !result && options[i]; i++)
2429 result = share_setopt(share, options[i]);
2430 return result;
2433CURLcode operate(int argc, argv_item_t argv[])
2435 CURLcode result = CURLE_OK;
2436 const char *first_arg;
2437 char *curlrc_path = NULL;
2438 bool found_curlrc = FALSE;
2440 first_arg = argc > 1 ? convert_tchar_to_UTF8(argv[1]) : NULL;
2442#ifdef HAVE_SETLOCALE
2443 /* Override locale for number parsing (only) */
2444 setlocale(LC_ALL, "");
2445 setlocale(LC_NUMERIC, "C");
2446#endif
2448 /* Parse .curlrc if necessary */
2449 if((argc == 1) ||
2450 (first_arg && strncmp(first_arg, "-q", 2) &&
2451 strcmp(first_arg, "--disable"))) {
2452 if(!parseconfig(NULL, CONFIG_MAX_LEVELS, &curlrc_path))
2453 found_curlrc = TRUE;
2455 /* If we had no arguments then make sure a URL was specified in .curlrc */
2456 if((argc < 2) && (!global->first->url_list)) {
2457 helpf(NULL);
2458 result = CURLE_FAILED_INIT;
2462 unicodefree(CURL_UNCONST(first_arg));
2464 if(!result) {
2465 /* Parse the command line arguments */
2466 ParameterError err = parse_args(argc, argv);
2467 if(found_curlrc) {
2468 /* After parse_args so notef knows the verbosity */
2469 notef("Read config file from '%s'", curlrc_path);
2471 if(err) {
2472 result = CURLE_OK;
2474 /* Check if we were asked for the help */
2475 if(err == PARAM_HELP_REQUESTED)
2476 ; /* already done */
2477 /* Check if we were asked for the manual */
2478 else if(err == PARAM_MANUAL_REQUESTED) {
2479#ifdef USE_MANUAL
2480 hugehelp();
2481#else
2482 warnf("built-in manual was disabled at build-time");
2483#endif
2485 /* Check if we were asked for the version information */
2486 else if(err == PARAM_VERSION_INFO_REQUESTED)
2487 tool_version_info();
2488 /* Check if we were asked to list the SSL engines */
2489 else if(err == PARAM_ENGINES_REQUESTED)
2490 tool_list_engines();
2491 /* Check if we were asked to dump the embedded CA bundle */
2492 else if(err == PARAM_CA_EMBED_REQUESTED) {
2493#ifdef CURL_CA_EMBED
2494 curl_mprintf("%s", curl_ca_embed);
2495#endif
2497 else if(err == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL)
2498 result = CURLE_UNSUPPORTED_PROTOCOL;
2499 else if(err == PARAM_READ_ERROR)
2500 result = CURLE_READ_ERROR;
2501 else
2502 result = CURLE_FAILED_INIT;
2504 else {
2505 if(global->libcurl) {
2506 /* Initialize the libcurl source output */
2507 result = easysrc_init();
2510 /* Perform the main operations */
2511 if(!result) {
2512 size_t count = 0;
2513 struct OperationConfig *operation = global->first;
2514 CURLSH *share = curl_share_init();
2515 if(!share) {
2516 if(global->libcurl) {
2517 /* Cleanup the libcurl source output */
2518 easysrc_cleanup();
2520 result = CURLE_OUT_OF_MEMORY;
2523 if(!result)
2524 result = share_setup(share);
2526 if(!result && global->ssl_sessions && feature_ssls_export)
2527 result = tool_ssls_load(global->first, share,
2528 global->ssl_sessions);
2530 if(!result) {
2531 /* Get the required arguments for each operation */
2532 do {
2533 result = get_args(operation, count++);
2535 operation = operation->next;
2536 } while(!result && operation);
2538 if(!result) {
2539 /* Set the current operation pointer */
2540 global->current = global->first;
2542 /* now run! */
2543 result = run_all_transfers(share, result);
2545 if(global->ssl_sessions && feature_ssls_export) {
2546 CURLcode r2 = tool_ssls_save(global->first, share,
2547 global->ssl_sessions);
2548 if(r2 && !result)
2549 result = r2;
2554 curl_share_cleanup(share);
2555 if(global->libcurl) {
2556 /* Cleanup the libcurl source output */
2557 easysrc_cleanup();
2559 /* Dump the libcurl code if previously enabled */
2560 dumpeasysrc();
2563 else
2564 errorf("out of memory");
2567 curlx_free(curlrc_path);
2568 varcleanup();
2570 return result;
Diagnostics4 git calls · 42 ms
CallStatusSizeTime042 ms
discover200152 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit13 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=13
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=13
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch20023 KiB · hit26 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=26
git.response.bytes=23656
git.response.cache=HIT
git.response.duration_ms=26
git.fetch.wants=dfd03f82dfcc5c3aeb6c453dd6673f963fa07659
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=23656
git.pkt.payload.max=23635
git.sideband.band1.bytes=23634
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e15f34bdea725b
Request
29bd7dd6…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format