Artifactspublic-v0

curl

master
ply4 git calls220 ms296 KiB2 cachedprotocol v2
1098 lines34 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Linus Nielsen Feltzing, <linus@haxx.se>
9 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
10 *
11 * This software is licensed as described in the file COPYING, which
12 * you should have received as part of this distribution. The terms
13 * are also available at https://curl.se/docs/copyright.html.
14 *
15 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
16 * copies of the Software, and permit persons to whom the Software is
17 * furnished to do so, under the terms of the COPYING file.
18 *
19 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20 * KIND, either express or implied.
21 *
22 * SPDX-License-Identifier: curl
23 *
24 ***************************************************************************/
25#include "curl_setup.h"
27#include "urldata.h"
28#include "url.h"
29#include "cfilters.h"
30#include "progress.h"
31#include "multiif.h"
32#include "curl_trc.h"
33#include "cshutdn.h"
34#include "conncache.h"
35#include "curl_share.h"
36#include "sigpipe.h"
37#include "uint-spbset.h"
40#define CPOOL_IS_LOCKED(c) ((c) && (c)->locked)
42#define CPOOL_LOCK(c, d) \
43 do { \
44 if(c) { \
45 if(CURL_SHARE_KEEP_CONNECT((c)->share)) \
46 Curl_share_lock_share((c)->share, (d), CURL_LOCK_DATA_CONNECT, \
47 CURL_LOCK_ACCESS_SINGLE); \
48 DEBUGASSERT(!(c)->locked); \
49 (c)->locked = TRUE; \
50 } \
51 } while(0)
53#define CPOOL_UNLOCK(c, d) \
54 do { \
55 if(c) { \
56 DEBUGASSERT((c)->locked); \
57 (c)->locked = FALSE; \
58 if(CURL_SHARE_KEEP_CONNECT((c)->share)) \
59 Curl_share_unlock_share((c)->share, (d), CURL_LOCK_DATA_CONNECT); \
60 } \
61 } while(0)
63/* A set of connection pool ids to the same destination. */
64struct cpool_bundle {
65 struct uint32_spbset ids; /* bitset of connection `pool_id`s */
66 uint32_t refcount;
67};
69static void cpool_bundle_unlink(struct cpool_bundle **pbundle)
71 if(*pbundle) {
72 struct cpool_bundle *bundle = *pbundle;
74 DEBUGASSERT(bundle->refcount);
75 *pbundle = NULL;
76 if(bundle->refcount)
77 bundle->refcount--;
78 if(!bundle->refcount) {
79 Curl_uint32_spbset_destroy(&bundle->ids);
80 curlx_free(bundle);
81 }
82 }
85static void cpool_bundle_link(struct cpool_bundle **pdest,
86 struct cpool_bundle *src)
88 if(*pdest != src) {
89 cpool_bundle_unlink(pdest);
90 *pdest = src;
91 if(src) {
92 DEBUGASSERT(src->refcount < UINT32_MAX);
93 src->refcount++;
94 }
95 }
98static struct cpool_bundle *cpool_create_bundle(void)
100 /* created with refcount == 1 */
101 struct cpool_bundle *bundle = curlx_calloc(1, sizeof(*bundle));
102 if(!bundle)
103 return NULL;
105 Curl_uint32_spbset_init(&bundle->ids);
106 bundle->refcount = 1;
107 return bundle;
110/* Add a connection to a bundle */
111static bool cpool_bundle_add_id(struct cpool *cpool,
112 struct cpool_bundle *bundle,
113 const char *destination,
114 uint32_t pool_id)
116 if(!Curl_uint32_spbset_add(&bundle->ids, pool_id))
117 return FALSE;
118 if(Curl_uint32_spbset_count(&bundle->ids) == 1) {
119 /* first one added, register bundle */
120 struct cpool_bundle *new_ref = NULL;
121 cpool_bundle_link(&new_ref, bundle);
122 if(!Curl_hash_add(&cpool->dest2bundle,
123 destination, strlen(destination), new_ref)) {
124 cpool_bundle_unlink(&new_ref);
125 return FALSE;
126 }
127 }
128 return TRUE;
131/* Remove a connection from a bundle */
132static void cpool_bundle_remove_id(struct cpool *cpool,
133 struct cpool_bundle *bundle,
134 const char *destination,
135 uint32_t pool_id)
137 Curl_uint32_spbset_remove(&bundle->ids, pool_id);
138 if(!Curl_uint32_spbset_count(&bundle->ids)) {
139 /* Now empty, unregister bundle */
140 Curl_hash_delete(&cpool->dest2bundle, destination, strlen(destination));
141 }
144static void cpool_bundle_hash_dtor(void *entry)
146 struct cpool_bundle *bundle = entry;
147 cpool_bundle_unlink(&bundle);
150static struct cpool_bundle *cpool_get_bundle(struct cpool *cpool,
151 const char *destination)
153 struct cpool_bundle *bundle = NULL;
154 cpool_bundle_link(&bundle, Curl_hash_pick(
155 &cpool->dest2bundle, destination, strlen(destination)));
156 return bundle;
159static void cpool_remove_conn(struct cpool *cpool,
160 struct connectdata *conn)
162 uint32_t cpid = conn->cpid;
164 DEBUGASSERT(cpool);
165 if(cpid != UINT32_MAX) {
166 struct cpool_bundle *bundle;
168#ifdef DEBUGBUILD
169 {
170 struct connectdata *pooled_conn =
171 Curl_uint32_tbl_get(&cpool->conns, cpid);
172 DEBUGASSERT(conn == pooled_conn);
173 }
174#endif
175 Curl_uint32_tbl_remove(&cpool->conns, cpid);
176 Curl_uint32_bset_remove(&cpool->idles, cpid);
177 conn->cpid = UINT32_MAX;
179 bundle = cpool_get_bundle(cpool, conn->destination);
180 DEBUGASSERT(bundle);
181 if(bundle)
182 cpool_bundle_remove_id(cpool, bundle, conn->destination, cpid);
183 cpool_bundle_unlink(&bundle);
184 }
187static void cpool_discard_conn(struct cpool *cpool,
188 struct Curl_easy *data,
189 struct connectdata *conn,
190 bool aborted)
192 struct Curl_easy *admin;
193 bool done = FALSE;
195 DEBUGASSERT(data);
196 DEBUGASSERT(!data->conn);
197 DEBUGASSERT(cpool);
198 DEBUGASSERT(conn->cpid == UINT32_MAX);
200 admin = Curl_get_admin(data);
201 /* We MUST NOT discard a connection that is still in use and would
202 * trigger Use-After-Free. */
203 if(CONN_INUSE(conn)) {
204 CURL_TRC_M(admin, "[CPOOL] not discarding #%" FMT_OFF_T
205 " still in use by %u transfers", conn->connection_id,
206 conn->attached_xfers);
207 DEBUGASSERT(0);
208 return;
209 }
211 /* treat the connection as aborted in CONNECT_ONLY situations, we do
212 * not know what the APP did with it. */
213 if(conn->bits.connect_only)
214 aborted = TRUE;
215 conn->bits.aborted = aborted;
217 /* We do not shutdown dead connections. The term 'dead' can be misleading
218 * here, as we also mark errored connections/transfers as 'dead'.
219 * If we do a shutdown for an aborted transfer, the server might think
220 * it was successful otherwise (for example an ftps: upload). This is
221 * not what we want. */
222 if(aborted)
223 done = TRUE;
224 if(!done) {
225 /* Attempt to shutdown the connection right away. */
226 Curl_cshutdn_try_once(admin, conn, &done);
227 }
229 if(done || !data->multi)
230 Curl_cshutdn_terminate(admin, conn, FALSE);
231 else {
232 struct Curl_multi *multi = data->multi;
233 size_t max_shutdowns = multi->max_total_connections;
234 uint32_t num_conns = Curl_uint32_tbl_count(&cpool->conns);
236 if(num_conns < max_shutdowns)
237 max_shutdowns -= num_conns;
238 else if(max_shutdowns)
239 max_shutdowns = 1;
240 else /* no connection limit set, let's restrict growth nevertheless */
241 max_shutdowns = CURLMAX(num_conns / 4, 128);
242 Curl_cshutdn_add(&multi->cshutdn, multi, conn, max_shutdowns);
243 }
246void Curl_cpool_init(struct cpool *cpool,
247 struct Curl_share *share,
248 size_t size)
250 Curl_uint32_tbl_init(&cpool->conns);
251 Curl_uint32_bset_init(&cpool->idles);
252 Curl_hash_init(&cpool->dest2bundle, size, CURL_HASH_TYPE_BYTES,
253 cpool_bundle_hash_dtor);
254 cpool->share = share;
255 cpool->initialized = TRUE;
256 cpool->in_shutdown = FALSE;
257 cpool->locked = FALSE;
260void Curl_cpool_destroy(struct cpool *cpool, struct Curl_easy *admin)
262 if(cpool && cpool->initialized && admin) {
263 void *entry;
264 uint32_t cpid;
266 CURL_TRC_M(admin, "%s[CPOOL] destroy, %u connections",
267 cpool->share ? "[SHARE] " : "",
268 Curl_uint32_tbl_count(&cpool->conns));
269 /* Move all connections to the shutdown list */
270 CPOOL_LOCK(cpool, admin);
271 if(Curl_uint32_tbl_first(&cpool->conns, &cpid, &entry)) {
272 struct Curl_sigpipe_ctx pipe_ctx;
274 sigpipe_init(&pipe_ctx);
275 sigpipe_apply(admin, &pipe_ctx);
276 do {
277 struct connectdata *conn = entry;
278 cpool_remove_conn(cpool, conn);
279 cpool_discard_conn(cpool, admin, conn, FALSE);
280 } while(Curl_uint32_tbl_next(&cpool->conns, cpid, &cpid, &entry));
281 sigpipe_restore(&pipe_ctx);
282 }
283 CPOOL_UNLOCK(cpool, admin);
284 Curl_hash_destroy(&cpool->dest2bundle);
285 Curl_uint32_bset_destroy(&cpool->idles);
286 Curl_uint32_tbl_destroy(&cpool->conns);
287 }
290static struct cpool *cpool_get_instance(struct Curl_easy *data)
292 /* admin handles do not necessarily find the correct pool */
293 DEBUGASSERT(data->mid);
294 if(CURL_SHARE_KEEP_CONNECT(data->share))
295 return &data->share->cpool;
296 else if(data->multi_easy)
297 return &data->multi_easy->cpool;
298 else if(data->multi)
299 return &data->multi->cpool;
300 return NULL;
303void Curl_cpool_xfer_init(struct Curl_easy *data)
305 struct cpool *cpool = cpool_get_instance(data);
307 if(cpool) {
308 CPOOL_LOCK(cpool, data);
309 /* the identifier inside the connection cache */
310 data->id = cpool->next_easy_id++;
311 if(cpool->next_easy_id == CURL_OFF_T_MAX)
312 cpool->next_easy_id = 0;
313 data->state.last_conn_id = -1;
314 data->state.last_cpid = UINT32_MAX;
316 CPOOL_UNLOCK(cpool, data);
317 }
318 else {
319 /* We should not get here, but in a non-debug build, do something */
320 DEBUGASSERT(0);
321 data->id = 0;
322 data->state.last_conn_id = -1;
323 data->state.last_cpid = UINT32_MAX;
324 }
327static bool cpool_conn_can_be_closed(struct connectdata *conn)
329 /* CONNECT_ONLY sockets remain in use by the application. */
330 return (conn && !CONN_INUSE(conn) && !conn->bits.connect_only);
333static struct connectdata *cpool_bundle_get_oldest_idle(
334 struct cpool *cpool,
335 struct cpool_bundle *bundle,
336 const struct curltime *pnow)
338 uint32_t cpid, oldest_id = UINT32_MAX;
339 timediff_t unused_ms;
340 timediff_t oldest_ms = -1;
342 if(Curl_uint32_spbset_first(&bundle->ids, &cpid)) {
343 do {
344 struct connectdata *conn = Curl_uint32_tbl_get(&cpool->conns, cpid);
345 if(cpool_conn_can_be_closed(conn)) {
346 /* Set higher score for the age passed since the connection was used */
347 unused_ms =
348 curlx_ptimediff_ms(pnow, &conn->created) - conn->lastused_ms;
349 if(unused_ms > oldest_ms) {
350 oldest_ms = unused_ms;
351 oldest_id = cpid;
352 }
353 }
354 else if(!conn) {
355 DEBUGASSERT(0);
356 Curl_uint32_spbset_remove(&bundle->ids, cpid);
357 }
358 } while(Curl_uint32_spbset_next(&bundle->ids, cpid, &cpid));
359 }
360 return Curl_uint32_tbl_get(&cpool->conns, oldest_id);
363static struct connectdata *cpool_get_oldest_idle(struct cpool *cpool,
364 const struct curltime *pnow,
365 timediff_t min_age_ms)
367 uint32_t cpid, oldest_id = UINT32_MAX;
368 timediff_t oldest_idle_ms = -1;
369 timediff_t idle_ms;
371 /* Iterate over all connections that once were idle. They may
372 * no longer be, in which case we remove them from the set again. */
373 if(Curl_uint32_bset_first(&cpool->idles, &cpid)) {
374 do {
375 struct connectdata *conn = Curl_uint32_tbl_get(&cpool->conns, cpid);
376 if(cpool_conn_can_be_closed(conn)) {
377 idle_ms = curlx_ptimediff_ms(pnow, &conn->created) - conn->lastused_ms;
378 if((idle_ms >= min_age_ms) && (idle_ms > oldest_idle_ms)) {
379 oldest_idle_ms = idle_ms;
380 oldest_id = cpid;
381 }
382 }
383 else {
384 DEBUGASSERT(conn);
385 Curl_uint32_bset_remove(&cpool->idles, cpid);
386 }
387 } while(Curl_uint32_bset_next(&cpool->idles, cpid, &cpid));
388 }
389 return Curl_uint32_tbl_get(&cpool->conns, oldest_id);
392static void cpool_conn_close(struct cpool *cpool,
393 struct Curl_easy *data,
394 struct connectdata *conn,
395 bool aborted)
397 struct Curl_easy *admin;
398 bool do_lock;
400 DEBUGASSERT(cpool);
401 DEBUGASSERT(data && !data->conn);
402 if(!cpool)
403 return;
405 /* If this connection is still used by transfers, we cannot close it. */
406 if(CONN_INUSE(conn)) {
407 DEBUGASSERT(0); /* we'd like to know when this happens */
408 DEBUGF(infof(data, "conn terminate when inuse: %u", conn->attached_xfers));
409 return;
410 }
412 /* This method may be called while we are under lock, e.g. from a
413 * user callback in find. */
414 admin = Curl_get_admin(data);
415 do_lock = !CPOOL_IS_LOCKED(cpool);
416 if(do_lock)
417 CPOOL_LOCK(cpool, admin);
419 if(conn->cpid != UINT32_MAX) {
420 cpool_remove_conn(cpool, conn);
421 DEBUGASSERT(conn->cpid == UINT32_MAX);
422 }
424 /* treat the connection as aborted in CONNECT_ONLY situations,
425 * so no graceful shutdown is attempted. */
426 if(conn->bits.connect_only)
427 aborted = TRUE;
429 if(data->multi) {
430 /* Add it to the multi's cpool for shutdown handling */
431 infof(data, "%s connection #%" FMT_OFF_T,
432 aborted ? "closing" : "shutting down", conn->connection_id);
433 cpool_discard_conn(&data->multi->cpool, data, conn, aborted);
434 }
435 else {
436 /* No multi available, terminate */
437 infof(data, "closing connection #%" FMT_OFF_T, conn->connection_id);
438 Curl_cshutdn_terminate(admin, conn, !aborted);
439 }
441 if(do_lock)
442 CPOOL_UNLOCK(cpool, admin);
445void Curl_conn_close(struct Curl_easy *data,
446 struct connectdata *conn,
447 bool aborted)
449 struct cpool *cpool = cpool_get_instance(data);
450 cpool_conn_close(cpool, data, conn, aborted);
453/* Evict an idle connection to make room in the pool. A pool owned by
454 * a share has no multi that could perform a controlled shutdown of the
455 * connection; terminate it right away. Otherwise, hand it to the
456 * transfer's multi for shutdown. Expects the pool to be locked. */
457static void cpool_evict_conn(struct cpool *cpool,
458 struct Curl_easy *admin,
459 struct connectdata *conn)
461 if(cpool->share) {
462 cpool_remove_conn(cpool, conn);
463 Curl_cshutdn_terminate(admin, conn, TRUE);
464 }
465 else
466 cpool_conn_close(cpool, admin, conn, FALSE);
469#define CPOOL_LIMIT_OK 0
470#define CPOOL_LIMIT_DEST 1
471#define CPOOL_LIMIT_TOTAL 2
473static int cpool_check_limits(struct Curl_easy *data,
474 uint32_t max_total,
475 struct connectdata *to_add,
476 uint32_t max_host,
477 struct cpool_bundle *bundle,
478 const struct curltime *pnow)
480 struct cpool *cpool = cpool_get_instance(data);
481 struct cshutdn *cshutdn = data->multi ? &data->multi->cshutdn : NULL;
482 struct Curl_easy *admin = NULL;
484 if(!cpool)
485 return CPOOL_LIMIT_OK;
487 if(!max_total && !max_host)
488 return CPOOL_LIMIT_OK;
490 if(max_host) {
491 size_t live = 0, shutdowns = 0;
492 /* if we are at or above `dest_limit`, try to get rid of connections
493 * in shutdown and, if that does not lower it, evict idle connections
494 * from the pool. */
495 admin = Curl_get_admin(data);
496 if(bundle) {
497 live = Curl_uint32_spbset_count(&bundle->ids);
498 if(live >= max_host) {
499 size_t over = live - max_host + 1;
500 for(; over; --over) {
501 struct connectdata *oldest_idle =
502 cpool_bundle_get_oldest_idle(cpool, bundle, pnow);
503 if(!oldest_idle)
504 break;
505 /* disconnect the old conn and continue */
506 CURL_TRC_M(admin, "Shutting down connection #%" FMT_OFF_T
507 " to '%s' due to destination limit of %u",
508 oldest_idle->connection_id, oldest_idle->destination,
509 max_host);
510 cpool_evict_conn(cpool, admin, oldest_idle);
511 }
512 live = Curl_uint32_spbset_count(&bundle->ids);
513 }
514 }
515 if(cshutdn) {
516 shutdowns = Curl_cshutdn_dest_count(cshutdn, to_add->destination);
517 if(shutdowns && ((live + shutdowns) >= max_host)) {
518 size_t over = live + shutdowns - max_host + 1;
519 shutdowns -= Curl_cshutdn_close_oldest(cshutdn, admin,
520 to_add->destination, over);
521 }
522 }
523 /* We may not have gotten rid of as many connections as we wanted to */
524 if((live + shutdowns) >= max_host)
525 return CPOOL_LIMIT_DEST;
526 }
528 /* Internal transfers are outside the total limit */
529 if(max_total && !data->state.internal) {
530 size_t shutdowns = 0;
531 uint32_t num_conns = Curl_uint32_tbl_count(&cpool->conns);
533 if(!admin)
534 admin = Curl_get_admin(data);
535 if(num_conns >= max_total) {
536 size_t over = num_conns - max_total + 1;
537 for(; over; --over) {
538 struct connectdata *oldest_idle =
539 cpool_get_oldest_idle(cpool, pnow, 0);
540 if(!oldest_idle)
541 break;
542 /* disconnect the old conn and continue */
543 CURL_TRC_M(admin, "Shutting down idle connection #%"
544 FMT_OFF_T " to '%s' due to total limit of %u",
545 oldest_idle->connection_id, oldest_idle->destination,
546 max_total);
547 cpool_evict_conn(cpool, admin, oldest_idle);
548 }
549 }
551 num_conns = Curl_uint32_tbl_count(&cpool->conns);
552 if(cshutdn) {
553 shutdowns = Curl_cshutdn_count(cshutdn);
554 if(shutdowns && (num_conns + shutdowns) >= max_total) {
555 size_t over = num_conns + shutdowns - max_total + 1;
556 shutdowns -= Curl_cshutdn_close_oldest(cshutdn, admin, NULL, over);
557 }
558 }
559 if((num_conns + shutdowns) >= max_total)
560 return CPOOL_LIMIT_TOTAL;
561 }
563 return CPOOL_LIMIT_OK;
566CURLcode Curl_cpool_add(struct Curl_easy *data,
567 struct connectdata *conn,
568 uint32_t max_total,
569 uint32_t max_host,
570 const struct curltime *pnow)
572 CURLcode result = CURLE_OK;
573 struct cpool_bundle *bundle = NULL;
574 struct cpool *cpool = cpool_get_instance(data);
575 uint32_t capacity, cpid;
576 DEBUGASSERT(conn);
578 DEBUGASSERT(cpool);
579 if(!cpool)
580 return CURLE_FAILED_INIT;
582 DEBUGASSERT(conn->cpid == UINT32_MAX);
583 conn->created = *pnow;
584 conn->connect_start_ms = -1;
585 conn->shutdown.start_ms[FIRSTSOCKET] =
586 conn->shutdown.start_ms[SECONDARYSOCKET] = -1;
588 CPOOL_LOCK(cpool, data);
589 /* Find the bundle, should it exist, and check the limits */
590 bundle = cpool_get_bundle(cpool, conn->destination);
592 switch(cpool_check_limits(data, max_total, conn, max_host, bundle, pnow)) {
593 case CPOOL_LIMIT_DEST:
594 infof(data, "No more connections allowed to host");
595 result = CURLE_NO_CONNECTION_AVAILABLE;
596 goto out;
597 case CPOOL_LIMIT_TOTAL:
598 infof(data, "No connections available, total of %u reached.",
599 data->multi->max_total_connections);
600 result = CURLE_NO_CONNECTION_AVAILABLE;
601 goto out;
602 default:
603 break;
604 }
606 /* We add the connection */
607 capacity = Curl_uint32_tbl_capacity(&cpool->conns);
608 if(capacity == UINT32_MAX) {
609 infof(data, "No connections available, pool at max capacity");
610 result = CURLE_NO_CONNECTION_AVAILABLE;
611 goto out;
612 }
613 if(Curl_uint32_tbl_count(&cpool->conns) >= capacity) {
614 uint32_t growth, ncapacity;
616 growth = CURLMAX(capacity, 16);
617 if((UINT32_MAX - growth) >= capacity)
618 ncapacity = capacity + growth;
619 else
620 ncapacity = UINT32_MAX;
621 /* Grow bitset first, so table resize failure does not leave a too
622 * short bitset should that one fail. */
623 if(Curl_uint32_bset_resize(&cpool->idles, ncapacity) ||
624 Curl_uint32_tbl_resize(&cpool->conns, ncapacity)) {
625 result = CURLE_OUT_OF_MEMORY;
626 goto out;
627 }
628 }
630 if(!bundle) {
631 bundle = cpool_create_bundle();
632 if(!bundle) {
633 result = CURLE_OUT_OF_MEMORY;
634 goto out;
635 }
636 }
638 if(!Curl_uint32_tbl_add(&cpool->conns, conn, &cpid)) {
639 DEBUGASSERT(0);
640 result = CURLE_OUT_OF_MEMORY;
641 goto out;
642 }
643 conn->cpid = cpid;
645 if(!cpool_bundle_add_id(cpool, bundle, conn->destination, cpid)) {
646 Curl_uint32_tbl_remove(&cpool->conns, cpid);
647 conn->cpid = UINT32_MAX;
648 result = CURLE_OUT_OF_MEMORY;
649 goto out;
650 }
652 conn->connection_id = cpool->next_connection_id++;
653 CURL_TRC_M(data, "[CPOOL] added connection %" FMT_OFF_T ". "
654 "The cache now contains %u members",
655 conn->connection_id, Curl_uint32_tbl_count(&cpool->conns));
656out:
657 cpool_bundle_unlink(&bundle);
658 CPOOL_UNLOCK(cpool, data);
660 return result;
663/* This function iterates the entire connection pool and calls the function
664 func() with the connection pointer as the first argument and the supplied
665 'param' argument as the other.
667 The cpool lock is still held when the callback is called. It needs it,
668 so that it can safely continue traversing the lists once the callback
669 returns.
671 Returns TRUE if the loop was aborted due to the callback's return code.
673 Return 0 from func() to continue the loop, return 1 to abort it.
674 */
675static bool cpool_foreach(struct Curl_easy *data,
676 struct cpool *cpool,
677 void *param,
678 int (*func)(struct cpool *cpool,
679 struct Curl_easy *data,
680 struct connectdata *conn, void *param))
682 void *entry;
683 uint32_t cpid;
685 if(!cpool)
686 return FALSE;
688 if(Curl_uint32_tbl_first(&cpool->conns, &cpid, &entry)) {
689 do {
690 struct connectdata *conn = entry;
691 if(conn && func(cpool, data, conn, param) == 1)
692 return TRUE;
693 } while(Curl_uint32_tbl_next(&cpool->conns, cpid, &cpid, &entry));
694 }
695 return FALSE;
698/*
699 * A connection (already in the pool) has become idle. Do any
700 * cleanups in regard to the pool's limits.
701 *
702 * Return TRUE if idle connection kept in pool, FALSE if closed.
703 */
704static bool cpool_conn_now_idle(struct cpool *cpool,
705 struct Curl_easy *data,
706 struct connectdata *conn,
707 const struct curltime *pnow)
709 struct connectdata *oldest_idle = NULL;
710 struct Curl_easy *admin;
711 uint32_t maxconnects;
712 bool kept = TRUE;
713 timediff_t min_age_ms = 0;
715 if(!data || !data->multi)
716 return kept;
718 Curl_uint32_bset_add(&cpool->idles, conn->cpid);
720 if(!data->multi->maxconnects) {
721 /* Attached transfers is a weak indicator of business. */
722 uint32_t attached = Curl_multi_xfers_attached(data->multi);
723 maxconnects = (attached <= UINT32_MAX / 2) ? attached * 2 : UINT32_MAX;
724 /* We are guessing. So, only evict a "seemingly superfluous" connection
725 * when has not been used for this long, */
726 min_age_ms = 1000;
727 }
728 else {
729 maxconnects = data->multi->maxconnects;
730 }
732 if(cpool && maxconnects && (maxconnects < UINT32_MAX)) {
733 uint32_t num_conns = Curl_uint32_tbl_count(&cpool->conns);
734 admin = Curl_get_admin(data);
735 if(num_conns > maxconnects) {
736 infof(data, "Connection pool is full, closing the oldest of %u/%u",
737 num_conns, maxconnects);
738 oldest_idle = cpool_get_oldest_idle(cpool, pnow, min_age_ms);
739 kept = (oldest_idle != conn);
740 if(oldest_idle) {
741 cpool_evict_conn(cpool, admin, oldest_idle);
742 }
743 }
744 }
746 return kept;
749static void cpool_prune_dead(struct cpool *cpool,
750 struct Curl_easy *data,
751 const struct curltime *pnow);
753static bool cpool_find_act(struct cpool *cpool,
754 struct Curl_easy *data,
755 struct connectdata *conn,
756 cpool_match_result match)
758 switch(match) {
759 case CPOOL_MATCH_FOUND:
760 return TRUE;
761 case CPOOL_MATCH_TOO_OLD:
762 if(!cpool_conn_can_be_closed(conn))
763 return FALSE;
764 FALLTHROUGH();
765 case CPOOL_MATCH_CLOSE:
766 cpool_conn_close(cpool, data, conn, FALSE);
767 break;
768 default:
769 break;
770 }
771 return FALSE;
774bool Curl_cpool_find(struct Curl_easy *data,
775 const char *destination,
776 bool prune_dead,
777 const struct curltime *pnow,
778 Curl_cpool_conn_match_cb *conn_cb,
779 Curl_cpool_done_match_cb *done_cb,
780 void *userdata)
782 struct cpool *cpool = cpool_get_instance(data);
783 struct cpool_bundle *bundle = NULL;
784 bool found = FALSE;
786 DEBUGASSERT(cpool);
787 DEBUGASSERT(conn_cb);
788 if(!cpool)
789 return FALSE;
791 CPOOL_LOCK(cpool, data);
793 if(prune_dead)
794 cpool_prune_dead(cpool, data, pnow);
796 if(data->state.last_cpid != UINT32_MAX) {
797 struct connectdata *conn =
798 Curl_uint32_tbl_get(&cpool->conns, data->state.last_cpid);
799 if(conn && (data->state.last_conn_id == conn->connection_id)) {
800 /* Indeed the last connection used by `data` still exists.
801 * Check that first, so that connection use gets "sticky". */
802 found = cpool_find_act(cpool, data, conn, conn_cb(conn, userdata));
803 if(found)
804 goto out;
805 }
806 else {
807 /* No longer found here. */
808 data->state.last_conn_id = -1;
809 data->state.last_cpid = UINT32_MAX;
810 }
811 }
813 bundle = cpool_get_bundle(cpool, destination);
814 if(bundle) {
815 uint32_t cpid;
816 if(Curl_uint32_spbset_first(&bundle->ids, &cpid)) {
817 do {
818 struct connectdata *conn = Curl_uint32_tbl_get(&cpool->conns, cpid);
819 if(conn) {
820 if(data->state.last_conn_id != conn->connection_id) {
821 found = cpool_find_act(cpool, data, conn, conn_cb(conn, userdata));
822 if(found)
823 goto out;
824 }
825 }
826 else {
827 DEBUGASSERT(0);
828 Curl_uint32_spbset_remove(&bundle->ids, cpid);
829 }
830 } while(Curl_uint32_spbset_next(&bundle->ids, cpid, &cpid));
831 }
832 }
834out:
835 if(done_cb) {
836 found = done_cb(userdata);
837 }
838 cpool_bundle_unlink(&bundle);
839 CPOOL_UNLOCK(cpool, data);
840 return found;
843struct cpool_reaper_ctx {
844 size_t reaped;
845 const struct curltime *pnow;
846};
848static int cpool_reap_dead_cb(struct cpool *cpool,
849 struct Curl_easy *admin,
850 struct connectdata *conn, void *param)
852 struct cpool_reaper_ctx *reaper = param;
854 if(cpool_conn_can_be_closed(conn)) {
855 if(conn->bits.no_reuse || conn->bits.close ||
856 !Curl_cpool_conn_seems_healthy(conn, admin, reaper->pnow)) {
857 /* terminate conn and stop the iteration */
858 reaper->reaped++;
859 cpool_conn_close(cpool, admin, conn, FALSE);
860 }
861 }
862 return 0; /* continue iteration */
865/*
866 * This function scans the data's connection pool for half-open/dead
867 * connections, closes and removes them.
868 * The cleanup is done at most once per second.
869 */
870static void cpool_prune_dead(struct cpool *cpool,
871 struct Curl_easy *data,
872 const struct curltime *pnow)
874 timediff_t elapsed_ms = curlx_ptimediff_ms(pnow, &cpool->last_cleanup);
876 if(elapsed_ms >= 1000L) {
877 struct Curl_easy *admin = Curl_get_admin(data);
878 struct cpool_reaper_ctx reaper;
880 memset(&reaper, 0, sizeof(reaper));
881 reaper.pnow = pnow;
882 cpool_foreach(admin, cpool, &reaper, cpool_reap_dead_cb);
883 cpool->last_cleanup = *pnow;
884 }
887static int conn_upkeep(struct cpool *cpool,
888 struct Curl_easy *admin,
889 struct connectdata *conn,
890 void *param)
892 const struct curltime *pnow = Curl_pgrs_now(admin);
894 (void)param;
895 if((curlx_ptimediff_ms(pnow, &conn->created) - conn->lastupkeep_ms) >=
896 admin->set.upkeep_interval_ms) {
897 CURLcode result;
899 conn->lastupkeep_ms = curlx_ptimediff_ms(pnow, &conn->created);
900 /* briefly attach for action */
901 Curl_attach_connection(admin, conn, FALSE);
902 result = Curl_conn_keep_alive(admin, conn);
903 Curl_detach_connection(admin);
905 if(result && cpool_conn_can_be_closed(conn)) {
906 cpool_conn_close(cpool, admin, conn, FALSE);
907 }
908 }
909 return 0; /* continue iteration */
912CURLcode Curl_cpool_upkeep(struct Curl_easy *data)
914 struct cpool *cpool = cpool_get_instance(data);
915 struct Curl_easy *admin = Curl_get_admin(data);
917 if(!cpool)
918 return CURLE_OK;
920 CPOOL_LOCK(cpool, admin);
921 cpool_foreach(admin, cpool, NULL, conn_upkeep);
922 CPOOL_UNLOCK(cpool, admin);
923 return CURLE_OK;
926struct connectdata *Curl_cpool_get_last_conn(struct Curl_easy *data)
928 struct cpool *cpool = cpool_get_instance(data);
929 if(cpool && (data->state.last_cpid != UINT32_MAX)) {
930 struct connectdata *conn;
932 CPOOL_LOCK(cpool, data);
933 conn = Curl_uint32_tbl_get(&cpool->conns, data->state.last_cpid);
934 CPOOL_UNLOCK(cpool, data);
935 if(conn && (data->state.last_conn_id == conn->connection_id))
936 return conn;
937 data->state.last_cpid = UINT32_MAX;
938 data->state.last_conn_id = -1;
939 }
940 return NULL;
943void Curl_cpool_return(struct Curl_easy *data,
944 struct connectdata *conn,
945 Curl_cpool_return_cb *cb, void *cbdata,
946 const struct curltime *pnow)
948 struct cpool *cpool = cpool_get_instance(data);
950 CPOOL_LOCK(cpool, data);
952 /* remember times, connection had been used just before */
953 conn->lastchecked_ms = conn->lastupkeep_ms = conn->lastused_ms =
954 curlx_ptimediff_ms(pnow, &conn->created);
956 switch(cb(data, conn, cbdata, pnow)) {
957 case CPOOL_DO_KEEP:
958 break;
959 case CPOOL_DO_IDLE:
960 /* the connection is no longer in use by any transfer */
961 if(cpool_conn_now_idle(cpool, data, conn, pnow)) {
962 /* connection kept in the cpool */
963 infof(data, "Connection #%" FMT_OFF_T " to host %s:%u left intact",
964 conn->connection_id, conn->origin->user_hostname,
965 conn->origin->port);
966 }
967 else { /* connection was removed from the cpool and destroyed. */
968 data->state.last_conn_id = -1;
969 data->state.last_cpid = UINT32_MAX;
970 }
971 break;
972 case CPOOL_DO_CLOSE:
973 Curl_conn_close(data, conn, FALSE);
974 break;
975 case CPOOL_DO_TERMINATE:
976 Curl_conn_close(data, conn, TRUE);
977 break;
978 }
980 CPOOL_UNLOCK(cpool, data);
983static int cpool_reap_no_reuse(struct cpool *cpool,
984 struct Curl_easy *admin,
985 struct connectdata *conn, void *param)
987 (void)param;
988 conn->bits.no_reuse = TRUE;
989 if(cpool_conn_can_be_closed(conn))
990 cpool_conn_close(cpool, admin, conn, FALSE);
991 return 0; /* continue iteration */
994void Curl_cpool_nw_changed(struct cpool *cpool, struct Curl_easy *admin)
996 if(cpool && admin) {
997 CPOOL_LOCK(cpool, admin);
998 cpool_foreach(admin, cpool, NULL, cpool_reap_no_reuse);
999 CPOOL_UNLOCK(cpool, admin);
1003/* A connection has to have been idle for less than 'conn_max_idle_ms'
1004 (the success rate is too low after this), or created less than
1005 'conn_max_age_ms' ago, to be subject for reuse. */
1006static bool cpool_conn_too_old(struct Curl_easy *data,
1007 struct connectdata *conn,
1008 const struct curltime *pnow)
1010 timediff_t age_ms;
1012 if(data->set.conn_max_idle_ms) {
1013 age_ms = curlx_ptimediff_ms(pnow, &conn->created) - conn->lastused_ms;
1014 if(age_ms > data->set.conn_max_idle_ms) {
1015 infof(data, "Too old connection (%" FMT_TIMEDIFF_T
1016 " ms idle, max idle is %" FMT_TIMEDIFF_T " ms), disconnect it",
1017 age_ms, data->set.conn_max_idle_ms);
1018 return TRUE;
1022 if(data->set.conn_max_age_ms) {
1023 age_ms = curlx_ptimediff_ms(pnow, &conn->created);
1024 if(age_ms > data->set.conn_max_age_ms) {
1025 infof(data,
1026 "Too old connection (created %" FMT_TIMEDIFF_T
1027 " ms ago, max lifetime is %" FMT_TIMEDIFF_T " ms), disconnect it",
1028 age_ms, data->set.conn_max_age_ms);
1029 return TRUE;
1033 return FALSE;
1036bool Curl_cpool_conn_seems_healthy(struct connectdata *conn,
1037 struct Curl_easy *data,
1038 const struct curltime *pnow)
1040 struct Curl_easy *admin;
1041 bool healthy = TRUE;
1043 DEBUGASSERT(!data->conn);
1044 if(cpool_conn_can_be_closed(conn) && cpool_conn_too_old(data, conn, pnow))
1045 return FALSE;
1047 /* An idle, non-multiplexed connection must not be reused when it has
1048 * input pending (RFC 9112, section 9.2). Check this:
1049 * - on a connection that is not in use
1050 * - or a connection having its own "is dead" callback
1051 * - or the last check was more than a second ago */
1052 if(CONN_INUSE(conn) &&
1053 !conn->scheme->run->connection_is_dead &&
1054 (curlx_ptimediff_ms(pnow, &conn->created) - conn->lastchecked_ms) < 1000)
1055 return TRUE;
1057 admin = Curl_get_admin(data);
1058 if(conn->scheme->run->connection_is_dead) {
1059 Curl_attach_connection(admin, conn, FALSE);
1060 healthy = !conn->scheme->run->connection_is_dead(admin, conn);
1061 Curl_detach_connection(admin);
1063 else {
1064 bool input_pending = FALSE;
1066 Curl_attach_connection(admin, conn, FALSE);
1067 healthy = Curl_conn_is_alive(admin, conn, &input_pending);
1068 Curl_detach_connection(admin);
1069 if(healthy && input_pending && !CONN_INUSE(conn) &&
1070 !Curl_conn_is_multiplex(conn, FIRSTSOCKET)) {
1071 /* Non-multiplexed connections without attached transfers should
1072 * not have input pending. The input might be a TLS Notify Close,
1073 * for all we know. */
1074 DEBUGF(infof(data, "connection has no transfer but input, not healthy"));
1075 healthy = FALSE;
1079 if(healthy)
1080 conn->lastchecked_ms = curlx_ptimediff_ms(pnow, &conn->created);
1081 return healthy;
1084void Curl_cpool_conn_was_used(struct Curl_easy *data,
1085 struct connectdata *conn,
1086 const struct curltime *pnow)
1088 (void)data;
1089 conn->lastupkeep_ms = curlx_ptimediff_ms(pnow, &conn->created);
1092timediff_t Curl_cpool_conn_age_ms(struct Curl_easy *data,
1093 struct connectdata *conn,
1094 const struct curltime *pnow)
1096 (void)data;
1097 return curlx_ptimediff_ms(pnow, &conn->created);
Diagnostics4 git calls · 220 ms
CallStatusSizeTime0220 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2009.2 KiB214 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=214
git.response.bytes=9404
git.response.cache=MISS
git.response.duration_ms=214
git.fetch.wants=905412a9c0161b12a6eb5c3d04f74089a6031c7e
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=9404
git.pkt.payload.max=9383
git.sideband.band1.bytes=9382
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2bb264996cfc3
Request
179cd384…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format