master
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24/* <DESC>
25 * multi socket interface with epoll and timerfd
26 * </DESC>
27 */
28/* Example application source code using the multi socket interface to
29 * download many files at once.
30 *
31 * This example features the same basic functionality as hiperfifo.c does,
32 * but this uses epoll and timerfd instead of libevent.
33 *
34 * Written by Jeff Pohlmeyer, converted to use epoll by Josh Bialkowski
35 *
36 * Requires a Linux system with epoll
37 *
38 * When running, the program creates the named pipe "hiper.fifo"
39 *
40 * Whenever there is input into the fifo, the program reads the input as a list
41 * of URL's and creates some new easy handles to fetch each URL via the
42 * curl_multi "hiper" API.
43 *
44 * Thus, you can try a single URL:
45 * % echo http://www.example.com > hiper.fifo
46 *
47 * Or a whole bunch of them:
48 * % cat my-url-list > hiper.fifo
49 *
50 * The fifo buffer is handled almost instantly, so you can even add more URL's
51 * while the previous requests are still being downloaded.
52 *
53 * Note:
54 * For the sake of simplicity, URL length is limited to 1023 chars.
55 *
56 * This is purely a demo app, all retrieved data is discarded by
57 * the write callback.
58 */
59#include <errno.h>
60#include <fcntl.h>
61#include <signal.h>
62#include <stdio.h>
63#include <stdlib.h>
64#include <string.h>
65#include <sys/epoll.h>
66#include <sys/stat.h>
67#include <sys/time.h>
68#include <sys/timerfd.h>
69#include <sys/types.h>
70#include <time.h>
71#include <unistd.h>
73#include <curl/curl.h>
75#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */
77/* Global information, common to all connections */
78struct GlobalInfo {
79 int epfd; /* epoll filedescriptor */
80 int tfd; /* timer filedescriptor */
81 int fifofd; /* fifo filedescriptor */
82 CURLM *multi;
83 int still_running;
84 FILE *input;
85};
87/* Information associated with a specific easy handle */
88struct ConnInfo {
89 CURL *curl;
90 char *url;
91 struct GlobalInfo *global;
92 char error[CURL_ERROR_SIZE];
93};
95/* Information associated with a specific socket */
96struct SockInfo {
97 curl_socket_t sockfd;
98 CURL *curl;
99 int action;
100 long timeout;
101 struct GlobalInfo *global;
102};
104/* Die if we get a bad CURLMcode somewhere */
105static void mcode_or_die(const char *where, CURLMcode code)
106{
107 if(CURLM_OK != code) {
108 const char *s;
109 switch(code) {
110 case CURLM_BAD_HANDLE:
111 s = "CURLM_BAD_HANDLE";
112 break;
113 case CURLM_BAD_EASY_HANDLE:
114 s = "CURLM_BAD_EASY_HANDLE";
115 break;
116 case CURLM_OUT_OF_MEMORY:
117 s = "CURLM_OUT_OF_MEMORY";
118 break;
119 case CURLM_INTERNAL_ERROR:
120 s = "CURLM_INTERNAL_ERROR";
121 break;
122 case CURLM_UNKNOWN_OPTION:
123 s = "CURLM_UNKNOWN_OPTION";
124 break;
125 case CURLM_LAST:
126 s = "CURLM_LAST";
127 break;
128 default:
129 s = "CURLM_unknown";
130 break;
131 case CURLM_BAD_SOCKET:
132 s = "CURLM_BAD_SOCKET";
133 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
134 /* ignore this error */
135 return;
136 }
137 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
138 exit(code);
139 }
140}
142/* Check for completed transfers, and remove their easy handles */
143static void check_multi_info(struct GlobalInfo *g)
144{
145 char *eff_url;
146 CURLMsg *msg;
147 int msgs_left;
148 struct ConnInfo *conn;
150 fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running);
151 while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
152 if(msg->msg == CURLMSG_DONE) {
153 CURL *curl = msg->easy_handle;
154 CURLcode result = msg->data.result;
155 curl_easy_getinfo(curl, CURLINFO_PRIVATE, &conn);
156 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &eff_url);
157 fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, result, conn->error);
158 curl_multi_remove_handle(g->multi, curl);
159 free(conn->url);
160 curl_easy_cleanup(curl);
161 free(conn);
162 }
163 }
164}
166/* Called by main loop when our timeout expires */
167static void timer_cb(struct GlobalInfo *g, int revents)
168{
169 CURLMcode mresult;
170 uint64_t count = 0;
171 ssize_t err = 0;
173 (void)revents;
175 err = read(g->tfd, &count, sizeof(uint64_t));
176 if(err == -1) {
177 /* Note that we may call the timer callback even if the timerfd is not
178 * readable. It is possible that there are multiple events stored in the
179 * epoll buffer (i.e. the timer may have fired multiple times). The event
180 * count is cleared after the first call so future events in the epoll
181 * buffer fails to read from the timer. */
182 if(errno == EAGAIN) {
183 fprintf(MSG_OUT, "EAGAIN on tfd %d\n", g->tfd);
184 return;
185 }
186 }
187 if(err != sizeof(uint64_t)) {
188 fprintf(stderr, "read(tfd) == %ld", err);
189 perror("read(tfd)");
190 }
192 mresult = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0,
193 &g->still_running);
194 mcode_or_die("timer_cb: curl_multi_socket_action", mresult);
195 check_multi_info(g);
196}
198/* Update the timer after curl_multi library does its thing. curl informs the
199 * application through this callback what it wants the new timeout to be,
200 * after it does some work. */
201static int multi_timer_cb(CURLM *multi, long timeout_ms, struct GlobalInfo *g)
202{
203 struct itimerspec its;
205 (void)multi;
207 fprintf(MSG_OUT, "multi_timer_cb: Setting timeout to %ld ms\n", timeout_ms);
209 if(timeout_ms > 0) {
210 its.it_interval.tv_sec = 0;
211 its.it_interval.tv_nsec = 0;
212 its.it_value.tv_sec = timeout_ms / 1000;
213 its.it_value.tv_nsec = (timeout_ms % 1000) * 1000 * 1000;
214 }
215 else if(timeout_ms == 0) {
216 /* libcurl wants us to timeout now, however setting both fields of
217 * new_value.it_value to zero disarms the timer. The closest we can
218 * do is to schedule the timer to fire in 1 ns. */
219 its.it_interval.tv_sec = 0;
220 its.it_interval.tv_nsec = 0;
221 its.it_value.tv_sec = 0;
222 its.it_value.tv_nsec = 1;
223 }
224 else {
225 memset(&its, 0, sizeof(its));
226 }
228 timerfd_settime(g->tfd, /* flags= */ 0, &its, NULL);
229 return 0;
230}
232/* Called by libevent when we get action on a multi socket filedescriptor */
233static void event_cb(struct GlobalInfo *g, int fd, int revents)
234{
235 CURLMcode mresult;
236 struct itimerspec its;
238 int action = ((revents & EPOLLIN) ? CURL_CSELECT_IN : 0) |
239 ((revents & EPOLLOUT) ? CURL_CSELECT_OUT : 0);
241 mresult = curl_multi_socket_action(g->multi, fd, action, &g->still_running);
242 mcode_or_die("event_cb: curl_multi_socket_action", mresult);
244 check_multi_info(g);
245 if(g->still_running <= 0) {
246 fprintf(MSG_OUT, "last transfer done, kill timeout\n");
247 memset(&its, 0, sizeof(its));
248 timerfd_settime(g->tfd, 0, &its, NULL);
249 }
250}
252/* Clean up the SockInfo structure */
253static void remsock(struct SockInfo *f, struct GlobalInfo *g)
254{
255 if(f) {
256 if(f->sockfd) {
257 if(epoll_ctl(g->epfd, EPOLL_CTL_DEL, f->sockfd, NULL))
258 fprintf(stderr, "EPOLL_CTL_DEL failed for fd: %d : %s\n",
259 f->sockfd, strerror(errno));
260 }
261 free(f);
262 }
263}
265/* Assign information to a SockInfo structure */
266static void setsock(struct SockInfo *f, curl_socket_t s, CURL *e, int act,
267 struct GlobalInfo *g)
268{
269 struct epoll_event ev;
270 int kind = ((act & CURL_POLL_IN) ? EPOLLIN : 0) |
271 ((act & CURL_POLL_OUT) ? EPOLLOUT : 0);
273 if(f->sockfd) {
274 if(epoll_ctl(g->epfd, EPOLL_CTL_DEL, f->sockfd, NULL))
275 fprintf(stderr, "EPOLL_CTL_DEL failed for fd: %d : %s\n",
276 f->sockfd, strerror(errno));
277 }
279 f->sockfd = s;
280 f->action = act;
281 f->curl = e;
283 ev.events = kind;
284 ev.data.fd = s;
285 if(epoll_ctl(g->epfd, EPOLL_CTL_ADD, s, &ev))
286 fprintf(stderr, "EPOLL_CTL_ADD failed for fd: %d : %s\n",
287 s, strerror(errno));
288}
290/* Initialize a new SockInfo structure */
291static void addsock(curl_socket_t s, CURL *curl, int action,
292 struct GlobalInfo *g)
293{
294 struct SockInfo *fdp = calloc(1, sizeof(struct SockInfo));
296 fdp->global = g;
297 setsock(fdp, s, curl, action, g);
298 curl_multi_assign(g->multi, s, fdp);
299}
301/* CURLMOPT_SOCKETFUNCTION */
302static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp)
303{
304 struct GlobalInfo *g = (struct GlobalInfo *)cbp;
305 struct SockInfo *fdp = (struct SockInfo *)sockp;
306 static const char * const whatstr[] = {
307 "none", "IN", "OUT", "INOUT", "REMOVE"
308 };
310 fprintf(MSG_OUT, "socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]);
311 if(what == CURL_POLL_REMOVE) {
312 fprintf(MSG_OUT, "\n");
313 remsock(fdp, g);
314 }
315 else {
316 if(!fdp) {
317 fprintf(MSG_OUT, "Adding data: %s\n", whatstr[what]);
318 addsock(s, e, what, g);
319 }
320 else {
321 fprintf(MSG_OUT, "Changing action from %s to %s\n",
322 whatstr[fdp->action], whatstr[what]);
323 setsock(fdp, s, e, what, g);
324 }
325 }
326 return 0;
327}
329/* CURLOPT_WRITEFUNCTION */
330static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *data)
331{
332 (void)ptr;
333 (void)data;
334 return size * nmemb;
335}
337/* CURLOPT_PROGRESSFUNCTION */
338static int prog_cb(void *p, double dltotal, double dlnow, double ult,
339 double uln)
340{
341 struct ConnInfo *conn = (struct ConnInfo *)p;
342 (void)ult;
343 (void)uln;
345 fprintf(MSG_OUT, "Progress: %s (%g/%g)\n", conn->url, dlnow, dltotal);
346 return 0;
347}
349/* Create a new easy handle, and add it to the global curl_multi */
350static void new_conn(const char *url, struct GlobalInfo *g)
351{
352 struct ConnInfo *conn;
353 CURLMcode mresult;
355 conn = calloc(1, sizeof(*conn));
356 conn->error[0] = '\0';
358 conn->curl = curl_easy_init();
359 if(!conn->curl) {
360 fprintf(MSG_OUT, "curl_easy_init() failed, exiting!\n");
361 exit(2);
362 }
363 conn->global = g;
364 conn->url = strdup(url);
365 curl_easy_setopt(conn->curl, CURLOPT_URL, conn->url);
366 curl_easy_setopt(conn->curl, CURLOPT_WRITEFUNCTION, write_cb);
367 curl_easy_setopt(conn->curl, CURLOPT_WRITEDATA, conn);
368 curl_easy_setopt(conn->curl, CURLOPT_VERBOSE, 1L);
369 curl_easy_setopt(conn->curl, CURLOPT_ERRORBUFFER, conn->error);
370 curl_easy_setopt(conn->curl, CURLOPT_PRIVATE, conn);
371 curl_easy_setopt(conn->curl, CURLOPT_NOPROGRESS, 0L);
372 curl_easy_setopt(conn->curl, CURLOPT_PROGRESSFUNCTION, prog_cb);
373 curl_easy_setopt(conn->curl, CURLOPT_PROGRESSDATA, conn);
374 curl_easy_setopt(conn->curl, CURLOPT_FOLLOWLOCATION, 1L);
375 curl_easy_setopt(conn->curl, CURLOPT_LOW_SPEED_TIME, 3L);
376 curl_easy_setopt(conn->curl, CURLOPT_LOW_SPEED_LIMIT, 10L);
378 fprintf(MSG_OUT, "Adding easy %p to multi %p (%s)\n",
379 conn->curl, g->multi, url);
380 mresult = curl_multi_add_handle(g->multi, conn->curl);
381 mcode_or_die("new_conn: curl_multi_add_handle", mresult);
383 /* note that add_handle() sets a timeout to trigger soon so that the
384 necessary socket_action() gets called */
385}
387/* This gets called whenever data is received from the fifo */
388static void fifo_cb(struct GlobalInfo *g, int revents)
389{
390 char s[1024];
391 long int rv = 0;
392 int n = 0;
394 (void)revents;
396 do {
397 s[0] = '\0';
398 rv = fscanf(g->input, "%1023s%n", s, &n);
399 s[n] = '\0';
400 if(n && s[0]) {
401 new_conn(s, g); /* if we read a URL, go get it! */
402 }
403 else
404 break;
405 } while(rv != EOF);
406}
408/* Create a named pipe and tell libevent to monitor it */
409static int init_fifo(struct GlobalInfo *g)
410{
411 struct stat st;
412 static const char fifo[] = "hiper.fifo";
413 curl_socket_t sockfd;
414 struct epoll_event epev;
416 fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo);
417 if(!lstat(fifo, &st)) {
418 if((st.st_mode & S_IFMT) == S_IFREG) {
419 errno = EEXIST;
420 perror("lstat");
421 return 1;
422 }
423 }
424 unlink(fifo);
425 if(mkfifo(fifo, 0600) == -1) {
426 perror("mkfifo");
427 return 1;
428 }
429 sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0);
430 if(sockfd == -1) {
431 perror("open");
432 return 1;
433 }
435 g->fifofd = sockfd;
436 g->input = fdopen(sockfd, "r");
438 epev.events = EPOLLIN;
439 epev.data.fd = sockfd;
440 epoll_ctl(g->epfd, EPOLL_CTL_ADD, sockfd, &epev);
442 fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo);
443 return 0;
444}
446static void clean_fifo(struct GlobalInfo *g)
447{
448 epoll_ctl(g->epfd, EPOLL_CTL_DEL, g->fifofd, NULL);
449 fclose(g->input);
450 unlink(fifo);
451}
453int g_should_exit_ = 0;
455void sigint_handler(int signo)
456{
457 g_should_exit_ = 1;
458}
460int main(void)
461{
462 CURLcode result;
463 struct GlobalInfo g;
464 struct itimerspec its;
465 struct epoll_event ev;
466 struct epoll_event events[10];
468 result = curl_global_init(CURL_GLOBAL_ALL);
469 if(result != CURLE_OK)
470 return (int)result;
472 g_should_exit_ = 0;
473 signal(SIGINT, sigint_handler);
475 memset(&g, 0, sizeof(g));
476 g.epfd = epoll_create1(EPOLL_CLOEXEC);
477 if(g.epfd == -1) {
478 perror("epoll_create1 failed");
479 curl_global_cleanup();
480 return 1;
481 }
483 g.tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
484 if(g.tfd == -1) {
485 perror("timerfd_create failed");
486 curl_global_cleanup();
487 return 1;
488 }
490 memset(&its, 0, sizeof(its));
491 its.it_interval.tv_sec = 0;
492 its.it_value.tv_sec = 1;
493 timerfd_settime(g.tfd, 0, &its, NULL);
495 ev.events = EPOLLIN;
496 ev.data.fd = g.tfd;
497 epoll_ctl(g.epfd, EPOLL_CTL_ADD, g.tfd, &ev);
499 if(init_fifo(&g)) {
500 curl_global_cleanup();
501 return 1;
502 }
503 g.multi = curl_multi_init();
505 /* setup the generic multi interface options we want */
506 curl_multi_setopt(g.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
507 curl_multi_setopt(g.multi, CURLMOPT_SOCKETDATA, &g);
508 curl_multi_setopt(g.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
509 curl_multi_setopt(g.multi, CURLMOPT_TIMERDATA, &g);
511 /* we do not call any curl_multi_socket*() function yet as we have no handles
512 added! */
514 fprintf(MSG_OUT, "Entering wait loop\n");
515 fflush(MSG_OUT);
516 while(!g_should_exit_) {
517 int idx;
518 int err = epoll_wait(g.epfd, events,
519 sizeof(events) / sizeof(struct epoll_event), 10000);
520 if(err == -1) {
521 /* !checksrc! disable ERRNOVAR 1 */
522 if(errno == EINTR) {
523 fprintf(MSG_OUT, "note: wait interrupted\n");
524 continue;
525 }
526 else {
527 perror("epoll_wait");
528 curl_global_cleanup();
529 return 1;
530 }
531 }
533 for(idx = 0; idx < err; ++idx) {
534 if(events[idx].data.fd == g.fifofd) {
535 fifo_cb(&g, events[idx].events);
536 }
537 else if(events[idx].data.fd == g.tfd) {
538 timer_cb(&g, events[idx].events);
539 }
540 else {
541 event_cb(&g, events[idx].data.fd, events[idx].events);
542 }
543 }
544 }
546 fprintf(MSG_OUT, "Exiting normally.\n");
547 fflush(MSG_OUT);
549 curl_multi_cleanup(g.multi);
550 clean_fifo(&g);
551 curl_global_cleanup();
552 return 0;
553}
Diagnostics4 git calls · 24 ms
CallStatusSizeTime024 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=19 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2005.3 KiB · hit19 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=5469 git.response.cache=HIT git.response.duration_ms=19 git.fetch.wants=de86690ef42f4500289463719eb80e5ff44c191c git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=5469 git.pkt.payload.max=5448 git.sideband.band1.bytes=5447 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e1ccb9cb0bc6f8
- Request
- 6a53e78a…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format