master
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24/* <DESC>
25 * multi socket interface with libev
26 * </DESC>
27 */
28/* Example application source code using the multi socket interface to
29 * download many files at once.
30 *
31 * This example features the same basic functionality as hiperfifo.c does,
32 * but this uses libev instead of libevent.
33 *
34 * Written by Jeff Pohlmeyer, converted to use libev by Markus Koetter
35 *
36 * Requires libev and a (POSIX?) system that has mkfifo().
37 *
38 * This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c"
39 * sample programs.
40 *
41 * When running, the program creates the named pipe "hiper.fifo"
42 *
43 * Whenever there is input into the fifo, the program reads the input as a list
44 * of URL's and creates some new easy handles to fetch each URL via the
45 * curl_multi "hiper" API.
46 *
47 * Thus, you can try a single URL:
48 * % echo http://www.example.com > hiper.fifo
49 *
50 * Or a whole bunch of them:
51 * % cat my-url-list > hiper.fifo
52 *
53 * The fifo buffer is handled almost instantly, so you can even add more URL's
54 * while the previous requests are still being downloaded.
55 *
56 * Note:
57 * For the sake of simplicity, URL length is limited to 1023 chars.
58 *
59 * This is purely a demo app, all retrieved data is discarded by
60 * the write callback.
61 */
62#include <errno.h>
63#include <fcntl.h>
64#include <stdio.h>
65#include <stdlib.h>
66#include <string.h>
67#include <sys/poll.h>
68#include <sys/stat.h>
69#include <sys/time.h>
70#include <time.h>
71#include <unistd.h>
73#include <ev.h>
75#include <curl/curl.h>
77#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */
79/* Global information, common to all connections */
80struct GlobalInfo {
81 struct ev_loop *loop;
82 struct ev_io fifo_event;
83 struct ev_timer timer_event;
84 CURLM *multi;
85 int still_running;
86 FILE *input;
87};
89/* Information associated with a specific easy handle */
90struct ConnInfo {
91 CURL *curl;
92 char *url;
93 struct GlobalInfo *global;
94 char error[CURL_ERROR_SIZE];
95};
97/* Information associated with a specific socket */
98struct SockInfo {
99 curl_socket_t sockfd;
100 CURL *curl;
101 int action;
102 long timeout;
103 struct ev_io ev;
104 int evset;
105 struct GlobalInfo *global;
106};
108/* Die if we get a bad CURLMcode somewhere */
109static void mcode_or_die(const char *where, CURLMcode code)
110{
111 if(CURLM_OK != code) {
112 const char *s;
113 switch(code) {
114 case CURLM_BAD_HANDLE:
115 s = "CURLM_BAD_HANDLE";
116 break;
117 case CURLM_BAD_EASY_HANDLE:
118 s = "CURLM_BAD_EASY_HANDLE";
119 break;
120 case CURLM_OUT_OF_MEMORY:
121 s = "CURLM_OUT_OF_MEMORY";
122 break;
123 case CURLM_INTERNAL_ERROR:
124 s = "CURLM_INTERNAL_ERROR";
125 break;
126 case CURLM_UNKNOWN_OPTION:
127 s = "CURLM_UNKNOWN_OPTION";
128 break;
129 case CURLM_LAST:
130 s = "CURLM_LAST";
131 break;
132 default:
133 s = "CURLM_unknown";
134 break;
135 case CURLM_BAD_SOCKET:
136 s = "CURLM_BAD_SOCKET";
137 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
138 /* ignore this error */
139 return;
140 }
141 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
142 exit(code);
143 }
144}
146/* Check for completed transfers, and remove their easy handles */
147static void check_multi_info(struct GlobalInfo *g)
148{
149 char *eff_url;
150 CURLMsg *msg;
151 int msgs_left;
152 struct ConnInfo *conn;
154 fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running);
155 while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
156 if(msg->msg == CURLMSG_DONE) {
157 CURL *curl = msg->easy_handle;
158 CURLcode result = msg->data.result;
159 curl_easy_getinfo(curl, CURLINFO_PRIVATE, &conn);
160 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &eff_url);
161 fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, result, conn->error);
162 curl_multi_remove_handle(g->multi, curl);
163 free(conn->url);
164 curl_easy_cleanup(curl);
165 free(conn);
166 }
167 }
168}
170/* Called by libevent when our timeout expires */
171static void timer_cb(EV_P_ struct ev_timer *w, int revents)
172{
173 CURLMcode mresult;
174 struct GlobalInfo *g;
176 printf("%s w %p revents %d\n", __PRETTY_FUNCTION__, (void *)w, revents);
178 g = (struct GlobalInfo *)w->data;
180 mresult = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0,
181 &g->still_running);
182 mcode_or_die("timer_cb: curl_multi_socket_action", mresult);
183 check_multi_info(g);
184}
186/* Update the event timer after curl_multi library calls */
187static int multi_timer_cb(CURLM *multi, long timeout_ms, struct GlobalInfo *g)
188{
189 (void)multi;
190 printf("%s %li\n", __PRETTY_FUNCTION__, timeout_ms);
191 ev_timer_stop(g->loop, &g->timer_event);
192 if(timeout_ms >= 0) {
193 /* -1 means delete, other values are timeout times in milliseconds */
194 double t = timeout_ms / 1000;
195 ev_timer_init(&g->timer_event, timer_cb, t, 0.);
196 ev_timer_start(g->loop, &g->timer_event);
197 }
198 return 0;
199}
201/* Called by libevent when we get action on a multi socket */
202static void event_cb(EV_P_ struct ev_io *w, int revents)
203{
204 CURLMcode mresult;
205 struct GlobalInfo *g;
207 int action = ((revents & EV_READ) ? CURL_POLL_IN : 0) |
208 ((revents & EV_WRITE) ? CURL_POLL_OUT : 0);
210 printf("%s w %p revents %d\n", __PRETTY_FUNCTION__, (void *)w, revents);
211 g = (struct GlobalInfo *)w->data;
213 mresult = curl_multi_socket_action(g->multi, w->fd, action,
214 &g->still_running);
215 mcode_or_die("event_cb: curl_multi_socket_action", mresult);
216 check_multi_info(g);
217 if(g->still_running <= 0) {
218 fprintf(MSG_OUT, "last transfer done, kill timeout\n");
219 ev_timer_stop(g->loop, &g->timer_event);
220 }
221}
223/* Clean up the SockInfo structure */
224static void remsock(struct SockInfo *f, struct GlobalInfo *g)
225{
226 printf("%s \n", __PRETTY_FUNCTION__);
227 if(f) {
228 if(f->evset)
229 ev_io_stop(g->loop, &f->ev);
230 free(f);
231 }
232}
234/* Assign information to a SockInfo structure */
235static void setsock(struct SockInfo *f, curl_socket_t s, CURL *e, int act,
236 struct GlobalInfo *g)
237{
238 int kind = ((act & CURL_POLL_IN) ? EV_READ : 0) |
239 ((act & CURL_POLL_OUT) ? EV_WRITE : 0);
241 printf("%s \n", __PRETTY_FUNCTION__);
243 f->sockfd = s;
244 f->action = act;
245 f->curl = e;
246 if(f->evset)
247 ev_io_stop(g->loop, &f->ev);
248 ev_io_init(&f->ev, event_cb, f->sockfd, kind);
249 f->ev.data = g;
250 f->evset = 1;
251 ev_io_start(g->loop, &f->ev);
252}
254/* Initialize a new SockInfo structure */
255static void addsock(curl_socket_t s, CURL *curl, int action,
256 struct GlobalInfo *g)
257{
258 struct SockInfo *fdp = calloc(1, sizeof(struct SockInfo));
260 fdp->global = g;
261 setsock(fdp, s, curl, action, g);
262 curl_multi_assign(g->multi, s, fdp);
263}
265/* CURLMOPT_SOCKETFUNCTION */
266static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp)
267{
268 struct GlobalInfo *g = (struct GlobalInfo *)cbp;
269 struct SockInfo *fdp = (struct SockInfo *)sockp;
270 static const char * const whatstr[] = {
271 "none", "IN", "OUT", "INOUT", "REMOVE"
272 };
274 printf("%s e %p s %d what %d cbp %p sockp %p\n",
275 __PRETTY_FUNCTION__, e, s, what, cbp, sockp);
277 fprintf(MSG_OUT, "socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]);
278 if(what == CURL_POLL_REMOVE) {
279 fprintf(MSG_OUT, "\n");
280 remsock(fdp, g);
281 }
282 else {
283 if(!fdp) {
284 fprintf(MSG_OUT, "Adding data: %s\n", whatstr[what]);
285 addsock(s, e, what, g);
286 }
287 else {
288 fprintf(MSG_OUT, "Changing action from %s to %s\n",
289 whatstr[fdp->action], whatstr[what]);
290 setsock(fdp, s, e, what, g);
291 }
292 }
293 return 0;
294}
296/* CURLOPT_WRITEFUNCTION */
297static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *data)
298{
299 size_t realsize = size * nmemb;
300 struct ConnInfo *conn = (struct ConnInfo *)data;
301 (void)ptr;
302 (void)conn;
303 return realsize;
304}
306/* CURLOPT_XFERINFOFUNCTION */
307static int xferinfo_cb(void *p, curl_off_t dltotal, curl_off_t dlnow,
308 curl_off_t ult, curl_off_t uln)
309{
310 struct ConnInfo *conn = (struct ConnInfo *)p;
311 (void)ult;
312 (void)uln;
314 fprintf(MSG_OUT, "Progress: %s (%" CURL_FORMAT_CURL_OFF_T "/"
315 "%" CURL_FORMAT_CURL_OFF_T ")\n",
316 conn->url, dlnow, dltotal);
317 return 0;
318}
320/* Create a new easy handle, and add it to the global curl_multi */
321static void new_conn(const char *url, struct GlobalInfo *g)
322{
323 struct ConnInfo *conn;
324 CURLMcode mresult;
326 conn = calloc(1, sizeof(*conn));
327 conn->error[0] = '\0';
329 conn->curl = curl_easy_init();
330 if(!conn->curl) {
331 fprintf(MSG_OUT, "curl_easy_init() failed, exiting!\n");
332 exit(2);
333 }
334 conn->global = g;
335 conn->url = strdup(url);
336 curl_easy_setopt(conn->curl, CURLOPT_URL, conn->url);
337 curl_easy_setopt(conn->curl, CURLOPT_WRITEFUNCTION, write_cb);
338 curl_easy_setopt(conn->curl, CURLOPT_WRITEDATA, conn);
339 curl_easy_setopt(conn->curl, CURLOPT_VERBOSE, 1L);
340 curl_easy_setopt(conn->curl, CURLOPT_ERRORBUFFER, conn->error);
341 curl_easy_setopt(conn->curl, CURLOPT_PRIVATE, conn);
342 curl_easy_setopt(conn->curl, CURLOPT_NOPROGRESS, 0L);
343 curl_easy_setopt(conn->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
344 curl_easy_setopt(conn->curl, CURLOPT_PROGRESSDATA, conn);
345 curl_easy_setopt(conn->curl, CURLOPT_LOW_SPEED_TIME, 3L);
346 curl_easy_setopt(conn->curl, CURLOPT_LOW_SPEED_LIMIT, 10L);
348 fprintf(MSG_OUT, "Adding easy %p to multi %p (%s)\n",
349 conn->curl, g->multi, url);
350 mresult = curl_multi_add_handle(g->multi, conn->curl);
351 mcode_or_die("new_conn: curl_multi_add_handle", mresult);
353 /* note that add_handle() sets a timeout to trigger soon so that the
354 necessary socket_action() gets called */
355}
357/* This gets called whenever data is received from the fifo */
358static void fifo_cb(EV_P_ struct ev_io *w, int revents)
359{
360 char s[1024];
361 long int rv = 0;
362 int n = 0;
363 struct GlobalInfo *g = (struct GlobalInfo *)w->data;
365 (void)revents;
367 do {
368 s[0] = '\0';
369 rv = fscanf(g->input, "%1023s%n", s, &n);
370 s[n] = '\0';
371 if(n && s[0]) {
372 new_conn(s, g); /* if we read a URL, go get it! */
373 }
374 else
375 break;
376 } while(rv != EOF);
377}
379/* Create a named pipe and tell libevent to monitor it */
380static int init_fifo(struct GlobalInfo *g)
381{
382 struct stat st;
383 static const char fifo[] = "hiper.fifo";
384 curl_socket_t sockfd;
386 fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo);
387 if(!lstat(fifo, &st)) {
388 if((st.st_mode & S_IFMT) == S_IFREG) {
389 errno = EEXIST;
390 perror("lstat");
391 return 1;
392 }
393 }
394 unlink(fifo);
395 if(mkfifo(fifo, 0600) == -1) {
396 perror("mkfifo");
397 return 1;
398 }
399 sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0);
400 if(sockfd == -1) {
401 perror("open");
402 return 1;
403 }
404 g->input = fdopen(sockfd, "r");
406 fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo);
407 ev_io_init(&g->fifo_event, fifo_cb, sockfd, EV_READ);
408 ev_io_start(g->loop, &g->fifo_event);
409 return 0;
410}
412int main(void)
413{
414 CURLcode result;
415 struct GlobalInfo g;
417 result = curl_global_init(CURL_GLOBAL_ALL);
418 if(result != CURLE_OK)
419 return (int)result;
421 memset(&g, 0, sizeof(g));
422 g.loop = ev_default_loop(0);
424 if(init_fifo(&g))
425 return 1;
426 g.multi = curl_multi_init();
428 ev_timer_init(&g.timer_event, timer_cb, 0., 0.);
429 g.timer_event.data = &g;
430 g.fifo_event.data = &g;
431 curl_multi_setopt(g.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
432 curl_multi_setopt(g.multi, CURLMOPT_SOCKETDATA, &g);
433 curl_multi_setopt(g.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
434 curl_multi_setopt(g.multi, CURLMOPT_TIMERDATA, &g);
436 /* we do not call any curl_multi_socket*() function yet as we have no handles
437 added! */
439 ev_loop(g.loop, 0);
440 curl_multi_cleanup(g.multi);
441 curl_global_cleanup();
442 return 0;
443}
Diagnostics4 git calls · 178 ms
CallStatusSizeTime0178 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2004.5 KiB173 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=173 git.response.bytes=4574 git.response.cache=MISS git.response.duration_ms=173 git.fetch.wants=32f92f9415c5fc49577da7c08910b688d44691cf git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4574 git.pkt.payload.max=4553 git.sideband.band1.bytes=4552 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e1c34dbd7c36cd
- Request
- e8e4fe8e…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format