Artifactspublic-v0

curl

master
ply4 git calls31 ms290 KiB3 cachedprotocol v2
274 lines8.1 KiBRaw
1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) Jeroen Ooms <jeroenooms@gmail.com>
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 * SPDX-License-Identifier: curl
22 *
23 ***************************************************************************/
24/* <DESC>
25 * Web crawler based on curl and libxml2 to stress-test curl with
26 * hundreds of concurrent connections to various servers.
27 * </DESC>
28 */
29/*
30 * To compile:
31 * gcc crawler.c $(pkg-config --cflags --libs libxml-2.0 libcurl)
32 */
33#include <libxml/HTMLparser.h>
34#include <libxml/xpath.h>
35#include <libxml/uri.h>
37#include <stdlib.h>
38#include <string.h>
39#include <math.h>
40#include <signal.h>
42#include <curl/curl.h>
44/* Parameters */
45static int max_con = 200;
46static int max_total = 20000;
47static int max_requests = 500;
48static size_t max_link_per_page = 5;
49static int follow_relative_links = 0;
50static const char start_page[] = "https://www.reuters.com/";
52static int pending_interrupt = 0;
53static void sighandler(int dummy)
55 (void)dummy;
56 pending_interrupt = 1;
59/* resizable buffer */
60struct memory {
61 char *buf;
62 size_t size;
63};
65static size_t write_cb(char *contents, size_t sz, size_t nmemb, void *ctx)
67 size_t realsize = sz * nmemb;
68 struct memory *mem = (struct memory *)ctx;
69 char *ptr = realloc(mem->buf, mem->size + realsize);
70 if(!ptr) {
71 /* out of memory */
72 printf("not enough memory (realloc returned NULL)\n");
73 return 0;
74 }
75 mem->buf = ptr;
76 memcpy(&(mem->buf[mem->size]), contents, realsize);
77 mem->size += realsize;
78 return realsize;
81static CURL *make_handle(const char *url)
83 CURL *curl = curl_easy_init();
84 struct memory *mem;
86 /* Important: use HTTP2 over HTTPS */
87 curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
88 curl_easy_setopt(curl, CURLOPT_URL, url);
90 /* buffer body */
91 mem = malloc(sizeof(*mem));
92 mem->size = 0;
93 mem->buf = malloc(1);
94 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
95 curl_easy_setopt(curl, CURLOPT_WRITEDATA, mem);
96 curl_easy_setopt(curl, CURLOPT_PRIVATE, mem);
98 /* For completeness */
99 curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
100 curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L);
101 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
102 /* only allow redirects to HTTP and HTTPS URLs */
103 curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
104 curl_easy_setopt(curl, CURLOPT_AUTOREFERER, 1L);
105 curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
106 /* each transfer needs to be done within 20 seconds! */
107 curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 20000L);
108 /* connect fast or fail */
109 curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 2000L);
110 /* skip files larger than a gigabyte */
111 curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
112 (curl_off_t)1024 * 1024 * 1024);
113 curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "");
114 curl_easy_setopt(curl, CURLOPT_FILETIME, 1L);
115 curl_easy_setopt(curl, CURLOPT_USERAGENT, "mini crawler");
116 curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
117 curl_easy_setopt(curl, CURLOPT_UNRESTRICTED_AUTH, 1L);
118 curl_easy_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
119 curl_easy_setopt(curl, CURLOPT_EXPECT_100_TIMEOUT_MS, 0L);
120 return curl;
123/* HREF finder implemented in libxml2 but could be any HTML parser */
124static size_t follow_links(CURLM *multi, struct memory *mem, const char *url)
126 int opts = HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING |
127 HTML_PARSE_NONET;
128 htmlDocPtr doc = htmlReadMemory(mem->buf, (int)mem->size, url, NULL, opts);
129 size_t count;
130 int i;
131 xmlChar *xpath;
132 xmlNodeSetPtr nodeset;
133 xmlXPathContextPtr context;
134 xmlXPathObjectPtr object;
135 if(!doc)
136 return 0;
137 xpath = (xmlChar *)"//a/@href";
138 context = xmlXPathNewContext(doc);
139 if(!context) {
140 xmlFreeDoc(doc);
141 return 0;
142 }
143 object = xmlXPathEvalExpression(xpath, context);
144 xmlXPathFreeContext(context);
145 if(!object) {
146 xmlFreeDoc(doc);
147 return 0;
148 }
149 nodeset = object->nodesetval;
150 if(xmlXPathNodeSetIsEmpty(nodeset)) {
151 xmlXPathFreeObject(object);
152 xmlFreeDoc(doc);
153 return 0;
154 }
155 count = 0;
156 for(i = 0; i < nodeset->nodeNr; i++) {
157 double r = rand();
158 int x = (int)(r * nodeset->nodeNr / RAND_MAX);
159 const xmlNode *node = nodeset->nodeTab[x]->xmlChildrenNode;
160 xmlChar *href = xmlNodeListGetString(doc, node, 1);
161 char *link;
162 if(follow_relative_links) {
163 xmlChar *orig = href;
164 href = xmlBuildURI(href, (xmlChar *)url);
165 xmlFree(orig);
166 }
167 link = (char *)href;
168 if(!link || strlen(link) < 20) {
169 xmlFree(link);
170 continue;
171 }
172 if(!strncmp(link, "http://", 7) || !strncmp(link, "https://", 8)) {
173 if(count >= max_link_per_page) {
174 xmlFree(link);
175 break;
176 }
177 curl_multi_add_handle(multi, make_handle(link));
178 count++;
179 }
180 xmlFree(link);
181 }
182 xmlXPathFreeObject(object);
183 xmlFreeDoc(doc);
184 return count;
187static int is_html(const char *ctype)
189 return ctype && strlen(ctype) > 10 && strstr(ctype, "text/html");
192int main(void)
194 CURLM *multi;
195 int msgs_left;
196 int pending;
197 int complete;
198 int still_running;
199 CURLcode result;
201 result = curl_global_init(CURL_GLOBAL_ALL);
202 if(result != CURLE_OK)
203 return (int)result;
205 signal(SIGINT, sighandler);
206 LIBXML_TEST_VERSION
207 multi = curl_multi_init();
208 if(multi) {
209 curl_multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
210 curl_multi_setopt(multi, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);
212 /* enables http/2 if available */
213#ifdef CURLPIPE_MULTIPLEX
214 curl_multi_setopt(multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
215#endif
217 /* sets html start page */
218 curl_multi_add_handle(multi, make_handle(start_page));
220 pending = 0;
221 complete = 0;
222 still_running = 1;
223 while(still_running && !pending_interrupt) {
224 int numfds;
225 CURLMsg *m;
227 curl_multi_wait(multi, NULL, 0, 1000, &numfds);
228 curl_multi_perform(multi, &still_running);
230 /* See how the transfers went */
231 m = NULL;
232 while((m = curl_multi_info_read(multi, &msgs_left))) {
233 if(m->msg == CURLMSG_DONE) {
234 CURL *curl = m->easy_handle;
235 char *url;
236 struct memory *mem;
237 curl_easy_getinfo(curl, CURLINFO_PRIVATE, &mem);
238 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
239 if(m->data.result == CURLE_OK) {
240 long res_status;
241 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &res_status);
242 if(res_status == 200) {
243 char *ctype;
244 curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ctype);
245 printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url);
246 if(is_html(ctype) && mem->size > 100) {
247 if(pending < max_requests &&
248 (complete + pending) < max_total) {
249 pending += follow_links(multi, mem, url);
250 still_running = 1;
251 }
252 }
253 }
254 else {
255 printf("[%d] HTTP %d: %s\n", complete, (int)res_status, url);
256 }
257 }
258 else {
259 printf("[%d] Connection failure: %s\n", complete, url);
260 }
261 curl_multi_remove_handle(multi, curl);
262 curl_easy_cleanup(curl);
263 free(mem->buf);
264 free(mem);
265 complete++;
266 pending--;
267 }
268 }
269 }
270 curl_multi_cleanup(multi);
271 }
272 curl_global_cleanup();
273 return 0;
Diagnostics4 git calls · 31 ms
CallStatusSizeTime031 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=25
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2003.2 KiB · hit25 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3232
git.response.cache=HIT
git.response.duration_ms=25
git.fetch.wants=46f4ebb531ed53eebc2a2fcf86f63f99f7a99b06
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3232
git.pkt.payload.max=3211
git.sideband.band1.bytes=3210
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2487d2f78ee28
Request
2a78c577…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format