Artifactspublic-v0

curl

master
ply4 git calls29 ms317 KiB3 cachedprotocol v2
3401 lines111 KiBexecutableRaw
1#!/usr/bin/env perl
2#***************************************************************************
3# _ _ ____ _
4# Project ___| | | | _ \| |
5# / __| | | | |_) | |
6# | (__| |_| | _ <| |___
7# \___|\___/|_| \_\_____|
8#
9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
11# This software is licensed as described in the file COPYING, which
12# you should have received as part of this distribution. The terms
13# are also available at https://curl.se/docs/copyright.html.
15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
16# copies of the Software, and permit persons to whom the Software is
17# furnished to do so, under the terms of the COPYING file.
19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20# KIND, either express or implied.
22# SPDX-License-Identifier: curl
24###########################################################################
26# For documentation, see docs/runtests.md and README.md.
28# Experimental hooks are available to run tests remotely on machines that
29# are able to run curl but are unable to run the test harness.
30# The following sections need to be modified:
32# $HOSTIP, $HOST6IP - Set to the address of the host running the test suite
33# $CLIENTIP, $CLIENT6IP - Set to the address of the host running curl
34# runclient, runclientoutput - Modify to copy all the files in the log/
35# directory to the system running curl, run the given command remotely
36# and save the return code or returned stdout (respectively), then
37# copy all the files from the remote system's log/ directory back to
38# the host running the test suite. This can be done a few ways, such
39# as using scp & ssh, rsync & telnet, or using a NFS shared directory
40# and ssh.
42# 'make && make test' needs to be done on both machines before making the
43# above changes and running runtests.pl manually. In the shared NFS case,
44# the contents of the tests/server/ directory must be from the host
45# running the test suite, while the rest must be from the host running curl.
47# Note that even with these changes a number of tests still fail (mainly
48# to do with cookies, those that set environment variables, or those that
49# do more than touch the file system in a <precheck> or <postcheck>
50# section). These can be added to the $TESTCASES line below,
51# e.g. $TESTCASES="!8 !31 !63 !cookies..."
53# Finally, to properly support -g and -n, checktestcmd needs to change
54# to check the remote system's PATH, and the places in the code where
55# the curl binary is read directly to determine its type also need to be
56# fixed. As long as the -g option is never given, and the -n is always
57# given, this is not a problem.
59use strict;
60use warnings;
61use 5.006;
62use POSIX qw(strftime);
64# These should be the only variables that might be needed to get edited:
66BEGIN {
67 # Define srcdir to the location of the tests source directory. This is
68 # usually set by the Makefile, but for out-of-tree builds with direct
69 # invocation of runtests.pl, it may not be set.
70 if(!defined $ENV{'srcdir'}) {
71 use File::Basename;
72 $ENV{'srcdir'} = dirname(__FILE__);
73 }
74 push(@INC, $ENV{'srcdir'});
75 # run time statistics needs Time::HiRes
76 eval {
77 no warnings "all";
78 require Time::HiRes;
79 import Time::HiRes qw( time );
80 }
83use Digest::MD5 qw(md5);
84use List::Util 'sum';
85use I18N::Langinfo qw(langinfo CODESET);
86use POSIX qw(setlocale LC_ALL);
88use serverhelp qw(
89 server_exe
90 );
91use pathhelp qw(
92 exe_ext
93 sys_native_current_path
94 shell_quote
95 );
97use appveyor;
98use azure;
99use getpart; # array functions
100use servers;
101use valgrind; # valgrind report parser
102use globalconfig;
103use runner;
104use testutil;
105use memanalyzer;
107my %custom_skip_reasons;
109my $ACURL = $VCURL; # what curl binary to use to talk to APIs (relevant for CI)
110 # ACURL is handy to set to the system one for reliability
111my $CURLCONFIG = "../curl-config"; # curl-config from current build
113# Normally, all test cases should be run, but at times it is handy to
114# run a particular one:
115my $TESTCASES = "all";
117# To run specific test cases, set them like:
118# $TESTCASES = "1 2 3 7 8";
120#######################################################################
121# No variables below this point should need to be modified
124my $libtool;
125my $repeat = 0;
126my $retry = 0;
128my $start; # time at which testing started
129my $args; # command-line arguments
131my $uname_release = qx(uname -r);
132my $is_wsl = $uname_release =~ /Microsoft$/;
133my $uname_release_ver = $uname_release =~ /^(\d+)\.(\d+)\.(\d+)/ ? $1 * 10000 + $2 * 100 + $3 : 0;
135my $http_ipv6; # set if HTTP server has IPv6 support
136my $http_unix; # set if HTTP server has Unix sockets support
137my $ftp_ipv6; # set if FTP server has IPv6 support
139my $resolver; # name of the resolver backend (for human presentation)
141my %skipped; # skipped{reason}=counter, reasons for skip
142my @teststat; # teststat[testnum]=reason, reasons for skip
143my %disabled_keywords; # key words of tests to skip
144my %ignored_keywords; # key words of tests to ignore results
145my %enabled_keywords; # key words of tests to run
146my %disabled; # disabled test cases
147my %ignored; # ignored results of test cases
148my %ignoretestcodes; # if test results are to be ignored
149my %may_retry; # tests we want to retry on failure
151my @global_strip_stderr; # global patterns added to stripfile before stderr check
152my @global_strip_file; # global patterns added to stripfile before file check
154my $passedign; # tests passed with results ignored
156my $timestats; # time stamping and stats generation
157my $fullstats; # show time stats for every single test
158my %timeprepini; # timestamp for each test preparation start
159my %timesrvrini; # timestamp for each test required servers verification start
160my %timesrvrend; # timestamp for each test required servers verification end
161my %timetoolini; # timestamp for each test command run starting
162my %timetoolend; # timestamp for each test command run stopping
163my %timesrvrlog; # timestamp for each test server logs lock removal
164my %timevrfyend; # timestamp for each test result verification end
165my $globalabort; # flag signalling program abort
167# values for $singletest_state
168use constant {
169 ST_INIT => 0,
170 ST_INITED => 2,
171 ST_PREPROCESS => 3,
172 ST_RUN => 4,
173};
174my %singletest_state; # current state of singletest() by runner ID
175my %singletest_logs; # log messages while in singletest array ref by runner
176my $singletest_bufferedrunner; # runner ID which is buffering logs
177my %runnerids; # runner IDs by number
178my %runnernums; # runner numbers by IDs (inverted %runnerids)
179my @runnersidle; # runner IDs idle and ready to execute a test
180my %countforrunner; # test count by runner ID
181my %runnersrunning; # tests currently running by runner ID
183#######################################################################
184# variables that command line options may set
186my $short;
187my $no_debuginfod;
188my $postmortem; # display detailed info about failed tests
189my $show_success_logs = 0; # display detailed info about passed tests
190my $run_disabled; # run the specific tests even if listed in DISABLED
191my $scrambleorder;
192my $jobs = 0;
194# Azure Pipelines specific variables
195my $AZURE_RUN_ID = 0;
196my $AZURE_RESULT_ID = 0;
198#######################################################################
199# logmsg is our general message logging subroutine.
201sub logmsg {
202 if($singletest_bufferedrunner) {
203 # Logs are currently being buffered
204 return singletest_logmsg(@_);
205 }
206 for(@_) {
207 my $line = $_;
208 if(!$line) {
209 next;
210 }
211 if($is_wsl) {
212 # use \r\n for WSL shell
213 $line =~ s/\r?\n$/\r\n/g;
214 }
215 print $line;
216 }
219#######################################################################
220# enable logmsg buffering for the given runner ID
222sub logmsg_bufferfortest {
223 my ($runnerid) = @_;
224 if($jobs) {
225 # Only enable buffering in multiprocess mode
226 $singletest_bufferedrunner = $runnerid;
227 }
230#######################################################################
231# Store a log message in a buffer for this test
232# The messages can then be displayed all at once at the end of the test
233# which prevents messages from different tests from being interleaved.
234sub singletest_logmsg {
235 if(!exists $singletest_logs{$singletest_bufferedrunner}) {
236 # initialize to a reference to an empty anonymous array
237 $singletest_logs{$singletest_bufferedrunner} = [];
238 }
239 my $logsref = $singletest_logs{$singletest_bufferedrunner};
240 push @$logsref, @_;
243#######################################################################
244# Stop buffering log messages, but do not touch them
245sub singletest_unbufferlogs {
246 undef $singletest_bufferedrunner;
249#######################################################################
250# Clear the buffered log messages & stop buffering after returning them
251sub singletest_dumplogs {
252 if(!defined $singletest_bufferedrunner) {
253 # probably not multiprocess mode and logs were not buffered
254 return undef;
255 }
256 my $logsref = $singletest_logs{$singletest_bufferedrunner};
257 my $msg = join("", @$logsref);
258 delete $singletest_logs{$singletest_bufferedrunner};
259 singletest_unbufferlogs();
260 return $msg;
263sub catch_zap {
264 my $signame = shift;
265 print "runtests.pl received SIG$signame, exiting\r\n";
266 $globalabort = 1;
268$SIG{INT} = \&catch_zap;
269$SIG{TERM} = \&catch_zap;
271sub catch_usr1 {
272 print "runtests.pl internal state:\r\n";
273 print scalar(%runnersrunning) . " busy test runner(s) of " . scalar(keys %runnerids) . "\r\n";
274 foreach my $rid (sort(keys(%runnersrunning))) {
275 my $runnernum = "unknown";
276 if(exists $runnernums{$rid}) {
277 $runnernum = $runnernums{$rid};
278 }
279 print "Runner $runnernum (id $rid) running test $runnersrunning{$rid} in state $singletest_state{$rid}\r\n";
280 }
283eval {
284 # some msys2 perl versions do not define SIGUSR1
285 $SIG{USR1} = \&catch_usr1;
286};
287$SIG{PIPE} = 'IGNORE'; # these errors are captured in the read/write calls
289##########################################################################
290# Clear all possible '*_proxy' environment variables for various protocols
291# to prevent them to interfere with our testing!
293foreach my $protocol (('ftp', 'http', 'ftps', 'https', 'no', 'all')) {
294 my $proxy = "${protocol}_proxy";
295 # clear lowercase version
296 delete $ENV{$proxy} if($ENV{$proxy});
297 # clear uppercase version
298 delete $ENV{uc($proxy)} if($ENV{uc($proxy)});
301# make sure we do not get affected by other variables that control our
302# behavior
304delete $ENV{'SSL_CERT_DIR'} if($ENV{'SSL_CERT_DIR'});
305delete $ENV{'SSL_CERT_PATH'} if($ENV{'SSL_CERT_PATH'});
306delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
308# provide defaults from our config file for ENV vars not explicitly
309# set by the caller
310if(open(my $fd, "<", "config")) {
311 while(my $line = <$fd>) {
312 next if($line =~ /^#/);
313 chomp $line;
314 my ($name, $val) = split(/\s*:\s*/, $line, 2);
315 $ENV{$name} = $val if(!$ENV{$name});
316 }
317 close($fd);
320# Check if we have nghttpx available and if it talks http/3
321my $nghttpx_h3 = 0;
322if(!$ENV{"NGHTTPX"}) {
323 $ENV{"NGHTTPX"} = checktestcmd("nghttpx");
325if($ENV{"NGHTTPX"}) {
326 my $cmd = "\"$ENV{'NGHTTPX'}\" -v 2>$dev_null";
327 my $nghttpx_version = join(' ', qx($cmd));
328 $nghttpx_h3 = $nghttpx_version =~ /nghttp3\//;
329 chomp $nghttpx_h3;
332#######################################################################
333# Get the list of tests that the tests/data/Makefile.am knows about!
335my $disttests = "";
336sub get_disttests {
337 # If a non-default $TESTDIR is being used there may not be any
338 # Makefile.am in which case there is nothing to do.
339 open(my $dh, "<", "$TESTDIR/Makefile.am") or return;
340 while(<$dh>) {
341 chomp $_;
342 if(($_ =~ /^#/) ||($_ !~ /test/)) {
343 next;
344 }
345 $disttests .= $_;
346 }
347 close($dh);
350#######################################################################
351# Remove all files in the specified directory
353sub cleardir {
354 my $dir = $_[0];
355 my $done = 1; # success
356 my $file;
358 # Get all files
359 opendir(my $dh, $dir) or return 0; # cannot open dir
360 while($file = readdir($dh)) {
361 # Do not clear the $PIDDIR or $LOCKDIR since those need to live beyond
362 # one test
363 if(($file !~ /^(\.|\.\.)\z/) &&
364 $file ne $PIDDIR && $file ne $LOCKDIR) {
365 if(-d "$dir/$file") {
366 if(!cleardir("$dir/$file")) {
367 $done = 0;
368 }
369 if(!rmdir("$dir/$file")) {
370 $done = 0;
371 }
372 }
373 else {
374 # Ignore stunnel since we cannot do anything about its locks
375 if(!unlink("$dir/$file") && $file !~ /_stunnel\.log$/) {
376 $done = 0;
377 }
378 }
379 }
380 }
381 closedir $dh;
382 return $done;
385#######################################################################
386# Given two array references, this function stores them in two temporary
387# files, run 'diff' on them, store the result and return the diff output!
388sub showdiff {
389 my ($logdir, $firstref, $secondref) = @_;
391 my $file1 = "$logdir/check-generated";
392 my $file2 = "$logdir/check-expected";
394 open(my $temp, ">", $file1) or die "Failure writing diff file";
395 for(@$firstref) {
396 my $l = $_;
397 $l =~ s/\r/[CR]/g;
398 $l =~ s/\n/[LF]/g;
399 $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
400 print $temp $l;
401 print $temp "\n";
402 }
403 close($temp) or die "Failure writing diff file";
405 open($temp, ">", $file2) or die "Failure writing diff file";
406 for(@$secondref) {
407 my $l = $_;
408 $l =~ s/\r/[CR]/g;
409 $l =~ s/\n/[LF]/g;
410 $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
411 print $temp $l;
412 print $temp "\n";
413 }
414 close($temp) or die "Failure writing diff file";
415 my @out = qx(diff -u $file2 $file1 2>$dev_null);
417 if(!$out[0]) {
418 @out = qx(diff -c $file2 $file1 2>$dev_null);
419 if(!$out[0]) {
420 logmsg "Failed to show diff. The diff tool may be missing.\n";
421 }
422 }
424 return @out;
427#######################################################################
428# compare test results with the expected output, we might filter off
429# some pattern that is allowed to differ, output test results
431sub compare {
432 my ($runnerid, $testnum, $testname, $subject, $firstref, $secondref) = @_;
434 my $result = compareparts($firstref, $secondref);
436 if($result) {
437 # timestamp test result verification end
438 $timevrfyend{$testnum} = Time::HiRes::time();
440 if(!$short) {
441 logmsg "\n $testnum: $subject FAILED:\n";
442 my $logdir = getrunnerlogdir($runnerid);
443 logmsg showdiff($logdir, $firstref, $secondref);
444 }
445 elsif(!$automakestyle) {
446 logmsg "FAILED\n";
447 }
448 else {
449 # automakestyle
450 logmsg "FAIL: $testnum - $testname - $subject\n";
451 }
452 }
453 return $result;
456#######################################################################
457# Numeric-sort words in a string
458sub numsortwords {
459 my ($string) = @_;
460 return join(' ', sort { $a <=> $b } split(' ', $string));
463#######################################################################
464# Parse and store the protocols in curl's Protocols: line
465sub parseprotocols {
466 my ($line) = @_;
468 @protocols = split(' ', lc($line));
470 # Generate a "proto-ipv6" version of each protocol to match the
471 # IPv6 <server> name and a "proto-unix" to match the variant which
472 # uses Unix domain sockets. This works even if support is not
473 # compiled in because the <features> test fails.
474 push @protocols, map(("$_-ipv6", "$_-unix"), @protocols);
476 # 'http-proxy' is used in test cases to do CONNECT through
477 push @protocols, 'http-proxy';
479 # 'https-mtls' is used for client certificate auth testing
480 push @protocols, 'https-mtls';
482 # 'none' is used in test cases to mean no server
483 push @protocols, 'none';
486#######################################################################
487# Check if the operating environment supports UTF-8.
488sub is_utf8_supported {
489 my $result;
490 my $old_LC_ALL;
491 my $was_defined = defined $ENV{'LC_ALL'};
492 if($was_defined) {
493 $old_LC_ALL = $ENV{'LC_ALL'};
494 }
495 setlocale(LC_ALL, $ENV{'LC_ALL'} = "C.UTF-8");
496 $result = lc(langinfo(CODESET())) eq "utf-8";
497 if($was_defined) {
498 $ENV{'LC_ALL'} = $old_LC_ALL;
499 }
500 else {
501 delete $ENV{'LC_ALL'};
502 }
503 return $result;
506#######################################################################
507# Check & display information about curl and the host the test suite runs on.
508# Information to do with servers is displayed in displayserverfeatures, after
509# the server initialization is performed.
510sub checksystemfeatures {
511 my $proto;
512 my $feat;
513 my $curl;
514 my $libcurl;
515 my @disabled;
516 my $dis = "";
518 my $versioncmd = exerunner() . shell_quote($CURL) . ' --disable --version 2>&1';
519 my $versout = qx($versioncmd);
520 my $versretval = $?;
521 print "CMD ($versretval): $versioncmd\n" if($verbose && !$torture);
522 my @version = split(/\n/, $versout);
524 my $current_time = int(time());
525 $ENV{'SOURCE_DATE_EPOCH'} = $current_time;
526 $DATE = strftime "%Y-%m-%d", gmtime($current_time);
528 open(my $disabledh, "-|", exerunner() . shell_quote($CURLINFO));
529 while(<$disabledh>) {
530 if($_ =~ /([^:]*): ([ONF]*)/) {
531 my ($val, $toggle) = ($1, $2);
532 push @disabled, $val if($toggle eq "OFF");
533 $feature{$val} = 1 if($toggle eq "ON");
534 }
535 }
536 close($disabledh);
538 if($disabled[0]) {
539 s/[\r\n]//g for @disabled;
540 $dis = join(", ", @disabled);
541 }
543 $resolver = "stock";
544 for(@version) {
545 chomp;
547 if($_ =~ /^curl ([^ ]*)/) {
548 $curl = $_;
549 $CURLVERSION = $1;
550 $CURLVERNUM = $CURLVERSION;
551 $CURLVERNUM =~ s/^([0-9.]+)(.*)/$1/; # leading digits and dots
552 $curl =~ s/^(.*)(libcurl.*)/$1/g or die "Failure determining curl binary version";
554 $libcurl = $2;
555 if($curl =~ /win32|Windows|windows|mingw(32|64)/) {
556 # This is a Windows MinGW build or native build, we need to use
557 # Windows-style path.
558 $pwd = sys_native_current_path();
559 $feature{"win32"} = 1;
560 }
561 if($curl =~ /cygwin|msys/i) {
562 $feature{"cygwin"} = 1;
563 }
564 # Assume set but empty environment variables are supported, except
565 # by MSYS2 <3.4.2 (2022-12-16) processes starting non-Cygwin ones.
566 if(($^O ne 'cygwin' && $^O ne 'msys') || $curl =~ /cygwin|msys/i || $uname_release_ver >= 30605) {
567 $feature{"blank-env"} = 1;
568 }
569 if($libcurl =~ /\sschannel\b/i) {
570 $feature{"Schannel"} = 1;
571 $feature{"SSLpinning"} = 1;
572 }
573 elsif($libcurl =~ /\sopenssl\b/i) {
574 $feature{"OpenSSL"} = 1;
575 $feature{"SSLpinning"} = 1;
576 }
577 elsif($libcurl =~ /\sgnutls\b/i) {
578 $feature{"GnuTLS"} = 1;
579 $feature{"SSLpinning"} = 1;
580 }
581 elsif($libcurl =~ /\srustls-ffi\b/i) {
582 $feature{"rustls"} = 1;
583 }
584 elsif($libcurl =~ /\swolfssl\b/i) {
585 $feature{"wolfssl"} = 1;
586 $feature{"SSLpinning"} = 1;
587 if($libcurl =~ /\swolfssl\/5\.9\.2\b/i) {
588 $feature{"wolfssl-5.9.2"} = 1;
589 }
590 }
591 elsif($libcurl =~ /\s(AWS-LC|BoringSSL)\b/i) {
592 # OpenSSL compatible API
593 $feature{"OpenSSL"} = 1;
594 $feature{"SSLpinning"} = 1;
595 }
596 elsif($libcurl =~ /\slibressl\b/i) {
597 # OpenSSL compatible API
598 $feature{"OpenSSL"} = 1;
599 $feature{"SSLpinning"} = 1;
600 if($libcurl =~ /\slibressl\/4\.4\.0\b/i) {
601 $feature{"libressl-4.4.0"} = 1;
602 }
603 }
604 elsif($libcurl =~ /\squictls\b/i) {
605 # OpenSSL compatible API
606 $feature{"OpenSSL"} = 1;
607 $feature{"SSLpinning"} = 1;
608 }
609 elsif($libcurl =~ /\smbedTLS\b/i) {
610 $feature{"mbedtls"} = 1;
611 $feature{"SSLpinning"} = 1;
612 }
613 if($libcurl =~ /ares/i) {
614 $feature{"c-ares"} = 1;
615 $resolver = "c-ares";
616 }
617 if($libcurl =~ /nghttp2/i) {
618 # nghttp2 supports h2c
619 $feature{"h2c"} = 1;
620 }
621 if($libcurl =~ /AppleIDN/) {
622 $feature{"AppleIDN"} = 1;
623 }
624 if($libcurl =~ /WinIDN/) {
625 $feature{"WinIDN"} = 1;
626 }
627 if($libcurl =~ /libidn2/) {
628 $feature{"libidn2"} = 1;
629 }
630 if($libcurl =~ /libssh2/i) {
631 $feature{"libssh2"} = 1;
632 $feature{"sshkeyalgo"} = ($ENV{'CURL_TEST_SSH_KEYALGO'} and
633 $ENV{'CURL_TEST_SSH_KEYALGO'} =~ /^(?:rsa|ecdsa|ed25519)$/) ? $ENV{'CURL_TEST_SSH_KEYALGO'} : 'rsa';
634 }
635 if($libcurl =~ /libssh\/([0-9.]*)\//i) {
636 $feature{"libssh"} = 1;
637 $feature{"sshkeyalgo"} = ($ENV{'CURL_TEST_SSH_KEYALGO'} and
638 $ENV{'CURL_TEST_SSH_KEYALGO'} =~ /^(?:rsa|ecdsa|ed25519)$/) ? $ENV{'CURL_TEST_SSH_KEYALGO'} : 'rsa';
639 # Detect simple cases of default libssh configuration files ending up
640 # setting 'StrictHostKeyChecking no'. include files, quoted values,
641 # '=value' format not implemented.
642 $feature{"badlibssh"} = 0;
643 foreach my $libssh_configfile (('/etc/ssh/ssh_config', $ENV{'HOME'} . '/.ssh/config')) {
644 if(open(my $fd, '<', $libssh_configfile)) {
645 while(my $line = <$fd>) {
646 chomp $line;
647 if($line =~ /^\s*StrictHostKeyChecking\s+(yes|no)\s*$/) {
648 $feature{"badlibssh"} = ($1 eq 'no' ? 1 : 0);
649 last; # Do as openssh and libssh
650 }
651 }
652 close($fd);
653 }
654 }
655 }
656 }
657 elsif($_ =~ /^Protocols: (.*)/i) {
658 $proto = $1;
659 # these are the protocols compiled in to this libcurl
660 parseprotocols($proto);
661 }
662 elsif($_ =~ /^Features: (.*)/i) {
663 $feat = $1;
665 # built with memory tracking support (--enable-debug); may be disabled later
666 $feature{"TrackMemory"} = $feat =~ /\bDebug/;
667 # curl was built with --enable-debug
668 $feature{"Debug"} = $feat =~ /\bDebug/;
669 # SSL enabled
670 $feature{"SSL"} = $feat =~ /SSL/i;
671 # multiple SSL backends available.
672 $feature{"MultiSSL"} = $feat =~ /MultiSSL/i;
673 # embedded CA certificate bundle
674 $feature{"CAcert"} = $feat =~ /CAcert/i;
675 # large file support
676 $feature{"Largefile"} = $feat =~ /Largefile/i;
677 # IDN support
678 $feature{"IDN"} = $feat =~ /IDN/i;
679 # IPv6 support
680 $feature{"IPv6"} = $feat =~ /IPv6/i;
681 # Unix sockets support
682 $feature{"UnixSockets"} = $feat =~ /UnixSockets/i;
683 # libz compression
684 $feature{"libz"} = $feat =~ /libz/i;
685 # Brotli compression
686 $feature{"brotli"} = $feat =~ /brotli/i;
687 # Zstd compression
688 $feature{"zstd"} = $feat =~ /zstd/i;
689 # SSPI enabled
690 $feature{"SSPI"} = $feat =~ /SSPI/i;
691 # GSS-API enabled
692 $feature{"GSS-API"} = $feat =~ /GSS-API/i;
693 # Kerberos enabled
694 $feature{"Kerberos"} = $feat =~ /Kerberos/i;
695 # SPNEGO enabled
696 $feature{"SPNEGO"} = $feat =~ /SPNEGO/i;
697 # PSL enabled
698 $feature{"PSL"} = $feat =~ /PSL/i;
699 # alt-svc enabled
700 $feature{"alt-svc"} = $feat =~ /alt-svc/i;
701 # HSTS support
702 $feature{"HSTS"} = $feat =~ /HSTS/i;
703 $feature{"asyn-rr"} = $feat =~ /asyn-rr/;
704 if($feat =~ /AsynchDNS/i) {
705 if(!$feature{"c-ares"} || $feature{"asyn-rr"}) {
706 # this means threaded resolver
707 $feature{"threaded-resolver"} = 1;
708 $resolver = "threaded";
710 # does not count as "real" c-ares
711 $feature{"c-ares"} = 0;
712 }
713 }
714 # http2 enabled
715 $feature{"http/2"} = $feat =~ /HTTP2/;
716 if($feature{"http/2"}) {
717 push @protocols, 'http/2';
718 }
719 # http3 enabled
720 $feature{"http/3"} = $feat =~ /HTTP3/;
721 if($feature{"http/3"}) {
722 push @protocols, 'http/3';
723 }
724 # https proxy support
725 $feature{"HTTPS-proxy"} = $feat =~ /HTTPS-proxy/;
726 if($feature{"HTTPS-proxy"}) {
727 # 'https-proxy' is used as "server" so consider it a protocol
728 push @protocols, 'https-proxy';
729 }
730 $feature{"SSLS-EXPORT"} = $feat =~ /SSLS-EXPORT/;
731 # Unicode support
732 $feature{"Unicode"} = $feat =~ /Unicode/i;
733 # Thread-safe init
734 $feature{"threadsafe"} = $feat =~ /threadsafe/i;
735 $feature{"HTTPSRR"} = $feat =~ /HTTPSRR/;
736 $feature{"ECH"} = $feat =~ /ECH/;
737 }
738 }
740 if($torture) {
741 $feature{"torture"} = 1;
742 }
744 if(!$curl) {
745 logmsg "unable to get curl's version, further details are:\n";
746 logmsg "issued command: \n";
747 logmsg "$versioncmd \n";
748 if($versretval == -1) {
749 logmsg "command failed with: \n";
750 logmsg "$versretval \n";
751 }
752 elsif($versretval & 127) {
753 logmsg sprintf("command died with signal %d, and %s coredump.\n",
754 ($versretval & 127), ($versretval & 128) ? "a" : "no");
755 }
756 else {
757 logmsg sprintf("command exited with value %d \n", $versretval >> 8);
758 }
759 logmsg "curl version output: \n";
760 logmsg "$versout \n";
761 die "Could not get curl's version";
762 }
764 if($feature{"IPv6"}) {
765 # client has IPv6 support
767 # check if the HTTP server has it!
768 my $cmd = server_exe('sws') . " --logfile $dev_null --version";
769 my @sws = qx($cmd);
770 if($sws[0] =~ /IPv6/) {
771 # HTTP server has IPv6 support!
772 $http_ipv6 = 1;
773 }
775 # check if the FTP server has it!
776 $cmd = server_exe('sockfilt') . " --logfile $dev_null --version";
777 @sws = qx($cmd);
778 if($sws[0] =~ /IPv6/) {
779 # FTP server has IPv6 support!
780 $ftp_ipv6 = 1;
781 }
782 }
784 if($feature{"UnixSockets"}) {
785 # client has Unix sockets support, check whether the HTTP server has it
786 my $cmd = server_exe('sws') . " --logfile $dev_null --version";
787 my @sws = qx($cmd);
788 $http_unix = 1 if($sws[0] =~ /unix/);
789 }
791 # strip line from stderr and file output to not confuse tests
792 if($feature{"CAcert"}) {
793 my $strip_cacert = 's/^Note: Using embedded CA bundle.*\n//';
794 push @global_strip_stderr, $strip_cacert;
795 push @global_strip_file, $strip_cacert;
796 }
798 open(my $manh, "-|", shell_quote($CURL) . " --disable --manual 2>&1");
799 while(my $s = <$manh>) {
800 if($s =~ /built-in manual was disabled at build-time/) {
801 $feature{"manual"} = 0;
802 last;
803 }
804 $feature{"manual"} = 1;
805 last;
806 }
807 close($manh);
809 $feature{"python"} = !$ENV{'CURL_TEST_NO_PYTHON'};
810 $feature{"unittest"} = 1;
811 $feature{"nghttpx"} = !!$ENV{'NGHTTPX'};
812 $feature{"nghttpx-h3"} = !!$nghttpx_h3;
814 # Use this as a proxy for any cryptographic authentication
815 $feature{"crypto"} = $feature{"SSL"} || $feature{"Kerberos"} || $feature{"SPNEGO"};
816 $feature{"local-http"} = servers::localhttp();
817 $feature{"codeset-utf8"} = is_utf8_supported();
818 if($feature{"codeset-utf8"}) {
819 $ENV{'CURL_TEST_HAVE_CODESET_UTF8'} = 1;
820 }
822 # make each protocol an enabled "feature"
823 for my $p (@protocols) {
824 $feature{$p} = 1;
825 }
826 # 'socks' was once here but is now removed
828 if($torture && !$feature{"TrackMemory"}) {
829 die "cannot run torture tests since curl was built without " .
830 "TrackMemory feature (--enable-debug)";
831 }
833 my $hostname = join(' ', runclientoutput("hostname"));
834 chomp $hostname;
835 my $hosttype = join(' ', runclientoutput("uname -a"));
836 chomp $hosttype;
837 my $hostos = $^O;
839 # display summary information about curl and the test host
840 logmsg("********* System characteristics ******** \n",
841 "* $curl\n",
842 "* $libcurl\n",
843 "* Protocols: $proto\n",
844 "* Features: $feat\n",
845 "* Disabled: $dis\n",
846 "* Host: $hostname\n",
847 "* System: $hosttype\n",
848 "* OS: $hostos\n",
849 "* Perl: $^V ($^X)\n",
850 "* Args: $args\n");
851 if($jobs) {
852 # Only show if not the default for now
853 logmsg "* Jobs: $jobs\n";
854 }
856 my $env = sprintf("%s%s%s%s%s",
857 $valgrind ? "Valgrind " : "",
858 $run_duphandle ? "test-duphandle " : "",
859 $run_event_based ? "event-based " : "",
860 $nghttpx_h3 ? "nghttpx-h3 " : "",
861 $libtool ? "Libtool " : "");
862 if($env) {
863 logmsg "* Env: $env\n";
864 }
865 logmsg "* Seed: $randseed\n";
866 if(system("diff $TESTDIR/DISABLED $TESTDIR/DISABLED 2>$dev_null") != 0) {
867 logmsg "* diff: missing\n";
868 }
869 if($mintotal) {
870 logmsg "* Min tests: $mintotal\n";
871 }
874#######################################################################
875# display information about server features
877sub displayserverfeatures {
878 logmsg sprintf("* Servers: %s", $stunnel ? "SSL " : "");
879 logmsg sprintf("%s", $http_ipv6 ? "HTTP-IPv6 " : "");
880 logmsg sprintf("%s", $http_unix ? "HTTP-unix " : "");
881 logmsg sprintf("%s\n", $ftp_ipv6 ? "FTP-IPv6 " : "");
882 logmsg "***************************************** \n";
885#######################################################################
886# Provide time stamps for single test skipped events
888sub timestampskippedevents {
889 my $testnum = $_[0];
891 return if((not defined($testnum)) || ($testnum < 1));
893 if($timestats) {
895 if($timevrfyend{$testnum}) {
896 return;
897 }
898 elsif($timesrvrlog{$testnum}) {
899 $timevrfyend{$testnum} = $timesrvrlog{$testnum};
900 return;
901 }
902 elsif($timetoolend{$testnum}) {
903 $timevrfyend{$testnum} = $timetoolend{$testnum};
904 $timesrvrlog{$testnum} = $timetoolend{$testnum};
905 }
906 elsif($timetoolini{$testnum}) {
907 $timevrfyend{$testnum} = $timetoolini{$testnum};
908 $timesrvrlog{$testnum} = $timetoolini{$testnum};
909 $timetoolend{$testnum} = $timetoolini{$testnum};
910 }
911 elsif($timesrvrend{$testnum}) {
912 $timevrfyend{$testnum} = $timesrvrend{$testnum};
913 $timesrvrlog{$testnum} = $timesrvrend{$testnum};
914 $timetoolend{$testnum} = $timesrvrend{$testnum};
915 $timetoolini{$testnum} = $timesrvrend{$testnum};
916 }
917 elsif($timesrvrini{$testnum}) {
918 $timevrfyend{$testnum} = $timesrvrini{$testnum};
919 $timesrvrlog{$testnum} = $timesrvrini{$testnum};
920 $timetoolend{$testnum} = $timesrvrini{$testnum};
921 $timetoolini{$testnum} = $timesrvrini{$testnum};
922 $timesrvrend{$testnum} = $timesrvrini{$testnum};
923 }
924 elsif($timeprepini{$testnum}) {
925 $timevrfyend{$testnum} = $timeprepini{$testnum};
926 $timesrvrlog{$testnum} = $timeprepini{$testnum};
927 $timetoolend{$testnum} = $timeprepini{$testnum};
928 $timetoolini{$testnum} = $timeprepini{$testnum};
929 $timesrvrend{$testnum} = $timeprepini{$testnum};
930 $timesrvrini{$testnum} = $timeprepini{$testnum};
931 }
932 }
935# Setup CI Test Run
936sub citest_starttestrun {
937 if(azure_check_environment()) {
938 $AZURE_RUN_ID = azure_create_test_run($ACURL);
939 logmsg "Azure Run ID: $AZURE_RUN_ID\n" if($verbose);
940 }
941 # Appveyor does not require anything here
944# Register the test case with the CI runner
945sub citest_starttest {
946 my $testnum = $_[0];
948 # get the name of the test early
949 my $testname = (getpart("client", "name"))[0];
950 chomp $testname;
952 if(length($testname) > 70) {
953 logmsg "ERROR: test $testnum has a too long name, wider than 70 columns\n";
954 return 1;
955 }
957 # create test result in CI services
958 if(azure_check_environment() && $AZURE_RUN_ID) {
959 $AZURE_RESULT_ID = azure_create_test_result($ACURL, $AZURE_RUN_ID, $testnum, $testname);
960 }
961 elsif(appveyor_check_environment()) {
962 appveyor_create_test_result($ACURL, $testnum, $testname);
963 }
964 return 0;
967# Submit the test case result with the CI runner
968sub citest_finishtest {
969 my ($testnum, $error) = @_;
970 # update test result in CI services
971 if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
972 $AZURE_RESULT_ID = azure_update_test_result($ACURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
973 $timeprepini{$testnum}, $timevrfyend{$testnum});
974 }
975 elsif(appveyor_check_environment()) {
976 appveyor_update_test_result($ACURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
977 }
980# Complete CI test run
981sub citest_finishtestrun {
982 if(azure_check_environment() && $AZURE_RUN_ID) {
983 $AZURE_RUN_ID = azure_update_test_run($ACURL, $AZURE_RUN_ID);
984 }
985 # Appveyor does not require anything here
988# add one set of test timings from the runner to global set
989sub updatetesttimings {
990 my ($testnum, %testtimings) = @_;
992 if(defined $testtimings{"timeprepini"}) {
993 $timeprepini{$testnum} = $testtimings{"timeprepini"};
994 }
995 if(defined $testtimings{"timesrvrini"}) {
996 $timesrvrini{$testnum} = $testtimings{"timesrvrini"};
997 }
998 if(defined $testtimings{"timesrvrend"}) {
999 $timesrvrend{$testnum} = $testtimings{"timesrvrend"};
1001 if(defined $testtimings{"timetoolini"}) {
1002 $timetoolini{$testnum} = $testtimings{"timetoolini"};
1004 if(defined $testtimings{"timetoolend"}) {
1005 $timetoolend{$testnum} = $testtimings{"timetoolend"};
1007 if(defined $testtimings{"timesrvrlog"}) {
1008 $timesrvrlog{$testnum} = $testtimings{"timesrvrlog"};
1012#######################################################################
1013# Return the log directory for the given test runner
1014sub getrunnernumlogdir {
1015 my $runnernum = $_[0];
1016 return $jobs > 1 ? "$LOGDIR/$runnernum" : $LOGDIR;
1019#######################################################################
1020# Return the log directory for the given test runner ID
1021sub getrunnerlogdir {
1022 my $runnerid = $_[0];
1023 if($jobs <= 1) {
1024 return $LOGDIR;
1026 if(exists $runnernums{$runnerid}) {
1027 return "$LOGDIR/$runnernums{$runnerid}";
1029 die "Internal error: runner ID $runnerid not found";
1032#######################################################################
1033# Verify that this test case should be run
1034sub singletest_shouldrun {
1035 my $testnum = $_[0];
1036 my $why; # why the test is not run
1037 my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
1038 my @what; # what features are needed
1039 my $may_retry_candidate;
1041 if($disttests !~ /test$testnum(\W|\z)/ ) {
1042 logmsg "Warning: test$testnum not present in tests/data/Makefile.am\n";
1044 if($disabled{$testnum}) {
1045 if(!$run_disabled) {
1046 $why = "listed in DISABLED";
1048 else {
1049 logmsg "Warning: test$testnum is explicitly disabled\n";
1052 if($ignored{$testnum}) {
1053 logmsg "Warning: test$testnum result is ignored\n";
1054 $errorreturncode = 2;
1057 if(loadtest("${TESTDIR}/test${testnum}", 1)) {
1058 if($verbose) {
1059 # this is not a test
1060 logmsg "RUN: $testnum does not look like a test case\n";
1062 $why = "no test";
1064 else {
1065 @what = getpart("client", "features");
1068 # We require a feature to be present
1069 for(@what) {
1070 my $f = $_;
1071 $f =~ s/\s//g;
1073 if($f =~ /^([^!].*)$/) {
1074 if($feature{$1}) {
1075 next;
1078 $why = "curl lacks $1 support";
1079 last;
1083 # We require a feature to not be present
1084 if(!$why) {
1085 for(@what) {
1086 my $f = $_;
1087 $f =~ s/\s//g;
1089 if($f =~ /^!(.*)$/) {
1090 if(!$feature{$1}) {
1091 next;
1094 else {
1095 next;
1098 $why = "curl has $1 support";
1099 last;
1103 my @info_keywords;
1104 if(!$why) {
1105 @info_keywords = getpart("info", "keywords");
1106 push @info_keywords, getpart("client", "server") ? "with-server" : "without-server";
1108 if(!$info_keywords[0]) {
1109 $why = "missing the <keywords> section!";
1111 # Only evaluate keywords if the section is present.
1112 else {
1113 # Prefix features with "feat:" and add to keywords list.
1114 push @info_keywords, map { "feat:" . lc($_) } @what;
1116 my $match;
1117 for my $k (@info_keywords) {
1118 chomp $k;
1120 $may_retry_candidate |= ($k =~ /^(flaky|may-retry|threads|timing-dependent|with-server)$/);
1121 $may_retry_candidate |= ($k =~ /^may-retry-on-valgrind$/) if($valgrind);
1123 if($disabled_keywords{lc($k)}) {
1124 if($k =~ /^feat:/) {
1125 $why = "disabled by feature";
1127 else {
1128 $why = "disabled by keyword";
1131 elsif($enabled_keywords{lc($k)}) {
1132 $match = 1;
1134 if($ignored_keywords{lc($k)}) {
1135 logmsg "Warning: test$testnum result is ignored due to $k\n";
1136 $errorreturncode = 2;
1140 if(!$why && !$match && %enabled_keywords) {
1141 if(grep { /^feat:/ } keys %enabled_keywords) {
1142 $why = "disabled by missing feature";
1144 else {
1145 $why = "disabled by missing keyword";
1151 if(!$why && defined $custom_skip_reasons{test}{$testnum}) {
1152 $why = $custom_skip_reasons{test}{$testnum};
1155 if(!$why && defined $custom_skip_reasons{tool}) {
1156 foreach my $tool (getpart("client", "tool")) {
1157 foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) {
1158 if($tool =~ /$tool_skip_pattern/i) {
1159 $why = $custom_skip_reasons{tool}{$tool_skip_pattern};
1165 if(!$why && defined $custom_skip_reasons{keyword}) {
1166 foreach my $keyword (@info_keywords) {
1167 foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) {
1168 if($keyword =~ /$keyword_skip_pattern/i) {
1169 $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern};
1175 if(!$why && $may_retry_candidate) {
1176 $may_retry{$testnum} = 1;
1179 if($why && checktest("${TESTDIR}/test${testnum}")) {
1180 logmsg "Warning: issue(s) found in test data: ${TESTDIR}/test${testnum}\n";
1183 return ($why, $errorreturncode);
1186my %allnames;
1188#######################################################################
1189# Print the test name and count tests
1190sub singletest_count {
1191 my ($testnum, $why) = @_;
1193 if($why && !$listonly) {
1194 # there is a problem, count it as "skipped"
1195 $skipped{$why}++;
1196 $teststat[$testnum] = $why; # store reason for this test case
1198 if(!$short) {
1199 if($skipped{$why} <= 3) {
1200 # show only the first three skips for each reason
1201 logmsg sprintf("test %04d SKIPPED: $why\n", $testnum);
1205 timestampskippedevents($testnum);
1206 return -1;
1209 # At this point we have committed to run this test
1210 logmsg sprintf("test %04d ", $testnum) if(!$automakestyle);
1212 # name of the test
1213 my $testname = (getpart("client", "name"))[0];
1214 chomp $testname;
1215 logmsg "[$testname]\n" if(!$short);
1217 if($listonly) {
1218 timestampskippedevents($testnum);
1220 else {
1221 if(exists $allnames{$testname} &&
1222 ($allnames{$testname} != $testnum)) {
1223 logmsg sprintf("ERROR: test %d has duplicate test name: \"%s\". " .
1224 "The same as test %d\n", $testnum,
1225 $testname, $allnames{$testname});
1226 exit 1;
1228 # store which test that uses this name
1229 $allnames{$testname} = $testnum;
1231 return 0;
1234# Make sure all line endings in the array are the same: CRLF
1235sub normalize_text {
1236 my ($ref) = @_;
1237 s/\r\n/\n/g for @$ref;
1238 s/\n/\r\n/g for @$ref;
1241#######################################################################
1242# Verify test succeeded
1243sub singletest_check {
1244 my ($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind) = @_;
1246 # Skip all the verification on torture tests
1247 if($torture) {
1248 # timestamp test result verification end
1249 $timevrfyend{$testnum} = Time::HiRes::time();
1250 return -2;
1253 my $logdir = getrunnerlogdir($runnerid);
1254 my @err = getpart("verify", "errorcode");
1255 my $errorcode = $err[0] || "0";
1256 my $ok = "";
1257 my $res;
1258 chomp $errorcode;
1259 my $testname = (getpart("client", "name"))[0];
1260 chomp $testname;
1262 # what parts to cut off from stdout/stderr
1263 my @stripfile = getpart("verify", "stripfile");
1265 my @validstdout = getpart("verify", "stdout");
1266 # get all attributes
1267 my %hash = getpartattr("verify", "stdout");
1269 my $loadfile = $hash{'loadfile'};
1270 if($loadfile) {
1271 open(my $tmp, "<", $loadfile) or die "Cannot open file $loadfile: $!";
1272 @validstdout = <$tmp>;
1273 close($tmp);
1275 # Enforce LF newlines on load
1276 s/\r\n/\n/g for @validstdout;
1279 if(@validstdout) {
1280 $validstdout[0] =~ s/^%EMPTY[\r\n]*//;
1282 # verify redirected stdout
1283 my @actual = loadarray(stdoutfilename($logdir, $testnum));
1285 foreach my $strip (@stripfile) {
1286 chomp $strip;
1287 my @newgen;
1288 for(@actual) {
1289 eval $strip;
1290 if($_) {
1291 push @newgen, $_;
1294 # this is to get rid of array entries that vanished (zero
1295 # length) because of replacements
1296 @actual = @newgen;
1299 # get the mode attribute
1300 my $filemode = $hash{'mode'};
1301 if($filemode && ($filemode eq "text")) {
1302 normalize_text(\@validstdout);
1303 normalize_text(\@actual);
1306 if($hash{'nonewline'}) {
1307 # Yes, we must cut off the final newline from the final line
1308 # of the protocol data
1309 chomp($validstdout[-1]);
1312 if($hash{'crlf'}) {
1313 if($hash{'crlf'} eq "headers") {
1314 subnewlines(0, \$_) for @validstdout;
1316 else {
1317 subnewlines(1, \$_) for @validstdout;
1321 $res = compare($runnerid, $testnum, $testname, "stdout", \@actual, \@validstdout);
1322 if($res) {
1323 return -1;
1325 $ok .= "s";
1327 else {
1328 $ok .= "-"; # stdout not checked
1331 my @validstderr = getpart("verify", "stderr");
1332 if(@validstderr) {
1333 # verify redirected stderr
1334 my @actual = loadarray(stderrfilename($logdir, $testnum));
1336 foreach my $strip (@global_strip_stderr, @stripfile) {
1337 chomp $strip;
1338 my @newgen;
1339 for(@actual) {
1340 eval $strip;
1341 if($_) {
1342 push @newgen, $_;
1345 # this is to get rid of array entries that vanished (zero
1346 # length) because of replacements
1347 @actual = @newgen;
1350 # get all attributes
1351 my %hash = getpartattr("verify", "stderr");
1353 # get the mode attribute
1354 my $filemode = $hash{'mode'};
1355 if($filemode && ($filemode eq "text")) {
1356 normalize_text(\@validstderr);
1357 normalize_text(\@actual);
1359 if($filemode && ($filemode eq "warn")) {
1360 for(@validstderr) {
1361 s/Warning: //;
1362 s/\r//;
1363 s/\n/ /;
1365 for(@actual) {
1366 s/Warning: //;
1367 s/\r//;
1368 s/\n/ /;
1372 if($hash{'nonewline'}) {
1373 # Yes, we must cut off the final newline from the final line
1374 # of the protocol data
1375 chomp($validstderr[-1]);
1378 if($hash{'crlf'}) {
1379 if($hash{'crlf'} eq "headers") {
1380 subnewlines(0, \$_) for @validstderr;
1382 else {
1383 subnewlines(1, \$_) for @validstderr;
1387 $res = compare($runnerid, $testnum, $testname, "stderr", \@actual, \@validstderr);
1388 if($res) {
1389 return -1;
1391 $ok .= "r";
1393 else {
1394 $ok .= "-"; # stderr not checked
1397 # what to cut off from the live protocol sent by curl
1398 my @strip = getpart("verify", "strip");
1400 # what parts to cut off from the protocol & upload
1401 my @strippart = getpart("verify", "strippart");
1403 # this is the valid protocol blurb curl should generate
1404 my @protocol = getpart("verify", "protocol");
1405 if(@protocol) {
1406 # Verify the sent request
1407 my @out = loadarray("$logdir/$SERVERIN");
1409 # check if there is any attributes on the verify/protocol section
1410 my %hash = getpartattr("verify", "protocol");
1412 if($hash{'nonewline'}) {
1413 # Yes, we must cut off the final newline from the final line
1414 # of the protocol data
1415 chomp($protocol[-1]);
1418 for(@strip) {
1419 # strip off all lines that match the patterns from both arrays
1420 chomp $_;
1421 @out = striparray( $_, \@out);
1422 @protocol = striparray( $_, \@protocol);
1425 for my $strip (@strippart) {
1426 chomp $strip;
1427 for(@out) {
1428 eval $strip;
1432 if($hash{'crlf'}) {
1433 if($hash{'crlf'} eq "headers") {
1434 subnewlines(0, \$_) for @protocol;
1436 else {
1437 subnewlines(1, \$_) for @protocol;
1441 if((!$out[0] || ($out[0] eq "")) && $protocol[0]) {
1442 logmsg "\n $testnum: protocol FAILED!\n" .
1443 " There was no content at all in the file $logdir/$SERVERIN.\n" .
1444 " Server glitch? Total curl failure? Returned: $cmdres\n";
1445 # timestamp test result verification end
1446 $timevrfyend{$testnum} = Time::HiRes::time();
1447 return -1;
1450 $res = compare($runnerid, $testnum, $testname, "protocol", \@out, \@protocol);
1451 if($res) {
1452 return -1;
1454 $ok .= "p";
1456 else {
1457 $ok .= "-"; # protocol not checked
1460 my %replyattr = getpartattr("reply", "data");
1461 my @reply;
1462 if(partexists("reply", "datacheck")) {
1463 for my $partsuffix (('', '1', '2', '3', '4')) {
1464 my @replycheckpart = getpart("reply", "datacheck" . $partsuffix);
1465 if(@replycheckpart) {
1466 my %replycheckpartattr = getpartattr("reply", "datacheck" . $partsuffix);
1467 # get the mode attribute
1468 my $filemode = $replycheckpartattr{'mode'};
1469 if($filemode && ($filemode eq "text")) {
1470 normalize_text(\@replycheckpart);
1472 if($replycheckpartattr{'nonewline'}) {
1473 # Yes, we must cut off the final newline from the final line
1474 # of the datacheck
1475 chomp($replycheckpart[-1]);
1477 if($replycheckpartattr{'crlf'}) {
1478 if($replycheckpartattr{'crlf'} eq "headers") {
1479 subnewlines(0, \$_) for @replycheckpart;
1481 else {
1482 subnewlines(1, \$_) for @replycheckpart;
1485 push(@reply, @replycheckpart);
1489 else {
1490 # check against the data section
1491 @reply = getpart("reply", "data");
1492 if(@reply) {
1493 if($replyattr{'nonewline'}) {
1494 # cut off the final newline from the final line of the data
1495 chomp($reply[-1]);
1498 # get the mode attribute
1499 my $filemode = $replyattr{'mode'};
1500 if($filemode && ($filemode eq "text")) {
1501 normalize_text(\@reply);
1503 if($replyattr{'crlf'}) {
1504 if($replyattr{'crlf'} eq "headers") {
1505 subnewlines(0, \$_) for @reply;
1507 else {
1508 subnewlines(1, \$_) for @reply;
1513 if(!$replyattr{'nocheck'} && (@reply || $replyattr{'sendzero'})) {
1514 # verify the received data
1515 my @out = loadarray($CURLOUT);
1517 # get the mode attribute
1518 my $filemode = $replyattr{'mode'};
1519 if($filemode && ($filemode eq "text")) {
1520 normalize_text(\@out);
1522 $res = compare($runnerid, $testnum, $testname, "data", \@out, \@reply);
1523 if($res) {
1524 return -1;
1526 $ok .= "d";
1528 else {
1529 $ok .= "-"; # data not checked
1532 # if this section exists, we verify upload
1533 my @upload = getpart("verify", "upload");
1534 if(@upload) {
1535 my %hash = getpartattr("verify", "upload");
1536 if($hash{'nonewline'}) {
1537 # cut off the final newline from the final line of the upload data
1538 chomp($upload[-1]);
1541 # verify uploaded data
1542 my @out = loadarray("$logdir/upload.$testnum");
1543 for my $strip (@strippart) {
1544 chomp $strip;
1545 for(@out) {
1546 eval $strip;
1549 if($hash{'crlf'}) {
1550 subnewlines(1, \$_) for @upload;
1552 if($hash{'nonewline'}) {
1553 # Yes, we must cut off the final newline from the final line
1554 # of the upload data
1555 chomp($upload[-1]);
1558 $res = compare($runnerid, $testnum, $testname, "upload", \@out, \@upload);
1559 if($res) {
1560 return -1;
1562 $ok .= "u";
1564 else {
1565 $ok .= "-"; # upload not checked
1568 # this is the valid protocol blurb curl should generate to a proxy
1569 my @proxyprot = getpart("verify", "proxy");
1570 if(@proxyprot) {
1571 # Verify the sent proxy request
1572 # check if there is any attributes on the verify/protocol section
1573 my %hash = getpartattr("verify", "proxy");
1575 if($hash{'nonewline'}) {
1576 # Yes, we must cut off the final newline from the final line
1577 # of the protocol data
1578 chomp($proxyprot[-1]);
1581 my @out = loadarray("$logdir/$PROXYIN");
1582 for(@strip) {
1583 # strip off all lines that match the patterns from both arrays
1584 chomp $_;
1585 @out = striparray( $_, \@out);
1586 @proxyprot = striparray( $_, \@proxyprot);
1589 for my $strip (@strippart) {
1590 chomp $strip;
1591 for(@out) {
1592 eval $strip;
1596 if($hash{'crlf'}) {
1597 if($hash{'crlf'} eq "headers") {
1598 subnewlines(0, \$_) for @proxyprot;
1600 else {
1601 subnewlines(1, \$_) for @proxyprot;
1605 $res = compare($runnerid, $testnum, $testname, "proxy", \@out, \@proxyprot);
1606 if($res) {
1607 return -1;
1609 $ok .= "P";
1611 else {
1612 $ok .= "-"; # proxy not checked
1615 my $outputok;
1616 for my $partsuffix (('', '1', '2', '3', '4')) {
1617 my @outfile = getpart("verify", "file" . $partsuffix);
1618 if(@outfile || partexists("verify", "file" . $partsuffix) ) {
1619 # we are supposed to verify a dynamically generated file!
1620 my %hash = getpartattr("verify", "file" . $partsuffix);
1622 my $filename = $hash{'name'};
1623 if(!$filename) {
1624 logmsg " $testnum: IGNORED: section verify=>file$partsuffix " .
1625 "has no name attribute\n";
1626 if(runnerac_stopservers($runnerid)) {
1627 logmsg "ERROR: runner $runnerid seems to have died\n";
1629 else {
1631 # TODO: this is a blocking call that stalls the controller,
1632 if($verbose) {
1633 logmsg "WARNING: blocking call in async function\n";
1635 # but this error condition should never happen except during
1636 # development.
1637 my ($rid, $unexpected, $logs) = runnerar($runnerid);
1638 if(!$rid) {
1639 logmsg "ERROR: runner $runnerid seems to have died\n";
1641 else {
1642 logmsg $logs;
1645 # timestamp test result verification end
1646 $timevrfyend{$testnum} = Time::HiRes::time();
1647 return -1;
1649 my @generated = loadarray($filename);
1651 # what parts to cut off from the file
1652 my @stripfilepar = getpart("verify", "stripfile" . $partsuffix);
1654 my $filemode = $hash{'mode'};
1655 if($filemode && ($filemode eq "text")) {
1656 normalize_text(\@outfile);
1657 normalize_text(\@generated);
1659 if($hash{'crlf'}) {
1660 if($hash{'crlf'} eq "headers") {
1661 subnewlines(0, \$_) for @outfile;
1663 else {
1664 subnewlines(1, \$_) for @outfile;
1668 for my $strip (@global_strip_file, @stripfilepar) {
1669 chomp $strip;
1670 my @newgen;
1671 for(@generated) {
1672 eval $strip;
1673 if($_) {
1674 push @newgen, $_;
1677 # this is to get rid of array entries that vanished (zero
1678 # length) because of replacements
1679 @generated = @newgen;
1682 if($hash{'nonewline'}) {
1683 # cut off the final newline from the final line of the
1684 # output data
1685 chomp($outfile[-1]);
1688 $res = compare($runnerid, $testnum, $testname, "output ($filename)",
1689 \@generated, \@outfile);
1690 if($res) {
1691 return -1;
1694 $outputok = 1; # output checked
1697 $ok .= ($outputok) ? "o" : "-"; # output checked or not
1699 # verify SOCKS proxy details
1700 my @socksprot = getpart("verify", "socks");
1701 if(@socksprot) {
1702 # Verify the sent SOCKS proxy details
1703 my @out = loadarray("$logdir/$SOCKSIN");
1704 $res = compare($runnerid, $testnum, $testname, "socks", \@out, \@socksprot);
1705 if($res) {
1706 return -1;
1710 my @dnsd = getpart("verify", "dns");
1711 if(@dnsd) {
1712 # we are supposed to verify a dynamically generated file!
1713 my %hash = getpartattr("verify", "dns");
1714 my $hostname = $hash{'host'};
1716 # Verify the sent DNS requests
1717 my @out = loadarray("$logdir/dnsd.input");
1718 my @sverify = sort @dnsd;
1719 my @sout = sort @out;
1721 if($hostname) {
1722 # when a hostname is set, we filter out requests to this pattern
1723 @sout = grep {/$hostname/} @sout;
1726 $res = compare($runnerid, $testnum, $testname, "DNS", \@sout, \@sverify);
1727 if($res) {
1728 return -1;
1732 # accept multiple comma-separated error codes
1733 my @splerr = split(/ *, */, $errorcode);
1734 my $errok;
1735 foreach my $e (@splerr) {
1736 if($e == $cmdres) {
1737 # a fine error code
1738 $errok = 1;
1739 last;
1743 if($errok) {
1744 $ok .= "e";
1746 else {
1747 if(!$short) {
1748 logmsg sprintf("\n%s returned $cmdres, when expecting %s\n",
1749 (!$tool) ? "curl" : $tool, $errorcode);
1751 logmsg " $testnum: exit FAILED\n";
1752 # timestamp test result verification end
1753 $timevrfyend{$testnum} = Time::HiRes::time();
1754 return -1;
1757 if($feature{"TrackMemory"}) {
1758 if(! -f "$logdir/$MEMDUMP") {
1759 my %cmdhash = getpartattr("client", "command");
1760 my $cmdtype = $cmdhash{'type'} || "default";
1761 if($cmdhash{'option'} && $cmdhash{'option'} !~ /no-memdebug/) {
1762 logmsg "\n** ALERT! memory tracking with no output file?\n"
1763 if($cmdtype ne "perl");
1765 $ok .= "-"; # problem with memory checking
1767 else {
1768 my @memdata = memanalyze("$logdir/$MEMDUMP", 0, 0, 0);
1769 my $leak = 0;
1770 for(@memdata) {
1771 if($_ ne "") {
1772 # well it could be other memory problems as well, but
1773 # we call it leak for short here
1774 $leak = 1;
1777 if($leak) {
1778 logmsg "\n** MEMORY FAILURE\n";
1779 logmsg @memdata;
1780 logmsg " $testnum: memory FAILED\n";
1781 # timestamp test result verification end
1782 $timevrfyend{$testnum} = Time::HiRes::time();
1783 return -1;
1785 else {
1786 $ok .= "m";
1788 my @more = memanalyze("$logdir/$MEMDUMP", 1, 0, 0);
1789 my $allocs = 0;
1790 my $max = 0;
1791 for(@more) {
1792 if(/^Allocations: (\d+)/) {
1793 $allocs = $1;
1795 elsif(/^Maximum allocated: (\d+)/) {
1796 $max = $1;
1799 my @limits = getpart("verify", "limits");
1800 my $lim_allocs = 1000; # high default values
1801 my $lim_max = 1000000;
1802 for(@limits) {
1803 if(/^Allocations: (\d+)/i) {
1804 $lim_allocs = $1;
1806 elsif(/^Maximum allocated: (\d+)/i) {
1807 $lim_max = $1;
1810 logmsg "did $allocs allocations, $lim_allocs allowed\n"
1811 if($verbose);
1813 logmsg "allocated $max maximum, $lim_max allowed\n"
1814 if($verbose);
1816 if($allocs > $lim_allocs) {
1817 logmsg "\n** TOO MANY ALLOCS\n";
1818 logmsg "$lim_allocs allocations allowed, did $allocs\n";
1819 logmsg " $testnum: allocs FAILED\n";
1820 # timestamp test result verification end
1821 $timevrfyend{$testnum} = Time::HiRes::time();
1822 return -1;
1824 if($max > $lim_max) {
1825 logmsg "\n** TOO MUCH TOTAL ALLOCATION\n";
1826 logmsg "$lim_max maximum allocation allowed, did $max\n";
1827 logmsg " $testnum: allocsize FAILED\n";
1828 # timestamp test result verification end
1829 $timevrfyend{$testnum} = Time::HiRes::time();
1830 return -1;
1834 else {
1835 $ok .= "-"; # memory not checked
1838 my @notexists = getpart("verify", "notexists");
1839 if(@notexists) {
1840 # a list of directory entries that must not exist
1841 my $err;
1842 while(@notexists) {
1843 my $fname = shift @notexists;
1844 chomp $fname;
1845 if(-e $fname) {
1846 logmsg "ERROR: Found '$fname' when not supposed to exist.\n";
1847 $err++;
1849 elsif($verbose) {
1850 logmsg "Found '$fname' confirmed to not exist.\n";
1853 if($err) {
1854 return -1;
1857 if($valgrind) {
1858 if($usedvalgrind) {
1859 if(!opendir(DIR, $logdir)) {
1860 logmsg "ERROR: unable to read $logdir\n";
1861 # timestamp test result verification end
1862 $timevrfyend{$testnum} = Time::HiRes::time();
1863 return -1;
1865 my @files = readdir(DIR);
1866 closedir(DIR);
1867 my $vgfile;
1868 foreach my $file (@files) {
1869 if($file =~ /^valgrind$testnum(\..*|)$/) {
1870 $vgfile = $file;
1871 last;
1874 if(!$vgfile) {
1875 logmsg "ERROR: valgrind log file missing for test $testnum\n";
1876 # timestamp test result verification end
1877 $timevrfyend{$testnum} = Time::HiRes::time();
1878 return -1;
1880 my @e = valgrindparse("$logdir/$vgfile");
1881 if(@e && $e[0]) {
1882 if($automakestyle) {
1883 logmsg "FAIL: $testnum - $testname - valgrind\n";
1885 else {
1886 logmsg " valgrind ERROR ";
1887 logmsg @e;
1889 # timestamp test result verification end
1890 $timevrfyend{$testnum} = Time::HiRes::time();
1891 return -1;
1893 $ok .= "v";
1895 else {
1896 if($verbose) {
1897 logmsg " valgrind SKIPPED\n";
1899 $ok .= "-"; # skipped
1902 else {
1903 $ok .= "-"; # valgrind not checked
1905 # add 'E' for event-based
1906 $ok .= $run_event_based ? "E" : "-";
1908 logmsg "$ok " if(!$short);
1910 # timestamp test result verification end
1911 $timevrfyend{$testnum} = Time::HiRes::time();
1913 return 0;
1916#######################################################################
1917# Report a successful test
1918sub singletest_success {
1919 my ($testnum, $count, $total, $errorreturncode) = @_;
1921 my $sofar = time() - $start;
1922 my $esttotal = $sofar / $count * $total;
1923 my $estleft = $esttotal - $sofar;
1924 my $timeleft = sprintf("remaining: %02d:%02d",
1925 $estleft / 60,
1926 $estleft % 60);
1927 my $took = $timevrfyend{$testnum} - $timeprepini{$testnum};
1928 my $duration = sprintf("duration: %02d:%02d",
1929 $sofar / 60, $sofar % 60);
1930 if(!$automakestyle) {
1931 logmsg sprintf("OK (%-3d out of %-3d, %s, took %.3fs, %s)\n",
1932 $count, $total, $timeleft, $took, $duration);
1934 else {
1935 my $testname = (getpart("client", "name"))[0];
1936 chomp $testname;
1937 logmsg "PASS: $testnum - $testname\n";
1940 if($errorreturncode == 2) {
1941 # ignored test success
1942 $passedign .= "$testnum ";
1943 logmsg "Warning: test$testnum result is ignored, but passed!\n";
1947#######################################################################
1948# Run a single specified test case
1949# This is structured as a state machine which changes state after an
1950# asynchronous call is made that awaits a response. The function returns with
1951# an error code and a flag that indicates if the state machine has completed,
1952# which means (if not) the function must be called again once the response has
1953# arrived.
1955sub singletest {
1956 my ($runnerid, $testnum, $count, $total) = @_;
1958 # start buffering logmsg; stop it on return
1959 logmsg_bufferfortest($runnerid);
1960 if(!exists $singletest_state{$runnerid}) {
1961 # First time in singletest() for this test
1962 $singletest_state{$runnerid} = ST_INIT;
1965 if($singletest_state{$runnerid} == ST_INIT) {
1966 my $logdir = getrunnerlogdir($runnerid);
1967 # first, remove all lingering log & lock files
1968 if(!cleardir($logdir)) {
1969 #logmsg "Warning: $runnerid: cleardir($logdir) failed\n";
1971 if(!cleardir("$logdir/$LOCKDIR")) {
1972 logmsg "Warning: $runnerid: cleardir($logdir/$LOCKDIR) failed\n";
1975 $singletest_state{$runnerid} = ST_INITED;
1976 # Recursively call the state machine again because there is no
1977 # event expected that would otherwise trigger a new call.
1978 return singletest(@_);
1980 elsif($singletest_state{$runnerid} == ST_INITED) {
1981 ###################################################################
1982 # Restore environment variables that were modified in a previous run.
1983 # Test definition may instruct to (un)set environment vars.
1984 # This is done this early so that leftover variables do not affect
1985 # starting servers or CI registration.
1986 # restore_test_env(1);
1988 ###################################################################
1989 # Load test file so CI registration can get the right data before the
1990 # runner is called
1991 loadtest("${TESTDIR}/test${testnum}", 1);
1993 ###################################################################
1994 # Register the test case with the CI environment
1995 if(citest_starttest($testnum)) {
1996 return (-1, 0);
1999 if(runnerac_test_preprocess($runnerid, $testnum)) {
2000 logmsg "ERROR: runner $runnerid seems to have died\n";
2001 $singletest_state{$runnerid} = ST_INIT;
2002 return (-1, 0);
2004 $singletest_state{$runnerid} = ST_PREPROCESS;
2006 elsif($singletest_state{$runnerid} == ST_PREPROCESS) {
2007 my ($rid, $why, $error, $logs, $testtimings) = runnerar($runnerid);
2008 if(!$rid) {
2009 logmsg "ERROR: runner $runnerid seems to have died\n";
2010 $singletest_state{$runnerid} = ST_INIT;
2011 return (-1, 0);
2013 logmsg $logs;
2014 updatetesttimings($testnum, %$testtimings);
2015 if($error == -2) {
2016 if($postmortem) {
2017 # Error indicates an actual problem starting the server, so
2018 # display the server logs
2019 displaylogs($rid, $testnum);
2022 elsif($show_success_logs) {
2023 displaylogs($rid, $testnum);
2026 #######################################################################
2027 # Load test file for this test number
2028 my $logdir = getrunnerlogdir($runnerid);
2029 loadtest("${logdir}/test${testnum}");
2031 #######################################################################
2032 # Print the test name and count tests
2033 $error = singletest_count($testnum, $why);
2034 if($error) {
2035 # Submit the test case result with the CI environment
2036 citest_finishtest($testnum, $error);
2037 $singletest_state{$runnerid} = ST_INIT;
2038 logmsg singletest_dumplogs();
2039 return ($error, 0);
2042 #######################################################################
2043 # Execute this test number
2044 my $cmdres;
2045 my $CURLOUT;
2046 my $tool;
2047 my $usedvalgrind;
2048 if(runnerac_test_run($runnerid, $testnum)) {
2049 logmsg "ERROR: runner $runnerid seems to have died\n";
2050 $singletest_state{$runnerid} = ST_INIT;
2051 return (-1, 0);
2053 $singletest_state{$runnerid} = ST_RUN;
2055 elsif($singletest_state{$runnerid} == ST_RUN) {
2056 my ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar($runnerid);
2057 if(!$rid) {
2058 logmsg "ERROR: runner $runnerid seems to have died\n";
2059 $singletest_state{$runnerid} = ST_INIT;
2060 return (-1, 0);
2062 logmsg $logs;
2063 updatetesttimings($testnum, %$testtimings);
2064 if($error == -1) {
2065 # no further verification occurs
2066 $timevrfyend{$testnum} = Time::HiRes::time();
2067 my $err = ignoreresultcode($testnum);
2068 # Submit the test case result with the CI environment
2069 citest_finishtest($testnum, $err);
2070 $singletest_state{$runnerid} = ST_INIT;
2071 logmsg singletest_dumplogs();
2072 # return a test failure, either to be reported or to be ignored
2073 return ($err, 0);
2075 elsif($error == -2) {
2076 # fill in the missing timings on error
2077 timestampskippedevents($testnum);
2078 # Submit the test case result with the CI environment
2079 citest_finishtest($testnum, $error);
2080 $singletest_state{$runnerid} = ST_INIT;
2081 logmsg singletest_dumplogs();
2082 return ($error, 0);
2084 elsif($error > 0) {
2085 # no further verification occurs
2086 $timevrfyend{$testnum} = Time::HiRes::time();
2087 # Submit the test case result with the CI environment
2088 citest_finishtest($testnum, $error);
2089 $singletest_state{$runnerid} = ST_INIT;
2090 logmsg singletest_dumplogs();
2091 return ($error, 0);
2094 #######################################################################
2095 # Verify that the test succeeded
2097 # Load test file for this test number
2098 my $logdir = getrunnerlogdir($runnerid);
2099 loadtest("${logdir}/test${testnum}");
2100 readtestkeywords();
2102 $error = singletest_check($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind);
2103 if($error == -1) {
2104 my $err = ignoreresultcode($testnum);
2105 # Submit the test case result with the CI environment
2106 citest_finishtest($testnum, $err);
2107 $singletest_state{$runnerid} = ST_INIT;
2108 logmsg singletest_dumplogs();
2109 # return a test failure, either to be reported or to be ignored
2110 return ($err, 0);
2112 elsif($error == -2) {
2113 # torture test; there is no verification, so the run result holds the
2114 # test success code
2115 # Submit the test case result with the CI environment
2116 citest_finishtest($testnum, $cmdres);
2117 $singletest_state{$runnerid} = ST_INIT;
2118 logmsg singletest_dumplogs();
2119 return ($cmdres, 0);
2122 #######################################################################
2123 # Report a successful test
2124 singletest_success($testnum, $count, $total, ignoreresultcode($testnum));
2126 # Submit the test case result with the CI environment
2127 citest_finishtest($testnum, 0);
2128 $singletest_state{$runnerid} = ST_INIT;
2130 logmsg singletest_dumplogs();
2131 return (0, 0); # state machine is finished
2133 singletest_unbufferlogs();
2134 return (0, 1); # state machine must be called again on event
2137#######################################################################
2138# runtimestats displays test-suite run time statistics
2140sub runtimestats {
2141 my $lasttest = $_[0];
2143 return if(not $timestats);
2145 logmsg "::group::Run Time Stats\n";
2147 logmsg "\nTest suite total running time breakdown per task...\n\n";
2149 my @timesrvr;
2150 my @timeprep;
2151 my @timetool;
2152 my @timelock;
2153 my @timevrfy;
2154 my @timetest;
2155 my $timesrvrtot = 0.0;
2156 my $timepreptot = 0.0;
2157 my $timetooltot = 0.0;
2158 my $timelocktot = 0.0;
2159 my $timevrfytot = 0.0;
2160 my $timetesttot = 0.0;
2161 my $counter;
2163 for my $testnum (1 .. $lasttest) {
2164 if($timesrvrini{$testnum}) {
2165 $timesrvrtot += $timesrvrend{$testnum} - $timesrvrini{$testnum};
2166 $timepreptot +=
2167 (($timetoolini{$testnum} - $timeprepini{$testnum}) -
2168 ($timesrvrend{$testnum} - $timesrvrini{$testnum}));
2169 $timetooltot += $timetoolend{$testnum} - $timetoolini{$testnum};
2170 $timelocktot += $timesrvrlog{$testnum} - $timetoolend{$testnum};
2171 $timevrfytot += $timevrfyend{$testnum} - $timesrvrlog{$testnum};
2172 $timetesttot += $timevrfyend{$testnum} - $timeprepini{$testnum};
2173 push @timesrvr, sprintf("%06.3f %04d",
2174 $timesrvrend{$testnum} - $timesrvrini{$testnum}, $testnum);
2175 push @timeprep, sprintf("%06.3f %04d",
2176 ($timetoolini{$testnum} - $timeprepini{$testnum}) -
2177 ($timesrvrend{$testnum} - $timesrvrini{$testnum}), $testnum);
2178 push @timetool, sprintf("%06.3f %04d",
2179 $timetoolend{$testnum} - $timetoolini{$testnum}, $testnum);
2180 push @timelock, sprintf("%06.3f %04d",
2181 $timesrvrlog{$testnum} - $timetoolend{$testnum}, $testnum);
2182 push @timevrfy, sprintf("%06.3f %04d",
2183 $timevrfyend{$testnum} - $timesrvrlog{$testnum}, $testnum);
2184 push @timetest, sprintf("%06.3f %04d",
2185 $timevrfyend{$testnum} - $timeprepini{$testnum}, $testnum);
2190 no warnings 'numeric';
2191 @timesrvr = sort { $b <=> $a } @timesrvr;
2192 @timeprep = sort { $b <=> $a } @timeprep;
2193 @timetool = sort { $b <=> $a } @timetool;
2194 @timelock = sort { $b <=> $a } @timelock;
2195 @timevrfy = sort { $b <=> $a } @timevrfy;
2196 @timetest = sort { $b <=> $a } @timetest;
2199 logmsg "Spent " . sprintf("%08.3f ", $timesrvrtot) .
2200 "seconds starting and verifying test harness servers.\n";
2201 logmsg "Spent " . sprintf("%08.3f ", $timepreptot) .
2202 "seconds reading definitions and doing test preparations.\n";
2203 logmsg "Spent " . sprintf("%08.3f ", $timetooltot) .
2204 "seconds actually running test tools.\n";
2205 logmsg "Spent " . sprintf("%08.3f ", $timelocktot) .
2206 "seconds awaiting server logs lock removal.\n";
2207 logmsg "Spent " . sprintf("%08.3f ", $timevrfytot) .
2208 "seconds verifying test results.\n";
2209 logmsg "Spent " . sprintf("%08.3f ", $timetesttot) .
2210 "seconds doing all of the above.\n";
2212 $counter = 25;
2213 logmsg "\nTest server starting and verification time per test " .
2214 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2215 logmsg "-time- test\n";
2216 logmsg "------ ----\n";
2217 foreach my $txt (@timesrvr) {
2218 last if((not $fullstats) && (not $counter--));
2219 logmsg "$txt\n";
2222 $counter = 10;
2223 logmsg "\nTest definition reading and preparation time per test " .
2224 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2225 logmsg "-time- test\n";
2226 logmsg "------ ----\n";
2227 foreach my $txt (@timeprep) {
2228 last if((not $fullstats) && (not $counter--));
2229 logmsg "$txt\n";
2232 $counter = 25;
2233 logmsg "\nTest tool execution time per test " .
2234 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2235 logmsg "-time- test\n";
2236 logmsg "------ ----\n";
2237 foreach my $txt (@timetool) {
2238 last if((not $fullstats) && (not $counter--));
2239 logmsg "$txt\n";
2242 $counter = 15;
2243 logmsg "\nTest server logs lock removal time per test " .
2244 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2245 logmsg "-time- test\n";
2246 logmsg "------ ----\n";
2247 foreach my $txt (@timelock) {
2248 last if((not $fullstats) && (not $counter--));
2249 logmsg "$txt\n";
2252 $counter = 10;
2253 logmsg "\nTest results verification time per test " .
2254 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2255 logmsg "-time- test\n";
2256 logmsg "------ ----\n";
2257 foreach my $txt (@timevrfy) {
2258 last if((not $fullstats) && (not $counter--));
2259 logmsg "$txt\n";
2262 $counter = 50;
2263 logmsg "\nTotal time per test " .
2264 sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2265 logmsg "-time- test\n";
2266 logmsg "------ ----\n";
2267 foreach my $txt (@timetest) {
2268 last if((not $fullstats) && (not $counter--));
2269 logmsg "$txt\n";
2272 logmsg "\n";
2274 logmsg "::endgroup::\n";
2277#######################################################################
2278# returns code indicating why a test was skipped
2279# 0=unknown test, 1=use test result, 2=ignore test result
2281sub ignoreresultcode {
2282 my ($testnum) = @_;
2283 if(defined $ignoretestcodes{$testnum}) {
2284 return $ignoretestcodes{$testnum};
2286 return 0;
2289#######################################################################
2290# Put the given runner ID onto the queue of runners ready for a new task
2292sub runnerready {
2293 my ($runnerid) = @_;
2294 push @runnersidle, $runnerid;
2297#######################################################################
2298# Create test runners
2300sub createrunners {
2301 my ($numrunners) = @_;
2302 if(! $numrunners) {
2303 $numrunners++;
2305 # create $numrunners runners with minimum 1
2306 for my $runnernum (1..$numrunners) {
2307 my $dir = getrunnernumlogdir($runnernum);
2308 cleardir($dir);
2309 mkdir($dir, 0777);
2310 my $id = runner_init($dir, $jobs);
2311 $runnerids{$runnernum} = $id;
2312 $runnernums{$id} = $runnernum;
2313 runnerready($id);
2317#######################################################################
2318# Pick a test runner for the given test
2320sub pickrunner {
2321 my ($testnum) = @_;
2322 scalar(@runnersidle) or die "No runners available";
2324 return pop @runnersidle;
2327#######################################################################
2328# Check options to this test program
2331# Special case for CMake: replace '$TFLAGS' by the contents of the
2332# environment variable (if any).
2333if(@ARGV && $ARGV[-1] eq '$TFLAGS') {
2334 pop @ARGV;
2335 push(@ARGV, split(' ', $ENV{'TFLAGS'})) if defined($ENV{'TFLAGS'});
2338$args = join(' ', @ARGV);
2340my $mintotalany = 0;
2342$valgrind = checktestcmd("valgrind");
2343my $number = 0;
2344my $fromnum = -1;
2345my $useshares;
2346my $usepart;
2347my @testthis;
2348while(@ARGV) {
2349 if($ARGV[0] eq "-v") {
2350 # verbose output
2351 $verbose = 1;
2353 elsif($ARGV[0] eq "-c") {
2354 # use this path to curl instead of default
2355 $DBGCURL = $CURL = $ARGV[1];
2356 shift @ARGV;
2358 elsif($ARGV[0] eq "-vc") {
2359 # use this path to a curl used to verify servers
2361 # Particularly useful when you introduce a crashing bug somewhere in
2362 # the development version as then it is not able to run any tests
2363 # since it cannot verify the servers!
2365 $VCURL = shell_quote($ARGV[1]);
2366 shift @ARGV;
2368 elsif($ARGV[0] eq "-ac") {
2369 # use this curl only to talk to APIs (currently only CI test APIs)
2370 $ACURL = shell_quote($ARGV[1]);
2371 shift @ARGV;
2373 elsif($ARGV[0] eq "-d") {
2374 # have the servers display protocol output
2375 $debugprotocol = 1;
2377 elsif(($ARGV[0] eq "-e") || ($ARGV[0] eq "--test-event")) {
2378 # run the tests cases event based if possible
2379 $run_event_based = 1;
2381 elsif($ARGV[0] eq "--test-duphandle") {
2382 # run the tests with --test-duphandle
2383 $run_duphandle = 1;
2385 elsif($ARGV[0] eq "-f") {
2386 # force - run the test case even if listed in DISABLED
2387 $run_disabled = 1;
2389 elsif($ARGV[0] eq "-E") {
2390 # load additional reasons to skip tests
2391 shift @ARGV;
2392 my $exclude_file = $ARGV[0];
2393 open(my $fd, "<", $exclude_file) or die "Could not open '$exclude_file': $!";
2394 while(my $line = <$fd>) {
2395 next if($line =~ /^#/);
2396 chomp $line;
2397 my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3);
2399 die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/);
2401 foreach my $pattern (split(/,/, $patterns)) {
2402 if($type eq "test") {
2403 # Strip leading zeros in the test number
2404 $pattern = int($pattern);
2406 $custom_skip_reasons{$type}{$pattern} = $skip_reason;
2409 close($fd);
2411 elsif($ARGV[0] eq "-g") {
2412 # run this test with gdb
2413 $gdbthis = 1;
2415 elsif($ARGV[0] eq "-gl") {
2416 # run this test with lldb
2417 $gdbthis = 2;
2419 elsif($ARGV[0] eq "-gw") {
2420 # run this test with windowed gdb
2421 $gdbthis = 1;
2422 $gdbxwin = 1;
2424 elsif($ARGV[0] eq "-s") {
2425 # short output
2426 $short = 1;
2428 elsif($ARGV[0] eq "-am") {
2429 # automake-style output
2430 $short = 1;
2431 $automakestyle = 1;
2433 elsif($ARGV[0] =~ /-m=(\d+)/) {
2434 my ($num) = ($1);
2435 $maxtime = $num;
2437 elsif($ARGV[0] =~ /--min=(\d+)/) {
2438 my ($num) = ($1);
2439 $mintotal = $num;
2440 $mintotalany = 1;
2442 elsif($ARGV[0] eq "-n") {
2443 # no valgrind
2444 undef $valgrind;
2446 elsif($ARGV[0] eq "--no-debuginfod") {
2447 # disable the valgrind debuginfod functionality
2448 $no_debuginfod = 1;
2450 elsif($ARGV[0] eq "-R") {
2451 # execute in scrambled order
2452 $scrambleorder = 1;
2454 elsif($ARGV[0] =~ /^-t(.*)/) {
2455 # torture
2456 $torture = 1;
2457 my $xtra = $1;
2459 if($xtra =~ s/(\d+)$//) {
2460 $tortalloc = $1;
2463 elsif($ARGV[0] =~ /^--subset=(\d+)\/(\d+)$/) {
2464 # split all tests into $2 parts.
2465 # this invoke then runs the part number $1 (0-indexed)
2466 ($usepart, $useshares) = ($1, $2);
2467 if($useshares < 1 || $usepart >= $useshares) {
2468 die "illegal subset specified";
2471 elsif($ARGV[0] =~ /--shallow=(\d+)/) {
2472 # Fail no more than this amount per tests when running
2473 # torture.
2474 my ($num) = ($1);
2475 $shallow = $num;
2477 elsif($ARGV[0] =~ /--repeat=(\d+)/) {
2478 # Repeat-run the given tests this many times
2479 $repeat = $1;
2481 elsif($ARGV[0] =~ /--retry=(\d+)/) {
2482 # Number of attempts for the whole test run to retry failed tests
2483 $retry = $1;
2485 elsif($ARGV[0] =~ /--seed=(\d+)/) {
2486 # Set a fixed random seed (used for -R and --shallow)
2487 $randseed = $1;
2489 elsif($ARGV[0] eq "-a") {
2490 # continue anyway, even if a test fail
2491 $anyway = 1;
2493 elsif($ARGV[0] eq "-o") {
2494 shift @ARGV;
2495 if($ARGV[0] =~ /^(\w+)=([\w.:\/\[\]-]+)$/) {
2496 my ($variable, $value) = ($1, $2);
2497 eval "\$$variable='$value'" or die "Failed to set \$$variable to $value: $@";
2499 else {
2500 die "Failed to parse '-o $ARGV[0]'. May contain unexpected characters.\n";
2503 elsif($ARGV[0] eq "-S") {
2504 $show_success_logs = 1;
2506 elsif($ARGV[0] eq "-p") {
2507 $postmortem = 1;
2509 elsif($ARGV[0] eq "-P") {
2510 shift @ARGV;
2511 $proxy_address = $ARGV[0];
2513 elsif($ARGV[0] eq "-L") {
2514 # require additional library file
2515 shift @ARGV;
2516 require $ARGV[0];
2518 elsif($ARGV[0] eq "-l") {
2519 # lists the test case names only
2520 $listonly = 1;
2522 elsif($ARGV[0] eq "--buildinfo") {
2523 $buildinfo = 1;
2525 elsif($ARGV[0] =~ /^-j(.*)/) {
2526 # parallel jobs
2527 $jobs = 1;
2528 my $xtra = $1;
2529 if($xtra =~ s/(\d+)$//) {
2530 $jobs = $1;
2533 elsif($ARGV[0] eq "-k") { # delete this check after December 2026
2534 print "Option -k became always-on in 7.65.2 (2019) and now a no-op. Delete it to continue.\n";
2535 exit 1;
2537 elsif($ARGV[0] eq "-r") {
2538 # run time statistics needs Time::HiRes
2539 if($Time::HiRes::VERSION) {
2540 # presize hashes appropriately to hold an entire test run
2541 keys(%timeprepini) = 2000;
2542 keys(%timesrvrini) = 2000;
2543 keys(%timesrvrend) = 2000;
2544 keys(%timetoolini) = 2000;
2545 keys(%timetoolend) = 2000;
2546 keys(%timesrvrlog) = 2000;
2547 keys(%timevrfyend) = 2000;
2548 $timestats = 1;
2549 $fullstats = 0;
2552 elsif($ARGV[0] eq "-rf") {
2553 # run time statistics needs Time::HiRes
2554 if($Time::HiRes::VERSION) {
2555 # presize hashes appropriately to hold an entire test run
2556 keys(%timeprepini) = 2000;
2557 keys(%timesrvrini) = 2000;
2558 keys(%timesrvrend) = 2000;
2559 keys(%timetoolini) = 2000;
2560 keys(%timetoolend) = 2000;
2561 keys(%timesrvrlog) = 2000;
2562 keys(%timevrfyend) = 2000;
2563 $timestats = 1;
2564 $fullstats = 1;
2567 elsif($ARGV[0] eq "-u") {
2568 # error instead of warning on server unexpectedly alive
2569 $err_unexpected = 1;
2571 elsif(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) {
2572 # show help text
2573 print <<"EOHELP"
2574Usage: runtests.pl [options] [test selection(s)]
2575 -a continue even if a test fails
2576 -ac path use this curl only to talk to APIs (currently only CI test APIs)
2577 -am automake style output PASS/FAIL: [number] [name]
2578 --buildinfo dump buildinfo.txt
2579 -c path use this curl executable
2580 -d display server debug info
2581 -e, --test-event event-based execution
2582 --test-duphandle duplicate handles before use
2583 -E file load the specified file to exclude certain tests
2584 -f forcibly run even if disabled
2585 -g run the test case with gdb
2586 -gw run the test case with gdb as a windowed application
2587 -h this help text
2588 -j[N] spawn this number of processes to run tests (default 0)
2589 -L path require an additional perl library file to replace certain functions
2590 -l list all test case names/descriptions
2591 -m=[seconds] set timeout for curl commands in tests
2592 --min=[count] minimum number of tests to run.
2593 -n no valgrind
2594 --no-debuginfod disable the valgrind debuginfod functionality
2595 -o variable=value set internal variable to the specified value
2596 -P proxy use the specified proxy
2597 -p print log file contents when a test fails
2598 -R scrambled order (uses the random seed, see --seed)
2599 -r run time statistics
2600 -rf full run time statistics
2601 --repeat=[num] run the given tests this many times
2602 --retry=[num] number of attempts for the whole test run to retry failed tests
2603 -S print log file contents when a test passes
2604 -s short output
2605 --seed=[num] set the random seed to a fixed number
2606 --shallow=[num] randomly makes the torture tests "thinner"
2607 -t[N] torture (simulate function failures); N means fail Nth function
2608 -u error instead of warning on server unexpectedly alive
2609 -v verbose output
2610 -vc path use this curl only to verify the existing servers
2611 [num] like "5 6 9" or " 5 to 22 " to run those tests only
2612 [!num] like "!5 !6 !9" to disable those tests
2613 [~num] like "~5 ~6 ~9" to ignore the result of those tests
2614 [keyword] like "IPv6" to select only tests containing the key word
2615 [!keyword] like "!cookies" to disable any tests containing the key word
2616 [~keyword] like "~cookies" to ignore results of tests containing key word
2617EOHELP
2619 exit;
2621 elsif($ARGV[0] =~ /^(\d+)/) {
2622 $number = $1;
2623 if($fromnum >= 0) {
2624 for my $n ($fromnum .. $number) {
2625 push @testthis, $n;
2627 $fromnum = -1;
2629 else {
2630 push @testthis, $1;
2633 elsif($ARGV[0] =~ /^to$/i) {
2634 $fromnum = $number + 1;
2636 elsif($ARGV[0] =~ /^!(\d+)/) {
2637 $fromnum = -1;
2638 $disabled{$1} = $1;
2640 elsif($ARGV[0] =~ /^~(\d+)/) {
2641 $fromnum = -1;
2642 $ignored{$1} = $1;
2644 elsif($ARGV[0] =~ /^!(.+)/) {
2645 $disabled_keywords{lc($1)} = $1;
2647 elsif($ARGV[0] =~ /^~(.+)/) {
2648 $ignored_keywords{lc($1)} = $1;
2650 elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
2651 $enabled_keywords{lc($1)} = $1;
2653 else {
2654 print "Unknown option: $ARGV[0]\n";
2655 exit;
2657 shift @ARGV;
2660delete $ENV{'DEBUGINFOD_URLS'} if($ENV{'DEBUGINFOD_URLS'} && $no_debuginfod);
2662if(!$randseed) {
2663 my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
2664 # seed of the month. December 2019 becomes 201912
2665 $randseed = ($year + 1900) * 100 + $mon + 1;
2666 print "Using curl: $CURL\n";
2667 open(my $curlvh, "-|", exerunner() . shell_quote($CURL) . " --disable --version 2>$dev_null") or
2668 die "could not get curl version!";
2669 my @c = <$curlvh>;
2670 close($curlvh) or die "could not get curl version!";
2671 # use the first line of output and get the md5 out of it
2672 my $str = md5($c[0]);
2673 $randseed += unpack('S', $str); # unsigned 16-bit value
2675srand $randseed;
2677if(@testthis && ($testthis[0] ne "")) {
2678 $TESTCASES = join(" ", @testthis);
2681if($valgrind) {
2682 # we have found valgrind on the host, use it
2684 # verify that we can invoke it fine
2685 my $code = runclient("valgrind >$dev_null 2>&1");
2687 if(($code>>8) != 1) {
2688 #logmsg "Valgrind failure, disable it\n";
2689 undef $valgrind;
2691 else {
2692 open(my $curlh, "<", $CURL);
2693 my $l = <$curlh>;
2694 if($l =~ /^\#\!/) {
2695 # A shell script. This is typically when built with libtool,
2696 $valgrind = "../libtool --mode=execute $valgrind";
2698 close($curlh);
2702if($gdbthis) {
2703 # open the executable curl and read the first 4 bytes of it
2704 open(my $check, "<", $CURL);
2705 my $c;
2706 sysread $check, $c, 4;
2707 close($check);
2708 if($c eq "#! /") {
2709 # A shell script. This is typically when built with libtool,
2710 $libtool = 1;
2711 $gdb = "../libtool --mode=execute gdb";
2715#######################################################################
2716# clear and create logging directory:
2719# TODO: figure how to get around this. This directory is needed for checksystemfeatures()
2720# Maybe create & use & delete a temporary directory in that function
2721cleardir($LOGDIR);
2722mkdir($LOGDIR, 0777);
2723mkdir("$LOGDIR/$LOCKDIR", 0777);
2725#######################################################################
2726# initialize some variables
2729get_disttests();
2730if(!$jobs) {
2731 # Disable buffered logging with only one test job
2732 setlogfunc(\&logmsg);
2735if(!$mintotalany && $ENV{"CURL_TEST_MIN"}) {
2736 $mintotal = $ENV{"CURL_TEST_MIN"};
2737 if($useshares) {
2738 $mintotal /= $useshares;
2739 # add safety margin to avoid potential underruns
2740 $mintotal = $mintotal - 30 < 1 ? 1 : $mintotal - 30;
2744#######################################################################
2745# Output curl version and host info being tested
2748if(!$listonly) {
2749 checksystemfeatures();
2752#######################################################################
2753# Output information about the curl build
2755if(!$listonly && $buildinfo) {
2756 if(open(my $fd, "<", '../buildinfo.txt')) {
2757 while(my $line = <$fd>) {
2758 chomp $line;
2759 if($line && $line !~ /^#/) {
2760 logmsg("* $line\n");
2763 close($fd);
2767#######################################################################
2768# initialize configuration needed to set up servers
2769# TODO: rearrange things so this can be called only in runner_init()
2771initserverconfig();
2773if(!$listonly) {
2774 # these can only be displayed after initserverconfig() has been called
2775 displayserverfeatures();
2777 # globally disabled tests
2778 disabledtests("$TESTDIR/DISABLED");
2781#######################################################################
2782# Fetch all disabled tests, if there are any
2784sub disabledtests {
2785 my ($file) = @_;
2786 my @input;
2788 if(open(my $disabledh, "<", $file)) {
2789 while(<$disabledh>) {
2790 if(/^ *\#/) {
2791 # allow comments
2792 next;
2794 push @input, $_;
2796 close($disabledh);
2798 # preprocess the input to make conditionally disabled tests depending
2799 # on variables
2800 my @pp = prepro(0, @input);
2801 for my $t (@pp) {
2802 if($t =~ /(\d+)/) {
2803 my ($n) = $1;
2804 $disabled{$n} = $n; # disable this test number
2805 if(! -f "$srcdir/data/test$n") {
2806 print STDERR "WARNING! Non-existing test $n in $file!\n";
2807 # fail hard to make user notice
2808 exit 1;
2810 logmsg "DISABLED: test $n\n" if($verbose);
2812 else {
2813 print STDERR "$file: rubbish content: $t\n";
2814 exit 2;
2818 else {
2819 print STDERR "Cannot open $file, exiting\n";
2820 exit 3;
2824#######################################################################
2825# If 'all' tests are requested, find out all test numbers
2828if($TESTCASES eq "all") {
2829 # Get all commands and find out their test numbers
2830 opendir(DIR, $TESTDIR) or die "cannot opendir $TESTDIR: $!";
2831 my @cmds = grep { /^test([0-9]+)$/ && -f "$TESTDIR/$_" } readdir(DIR);
2832 closedir(DIR);
2834 $TESTCASES = ""; # start with no test cases
2836 # cut off everything but the digits
2837 for(@cmds) {
2838 $_ =~ s/[a-z\/\.]*//g;
2840 # sort the numbers from low to high
2841 foreach my $n (sort { $a <=> $b } @cmds) {
2842 if($disabled{$n}) {
2843 # skip disabled test cases
2844 my $why = "configured as DISABLED";
2845 $skipped{$why}++;
2846 $teststat[$n] = $why; # store reason for this test case
2847 next;
2849 $TESTCASES .= " $n";
2852else {
2853 my $verified = "";
2854 for(split(" ", $TESTCASES)) {
2855 if(-e "$TESTDIR/test$_") {
2856 $verified .= "$_ ";
2859 if($verified eq "") {
2860 print "No existing test cases were specified\n";
2861 exit;
2863 $TESTCASES = $verified;
2865if($repeat) {
2866 my $s;
2867 for(1 .. $repeat) {
2868 $s .= $TESTCASES;
2870 $TESTCASES = $s;
2873if($scrambleorder) {
2874 # scramble the order of the test cases
2875 my @rand;
2876 while($TESTCASES) {
2877 my @all = split(/ +/, $TESTCASES);
2878 if(!$all[0]) {
2879 # if the first is blank, shift away it
2880 shift @all;
2882 my $r = rand @all;
2883 push @rand, $all[$r];
2884 $all[$r] = "";
2885 $TESTCASES = join(" ", @all);
2887 $TESTCASES = join(" ", @rand);
2890if($useshares) {
2891 my @a = grep { length($_) } split(/ +/, $TESTCASES);
2892 my $n = scalar(@a);
2894 if($useshares < 1 || $usepart >= $useshares) {
2895 die "illegal subset specified";
2898 my $start = int(($n * $usepart) / $useshares);
2899 my $end = int(($n * ($usepart + 1)) / $useshares); # one past last index
2900 my $run = $end - $start;
2902 printf STDERR "Subset: 1/%u of the tests (run %u tests out of %u). Part %u\n",
2903 $useshares, $run, $n, $usepart;
2905 my @s = $run ? @a[$start .. $end - 1] : ();
2906 $TESTCASES = join(" ", @s);
2909# Display the contents of the given file. Line endings are canonicalized
2910# and excessively long files are elided
2911sub displaylogcontent {
2912 my ($file) = @_;
2913 if(open(my $single, "<", $file)) {
2914 my $linecount = 0;
2915 my $truncate;
2916 my @tail;
2917 while(my $string = <$single>) {
2918 $string =~ s/\r\n/\n/g;
2919 $string =~ s/[\r\f\032]/\n/g;
2920 $string .= "\n" unless ($string =~ /\n$/);
2921 $string =~ tr/\n//;
2922 for my $line (split(m/\n/, $string)) {
2923 $line =~ s/\s*\!$//;
2924 if($truncate) {
2925 push @tail, " $line\n";
2927 else {
2928 logmsg " $line\n";
2930 $linecount++;
2931 $truncate = $linecount > 1200;
2934 close($single);
2935 if(@tail) {
2936 my $tailshow = 200;
2937 my $tailskip = 0;
2938 my $tailtotal = scalar @tail;
2939 if($tailtotal > $tailshow) {
2940 $tailskip = $tailtotal - $tailshow;
2941 logmsg "=== File too long: $tailskip lines omitted here\n";
2943 for($tailskip .. $tailtotal-1) {
2944 logmsg "$tail[$_]";
2950sub displaylogs {
2951 my ($runnerid, $testnum) = @_;
2952 my $logdir = getrunnerlogdir($runnerid);
2953 opendir(DIR, $logdir) or die "cannot open dir: $!";
2954 my @logs = readdir(DIR);
2955 closedir(DIR);
2957 logmsg "== Contents of files in the $logdir/ directory after test $testnum\n";
2958 foreach my $log (sort @logs) {
2959 if($log =~ /\.(\.|)$/) {
2960 next; # skip "." and ".."
2962 if($log =~ /^\.nfs/) {
2963 next; # skip ".nfs"
2965 if(($log eq "memdump") || ($log eq "core")) {
2966 next; # skip "memdump" and "core"
2968 if((-d "$logdir/$log") || (! -s "$logdir/$log")) {
2969 next; # skip directory and empty files
2971 if(($log =~ /^stdout\d+/) && ($log !~ /^stdout$testnum/)) {
2972 next; # skip stdoutNnn of other tests
2974 if(($log =~ /^stderr\d+/) && ($log !~ /^stderr$testnum/)) {
2975 next; # skip stderrNnn of other tests
2977 if(($log =~ /^upload\d+/) && ($log !~ /^upload$testnum/)) {
2978 next; # skip uploadNnn of other tests
2980 if(($log =~ /^curl\d+\.out/) && ($log !~ /^curl$testnum\.out/)) {
2981 next; # skip curlNnn.out of other tests
2983 if(($log =~ /^test\d+\.txt/) && ($log !~ /^test$testnum\.txt/)) {
2984 next; # skip testNnn.txt of other tests
2986 if(($log =~ /^file\d+\.txt/) && ($log !~ /^file$testnum\.txt/)) {
2987 next; # skip fileNnn.txt of other tests
2989 if(($log =~ /^netrc\d+/) && ($log !~ /^netrc$testnum/)) {
2990 next; # skip netrcNnn of other tests
2992 if(($log =~ /^trace\d+/) && ($log !~ /^trace$testnum/)) {
2993 next; # skip traceNnn of other tests
2995 if(($log =~ /^valgrind\d+/) && ($log !~ /^valgrind$testnum(?:\..*)?$/)) {
2996 next; # skip valgrindNnn of other tests
2998 if(($log =~ /^test$testnum$/)) {
2999 next; # skip test$testnum since it can be big
3001 logmsg "=== Start of file $log\n";
3002 displaylogcontent("$logdir/$log");
3003 logmsg "=== End of file $log\n";
3007#######################################################################
3008# Scan tests to find suitable candidates
3011my $failed;
3012my $failedign;
3013my $failedre;
3014my $ok = 0;
3015my $ign = 0;
3016my $total = 0;
3017my $executed = 0;
3018my $retry_done = 0;
3019my $lasttest = 0;
3020my @at = split(" ", $TESTCASES);
3021my $count = 0;
3022my $endwaitcnt = 0;
3024$start = time();
3026# scan all tests to find ones we should try to run
3027my @runtests;
3028foreach my $testnum (@at) {
3029 $lasttest = $testnum if($testnum > $lasttest);
3030 my ($why, $errorreturncode) = singletest_shouldrun($testnum);
3031 if($why || $listonly) {
3032 # Display test name now--test is completely skipped later
3033 my $error = singletest_count($testnum, $why);
3034 next;
3036 $ignoretestcodes{$testnum} = $errorreturncode;
3037 push(@runtests, $testnum);
3039my $totaltests = scalar(@runtests);
3041if($listonly) {
3042 exit(0);
3045#######################################################################
3046# Setup CI Test Run
3047citest_starttestrun();
3049#######################################################################
3050# Start test runners
3052my $numrunners = $jobs < scalar(@runtests) ? $jobs : scalar(@runtests);
3053createrunners($numrunners);
3055#######################################################################
3056# The main test-loop
3058# Every iteration through the loop consists of these steps:
3059# - if the global abort flag is set, exit the loop; we are done
3060# - if a runner is idle, start a new test on it
3061# - if all runners are idle, exit the loop; we are done
3062# - if a runner has a response for us, process the response
3064# run through each candidate test and execute it
3065my $runner_wait_cnt = 0;
3067# number of retry attempts for the whole test run
3068my $retry_left;
3069if($torture) {
3070 $retry_left = 0; # No use of retrying torture tests
3072else {
3073 $retry_left = $retry;
3076while(1) {
3077 # check the abort flag
3078 if($globalabort) {
3079 logmsg singletest_dumplogs();
3080 logmsg "Aborting tests\n";
3081 logmsg "Waiting for " . scalar((keys %runnersrunning)) . " outstanding test(s) to finish...\n";
3082 # Wait for the last requests to complete and throw them away so
3083 # that IPC calls & responses stay in sync
3084 # TODO: send a signal to the runners to interrupt a long test
3085 foreach my $rid (keys %runnersrunning) {
3086 runnerar($rid);
3087 delete $runnersrunning{$rid};
3088 logmsg ".";
3089 $| = 1;
3091 logmsg "\n";
3092 last;
3095 # Start a new test if possible
3096 if(scalar(@runnersidle) && scalar(@runtests)) {
3097 # A runner is ready to run a test, and tests are still available to run
3098 # so start a new test.
3099 $count++;
3100 my $testnum = shift(@runtests);
3102 # pick a runner for this new test
3103 my $runnerid = pickrunner($testnum);
3104 $countforrunner{$runnerid} = $count;
3106 # Start the test
3107 my ($error, $again) = singletest($runnerid, $testnum, $countforrunner{$runnerid}, $totaltests);
3108 if($again) {
3109 # this runner is busy running a test
3110 $runnersrunning{$runnerid} = $testnum;
3112 else {
3113 runnerready($runnerid);
3114 if($error >= 0) {
3115 # We make this simplifying assumption to avoid having to handle
3116 # $error properly here, but we must handle the case of runner
3117 # death without abending here.
3118 die "Internal error: test must not complete on first call";
3123 # See if we have completed all the tests
3124 if(!scalar(%runnersrunning)) {
3125 # No runners are running; we must be done
3126 scalar(@runtests) && die 'Internal error: still have tests to run';
3127 last;
3130 # See if a test runner needs attention
3131 # If we could be running more tests, do not wait so we can schedule a new
3132 # one immediately. If all runners are busy, wait a fraction of a second
3133 # for one to finish so we can still loop around to check the abort flag.
3134 my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0.1 : 1.0;
3135 my (@ridsready, $riderror) = runnerar_ready($runnerwait);
3136 if(@ridsready) {
3137 for my $ridready (@ridsready) {
3138 if($ridready && ! defined $runnersrunning{$ridready}) {
3139 # On Linux, a closed pipe still shows up as ready instead of error.
3140 # Detect this here by seeing if we are expecting it to be ready and
3141 # treat it as an error if not.
3142 logmsg "ERROR: Runner $ridready is unexpectedly ready; is probably actually dead\n";
3143 $riderror = $ridready;
3144 undef $ridready;
3146 if($ridready) {
3147 $endwaitcnt = 0;
3148 # This runner is ready to be serviced
3149 my $testnum = $runnersrunning{$ridready};
3150 defined $testnum or die "Internal error: test for runner $ridready unknown";
3151 delete $runnersrunning{$ridready};
3152 my ($error, $again) = singletest($ridready, $testnum, $countforrunner{$ridready}, $totaltests);
3153 if($again) {
3154 # this runner is busy running a test
3155 $runnersrunning{$ridready} = $testnum;
3157 else {
3158 # Test is complete
3159 $runner_wait_cnt = 0;
3160 runnerready($ridready);
3162 if($error < 0) {
3163 # not a test we can run
3164 next;
3167 $total++; # number of tests we have run
3168 $executed++;
3170 if($error > 0) {
3171 if($error == 2) {
3172 # ignored test failures
3173 $failedign .= "$testnum ";
3175 else {
3176 # make another attempt to counteract flaky failures
3177 if($may_retry{$testnum} && $retry_left > 0) {
3178 $retry_left--;
3179 $retry_done++;
3180 $total--;
3181 push(@runtests, $testnum);
3182 $failedre .= "$testnum ";
3184 else {
3185 $failed.= "$testnum ";
3188 if($postmortem) {
3189 # display all files in $LOGDIR/ in a nice way
3190 displaylogs($ridready, $testnum);
3192 if($error == 2) {
3193 $ign++; # ignored test result counter
3195 elsif(!$anyway) {
3196 # a test failed, abort
3197 logmsg "\n - abort tests\n";
3198 undef @runtests; # empty out the remaining tests
3201 elsif(!$error) {
3202 $ok++; # successful test counter
3203 if($show_success_logs) {
3204 displaylogs($ridready, $testnum);
3211 if(!@ridsready && $runnerwait && !$torture && scalar(%runnersrunning)) {
3212 $runner_wait_cnt++;
3213 if($runner_wait_cnt >= 5) {
3214 my $msg = "waiting for " . scalar(%runnersrunning) . " results:";
3215 my $sep = " ";
3216 foreach my $rid (keys %runnersrunning) {
3217 $msg .= $sep . $runnersrunning{$rid} . "[$rid]";
3218 $sep = ", "
3220 logmsg "$msg\n";
3222 if($runner_wait_cnt >= 10) {
3223 $runner_wait_cnt = 0;
3224 foreach my $rid (keys %runnersrunning) {
3225 my $testnum = $runnersrunning{$rid};
3226 logmsg "current state of test $testnum in [$rid]:\n";
3227 displaylogs($rid, $testnum);
3231 if($riderror) {
3232 logmsg "ERROR: runner $riderror is dead! aborting test run\n";
3233 delete $runnersrunning{$riderror} if(defined $runnersrunning{$riderror});
3234 $globalabort = 1;
3236 $endwaitcnt += $runnerwait;
3237 if($endwaitcnt >= 10) {
3238 # Once all tests have been scheduled on a runner at the end of a test
3239 # run, we wait for their results to come in. If we are still
3240 # waiting after a couple of minutes ($endwaitcnt multiplied by
3241 # $runnerwait, plus $jobs because that number does not time out), display
3242 # the same test runner status as we give with a SIGUSR1. This likely
3243 # points to a single test that has hung.
3244 logmsg "Hmmm, the tests are taking a while to finish. Here is the status:\n";
3245 catch_usr1();
3246 $endwaitcnt = 0;
3250my $sofar = time() - $start;
3252#######################################################################
3253# Finish CI Test Run
3254citest_finishtestrun();
3256# Tests done, stop the servers
3257foreach my $runnerid (values %runnerids) {
3258 runnerac_stopservers($runnerid);
3261# Wait for servers to stop
3262my $unexpected;
3263foreach my $runnerid (values %runnerids) {
3264 my ($rid, $unexpected_for_runner, $logs) = runnerar($runnerid);
3265 $unexpected ||= $unexpected_for_runner;
3266 logmsg $logs;
3269# Kill the runners
3270# There is a race condition here since we do not know exactly when the runners
3271# have each finished shutting themselves down, but we are about to exit so it
3272# does not make much difference.
3273foreach my $runnerid (values %runnerids) {
3274 runnerac_shutdown($runnerid);
3275 sleep 0; # give runner a context switch so it can shut itself down
3278my $numskipped = %skipped ? sum values %skipped : 0;
3279my $all = $total + $numskipped;
3281runtimestats($lasttest);
3283if($all) {
3284 logmsg "TESTDONE: $all tests were considered during " .
3285 sprintf("%.0f", $sofar) . " seconds.\n";
3288if(%skipped && !$short) {
3289 my $s = 0;
3290 # Temporary hash to print the restraints sorted by the number
3291 # of their occurrences
3292 my %restraints;
3293 logmsg "TESTINFO: $numskipped tests were skipped due to these restraints:\n";
3295 for(keys %skipped) {
3296 my $r = $_;
3297 my $skip_count = $skipped{$r};
3298 my $log_line = sprintf("TESTINFO: \"%s\" %d time%s (", $r, $skip_count,
3299 ($skip_count == 1) ? "" : "s");
3301 # now gather all test case numbers that had this reason for being
3302 # skipped
3303 my $c = 0;
3304 my $max = 9;
3305 for(0 .. scalar @teststat) {
3306 my $t = $_;
3307 if($teststat[$t] && ($teststat[$t] eq $r)) {
3308 if($c < $max) {
3309 $log_line .= ", " if($c);
3310 $log_line .= $t;
3312 $c++;
3315 if($c > $max) {
3316 $log_line .= " and " . ($c - $max) . " more";
3318 $log_line .= ")\n";
3319 $restraints{$log_line} = $skip_count;
3321 foreach my $log_line (sort {$restraints{$b} <=> $restraints{$a} || uc($a) cmp uc($b)} keys %restraints) {
3322 logmsg $log_line;
3326sub testnumdetails {
3327 my ($desc, $numlist) = @_;
3328 foreach my $testnum (split(' ', $numlist)) {
3329 if(!loadtest("${TESTDIR}/test${testnum}", 1)) {
3330 my @info_keywords = getpart("info", "keywords");
3331 my $testname = (getpart("client", "name"))[0];
3332 chomp $testname;
3333 logmsg "$desc $testnum: '$testname'";
3334 my $first = 1;
3335 for my $k (@info_keywords) {
3336 chomp $k;
3337 my $sep = ($first == 1) ? " " : ", ";
3338 logmsg "$sep$k";
3339 $first = 0;
3341 logmsg "\n";
3346if($executed) {
3347 if($failedre) {
3348 my $sorted = numsortwords($failedre);
3349 logmsg "::group::Failed Retried Test details\n";
3350 testnumdetails("FAIL-RETRIED", $sorted);
3351 logmsg "RETRIED: failed tests: $sorted\n";
3352 logmsg "::endgroup::\n";
3355 if($passedign) {
3356 my $sorted = numsortwords($passedign);
3357 logmsg "::group::Passed Ignored Test details\n";
3358 testnumdetails("PASSED-IGNORED", $sorted);
3359 logmsg "IGNORED: passed tests: $sorted\n";
3360 logmsg "::endgroup::\n";
3363 if($failedign) {
3364 my $sorted = numsortwords($failedign);
3365 testnumdetails("FAIL-IGNORED", $sorted);
3366 logmsg "IGNORED: failed tests: $sorted\n";
3368 logmsg sprintf("TESTDONE: $ok tests out of $total reported OK: %d%%\n",
3369 $ok / $total * 100);
3371 if($failed && ($ok != $total)) {
3372 my $failedsorted = numsortwords($failed);
3373 logmsg "\n";
3374 testnumdetails("FAIL", $failedsorted);
3375 logmsg "\nTESTFAIL: These test cases failed: $failedsorted\n\n";
3378else {
3379 logmsg "\nTESTFAIL: No tests were performed\n\n";
3380 if(scalar(keys %enabled_keywords)) {
3381 logmsg "TESTFAIL: Nothing matched these keywords: ";
3382 for(keys %enabled_keywords) {
3383 logmsg "$_ ";
3385 logmsg "\n";
3389if($mintotal) {
3390 if($total < $mintotal) {
3391 logmsg "TESTFAIL: number of tests run ($total) was below the minimum of: $mintotal\n";
3392 exit 1;
3394 else {
3395 logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
3399if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
3400 exit 1;
Diagnostics4 git calls · 29 ms
CallStatusSizeTime029 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch20030 KiB · hit24 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=24
git.response.bytes=31131
git.response.cache=HIT
git.response.duration_ms=24
git.fetch.wants=98ac6f403f91a88ab50c8044dc7829a315064c53
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=31131
git.pkt.payload.max=31110
git.sideband.band1.bytes=31109
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e2dec15c592308
Request
c26a2847…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format