5 # Script to analyze failed build logs. Look for specific regexps to
9 # (c) 2018 Steve McIntyre <steve@einval.com> GPL v2+
13 use POSIX qw(strftime);
18 my $name = "analyze_results";
19 my $repo = "https://git.einval.com/cgi-bin/gitweb.cgi?p=buildd-scripts.git";
22 my $time_start = time();
26 my $awaiting_analysis = 0;
28 my $existing_bugs = 0;
29 my $buildd_base_url="https://buildd.debian.org/status/package.php?p=";
32 ERR_ARCH_MISMATCH => 1,
36 ERR_DETECT_WRONG_ARCH => 5,
38 ERR_BUILD_PROBLEM => 7,
39 ERR_TEST_PROBLEM => 8,
40 ERR_BUILD_TIMEOUT => 9,
44 $err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches";
45 $err_descriptions [ERR_NO_SOURCE] = "No source found";
46 $err_descriptions [ERR_INFRA] = "Infrastructure errors";
47 $err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps";
48 $err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture";
49 $err_descriptions [ERR_CRASH] = "Crashes detected";
50 $err_descriptions [ERR_BUILD_PROBLEM] = "Problems detected during build phase";
51 $err_descriptions [ERR_TEST_PROBLEM] = "Problems detected during test phase";
52 $err_descriptions [ERR_BUILD_TIMEOUT] = "Package builds timed out";
54 # Known failure modes to look for
57 # "rchitecture mismatch" -> should never build on this arch
58 # Stop working on this log at this point
59 string => 'rchitecture mismatch',
60 message => 'Architecture mismatch',
64 type => ERR_ARCH_MISMATCH,
67 string => 'not in arch list or does not match any',
68 message => 'Architecture mismatch',
72 type => ERR_ARCH_MISMATCH,
75 # "binary build with no binary artifacts found" -> no packages
76 # built. Why not picked up already above?.
77 # Stop working on this log at this point
78 string => 'binary build with no binary artifacts found',
79 message => 'No binaries built',
83 type => ERR_ARCH_MISMATCH,
86 string => 'E: Can not find version \S+ of package',
87 message => 'Could not find specified source package',
91 type => ERR_NO_SOURCE,
94 string => 'schroot.*File is not owned by user root',
95 message => 'Schroot setup failure',
102 string => 'No space left on device',
103 message => 'Build ran out of disk space',
110 string => 'The system has no more ptys',
111 message => 'Build ran out of ptys',
118 string => 'aarch64-unknown-linux-gnu',
119 message => 'Wrong arch detected',
123 type => ERR_DETECT_WRONG_ARCH,
126 string => 'binutils-aarch64',
127 message => 'Wrong arch detected',
131 type => ERR_DETECT_WRONG_ARCH,
134 string => 'lib.linux-aarch64',
135 message => 'Wrong arch detected',
139 type => ERR_DETECT_WRONG_ARCH,
142 # "Bus error" -> alignment bug
143 string => 'Bus error',
144 message => 'Alignment problem',
151 # "Segmentation fault" -> code problem
152 string => 'Segmentation fault',
153 pstring => 'Setting up (\S+)',
154 message => 'Segmentation fault when installing RESULT',
161 # "Segmentation fault" -> code problem
162 string => 'Segmentation fault',
163 message => 'Segmentation fault',
170 # "Illegal instruction" -> bad build target?
171 string => 'Illegal instruction',
172 pstring => 'Setting up (\S+)',
173 message => 'Illegal instruction when installing RESULT',
180 # "Illegal instruction" -> bad build target?
181 string => 'Illegal instruction',
182 message => 'Illegal instruction',
189 # Installing build-deps failed
190 string => 'dpkg: error processing package (\S+)',
191 message => 'Build-dep failed to install (RESULT)',
195 type => ERR_BD_PROBLEM,
198 # Installing build-deps failed
199 string => 'E: pbuilder-satisfydepends failed.',
200 message => 'Pbuilder build-deps failed',
204 type => ERR_BD_PROBLEM,
207 # Installing build-deps failed
208 string => 'E: Unmet dependencies',
209 message => 'Build-deps failed',
213 type => ERR_BD_PROBLEM,
216 # Installing build-deps failed
217 string => 'unsat-dependency: (\S+)',
218 message => 'Missing build-dep (RESULT)',
222 type => ERR_BD_PROBLEM,
225 # Installing build-deps failed
226 string => 'unsat-conflict: (\S+)',
227 message => 'Unsatisfiable build-dep conflict (RESULT)',
231 type => ERR_BD_PROBLEM,
234 # Build failed - missing build-dep?
235 string => 'build dependencies/conflicts unsatisfied',
236 message => 'Build-deps not satisfiable',
240 type => ERR_BD_PROBLEM,
244 string => 'dpkg-source: error: unrepresentable changes to source',
245 message => 'dpkg-source failure',
249 type => ERR_BUILD_PROBLEM,
252 # Build failed - missing build-dep?
253 string => 'ld: cannot find',
254 message => 'Build failure: missing library - missing build-dep?',
258 type => ERR_BUILD_PROBLEM,
261 # Build failed - missing build-dep?
262 string => 'fatal error:.*No such file or directory',
263 message => 'Build failure: missing header - missing build-dep?',
267 type => ERR_BUILD_PROBLEM,
270 # Build failed - missing build-dep?
271 string => 'SEVERE: Cannot resolve dependencies',
272 message => 'Build failure - missing build-dep?',
276 type => ERR_BUILD_PROBLEM,
279 # Build failed - can't exec something...
280 string => 'error trying to exec.*execvp: No',
281 message => 'Build failure (missing binary)',
285 type => ERR_BUILD_PROBLEM,
289 string => 'BUILD FAIL',
290 message => 'Build failure (java/javadoc)',
294 type => ERR_BUILD_PROBLEM,
298 string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$',
299 message => 'Build failure (other)',
303 type => ERR_BUILD_PROBLEM,
307 string => 'make.*returned exit code',
308 message => 'Build failure (other)',
312 type => ERR_BUILD_PROBLEM,
316 string => 'dh_auto_build:.*returned exit code \d+',
317 message => 'Build failure (other)',
321 type => ERR_BUILD_PROBLEM,
325 string => 'dh_auto_clean:.*returned exit code \d+',
326 message => 'Build failure (clean failed)',
330 type => ERR_BUILD_PROBLEM,
334 string => 'dh_auto_install:.*returned exit code \d+',
335 message => 'Build failure (install failed)',
339 type => ERR_BUILD_PROBLEM,
343 string => 'dh_auto_configure:.*returned exit code \d+',
344 message => 'Build failure (configure failed)',
348 type => ERR_BUILD_PROBLEM,
352 string => '^debian/.*recipe for target (\S+) failed',
353 message => 'Build failure (RESULT)',
357 type => ERR_BUILD_PROBLEM,
361 string => '^debian/.*\*\*\* (.*). Stop.',
362 message => 'Build error (RESULT)',
366 type => ERR_BUILD_PROBLEM,
370 string => 'dpkg-buildpackage: error:.*(debian/rules \S+) subprocess returned exit status 2',
371 message => 'Build error (RESULT)',
375 type => ERR_BUILD_PROBLEM,
379 string => 'fakeroot debian/rules binary',
380 message => 'Build failure (other)',
384 type => ERR_BUILD_PROBLEM,
388 string => 'constant \d+ overflows.*int',
389 message => 'Go 32-bit integer overflow',
393 type => ERR_BUILD_PROBLEM,
397 string => 'OSError: \[Errno 13\] Permission denied',
398 message => 'Python EPERM test failure',
402 type => ERR_TEST_PROBLEM,
406 string => 'dh_auto_test:.*returned exit code \d+',
407 message => 'Test failure',
411 type => ERR_TEST_PROBLEM,
414 # Timeout. pbuilder is too dumb to do this properly :-(
415 string => 'I: Terminating build process due to timeout',
416 message => 'Pbuilder build timeout',
420 type => ERR_BUILD_TIMEOUT,
423 # Timeout from sbuild
424 string => 'Build killed with signal TERM after \d+ minutes of inactivity',
425 message => 'Sbuild build timeout',
429 type => ERR_BUILD_TIMEOUT,
436 #foreach my $checktmp (@logcheck) {
437 # my %check = %$checktmp;
438 # print "looking for \"$check{string}\"\n";
439 # print " with log message \"$check{message}\"\n";
440 # print " check this regexp: $check{check}\n";
441 # print " stop if found: $check{stop}\n";
446 print "<title>Build log analysis</title>\n";
449 print "<h1>Build log analysis</h1>\n";
450 print "<p><a href=\"#summary\">Summary</a></p>\n";
451 print "<h2>Packages</h2>\n";
457 getopts('u:') or die "getopts failure\n";
459 if (defined $opt_u) {
460 open (UDD, "< $opt_u") or die "Can't open UDD list file $opt_u for reading\n";
461 while (defined ($line = <UDD>)) {
463 my ($pkg, $bugno, $bugsubj) = split (/ /, $line, 3);
465 my $tmp = $udd{"$pkg"};
469 push (@buglist, "$bugno: $bugsubj");
470 $udd{"$pkg"} = \@buglist;
474 # foreach my $key (keys %udd) {
475 # my $tmp = $udd{$key};
476 # my $num = scalar (@$tmp);
477 # print "$key has $num bugs:\n";
478 # foreach my $entry (@$tmp) {
484 sub check_buildd_status {
488 my $url = "$buildd_base_url$pkg";
490 # print "Looking at URL $url\n";
492 my $built_on_arches = "";
493 my $built_this_arch = 0;
494 my $ftbfs_this_arch = 0;
495 my $not_this_arch = 0;
497 # print "Checking buildd data for package $pkg, arch $arch\n";
499 my $response = HTTP::Tiny->new->get($url);
501 if ($response->{success}) {
502 # print "$response->{status} $response->{reason}\n";
503 my $content = $response->{content};
504 # print "$content\n";
506 # Split the content up into lines
507 my @lines = split("\n", $content);
509 # Now look for two things:
510 # 1. how many arches have built this package?
511 # 2. has *our* arch built it?
512 foreach my $line (@lines) {
514 if ($line =~ /fetch.php/) {
515 if ($line =~ /arch=([^&]*)&\;.*Installed/) {
516 $built_on_arches .= "$1 ";
518 if ($line =~ /a=$arch.*Installed/) {
521 if ($line =~ /a=$arch.*Build-Attempted/) {
525 if ($line =~ /$arch is not present in the architecture list set by the maintainer/) {
528 if ($line =~ /No entry in $arch database/) {
532 print " <li>Built on this arch: $built_this_arch\n";
533 print " <li>Built on arches: $built_on_arches\n";
534 if ($not_this_arch) {
535 print " <li>Not set to build on $arch\n";
537 if ($ftbfs_this_arch) {
538 print " <li>Already seen to FTBFS on $arch\n";
544 foreach my $input (@ARGV) {
545 open (IN, "< $input") or die "Can't read $input: $!\n";
550 my $errors_need_analysis = 0;
561 ($pkg, $version, $arch) = split (/_/, $pav);
563 print "<li>Looking at <a href=\"$input\">$pkg version $version on arch $arch</a>:\n";
565 while (defined ($line = <IN>) and !$stop) {
568 foreach my $checktmp (@logcheck) {
569 my %check = %$checktmp;
572 and ($line =~ m/$check{string}/)
573 and (!$check{pstring} or $oldline =~ m/$check{pstring}/)) {
574 # print " Line $lineno: found \"$check{string}\"\n";
576 my $message = $check{message};
577 $message =~ s,RESULT,$match,g;
578 $file_results{$message} = $check{type};
579 if (($check{type} == ERR_BUILD_TIMEOUT) and ($errors == 0)) {
580 print " <li>Line $lineno: $message\n";
581 print " <li>Build killed by timeout before any errors at line $lineno\n";
583 print " <li>Line $lineno: $message\n";
586 if ($check{analyze}) {
587 $errors_need_analysis++;
590 # print " stopping processing\n";
599 # End of checking this package
600 print " <li><strong>Found errors: $errors</strong>\n";
602 print " (maybe just timed out during build?)\n";
605 # Look for a note for manually-added logfile analysis
607 $note =~ s,\.log$,.note,;
609 open (IN, "< $note") or die "Can't open $note for reading: $!\n";
610 while (defined (my $line = <IN>)) {
612 if ($line =~ m,#(\d+),) {
615 $line =~ s,#(\d+),<a href="https://bugs.debian.org/$1">#$1</a>,g;
617 print " <li>$line\n";
621 if ($errors_need_analysis) {
622 print " <li><strong>Needs analysis</strong>\n";
624 my $tmp = $udd{$pkg};
625 my $num = scalar (@$tmp);
626 print " <li>$num reported FTBFS bugs:\n";
628 foreach my $entry (@$tmp) {
629 print " <li>#$entry\n";
633 check_buildd_status($pkg, $arch);
638 foreach my $key (keys %file_results) {
639 $log_results{$file_results{$key}}{$key} += 1;
640 # print "now have $log_results{$key} for \"$key\"\n";
642 if ($errors_need_analysis) {
643 $awaiting_analysis++;
648 print "<a name=\"summary\"</a>\n";
649 print "<h2>Summary of results from $num_fail failed builds:</h2>\n";
650 print "<p>$awaiting_analysis logs still need analysis</p>\n";
652 print " <li>Linked to $existing_bugs bugs in the Debian BTS</li>\n";
653 foreach my $type (sort keys %log_results) {
654 print " <h3>$err_descriptions[$type]</h3>\n";
655 my $tmp = $log_results{$type};
657 foreach my $key (sort { $result{$b} <=> $result{$a} }keys %result) {
658 print " <li>Found $result{$key} log(s) showing $key\n";
664 $time_taken = $time_end - $time_start;
665 $date = strftime "%a %b %e %H:%M:%S %Z %Y", localtime;
666 $hostname = `hostname`;
670 print "<p>Log analysis generated on $hostname, $date.\n";
671 print "<br>Output from $name - see <a href=\"$repo\">$repo</a> for source.\n";
672 print "<br>Read $lines_read lines of logs and took $time_taken seconds to complete.\n";