5 # Script to analyze failed build logs. Look for specific regexps to
9 # (c) 2018 Steve McIntyre <steve@einval.com> GPL v2+
13 use POSIX qw(strftime);
17 my $name = "analyze_results";
18 my $repo = "https://git.einval.com/cgi-bin/gitweb.cgi?p=buildd-scripts.git";
21 my $time_start = time();
25 my $awaiting_analysis = 0;
27 my $existing_bugs = 0;
30 ERR_ARCH_MISMATCH => 1,
34 ERR_DETECT_WRONG_ARCH => 5,
36 ERR_BUILD_PROBLEM => 7,
37 ERR_TEST_PROBLEM => 8,
38 ERR_BUILD_TIMEOUT => 9,
42 $err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches";
43 $err_descriptions [ERR_NO_SOURCE] = "No source found";
44 $err_descriptions [ERR_INFRA] = "Infrastructure errors";
45 $err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps";
46 $err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture";
47 $err_descriptions [ERR_CRASH] = "Crashes detected";
48 $err_descriptions [ERR_BUILD_PROBLEM] = "Problems detected during build phase";
49 $err_descriptions [ERR_TEST_PROBLEM] = "Problems detected during test phase";
50 $err_descriptions [ERR_BUILD_TIMEOUT] = "Package builds timed out";
52 # Known failure modes to look for
55 # "rchitecture mismatch" -> should never build on this arch
56 # Stop working on this log at this point
57 string => 'rchitecture mismatch',
58 message => 'Architecture mismatch',
62 type => ERR_ARCH_MISMATCH,
65 string => 'not in arch list or does not match any',
66 message => 'Architecture mismatch',
70 type => ERR_ARCH_MISMATCH,
73 # "binary build with no binary artifacts found" -> no packages
74 # built. Why not picked up already above?.
75 # Stop working on this log at this point
76 string => 'binary build with no binary artifacts found',
77 message => 'No binaries built',
81 type => ERR_ARCH_MISMATCH,
84 string => 'E: Can not find version \S+ of package',
85 message => 'Could not find specified source package',
89 type => ERR_NO_SOURCE,
92 string => 'schroot.*File is not owned by user root',
93 message => 'Schroot setup failure',
100 string => 'No space left on device',
101 message => 'Build ran out of disk space',
108 string => 'The system has no more ptys',
109 message => 'Build ran out of ptys',
116 string => 'aarch64-unknown-linux-gnu',
117 message => 'Wrong arch detected',
121 type => ERR_DETECT_WRONG_ARCH,
124 string => 'binutils-aarch64',
125 message => 'Wrong arch detected',
129 type => ERR_DETECT_WRONG_ARCH,
132 string => 'lib.linux-aarch64',
133 message => 'Wrong arch detected',
137 type => ERR_DETECT_WRONG_ARCH,
140 # "Bus error" -> alignment bug
141 string => 'Bus error',
142 message => 'Alignment problem',
149 # "Segmentation fault" -> code problem
150 string => 'Segmentation fault',
151 pstring => 'Setting up (\S+)',
152 message => 'Segmentation fault when installing RESULT',
159 # "Segmentation fault" -> code problem
160 string => 'Segmentation fault',
161 message => 'Segmentation fault',
168 # "Illegal instruction" -> bad build target?
169 string => 'Illegal instruction',
170 pstring => 'Setting up (\S+)',
171 message => 'Illegal instruction when installing RESULT',
178 # "Illegal instruction" -> bad build target?
179 string => 'Illegal instruction',
180 message => 'Illegal instruction',
187 # Installing build-deps failed
188 string => 'dpkg: error processing package (\S+)',
189 message => 'Build-dep failed to install (RESULT)',
193 type => ERR_BD_PROBLEM,
196 # Installing build-deps failed
197 string => 'E: pbuilder-satisfydepends failed.',
198 message => 'Pbuilder build-deps failed',
202 type => ERR_BD_PROBLEM,
205 # Installing build-deps failed
206 string => 'E: Unmet dependencies',
207 message => 'Build-deps failed',
211 type => ERR_BD_PROBLEM,
214 # Installing build-deps failed
215 string => 'unsat-dependency: (\S+)',
216 message => 'Missing build-dep (RESULT)',
220 type => ERR_BD_PROBLEM,
223 # Installing build-deps failed
224 string => 'unsat-conflict: (\S+)',
225 message => 'Unsatisfiable build-dep conflict (RESULT)',
229 type => ERR_BD_PROBLEM,
232 # Build failed - missing build-dep?
233 string => 'build dependencies/conflicts unsatisfied',
234 message => 'Build-deps not satisfiable',
238 type => ERR_BD_PROBLEM,
242 string => 'dpkg-source: error: unrepresentable changes to source',
243 message => 'dpkg-source failure',
247 type => ERR_BUILD_PROBLEM,
250 # Build failed - missing build-dep?
251 string => 'ld: cannot find',
252 message => 'Build failure: missing library - missing build-dep?',
256 type => ERR_BUILD_PROBLEM,
259 # Build failed - missing build-dep?
260 string => 'fatal error:.*No such file or directory',
261 message => 'Build failure: missing header - missing build-dep?',
265 type => ERR_BUILD_PROBLEM,
268 # Build failed - missing build-dep?
269 string => 'SEVERE: Cannot resolve dependencies',
270 message => 'Build failure - missing build-dep?',
274 type => ERR_BUILD_PROBLEM,
277 # Build failed - can't exec something...
278 string => 'error trying to exec.*execvp: No',
279 message => 'Build failure (missing binary)',
283 type => ERR_BUILD_PROBLEM,
287 string => 'BUILD FAIL',
288 message => 'Build failure (java/javadoc)',
292 type => ERR_BUILD_PROBLEM,
296 string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$',
297 message => 'Build failure (other)',
301 type => ERR_BUILD_PROBLEM,
305 string => 'make.*returned exit code',
306 message => 'Build failure (other)',
310 type => ERR_BUILD_PROBLEM,
314 string => 'dh_auto_build:.*returned exit code \d+',
315 message => 'Build failure (other)',
319 type => ERR_BUILD_PROBLEM,
323 string => 'dh_auto_clean:.*returned exit code \d+',
324 message => 'Build failure (clean failed)',
328 type => ERR_BUILD_PROBLEM,
332 string => 'dh_auto_install:.*returned exit code \d+',
333 message => 'Build failure (install failed)',
337 type => ERR_BUILD_PROBLEM,
341 string => 'dh_auto_configure:.*returned exit code \d+',
342 message => 'Build failure (configure failed)',
346 type => ERR_BUILD_PROBLEM,
350 string => '^debian/.*recipe for target (\S+) failed',
351 message => 'Build failure (RESULT)',
355 type => ERR_BUILD_PROBLEM,
359 string => '^debian/.*\*\*\* (.*). Stop.',
360 message => 'Build error (RESULT)',
364 type => ERR_BUILD_PROBLEM,
368 string => 'dpkg-buildpackage: error:.*(debian/rules \S+) subprocess returned exit status 2',
369 message => 'Build error (RESULT)',
373 type => ERR_BUILD_PROBLEM,
377 string => 'fakeroot debian/rules binary',
378 message => 'Build failure (other)',
382 type => ERR_BUILD_PROBLEM,
386 string => 'OSError: \[Errno 13\] Permission denied',
387 message => 'Python EPERM test failure',
391 type => ERR_TEST_PROBLEM,
395 string => 'dh_auto_test:.*returned exit code \d+',
396 message => 'Test failure',
400 type => ERR_TEST_PROBLEM,
403 # Timeout. pbuilder is too dumb to do this properly :-(
404 string => 'I: Terminating build process due to timeout',
405 message => 'Pbuilder build timeout',
409 type => ERR_BUILD_TIMEOUT,
412 # Timeout from sbuild
413 string => 'Build killed with signal TERM after \d+ minutes of inactivity',
414 message => 'Sbuild build timeout',
418 type => ERR_BUILD_TIMEOUT,
425 #foreach my $checktmp (@logcheck) {
426 # my %check = %$checktmp;
427 # print "looking for \"$check{string}\"\n";
428 # print " with log message \"$check{message}\"\n";
429 # print " check this regexp: $check{check}\n";
430 # print " stop if found: $check{stop}\n";
435 print "<title>Build log analysis</title>\n";
438 print "<h1>Build log analysis</h1>\n";
439 print "<p><a href=\"#summary\">Summary</a></p>\n";
440 print "<h2>Packages</h2>\n";
446 getopts('u:') or die "getopts failure\n";
448 if (defined $opt_u) {
449 open (UDD, "< $opt_u") or die "Can't open UDD list file $opt_u for reading\n";
450 while (defined ($line = <UDD>)) {
452 my ($pkg, $bugno, $bugsubj) = split (/ /, $line, 3);
454 my $tmp = $udd{"$pkg"};
458 push (@buglist, "$bugno: $bugsubj");
459 $udd{"$pkg"} = \@buglist;
463 # foreach my $key (keys %udd) {
464 # my $tmp = $udd{$key};
465 # my $num = scalar (@$tmp);
466 # print "$key has $num bugs:\n";
467 # foreach my $entry (@$tmp) {
473 foreach my $input (@ARGV) {
474 open (IN, "< $input") or die "Can't read $input: $!\n";
479 my $errors_need_analysis = 0;
490 ($pkg, $version, $arch) = split (/_/, $pav);
492 print "<li>Looking at <a href=\"$input\">$pkg version $version on arch $arch</a>:\n";
494 while (defined ($line = <IN>) and !$stop) {
497 foreach my $checktmp (@logcheck) {
498 my %check = %$checktmp;
501 and ($line =~ m/$check{string}/)
502 and (!$check{pstring} or $oldline =~ m/$check{pstring}/)) {
503 # print " Line $lineno: found \"$check{string}\"\n";
505 my $message = $check{message};
506 $message =~ s,RESULT,$match,g;
507 $file_results{$message} = $check{type};
508 if (($check{type} == ERR_BUILD_TIMEOUT) and ($errors == 0)) {
509 print " <li>Line $lineno: $message\n";
510 print " <li>Build killed by timeout before any errors at line $lineno\n";
512 print " <li>Line $lineno: $message\n";
515 if ($check{analyze}) {
516 $errors_need_analysis++;
519 # print " stopping processing\n";
528 # End of checking this package
529 print " <li><strong>Found errors: $errors</strong>\n";
531 print " (maybe just timed out during build?)\n";
534 # Look for a note for manually-added logfile analysis
536 $note =~ s,\.log$,.note,;
538 open (IN, "< $note") or die "Can't open $note for reading: $!\n";
539 while (defined (my $line = <IN>)) {
541 if ($line =~ m,#(\d+),) {
544 $line =~ s,#(\d+),<a href="https://bugs.debian.org/$1">#$1</a>,g;
546 print " <li>$line\n";
550 if ($errors_need_analysis) {
551 print " <li><strong>Needs analysis</strong>\n";
553 my $tmp = $udd{$pkg};
554 my $num = scalar (@$tmp);
555 print " <li>$num reported FTBFS bugs:\n";
557 foreach my $entry (@$tmp) {
558 print " <li>#$entry\n";
566 foreach my $key (keys %file_results) {
567 $log_results{$file_results{$key}}{$key} += 1;
568 # print "now have $log_results{$key} for \"$key\"\n";
570 if ($errors_need_analysis) {
571 $awaiting_analysis++;
576 print "<a name=\"summary\"</a>\n";
577 print "<h2>Summary of results from $num_fail failed builds:</h2>\n";
578 print "<p>$awaiting_analysis logs still need analysis</p>\n";
580 print " <li>Found $existing_bugs existing bugs in the Debian BTS</li>\n";
581 foreach my $type (sort keys %log_results) {
582 print " <h3>$err_descriptions[$type]</h3>\n";
583 my $tmp = $log_results{$type};
585 foreach my $key (sort { $result{$b} <=> $result{$a} }keys %result) {
586 print " <li>Found $result{$key} log(s) showing $key\n";
592 $time_taken = $time_end - $time_start;
593 $date = strftime "%a %b %e %H:%M:%S %Z %Y", localtime;
594 $hostname = `hostname`;
598 print "<p>Log analysis generated on $hostname, $date.\n";
599 print "<br>Output from $name - see <a href=\"$repo\">$repo</a> for source.\n";
600 print "<br>Read $lines_read lines of logs and took $time_taken seconds to complete.\n";