#! /usr/bin/perl
-
+#
+# analyze_results
+#
# Script to analyze failed build logs. Look for specific regexps to
# classify things
+#
+#
+# (c) 2018 Steve McIntyre <steve@einval.com> GPL v2+
use strict;
use warnings;
+use POSIX qw(strftime);
+use Data::Dumper;
-my $logs = "/home/build/logs";
-my $logs_fail = "$logs/FAIL";
+my $name = "analyze_results";
+my $repo = "https://git.einval.com/cgi-bin/gitweb.cgi?p=buildd-scripts.git";
+my $hostname;
+my $date;
+my $time_start = time();
+my $time_end;
+my $time_taken;
my $num_fail = 0;
+my $awaiting_analysis = 0;
+my $lines_read = 0;
+my $existing_bugs = 0;
+
+use constant {
+ ERR_ARCH_MISMATCH => 1,
+ ERR_NO_SOURCE => 2,
+ ERR_INFRA => 3,
+ ERR_BD_PROBLEM => 4,
+ ERR_DETECT_WRONG_ARCH => 5,
+ ERR_CRASH => 6,
+ ERR_BUILD_PROBLEM => 7,
+ ERR_TEST_PROBLEM => 8,
+ ERR_BUILD_TIMEOUT => 9,
+};
+
+my @err_descriptions;
+$err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches";
+$err_descriptions [ERR_NO_SOURCE] = "No source found";
+$err_descriptions [ERR_INFRA] = "Infrastructure errors";
+$err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps";
+$err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture";
+$err_descriptions [ERR_CRASH] = "Crashes detected";
+$err_descriptions [ERR_BUILD_PROBLEM] = "Problems detected during build phase";
+$err_descriptions [ERR_TEST_PROBLEM] = "Problems detected during test phase";
+$err_descriptions [ERR_BUILD_TIMEOUT] = "Package builds timed out";
# Known failure modes to look for
my @logcheck = (
{
- # Couldn't install build-deps
- string => '^Unable to resolve dependencies',
- pstring => '.*',
- message => 'build-deps failed',
+ # "rchitecture mismatch" -> should never build on this arch
+ # Stop working on this log at this point
+ string => 'rchitecture mismatch',
+ message => 'Architecture mismatch',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_ARCH_MISMATCH,
},
{
- # "rchitecture mismatch" -> should never build on this arch
+ string => 'not in arch list or does not match any',
+ message => 'Architecture mismatch',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_ARCH_MISMATCH,
+ },
+ {
+ # "binary build with no binary artifacts found" -> no packages
+ # built. Why not picked up already above?.
# Stop working on this log at this point
- string => 'rchitecture mismatch',
- pstring => '.*',
- message => 'architecture mismatch',
+ string => 'binary build with no binary artifacts found',
+ message => 'No binaries built',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_ARCH_MISMATCH,
},
{
- string => 'not in arch list or does not match any',
- pstring => '.*',
- message => 'architecture mismatch',
+ string => 'E: Can not find version \S+ of package',
+ message => 'Could not find specified source package',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_NO_SOURCE,
+ },
+ {
+ string => 'schroot.*File is not owned by user root',
+ message => 'Schroot setup failure',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_INFRA,
},
{
string => 'No space left on device',
- pstring => '.*',
- message => 'ran out of disk space',
+ message => 'Build ran out of disk space',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_INFRA,
},
{
- # "binary build with no binary artifacts found" -> no packages
- # built. Why not picked up already above?.
- # Stop working on this log at this point
- string => 'binary build with no binary artifacts found',
- pstring => '.*',
- message => 'no binaries built',
+ string => 'The system has no more ptys',
+ message => 'Build ran out of ptys',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_INFRA,
+ },
+ {
+ string => 'aarch64-unknown-linux-gnu',
+ message => 'Wrong arch detected',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_DETECT_WRONG_ARCH,
+ },
+ {
+ string => 'binutils-aarch64',
+ message => 'Wrong arch detected',
+ check => 0,
+ stop => 1,
+ analyze => 1,
+ type => ERR_DETECT_WRONG_ARCH,
+ },
+ {
+ string => 'lib.linux-aarch64',
+ message => 'Wrong arch detected',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_DETECT_WRONG_ARCH,
},
{
# "Bus error" -> alignment bug
string => 'Bus error',
- pstring => '.*',
- message => 'alignment problem',
+ message => 'Alignment problem',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_CRASH,
},
{
# "Segmentation fault" -> code problem
string => 'Segmentation fault',
- pstring => 'Setting up libglib2.0-cil',
- message => 'Segmentation fault (mono)',
+ pstring => 'Setting up (\S+)',
+ message => 'Segmentation fault when installing RESULT',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_CRASH,
},
{
# "Segmentation fault" -> code problem
string => 'Segmentation fault',
- pstring => '.*',
message => 'Segmentation fault',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_CRASH,
},
{
# "Illegal instruction" -> bad build target?
string => 'Illegal instruction',
- pstring => 'Setting up ghc',
- message => 'Illegal instruction (ghc)',
+ pstring => 'Setting up (\S+)',
+ message => 'Illegal instruction when installing RESULT',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_CRASH,
},
{
# "Illegal instruction" -> bad build target?
string => 'Illegal instruction',
- pstring => '.*',
message => 'Illegal instruction',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_CRASH,
},
{
# Installing build-deps failed
- string => 'dpkg: error processing package',
- pstring => '.*',
- message => 'build-deps failed',
+ string => 'dpkg: error processing package (\S+)',
+ message => 'Build-dep failed to install (RESULT)',
+ check => 1,
+ stop => 0,
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
+ },
+ {
+ # Installing build-deps failed
+ string => 'E: pbuilder-satisfydepends failed.',
+ message => 'Pbuilder build-deps failed',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
},
{
# Installing build-deps failed
string => 'E: Unmet dependencies',
- pstring => '.*',
- message => 'build-deps failed',
+ message => 'Build-deps failed',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
},
{
# Installing build-deps failed
- string => 'unsat-dependency:',
- pstring => '.*',
- message => 'build-deps failed',
+ string => 'unsat-dependency: (\S+)',
+ message => 'Missing build-dep (RESULT)',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
+ },
+ {
+ # Installing build-deps failed
+ string => 'unsat-conflict: (\S+)',
+ message => 'Unsatisfiable build-dep conflict (RESULT)',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
},
{
# Build failed - missing build-dep?
string => 'build dependencies/conflicts unsatisfied',
- pstring => '.*',
- message => 'build-deps not satisfiable',
+ message => 'Build-deps not satisfiable',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_BD_PROBLEM,
+ },
+ {
+ # Build failed
+ string => 'dpkg-source: error: unrepresentable changes to source',
+ message => 'dpkg-source failure',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed - missing build-dep?
string => 'ld: cannot find',
- pstring => '.*',
- message => 'build failure: missing library - missing build-dep?',
+ message => 'Build failure: missing library - missing build-dep?',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed - missing build-dep?
string => 'fatal error:.*No such file or directory',
- pstring => '.*',
- message => 'build failure: missing header - missing build-dep?',
+ message => 'Build failure: missing header - missing build-dep?',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed - missing build-dep?
string => 'SEVERE: Cannot resolve dependencies',
- pstring => '.*',
- message => 'build failure - missing build-dep?',
+ message => 'Build failure - missing build-dep?',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed - can't exec something...
string => 'error trying to exec.*execvp: No',
- pstring => '.*',
- message => 'build failure (missing binary)',
+ message => 'Build failure (missing binary)',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failed
+ string => 'BUILD FAIL',
+ message => 'Build failure (java/javadoc)',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed
string => 'make.*returned exit code',
- pstring => '.*',
- message => 'build failure (other)',
+ message => 'Build failure (other)',
check => 1,
stop => 0,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed
string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$',
- pstring => '.*',
- message => 'build failure (other)',
+ message => 'Build failure (other)',
check => 1,
- stop => 0,
- timeout => 0, # This is a real error, not a build timeout
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failure
+ string => 'dh_auto_build:.*returned exit code \d+',
+ message => 'Build failure (other)',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failure
+ string => 'dh_auto_clean:.*returned exit code \d+',
+ message => 'Build failure (clean failed)',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failure
+ string => 'dh_auto_install:.*returned exit code \d+',
+ message => 'Build failure (install failed)',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failure
+ string => 'dh_auto_configure:.*returned exit code \d+',
+ message => 'Build failure (configure failed)',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed
- string => 'dpkg-source: error: unrepresentable changes to source',
- pstring => '.*',
- message => 'dpkg-source failure',
+ string => '^debian/.*recipe for target (\S+) failed',
+ message => 'Build failure (RESULT)',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failed
+ string => '^debian/.*\*\*\* (.*). Stop.',
+ message => 'Build error (RESULT)',
check => 1,
stop => 1,
- timeout => 0, # This is a real error, not a build timeout
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failed
+ string => 'dpkg-buildpackage: error:.*(debian/rules \S+) subprocess returned exit status 2',
+ message => 'Build error (RESULT)',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
},
{
# Build failed
string => 'fakeroot debian/rules binary',
- pstring => '.*',
- message => 'build failure (other)',
+ message => 'Build failure (other)',
check => 0,
stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Test failure
+ string => 'OSError: \[Errno 13\] Permission denied',
+ message => 'Python EPERM test failure',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_TEST_PROBLEM,
+ },
+ {
+ # Test failure
+ string => 'dh_auto_test:.*returned exit code \d+',
+ message => 'Test failure',
+ check => 1,
+ stop => 1,
+ analyze => 1,
+ type => ERR_TEST_PROBLEM,
},
{
# Timeout. pbuilder is too dumb to do this properly :-(
string => 'I: Terminating build process due to timeout',
- pstring => '.*',
- message => 'build timeout',
+ message => 'Pbuilder build timeout',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_BUILD_TIMEOUT,
+ },
+ {
+ # Timeout from sbuild
+ string => 'Build killed with signal TERM after \d+ minutes of inactivity',
+ message => 'Sbuild build timeout',
check => 1,
stop => 1,
- timeout => 1, # This is a build timeout. If this happened
- # before other errors, then we should retry
+ analyze => 0,
+ type => ERR_BUILD_TIMEOUT,
},
);
print "</head>\n";
print "<body>\n";
print "<h1>Build log analysis</h1>\n";
+print "<p><a href=\"#summary\">Summary</a></p>\n";
print "<h2>Packages</h2>\n";
print "<ol>\n";
my $stop = 0;
my $lineno = 0;
my $errors = 0;
+ my $errors_need_analysis = 0;
my $oldline = "";
my %file_results;
print "<li>Looking at <a href=\"$input\">$input</a>:\n";
print "<ul>\n";
while (defined (my $line = <IN>) and !$stop) {
$lineno++;
+ $lines_read++;
foreach my $checktmp (@logcheck) {
my %check = %$checktmp;
if ($check{check}) {
if (!$stop
- and $line =~ m/$check{string}/
- and $oldline =~ m/$check{pstring}/) {
+ and ($line =~ m/$check{string}/)
+ and (!$check{pstring} or $oldline =~ m/$check{pstring}/)) {
# print " Line $lineno: found \"$check{string}\"\n";
- $file_results{$check{message}} = 1;
- if ($check{timeout} and $errors == 0) {
+ my $match = $1;
+ my $message = $check{message};
+ $message =~ s,RESULT,$match,g;
+ $file_results{$message} = $check{type};
+ if (($check{type} == ERR_BUILD_TIMEOUT) and ($errors == 0)) {
+ print " <li>Line $lineno: $message\n";
print " <li>Build killed by timeout before any errors at line $lineno\n";
} else {
- print " <li>Line $lineno: $check{message}\n";
+ print " <li>Line $lineno: $message\n";
$errors++;
}
+ if ($check{analyze}) {
+ $errors_need_analysis++;
+ }
if ($check{stop}) {
# print " stopping processing\n";
$stop = 1;
}
$oldline = $line;
}
+ close IN;
# End of checking this package
- print " <li><strong>found errors: $errors</strong>\n";
+ print " <li><strong>Found errors: $errors</strong>\n";
if (!$errors) {
print " (maybe just timed out during build?)\n";
}
+
+ # Look for a note for manually-added logfile analysis
+ my $note = $input;
+ $note =~ s,\.log$,.note,;
+ if (-f $note) {
+ open (IN, "< $note") or die "Can't open $note for reading: $!\n";
+ while (defined (my $line = <IN>)) {
+ chomp $line;
+ if ($line =~ m,#(\d+),) {
+ my $bugno = $1;
+ $existing_bugs++;
+ $line =~ s,#(\d+),<a href="https://bugs.debian.org/$1">#$1</a>,g;
+ }
+ print " <li>$line\n";
+ }
+ close IN;
+ } else {
+ if ($errors_need_analysis) {
+ print " <li><strong>Needs analysis</strong>\n";
+ }
+ }
+
print "</ul>\n";
foreach my $key (keys %file_results) {
- $log_results{$key} += 1;
+ $log_results{$file_results{$key}}{$key} += 1;
# print "now have $log_results{$key} for \"$key\"\n";
}
+ if ($errors_need_analysis) {
+ $awaiting_analysis++;
+ }
}
print "</ol>\n";
+print "<a name=\"summary\"</a>\n";
print "<h2>Summary of results from $num_fail failed builds:</h2>\n";
-print "<ol>\n";
-foreach my $key (keys %log_results) {
- print " <li>Found $log_results{$key} logs showing $key\n";
+print "<p>$awaiting_analysis logs still need analysis</p>\n";
+print "<ul>\n";
+print " <li>Found $existing_bugs existing bugs in the Debian BTS</li>\n";
+foreach my $type (sort keys %log_results) {
+ print " <h3>$err_descriptions[$type]</h3>\n";
+ my $tmp = $log_results{$type};
+ my %result = %$tmp;
+ foreach my $key (sort { $result{$b} <=> $result{$a} }keys %result) {
+ print " <li>Found $result{$key} log(s) showing $key\n";
+ }
}
-print "</ol>\n";
+print "</ul>\n";
+
+$time_end = time();
+$time_taken = $time_end - $time_start;
+$date = strftime "%a %b %e %H:%M:%S %Z %Y", localtime;
+$hostname = `hostname`;
+chomp $hostname;
+
+print "<hr>\n";
+print "<p>Log analysis generated on $hostname, $date.\n";
+print "<br>Output from $name - see <a href=\"$repo\">$repo</a> for source.\n";
+print "<br>Read $lines_read lines of logs and took $time_taken seconds to complete.\n";
+
print "</body>\n";
print "</html>\n";