#! /usr/bin/perl # # analyze_results # # Script to analyze failed build logs. Look for specific regexps to # classify things # # # (c) 2018 Steve McIntyre GPL v2+ use strict; use warnings; use POSIX qw(strftime); use Data::Dumper; my $name = "analyze_results"; my $repo = "https://git.einval.com/cgi-bin/gitweb.cgi?p=buildd-scripts.git"; my $hostname; my $date; my $time_start = time(); my $time_end; my $time_taken; my $num_fail = 0; my $lines_read = 0; my $existing_bugs = 0; use constant { ERR_ARCH_MISMATCH => 1, ERR_INFRA => 2, ERR_BD_PROBLEM => 3, ERR_DETECT_WRONG_ARCH => 4, ERR_CRASH => 5, ERR_BUILD_PROBLEM => 6, ERR_TEST_PROBLEM => 7, ERR_BUILD_TIMEOUT => 8, }; my @err_descriptions; $err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches"; $err_descriptions [ERR_INFRA] = "Infrastructure errors"; $err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps"; $err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture"; $err_descriptions [ERR_CRASH] = "Crashes detected"; $err_descriptions [ERR_BUILD_PROBLEM] = "Problems detected during build phase"; $err_descriptions [ERR_TEST_PROBLEM] = "Problems detected during test phase"; $err_descriptions [ERR_BUILD_TIMEOUT] = "Package builds timed out"; # Known failure modes to look for my @logcheck = ( { # "rchitecture mismatch" -> should never build on this arch # Stop working on this log at this point string => 'rchitecture mismatch', message => 'Architecture mismatch', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { string => 'not in arch list or does not match any', message => 'Architecture mismatch', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { # "binary build with no binary artifacts found" -> no packages # built. Why not picked up already above?. # Stop working on this log at this point string => 'binary build with no binary artifacts found', message => 'No binaries built', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { string => 'schroot.*File is not owned by user root', message => 'Schroot setup failure', check => 1, stop => 1, type => ERR_INFRA, }, { string => 'No space left on device', message => 'Build ran out of disk space', check => 1, stop => 1, type => ERR_INFRA, }, { string => 'aarch64-unknown-linux-gnu', message => 'Wrong arch detected', check => 1, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { string => 'binutils-aarch64', message => 'Wrong arch detected', check => 0, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { string => 'lib.linux-aarch64', message => 'Wrong arch detected', check => 1, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { # "Bus error" -> alignment bug string => 'Bus error', message => 'Alignment problem', check => 1, stop => 1, type => ERR_CRASH, }, { # "Segmentation fault" -> code problem string => 'Segmentation fault', pstring => 'Setting up (\S+)', message => 'Segmentation fault when installing RESULT', check => 1, stop => 1, type => ERR_CRASH, }, { # "Segmentation fault" -> code problem string => 'Segmentation fault', message => 'Segmentation fault', check => 1, stop => 1, type => ERR_CRASH, }, { # "Illegal instruction" -> bad build target? string => 'Illegal instruction', pstring => 'Setting up (\S+)', message => 'Illegal instruction when installing RESULT', check => 1, stop => 1, type => ERR_CRASH, }, { # "Illegal instruction" -> bad build target? string => 'Illegal instruction', message => 'Illegal instruction', check => 1, stop => 1, type => ERR_CRASH, }, { # Installing build-deps failed string => 'dpkg: error processing package (\S+)', message => 'Build-dep failed to install (RESULT)', check => 1, stop => 0, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'E: pbuilder-satisfydepends failed.', message => 'Pbuilder build-deps failed', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'E: Unmet dependencies', message => 'Build-deps failed', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'unsat-dependency: (\S+)', message => 'Missing build-dep (RESULT)', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Build failed - missing build-dep? string => 'build dependencies/conflicts unsatisfied', message => 'Build-deps not satisfiable', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Build failed - missing build-dep? string => 'ld: cannot find', message => 'Build failure: missing library - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - missing build-dep? string => 'fatal error:.*No such file or directory', message => 'Build failure: missing header - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - missing build-dep? string => 'SEVERE: Cannot resolve dependencies', message => 'Build failure - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - can't exec something... string => 'error trying to exec.*execvp: No', message => 'Build failure (missing binary)', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'BUILD FAILED', message => 'Build failure (java/javadoc)', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'make.*returned exit code', message => 'Build failure (other)', check => 1, stop => 0, type => ERR_BUILD_PROBLEM, }, { # Build failed string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$', message => 'Build failure (other)', check => 1, stop => 0, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'dpkg-source: error: unrepresentable changes to source', message => 'dpkg-source failure', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'fakeroot debian/rules binary', message => 'Build failure (other)', check => 0, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Test failure string => 'OSError: \[Errno 13\] Permission denied', message => 'Python EPERM test failure', check => 1, stop => 1, type => ERR_TEST_PROBLEM, }, { # Test failure string => 'dh_auto_test:.*returned exit code \d+', message => 'Test failure', check => 1, stop => 1, type => ERR_TEST_PROBLEM, }, { # Timeout. pbuilder is too dumb to do this properly :-( string => 'I: Terminating build process due to timeout', message => 'Pbuilder build timeout', check => 1, stop => 1, type => ERR_BUILD_TIMEOUT, }, { # Timeout from sbuild string => 'Build killed with signal TERM after \d+ minutes of inactivity', message => 'Sbuild build timeout', check => 1, stop => 1, type => ERR_BUILD_TIMEOUT, }, ); my %log_results; #foreach my $checktmp (@logcheck) { # my %check = %$checktmp; # print "looking for \"$check{string}\"\n"; # print " with log message \"$check{message}\"\n"; # print " check this regexp: $check{check}\n"; # print " stop if found: $check{stop}\n"; #} print "\n"; print "\n"; print "Build log analysis\n"; print "\n"; print "\n"; print "

Build log analysis

\n"; print "

Summary

\n"; print "

Packages

\n"; print "
    \n"; foreach my $input (@ARGV) { open (IN, "< $input") or die "Can't read $input: $!\n"; $num_fail++; my $stop = 0; my $lineno = 0; my $errors = 0; my $oldline = ""; my %file_results; print "
  1. Looking at $input:\n"; print "
      \n"; while (defined (my $line = ) and !$stop) { $lineno++; $lines_read++; foreach my $checktmp (@logcheck) { my %check = %$checktmp; if ($check{check}) { if (!$stop and ($line =~ m/$check{string}/) and (!$check{pstring} or $oldline =~ m/$check{pstring}/)) { # print " Line $lineno: found \"$check{string}\"\n"; my $match = $1; my $message = $check{message}; $message =~ s,RESULT,$match,g; $file_results{$message} = $check{type}; if ($check{type} == ERR_BUILD_TIMEOUT and $errors == 0) { print "
    • Line $lineno: $message\n"; print "
    • Build killed by timeout before any errors at line $lineno\n"; } else { print "
    • Line $lineno: $message\n"; $errors++; } if ($check{stop}) { # print " stopping processing\n"; $stop = 1; } } } } $oldline = $line; } close IN; # End of checking this package print "
    • Found errors: $errors\n"; if (!$errors) { print " (maybe just timed out during build?)\n"; } # Look for a note for manually-added logfile analysis my $note = $input; $note =~ s,\.log$,.note,; if (-f $note) { open (IN, "< $note") or die "Can't open $note for reading: $!\n"; while (defined (my $line = )) { chomp $line; if ($line =~ m,#(\d+),) { my $bugno = $1; $existing_bugs++; $line =~ s,#(\d+),#$1,g; } print "
    • $line\n"; } close IN; } print "
    \n"; foreach my $key (keys %file_results) { $log_results{$file_results{$key}}{$key} += 1; # print "now have $log_results{$key} for \"$key\"\n"; } } print "
\n"; print "\n"; print "

Summary of results from $num_fail failed builds:

\n"; print "\n"; $time_end = time(); $time_taken = $time_end - $time_start; $date = strftime "%a %b %e %H:%M:%S %Z %Y", localtime; $hostname = `hostname`; print "
\n"; print "

Log analysis generated on $hostname, $date.\n"; print "
Output from $name - see
$repo for source.\n"; print "
Read $lines_read lines of logs and took $time_taken seconds to complete.\n"; print "\n"; print "\n";