#! /usr/bin/perl # Script to analyze failed build logs. Look for specific regexps to # classify things use strict; use warnings; use Data::Dumper; my $logs = "/home/build/logs"; my $logs_fail = "$logs/FAIL"; my $num_fail = 0; my $existing_bugs = 0; use constant { ERR_ARCH_MISMATCH => 1, ERR_INFRA => 2, ERR_BD_PROBLEM => 3, ERR_DETECT_WRONG_ARCH => 4, ERR_CRASH => 5, ERR_BUILD_PROBLEM => 6, ERR_TEST_PROBLEM => 7, ERR_BUILD_TIMEOUT => 8, }; my @err_descriptions; $err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches"; $err_descriptions [ERR_INFRA] = "Infrastructure errors"; $err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps"; $err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture"; $err_descriptions [ERR_CRASH] = "Crashes detected"; $err_descriptions [ERR_BUILD_PROBLEM] = "Problems detected during build phase"; $err_descriptions [ERR_TEST_PROBLEM] = "Problems detected during test phase"; $err_descriptions [ERR_BUILD_TIMEOUT] = "Package builds timed out"; # Known failure modes to look for my @logcheck = ( { # "rchitecture mismatch" -> should never build on this arch # Stop working on this log at this point string => 'rchitecture mismatch', message => 'Architecture mismatch', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { string => 'not in arch list or does not match any', message => 'Architecture mismatch', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { # "binary build with no binary artifacts found" -> no packages # built. Why not picked up already above?. # Stop working on this log at this point string => 'binary build with no binary artifacts found', message => 'No binaries built', check => 1, stop => 1, type => ERR_ARCH_MISMATCH, }, { string => 'schroot.*File is not owned by user root', message => 'Schroot setup failure', check => 1, stop => 1, type => ERR_INFRA, }, { string => 'No space left on device', message => 'Build ran out of disk space', check => 1, stop => 1, type => ERR_INFRA, }, { string => 'aarch64-unknown-linux-gnu', message => 'Wrong arch detected', check => 1, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { string => 'binutils-aarch64', message => 'Wrong arch detected', check => 0, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { string => 'lib.linux-aarch64', message => 'Wrong arch detected', check => 1, stop => 1, type => ERR_DETECT_WRONG_ARCH, }, { # "Bus error" -> alignment bug string => 'Bus error', message => 'Alignment problem', check => 1, stop => 1, type => ERR_CRASH, }, { # "Segmentation fault" -> code problem string => 'Segmentation fault', pstring => 'Setting up (\S+)', message => 'Segmentation fault when installing RESULT', check => 1, stop => 1, type => ERR_CRASH, }, { # "Segmentation fault" -> code problem string => 'Segmentation fault', message => 'Segmentation fault', check => 1, stop => 1, type => ERR_CRASH, }, { # "Illegal instruction" -> bad build target? string => 'Illegal instruction', pstring => 'Setting up (\S+)', message => 'Illegal instruction when installing RESULT', check => 1, stop => 1, type => ERR_CRASH, }, { # "Illegal instruction" -> bad build target? string => 'Illegal instruction', message => 'Illegal instruction', check => 1, stop => 1, type => ERR_CRASH, }, { # Installing build-deps failed string => 'dpkg: error processing package (\S+)', message => 'Build-dep failed to install (RESULT)', check => 1, stop => 0, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'E: pbuilder-satisfydepends failed.', message => 'Pbuilder build-deps failed', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'E: Unmet dependencies', message => 'Build-deps failed', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Installing build-deps failed string => 'unsat-dependency: (\S+)', message => 'Missing build-dep (RESULT)', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Build failed - missing build-dep? string => 'build dependencies/conflicts unsatisfied', message => 'Build-deps not satisfiable', check => 1, stop => 1, type => ERR_BD_PROBLEM, }, { # Build failed - missing build-dep? string => 'ld: cannot find', message => 'Build failure: missing library - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - missing build-dep? string => 'fatal error:.*No such file or directory', message => 'Build failure: missing header - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - missing build-dep? string => 'SEVERE: Cannot resolve dependencies', message => 'Build failure - missing build-dep?', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed - can't exec something... string => 'error trying to exec.*execvp: No', message => 'Build failure (missing binary)', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'BUILD FAILED', message => 'Build failure (java/javadoc)', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'make.*returned exit code', message => 'Build failure (other)', check => 1, stop => 0, type => ERR_BUILD_PROBLEM, }, { # Build failed string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$', message => 'Build failure (other)', check => 1, stop => 0, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'dpkg-source: error: unrepresentable changes to source', message => 'dpkg-source failure', check => 1, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Build failed string => 'fakeroot debian/rules binary', message => 'Build failure (other)', check => 0, stop => 1, type => ERR_BUILD_PROBLEM, }, { # Test failure string => 'OSError: \[Errno 13\] Permission denied', message => 'Python EPERM test failure', check => 1, stop => 1, type => ERR_TEST_PROBLEM, }, { # Test failure string => 'dh_auto_test:.*returned exit code \d+', message => 'Test failure', check => 1, stop => 1, type => ERR_TEST_PROBLEM, }, { # Timeout. pbuilder is too dumb to do this properly :-( string => 'I: Terminating build process due to timeout', message => 'Pbuilder build timeout', check => 1, stop => 1, type => ERR_BUILD_TIMEOUT, }, { # Timeout from sbuild string => 'Build killed with signal TERM after \d+ minutes of inactivity', message => 'Sbuild build timeout', check => 1, stop => 1, type => ERR_BUILD_TIMEOUT, }, ); my %log_results; #foreach my $checktmp (@logcheck) { # my %check = %$checktmp; # print "looking for \"$check{string}\"\n"; # print " with log message \"$check{message}\"\n"; # print " check this regexp: $check{check}\n"; # print " stop if found: $check{stop}\n"; #} print "\n"; print "\n"; print "Build log analysis\n"; print "\n"; print "\n"; print "

Build log analysis

\n"; print "

Summary

\n"; print "

Packages

\n"; print "
    \n"; foreach my $input (@ARGV) { open (IN, "< $input") or die "Can't read $input: $!\n"; $num_fail++; my $stop = 0; my $lineno = 0; my $errors = 0; my $oldline = ""; my %file_results; print "
  1. Looking at $input:\n"; print "\n"; foreach my $key (keys %file_results) { $log_results{$file_results{$key}}{$key} += 1; # print "now have $log_results{$key} for \"$key\"\n"; } } print "
\n"; print "\n"; print "

Summary of results from $num_fail failed builds:

\n"; print "\n"; print "\n"; print "\n";