use warnings;
use POSIX qw(strftime);
use Data::Dumper;
+use Getopt::Std;
+use HTTP::Tiny;
+use URI::Encode qw(uri_encode uri_decode);
my $name = "analyze_results";
my $repo = "https://git.einval.com/cgi-bin/gitweb.cgi?p=buildd-scripts.git";
my $time_end;
my $time_taken;
my $num_fail = 0;
+my $awaiting_analysis = 0;
my $lines_read = 0;
my $existing_bugs = 0;
+my $buildd_base_url="https://buildd.debian.org/status/package.php?p=";
use constant {
ERR_ARCH_MISMATCH => 1,
- ERR_INFRA => 2,
- ERR_BD_PROBLEM => 3,
- ERR_DETECT_WRONG_ARCH => 4,
- ERR_CRASH => 5,
- ERR_BUILD_PROBLEM => 6,
- ERR_TEST_PROBLEM => 7,
- ERR_BUILD_TIMEOUT => 8,
+ ERR_NO_SOURCE => 2,
+ ERR_INFRA => 3,
+ ERR_BD_PROBLEM => 4,
+ ERR_DETECT_WRONG_ARCH => 5,
+ ERR_CRASH => 6,
+ ERR_BUILD_PROBLEM => 7,
+ ERR_TEST_PROBLEM => 8,
+ ERR_BUILD_TIMEOUT => 9,
};
my @err_descriptions;
$err_descriptions [ERR_ARCH_MISMATCH] = "Architecture mismatches";
+$err_descriptions [ERR_NO_SOURCE] = "No source found";
$err_descriptions [ERR_INFRA] = "Infrastructure errors";
$err_descriptions [ERR_BD_PROBLEM] = "Problems with build-deps";
$err_descriptions [ERR_DETECT_WRONG_ARCH] = "Builds detected wrong architecture";
message => 'Architecture mismatch',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_ARCH_MISMATCH,
},
{
message => 'Architecture mismatch',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_ARCH_MISMATCH,
},
{
message => 'No binaries built',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_ARCH_MISMATCH,
},
+ {
+ string => 'E: Can not find version \S+ of package',
+ message => 'Could not find specified source package',
+ check => 1,
+ stop => 1,
+ analyze => 0,
+ type => ERR_NO_SOURCE,
+ },
{
string => 'schroot.*File is not owned by user root',
message => 'Schroot setup failure',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_INFRA,
},
{
message => 'Build ran out of disk space',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_INFRA,
},
{
message => 'Build ran out of ptys',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_INFRA,
},
{
message => 'Wrong arch detected',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_DETECT_WRONG_ARCH,
},
{
message => 'Wrong arch detected',
check => 0,
stop => 1,
+ analyze => 1,
type => ERR_DETECT_WRONG_ARCH,
},
{
message => 'Wrong arch detected',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_DETECT_WRONG_ARCH,
},
{
message => 'Alignment problem',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_CRASH,
},
{
message => 'Segmentation fault when installing RESULT',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_CRASH,
},
{
message => 'Segmentation fault',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_CRASH,
},
{
message => 'Illegal instruction when installing RESULT',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_CRASH,
},
{
message => 'Illegal instruction',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_CRASH,
},
{
message => 'Build-dep failed to install (RESULT)',
check => 1,
stop => 0,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'Pbuilder build-deps failed',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'Build-deps failed',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'Missing build-dep (RESULT)',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'Unsatisfiable build-dep conflict (RESULT)',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'Build-deps not satisfiable',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BD_PROBLEM,
},
{
message => 'dpkg-source failure',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure: missing library - missing build-dep?',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure: missing header - missing build-dep?',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure - missing build-dep?',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (missing binary)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (java/javadoc)',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
# Build failed
- string => 'make.*returned exit code',
+ string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$',
message => 'Build failure (other)',
check => 1,
- stop => 0,
+ stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
# Build failed
- string => '^make.*\*\*\*.* \[debian/rules.*Error \d+$',
+ string => 'make.*returned exit code',
message => 'Build failure (other)',
check => 1,
- stop => 1,
+ stop => 0,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (other)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (clean failed)',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (install failed)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (configure failed)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (RESULT)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build error (RESULT)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
# Build failed
- string => 'dpkg-buildpackage: error: (debian/rules \S+) subprocess returned exit status 2',
+ string => 'dpkg-buildpackage: error:.*(debian/rules \S+) subprocess returned exit status 2',
message => 'Build error (RESULT)',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Build failure (other)',
check => 0,
stop => 1,
+ analyze => 1,
+ type => ERR_BUILD_PROBLEM,
+ },
+ {
+ # Build failure
+ string => 'constant \d+ overflows.*int',
+ message => 'Go 32-bit integer overflow',
+ check => 1,
+ stop => 0,
+ analyze => 1,
type => ERR_BUILD_PROBLEM,
},
{
message => 'Python EPERM test failure',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_TEST_PROBLEM,
},
{
message => 'Test failure',
check => 1,
stop => 1,
+ analyze => 1,
type => ERR_TEST_PROBLEM,
},
{
message => 'Pbuilder build timeout',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_TIMEOUT,
},
{
message => 'Sbuild build timeout',
check => 1,
stop => 1,
+ analyze => 0,
type => ERR_BUILD_TIMEOUT,
},
);
my %log_results;
+my %udd;
#foreach my $checktmp (@logcheck) {
# my %check = %$checktmp;
print "<h2>Packages</h2>\n";
print "<ol>\n";
+my $line;
+
+our($opt_u);
+getopts('u:') or die "getopts failure\n";
+
+if (defined $opt_u) {
+ open (UDD, "< $opt_u") or die "Can't open UDD list file $opt_u for reading\n";
+ while (defined ($line = <UDD>)) {
+ chomp $line;
+ my ($pkg, $bugno, $bugsubj) = split (/ /, $line, 3);
+ my @buglist;
+ my $tmp = $udd{"$pkg"};
+ if ($tmp) {
+ @buglist = @$tmp;
+ }
+ push (@buglist, "$bugno: $bugsubj");
+ $udd{"$pkg"} = \@buglist;
+ }
+ close UDD;
+
+# foreach my $key (keys %udd) {
+# my $tmp = $udd{$key};
+# my $num = scalar (@$tmp);
+# print "$key has $num bugs:\n";
+# foreach my $entry (@$tmp) {
+# print " $entry\n";
+# }
+# }
+}
+
+sub check_buildd_status {
+ my $pkg = shift;
+ my $version = shift;
+ my $arch = shift;
+
+ my $url = "$buildd_base_url$pkg";
+
+# print "Looking at URL $url\n";
+
+ my $built_on_arches = "";
+ my $built_my_arch = 0;
+ my $ftbfs_my_arch = 0;
+ my $not_my_arch = 0;
+ my $current_version = "";
+
+# print "Checking buildd data for package $pkg, arch $arch\n";
+
+ my $response = HTTP::Tiny->new->get($url);
+
+ if ($response->{success}) {
+# print "$response->{status} $response->{reason}\n";
+ my $content = $response->{content};
+# print "$content\n";
+
+ # Split the content up into lines
+ my @lines = split("\n", $content);
+
+ # Now look for two things:
+ # 1. how many arches have built this package?
+ # 2. has *our* arch built it?
+# print "we tried $version on arch $arch\n";
+ foreach my $line (@lines) {
+ chomp $line;
+ $line = uri_decode($line);
+ if ($line =~ /https:\/\/buildd.debian.org\/stats\/graph/) {
+ my @info = split (/<\/td>/, $line);
+ # Grab the info from our fields
+ my $this_arch = "";
+ if ($info[0] =~ /architecture.php\?a=([^&]*)&/) {
+ $this_arch = $1;
+# print "Found entry for arch $this_arch\n";
+ } else {
+ print " <li>FAILED TO PARSE $url\n";
+ }
+ $current_version = $info[1];
+ $current_version =~ s/^<td>//;
+# print "current_version for $this_arch is $current_version, we tried $version\n";
+ my $status = "";
+ if ($info[2] =~ /class="status status-([a-z]*)/) {
+ $status = $1;
+# print "Found status $status\n";
+ } else {
+ print " <li>FAILED TO PARSE $url\n";
+ }
+# foreach my $field(@info) {
+# print "$field\n";
+# }
+# print " On $this_arch we have version $current_version, status $status\n";
+ if ($arch =~ /^\Q$this_arch\E$/) {
+ if ($status =~ /installed/) {
+ $built_my_arch++;
+ } else {
+ $ftbfs_my_arch++;
+ }
+ }
+ if ($status =~ /installed/) {
+ $built_on_arches .= "$this_arch ";
+ }
+ }
+
+ if ($line =~ /$arch is not present in the architecture list set by the maintainer/) {
+ $not_my_arch++;
+ }
+ if ($line =~ /No entry in $arch database/) {
+ $not_my_arch++;
+ }
+ }
+ print " <li>Built on this arch: $built_my_arch\n";
+ print " <li>Built on arches: $built_on_arches\n";
+ if ($not_my_arch) {
+ print " <li>Not set to build on $arch\n";
+ }
+ if ($ftbfs_my_arch) {
+ print " <li>Already seen to FTBFS on $arch\n";
+ }
+ if (length($current_version) < 2) {
+ print " <li>No current version found - has this been removed\n";
+ } elsif (! ($current_version =~ /\Q$version\E/)) {
+ print " <li><strong>New version $current_version available - retry?</strong>\n";
+ }
+ } else {
+ print " <LI>FAILED TO RETRIEVE buildd URL $url\n";
+ }
+}
+
+
foreach my $input (@ARGV) {
open (IN, "< $input") or die "Can't read $input: $!\n";
$num_fail++;
my $stop = 0;
my $lineno = 0;
my $errors = 0;
+ my $errors_need_analysis = 0;
my $oldline = "";
my %file_results;
- print "<li>Looking at <a href=\"$input\">$input</a>:\n";
+ my $pav;
+ my $pkg;
+ my $arch;
+ my $version;
+
+ $pav = $input;
+ $pav =~ s,^.*/,,g;
+ $pav =~ s,\.log,,g;
+ ($pkg, $version, $arch) = split (/_/, $pav);
+
+ print "<li>Looking at <a href=\"$input\">$pkg version $version on arch $arch</a>:\n";
print "<ul>\n";
- while (defined (my $line = <IN>) and !$stop) {
+ while (defined ($line = <IN>) and !$stop) {
$lineno++;
$lines_read++;
foreach my $checktmp (@logcheck) {
print " <li>Line $lineno: $message\n";
$errors++;
}
+ if ($check{analyze}) {
+ $errors_need_analysis++;
+ }
if ($check{stop}) {
# print " stopping processing\n";
$stop = 1;
print " <li>$line\n";
}
close IN;
+ } else {
+ if ($errors_need_analysis) {
+ print " <li><strong>Needs analysis</strong>\n";
+ if ($udd{$pkg}) {
+ my $tmp = $udd{$pkg};
+ my $num = scalar (@$tmp);
+ print " <li>$num reported FTBFS bugs:\n";
+ print " <ul>\n";
+ foreach my $entry (@$tmp) {
+ print " <li>#$entry\n";
+ }
+ print " </ul>\n";
+ }
+ check_buildd_status($pkg, $version, $arch);
+ }
}
print "</ul>\n";
$log_results{$file_results{$key}}{$key} += 1;
# print "now have $log_results{$key} for \"$key\"\n";
}
+ if ($errors_need_analysis) {
+ $awaiting_analysis++;
+ }
}
print "</ol>\n";
print "<a name=\"summary\"</a>\n";
print "<h2>Summary of results from $num_fail failed builds:</h2>\n";
+print "<p>$awaiting_analysis logs still need analysis</p>\n";
print "<ul>\n";
-print "<li> Found $existing_bugs existing bugs in the Debian BTS</li>\n";
+print " <li>Linked to $existing_bugs bugs in the Debian BTS</li>\n";
foreach my $type (sort keys %log_results) {
print " <h3>$err_descriptions[$type]</h3>\n";
my $tmp = $log_results{$type};