--- /dev/null
+#!/usr/bin/perl -w
+
+use strict;
+use Fcntl qw(:flock SEEK_END);
+use File::stat;
+use File::Find;
+use File::Basename;
+use Compress::Zlib;
+use POSIX qw(ENOENT EROFS ENOSYS EEXIST EPERM EBUSY O_RDONLY O_RDWR O_APPEND O_CREAT);
+use Fcntl qw(O_RDWR O_WRONLY);
+use DB_File;
+
+my $dbdir = "/home/debian-cd/search-db";
+my $lock = "$dbdir/.update.lock";
+my @areas = qw(daily-builds release weekly-builds archive);
+#my @areas = qw(weekly-builds);
+my $update_needed = 0;
+my $num_list_files;
+my $db_mtime = 0;
+my $db_file_name;
+my $list_file_name;
+my $list_file;
+my $verbose = 0;
+
+while (1) {
+ my $arg = shift;
+ if (!defined($arg)) {
+ last;
+ }
+ if ("-v" eq $arg) {
+ $verbose++;
+ }
+}
+
+sub print_log {
+ my $level = shift;
+ my $msg = shift;
+ if ($level <= $verbose) {
+ print $msg;
+ }
+}
+
+sub lock {
+ my ($fh) = @_;
+ flock($fh, LOCK_EX) or die "Cannot lock lockfile - $!\n";
+
+ # and, in case someone appended while we were waiting...
+ seek($fh, 0, SEEK_END) or die "Cannot seek - $!\n";
+}
+
+sub unlock {
+ my ($fh) = @_;
+ flock($fh, LOCK_UN) or die "Cannot unlock lockfile - $!\n";
+}
+
+sub file_mtime {
+ my ($file) = shift;
+ my $sb = lstat($file);
+ if (! -e $file) {
+ print "ENOENT $file!\n";
+ }
+ return $sb->mtime;
+}
+
+sub check_newer {
+ my ($filename);
+ $filename = $File::Find::name;
+ if ($filename =~ m/\.list\.gz$/) {
+ my $mtime = file_mtime("/mnt/nfs-cdimage/$filename");
+ print_log(4, " check_newer: found $filename\n");
+ print LISTS "$filename\n";
+ if ($mtime > $db_mtime) {
+ $update_needed = 1;
+ }
+ $num_list_files++;
+ if (!($num_list_files % 1000)) {
+ print_log(3, " check_newer: found $num_list_files list files\n");
+ }
+ }
+}
+
+chdir "/mnt/nfs-cdimage";
+
+open(my $lockfile, ">>", "$lock") or die "Can't open lockfile: $!";
+print_log(1, "waiting on lock for $lock\n");
+lock($lockfile);
+print_log(1, "lock acquired for $lock\n");
+
+foreach my $area (@areas) {
+ $update_needed = 0;
+ $num_list_files = 0;
+ $db_file_name = "$dbdir/$area.db";
+ $list_file_name = "$dbdir/$area.lists";
+
+ print_log(1, "Working on area $area:\n");
+ unlink "$list_file_name.new";
+ open(LISTS, ">> $list_file_name.new") or die ("Can't open lists file $list_file_name.new for writing: $!\n");
+ if (-f $db_file_name) {
+ $db_mtime = file_mtime($db_file_name);
+ } else {
+ $db_mtime = 0;
+ }
+ find (\&check_newer, "$area");
+ close LISTS;
+ print_log(2, " found $num_list_files list files total, update_needed $update_needed\n");
+
+ if ($update_needed) {
+ my $current_list_num = 0;
+ my $num_files = 0;
+ my $current_file = 0;
+ my $unique_files = 0;
+ my %fileinfo;
+ my %dbinfo;
+
+ # Two passes; work in memory first, then push to the DB
+ # file. Will this work better?
+ undef %fileinfo;
+ undef %dbinfo;
+
+ unlink "$db_file_name.new";
+
+ open(LISTS, "< $list_file_name.new") or die ("Can't open lists file $list_file_name.new for reading: $!\n");
+ while (my $listfile = <LISTS>) {
+ $current_list_num++;
+ chomp $listfile;
+ my $gz = gzopen($listfile, "rb") or die "Cannot open $listfile: $gzerrno\n";
+ my $file;
+ while ($gz->gzreadline($file) > 0) {
+ chomp $file;
+ $num_files++;
+ if (defined($fileinfo{$file})) {
+ $fileinfo{$file} = "$fileinfo{$file} $listfile";
+ } else {
+ $fileinfo{$file} = "$listfile";
+ $unique_files++;
+ }
+ }
+ $gz->gzclose();
+ if (!($current_list_num % 100)) {
+ print_log(3, " processing $area in memory: $current_list_num/$num_list_files list files done, $num_files files ($unique_files unique)\n");
+ }
+ }
+
+ # now push to the db
+ tie %dbinfo, 'DB_File', "$db_file_name.new";
+ foreach my $file (keys %fileinfo) {
+ $dbinfo{$file} = $fileinfo{$file};
+ $current_file++;
+ if (!($current_file % 10000)) {
+ print_log(3, " storing $area to db: $current_file/$unique_files files added\n");
+ }
+ }
+ untie %dbinfo;
+
+ rename("$db_file_name.new", "$db_file_name");
+ print_log(2, " $db_file_name created: $num_list_files list files, $num_files files referenced\n");
+ }
+ rename("$list_file_name.new", "$list_file_name");
+}
+print_log(1, "dropping lock for $lock\n");
+unlock($lockfile);
+close($lockfile);