-#!/usr/bin/perl -w
+#!/usr/bin/perl
#
# find_file.cgi
#
# Look through a provided database to find which CD/DVD image(s)
# contain a specified Debian package or source file.
#
-# Copyright (c) 2011 Steve McIntyre <93sam@debian.org>
+# Copyright (c) 2011-2017 Steve McIntyre <93sam@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
use strict;
+use warnings;
+use threads;
use DB_File;
use CGI;
use ConfigReader::Simple;
+use DBI;
+use DBD::SQLite;
my %conf;
my $cdimage_url = "http://cdimage.debian.org/cdimage/";
-my $source_url = "http://git.einval.com/cgi-bin/gitweb.cgi?p=debian-cd-search.git";
+my $source_url = "https://git.einval.com/cgi-bin/gitweb.cgi?p=debian-cd-search.git";
my @AREAS;
my %num_files;
my %fileinfo;
my %imageinfo;
my $query_term;
my $query_type;
-my @results;
+my %results;
my @chosen_areas;
my $l = "";
my $max_count = 1000;
my $footer2 = "";
my $last_update = 0;
-my $version = "0.8";
+my $version = "0.11";
my $title_base = "Debian CD search engine";
my $q = new CGI;
$conf{'dbdir'} = "/home/steve/debian/debian-cd/search/search-db";
$conf{'htmldir'} = "/home/steve/debian/debian-cd/html";
$conf{'debug'} = 0;
+ $conf{'dbtype'} = "sqlite";
}
# If we can find an appropriately-name config file, read it and
print $q->start_html(
-title=>"$title",
-author=>"$authormail",
- -style=>{'src'=>'http://www.debian.org/debian.css'},
+ -style=>{'src'=>'https://www.debian.org/debian.css'},
);
print $header1;
print '<p id="breadcrumbs">cdimage-search.debian.org</p>';
return $name;
}
+sub keepalive_thread () {
+ $| = 1;
+ $SIG{'KILL'} = sub { threads->exit(); };
+ while(1) {
+ sleep 10;
+ print "<!-- processing, please wait... -->\n";
+ }
+}
+
set_default_config();
read_config();
read_files("en");
chdir($conf{'dbdir'}) || log_error(500, "Failed to cd to $conf{'dbdir'}: $!\n");
opendir(my $dh, ".") || log_error(500, "Failed to open $conf{'dbdir'}: $!\n");
+
+
while (defined($_ = readdir($dh))) {
- if (m/(.*)\.db$/) {
- my $mtime = file_mtime("$1.db");
- if ($mtime > $last_update) {
- $last_update = $mtime;
- }
- push(@AREAS, "$1");
+ if ($conf{'dbtype'} eq "hashdb") {
+ if (m/(.*)\.db$/) {
+ my $mtime = file_mtime("$1.db");
+ if ($mtime > $last_update) {
+ $last_update = $mtime;
+ }
+ push(@AREAS, "$1");
+ }
+ }
+ else {
+ if (m/(.*)\.db.sqlite$/) {
+ my $mtime = file_mtime("$1.db.sqlite");
+ if ($mtime > $last_update) {
+ $last_update = $mtime;
+ }
+ push(@AREAS, "$1");
+ }
}
}
closedir($dh);
blank_form("");
}
+if (defined($query_term)) {
+ if ($query_term =~ m/[\@\~\[\]\{\}|#\%\<\>\'\";\\\/]/) {
+ $q->param(-name=>'query', -value=>'');
+ $q->param(-name=>'type', -value=>'');
+ $q->param(-name=>'search_area', -value=>());
+ blank_form("Invalid query string");
+ }
+}
+
+if (defined($query_type)) {
+ if (!($query_type eq "exact" or $query_type eq "simple")) {
+ $q->param(-name=>'query', -value=>'');
+ $q->param(-name=>'type', -value=>'');
+ $q->param(-name=>'search_area', -value=>());
+ blank_form("Invalid query string");
+ }
+}
+
if (!(@chosen_areas) && defined($query_term)) {
blank_form("No search areas chosen");
}
}
my $count = 0;
+my $count_images = 0;
my $re_search;
my $using_glob = "";
if ($query_type eq "simple") {
}
# If we get here, we have stuff to work with. Yay!
-foreach my $area (@chosen_areas) {
- my $db_file_name = "$conf{'dbdir'}/$area.db";
- $l .= "Looking in area $area, file $db_file_name<br>\n";
- dbmopen(%fileinfo, "$db_file_name", 0000) ||
- log_error(500, "Failed to open db file: $!\n");
-
- if ($query_term =~ /[\*\?]/ || $query_type eq "simple") {
- $using_glob = "(using globs)";
- # Will need to search through all the keys to allow for glob
- foreach my $file (keys %fileinfo) {
- if ($file =~ $re_search) {
- $count++;
- push(@results, "$file $fileinfo{$file}");
- if ($count >= $max_count) {
- last;
- }
- }
- }
- } else {
- # We've been given an exact name - do the exact key lookup \o/
- if (defined($fileinfo{$query_term})) {
- push (@results, "$query_term $fileinfo{$query_term}");
- $count++;
- }
+
+select STDERR; $| = 1; # make unbuffered
+select STDOUT; $| = 1; # make unbuffered
+
+my $start_time = time();
+print_header();
+print_html_header("$title_base results");
+
+# Now start the keepalive thread to print something every few seconds
+my $thr = threads->create(\&keepalive_thread);
+
+if ($conf{'dbtype'} eq "hashdb") {
+ foreach my $area (@chosen_areas) {
+ my $db_file_name = "$conf{'dbdir'}/$area.db";
+ $l .= "Looking in area $area, file $db_file_name<br>\n";
+ dbmopen(%fileinfo, "$db_file_name", 0000) ||
+ log_error(500, "Failed to open db file: $!\n");
+
+ if ($query_term =~ /[\*\?]/ || $query_type eq "simple") {
+ $using_glob = "(using globs)";
+ # Will need to search through all the keys to allow for glob
+ foreach my $file (keys %fileinfo) {
+ if ($file =~ $re_search) {
+ $count++;
+ $count_images += scalar (split / /, $fileinfo{$file});
+ $results{$file} = $fileinfo{$file};
+ if ($count >= $max_count) {
+ last;
+ }
+ }
+ }
+ } else {
+ # We've been given an exact name - do the exact key lookup \o/
+ if (defined($fileinfo{$query_term})) {
+ $results{$query_term} = $fileinfo{$query_term};
+ $count_images += scalar (split / /, $fileinfo{$query_term});
+ $count++;
+ }
+ }
+ if ($count >= $max_count) {
+ last;
+ }
+ dbmclose %fileinfo;
}
- if ($count >= $max_count) {
- last;
+} else {
+ foreach my $area (@chosen_areas) {
+ my $sth;
+ my @db_results;
+ my $db_file_name = "$conf{'dbdir'}/$area.db.sqlite";
+ my $dbh = DBI->connect("dbi:SQLite:dbname=$db_file_name","","", {
+ sqlite_open_flags => DBD::SQLite::OPEN_READONLY,
+ }) or log_error (500, "Failed to open DB file $db_file_name: $!\n");
+ $dbh->do("PRAGMA synchronous = OFF");
+
+ if ($query_term =~ /[\*\?]/ || $query_type eq "simple") {
+ $using_glob = "(using globs)";
+
+ # Will need to use sql LIKE and SQL wildcards
+ my $sql_term = "*" . $query_term . "*";
+ $sql_term =~ s,\*,\%,g;
+ $sql_term =~ s,\?,\_,g;
+ $sth = $dbh->prepare("SELECT * FROM entries WHERE filename LIKE ? ORDER BY filename ASC, jigdo ASC");
+ $sth->execute($sql_term);
+ } else {
+ # We've been given an exact name - do the exact lookup
+ $sth = $dbh->prepare("SELECT * FROM entries WHERE filename=? ORDER BY filename ASC");
+ $sth->execute($query_term);
+ }
+ while (@db_results = $sth->fetchrow_array) {
+ my $file = $db_results[0];
+ my $image = $db_results[1];
+ if (defined($results{$file})) {
+ $results{$file} = "$results{$file} $image";
+ $count_images++;
+ } else {
+ $results{$file} = "$image";
+ $count++;
+ $count_images++;
+ }
+ if ($count >= $max_count) {
+ last;
+ }
+ }
+ $dbh->disconnect();
}
- dbmclose %fileinfo;
}
-print_header();
-print_html_header("$title_base: $count results");
+# Kill the keepalive thread
+$thr->kill('KILL')->detach();
+
+my $end_time = time();
+my $time_taken = $end_time - $start_time;
+
print
- $q->start_html("$title_base: $count results"),
+ $q->start_html("$title_base: $count results from $count_images images"),
$q->h1($title_base), "\n";
print_config_if_debug();
if ($conf{'debug'}) {
print $q->li("query type: $query_type");
print $q->li("query term: \"$query_term\" $using_glob");
print $q->li("re_search: \"$re_search\"");
+ print $q->li("time taken: $time_taken sec\n");
print "</ul>\n";
}
print $q->p("<a href=\"" . $q->url . "\">Search again.</a>");
if ($count >= $max_count) {
print $q->p("More than $max_count results for $query_type search \"$query_term\". Showing the first $count only\n");
} else {
- print $q->p("$count result(s) for $query_type search \"$query_term\".\n");
+ print $q->p("$count result(s), $count_images image(s) for \"$query_term\":\n");
}
if ($count > 0) {
print "<ol>\n";
- foreach my $result (sort (@results)) {
- my($found, @list) = split(' ', $result);
+ foreach my $found (sort (keys %results)) {
+ my @list = split(' ', $results{$found});
print "<li> $found appears in:\n";
print "<ul>\n";
foreach my $image (sort(@list)) {