5 # (c) 2004-2011 Steve McIntyre <steve@einval.com>
7 # Server-side wrapper; run this on a machine with a mirror to set up
8 # the snapshots for jigit / jigdo downloading
10 # GPL v2 - see COPYING
12 # This script can be run in two modes:
14 # 1. To build a jigit .conf file for a single jigdo file:
15 # add the "-n" option with a CD name on the command line
16 # and only specify a single jigdo to work with using "-j".
18 # 2. To build a snapshot tree for (potentially multiple) jigdo files:
19 # do *not* specify the "-n" option, and list as many jigdo files as
20 # desired, either on the command line using multiple "-j <jigdo>" options
21 # or (better) via a file listing them with the "-J" option.
24 # (single-jigdo mode only) the CD name of the jigit
25 # (single-jigdo mode only) the output location; where the jigdo, template
26 # file and snapshot will be written
27 # (single-jigdo mode only) the locations of the input jigdo and template
29 # the location of the mirror
30 # the keyword(s) to look for (e.g. Debian)
31 # the snapshot dirname (e.g. today's date)
33 # Example #1: (single-jigdo mode, used for Ubuntu jigit generation)
35 # mkjigsnap -o /tmp/mjs-test -n mjs-test -m /tmp/mirror \
36 # -j ~/jigdo/update/debian-update-3.0r2.01-i386.jigdo \
37 # -t ~/jigdo/update/debian-update-3.0r2.01-i386.template \
41 # (This creates a single jigit conf file using the supplied jigdo/template
42 # file pair, looking for jigdo references to files in the "Debian" and
43 # "Non-US" areas. Output the files into /tmp/mjs-test and call them
44 # "mjs-test.<ext>", creating a snapshot of the needed files in
45 # /tmp/mjs-test/20041017 by linking files from /tmp/mirror as needed.)
47 # Example #2: (multi-jigdo mode, as run to keep
48 # http://us.cdimage.debian.org/cdimage/snapshot/ up to date)
50 # mkjigsnap -m /org/ftp/debian -J ~/jigdo.list -T ~/tmp \
52 # -d /org/jigdo-area/snapshot/Debian \
53 # -f ~/mkjigsnap-failed.log \
54 # -i ~/mkjigsnap-ignore.list
56 # (This reads in all the jigdo files listed in ~/jigdo.list, building a
57 # list of all the files referenced in the "Debian" area. It will then
58 # attempt to build a snapshot tree of all those files under
59 # /org/jigdo-area/snapshot/Debian by linking from /org/ftp/debian. Any
60 # files that are missing will be listed into the output "missing" file
61 # ~/mkjigsnap-failed.log for later checking, UNLESS they are already listed
62 # in the "ignore" file ~/mkjigsnap-ignore.list.)
102 KEYWORDS="$KEYWORDS $1"
127 TF_TMPDIR="--directory $1"
141 echo "Input error! ($1)"
150 generate_snapshot_tree () {
158 (echo $NUM; echo $MIRROR; echo $DIRNAME; echo $FAILEDFILE; echo $IGNOREFILE; cat $TMPFILE ) | perl -we '
161 my $dryrun = $ENV{"DRYRUN"};
162 my $verbose = $ENV{"VERBOSE"};
189 print "mkdirs($input)\n";
192 @components = split /\//,$input;
193 foreach $component (@components) {
195 $dir = join ("/", $dir, $component);
203 print "DRYRUN: not making directory tree $input\n";
208 sub delete_redundant {
212 $ref = $file_list{$File::Find::name};
213 if (!defined($ref)) {
215 print "delete_redundant($File::Find::name)\n";
218 unlink($File::Find::name);
220 print "DRYRUN: not deleting $File::Find::name\n";
223 if ( !($old_deleted % 1000) ) {
224 print "$old_deleted\n";
230 sub parse_ignore_file {
231 my $inputfile = shift;
232 my $num_ignored_loaded = 0;
233 open(INLIST, "$inputfile") or return;
234 while (defined (my $pkg = <INLIST>)) {
236 $ignored_fails{$pkg}++;
237 $num_ignored_loaded++;
239 print "parse_ignore_file: loaded $num_ignored_loaded entries from file $inputfile\n";
245 if (!defined($num)) {
249 if (!defined($mirrorpath)) {
253 if (!defined($outdir)) {
255 print "Linking $num files from $mirrorpath to $outdir\n";
259 print "$outdir does not exist\n";
263 die "DRYRUN: not making it, so aborting\n";
266 if (!defined($failedlog)) {
270 if (!defined($ignorefile)) {
272 print "ignorefile is \"$ignorefile\"\n";
273 if (length($ignorefile) > 2) {
274 parse_ignore_file($ignorefile);
279 $outfile = $outdir . "/" . $_;
280 $file_list{$outfile}++;
282 $dirname = dirname($_);
283 $filename = basename($_);
284 mkdirs($outdir . "/" . $dirname);
287 foreach $mirror (split /:/,$mirrorpath) {
288 $infile = $mirror . "/" . $_;
290 $link = readlink($infile);
291 if ($link =~ m#^/#) {
294 $infile = dirname($infile) . "/" . $link;
298 print "look for $_:\n";
300 $outfile = $outdir . "/" . $_;
303 print " try $infile\n";
305 if (link ($infile, $outfile)) {
310 print "DRYRUN: not linking $infile to $outfile\n";
314 $infile = $mirror . "/" . $filename;
316 print " fallback: try $infile\n";
319 if (link ($infile, $outfile)) {
324 print "DRYRUN: not linking $infile to $outfile\n";
330 if ($ignored_fails{$_}) {
333 if (length($failedlog) <= 2) {
334 # No logfile, print to stdout then
335 print "\nFailed to create link $outfile\n";
338 push (@failed_files, $_);
341 if ($ignored_fails{$_}) {
342 print "\n$_ marked as failed, but we found it anyway!\n";
347 if ( !($done % 10000) ) {
348 print "$done done, ignored $ignored, failed $failed out of $num\n";
351 print " Finished: $done/$num, $failed failed, ignored $ignored\n\n";
353 if ((length($failedlog) > 2) && ($failed > 0)) {
354 print "Writing list of failed files to $failedlog\n";
355 open(FAIL_LOG, "> $failedlog") or die "Failed to open $failedlog: $!\n";
356 foreach my $missing (@failed_files) {
357 print FAIL_LOG "$missing\n";
362 # Now walk the tree and delete files that we no longer need
363 print "Scanning for now-redundant files\n";
364 find(\&delete_redundant, $outdir);
365 print " Finished: $old_deleted old files removed\n";
369 # Sanity-check arguments
370 if [ "$DIRNAME"x = ""x ] ; then
371 echo "You must specify the snapshot directory name!"
374 if [ "$KEYWORDS"x = ""x ] ; then
375 echo "You must specify the keywords to match!"
378 if [ "$MIRROR"x = ""x ] ; then
379 echo "You must specify the location of the mirror!"
383 if [ "$JIGDOS"x != ""x ] ; then
384 NUM_JIGDOS=$(($NUM_JIGDOS+`echo $JIGDOS | wc -w`))
386 if [ "$JIGDOLIST"x != ""x ] ; then
387 NUM_JIGDOS=$(($NUM_JIGDOS+`cat $JIGDOLIST | wc -w`))
389 if [ "$NUM_JIGDOS" -eq 0 ] ; then
390 echo "No jigdo file(s) specified!"
393 if [ $MODE = "single" ] ; then
394 if [ "$CDNAME"x = ""x ] ; then
395 echo "You must specify the output name for the jigit conf!"
398 if [ "$OUT"x = ""x ] ; then
399 echo "You must specify where to set up the snapshot!"
402 if [ "$TEMPLATE"x = ""x ] ; then
403 echo "You must specify the template file!"
406 if [ "$NUM_JIGDOS" -ne 1 ] ; then
407 echo "More than one jigdo file specified ($NUM_JIGDOS) in single-jigdo mode!"
410 # In single-jigdo mode, the snapshot directory is relative to the
412 DIRNAME="$OUT"/"$DIRNAME"
414 if [ "$CDNAME"x != ""x ] ; then
415 echo "Output name is meaningless for multi-jigdo mode!"
418 if [ "$OUT"x != ""x ] ; then
419 echo "Output dir is meaningless for multi-jigdo mode!"
422 if [ "$TEMPLATE"x != ""x ] ; then
423 echo "Template file name is meaningless for multi-jigdo mode!"
428 # Build the snapshot(s)
429 for KEYWORD in $KEYWORDS; do
430 echo "Looking for keyword $KEYWORD in $NUM_JIGDOS jigdo file(s):"
431 TMPFILE=`tempfile $TF_TMPDIR`
432 if [ "$JIGDOLIST"x != ""x ] ; then
433 cat $JIGDOLIST | xargs zcat -f | sed -n "s/^.*${KEYWORD}://gp" >> $TMPFILE
435 if [ "$JIGDOS"x != ""x ] ; then
436 zcat -f $JIGDOS | sed -n "s/^.*${KEYWORD}://gp" >> $TMPFILE
438 LISTDONEDATE=`date -u`
439 TMPFILE1=`tempfile $TF_TMPDIR`
440 TOTAL_FILES=`wc -l < $TMPFILE`
441 echo " $LISTDONEDATE: Total references for \"$KEYWORD\": $TOTAL_FILES"
443 # sort is prone to running out of space, so bail if it fails. We
444 # don't want to destroy an existing snapshot if we end up with an
445 # empty list of files!
446 sort -u $SORT_TMPDIR $TMPFILE > $TMPFILE1
447 SORTDONEDATE=`date -u`
448 mv -f $TMPFILE1 $TMPFILE
449 NUM_FILES=`wc -l < $TMPFILE`
450 echo " $SORTDONEDATE: Unique references for \"$KEYWORD\": $NUM_FILES"
452 if [ $NUM_FILES -lt 5 ] ; then
453 echo " Only $NUM_FILES for the snapshot? Something is wrong; abort!"
457 echo "Creating snapshot tree in $DIRNAME:"
458 generate_snapshot_tree "$NUM_FILES" "$MIRROR" "$DIRNAME" "$FAILEDFILE" "$IGNOREFILE" "$TMPFILE"
459 SNAPDONEDATE=`date -u`
461 echo "$STARTDATE: startup"
462 echo "$LISTDONEDATE: $TOTAL_FILES files found"
463 echo "$SORTDONEDATE: $NUM_FILES unique files after sorting"
464 echo "$SNAPDONEDATE: snapshot done"
467 if [ $MODE = "single" ] ; then
468 if [ "$DRYRUN" = "0" ] ; then
469 zcat -f $JIGDOS | sed "s:^Template=.*$:Template=$CDNAME.template:" | gzip -9 > $OUT/$CDNAME.jigdo
470 cp $TEMPLATE $OUT/$CDNAME.template
471 echo "JIGDO=$CDNAME.jigdo" > $OUT/$CDNAME.conf
472 echo "TEMPLATE=$CDNAME.template" >> $OUT/$CDNAME.conf
473 echo "SNAPSHOT=snapshot/$DIRNAME" >> $OUT/$CDNAME.conf
474 echo "Jigdo files, config and snapshot made in $OUT"
476 echo "DRYRUN: Not creating files in $OUT"