3 # |_) /| Copyright (C) 2002 | richard@
4 # | \/¯| Richard Atterer | atterer.org
6 # Copyright (C) 2016-2019 Steve McIntyre <93sam@debian.org>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License, version 2. See
9 # the file COPYING for details.
11 # Mirror script for Debian CD images, using Jigsaw Download. You first
12 # need to set up a conventional mirror (rsync/http/ftp-based) for the
13 # jigdo and template files, then this script can use those files and
14 # your local Debian mirror to create the full images automatically.
17 # This directory will be scanned for .jigdo files
18 jigdoDir="/home/ftp/debian/jigdo"
20 # For any file $jigdoDir/somedir/file.jigdo, an attempt will be made
21 # to create all the images offered by file.jigdo in $imageDir/somedir/
22 imageDir="/home/ftp/debian-cd"
24 # Temporary dir to use for creating images. Should be on the same
25 # partition as $imageDir, because mv is used to put finished images
27 tmpDir="/home/jigdo-mirror-tmpdir"
29 # Local Debian/Non-US mirrors. Can use http/ftp URLs, but beware that
30 # this may cause huge amounts of data to be downloaded repeatedly -
31 # the ftp/http server had better be on your LAN.
32 debianMirror="file:/home/ftp/debian"
33 nonusMirror="file:/home/ftp/debian/non-US"
35 # Where to put the logfile. If undefined, log output goes to stdout.
36 # If defined, stderr is also redirected to the logfile
37 #logfile="$tmpDir/jigdo-mirror-`date +%y%m%d`.log"
39 # Include and exclude certain files. These are two regular extended
40 # expressions, matched case-sensitively. The input given to them is
41 # the filename of the .iso files relative to $imageDir, in the form
42 # "somedir/image.iso", e.g. "3.0rev0/i386/woody-i386-1.iso". The
43 # filtering is equivalent to
44 # echo $name | egrep $include | egrep -v $exclude
45 # That is, first the list of files is restricted to those matching
46 # $include, then anything matching $exclude is removed.
47 include='.' # include all files,
48 exclude='$^' # then exclude none
50 # US sites: Exclude non-US stuff
52 # US sites: All i386 images, images 1 and 2 of the rest
53 #include='i386/|-[12]'; exclude='source/|_NONUS'
54 # Sites outside the US: All i386 images, images 1 and 2 of the rest
55 #include='i386/|-[12]'; exclude='source/|-1\.'
57 # How to call jigdo-file or jigdo-port.
58 # CAREFUL: Make sure that jigdo-cache.db is not publically accessible
59 # from the internet since it contains local path info.
60 jigdoFile="jigdo-file --cache=$tmpDir/jigdo-cache.db --cache-expiry=1w --report=noprogress --no-check-files"
61 #jigdoFile="jigdo-port"; havePMA=false
63 # Any files older than $maxAge days are deleted from $imageDir, except
64 # when the variable is unset; in that case, nothing happens. WARNING:
65 # This really means *any* files, not just files generated by
69 # In case only a few files are missing for the image to be complete,
70 # will download them from any fallback servers specified in the .jigdo
71 # file. Maximum number of missing files to download:
75 wgetOpts="--passive-ftp --no-directories --no-verbose"
77 # To find the template file, first the leafname of the template's URL
78 # is extracted from the .jigdo file. Next, for a .jigdo file named
79 # $jigdoDir/somedir/file.jigdo, the file
80 # $templateDir/somedir/leafnameFromURL is tried. If that isn't
81 # present, either the template is downloaded (if the URL is absolute)
82 # or looked for in $jigdoDir/somedir/templateURL (if the URL is
83 # relative). Default if unset is templateDir=$jigdoDir.
84 #templateDir="$jigdoDir"
86 # If it is inconvenient for you to set the variables above, you can
87 # either specify a config file with settings on the command line...
90 elif test -r ~/.jigdo-mirror; then
91 # ... or put the commands in "~/.jigdo-mirror"
94 #======================================================================
95 # No user-serviceable parts below
96 #======================================================================
99 # Download a file, storing it in the current dir
101 if test "$#" -eq 0; then return 0; fi
102 wget --user-agent="$userAgent" $wgetOpts "$@" || return 1
104 userAgent="jigdo-mirror/1.0 (`wget --version 2>/dev/null | (read ver; echo $ver)`)"
105 #______________________________________________________________________
108 # Returns 0 (true) if the supplied string is a HTTP/FTP URL, otherwise 1
111 http:*|ftp:*|HTTP:*|FTP:*|file:*|FILE:*) return 0;;
115 #______________________________________________________________________
118 rm -f "image" "image.tmp"
119 template=`basename "$templateURI"`
120 if test -f "$templateDir/$dirName/$template"; then
121 # Check for template in $templateDir
122 log " Found template \`\$templateDir/$dirName/$template'"
123 ln -s "$templateDir/$dirName/$template" "template"
124 elif isURI "$templateURI"; then
125 # Absolute template URL - download
126 log " Template \`\$templateDir/$dirName/$template' not found, will download"
127 if fetch "$templateURI" -O "template"; then true; else
128 log " Error getting template file"
130 rm -f "image" "template"
133 elif test -f "$jigdoDir/$dirName/$templateURI"; then
134 log " Found template \`\$jigdoDir/$dirName/$templateURI'"
135 ln -s "$jigdoDir/$dirName/$templateURI" "template"
137 log " Template file \`\$templateDir/$dirName/$template' not found"
138 log " Template file \`\$jigdoDir/$dirName/$templateURI' not found"
140 rm -f "image" "template"
144 # If possible, check md5sum of template data
145 if test "$templateMD5"; then
146 set -- `$jigdoFile md5sum --report=quiet "template"`
147 if test "$1" = "$templateMD5"; then
148 log " Template checksum is correct"
150 log " Error - template checksum mismatch"
152 rm -f "image" "template"
155 elif test "$templateSHA256"; then
156 set -- `$jigdoFile sha256sum --report=quiet "template"`
157 if test "$1" = "$templateSHA256"; then
158 log " Template checksum is correct"
160 log " Error - template checksum mismatch"
162 rm -f "image" "template"
166 log " [WARNING - \`Template-MD5Sum' and \`Template-SHA256Sum' missing from image section]"
169 # Try to merge any files into the image.
171 $jigdoFile print-missing-all $ijtOpts $uriOpts \
172 | egrep -v '^([a-zA-Z0-9.+_-]+:|$)' \
173 | $jigdoFile make-image $ijtOpts --files-from=-
176 $jigdoFile print-missing $ijtOpts $uriOpts \
177 | egrep -v '^([a-zA-Z0-9.+_-]+:|$)' \
178 | $jigdoFile make-image $ijtOpts --files-from=-
181 if test "$jigdoErr" -ge 2; then
182 log " Error merging data from local filesystem"
184 rm -f "image" "template"
188 # First try to download all files using the first URL in the
189 # print-missing-all list. If any files remain missing, add another
190 # pass, this time try to download the missing files using the 2nd
192 noMorePasses=$localMirror
193 for pass in x xx xxx xxxx xxxxx xxxxxx xxxxxxx xxxxxxxx; do
195 $jigdoFile print-missing-all $ijtOpts $jigdoOpts $uriOpts \
196 | egrep -i '^(http:|ftp:|$)' >"list"
198 # Quick hack until jigdo-port supports print-missing-all
199 $jigdoFile print-missing $ijtOpts $jigdoOpts $uriOpts \
200 | egrep -i '^(http:|ftp:|$)' \
201 | sed -n '/./{p;s/^.*$//;p;}' >"list"
203 missingCount=`egrep '^$' <"list" | wc -l | sed -e 's/ *//g'`
204 if test "$pass" = "x"; then
205 if $localMirror; then true; else missingCount=0; fi
207 if test "$missingCount" -gt "$maxMissing"; then
208 log " Too many files ($missingCount) missing in local mirror"
210 rm -f "list" "image" "template"
213 # Accumulate URLs in $@, pass them to fetchAndMerge in batches
218 if test "$url" = ""; then count=""; continue; fi
219 if test "$count" != "$pass"; then continue; fi
220 if $noMorePasses; then
221 log " $missingCount parts still missing from image"
225 if test "$#" -ge "$filesPerFetch"; then
226 if fetchAndMerge "$@"; then true; else
227 set --; noMorePasses=true
232 if test "$#" -ge 1; then
233 if fetchAndMerge "$@"; then true; else break; fi
235 if $noMorePasses; then break; fi
236 if test -r "image"; then break; fi
240 if test -r "image"; then
241 # Finished - verify checksum
242 if $jigdoFile verify $ijtOpts; then
243 log " Image checksum is correct, moving image into place"
244 mkdir -p "$imageDir/$dirName"
245 mv "image" "$imageDir/$dirName/$image"
247 log " Error - image checksum mismatch"
251 log " Image creation failed, list of missing files follows"
252 $jigdoFile print-missing $ijtOpts $jigdoOpts $uriOpts
255 rm -f "list" "image" "image.tmp" "template"
258 #______________________________________________________________________
260 # Given URLs, fetch them into $tmpDir, then merge them into image
262 (mkdir "$tmpDir/files"; cd "$tmpDir/files"; fetch "$@")
263 # Merge into the image
264 find "$tmpDir/files" -type f \
265 | $jigdoFile make-image $ijtOpts --no-cache --files-from=-
267 if test "$jigdoErr" -ge 2; then
271 # Delete tmpDir, to avoid taking up more space than necessary
272 rm -rf "$tmpDir/files"
274 #______________________________________________________________________
277 if test "$section" = "[Image]" -a "$image" \
278 -a "$templateURI"; then
279 log " Image \`$image', template \`$templateURI'"
280 set -- `echo "$dirName/$image" | egrep -- "$include" | egrep -v -- "$exclude"`
281 if test "$#" -eq 0; then
282 log " \`\$imageDir/$dirName/$image' excluded by \$include/\$exclude"
285 if test -f "$imageDir/$dirName/$image"; then
286 if test "$jigdoDir/$jigdo" -nt "$imageDir/$dirName/$image";then
287 log " jigdo is newer - updating \`\$imageDir/$dirName/$image'"
288 # Remove outdated image *immediately*, even in case
289 # the subsequent attempt to regenerate it fails
290 rm -f "$imageDir/$dirName/$image"
293 log " \`\$imageDir/$dirName/$image' is up to date"
294 test "$maxAge" && touch "$imageDir/$dirName/$image"
297 log " Attempting to create \`\$imageDir/$dirName/$image'"
302 #______________________________________________________________________
304 if test "$logfile"; then
309 log() { printf "%s: %s\n" "`date +'%Y-%m-%d %H:%M:%S'`" "$1"; }
310 #________________________________________
312 log "imageDir: $imageDir"
313 log "jigdoDir: $jigdoDir"
314 log "templateDir: $templateDir"
315 # Remove slashes from dir names
316 jigdoDir=${jigdoDir%/}
317 imageDir=${imageDir%/}
318 templateDir=${templateDir%/}
320 debianMirror=${debianMirror%/}
321 nonusMirror=${nonusMirror%/}
322 uriOpts="--uri Debian='$debianMirror/' --uri Non-US='$nonusMirror/'"
323 ijtOpts="--image=image --jigdo=jigdo --template=template"
324 # Is the main mirror on the local disc?
325 case "$debianMirror $nonusMirror" in
326 "file:"*" file:"*|/*" /"*) localMirror=true;;
327 *) localMirror=false;;
329 if test -z "$havePMA"; then havePMA=true; fi
330 if test -z "$templateDir"; then templateDir="$jigdoDir"; fi
332 mkdir -p "$tmpDir" || true
334 #________________________________________
336 find "$jigdoDir" -name "*.jigdo" \
337 | sed -e "sÿ^$jigdoDir/ÿÿ" \
338 | while read jigdo; do
340 log "Found \`\$jigdoDir/$jigdo'"
341 dirName=`dirname "$jigdo"`
342 if gzip -cd "$jigdoDir/$jigdo" >"jigdo" 2>"/dev/null"; then
344 elif test -f "$jigdoDir/$jigdo"; then
345 rm -f "jigdo" "template"
346 ln -s "$jigdoDir/$jigdo" "jigdo"
348 log " jigdo file not present/could not be unpacked - ignored"
353 # Parse jigdo file, look for images
356 set -- `echo "$REPLY" | sed -e 's/^ *\[ *\([^ ]*\) *\] *$/[\1]/; s/ *= */ /; s/['\''"$]//g'`
360 unset image templateURI templateMD5 templateSHA256 shortInfo info
362 Filename) image="$2";;
363 Template) templateURI="$2";;
364 Template-MD5Sum) templateMD5="$2";;
365 Template-SHA256Sum) templateSHA256="$2";;
366 ShortInfo) shift; shortInfo="$*";;
367 Info) shift; info="$*";;
375 #________________________________________
377 if cd "$imageDir"; then
378 if test "$maxAge"; then
379 log "Expiring images older than $maxAge days"
380 find . -type f -mtime +"$maxAge" \
381 | while read file; do
382 log " Deleting \`\$imageDir/${file#./}'"
385 # Remove empty directories
386 find "$imageDir" -depth -mindepth 1 -type d -empty \
390 #________________________________________