Changelog for Jigsaw Download -*- Text -*-
------------------------------------------------------------------------
+jigdo 0.8.0 -- Steve McIntyre, XXXXX
+
+ - Roll up lots of older fixes from Debian packaging
+ - Fixed lots of warnings thrown by newer compilers
+ - Extensive changes throughout to add support for using SHA256 as
+ well as / instead of MD5 checksums
+
jigdo 0.7.3 -- Richard Atterer, 19 May 2006
- A maintenance release with some bug fixes
net/proxyguess-test@exe@ \
util/autonullptr-test@exe@ util/rsyncsum-test@exe@ \
util/gunzip-test@exe@ util/log-test@exe@ \
- util/md5sum-test@exe@ util/mimestream-test@exe@ \
+ util/md5sum-test@exe@ util/sha256sum-test@exe@ util/mimestream-test@exe@ \
util/string-utf-test@exe@
# net/uri-test@exe@ needs curl
job/makeimagedl.o job/single-url.o \
job/url-mapping.o net/download.o net/uri.o net/proxyguess.o \
util/bstream.o util/configfile.o util/glibc-getopt.o \
- util/glibc-getopt1.o util/glibc-md5.o util/gunzip.o \
- util/log.o util/md5sum.o util/progress.o util/string-utf.o \
+ util/glibc-getopt1.o util/glibc-md5.o util/glibc-sha256.o util/gunzip.o \
+ util/log.o util/md5sum.o util/sha256sum.o util/progress.o util/string-utf.o \
$(windows-res) \
util/debug.o # this must come last!
#^ net/glibwww-callbacks.o net/glibwww-init.o
jigdoconfig.o mkimage.o mkjigdo.o mktemplate.o \
partialmatch.o recursedir.o scan.o util/bstream.o \
util/configfile.o util/glibc-getopt.o util/glibc-getopt1.o \
- util/glibc-md5.o util/log.o util/md5sum.o util/rsyncsum.o \
+ util/glibc-md5.o util/glibc-sha256.o util/log.o util/md5sum.o \
+ util/sha256sum.o util/rsyncsum.o \
util/string.o zstream.o zstream-bz.o zstream-gz.o \
util/debug.o # this must come last!
objects-torture = cachefile.o compat.o jigdoconfig.o mkimage.o mkjigdo.o \
mktemplate.o partialmatch.o recursedir.o scan.o torture.o \
- util/bstream.o util/configfile.o util/glibc-md5.o \
- util/log.o util/md5sum.o util/rsyncsum.o util/string.o \
+ util/bstream.o util/configfile.o util/glibc-md5.o util/glibc-sha256.o \
+ util/log.o util/md5sum.o util/sha256sum.o util/rsyncsum.o util/string.o \
zstream.o zstream-bz.o zstream-gz.o \
util/debug.o # this must come last!
-objects-random = util/glibc-md5.o util/log.o util/md5sum.o util/random.o \
+objects-random = util/glibc-md5.o util/glibc-sha256.o util/log.o util/md5sum.o \
+ util/sha256sum.o util/random.o \
util/string.o \
util/debug.o # this must come last!
#______________________________
it under the terms of the GNU General Public License, version 2. See
the file COPYING for details.
- Cache with MD5 sums of file contents - used by JigdoCache in scan.hh
+ Cache with checksums of file contents - used by JigdoCache in scan.hh
*/
db->set_cachesize(db, 0, 4*1024*1024, 1);
// Use a btree, create database file if not yet present
- e = compat_dbOpen(db, dbName, "jigdo filecache v0", DB_BTREE, DB_CREATE,
+ e = compat_dbOpen(db, dbName, "jigdo filecache v1", DB_BTREE, DB_CREATE,
0666);
if (e != 0) {
// Re-close, in case it is necessary
/* If the DB file is old or corrupted, just regenerate it from
scratch, otherwise throw error. */
debug("Cache file corrupt, recreating it");
- if (compat_dbOpen(db, dbName, "jigdo filecache v0", DB_BTREE,
+ if (compat_dbOpen(db, dbName, "jigdo filecache v1", DB_BTREE,
DB_CREATE | DB_TRUNCATE, 0666) != 0)
throw DbError(e);
}
/** @file
- Cache with MD5 sums of file contents - used by JigdoCache in scan.hh
+ Cache with checksums of file contents - used by JigdoCache in scan.hh
The created libdb3 database contains one table with a mapping from
filenames (without trailing zero byte) to a binary structure. The
This is not handled by CacheFile; it is passed as an opaque string of
bytes to scan.hh classes:<pre>
4 blockLength (of rsync sum)
- 4 md5BlockLength
- 4 blocks (number of valid md5 blocks in this entry)
+ 4 csumBlockLength
+ 4 blocks (number of valid checksum blocks in this entry)
8 rsyncSum of file start (only valid if blocks > 0)
16 fileMD5Sum (only valid if
- blocks == (fileSize+md5BlockLength-1)/md5BlockLength )
+ blocks == (fileSize+csumBlockLength-1)/csumBlockLength )
+ 32 fileSHA256Sum (only valid if
+ blocks == (fileSize+csumBlockLength-1)/csumBlockLength )
followed by n entries:
- 16 md5sum of block of size md5BlockLength</pre>
+ 16 md5sum of block of size csumBlockLength</pre>
+ 32 sha256sum of block of size csumBlockLength</pre>
Why is mtime and size not part of the key? Because we only want to
store one entry per file, not an additional entry whenever the file
};
//______________________________________________________________________
-/** Cache with MD5 sums of file contents */
+/** Cache with checksums of file contents */
class CacheFile {
public:
/** Create new database or open existing database */
//____________________
JigdoCache cache(cacheFile, optCacheExpiry, readAmount, *optReporter);
- cache.setParams(blockLength, md5BlockLength);
+ cache.setParams(blockLength, csumBlockLength);
cache.setCheckFiles(optCheckFiles);
if (addLabels(cache)) return 3;
while (true) {
if (imageFile != "-" && willOutputTo(imageFile, optForce) > 0) return 3;
JigdoCache cache(cacheFile, optCacheExpiry, readAmount, *optReporter);
- cache.setParams(blockLength, md5BlockLength);
+ cache.setParams(blockLength, csumBlockLength);
while (true) {
try { cache.readFilenames(fileNames); } // Recurse through directories
catch (RecurseError e) { optReporter->error(e.message); continue; }
}
//______________________________________________________________________
-int JigdoFileCmd::verifyImage() {
+int JigdoFileCmd::verifyImageMD5() {
if (imageFile.empty() || templFile.empty()) {
cerr << subst(_(
"%1 verify: Not both --image and --template specified.\n"
}
//______________________________________________________________________
+int JigdoFileCmd::verifyImageSHA256() {
+ if (imageFile.empty() || templFile.empty()) {
+ cerr << subst(_(
+ "%1 verify: Not both --image and --template specified.\n"
+ "(Attempt to deduce missing names failed.)\n"), binaryName);
+ exit_tryHelp();
+ }
+
+ bistream* image;
+ auto_ptr<bistream> imageDel(openForInput(image, imageFile));
+
+ JigdoDescVec contents;
+ JigdoDesc::ImageInfoSHA256* info;
+ try {
+ bistream* templ;
+ auto_ptr<bistream> templDel(openForInput(templ, templFile));
+
+ if (JigdoDesc::isTemplate(*templ) == false)
+ optReporter->info(
+ _("Warning: This does not seem to be a template file"));
+
+ JigdoDesc::seekFromEnd(*templ);
+ *templ >> contents;
+ if (!*templ) {
+ string err = subst(_("%1 verify: %2"), binaryName, strerror(errno));
+ optReporter->error(err);
+ return 3;
+ }
+ info = dynamic_cast<JigdoDesc::ImageInfoSHA256*>(contents.back());
+ if (info == 0) {
+ string err = subst(_("%1 verify: Invalid template data - "
+ "corrupted file?"), binaryName);
+ optReporter->error(err);
+ return 3;
+ }
+ } catch (JigdoDescError e) {
+ string err = subst(_("%1: %2"), binaryName, e.message);
+ optReporter->error(err);
+ return 3;
+ }
+
+ SHA256Sum md; // SHA256Sum of image
+ md.updateFromStream(*image, info->size(), readAmount, *optReporter);
+ md.finish();
+ if (*image) {
+ image->get();
+ if (image->eof() && md == info->sha256()) {
+ optReporter->info(_("OK: Checksums match, image is good!"));
+ return 0;
+ }
+ }
+ optReporter->error(_(
+ "ERROR: Checksums do not match, image might be corrupted!"));
+ return 2;
+}
+//______________________________________________________________________
+
/* Look up a query (e.g. "MyServer:foo/path/bar") in the JigdoConfig
mapping. Returns true if something was found, and prints out all
resulting URIs. */
}
JigdoCache cache(cacheFile, optCacheExpiry, readAmount, *optReporter);
- cache.setParams(blockLength, md5BlockLength);
+ cache.setParams(blockLength, csumBlockLength);
if (addLabels(cache)) return 3;
while (true) {
try { cache.readFilenames(fileNames); } // Recurse through directories
while (ci != ce) { ci->getMD5Sum(&cache); ++ci; }
} else {
// Only cause first md5 block to be read; not scanning the whole file
- while (ci != ce) { ci->getSums(&cache, 0); ++ci; }
+ while (ci != ce) { ci->getMD5Sums(&cache, 0); ++ci; }
}
return 0;
// Cache data is written out when the JigdoCache is destroyed
actually very similar to scanFiles() above. */
int JigdoFileCmd::md5sumFiles() {
JigdoCache cache(cacheFile, optCacheExpiry, readAmount, *optReporter);
- cache.setParams(blockLength, md5BlockLength);
+ cache.setParams(blockLength, csumBlockLength);
cache.setCheckFiles(optCheckFiles);
while (true) {
try { cache.readFilenames(fileNames); } // Recurse through directories
return 0;
// Cache data is written out when the JigdoCache is destroyed
}
+//______________________________________________________________________
+
+/* Print SHA256 checksums of arguments like sha256sum(1), but using our
+ Base64-like encoding for the checksum, not hexadecimal like
+ sha256sum(1). Additionally, try to make use of the cache, and only
+ print out the part of any filename following any "//". This is
+ actually very similar to scanFiles() above. */
+int JigdoFileCmd::sha256sumFiles() {
+ JigdoCache cache(cacheFile, optCacheExpiry, readAmount, *optReporter);
+ cache.setParams(blockLength, csumBlockLength);
+ cache.setCheckFiles(optCheckFiles);
+ while (true) {
+ try { cache.readFilenames(fileNames); } // Recurse through directories
+ catch (RecurseError e) { optReporter->error(e.message); continue; }
+ break;
+ }
+
+ if (JigdoFileCmd::optHex) Base64String::hex = true;
+
+ JigdoCache::iterator ci = cache.begin(), ce = cache.end();
+ while (ci != ce) {
+ Base64String m;
+ // Causes whole file to be read
+ const SHA256Sum* md = ci->getSHA256Sum(&cache);
+ if (md != 0) {
+ m.write(md->digest(), 32).flush();
+ string& s(m.result());
+ s += " ";
+ if (ci->getPath() == "/") s += '/';
+ s += ci->leafName();
+ // Output checksum line
+ optReporter->coutInfo(s);
+ }
+ ++ci;
+ }
+ return 0;
+ // Cache data is written out when the JigdoCache is destroyed
+}
#include <jigdoconfig.hh>
#include <scan.hh>
#include <md5sum.hh>
+#include <sha256sum.hh>
#include <mkimage.hh>
#include <mktemplate.hh>
//______________________________________________________________________
public JigdoCache::ProgressReporter,
public JigdoDesc::ProgressReporter,
public MD5Sum::ProgressReporter,
+ public SHA256Sum::ProgressReporter,
public JigdoConfig::ProgressReporter {
virtual void error(const string& message) {
MD5Sum::ProgressReporter::error(message);
enum Command {
MAKE_TEMPLATE, MAKE_IMAGE,
PRINT_MISSING, PRINT_MISSING_ALL,
- SCAN, VERIFY, LIST_TEMPLATE, MD5SUM
+ SCAN, VERIFY, LIST_TEMPLATE, MD5SUM, SHA256SUM
};
//________________________________________
static vector<string> optLabels; // Strings of the form "Label=/some/path"
static vector<string> optUris; // "Label=http://some.server/"
static size_t blockLength; // of rsync algorithm, is also minimum file size
- static size_t md5BlockLength;
+ static size_t csumBlockLength;
static size_t readAmount;
static int optZipQuality;
static bool optBzip2;
static bool optForce; // true => Silently delete existent output
- static bool optMkImageCheck; // true => check MD5sums
+ static bool optMkImageCheck; // true => check checksums
static bool optCheckFiles; // true => check if files exist
static bool optScanWholeFile; // false => read only first block
// true => skip smaller matches if a larger match could be possible
static bool optGreedyMatching;
static bool optAddImage; // true => Add [Image] section to output .jigdo
static bool optAddServers; // true => Add [Servers] to output .jigdo
- static bool optHex; // true => Use hex not base64 output for md5/ls cmds
+ static bool optHex; // true => Use hex not base64 output for checksum/ls cmds
static string optDebug; // list of debug msg to turn on, or all/help
// Reporter is defined in config.h and is the base of all other *Reporter's
static AnyReporter* optReporter;
static int makeImage();
static int printMissing(Command command = PRINT_MISSING);
static int scanFiles();
- static int verifyImage();
+ static int verifyImageMD5();
+ static int verifyImageSHA256();
static int listTemplate();
static int md5sumFiles();
+ static int sha256sumFiles();
//@}
/** @name
vector<string> JigdoFileCmd::optLabels;
vector<string> JigdoFileCmd::optUris;
size_t JigdoFileCmd::blockLength = 1*1024U;
-size_t JigdoFileCmd::md5BlockLength = 128*1024U - 55;
+size_t JigdoFileCmd::csumBlockLength = 128*1024U - 55;
size_t JigdoFileCmd::readAmount = 128*1024U;
int JigdoFileCmd::optZipQuality = Z_BEST_COMPRESSION;
bool JigdoFileCmd::optBzip2 = false;
m += _("scanning image");
print(m, false);
}
- virtual void readingMD5(uint64 offInStream, uint64 size) {
+ virtual void readingChecksum(uint64 offInStream, uint64 size) {
if (!printProgress) return;
string m;
append(m, 100 * offInStream / size, 3); // 3
//______________________________________________________________________
inline void printUsage(bool detailed, size_t blockLength,
- size_t md5BlockLength, size_t readAmount) {
+ size_t csumBlockLength, size_t readAmount) {
if (detailed) {
cout << subst(_(
"\n"
" scan sc Update cache with information about supplied files\n");
cout << _(
" verify ver Check whether image matches checksum from template\n"
- " md5sum md5 Print MD5 checksums similar to md5sum(1)\n");
+ " md5sum md5 Print MD5 checksums similar to md5sum(1)\n"
+ " sha256sum sha256 Print SHA256 checksums similar to sha256sum(1)\n");
if (detailed) {
cout << _(
" list-template ls Print low-level listing of contents of template\n"
" --min-length=BYTES [default %1]\n"
" [make-template] Minimum length of files to search\n"
" for in image data\n"
- " --md5-block-size=BYTES [default %2]\n"
+ " --checksum-block-size=BYTES [default %2]\n"
" Uninteresting internal parameter -\n"
- " jigdo-file enforces: min-length < md5-block-size\n"
+ " jigdo-file enforces: min-length < checksum-block-size\n"
+ " --md5-block-size=BYTES\n"
+ " Alias for --checksum-block-size, deprecated.\n"
" --readbuffer=BYTES [default %3k]\n"
" Amount of data to read at a time\n"
" --check-files [default]\n"
- " [make-template,md5sum] Check if files exist and\n"
+ " [make-template,md5sum,sha256sum] Check if files exist and\n"
" get or verify checksums, date and size\n"
" [make-image] Verify checksum of files written to\n"
" image\n"
- " --no-check-files [make-template,md5sum] when used with --cache,\n"
+ " --no-check-files [make-template,md5sum,sha256sum] when used with --cache,\n"
" [make-image] Do not verify checksums of files\n"
" --scan-whole-file [scan] Scan whole file instead of only first block\n"
" --no-scan-whole-file [scan] Scan only first block [default]\n"
" --no-debug No debugging info [default]\n"
" --match-exec=CMD [make-template] Execute command when files match\n"
" CMD is passed to a shell, with environment set up:\n"
- " LABEL, LABELPATH, MATCHPATH, LEAF, MD5SUM, FILE\n"
+ " LABEL, LABELPATH, MATCHPATH, LEAF, MD5SUM, SHA256SUM, FILE\n"
" e.g. 'mkdir -p \"${LABEL:-.}/$MATCHPATH\" && ln -f \"$FILE\" \"${LABEL:-.}/$MATCHPATH$LEAF\"'\n"
" --no-hex [default]\n"
- " --hex [md5sum, list-template] Output checksums in\n"
+ " --hex [md5sum,sha256sum,list-template] Output checksums in\n"
" hexadecimal, not Base64\n"
" --gzip [default] Use gzip compression, not --bzip2\n"),
- blockLength, md5BlockLength, readAmount / 1024) << endl;
+ blockLength, csumBlockLength, readAmount / 1024) << endl;
}
return;
}
enum {
LONGOPT_BUFSIZE = 0x100, LONGOPT_NOFORCE, LONGOPT_MINSIZE,
- LONGOPT_MD5SIZE, LONGOPT_MKIMAGECHECK, LONGOPT_NOMKIMAGECHECK,
+ LONGOPT_CHECKSUMSIZE, LONGOPT_MKIMAGECHECK, LONGOPT_NOMKIMAGECHECK,
LONGOPT_LABEL, LONGOPT_URI, LONGOPT_ADDSERVERS, LONGOPT_NOADDSERVERS,
LONGOPT_ADDIMAGE, LONGOPT_NOADDIMAGE, LONGOPT_NOCACHE, LONGOPT_CACHEEXPIRY,
LONGOPT_MERGE, LONGOPT_HEX, LONGOPT_NOHEX, LONGOPT_DEBUG, LONGOPT_NODEBUG,
{ "jigdo", required_argument, 0, 'j' },
{ "label", required_argument, 0, LONGOPT_LABEL },
{ "match-exec", required_argument, 0, LONGOPT_MATCHEXEC },
- { "md5-block-size", required_argument, 0, LONGOPT_MD5SIZE },
+ { "md5-block-size", required_argument, 0, LONGOPT_CHECKSUMSIZE },
+ { "checksum-block-size",required_argument, 0, LONGOPT_CHECKSUMSIZE },
{ "merge", required_argument, 0, LONGOPT_MERGE },
{ "min-length", required_argument, 0, LONGOPT_MINSIZE },
{ "no-cache", no_argument, 0, LONGOPT_NOCACHE },
case 'f': optForce = true; break;
case LONGOPT_NOFORCE: optForce = false; break;
case LONGOPT_MINSIZE: blockLength = scanMemSize(optarg); break;
- case LONGOPT_MD5SIZE: md5BlockLength = scanMemSize(optarg); break;
+ case LONGOPT_CHECKSUMSIZE: csumBlockLength = scanMemSize(optarg); break;
case LONGOPT_BUFSIZE: readAmount = scanMemSize(optarg); break;
case 'r':
if (strcmp(optarg, "default") == 0) {
if (optHelp != '\0' || optVersion) {
if (optVersion) cout << "jigdo-file version " JIGDO_VERSION << endl;
if (optHelp != '\0') printUsage(optHelp == 'H', blockLength,
- md5BlockLength, readAmount);
+ csumBlockLength, readAmount);
throw Cleanup(0);
}
# endif
//______________________________
- // Silently correct invalid blockLength/md5BlockLength args
+ // Silently correct invalid blockLength/csumBlockLength args
if (blockLength < MINIMUM_BLOCKLENGTH) blockLength = MINIMUM_BLOCKLENGTH;
- if (blockLength >= md5BlockLength) md5BlockLength = blockLength + 1;
- // Round to next k*64+55 for efficient MD5 calculation
- md5BlockLength = ((md5BlockLength + 63 - 55) & ~63U) + 55;
+ if (blockLength >= csumBlockLength) csumBlockLength = blockLength + 1;
+ // Round to next k*64+55 for efficient checksum calculation
+ csumBlockLength = ((csumBlockLength + 63 - 55) & ~63U) + 55;
Paranoid(blockLength >= MINIMUM_BLOCKLENGTH
- && blockLength < md5BlockLength);
+ && blockLength < csumBlockLength);
//______________________________
// Complain if name of command isn't there
{ (char *)"list-template", LIST_TEMPLATE },
{ (char *)"ls", LIST_TEMPLATE },
{ (char *)"md5sum", MD5SUM },
- { (char *)"md5", MD5SUM }
+ { (char *)"md5", MD5SUM },
+ { (char *)"sha256sum", SHA256SUM },
+ { (char *)"sha256", SHA256SUM }
};
const CodesEntry *c = codes;
case JigdoFileCmd::SCAN:
returnValue = JigdoFileCmd::scanFiles(); break;
case JigdoFileCmd::VERIFY:
- returnValue = JigdoFileCmd::verifyImage(); break;
+ returnValue = JigdoFileCmd::verifyImageMD5(); break;
+ // FIXME! Needs update to use SHA256
case JigdoFileCmd::LIST_TEMPLATE:
returnValue = JigdoFileCmd::listTemplate(); break;
case JigdoFileCmd::MD5SUM:
JigdoFileCmd::optCheckFiles = true; // Quick fix, possibly not 100% correct
returnValue = JigdoFileCmd::md5sumFiles(); break;
+ case JigdoFileCmd::SHA256SUM:
+ JigdoFileCmd::optCheckFiles = true; // Quick fix, possibly not 100% correct
+ returnValue = JigdoFileCmd::sha256sumFiles(); break;
}
}
catch (bad_alloc &) { outOfMemory(); }
uint64 off = 0; // Offset in image
uint64 read = 0; // Nr of bytes read
MD5 entryMd5;
+ SHA256 entrySha256;
uint64 entryLen;
RsyncSum64 rsum;
size_t blockLength;
read += 1 + 6 + entryMd5.serialSizeOf() + 4;
break;
+ case JigdoDesc::IMAGE_INFO_SHA256:
+ unserialize6(entryLen, f);
+ unserialize(entrySha256, f);
+ unserialize4(blockLength, f);
+ if (!file) break;
+ debug("JigdoDesc::read: ImageInfo %1 %2",
+ entryLen, entrySha256.toString());
+ desc.reset(new JigdoDesc::ImageInfoSHA256(entryLen, entrySha256, blockLength));
+ push_back(desc.release());
+ read += 1 + 6 + entrySha256.serialSizeOf() + 4;
+ break;
+
case JigdoDesc::UNMATCHED_DATA:
unserialize6(entryLen, f);
if (!file) break;
off += entryLen;
break;
+ case JigdoDesc::MATCHED_FILE_SHA256:
+ case JigdoDesc::WRITTEN_FILE_SHA256:
+ unserialize6(entryLen, f);
+ unserialize(rsum, f);
+ unserialize(entrySha256, f);
+ if (!file) break;
+ debug("JigdoDesc::read: %1 %2File %3 %4",
+ off, (type == JigdoDesc::MATCHED_FILE_SHA256 ? "Matched" : "Written"),
+ entryLen, entrySha256.toString());
+ if (type == JigdoDesc::MATCHED_FILE_SHA256)
+ desc.reset(new JigdoDesc::MatchedFileSHA256(off, entryLen, rsum,entrySha256));
+ else
+ desc.reset(new JigdoDesc::WrittenFileSHA256(off, entryLen, rsum,entrySha256));
+ push_back(desc.release());
+ read += 1 + 6 + rsum.serialSizeOf() + entrySha256.serialSizeOf();
+ off += entryLen;
+ break;
+
// Template entry types that were obsoleted with version 0.6.3:
case JigdoDesc::OBSOLETE_IMAGE_INFO:
}
//______________________________________________________________________
-bostream& JigdoDescVec::put(bostream& file, MD5Sum* md) const {
+bostream& JigdoDescVec::put(bostream& file, MD5Sum* md, SHA256Sum* sd) const {
// Pass 1: Accumulate sizes of entries, calculate descLen
// 4 for DESC, 6 each for length of part at start & end
uint64 descLen = 4 + 6*2; // Length of DESC part
p = serialize4(0x43534544, buf); // "DESC" in little-endian order
p = serialize6(descLen, p);
writeBytes(file, buf, 4 + 6);
- if (md != 0) md->update(buf, 4 + 6);
+ if (md != 0)
+ md->update(buf, 4 + 6);
+ if (sd != 0)
+ sd->update(buf, 4 + 6);
for (const_iterator i = begin(), e = end(); i != e; ++i) {
JigdoDesc::ImageInfo* info;
JigdoDesc::UnmatchedData* unm;
JigdoDesc::MatchedFile* matched;
JigdoDesc::WrittenFile* written;
+ JigdoDesc::MatchedFileSHA256* matchedsha;
+ JigdoDesc::WrittenFileSHA256* writtensha;
/* NB we must first try to cast to WrittenFile, then to
MatchedFile, because WrittenFile derives from MatchedFile. */
if ((info = dynamic_cast<JigdoDesc::ImageInfo*>(*i)) != 0)
p = written->serialize(buf);
else if ((matched = dynamic_cast<JigdoDesc::MatchedFile*>(*i)) != 0)
p = matched->serialize(buf);
+ else if ((writtensha = dynamic_cast<JigdoDesc::WrittenFileSHA256*>(*i)) != 0)
+ p = writtensha->serialize(buf);
+ else if ((matchedsha = dynamic_cast<JigdoDesc::MatchedFileSHA256*>(*i)) != 0)
+ p = matchedsha->serialize(buf);
else { Assert(false); continue; }
writeBytes(file, buf, p - buf);
- if (md != 0) md->update(buf, p - buf);
+ if (md != 0)
+ md->update(buf, p - buf);
+ if (sd != 0)
+ sd->update(buf, p - buf);
}
p = serialize6(descLen, buf);
writeBytes(file, buf, 6);
- if (md != 0) md->update(buf, 6);
+ if (md != 0)
+ md->update(buf, 6);
+ if (sd != 0)
+ sd->update(buf, p - buf);
if (DEBUG) { Assert(buf[bufLen - 1] == 0xa5); }
- return file;
+ return file;
}
//______________________________________________________________________
namespace {
const int J_SIZE_WIDTH = 12;
+ const int J_CSUM_WIDTH = 46;
+ const int J_RSYNC_WIDTH = 12;
}
ostream& JigdoDesc::ImageInfo::put(ostream& s) const {
- s << "image-info " << setw(J_SIZE_WIDTH) << size() << " "
- << md5() << ' ' << blockLength() << '\n';
+ s << "image-info-md5 "
+ << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_SIZE_WIDTH) << blockLength()
+ << " " << setw(J_CSUM_WIDTH) << md5()
+ << "\n";
+ return s;
+}
+ostream& JigdoDesc::ImageInfoSHA256::put(ostream& s) const {
+ s << "image-info-sha256 "
+ << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_SIZE_WIDTH) << blockLength()
+ << " " << setw(J_CSUM_WIDTH) << sha256()
+ << "\n";
return s;
}
ostream& JigdoDesc::UnmatchedData::put(ostream& s) const {
- s << "in-template " << setw(J_SIZE_WIDTH) << offset() << ' '
- << setw(J_SIZE_WIDTH) << size() << '\n';
+ s << "in-template "
+ << setw(J_SIZE_WIDTH) << offset()
+ << " " << setw(J_SIZE_WIDTH) << size()
+ << "\n";
return s;
}
ostream& JigdoDesc::MatchedFile::put(ostream& s) const {
- s << "need-file " << setw(J_SIZE_WIDTH) << offset() << ' '
- << setw(J_SIZE_WIDTH) << size() << ' ' << md5() << ' ' << rsync() << '\n';
+ s << "need-file-md5 "
+ << setw(J_SIZE_WIDTH) << offset()
+ << " " << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_CSUM_WIDTH) << md5()
+ << " " << setw(J_RSYNC_WIDTH) << rsync()
+ << "\n";
+ return s;
+}
+ostream& JigdoDesc::MatchedFileSHA256::put(ostream& s) const {
+ s << "need-file-sha256 "
+ << setw(J_SIZE_WIDTH) << offset()
+ << " " << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_CSUM_WIDTH) << sha256()
+ << " " << setw(J_RSYNC_WIDTH) << rsync()
+ << "\n";
return s;
}
+
ostream& JigdoDesc::WrittenFile::put(ostream& s) const {
- s << "have-file " << setw(J_SIZE_WIDTH) << offset() << ' '
- << setw(J_SIZE_WIDTH) << size() << ' ' << md5() << ' ' << rsync() << '\n';
+ s << "have-file "
+ << setw(J_SIZE_WIDTH) << offset()
+ << " " << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_CSUM_WIDTH) << md5()
+ << " " << setw(J_RSYNC_WIDTH) << rsync()
+ << "\n";
+ return s;
+}
+ostream& JigdoDesc::WrittenFileSHA256::put(ostream& s) const {
+ s << "have-file-sha256 "
+ << setw(J_SIZE_WIDTH) << offset()
+ << " " << setw(J_SIZE_WIDTH) << size()
+ << " " << setw(J_CSUM_WIDTH) << sha256()
+ << " " << setw(J_RSYNC_WIDTH) << rsync()
+ << "\n";
return s;
}
/* Read up to file.size() of bytes from file, write it to image
stream. Check MD5/rsync sum if requested. Take care not to write
more than specified amount to image, even if file is longer. */
- int fileToImage(bostream* img, FilePart& file,
- const JigdoDesc::MatchedFile& matched, bool checkMD5, size_t rsyncLen,
+ int fileToImageMD5(bostream* img, FilePart& file,
+ const JigdoDesc::MatchedFile& matched, bool checkChecksum, size_t rsyncLen,
ProgressReporter& reporter, byte* buf, size_t readAmount, uint64& off,
uint64& nextReport, const uint64 totalBytes) {
uint64 toWrite = file.size();
// Read from file, write to image
// First couple of k: Calculate RsyncSum rs and MD5Sum md
- if (checkMD5 && rsyncLen > 0) {
+ if (checkChecksum && rsyncLen > 0) {
while (*img && f && !f.eof() && toWrite > 0) {
size_t n = (toWrite < readAmount ? toWrite : readAmount);
readBytes(f, buf, n);
writeBytes(*img, buf, n);
reportBytesWritten(n, off, nextReport, totalBytes, reporter);
toWrite -= n;
- if (checkMD5) md.update(buf, n);
+ if (checkChecksum) md.update(buf, n);
}
if (toWrite > 0 && (!f || f.eof())) {
reportBytesWritten(n, off, nextReport, totalBytes, reporter);
toWrite -= n;
}
- } else if (checkMD5
+ } else if (checkChecksum
&& (md.finish() != matched.md5()
|| (rsyncLen > 0 && rs != matched.rsync()))) {
err = subst(_("Error: `%1' does not match checksum in template data"),
}
//______________________________
+ /* Read up to file.size() of bytes from file, write it to image
+ stream. Check MD5/rsync sum if requested. Take care not to write
+ more than specified amount to image, even if file is longer. */
+ int fileToImageSHA256(bostream* img, FilePart& file,
+ const JigdoDesc::MatchedFileSHA256& matched, bool checkChecksum, size_t rsyncLen,
+ ProgressReporter& reporter, byte* buf, size_t readAmount, uint64& off,
+ uint64& nextReport, const uint64 totalBytes) {
+ uint64 toWrite = file.size();
+ SHA256Sum md;
+ RsyncSum64 rs;
+ size_t rl = 0; // Length covered by rs so far
+ string fileName(file.getPath());
+ fileName += file.leafName();
+ bifstream f(fileName.c_str(), ios::binary);
+ string err; // !err.empty() => error occurred
+
+ // Read from file, write to image
+ // First couple of k: Calculate RsyncSum rs and MD5Sum md
+ if (checkChecksum && rsyncLen > 0) {
+ while (*img && f && !f.eof() && toWrite > 0) {
+ size_t n = (toWrite < readAmount ? toWrite : readAmount);
+ readBytes(f, buf, n);
+ n = f.gcount();
+ writeBytes(*img, buf, n);
+ reportBytesWritten(n, off, nextReport, totalBytes, reporter);
+ toWrite -= n;
+ md.update(buf, n);
+ // Update RsyncSum
+ Paranoid(rl < rsyncLen);
+ size_t rsyncToAdd = rsyncLen - rl;
+ if (rsyncToAdd > n) rsyncToAdd = n;
+ rs.addBack(buf, rsyncToAdd);
+ rl += rsyncToAdd;
+ Paranoid(rl <= rsyncLen);
+ if (rl >= rsyncLen) break;
+ }
+ }
+ // Rest of file: Only calculate MD5Sum md
+ while (*img && f && !f.eof() && toWrite > 0) {
+ size_t n = (toWrite < readAmount ? toWrite : readAmount);
+ readBytes(f, buf, n);
+ n = f.gcount();
+ writeBytes(*img, buf, n);
+ reportBytesWritten(n, off, nextReport, totalBytes, reporter);
+ toWrite -= n;
+ if (checkChecksum) md.update(buf, n);
+ }
+
+ if (toWrite > 0 && (!f || f.eof())) {
+ const char* errDetail = "";
+ if (errno != 0) errDetail = strerror(errno);
+ else if (f.eof()) errDetail = _("file is too short");
+ err = subst(_("Error reading from `%1' (%2)"), fileName, errDetail);
+ // Even if there was an error - always try to write right amount
+ memClear(buf, readAmount);
+ while (*img && toWrite > 0) {
+ size_t n = (toWrite < readAmount ? toWrite : readAmount);
+ writeBytes(*img, buf, n);
+ reportBytesWritten(n, off, nextReport, totalBytes, reporter);
+ toWrite -= n;
+ }
+ } else if (checkChecksum
+ && (md.finish() != matched.sha256()
+ || (rsyncLen > 0 && rs != matched.rsync()))) {
+ err = subst(_("Error: `%1' does not match checksum in template data"),
+ fileName);
+ }
+
+ if (err.empty()) return 0; // Success
+ reporter.error(err);
+ if (toWrite == 0)
+ return 2; // "May have to fix something before you can continue"
+ else
+ return 3; // Yaargh, disaster! Please delete the .tmp file for me
+ }
+ //______________________________
+
/* Write all bytes of the image data, i.e. both UnmatchedData and
MatchedFiles. If any UnmatchedFiles are present in 'files', write
zeroes instead of the file content and also append a DESC section
cout, caller should rename file to remove .tmp extension. */
inline int writeAll(const Task& task, JigdoDescVec& files,
queue<FilePart*>& toCopy, bistream* templ, const size_t readAmount,
- bostream* img, const char* name, bool checkMD5,
+ bostream* img, const char* name, bool checkChecksum,
ProgressReporter& reporter, JigdoCache* cache,
const uint64 totalBytes) {
nice, but using virtual methods looks even worse.] */
switch ((*i)->type()) {
case JigdoDesc::IMAGE_INFO:
+ case JigdoDesc::IMAGE_INFO_SHA256:
break;
case JigdoDesc::UNMATCHED_DATA: {
// Copy data from Zibstream to image.
} else {
/* Copy data from file to image, taking care not to
write beyond toWrite. */
- int status = fileToImage(img, *mfile, *self, checkMD5,
+ int status = fileToImageMD5(img, *mfile, *self, checkChecksum,
imageInfo.blockLength(), reporter, buf, readAmount, off,
nextReport, totalBytes);
toCopy.pop();
}
break;
}
+ case JigdoDesc::MATCHED_FILE_SHA256: {
+ /* If file present in cache, copy its data to image, if
+ not, copy zeroes. if check==true, verify SHA256 sum match.
+ If successful, turn MatchedFileSHA256 into WrittenFileSHA256. */
+ JigdoDesc::MatchedFileSHA256* self =
+ dynamic_cast<JigdoDesc::MatchedFileSHA256*>(*i);
+ uint64 toWrite = self->size();
+ FilePart* mfile = 0;
+ if (!toCopy.empty()) mfile = toCopy.front();
+ debug("mkimage writeAll(): FilePart@%1, %2 of matched file `%3',"
+ " toCopy size %4", mfile, toWrite,
+ (mfile != 0 ? mfile->leafName() : ""), toCopy.size());
+ if (mfile == 0 || self->sha256() != *(mfile->getSHA256Sum(cache))) {
+ // Write right amount of zeroes
+ memClear(buf, readAmount);
+ while (*img && toWrite > 0) {
+ size_t n = (toWrite < readAmount ? toWrite : readAmount);
+ writeBytes(*img, buf, n);
+ reportBytesWritten(n, off, nextReport, totalBytes, reporter);
+ toWrite -= n;
+ }
+ if (result == 0) result = 1; // Soft failure
+ } else {
+ /* Copy data from file to image, taking care not to
+ write beyond toWrite. */
+ int status = fileToImageSHA256(img, *mfile, *self, checkChecksum,
+ imageInfo.blockLength(), reporter, buf, readAmount, off,
+ nextReport, totalBytes);
+ toCopy.pop();
+ if (result < status) result = status;
+ if (status == 0) { // Mark file as written to image
+ *i = new JigdoDesc::WrittenFileSHA256(self->offset(), self->size(),
+ self->rsync(), self->sha256());
+ delete self;
+ } else if (*img && (status > 2 || task == SINGLE_PASS)) {
+ // If !*img, exit after error msg below
+ /* If status <= 2 and task == {CREATE_TMP,MERGE_TMP},
+ we can continue; there has been an error copying
+ this individual file, but the right *amount* of
+ data has been written to the .tmp output file, and
+ the user may retry the failed one later. */
+ return result;
+ }
+ }
+ break;
+ }
case JigdoDesc::WRITTEN_FILE:
+ case JigdoDesc::WRITTEN_FILE_SHA256:
// These are never present in memory, cannot occur:
case JigdoDesc::OBSOLETE_IMAGE_INFO:
case JigdoDesc::OBSOLETE_MATCHED_FILE:
returned, caller should rename file to remove .tmp extension. */
inline int writeMerge(JigdoDescVec& files, queue<FilePart*>& toCopy,
const int missing, const size_t readAmount, bfstream* img,
- const string& imageTmpFile, bool checkMD5, ProgressReporter& reporter,
+ const string& imageTmpFile, bool checkChecksum, ProgressReporter& reporter,
JigdoCache* cache, const uint64 totalBytes) {
vector<byte> bufVec(readAmount);
byte* buf = &bufVec[0];
result = 2;
break;
}
- int status = fileToImage(img, *mfile, *self, checkMD5,
+ int status = fileToImageMD5(img, *mfile, *self, checkChecksum,
imageInfo.blockLength(), reporter, buf, readAmount, bytesWritten,
nextReport, totalBytes);
toCopy.pop();
#include <bstream.hh>
#include <debug.hh>
#include <md5sum.hh>
+#include <sha256sum.hh>
#include <scan.hh>
#include <serialize.hh>
//______________________________________________________________________
public:
/** Types of entries in a description section */
enum Type {
- IMAGE_INFO = 5, UNMATCHED_DATA = 2, MATCHED_FILE = 6, WRITTEN_FILE = 7,
- OBSOLETE_IMAGE_INFO = 1, OBSOLETE_MATCHED_FILE = 3,
- OBSOLETE_WRITTEN_FILE = 4
+ OBSOLETE_IMAGE_INFO = 1,
+ UNMATCHED_DATA = 2,
+ OBSOLETE_MATCHED_FILE = 3,
+ OBSOLETE_WRITTEN_FILE = 4,
+ IMAGE_INFO = 5,
+ MATCHED_FILE = 6,
+ WRITTEN_FILE = 7,
+ IMAGE_INFO_SHA256 = 8,
+ MATCHED_FILE_SHA256 = 9,
+ WRITTEN_FILE_SHA256 = 10
};
class ProgressReporter;
//____________________
const string& templFile, bistream* templ, ProgressReporter& reporter);
class ImageInfo;
+ class ImageInfoSHA256;
class UnmatchedData;
class MatchedFile;
+ class MatchedFileSHA256;
class WrittenFile;
+ class WrittenFileSHA256;
private:
static ProgressReporter noReport;
};
//________________________________________
+/** Information about the image file */
+class JigdoDesc::ImageInfoSHA256 : public JigdoDesc {
+public:
+ inline ImageInfoSHA256(uint64 s, const SHA256& m, size_t b);
+ inline ImageInfoSHA256(uint64 s, const SHA256Sum& m, size_t b);
+ inline bool operator==(const JigdoDesc& x) const;
+ Type type() const { return IMAGE_INFO_SHA256; }
+ uint64 size() const { return sizeVal; }
+ const SHA256& sha256() const { return sha256Val; }
+ size_t blockLength() const { return blockLengthVal; }
+ // Default dtor, operator==
+ virtual ostream& put(ostream& s) const;
+
+ template<class Iterator>
+ inline Iterator serialize(Iterator i) const;
+ inline size_t serialSizeOf() const;
+
+private:
+ uint64 sizeVal;
+ SHA256 sha256Val;
+ size_t blockLengthVal;
+};
+//________________________________________
+
/** Info about data that was not matched by any input file, i.e.
that is included in the template data verbatim */
class JigdoDesc::UnmatchedData : public JigdoDesc {
};
//______________________________________________________________________
+/** Info about data that *was* matched by an input file */
+class JigdoDesc::MatchedFileSHA256 : public JigdoDesc {
+public:
+ inline MatchedFileSHA256(uint64 o, uint64 s, const RsyncSum64& r, const SHA256& m);
+ inline MatchedFileSHA256(uint64 o, uint64 s, const RsyncSum64& r,
+ const SHA256Sum& m);
+ inline bool operator==(const JigdoDesc& x) const;
+ Type type() const { return MATCHED_FILE_SHA256; }
+ uint64 offset() const { return offsetVal; }
+ uint64 size() const { return sizeVal; }
+ const SHA256& sha256() const { return sha256Val; }
+ const RsyncSum64& rsync() const { return rsyncVal; }
+ // Default dtor, operator==
+ virtual ostream& put(ostream& s) const;
+
+ template<class Iterator>
+ inline Iterator serialize(Iterator i) const;
+ inline size_t serialSizeOf() const;
+
+private:
+ uint64 offsetVal; // Offset in image
+ uint64 sizeVal;
+ RsyncSum64 rsyncVal;
+ SHA256 sha256Val;
+};
+//________________________________________
+
+/** Like MatchedFileSHA256 - used only in .tmp files to express that the
+ file data was successfully written to the image. NB: Because this
+ derives from MatchedFileSHA256 and because of the implementation of
+ JigdoDesc::operator==, MatchedFileSHA256's and WrittenFileSHA256's will
+ compare equal if their data fields are identical. */
+class JigdoDesc::WrittenFileSHA256 : public MatchedFileSHA256 {
+public:
+ WrittenFileSHA256(uint64 o, uint64 s, const RsyncSum64& r, const SHA256& m)
+ : MatchedFileSHA256(o, s, r, m) { }
+ // Implicit cast to allow MatchedFileSHA256 and WrittenFileSHA256 to compare equal
+ inline bool operator==(const JigdoDesc& x) const;
+ Type type() const { return WRITTEN_FILE_SHA256; }
+ virtual ostream& put(ostream& s) const;
+
+ template<class Iterator>
+ inline Iterator serialize(Iterator i) const;
+ inline size_t serialSizeOf() const;
+};
+//______________________________________________________________________
+
/** Class allowing JigdoDesc to convey information back to the caller.
The default versions of the methods do nothing at all (except for
error(), which prints the error to cerr) - you need to supply an
not be two contiguous Unmatched regions - this is not checked.
Similarly, the length of the ImageInfo part must match the
accumulated lengths of the other parts. */
- bostream& put(bostream& file, MD5Sum* md = 0) const;
+ bostream& put(bostream& file, MD5Sum* md = 0, SHA256Sum* sd = 0) const;
/** List contents of a JigdoDescVec to a stream in human-readable format. */
void list(ostream& s) throw();
: sizeVal(s), md5Val(m), blockLengthVal(b) { }
JigdoDesc::ImageInfo::ImageInfo(uint64 s, const MD5Sum& m, size_t b)
: sizeVal(s), md5Val(m), blockLengthVal(b) { }
+JigdoDesc::ImageInfoSHA256::ImageInfoSHA256(uint64 s, const SHA256& m, size_t b)
+ : sizeVal(s), sha256Val(m), blockLengthVal(b) { }
+JigdoDesc::ImageInfoSHA256::ImageInfoSHA256(uint64 s, const SHA256Sum& m, size_t b)
+ : sizeVal(s), sha256Val(m), blockLengthVal(b) { }
JigdoDesc::MatchedFile::MatchedFile(uint64 o, uint64 s, const RsyncSum64& r,
const MD5& m)
: offsetVal(o), sizeVal(s), rsyncVal(r), md5Val(m) { }
+JigdoDesc::MatchedFileSHA256::MatchedFileSHA256(uint64 o, uint64 s, const RsyncSum64& r,
+ const SHA256& m)
+ : offsetVal(o), sizeVal(s), rsyncVal(r), sha256Val(m) { }
JigdoDesc::MatchedFile::MatchedFile(uint64 o, uint64 s, const RsyncSum64& r,
const MD5Sum& m)
: offsetVal(o), sizeVal(s), rsyncVal(r), md5Val(m) { }
+JigdoDesc::MatchedFileSHA256::MatchedFileSHA256(uint64 o, uint64 s, const RsyncSum64& r,
+ const SHA256Sum& m)
+ : offsetVal(o), sizeVal(s), rsyncVal(r), sha256Val(m) { }
//________________________________________
else return size() == i->size() && md5() == i->md5();
}
+bool JigdoDesc::ImageInfoSHA256::operator==(const JigdoDesc& x) const {
+ const ImageInfoSHA256* i = dynamic_cast<const ImageInfoSHA256*>(&x);
+ if (i == 0) return false;
+ else return size() == i->size() && sha256() == i->sha256();
+}
+
bool JigdoDesc::UnmatchedData::operator==(const JigdoDesc& x) const {
const UnmatchedData* u = dynamic_cast<const UnmatchedData*>(&x);
if (u == 0) return false;
&& md5() == m->md5();
}
+bool JigdoDesc::MatchedFileSHA256::operator==(const JigdoDesc& x) const {
+ const MatchedFileSHA256* m = dynamic_cast<const MatchedFileSHA256*>(&x);
+ if (m == 0) return false;
+ else return offset() == m->offset() && size() == m->size()
+ && sha256() == m->sha256();
+}
+
bool JigdoDesc::WrittenFile::operator==(const JigdoDesc& x) const {
// NB MatchedFile and WrittenFile considered equal!
const MatchedFile* m = dynamic_cast<const MatchedFile*>(&x);
else return offset() == m->offset() && size() == m->size()
&& md5() == m->md5();
}
+
+bool JigdoDesc::WrittenFileSHA256::operator==(const JigdoDesc& x) const {
+ // NB MatchedFileSHA256 and WrittenFileSHA256 considered equal!
+ const MatchedFileSHA256* m = dynamic_cast<const MatchedFileSHA256*>(&x);
+ if (m == 0) return false;
+ else return offset() == m->offset() && size() == m->size()
+ && sha256() == m->sha256();
+}
//________________________________________
inline bistream& operator>>(bistream& s, JigdoDescVec& v) {
}
size_t JigdoDesc::ImageInfo::serialSizeOf() const { return 1 + 6 + 16 + 4; }
+template<class Iterator>
+Iterator JigdoDesc::ImageInfoSHA256::serialize(Iterator i) const {
+ i = serialize1(IMAGE_INFO, i);
+ i = serialize6(size(), i);
+ i = ::serialize(sha256(), i);
+ i = serialize4(blockLength(), i);
+ return i;
+}
+size_t JigdoDesc::ImageInfoSHA256::serialSizeOf() const { return 1 + 6 + 32 + 4; }
+
template<class Iterator>
Iterator JigdoDesc::UnmatchedData::serialize(Iterator i) const {
i = serialize1(UNMATCHED_DATA, i);
}
size_t JigdoDesc::MatchedFile::serialSizeOf() const { return 1 + 6 + 8 + 16;}
+template<class Iterator>
+Iterator JigdoDesc::MatchedFileSHA256::serialize(Iterator i) const {
+ i = serialize1(MATCHED_FILE, i);
+ i = serialize6(size(), i);
+ i = ::serialize(rsync(), i);
+ i = ::serialize(sha256(), i);
+ return i;
+}
+size_t JigdoDesc::MatchedFileSHA256::serialSizeOf() const { return 1 + 6 + 8 + 32;}
+
template<class Iterator>
Iterator JigdoDesc::WrittenFile::serialize(Iterator i) const {
i = serialize1(WRITTEN_FILE, i);
}
size_t JigdoDesc::WrittenFile::serialSizeOf() const { return 1 + 6 + 8 + 16;}
+template<class Iterator>
+Iterator JigdoDesc::WrittenFileSHA256::serialize(Iterator i) const {
+ i = serialize1(WRITTEN_FILE_SHA256, i);
+ i = serialize6(size(), i);
+ i = ::serialize(rsync(), i);
+ i = ::serialize(sha256(), i);
+ return i;
+}
+size_t JigdoDesc::WrittenFileSHA256::serialSizeOf() const { return 1 + 6 + 8 + 32;}
+
+
#endif
files.reserve((files.size() + 16) % 16);
files.push_back(new JigdoDesc::ImageInfo(len, md5, blockLength));
}
+ // Alternative: insert in DESC section: information about whole image
+ inline void imageInfoSha256(uint64 len, const SHA256Sum& sha256, size_t blockLength) {
+ files.reserve((files.size() + 32) % 32);
+ files.push_back(new JigdoDesc::ImageInfoSHA256(len, sha256, blockLength));
+ }
// Insert in DESC section: info about some data that was not matched
inline void unmatchedData(uint64 len) {
JigdoDesc::UnmatchedData* u;
files.push_back(new JigdoDesc::MatchedFile(offset, len, r, md5));
offset += len;
}
+ // Alternative: Insert in DESC section: information about a file that matched
+ inline void matchedFileSHA256(uint64 len, const RsyncSum64& r,
+ const SHA256Sum& sha256) {
+ files.reserve((files.size() + 32) % 32);
+ files.push_back(new JigdoDesc::MatchedFileSHA256(offset, len, r, sha256));
+ offset += len;
+ }
inline bostream& put(bostream& s, MD5Sum* md) {
files.put(s, md);
return s;
them, anyway. */
inline bool MkTemplate::scanFiles(size_t blockLength, uint32 blockMask,
- size_t md5BlockLength) {
+ size_t csumBlockLength) {
bool result = SUCCESS;
- cache->setParams(blockLength, md5BlockLength);
+ cache->setParams(blockLength, csumBlockLength);
FileVec::iterator hashPos;
for (JigdoCache::iterator file = cache->begin();
//________________________________________
void MkTemplate::checkRsyncSumMatch2(const size_t blockLen,
- const size_t back, const size_t md5BlockLength, uint64& nextEvent,
+ const size_t back, const size_t csumBlockLength, uint64& nextEvent,
FilePart* file) {
/* Don't schedule match if its startOff (== off - blockLen) is impossible.
/* Rolling rsum matched - schedule an MD5Sum match. NB: In extreme cases,
nextEvent may be equal to off */
x->setStartOffset(off - blockLen);
- size_t eventLen = (file->size() < md5BlockLength ?
- file->size() : md5BlockLength);
+ size_t eventLen = (file->size() < csumBlockLength ?
+ file->size() : csumBlockLength);
x->setNextEvent(matches, x->startOffset() + eventLen);
debug(" %1: Head of %2 match at offset %3, my next event %4",
off, file->leafName(), x->startOffset(), x->nextEvent());
appropriate entry in "matches". */
void MkTemplate::checkRsyncSumMatch(const RsyncSum64& sum,
const uint32& bitMask, const size_t blockLen, const size_t back,
- const size_t md5BlockLength, uint64& nextEvent) {
+ const size_t csumBlockLength, uint64& nextEvent) {
typedef const vector<FilePart*> FVec;
FVec& hashEntry = block[sum.getHi() & bitMask];
const RsyncSum64* fileSum = file->getRsyncSum(cache);
if (fileSum != 0 && *fileSum == sum)
// Insert new partial file match in "matches" queue
- checkRsyncSumMatch2(blockLen, back, md5BlockLength, nextEvent, file);
+ checkRsyncSumMatch2(blockLen, back, csumBlockLength, nextEvent, file);
++i;
} while (i != e);
return;
buf[(data+stillBuffered-1)%bufferLength] */
bool MkTemplate::checkMD5Match(byte* const buf,
const size_t bufferLength, const size_t data,
- const size_t md5BlockLength, uint64& nextEvent,
+ const size_t csumBlockLength, uint64& nextEvent,
const size_t stillBuffered, Desc& desc) {
PartialMatch* x = matches->front();
Paranoid(x != 0 && matches->nextEvent() == off);
md.finishForReuse();
//____________________
- const MD5* xfileSum = x->file()->getSums(cache, x->blockNumber());
+ const MD5* xfileSum = x->file()->getMD5Sums(cache, x->blockNumber());
if (debug)
debug("checkMD5Match?: image %1, file %2 block #%3 %4",
md.toString(), x->file()->leafName(), x->blockNumber(),
// Still some more to go - update x and its position in queue
x->setBlockOffset(data);
x->setBlockNumber(x->blockNumber() + 1);
- x->setNextEvent(matches, min(x->nextEvent() + md5BlockLength,
+ x->setNextEvent(matches, min(x->nextEvent() + csumBlockLength,
x->startOffset() + x->file()->size()));
nextEvent = min(nextEvent, x->nextEvent());
debug("checkMD5Match: match and more to go, next at off %1",
void MkTemplate::scanImage_mainLoop_fastForward(uint64 nextEvent,
RsyncSum64* rsum, byte* buf, size_t* data, size_t* n, size_t* rsumBack,
size_t bufferLength, size_t blockLength, uint32 blockMask,
- size_t md5BlockLength) {
+ size_t csumBlockLength) {
# if 0
// Simple version
*rsumBack = modAdd(*rsumBack, 1, bufferLength);
if (((off - blockLength) & sectorMask) == 0) {
checkRsyncSumMatch(*rsum, blockMask, blockLength, *rsumBack,
- md5BlockLength, nextEvent);
+ csumBlockLength, nextEvent);
sectorMask = sectorLength - 1;
Paranoid(matches->empty()
|| matches->front()->startOffset() >= unmatchedStart);
if (off == nextAlignedOff) {
Paranoid(((off - blockLength) & sectorMask) == 0);
checkRsyncSumMatch(*rsum, blockMask, blockLength, *rsumBack,
- md5BlockLength, nextEvent);
+ csumBlockLength, nextEvent);
Paranoid(matches->empty()
|| matches->front()->startOffset() >= unmatchedStart);
sectorMask = sectorLength - 1;
Treat buf as a circular buffer. Read new data into at most half the
buffer. Calculate a rolling checksum covering blockLength bytes. When it
matches an entry in block, start calculating MD5Sums of blocks of length
- md5BlockLength.
+ csumBlockLength.
Since both image and templ can be non-seekable, we run into a problem in
the following case: After the initial RsyncSum match, a few of the
- md5BlockLength-sized chunks of one input file were matched, but not all,
+ csumBlockLength-sized chunks of one input file were matched, but not all,
so in the end, there is no match. Consequently, we would now need to
re-read that part of the image and pump it through zlib to templ - but we
can't if the image is stdin! Solution: Since we know that the MD5Sum of a
block matched part of an input file, we can re-read from there. */
inline bool MkTemplate::scanImage(byte* buf, size_t bufferLength,
- size_t blockLength, uint32 blockMask, size_t md5BlockLength,
+ size_t blockLength, uint32 blockMask, size_t csumBlockLength,
MD5Sum& templMd5Sum) {
bool result = SUCCESS;
/* Cause input files to be analysed */
- if (scanFiles(blockLength, blockMask, md5BlockLength))
+ if (scanFiles(blockLength, blockMask, csumBlockLength))
result = FAILURE;
/* Initialise rolling sums with blockSize bytes 0x7f, and do the same with
} else {
dataOld = data - dataOld; off += dataOld; n -= dataOld;
checkRsyncSumMatch(rsum, blockMask, blockLength, rsumBack,
- md5BlockLength, nextEvent);
+ csumBlockLength, nextEvent);
}
if (matches->full()) break;
}
/* Look for matches of rsum. If found, insert appropriate
entry in matches list and maybe modify nextEvent. */
checkRsyncSumMatch(rsum, blockMask, blockLength, rsumBack,
- md5BlockLength, nextEvent);
+ csumBlockLength, nextEvent);
/* We mustn't by accident schedule an event for a part of
the image that has already been flushed out of the
// Innermost loop - MATCHES IS FULL
scanImage_mainLoop_fastForward(nextEvent, &rsum, buf, &data, &n,
&rsumBack, bufferLength, blockLength, blockMask,
- md5BlockLength);
+ csumBlockLength);
} // endif (matches->full())
if (matches->empty())
debug(" %1: Event, matches empty", off);
while (!matches->empty() && matches->nextEvent() == off) {
size_t stillBuffered = bufferLength - n;
if (stillBuffered > off) stillBuffered = off;
- if (checkMD5Match(buf, bufferLength, data, md5BlockLength,
+ if (checkMD5Match(buf, bufferLength, data, csumBlockLength,
nextEvent, stillBuffered, desc))
return FAILURE; // no recovery possible, exit immediately
}
be able to read readAmount bytes into one buffer half in one go;
must be able to start calculating an MD5Sum at a position that is
blockLength bytes back in input; must be able to write out at
- least previous md5BlockLength bytes in case there is no match. */
+ least previous csumBlockLength bytes in case there is no match. */
size_t bufferLength = 2 *
(max_MD5Len_blockLen > readAmount ? max_MD5Len_blockLen : readAmount);
// Avoid reading less bytes than readAmount at any time
debug("Nr of files: %1 (%2 bits)", fileCount, blockBits);
debug("Total bytes: %1", fileSizeTotal);
debug("blockLength: %1", cache->getBlockLen());
- debug("md5BlockLen: %1", cache->getMD5BlockLen());
+ debug("csumBlockLen: %1", cache->getMD5BlockLen());
debug("bufLen (kB): %1", bufferLength/1024);
debug("zipQual: %1", zipQual);
}
void finalizeJigdo(const string& imageLeafName,
const string& templLeafName, const MD5Sum& templMd5Sum);
INLINE bool scanFiles(size_t blockLength, uint32 blockMask,
- size_t md5BlockLength);
+ size_t csumBlockLength);
INLINE bool scanImage(byte* buf, size_t bufferLength, size_t blockLength,
- uint32 blockMask, size_t md5BlockLength, MD5Sum&);
+ uint32 blockMask, size_t csumBlockLength, MD5Sum&);
static INLINE void insertInTodo(PartialMatchQueue& matches,
PartialMatch* x);
void checkRsyncSumMatch2(const size_t blockLen, const size_t back,
- const size_t md5BlockLength, uint64& nextEvent, FilePart* file);
+ const size_t csumBlockLength, uint64& nextEvent, FilePart* file);
INLINE void checkRsyncSumMatch(const RsyncSum64& sum,
const uint32& bitMask, const size_t blockLen, const size_t back,
- const size_t md5BlockLength, uint64& nextEvent);
+ const size_t csumBlockLength, uint64& nextEvent);
INLINE bool checkMD5Match(byte* const buf,
const size_t bufferLength, const size_t data,
- const size_t md5BlockLength, uint64& nextEvent,
+ const size_t csumBlockLength, uint64& nextEvent,
const size_t stillBuffered, Desc& desc);
INLINE bool checkMD5Match_mismatch(const size_t stillBuffered,
PartialMatch* x, Desc& desc);
INLINE void scanImage_mainLoop_fastForward(uint64 nextEvent,
RsyncSum64* rsum, byte* buf, size_t* data, size_t* n, size_t* rsumBack,
size_t bufferLength, size_t blockLength, uint32 blockMask,
- size_t md5BlockLength);
+ size_t csumBlockLength);
INLINE bool matchExecCommands(PartialMatch* x);
inline void debugRangeInfo(uint64 start, uint64 end, const char* msg,
//______________________________________________________________________
#if HAVE_LIBDB
+
+// How much checksum data do we have per checksum entry?
+#define CSUM_SIZE (16 + 32) // md5 size + sha256 size
+
/* Interpret a string of bytes (out of the file cache) like this:
4 blockLength (of rsync sum)
- 4 md5BlockLength
+ 4 csumBlockLength
4 blocks (number of valid md5 blocks in this entry), curr. always >0
8 rsyncSum of file start (only valid if blocks > 0)
16 fileMD5Sum (only valid if
- blocks == (fileSize+md5BlockLength-1)/md5BlockLength )
+ blocks == (fileSize+csumBlockLength-1)/csumBlockLength )
+ 32 fileSHA256Sum (only valid if
+ blocks == (fileSize+csumBlockLength-1)/csumBlockLength )
followed by n entries:
- 16 md5sum of block of size md5BlockLength
+ 16 md5sum of block of size csumBlockLength
+ 32 sha256sum of block of size csumBlockLength
- If stored md5BlockLength doesn't match supplied length, do nothing.
+ If stored csumBlockLength doesn't match supplied length, do nothing.
Otherwise, restore *this from cached data and return cached
blockLength (0 if not cached). The caller needs to make sure the
blockLength matches.
iter is missing. It only creates a cache entry. */
size_t FilePart::unserializeCacheEntry(const byte* data, size_t dataSize,
- size_t md5BlockLength){
- Assert(dataSize > PART_MD5SUM);
+ size_t csumBlockLength){
+ Assert(dataSize > PART_MD5SUMS);
// The resize() must have been made by the caller
- Paranoid(sums.size() == (size() + md5BlockLength - 1) / md5BlockLength);
+ Paranoid(MD5sums.size() == (size() + csumBlockLength - 1) / csumBlockLength);
+ Paranoid(SHA256sums.size() == (size() + csumBlockLength - 1) / csumBlockLength);
size_t cachedBlockLength;
data = unserialize4(cachedBlockLength, data);
- size_t cachedMd5BlockLength;
- data = unserialize4(cachedMd5BlockLength, data);
- if (cachedMd5BlockLength != md5BlockLength) return 0;
+ size_t cachedCsumBlockLength;
+ data = unserialize4(cachedCsumBlockLength, data);
+ if (cachedCsumBlockLength != csumBlockLength) return 0;
size_t blocks;
data = unserialize4(blocks, data);
// Ignore strange-looking entries
- if (blocks * serialSizeOf(md5Sum) != dataSize - PART_MD5SUM
- || blocks == 0) {
- if (blocks == 0) debug("ERR #blocks == 0");
- else debug("ERR wrong entry size (%1 vs %2)",
- blocks * 16, dataSize - PART_MD5SUM);
+ if (blocks == 0) {
+ debug("ERR #blocks == 0");
+ return 0;
+ }
+ if (dataSize - PART_MD5SUMS != (blocks * CSUM_SIZE)) {
+ debug("ERR wrong entry size (%1 vs %2)",
+ blocks * CSUM_SIZE, dataSize - PART_MD5SUMS);
return 0;
}
Paranoid(serialSizeOf(rsyncSum) == 8);
data = unserialize(rsyncSum, data);
Paranoid(serialSizeOf(md5Sum) == 16);
+ Paranoid(serialSizeOf(sha256Sum) == 32);
// All blocks of file present?
- if (blocks == sums.size()) {
+ if (blocks == MD5sums.size() + SHA256sums.size()) {
setFlag(MD_VALID);
data = unserialize(md5Sum, data);
+ data = unserialize(sha256Sum, data);
} else {
clearFlag(MD_VALID);
- data += 16;
+ data += CSUM_SIZE;
}
- // Read md5 sums of individual chunks of file
- vector<MD5>::iterator sum = sums.begin();
+ // Read md5sums of individual chunks of file
+ vector<MD5>::iterator sum = MD5sums.begin();
for (size_t i = blocks; i > 0; --i) {
data = unserialize(*sum, data);
++sum;
}
+ vector<SHA256>::iterator sum2 = SHA256sums.begin();
+ for (size_t i = blocks; i > 0; --i) {
+ data = unserialize(*sum2, data);
+ ++sum2;
+ }
return cachedBlockLength;
}
struct FilePart::SerializeCacheEntry {
SerializeCacheEntry(const FilePart& f, JigdoCache* c, size_t blockLen,
size_t md5Len)
- : file(f), cache(c), blockLength(blockLen), md5BlockLength(md5Len) { }
+ : file(f), cache(c), blockLength(blockLen), csumBlockLength(md5Len) { }
const FilePart& file;
JigdoCache* cache;
size_t blockLength;
- size_t md5BlockLength;
+ size_t csumBlockLength;
size_t serialSizeOf() {
- return PART_MD5SUM + (file.mdValid() ? file.sums.size() * 16 : 16);
+ return PART_MD5SUMS + (file.mdValid() ? file.MD5sums.size() * CSUM_SIZE : CSUM_SIZE);
}
void operator()(byte* data) {
Paranoid(file.getFlag(TO_BE_WRITTEN));
// If empty(), shouldn't have been marked TO_BE_WRITTEN:
- Assert(!file.sums.empty());
+ Assert(!file.MD5sums.empty());
+ Assert(!file.SHA256sums.empty());
data = serialize4(blockLength, data);
- data = serialize4(md5BlockLength, data);
+ data = serialize4(csumBlockLength, data);
// Nr of valid blocks - either 1 or all
- size_t blocks = (file.mdValid() ? file.sums.size() : 1);
+ size_t blocks = (file.mdValid() ? file.MD5sums.size() : 1);
data = serialize4(blocks, data);
data = serialize(file.rsyncSum, data);
data = serialize(file.md5Sum, data);
- // Write md5 sums of individual chunks of file
- vector<MD5>::const_iterator sum = file.sums.begin();
+ data = serialize(file.sha256Sum, data);
+ // Write md5sums of individual chunks of file
+ vector<MD5>::const_iterator sum = file.MD5sums.begin();
for (size_t i = blocks; i > 0; --i) {
data = serialize(*sum, data);
++sum;
}
+ // Write md5sums of individual chunks of file
+ vector<SHA256>::const_iterator sum2 = file.SHA256sums.begin();
+ for (size_t i = blocks; i > 0; --i) {
+ data = serialize(*sum2, data);
+ ++sum2;
+ }
}
};
#endif
#if HAVE_LIBDB
JigdoCache::JigdoCache(const string& cacheFileName, size_t expiryInSeconds,
size_t bufLen, ProgressReporter& pr)
- : blockLength(0), md5BlockLength(0), checkFiles(true), files(), nrOfFiles(0),
+ : blockLength(0), csumBlockLength(0), checkFiles(true), files(), nrOfFiles(0),
locationPaths(), readAmount(bufLen), buffer(), reporter(pr),
cacheExpiry(expiryInSeconds) {
cacheFile = 0;
#else
JigdoCache::JigdoCache(const string&, size_t, size_t bufLen,
ProgressReporter& pr)
- : blockLength(0), md5BlockLength(0), files(), nrOfFiles(0),
+ : blockLength(0), csumBlockLength(0), files(), nrOfFiles(0),
locationPaths(), readAmount(bufLen), buffer(), reporter(pr) { }
#endif
//______________________________________________________________________
if (i->deleted() || !i->getFlag(FilePart::TO_BE_WRITTEN)) continue;
debug("Writing %1", i->leafName());
FilePart::SerializeCacheEntry serializer(*i, this, blockLength,
- md5BlockLength);
+ csumBlockLength);
try {
cacheFile->insert(serializer, serializer.serialSizeOf(),
i->leafName(), i->mtime(), i->size());
}
//______________________________________________________________________
-/* Either reads data for the first MD5 block and creates sums[0] and
- rsyncSum, or reads whole file and creates all sums[] entries and
- rsyncSum and the whole file's MD5 sum. */
-const MD5* FilePart::getSumsRead(JigdoCache* c, size_t blockNr) {
+/* Either:
+
+ 1. read data for the first block and create rsyncSum, MD5sums[0]
+ SHA256sums[0], or;
+
+ 2. read the whole file and create rsyncSum, plus both checksums for
+ all the blocks and both checksums for the whole file.
+*/
+
+bool FilePart::getChecksumsRead(JigdoCache* c, size_t blockNr) {
// Should do this check before calling:
- Paranoid((blockNr == 0 && sums.empty()) || !mdValid());
+ Paranoid((blockNr == 0 && MD5sums.empty() && SHA256sums.empty()) || !mdValid());
// Do not forget to setParams() before calling this!
- Assert(c->md5BlockLength != 0);
+ Assert(c->csumBlockLength != 0);
const size_t thisBlockLength = c->blockLength;
- sums.resize((size() + c->md5BlockLength - 1) / c->md5BlockLength);
+ MD5sums.resize((size() + c->csumBlockLength - 1) / c->csumBlockLength);
+ SHA256sums.resize((size() + c->csumBlockLength - 1) / c->csumBlockLength);
//____________________
# if HAVE_LIBDB
const byte* data;
size_t dataSize;
try {
- /* Unserialize will do nothing if md5BlockLength differs. If
- md5BlockLength matches, but returned blockLength doesn't, we
+ /* Unserialize will do nothing if csumBlockLength differs. If
+ csumBlockLength matches, but returned blockLength doesn't, we
need to re-read the first block. */
if (c->cacheFile->find(data, dataSize, leafName(), size(), mtime())
.ok()) {
debug("%1 found, want block#%2", leafName(), blockNr);
size_t cachedBlockLength = unserializeCacheEntry(data, dataSize,
- c->md5BlockLength);
+ c->csumBlockLength);
// Was all necessary data in cache? Yes => return it now.
if (cachedBlockLength == thisBlockLength
&& (blockNr == 0 || mdValid())) {
debug("%1 loaded, blockLen (%2) matched, %3/%4 in cache",
- leafName(), thisBlockLength, (mdValid() ? sums.size() : 1),
- sums.size());
- return &sums[blockNr];
+ leafName(), thisBlockLength, (mdValid() ? MD5sums.size() : 1),
+ MD5sums.size());
+ return true;
}
/* blockLengths didn't match and/or the cache only contained
- the first md5 sum while we asked for a later one. It's as
- if we never queried the cache, except for the case when we
- need to re-read the first block because the blockLength
- changed, but *all* blocks' md5sums were in the cache. */
+ the checksum for the first block while we asked for a later
+ one. It's as if we never queried the cache, except for the
+ case when we need to re-read the first block because the
+ blockLength changed, but *all* blocks' checksums were in the
+ cache. */
debug("%1 loaded, NO match (blockLen %2 vs %3), %4/%5 in cache",
leafName(), cachedBlockLength, thisBlockLength,
- (mdValid() ? sums.size() : 1), sums.size());
+ (mdValid() ? MD5sums.size() : 1), MD5sums.size());
}
} catch (DbError e) {
string err = subst(_("Error accessing cache: %1"), e.message);
setFlag(TO_BE_WRITTEN);
// Allocate or resize buffer, or do nothing if already right size
- c->buffer.resize(c->readAmount > c->md5BlockLength ?
- c->readAmount : c->md5BlockLength);
+ c->buffer.resize(c->readAmount > c->csumBlockLength ?
+ c->readAmount : c->csumBlockLength);
//______________________________
- // Read data and create sums
+ // Read data and create checksums
uint64 off = 0; // File offset of first byte in buf
// Nr of bytes before we are to reset() md
- size_t mdLeft = c->md5BlockLength;
+ size_t mdLeft = c->csumBlockLength;
/* Call reporter once off reaches this value - only report something
- if scanning >1 md5 block */
+ if scanning >1 checksum block */
uint64 nextReport = mdLeft;
MD5Sum md;
md5Sum.reset();
- vector<MD5>::iterator sum = sums.begin();
+ vector<MD5>::iterator sum = MD5sums.begin();
+ SHA256Sum sd;
+ sha256Sum.reset();
+ vector<SHA256>::iterator sum2 = SHA256sums.begin();
//____________________
- // Calculate RsyncSum of head of file and MD5Sums for all blocks
+ // Calculate RsyncSum of head of file and MD5 and SHA256 for all blocks
- Assert(thisBlockLength <= c->md5BlockLength);
+ Assert(thisBlockLength <= c->csumBlockLength);
byte* buf = &c->buffer[0];
byte* bufpos = buf;
byte* bufend = buf + (c->readAmount > thisBlockLength ?
debug("Read %1", nn);
}
size_t n = bufpos - buf;
+
// Create RsyncSum of 1st bytes of file, or leave at 0 if file too small
rsyncSum.reset();
- if (n >= thisBlockLength) rsyncSum.addBack(buf, thisBlockLength);
+ if (n >= thisBlockLength)
+ rsyncSum.addBack(buf, thisBlockLength);
//__________
while (true) { // Will break out if error or whole file read
// n is number of valid bytes in buf[]
off += n;
- if (off > size()) break; // Argh - file size changed
+ if (off > size())
+ break; // Argh - file size changed
if (off >= nextReport) {
c->reporter.scanningFile(this, off);
nextReport += REPORT_INTERVAL;
}
- // Create MD5 for chunks of size md5BlockLength
+ // Create checksums for chunks of size csumBlockLength
if (n < mdLeft) {
md.update(buf, n);
+ sd.update(buf, n);
mdLeft -= n;
} else {
md.update(buf, mdLeft);
+ sd.update(buf, mdLeft);
byte* cur = buf + mdLeft;
size_t nn = n - mdLeft;
do {
md.finishForReuse();
+ sd.finishForReuse();
debug("%1: mdLeft (0), switching to next md at off %2, left %3, "
- "writing sum#%4: %5", name, off - n + cur - buf, nn,
- sum - sums.begin(), md.toString());
- Paranoid(sum != sums.end());
+ "writing sum#%4: %5/%6", name, off - n + cur - buf, nn,
+ sum - MD5sums.begin(), md.toString(), sd.toString());
+ Paranoid(sum != MD5sums.end());
*sum = md;
+ *sum2 = sd;
++sum;
- size_t m = (nn < c->md5BlockLength ? nn : c->md5BlockLength);
+ ++sum2;
+ size_t m = (nn < c->csumBlockLength ? nn : c->csumBlockLength);
md.reset().update(cur, m);
- cur += m; nn -= m;
- mdLeft = c->md5BlockLength - m;
+ sd.reset().update(cur, m);
+ cur += m;
+ nn -= m;
+ mdLeft = c->csumBlockLength - m;
} while (nn > 0);
}
md5Sum.update(buf, n); // Create MD5 for the whole file
+ sha256Sum.update(buf, n); // Create MD5 for the whole file
+
+ if (blockNr == 0 && sum != MD5sums.begin())
+ break; // Only wanted 1st block
- if (blockNr == 0 && sum != sums.begin()) break; // Only wanted 1st block
- if (!input) break; // End of file or error
+ if (!input)
+ break; // End of file or error
// Read more data
readBytes(input, buf, c->readAmount);
} // Endwhile (true), will break out if error or whole file read
- Paranoid(sum != sums.end() // >=1 trailing bytes
- || mdLeft == c->md5BlockLength); // 0 trailing bytes
+ Paranoid(sum != MD5sums.end() // >=1 trailing bytes
+ || mdLeft == c->csumBlockLength); // 0 trailing bytes
if (off == size() && input.eof()) {
// Whole file was read
c->reporter.scanningFile(this, size()); // 100% scanned
- if (mdLeft < c->md5BlockLength) {
+ if (mdLeft < c->csumBlockLength) {
(*sum) = md.finish(); // Digest of trailing bytes
- debug("%1: writing trailing sum#%2: %3",
- name, sum - sums.begin(), md.toString());
+ (*sum2) = sd.finish(); // Digest of trailing bytes
+ debug("%1: writing trailing sum#%2: %3/%4",
+ name, sum - MD5sums.begin(), md.toString(), sd.toString());
}
md5Sum.finish(); // Digest of whole file
+ sha256Sum.finish(); // Digest of whole file
setFlag(MD_VALID);
- return &sums[blockNr];
- } else if (blockNr == 0 && sum != sums.begin()) {
+ return true;
+ } else if (blockNr == 0 && sum != MD5sums.begin()) {
// Only first md5 block of file was read
debug("%1: file header read, sum#0 written", name);
# if DEBUG
md5Sum.finish(); // else failed assert in FilePart::SerializeCacheEntry
+ sha256Sum.finish();
# else
md5Sum.abort(); // Saves the memory until whole file is read
+ sha256Sum.abort(); // Saves the memory until whole file is read
# endif
- return &sums[0];
+ return true;
}
//____________________
//______________________________________________________________________
const MD5Sum* FilePart::getMD5SumRead(JigdoCache* c) {
- if (getSumsRead(c,
- (fileSize + c->md5BlockLength - 1) / c->md5BlockLength - 1)
- == 0) return 0;
+ if (!getChecksumsRead(c, (fileSize + c->csumBlockLength - 1) / c->csumBlockLength - 1))
+ return 0;
Paranoid(mdValid());
return &md5Sum;
}
//______________________________________________________________________
-void JigdoCache::setParams(size_t blockLen, size_t md5BlockLen) {
- if (blockLen == blockLength && md5BlockLen == md5BlockLength) return;
+const SHA256Sum* FilePart::getSHA256SumRead(JigdoCache* c) {
+ if (!getChecksumsRead(c, (fileSize + c->csumBlockLength - 1) / c->csumBlockLength - 1))
+ return 0;
+ Paranoid(mdValid());
+ return &sha256Sum;
+}
+//______________________________________________________________________
+
+void JigdoCache::setParams(size_t blockLen, size_t csumBlockLen) {
+ if (blockLen == blockLength && csumBlockLen == csumBlockLength) return;
blockLength = blockLen;
- md5BlockLength = md5BlockLen;
- Assert(blockLength <= md5BlockLength);
+ csumBlockLength = csumBlockLen;
+ Assert(blockLength <= csumBlockLength);
for (list<FilePart>::iterator file = files.begin(), end = files.end();
file != end; ++file) {
- file->sums.resize(0);
+ file->MD5sums.resize(0);
+ file->SHA256sums.resize(0);
}
}
//______________________________________________________________________
void JigdoCache::addFile(const string& name) {
// Do not forget to setParams() before calling this!
- Assert(md5BlockLength != 0);
+ Assert(csumBlockLength != 0);
// Assumes nonempty filenames
Paranoid(name.length() > 0);
#include <cachefile.hh>
#include <debug.hh>
#include <md5sum.hh>
+#include <sha256sum.hh>
#include <recursedir.fh>
#include <rsyncsum.hh>
#include <scan.fh>
consists of. The object is also responsible for scanning the file
and creating a) an RsyncSum64 of the first blockLength bytes (as
determined by JigdoCache). Caching is done in a "lazy" manner;
- first, only the file size and name is read, then the MD5 sum of
+ first, only the file size and name is read, then the checksum of
the file's first block as well as the Rsync sum is calculated, and
- only when the 2nd MD5 sum or the entire file's sum is requested,
+ only when the 2nd checksum or the entire file's sum is requested,
the file is scanned til EOF. */
class FilePart {
/** Objects are only created by JigdoCache */
inline time_t mtime() const;
/** Returns null ptr if error and you don't throw it in your
JigdoCache error handler */
- inline const MD5* getSums(JigdoCache* c, size_t blockNr);
+ inline const MD5* getMD5Sums(JigdoCache* c, size_t blockNr);
/** Returns null ptr if error and you don't throw it */
inline const MD5Sum* getMD5Sum(JigdoCache* c);
+ /** Returns null ptr if error and you don't throw it in your
+ JigdoCache error handler */
+ inline const SHA256* getSHA256Sums(JigdoCache* c, size_t blockNr);
+ /** Returns null ptr if error and you don't throw it */
+ inline const SHA256Sum* getSHA256Sum(JigdoCache* c);
/** Returns null ptr if error and you don't throw it */
inline const RsyncSum64* getRsyncSum(JigdoCache* c);
inline FilePart(LocationPathSet::iterator p, string rest, uint64 fSize,
time_t fMtime);
- /* Called when the methods getSums/getMD5Sum() need data read from
- file. Might return null. */
- const MD5* getSumsRead(JigdoCache* c, size_t blockNr);
+ /* Called when the methods getMD5Sums/getMD5Sum() need data read from
+ file. Might return false on failure. */
+ bool getChecksumsRead(JigdoCache* c, size_t blockNr);
+
const MD5Sum* getMD5SumRead(JigdoCache* c);
+ const SHA256Sum* getSHA256SumRead(JigdoCache* c);
//__________
/* There are 3 states of a FilePart:
- a) sums.empty(): File has not been read from so far
- b) !sums.empty() && !mdValid: sums[0] and rsyncSum are valid
- c) !sums.empty() && mdValid: all sums[] and rsyncSum and md5Sum valid*/
+ a) MD5sums.empty(): File has not been read from so far
+ b) !MD5sums.empty() && !mdValid: MD5sums[0] and rsyncSum are valid
+ c) !MD5sums.empty() && mdValid: all MD5sums[] and rsyncSum and md5Sum valid*/
LocationPathSet::iterator path;
string pathRest; // further dir names after "path", and leafname of file
/* RsyncSum64 of the first MkTemplate::blockLength bytes of the
file. */
RsyncSum64 rsyncSum;
- bool rsyncValid() const { return sums.size() > 0; }
+ bool rsyncValid() const { return MD5sums.size() > 0; }
- /* File is split up into chunks of length md5BlockLength (the last
+ /* File is split up into chunks of length csumBlockLength (the last
one may be smaller) and the MD5 checksum of each is
calculated. */
- vector<MD5> sums;
+ vector<MD5> MD5sums;
+ vector<SHA256> SHA256sums;
/* Hash of complete file contents. mdValid is true iff md5Sum is
cached and valid. NB, it is possible that mdValid==true, but
- sums.size()==0, after md5BlockSize has been changed. If
- sums.size()==fileSize/md5ChunkSize, then not necessarily
+ MD5sums.size()==0, after csumBlockLength has been changed. If
+ MD5sums.size()==fileSize/csumBlockLength, then not necessarily
mdValid==true */
MD5Sum md5Sum;
+ SHA256Sum sha256Sum;
+
enum Flags {
EMPTY = 0,
- // Bit flag is set iff md5Sum contains the whole file's md5 sum
+ // Bit flag is set iff md5Sum and sha256Sum contains the whole file's checksum
MD_VALID = 1,
/* This file was looked up in the cache file (whether successfully
or not doesn't matter) - don't look it up again. */
# if HAVE_LIBDB
// Offsets for binary representation in database (see cachefile.hh)
enum {
- BLOCKLEN = 0, MD5BLOCKLEN = 4, MD5BLOCKS = 8, RSYNCSUM = 12,
- FILE_MD5SUM = 20, PART_MD5SUM = 36
+ BLOCKLEN = 0,
+ CSUMBLOCKLEN = 4,
+ CSUMBLOCKS = 8,
+ RSYNCSUM = 12,
+ FILE_MD5SUM = 20,
+ FILE_SHA256SUM = 36,
+ PART_MD5SUMS = 68
+ // Can't point directly to the offset for PART_SHA256SUM, as
+ // things are dynamic after PART_MD5SUMS
};
size_t unserializeCacheEntry(const byte* data, size_t dataSize,
- size_t md5BlockLength); // Byte stream => FilePart
+ size_t csumBlockLength); // Byte stream => FilePart
struct SerializeCacheEntry; // FilePart => byte stream
friend struct SerializeCacheEntry;
# endif
~JigdoCache();
/** Read a list of filenames from the object and store them in the
- JigdoCache. Only the file size is read during the call, Rsync/MD5
- sums are calculated later, if/when needed. */
+ JigdoCache. Only the file size is read during the call,
+ Checksums are calculated later, if/when needed. */
template <class RecurseDir>
inline void readFilenames(RecurseDir& rd);
- /** Set the sizes of cache's blockLength and md5BlockLength
+ /** Set the sizes of cache's blockLength and csumBlockLength
parameters. This means that the checksum returned by a
FilePart's getRsyncSum() method will cover the specified size.
NB: If the cache already contains files and the new parameters
are not the same as the old, the start of the files will
(eventually) have to be re-read to re-calculate the checksums
for blockLength, and the *whole* file will have to be re-read
- for a changed md5BlockLength. */
- void setParams(size_t blockLen,size_t md5BlockLen);
+ for a changed csumBlockLength. */
+ void setParams(size_t blockLen,size_t csumBlockLen);
size_t getBlockLen() const { return blockLength; }
- size_t getMD5BlockLen() const { return md5BlockLength; }
+ size_t getMD5BlockLen() const { return csumBlockLength; }
/** Amount of data that JigdoCache will attempt to read per call to
ifstream::read(), and size of buffer allocated. Minimum: 64k */
/// Default reporter: Only prints error messages to stderr
static ProgressReporter noReport;
- size_t blockLength, md5BlockLength;
+ size_t blockLength, csumBlockLength;
/* Check if files exist in the filesystem */
bool checkFiles;
cerr */
virtual void info(const string& message);
/** Called when the individual files are read. JigdoCache sometimes
- reads only the first md5BlockLength bytes and sometimes the
+ reads only the first csumBlockLength bytes and sometimes the
whole file. This is *only* called when the whole file is read
(if the file only consists of one MD5 block, it is also called).
@param file File being scanned, or null if not applicable
FilePart::FilePart(LocationPathSet::iterator p, string rest, uint64 fSize,
time_t fMtime)
: path(p), pathRest(rest), fileSize(fSize), fileMtime(fMtime),
- rsyncSum(), sums(), md5Sum(), flags(EMPTY) {
+ rsyncSum(), MD5sums(), md5Sum(), flags(EMPTY) {
//pathRest.reserve(0);
}
return fileMtime;
}
-const MD5* FilePart::getSums(JigdoCache* c, size_t blockNr) {
+const MD5* FilePart::getMD5Sums(JigdoCache* c, size_t blockNr) {
Paranoid(!deleted());
- if (mdValid() || (blockNr == 0 && !sums.empty())) return &sums[blockNr];
- else return getSumsRead(c, blockNr);
+ if ( ((blockNr > 0) || MD5sums.empty()) && !mdValid() )
+ if (!getChecksumsRead(c, blockNr))
+ return 0;
+ return &MD5sums[blockNr];
}
const MD5Sum* FilePart::getMD5Sum(JigdoCache* c) {
Paranoid(!deleted());
- if (mdValid()) return &md5Sum;
- else return getMD5SumRead(c);
+ if (mdValid())
+ return &md5Sum;
+ else
+ return getMD5SumRead(c);
+}
+
+const SHA256* FilePart::getSHA256Sums(JigdoCache* c, size_t blockNr) {
+ Paranoid(!deleted());
+ if ( ((blockNr > 0) || SHA256sums.empty()) && !mdValid() )
+ if (!getChecksumsRead(c, blockNr))
+ return 0;
+ return &SHA256sums[blockNr];
+}
+
+const SHA256Sum* FilePart::getSHA256Sum(JigdoCache* c) {
+ Paranoid(!deleted());
+ if (mdValid()) return &sha256Sum;
+ else return getSHA256SumRead(c);
}
const RsyncSum64* FilePart::getRsyncSum(JigdoCache* c) {
Paranoid(!deleted());
if (!rsyncValid()) {
- if (getSumsRead(c, 0) == 0) return 0;
+ if (! getChecksumsRead(c, 0))
+ return 0;
}
Paranoid(rsyncValid());
return &rsyncSum;
}
void FilePart::markAsDeleted(JigdoCache* c) {
- fileSize = 0; sums.resize(0); --(c->nrOfFiles);
+ fileSize = 0;
+ MD5sums.resize(0);
+ SHA256sums.resize(0);
+ --(c->nrOfFiles);
}
void FilePart::setFlag(Flags f) {
// Run MkTemplate operation over it
// Try default parameters plus 2 other random cases
size_t blockLen = 4096;
- size_t md5BlockLen = 128*1024U - 55;
+ size_t csumBlockLen = 128*1024U - 55;
size_t readAmount = 128U*1024;
for (int i = 0; i < 2; ++i) {
cerr << " case=" << tc << static_cast<char>('a' + i)
<< " blockLen=" << blockLen
- << " md5BlockLen=" << md5BlockLen
+ << " csumBlockLen=" << csumBlockLen
<< " readAmount=" << readAmount << endl;
cheatMrNice();
TortureReport reporter;
fileNames.addFile(f);
}
JigdoCache cache(cacheFile, 60*60*24, readAmount);
- cache.setParams(blockLen, md5BlockLen);
+ cache.setParams(blockLen, csumBlockLen);
while (true) {
try { cache.readFilenames(fileNames); }
catch (RecurseError e) { cerr << e.message << endl; continue; }
report << (allChecksOK ? "OK" : "FAIL") << " case=" << tc
<< ", blockLen=" << blockLen
- << ", md5BlockLen=" << md5BlockLen
+ << ", csumBlockLen=" << csumBlockLen
<< ", readAmount=" << readAmount << endl;
blockLen = 1024 + rand.get(16);
- md5BlockLen = 1024 + rand.get(18);
- if (md5BlockLen <= blockLen) md5BlockLen = blockLen + 1;
+ csumBlockLen = 1024 + rand.get(18);
+ if (csumBlockLen <= blockLen) csumBlockLen = blockLen + 1;
readAmount = 16384 + rand.get(19);
}
}
--- /dev/null
+// Taken from glibc 2.30
+
+/* Functions to compute SHA256 message digest of files or memory blocks.
+ according to the definition of SHA256 in FIPS 180-2.
+ Copyright (C) 2007-2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Written by Ulrich Drepper <drepper@redhat.com>, 2007. */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <endian.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+// #include "sha256.h" - SAM
+#include <sha256sum.hh>
+#include <glibc-sha256.hh>
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+# ifdef _LIBC
+# include <byteswap.h>
+# define SWAP(n) bswap_32 (n)
+# define SWAP64(n) bswap_64 (n)
+# else
+# define SWAP(n) \
+ (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24))
+# define SWAP64(n) \
+ (((n) << 56) \
+ | (((n) & 0xff00) << 40) \
+ | (((n) & 0xff0000) << 24) \
+ | (((n) & 0xff000000) << 8) \
+ | (((n) >> 8) & 0xff000000) \
+ | (((n) >> 24) & 0xff0000) \
+ | (((n) >> 40) & 0xff00) \
+ | ((n) >> 56))
+# endif
+#else
+# define SWAP(n) (n)
+# define SWAP64(n) (n)
+#endif
+
+
+/* This array contains the bytes used to pad the buffer to the next
+ 64-byte boundary. (FIPS 180-2:5.1.1) */
+static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ };
+
+
+/* Constants for SHA256 from FIPS 180-2:4.2.2. */
+static const uint32_t K[64] =
+ {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
+ };
+
+void __sha256_process_block (const void *, size_t, struct sha256_ctx *);
+
+/* Initialize structure containing state of computation.
+ (FIPS 180-2:5.3.2) */
+void
+SHA256Sum::sha256_init_ctx (sha256_ctx *ctx)
+{
+ ctx->H[0] = 0x6a09e667;
+ ctx->H[1] = 0xbb67ae85;
+ ctx->H[2] = 0x3c6ef372;
+ ctx->H[3] = 0xa54ff53a;
+ ctx->H[4] = 0x510e527f;
+ ctx->H[5] = 0x9b05688c;
+ ctx->H[6] = 0x1f83d9ab;
+ ctx->H[7] = 0x5be0cd19;
+
+ ctx->total64 = 0;
+ ctx->buflen = 0;
+}
+
+
+/* Process the remaining bytes in the internal buffer and the usual
+ prolog according to the standard and write the result to RESBUF.
+
+ IMPORTANT: On some systems it is required that RESBUF is correctly
+ aligned for a 32 bits value. */
+byte*
+SHA256Sum::sha256_finish_ctx (sha256_ctx *ctx, byte* resbuf)
+{
+ /* Take yet unprocessed bytes into account. */
+ uint32_t bytes = ctx->buflen;
+ size_t pad;
+
+ /* Now count remaining bytes. */
+ ctx->total64 += bytes;
+
+ pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes;
+ memcpy (&ctx->buffer[bytes], fillbuf, pad);
+
+ /* Put the 64-bit file length in *bits* at the end of the buffer. */
+#if _STRING_ARCH_unaligned
+ ctx->buffer64[(bytes + pad) / 8] = SWAP64 (ctx->total64 << 3);
+#else
+ ctx->buffer32[(bytes + pad + 4) / 4] = SWAP (ctx->total[TOTAL64_low] << 3);
+ ctx->buffer32[(bytes + pad) / 4] = SWAP ((ctx->total[TOTAL64_high] << 3)
+ | (ctx->total[TOTAL64_low] >> 29));
+#endif
+
+ /* Process last bytes. */
+ sha256_process_block (ctx->buffer, bytes + pad + 8, ctx);
+
+ /* Put result from CTX in first 32 bytes following RESBUF. */
+ for (unsigned int i = 0; i < 8; ++i)
+ ((uint32_t *) resbuf)[i] = SWAP (ctx->H[i]);
+
+ return resbuf;
+}
+
+void
+SHA256Sum::sha256_process_bytes (const void *buffer, size_t len, struct sha256_ctx *ctx)
+{
+ /* When we already have some bits in our internal buffer concatenate
+ both inputs first. */
+ if (ctx->buflen != 0)
+ {
+ size_t left_over = ctx->buflen;
+ size_t add = 128 - left_over > len ? len : 128 - left_over;
+
+ memcpy (&ctx->buffer[left_over], buffer, add);
+ ctx->buflen += (uint32_t)add;
+
+ if (ctx->buflen > 64)
+ {
+ sha256_process_block (ctx->buffer, ctx->buflen & ~63, ctx);
+
+ ctx->buflen &= 63;
+ /* The regions in the following copy operation cannot overlap. */
+ memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63],
+ ctx->buflen);
+ }
+
+ buffer = (const char *) buffer + add;
+ len -= add;
+ }
+
+ /* Process available complete blocks. */
+ if (len >= 64)
+ {
+#if !_STRING_ARCH_unaligned
+/* To check alignment gcc has an appropriate operator. Other
+ compilers don't. */
+# if __GNUC__ >= 2
+# define UNALIGNED_P(p) (((uintptr_t) p) % __alignof__ (uint32_t) != 0)
+# else
+# define UNALIGNED_P(p) (((uintptr_t) p) % sizeof (uint32_t) != 0)
+# endif
+ if (UNALIGNED_P (buffer))
+ while (len > 64)
+ {
+ sha256_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx);
+ buffer = (const char *) buffer + 64;
+ len -= 64;
+ }
+ else
+#endif
+ {
+ sha256_process_block (buffer, len & ~63, ctx);
+ buffer = (const char *) buffer + (len & ~63);
+ len &= 63;
+ }
+ }
+
+ /* Move remaining bytes into internal buffer. */
+ if (len > 0)
+ {
+ size_t left_over = ctx->buflen;
+
+ memcpy (&ctx->buffer[left_over], buffer, len);
+ left_over += len;
+ if (left_over >= 64)
+ {
+ sha256_process_block (ctx->buffer, 64, ctx);
+ left_over -= 64;
+ memcpy (ctx->buffer, &ctx->buffer[64], left_over);
+ }
+ ctx->buflen = (uint32_t)left_over;
+ }
+}
+
+/* Process LEN bytes of BUFFER, accumulating context into CTX.
+ It is assumed that LEN % 64 == 0. */
+void
+SHA256Sum::sha256_process_block (const void *buffer, size_t len, struct sha256_ctx *ctx)
+{
+ const uint32_t *words = (uint32 *)buffer;
+ size_t nwords = len / sizeof (uint32_t);
+ uint32_t a = ctx->H[0];
+ uint32_t b = ctx->H[1];
+ uint32_t c = ctx->H[2];
+ uint32_t d = ctx->H[3];
+ uint32_t e = ctx->H[4];
+ uint32_t f = ctx->H[5];
+ uint32_t g = ctx->H[6];
+ uint32_t h = ctx->H[7];
+
+ /* First increment the byte count. FIPS 180-2 specifies the possible
+ length of the file up to 2^64 bits. Here we only compute the
+ number of bytes. */
+ ctx->total64 += len;
+
+ /* Process all bytes in the buffer with 64 bytes in each round of
+ the loop. */
+ while (nwords > 0)
+ {
+ uint32_t W[64];
+ uint32_t a_save = a;
+ uint32_t b_save = b;
+ uint32_t c_save = c;
+ uint32_t d_save = d;
+ uint32_t e_save = e;
+ uint32_t f_save = f;
+ uint32_t g_save = g;
+ uint32_t h_save = h;
+
+ /* Operators defined in FIPS 180-2:4.1.2. */
+#define Ch(x, y, z) ((x & y) ^ (~x & z))
+#define Maj(x, y, z) ((x & y) ^ (x & z) ^ (y & z))
+#define S0(x) (CYCLIC (x, 2) ^ CYCLIC (x, 13) ^ CYCLIC (x, 22))
+#define S1(x) (CYCLIC (x, 6) ^ CYCLIC (x, 11) ^ CYCLIC (x, 25))
+#define R0(x) (CYCLIC (x, 7) ^ CYCLIC (x, 18) ^ (x >> 3))
+#define R1(x) (CYCLIC (x, 17) ^ CYCLIC (x, 19) ^ (x >> 10))
+
+ /* It is unfortunate that C does not provide an operator for
+ cyclic rotation. Hope the C compiler is smart enough. */
+#define CYCLIC(w, s) ((w >> s) | (w << (32 - s)))
+
+ /* Compute the message schedule according to FIPS 180-2:6.2.2 step 2. */
+ for (unsigned int t = 0; t < 16; ++t)
+ {
+ W[t] = SWAP (*words);
+ ++words;
+ }
+ for (unsigned int t = 16; t < 64; ++t)
+ W[t] = R1 (W[t - 2]) + W[t - 7] + R0 (W[t - 15]) + W[t - 16];
+
+ /* The actual computation according to FIPS 180-2:6.2.2 step 3. */
+ for (unsigned int t = 0; t < 64; ++t)
+ {
+ uint32_t T1 = h + S1 (e) + Ch (e, f, g) + K[t] + W[t];
+ uint32_t T2 = S0 (a) + Maj (a, b, c);
+ h = g;
+ g = f;
+ f = e;
+ e = d + T1;
+ d = c;
+ c = b;
+ b = a;
+ a = T1 + T2;
+ }
+
+ /* Add the starting values of the context according to FIPS 180-2:6.2.2
+ step 4. */
+ a += a_save;
+ b += b_save;
+ c += c_save;
+ d += d_save;
+ e += e_save;
+ f += f_save;
+ g += g_save;
+ h += h_save;
+
+ /* Prepare for the next round. */
+ nwords -= 16;
+ }
+
+ /* Put checksum in context given as argument. */
+ ctx->H[0] = a;
+ ctx->H[1] = b;
+ ctx->H[2] = c;
+ ctx->H[3] = d;
+ ctx->H[4] = e;
+ ctx->H[5] = f;
+ ctx->H[6] = g;
+ ctx->H[7] = h;
+}
--- /dev/null
+/* Declaration of functions and data types used for SHA256 sum computing
+ library functions.
+ Copyright (C) 2007-2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef _SHA256_H
+#define _SHA256_H 1
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <endian.h>
+
+
+/* Structure to save state of computation between the single steps. */
+/* SAM moved this to sha256sum.hh
+struct sha256_ctx
+{
+ uint32_t H[8];
+
+ union
+ {
+ uint64_t total64;
+#define TOTAL64_low (1 - (BYTE_ORDER == LITTLE_ENDIAN))
+#define TOTAL64_high (BYTE_ORDER == LITTLE_ENDIAN)
+ uint32_t total[2];
+ };
+ uint32_t buflen;
+ union
+ {
+ char buffer[128];
+ uint32_t buffer32[32];
+ uint64_t buffer64[16];
+ };
+};
+*/
+
+/* Initialize structure containing state of computation.
+ (FIPS 180-2: 5.3.2) */
+// SAM extern void __sha256_init_ctx (struct sha256_ctx *ctx) __THROW;
+extern void sha256_init_ctx (struct sha256_ctx *ctx) __THROW;
+
+/* Starting with the result of former calls of this function (or the
+ initialization function update the context for the next LEN bytes
+ starting at BUFFER.
+ It is NOT required that LEN is a multiple of 64. */
+// SAM extern void __sha256_process_bytes (const void *buffer, size_t len,
+// struct sha256_ctx *ctx) __THROW;
+extern void sha256_process_bytes (const void *buffer, size_t len,
+ struct sha256_ctx *ctx) __THROW;
+
+/* Process the remaining bytes in the buffer and put result from CTX
+ in first 32 bytes following RESBUF.
+
+ IMPORTANT: On some systems it is required that RESBUF is correctly
+ aligned for a 32 bits value. */
+// SAM extern void *__sha256_finish_ctx (struct sha256_ctx *ctx, void *resbuf)
+// __THROW;
+extern void *sha256_finish_ctx (struct sha256_ctx *ctx, void *resbuf)
+ __THROW;
+
+#endif /* sha256.h */
void MD5Sum::ProgressReporter::info(const string& message) {
cerr << message << endl;
}
-void MD5Sum::ProgressReporter::readingMD5(uint64, uint64) { }
+void MD5Sum::ProgressReporter::readingChecksum(uint64, uint64) { }
MD5Sum::ProgressReporter MD5Sum::noReport;
//______________________________________________________________________
bytesRead += n;
toRead -= n;
if (bytesRead >= nextReport) {
- pr.readingMD5(bytesRead, size);
+ pr.readingChecksum(bytesRead, size);
nextReport += REPORT_INTERVAL;
}
}
/// Like error(), but for purely informational messages.
virtual void info(const string& message);
/// Called when data is read during updateFromStream()
- virtual void readingMD5(uint64 offInStream, uint64 size);
+ virtual void readingChecksum(uint64 offInStream, uint64 size);
};
//______________________________________________________________________
--- /dev/null
+/* $Id: sha256sum-test.cc,v 1.2 2003/09/27 21:31:04 atterer Exp $ -*- C++ -*-
+ __ _
+ |_) /| Copyright (C) 2000-2002 | richard@
+ | \/¯| Richard Atterer | atterer.org
+ ¯ '` ¯
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2. See
+ the file COPYING for details.
+
+ Quite secure 256-bit checksum
+
+ #test-deps util/glibc-sha256.o util/sha256sum.o
+
+*/
+
+#include <config.h>
+
+#include <iomanip>
+#include <iostream>
+#include <fstream>
+
+#include <bstream.hh>
+#include <log.hh>
+#include <sha256sum.hh>
+#include <mimestream.hh>
+//______________________________________________________________________
+
+namespace {
+
+int returnCode = 0;
+
+// SHA256 test suite (see end of RFC1321)
+
+const byte t1[] = "";
+const byte s1[] =
+"\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04\xe9\x80\x09\x98\xec\xf8\x42\x7e";
+const byte t2[] = "a";
+const byte s2[] =
+"\x0c\xc1\x75\xb9\xc0\xf1\xb6\xa8\x31\xc3\x99\xe2\x69\x77\x26\x61";
+const byte t3[] = "abc";
+const byte s3[] =
+"\x90\x01\x50\x98\x3c\xd2\x4f\xb0\xd6\x96\x3f\x7d\x28\xe1\x7f\x72";
+const byte t4[] = "message digest";
+const byte s4[] =
+"\xf9\x6b\x69\x7d\x7c\xb7\x93\x8d\x52\x5a\x2f\x31\xaa\xf1\x61\xd0";
+const byte t5[] = "abcdefghijklmnopqrstuvwxyz";
+const byte s5[] =
+"\xc3\xfc\xd3\xd7\x61\x92\xe4\x00\x7d\xfb\x49\x6c\xca\x67\xe1\x3b";
+const byte t6[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+const byte s6[] =
+"\xd1\x74\xab\x98\xd2\x77\xd9\xf5\xa5\x61\x1c\x2c\x9f\x41\x9d\x9f";
+const byte t7[] = "12345678901234567890123456789012345678901234567890123456789012345678901234567890";
+const byte s7[] =
+"\x57\xed\xf4\xa2\x2b\xe3\xc9\x55\xac\x49\xda\x2e\x21\x07\xb6\x7a";
+const byte sAll[] =
+"\x6f\xec\x75\xd4\xe7\xfc\xd7\xe9\x66\x46\xb4\xc7\xaf\x96\xbc\xe2";
+
+const char* const hexDigits = "0123456789abcdef";
+
+}
+//______________________________________________________________________
+
+string toHex(const byte* sum) {
+ string result;
+ for (int i = 0; i < 32; ++i) {
+ result += hexDigits[sum[i] >> 4];
+ result += hexDigits[sum[i] & 0xfU];
+ }
+ return result;
+}
+
+void compare(const byte* suite, const byte* mine) {
+ for (int i = 0; i < 32; ++i) {
+ if (suite[i] != mine[i]) {
+ msg("ERROR: Expected %1, but got %2", toHex(suite), toHex(mine));
+ returnCode = 1;
+ return;
+ }
+ }
+ msg("OK: %1", toHex(suite));
+}
+
+void printBlockSums(size_t blockSize, const char* fileName) {
+ bifstream file(fileName, ios::binary);
+ byte buf[blockSize];
+ byte* bufEnd = buf + blockSize;
+
+ while (file) {
+ // read another block
+ byte* cur = buf;
+ while (cur < bufEnd && file) {
+ readBytes(file, cur, bufEnd - cur); // Fill buffer
+ cur += file.gcount();
+ }
+ SHA256Sum sum;
+ sum.update(buf, cur - buf).finishForReuse();
+ cout << ' ' << sum << endl;
+ }
+ exit(0);
+}
+
+int main(int argc, char* argv[]) {
+ if (argc == 2) Logger::scanOptions(argv[1], argv[0]);
+ if (argc == 3) {
+ // 2 cmdline args, blocksize and filename. Print RsyncSums of all blocks
+ printBlockSums(atoi(argv[1]), argv[2]);
+ exit(0);
+ }
+ const byte* sum;
+ SHA256Sum x;
+ SHA256Sum all;
+
+ sum = x.reset().update(t1, sizeof(t1) - 1).finish().digest();
+ compare(s1, sum);
+ all.update(t1, sizeof(t1) - 1);
+
+ sum = x.reset().update(t2, sizeof(t2) - 1).finish().digest();
+ compare(s2, sum);
+ all.update(t2, sizeof(t2) - 1);
+
+ sum = x.reset().update(t3, sizeof(t3) - 1).finish().digest();
+ compare(s3, sum);
+ all.update(t3, sizeof(t3) - 1);
+
+ sum = x.reset().update(t4, sizeof(t4) - 1).finish().digest();
+ compare(s4, sum);
+ all.update(t4, sizeof(t4) - 1);
+
+ sum = x.reset().update(t5, sizeof(t5) - 1).finish().digest();
+ compare(s5, sum);
+ all.update(t5, sizeof(t5) - 1);
+
+ sum = x.reset().update(t6, sizeof(t6) - 1).finish().digest();
+ compare(s6, sum);
+ all.update(t6, sizeof(t6) - 1);
+
+ sum = x.reset().update(t7, sizeof(t7) - 1).finish().digest();
+ compare(s7, sum);
+ all.update(t7, sizeof(t7) - 1);
+
+ sum = all.finish().digest();
+ compare(sAll, sum);
+
+ return returnCode;
+}
--- /dev/null
+/* $Id: sha256sum.cc,v 1.4 2004/06/20 20:35:15 atterer Exp $ -*- C++ -*-
+ __ _
+ |_) /| Copyright (C) 2000-2004 | richard@
+ | \/¯| Richard Atterer | atterer.org
+ ¯ '` ¯
+ "Ported" to C++ by RA. Uses glibc code for the actual algorithm.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2. See
+ the file COPYING for details.
+
+ Quite secure 128-bit checksum
+
+*/
+
+#include <config.h>
+
+#include <iostream>
+#include <vector>
+
+#include <glibc-sha256.hh>
+#include <sha256sum.hh>
+#include <sha256sum.ih>
+//______________________________________________________________________
+
+void SHA256Sum::ProgressReporter::error(const string& message) {
+ cerr << message << endl;
+}
+void SHA256Sum::ProgressReporter::info(const string& message) {
+ cerr << message << endl;
+}
+void SHA256Sum::ProgressReporter::readingSHA256(uint64, uint64) { }
+
+SHA256Sum::ProgressReporter SHA256Sum::noReport;
+//______________________________________________________________________
+
+SHA256Sum::SHA256Sum(const SHA256Sum& md) {
+ if (md.p == 0) {
+ p = 0;
+ for (int i = 0; i < 32; ++i) sum[i] = md.sum[i];
+ } else {
+ p = new sha256_ctx();
+ *p = *md.p;
+ }
+}
+//________________________________________
+
+// NB must work with self-assign
+SHA256Sum& SHA256Sum::operator=(const SHA256Sum& md) {
+# if DEBUG
+ finished = md.finished;
+# endif
+ if (md.p == 0) {
+ delete p;
+ p = 0;
+ for (int i = 0; i < 32; ++i) sum[i] = md.sum[i];
+ } else {
+ if (p == 0) p = new sha256_ctx();
+ *p = *md.p;
+ }
+ return *this;
+}
+//______________________________________________________________________
+
+string SHA256::toString() const {
+ Base64String m;
+ m.write(sum, 32).flush();
+ return m.result();
+}
+//______________________________________________________________________
+
+bool SHA256::operator_less2(const SHA256& x) const {
+ if (sum[1] < x.sum[1]) return true;
+ if (sum[1] > x.sum[1]) return false;
+ if (sum[2] < x.sum[2]) return true;
+ if (sum[2] > x.sum[2]) return false;
+ if (sum[3] < x.sum[3]) return true;
+ if (sum[3] > x.sum[3]) return false;
+ if (sum[4] < x.sum[4]) return true;
+ if (sum[4] > x.sum[4]) return false;
+ if (sum[5] < x.sum[5]) return true;
+ if (sum[5] > x.sum[5]) return false;
+ if (sum[6] < x.sum[6]) return true;
+ if (sum[6] > x.sum[6]) return false;
+ if (sum[7] < x.sum[7]) return true;
+ if (sum[7] > x.sum[7]) return false;
+ if (sum[8] < x.sum[8]) return true;
+ if (sum[8] > x.sum[8]) return false;
+ if (sum[9] < x.sum[9]) return true;
+ if (sum[9] > x.sum[9]) return false;
+ if (sum[10] < x.sum[10]) return true;
+ if (sum[10] > x.sum[10]) return false;
+ if (sum[11] < x.sum[11]) return true;
+ if (sum[11] > x.sum[11]) return false;
+ if (sum[12] < x.sum[12]) return true;
+ if (sum[12] > x.sum[12]) return false;
+ if (sum[13] < x.sum[13]) return true;
+ if (sum[13] > x.sum[13]) return false;
+ if (sum[14] < x.sum[14]) return true;
+ if (sum[14] > x.sum[14]) return false;
+ if (sum[15] < x.sum[15]) return true;
+ if (sum[15] > x.sum[15]) return false;
+ if (sum[16] < x.sum[16]) return true;
+ if (sum[16] > x.sum[16]) return false;
+ if (sum[17] < x.sum[17]) return true;
+ if (sum[17] > x.sum[17]) return false;
+ if (sum[18] < x.sum[18]) return true;
+ if (sum[18] > x.sum[18]) return false;
+ if (sum[19] < x.sum[19]) return true;
+ if (sum[19] > x.sum[19]) return false;
+ if (sum[20] < x.sum[20]) return true;
+ if (sum[20] > x.sum[20]) return false;
+ if (sum[21] < x.sum[21]) return true;
+ if (sum[21] > x.sum[21]) return false;
+ if (sum[22] < x.sum[22]) return true;
+ if (sum[22] > x.sum[22]) return false;
+ if (sum[23] < x.sum[23]) return true;
+ if (sum[23] > x.sum[23]) return false;
+ if (sum[24] < x.sum[24]) return true;
+ if (sum[24] > x.sum[24]) return false;
+ if (sum[25] < x.sum[25]) return true;
+ if (sum[25] > x.sum[25]) return false;
+ if (sum[26] < x.sum[26]) return true;
+ if (sum[26] > x.sum[26]) return false;
+ if (sum[27] < x.sum[27]) return true;
+ if (sum[27] > x.sum[27]) return false;
+ if (sum[28] < x.sum[28]) return true;
+ if (sum[28] > x.sum[28]) return false;
+ if (sum[29] < x.sum[29]) return true;
+ if (sum[29] > x.sum[29]) return false;
+ if (sum[30] < x.sum[30]) return true;
+ if (sum[30] > x.sum[30]) return false;
+ if (sum[31] < x.sum[31]) return true;
+ return false;
+}
+//______________________________________________________________________
+
+uint64 SHA256Sum::updateFromStream(bistream& s, uint64 size, size_t bufSize,
+ ProgressReporter& pr) {
+ uint64 nextReport = REPORT_INTERVAL; // When next to call reporter
+ uint64 toRead = size;
+ uint64 bytesRead = 0;
+ vector<byte> buffer;
+ buffer.resize(bufSize);
+ byte* buf = &buffer[0];
+ // Read from stream and update *this
+ while (s && !s.eof() && toRead > 0) {
+ size_t n = (toRead < bufSize ? toRead : bufSize);
+ readBytes(s, buf, n);
+ n = s.gcount();
+ update(buf, n);
+ bytesRead += n;
+ toRead -= n;
+ if (bytesRead >= nextReport) {
+ pr.readingSHA256(bytesRead, size);
+ nextReport += REPORT_INTERVAL;
+ }
+ }
+ return bytesRead;
+}
--- /dev/null
+/* $Id: sha256sum.fh,v 1.2 2003/09/27 21:31:04 atterer Exp $ -*- C++ -*-
+ __ _
+ |_) /| Copyright (C) 2000-2002 | richard@
+ | \/¯| Richard Atterer | atterer.org
+ ¯ '` ¯
+ "Ported" to C++ by RA. Actual SHA256 code taken from glibc
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2. See
+ the file COPYING for details.
+
+ Quite secure 256-bit checksum
+
+*/
+
+class SHA256;
+class SHA256Sum;
--- /dev/null
+/* $Id: sha256sum.hh,v 1.5 2005/04/09 23:09:52 atterer Exp $ -*- C++ -*-
+ __ _
+ |_) /| Copyright (C) 2000-2004 | richard@
+ | \/¯| Richard Atterer | atterer.org
+ ¯ '` ¯
+ "Ported" to C++ by RA. Actual SHA256 code taken from glibc
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2. See
+ the file COPYING for details.
+
+*//** @file
+
+ 128-bit checksum, secure enough for our purposes
+
+*/
+
+#ifndef SHA256SUM_HH
+#define SHA256SUM_HH
+
+#ifndef INLINE
+# ifdef NOINLINE
+# define INLINE
+# else
+# define INLINE inline
+# endif
+#endif
+
+#include <config.h>
+
+#include <cstdlib>
+#include <cstring>
+#include <iosfwd>
+#include <string>
+
+#include <bstream.hh>
+#include <debug.hh>
+#include <sha256sum.fh>
+//______________________________________________________________________
+
+/** Container for an already computed SHA256Sum.
+
+ Objects of this class are smaller than SHA256Sum objects by one
+ pointer. As soon as the checksum calculation of an SHA256Sum object
+ has finish()ed, the pointer is no longer needed. If you need to
+ store a large number of calculated SHA256Sums, it may be beneficial
+ to assign the SHA256Sum to an SHA256 to save space. */
+class SHA256 {
+public:
+ SHA256() { }
+ inline SHA256(const SHA256Sum& md);
+ /** 32 bytes of SHA256 checksum */
+ byte sum[32];
+ /** Allows you to treat the object exactly like a pointer to a byte
+ array */
+ operator byte*() { return sum; }
+ operator const byte*() const { return sum; }
+ /** Assign an SHA256Sum */
+ inline SHA256& operator=(const SHA256Sum& md);
+ inline bool operator<(const SHA256& x) const;
+ /** Clear contents to zero */
+ inline SHA256& clear();
+ /** Convert to string */
+ string toString() const;
+
+ template<class Iterator>
+ inline Iterator serialize(Iterator i) const;
+ template<class ConstIterator>
+ inline ConstIterator unserialize(ConstIterator i);
+ inline size_t serialSizeOf() const { return 32; }
+
+ // Default copy ctor
+private:
+ bool operator_less2(const SHA256& x) const;
+ static const byte zero[32];
+};
+
+inline bool operator==(const SHA256& a, const SHA256& b);
+inline bool operator!=(const SHA256& a, const SHA256& b) { return !(a == b); }
+
+/// Output SHA256 as Base64 digest
+INLINE ostream& operator<<(ostream& s, const SHA256& r);
+//______________________________________________________________________
+
+/** A 256-bit, cryptographically strong message digest algorithm.
+
+ Unless described otherwise, if a method returns an SHA256Sum&, then
+ this is a reference to the object itself, to allow chaining of
+ calls. */
+class SHA256Sum {
+ friend class SHA256;
+public:
+ class ProgressReporter;
+
+ /** Initialise the checksum */
+ inline SHA256Sum();
+ /** Initialise with another checksum instance */
+ SHA256Sum(const SHA256Sum& md);
+ ~SHA256Sum() { delete p; }
+ /** Assign another checksum instance */
+ SHA256Sum& operator=(const SHA256Sum& md);
+ /** Tests for equality. Note: Will only return true if both message
+ digest operations have been finished and their SHA256 sums are the
+ same. */
+ inline bool operator==(const SHA256Sum& md) const;
+ inline bool operator!=(const SHA256Sum& md) const;
+ inline bool operator==(const SHA256& md) const { return sum == md; }
+ inline bool operator!=(const SHA256& md) const { return sum != md; }
+ /** Reset checksum object to the same state as immediately after its
+ creation. You must call when reusing an SHA256Sum object - call it
+ just before the first update() for the new checksum. */
+ inline SHA256Sum& reset();
+ /** Process bytes with the checksum algorithm. May lead to some
+ bytes being temporarily buffered internally. */
+ inline SHA256Sum& update(const byte* mem, size_t len);
+ /// Add a single byte. NB, not implemented efficiently ATM
+ inline SHA256Sum& update(byte x) { update(&x, 1); return *this; }
+ /** Process remaining bytes in internal buffer and create the final
+ checksum.
+ @return Pointer to the 32-byte checksum. */
+ inline SHA256Sum& finish();
+ /** Exactly the same behaviour as finish(), but is more efficient if
+ you are going to call reset() again in the near future to re-use
+ the SHA256Sum object.
+ @return Pointer to the 32-byte checksum. */
+ inline SHA256Sum& finishForReuse();
+ /** Deallocate buffers like finish(), but don't generate the final
+ sum */
+ inline SHA256Sum& abort();
+ /** Return 32 byte buffer with checksum. Warning: Returns junk if
+ checksum not yet finish()ed or flush()ed. */
+ inline const byte* digest() const;
+
+ /** Convert to string */
+ INLINE string toString() const;
+
+ /** Read data from file and update() this checksum with it.
+ @param s The stream to read from
+ @param size Total number of bytes to read
+ @param bufSize Size of temporary read buffer
+ @param pr Reporter object
+ @return Number of bytes read (==size if no error) */
+ uint64 updateFromStream(bistream& s, uint64 size,
+ size_t bufSize = 128*1024, ProgressReporter& pr = noReport);
+
+ /* Serializing an SHA256Sum is only allowed after finish(). The
+ serialization is compatible with that of SHA256. */
+ template<class Iterator>
+ inline Iterator serialize(Iterator i) const;
+ template<class ConstIterator>
+ inline ConstIterator unserialize(ConstIterator i);
+ inline size_t serialSizeOf() const { return sum.serialSizeOf(); }
+
+private:
+/* Structure to save state of computation between the single steps. */
+ struct sha256_ctx
+ {
+ uint32_t H[8];
+ union
+ {
+ uint64_t total64;
+#define TOTAL64_low (1 - (BYTE_ORDER == LITTLE_ENDIAN))
+#define TOTAL64_high (BYTE_ORDER == LITTLE_ENDIAN)
+ uint32_t total[2];
+ };
+ uint32_t buflen;
+ union
+ {
+ char buffer[128];
+ uint32_t buffer32[32];
+ uint64_t buffer64[16];
+ };
+ };
+
+ /// Default reporter: Only prints error messages to stderr
+ static ProgressReporter noReport;
+
+ // These functions are the original glibc API
+ static void sha256_init_ctx(sha256_ctx* ctx);
+ static void sha256_process_bytes(const void* buffer, size_t len,
+ struct sha256_ctx* ctx);
+ static byte* sha256_finish_ctx(struct sha256_ctx* ctx, byte* resbuf);
+ static byte* sha256_read_ctx(const sha256_ctx *ctx, byte* resbuf);
+ static void sha256_process_block(const void* buffer, size_t len,
+ sha256_ctx* ctx);
+ SHA256 sum;
+ struct sha256_ctx* p; // null once MD creation is finished
+
+# if DEBUG
+ /* After finish(ForReuse)(), must call reset() before the next
+ update(). OTOH, must only call digest() after finish(). Enforce
+ this rule by keeping track of the state. */
+ bool finished;
+# endif
+};
+
+inline bool operator==(const SHA256& a, const SHA256Sum& b) { return b == a; }
+inline bool operator!=(const SHA256& a, const SHA256Sum& b) { return b != a; }
+
+/// Output SHA256Sum as Base64 digest
+INLINE ostream& operator<<(ostream& s, const SHA256Sum& r);
+//______________________________________________________________________
+
+/** Class allowing JigdoCache to convey information back to the
+ creator of a JigdoCache object. */
+class SHA256Sum::ProgressReporter {
+public:
+ virtual ~ProgressReporter() { }
+ /// General-purpose error reporting.
+ virtual void error(const string& message);
+ /// Like error(), but for purely informational messages.
+ virtual void info(const string& message);
+ /// Called when data is read during updateFromStream()
+ virtual void readingSHA256(uint64 offInStream, uint64 size);
+};
+//______________________________________________________________________
+
+bool SHA256Sum::operator==(const SHA256Sum& md) const {
+# if DEBUG
+ Paranoid(this->finished && md.finished);
+# endif
+ return sum == md.sum;
+}
+bool SHA256Sum::operator!=(const SHA256Sum& md) const {
+# if DEBUG
+ Paranoid(this->finished && md.finished);
+# endif
+ return sum != md.sum;
+}
+
+SHA256Sum::SHA256Sum() {
+ p = new sha256_ctx();
+ sha256_init_ctx(p);
+# if DEBUG
+ finished = false;
+# endif
+}
+
+SHA256Sum& SHA256Sum::reset() {
+ if (p == 0) p = new sha256_ctx();
+ sha256_init_ctx(p);
+# if DEBUG
+ finished = false;
+# endif
+ return *this;
+}
+
+SHA256Sum& SHA256Sum::update(const byte* mem, size_t len) {
+ Paranoid(p != 0);
+# if DEBUG
+ Paranoid(!finished); // Don't forget to call reset() before update()
+# endif
+ sha256_process_bytes(mem, len, p);
+ return *this;
+}
+
+SHA256Sum& SHA256Sum::finish() {
+ Paranoid(p != 0 );
+ sha256_finish_ctx(p, sum);
+ delete p;
+ p = 0;
+# if DEBUG
+ finished = true;
+# endif
+ return *this;
+}
+
+SHA256Sum& SHA256Sum::finishForReuse() {
+ Paranoid(p != 0 );
+ sha256_finish_ctx(p, sum);
+# if DEBUG
+ finished = true;
+# endif
+ return *this;
+}
+
+SHA256Sum& SHA256Sum::abort() {
+ delete p;
+ p = 0;
+# if DEBUG
+ finished = false;
+# endif
+ return *this;
+}
+
+const byte* SHA256Sum::digest() const {
+# if DEBUG
+ Paranoid(finished); // Call finish() first
+# endif
+ return sum.sum;
+}
+
+template<class Iterator>
+inline Iterator SHA256Sum::serialize(Iterator i) const {
+# if DEBUG
+ Paranoid(finished ); // Call finish() first
+# endif
+ return sum.serialize(i);
+}
+template<class ConstIterator>
+inline ConstIterator SHA256Sum::unserialize(ConstIterator i) {
+# if DEBUG
+ finished = true;
+# endif
+ return sum.unserialize(i);
+}
+//____________________
+
+SHA256& SHA256::operator=(const SHA256Sum& md) {
+# if DEBUG
+ Paranoid(md.finished); // Call finish() first
+# endif
+ *this = md.sum;
+ return *this;
+}
+
+bool SHA256::operator<(const SHA256& x) const {
+ if (sum[0] < x.sum[0]) return true;
+ if (sum[0] > x.sum[0]) return false;
+ return operator_less2(x);
+}
+
+// inline bool operator<(const SHA256& a, const SHA256& b) {
+// return a.operator<(b);
+// }
+
+SHA256::SHA256(const SHA256Sum& md) { *this = md.sum; }
+
+bool operator==(const SHA256& a, const SHA256& b) {
+ // How portable is this?
+ return memcmp(a.sum, b.sum, 32 * sizeof(byte)) == 0;
+# if 0
+ return a.sum[0] == b.sum[0] && a.sum[1] == b.sum[1]
+ && a.sum[2] == b.sum[2] && a.sum[3] == b.sum[3]
+ && a.sum[4] == b.sum[4] && a.sum[5] == b.sum[5]
+ && a.sum[6] == b.sum[6] && a.sum[7] == b.sum[7]
+ && a.sum[8] == b.sum[8] && a.sum[9] == b.sum[9]
+ && a.sum[10] == b.sum[10] && a.sum[11] == b.sum[11]
+ && a.sum[12] == b.sum[12] && a.sum[13] == b.sum[13]
+ && a.sum[14] == b.sum[14] && a.sum[15] == b.sum[15];
+# endif
+}
+
+SHA256& SHA256::clear() {
+ byte* x = sum;
+ *x++ = 0; *x++ = 0; *x++ = 0; *x++ = 0;
+ *x++ = 0; *x++ = 0; *x++ = 0; *x++ = 0;
+ *x++ = 0; *x++ = 0; *x++ = 0; *x++ = 0;
+ *x++ = 0; *x++ = 0; *x++ = 0; *x++ = 0;
+ return *this;
+}
+
+template<class Iterator>
+inline Iterator SHA256::serialize(Iterator i) const {
+ for (int j = 0; j < 32; ++j) { *i = sum[j]; ++i; }
+ return i;
+}
+template<class ConstIterator>
+inline ConstIterator SHA256::unserialize(ConstIterator i) {
+ for (int j = 0; j < 32; ++j) { sum[j] = *i; ++i; }
+ return i;
+}
+
+#ifndef NOINLINE
+# include <sha256sum.ih> /* NOINLINE */
+#endif
+
+#endif
--- /dev/null
+/* $Id: sha256sum.ih,v 1.3 2004/06/20 20:35:15 atterer Exp $ -*- C++ -*-
+ __ _
+ |_) /| Copyright (C) 2000-2004 | richard@
+ | \/¯| Richard Atterer | atterer.org
+ ¯ '` ¯
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2. See
+ the file COPYING for details.
+
+ Quite secure 256-bit checksum
+
+*/
+
+#ifndef SHA256SUM_IH
+#define SHA256SUM_IH
+
+#include <mimestream.hh>
+//______________________________________________________________________
+
+ostream& operator<<(ostream& s, const SHA256& r) {
+ Base64String m;
+ m.write(r.sum, 32).flush();
+ s << m.result();
+ return s;
+}
+
+string SHA256Sum::toString() const {
+ const byte* d = digest();
+ if (d == 0) {
+ return "[SHA256_creation_not_finished]";
+ } else {
+ Base64String m;
+ m.write(d, 32).flush();
+ return m.result();
+ }
+}
+
+ostream& operator<<(ostream& s, const SHA256Sum& r) {
+ const byte* d = r.digest();
+ if (d == 0) {
+ s << "[SHA256_creation_not_finished]";
+ } else {
+ Base64String m;
+ m.write(d, 32).flush();
+ s << m.result();
+ }
+ return s;
+}
+
+#endif
projects / jigdo.git / commitdiff