23 INT64 offset_in_curr_buf;
27 /* Grab the file component from a full path */
28 static char *file_base_name(char *path)
43 static int read_data_block(FILE *template_file)
47 static INT64 template_offset = -1;
48 INT64 compressed_len = 0;
49 INT64 uncompressed_len = 0;
50 char *comp_buf = NULL;
54 if (-1 == template_offset)
56 fseek(template_file, 0, SEEK_SET);
57 fread(inbuf, sizeof(inbuf), 1, template_file);
58 for (i = 0; i < sizeof(inbuf); i++)
60 if (!strncmp(&inbuf[i], "DATA", 4))
62 zip_state.algorithm = CT_GZIP;
66 if (!strncmp(&inbuf[i], "BZIP", 4))
68 zip_state.algorithm = CT_BZIP2;
73 if (-1 == template_offset)
75 fprintf(G_logfile, "Unable to locate DATA block in template (offset %lld)\n",
81 fseek(template_file, template_offset, SEEK_SET);
82 fread(inbuf, 16, 1, template_file);
83 if (strncmp(inbuf, "DATA", 4) && strncmp(inbuf, "BZIP", 4))
85 fprintf(G_logfile, "Unable to locate DATA block in template (offset %lld)\n",
90 compressed_len = read_le48((unsigned char *)&inbuf[4]);
91 uncompressed_len = read_le48((unsigned char *)&inbuf[10]);
93 comp_buf = calloc(1, compressed_len);
96 fprintf(G_logfile, "Unable to locate DATA block in template (offset %lld)\n",
101 zip_state.data_buf = calloc(1, uncompressed_len);
102 if (!zip_state.data_buf)
104 fprintf(G_logfile, "Unable to allocate %lld bytes for decompression\n",
109 read_num = fread(comp_buf, compressed_len, 1, template_file);
112 fprintf(G_logfile, "Unable to read %lld bytes for decompression\n",
117 error = decompress_data_block(comp_buf, compressed_len,
118 zip_state.data_buf, uncompressed_len, zip_state.algorithm);
121 fprintf(G_logfile, "Unable to decompress data block, error %d\n", error);
125 template_offset += compressed_len;
126 zip_state.buf_size = uncompressed_len;
127 zip_state.offset_in_curr_buf = 0;
132 static int skip_data_block(INT64 data_size, FILE *template_file)
135 INT64 remaining = data_size;
138 /* If we're coming in in the middle of the image, we'll need to
139 skip through some compressed data */
142 if (!zip_state.data_buf)
144 error = read_data_block(template_file);
147 fprintf(G_logfile, "Unable to decompress template data, error %d\n",
152 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
153 zip_state.offset_in_curr_buf += size;
156 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
158 free(zip_state.data_buf);
159 zip_state.data_buf = NULL;
163 fprintf(G_logfile, "skip_data_block: skipped %lld bytes of unmatched data\n", data_size);
167 static int parse_data_block(INT64 data_size, FILE *template_file, struct mk_MD5Context *context, FILE *outfile)
170 INT64 remaining = data_size;
176 if (!zip_state.data_buf)
178 error = read_data_block(template_file);
181 fprintf(G_logfile, "Unable to decompress template data, error %d\n",
186 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
187 write_size = fwrite(&zip_state.data_buf[zip_state.offset_in_curr_buf], size, 1, outfile);
190 fprintf(G_logfile, "parse_data_block: fwrite %lld failed with error %d; aborting\n", size, ferror(outfile));
191 return ferror(outfile);
195 display_progress(outfile, "template data");
198 mk_MD5Update(context,
199 (unsigned char *)&zip_state.data_buf[zip_state.offset_in_curr_buf],
201 zip_state.offset_in_curr_buf += size;
204 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
206 free(zip_state.data_buf);
207 zip_state.data_buf = NULL;
211 fprintf(G_logfile, "parse_data_block: wrote %lld bytes of unmatched data\n", data_size);
215 static int read_file_data(char *filename, char *missing, INT64 offset, INT64 data_size,
216 struct mk_MD5Context *file_context, struct mk_MD5Context *image_context,
219 FILE *input_file = NULL;
220 INT64 remaining = data_size;
225 input_file = fopen(filename, "rb");
228 fprintf(G_logfile, "Unable to open mirror file %s, error %d\n",
239 fseek(input_file, offset, SEEK_SET);
242 int size = MIN(BUF_SIZE, remaining);
243 memset(buf, 0, BUF_SIZE);
245 num_read = fread(buf, size, 1, input_file);
248 fprintf(G_logfile, "Unable to read from mirror file %s, error %d (offset %ld, length %d)\n",
249 filename, errno, ftell(input_file), size);
255 mk_MD5Update(image_context, (unsigned char *)buf, size);
256 mk_MD5Update(file_context, (unsigned char *)buf, size);
259 write_size = fwrite(buf, size, 1, outfile);
262 fprintf(G_logfile, "read_file_data: fwrite %d failed with error %d; aborting\n", size, ferror(outfile));
263 return ferror(outfile);
267 display_progress(outfile, file_base_name(filename));
272 fprintf(G_logfile, "read_file_data: wrote %lld bytes of data from %s\n",
273 data_size, filename);
278 static int parse_file_block(INT64 offset, INT64 data_size, INT64 file_size, FILE *outfile,
279 JIGDB *dbp, unsigned char *md5, struct mk_MD5Context *image_context,
282 char *base64_md5 = base64_dump(md5, 16);
283 struct mk_MD5Context file_context;
284 struct mk_MD5Context *use_context = NULL;
285 unsigned char file_md5[16];
286 md5_list_t *md5_list_entry = NULL;
287 db_file_entry_t db_entry;
289 char *filename = NULL;
293 use_context = &file_context;
294 mk_MD5Init(use_context);
297 /* Try the DB first if we have one */
300 error = db_lookup_file_by_md5(dbp, base64_md5, &db_entry);
302 filename = db_entry.filename;
305 /* No joy; fall back to the MD5 list */
308 md5_list_entry = find_file_in_md5_list(base64_md5);
309 if (md5_list_entry && file_size == md5_list_entry->file_size)
310 filename = md5_list_entry->full_path;
315 error = read_file_data(filename, missing, offset, data_size,
316 use_context, image_context, outfile);
318 if (error && (ENOENT != error))
320 fprintf(G_logfile, "Failed to read file %s, error %d\n", filename, error);
327 mk_MD5Final(file_md5, &file_context);
329 if (memcmp(file_md5, md5, 16))
331 char *tmp_md5 = NULL;
333 fprintf(G_logfile, "MD5 MISMATCH for file %s\n", filename);
335 tmp_md5 = base64_dump(md5, 16);
336 fprintf(G_logfile, " template looking for %s\n", tmp_md5);
339 tmp_md5 = base64_dump(file_md5, 16);
340 fprintf(G_logfile, " file %s is %s\n", filename, tmp_md5);
350 /* No file found. Add it to the list of missing files, or complain */
353 (MISSING == md5_list_entry->file_size) &&
354 (!memcmp(md5_list_entry->md5, base64_md5, 16) ) )
356 file_missing(missing, md5_list_entry->full_path);
365 int parse_template_file(char *filename, int sizeonly, char *missing,
366 FILE *outfile, char *output_name, JIGDB *dbp)
368 INT64 template_offset = 0;
373 INT64 desc_start = 0;
374 INT64 written_length = 0;
375 INT64 output_offset = 0;
378 struct mk_MD5Context template_context;
379 unsigned char image_md5sum[16];
381 bzero(&zip_state, sizeof(zip_state));
383 file = fopen(filename, "rb");
386 fprintf(G_logfile, "Failed to open template file %s, error %d!\n", filename, errno);
390 buf = malloc(BUF_SIZE);
393 fprintf(G_logfile, "Failed to malloc %d bytes. Abort!\n", BUF_SIZE);
398 /* Find the beginning of the desc block */
399 file_size = get_file_size(filename);
400 fseek(file, file_size - 6, SEEK_SET);
401 fread(buf, 6, 1, file);
402 desc_start = file_size - read_le48((unsigned char *)buf);
404 /* Now seek back to the beginning of the image desc block to grab
405 the MD5 and image length */
406 fseek(file, file_size - 33, SEEK_SET);
407 fread(buf, BUF_SIZE, 1, file);
408 if (buf[0] != 5) /* image data */
410 fprintf(G_logfile, "Failed to find image desc in the template file\n");
415 memcpy(image_md5sum, &buf[7], 16);
417 G_out_size = read_le48((unsigned char *)&buf[1]);
419 /* Now seek back to the start of the desc block */
420 fseek(file, desc_start, SEEK_SET);
421 fread(buf, 10, 1, file);
422 if (strncmp(buf, "DESC", 4))
424 fprintf(G_logfile, "Failed to find desc start in the template file\n");
428 if ((file_size - desc_start) != read_le48((unsigned char *)&buf[4]))
430 fprintf(G_logfile, "Inconsistent desc length in the template file!\n");
431 fprintf(G_logfile, "Final chunk says %lld, first chunk says %lld\n",
432 file_size - desc_start, read_le48((unsigned char *)&buf[4]));
438 mk_MD5Init(&template_context);
439 template_offset = desc_start + 10;
442 fprintf(G_logfile, "Creating ISO image %s\n", output_name);
444 /* Main loop - walk through the template file and expand each entry we find */
449 INT64 read_length = 0;
451 if (template_offset >= (file_size - 33))
454 fprintf(G_logfile, "Reached end of template file\n");
455 break; /* Finished! */
458 if (output_offset > G_end_offset) /* Past the range we were asked for */
460 fprintf(G_logfile, "Reached end of range requested\n");
464 fseek(file, template_offset, SEEK_SET);
465 bytes = fread(buf, (MIN (BUF_SIZE, file_size - template_offset)), 1, file);
468 fprintf(G_logfile, "Failed to read template file!\n");
473 extent_size = read_le48((unsigned char *)&buf[1]);
474 read_length = extent_size;
476 if (G_start_offset > output_offset)
477 skip = G_start_offset - output_offset;
478 if ((output_offset + extent_size) > G_end_offset)
479 read_length -= (output_offset + extent_size - G_end_offset - 1);
485 case 2: /* unmatched data */
486 template_offset += 7;
489 if ((output_offset + extent_size) >= G_start_offset)
492 error = skip_data_block(skip, file);
495 fprintf(G_logfile, "Unable to read data block to skip, error %d\n", error);
499 error = parse_data_block(read_length, file, &template_context, outfile);
502 fprintf(G_logfile, "Unable to read data block, error %d\n", error);
506 written_length += read_length;
509 error = skip_data_block(extent_size, file);
512 template_offset += 31;
513 if ((output_offset + extent_size) >= G_start_offset)
515 error = parse_file_block(skip, read_length, extent_size, outfile, dbp,
516 (unsigned char *)&buf[15], &template_context, missing);
519 fprintf(G_logfile, "Unable to read file block, error %d\n", error);
523 written_length += read_length;
527 fprintf(G_logfile, "Unknown block type %d!\n", buf[0]);
531 output_offset += extent_size;
534 if (missing && G_missing_file)
540 fprintf(G_logfile, "\n");
543 mk_MD5Final (image_md5sum, &template_context);
544 fprintf(G_logfile, "Output image MD5 is ");
545 for (i = 0; i < 16; i++)
546 fprintf(G_logfile, "%2.2x", image_md5sum[i]);
547 fprintf(G_logfile, "\n");
549 fprintf(G_logfile, "Output image length is %lld bytes\n", written_length);
556 int add_new_template_file(JIGDB *dbp, char *filename)
559 INT64 template_offset = 0;
564 INT64 desc_start = 0;
565 INT64 image_offset = 0;
566 INT64 comp_offset = 0;
567 INT64 uncomp_offset = 0;
568 unsigned char tmp_md5sum[16];
569 char *md5_out = NULL;
570 db_template_entry_t template;
571 db_block_entry_t block;
572 db_compressed_entry_t compressed;
573 int num_compressed_blocks = 0;
574 int num_data_blocks = 0;
575 int num_file_blocks = 0;
577 file = fopen(filename, "rb");
580 fprintf(G_logfile, "add_new_template_file: Failed to open template file %s, error %d!\n", filename, errno);
584 buf = malloc(BUF_SIZE);
587 fprintf(G_logfile, "add_new_template_file: Failed to malloc %d bytes. Abort!\n", BUF_SIZE);
592 /* Find the beginning of the desc block */
593 file_size = get_file_size(filename);
594 fseek(file, file_size - 6, SEEK_SET);
595 fread(buf, 6, 1, file);
596 desc_start = file_size - read_le48((unsigned char *)buf);
598 /* Now seek back to the beginning image desc block to grab the MD5
600 fseek(file, file_size - 33, SEEK_SET);
601 fread(buf, BUF_SIZE, 1, file);
602 if (buf[0] != 5) /* image data */
604 fprintf(G_logfile, "add_new_template_file: Failed to find image desc in the template file\n");
609 /* Set up an entry in the template table for this template */
610 template.template_size = get_file_size(filename);
611 template.image_size = read_le48((unsigned char *)&buf[1]);
612 template.template_mtime = get_file_mtime(filename);
613 strncpy(template.template_name, filename, sizeof(template.template_name));
615 error = mk_MD5File(filename, tmp_md5sum);
618 fprintf(G_logfile, "add_new_template_file: failed to get md5sum of template file %s, error %d\n", filename, error);
621 md5_out = hex_dump(tmp_md5sum, 16);
622 strncpy(template.template_md5, md5_out, sizeof(template.template_md5));
625 md5_out = hex_dump(&buf[7], 16);
626 strncpy(template.image_md5, md5_out, sizeof(template.image_md5));
629 error = db_store_template(dbp, &template);
632 fprintf(G_logfile, "add_new_template_file: failed to store template entry for %s in the DB, error %d\n", filename, error);
636 /* Now seek back to the start of the desc block and start parsing
637 * the file/data entries to feed into the block table. */
638 fseek(file, desc_start, SEEK_SET);
639 fread(buf, 10, 1, file);
640 if (strncmp(buf, "DESC", 4))
642 fprintf(G_logfile, "Failed to find desc start in template file %s\n", filename);
646 if ((file_size - desc_start) != read_le48((unsigned char *)&buf[4]))
648 fprintf(G_logfile, "Inconsistent desc length in template file %s!\n", filename);
649 fprintf(G_logfile, "Final chunk says %lld, first chunk says %lld\n",
650 file_size - desc_start, read_le48((unsigned char *)&buf[4]));
655 template_offset = desc_start + 10;
657 strncpy(block.template_id, template.template_md5, sizeof(block.template_id));
659 /* Main loop - walk through the template file and dump each entry into the DB */
663 INT64 read_length = 0;
665 if (template_offset >= (file_size - 33))
668 fprintf(G_logfile, "Reached end of template file\n");
669 break; /* Finished! */
672 fseek(file, template_offset, SEEK_SET);
673 bytes = fread(buf, (MIN (BUF_SIZE, file_size - template_offset)), 1, file);
676 fprintf(G_logfile, "Failed to read template file %s!\n", filename);
681 extent_size = read_le48((unsigned char *)&buf[1]);
682 read_length = extent_size;
686 case 2: /* unmatched data */
687 template_offset += 7;
688 block.image_offset = image_offset;
689 block.size = extent_size;
690 block.uncomp_offset = uncomp_offset;
691 strncpy(block.template_id, template.template_md5, sizeof(block.template_id));
693 bzero(block.md5, sizeof(block.md5));
694 error = db_store_block(dbp, &block);
697 fprintf(G_logfile, "Failed to store unmatched data block at offset %lld in template file %s!\n", template_offset, filename);
702 uncomp_offset += extent_size;
706 template_offset += 31;
707 block.image_offset = image_offset;
708 block.size = extent_size;
709 block.uncomp_offset = 0;
711 md5_out = hex_dump(&buf[15], 16);
712 strncpy(block.md5, md5_out, sizeof(block.md5));
714 error = db_store_block(dbp, &block);
717 fprintf(G_logfile, "Failed to store file block at offset %lld in template file %s!\n", template_offset, filename);
725 fprintf(G_logfile, "Unknown block type %d in template file %s\n", buf[0], filename);
729 image_offset += extent_size;
733 fprintf(G_logfile, "Template file %s contains %d data blocks and %d file blocks\n",
734 filename, num_data_blocks, num_file_blocks);
736 /* Now go back to the start of the template file. Look at all the
737 * compressed blocks and add those to the "compressed" table */
738 /* Find the first compressed block */
739 fseek(file, 0, SEEK_SET);
740 fread(buf, BUF_SIZE, 1, file);
741 for (template_offset = 0; template_offset < BUF_SIZE; template_offset++)
743 if (!strncmp(&buf[template_offset], "DATA", 4))
745 if (!strncmp(&buf[template_offset], "BZIP", 4))
749 strncpy(compressed.template_id, template.template_md5, sizeof(compressed.template_id));
751 comp_offset = template_offset;
755 /* Now walk through the compressed blocks */
756 fseek(file, template_offset, SEEK_SET);
757 fread(buf, 16, 1, file);
759 if (!strncmp(buf, "DATA", 4))
760 compressed.comp_type = CT_GZIP;
761 else if (!strncmp(buf, "BZIP", 4))
762 compressed.comp_type = CT_BZIP2;
763 else if (!strncmp(buf, "DESC", 4))
767 fprintf(G_logfile, "Failed to find compressed block at offset %lld in template file %s!\n", template_offset, filename);
772 num_compressed_blocks++;
773 compressed.comp_offset = comp_offset + 16;
774 compressed.comp_size = read_le48((unsigned char *)&buf[4]) - 16;
775 compressed.uncomp_offset = uncomp_offset;
776 compressed.uncomp_size = read_le48((unsigned char *)&buf[10]);
778 uncomp_offset += compressed.uncomp_size;
779 comp_offset += read_le48((unsigned char *)&buf[4]);
780 template_offset += read_le48((unsigned char *)&buf[4]);
782 error = db_store_compressed(dbp, &compressed);
785 fprintf(G_logfile, "Failed to store file block at offset %lld in template file %s!\n", template_offset, filename);
792 fprintf(G_logfile, "Template file %s contains %d compressed blocks\n",
793 filename, num_compressed_blocks);