4 * Tool to create an ISO image from jigdo files
6 * Copyright (c) 2004 Steve McIntyre <steve@einval.com>
19 #include <sys/types.h>
31 FILE *missing_file = NULL;
32 long long start_offset = 0;
33 long long end_offset = 0;
37 char *missing_filename = NULL;
49 match_list_t *match_list_head = NULL;
50 match_list_t *match_list_tail = NULL;
52 md5_list_t *md5_list_head = NULL;
53 md5_list_t *md5_list_tail = NULL;
66 INT64 offset_in_curr_buf;
70 /* Grab the file component from a full path */
71 static char *file_base_name(char *path)
86 static void write_missing_entry(char *missing, char *filename)
90 missing_file = fopen(missing, "wb");
93 fprintf(logfile, "write_missing_entry: Unable to open missing log %s; error %d\n", missing, errno);
97 fprintf(missing_file, "%s\n", filename);
100 static void display_progress(FILE *file, char *text)
102 INT64 written = ftello(file);
104 fprintf(logfile, "\r %5.2f%% %-60.60s",
105 100.0 * written / out_size, text);
108 static int add_match_entry(char *match)
110 match_list_t *entry = NULL;
111 char *mirror_path = NULL;
114 /* Split "Foo=/mirror/foo" into its components */
129 fprintf(logfile, "Could not parse malformed match entry \"%s\"\n", match);
133 entry = calloc(1, sizeof(*entry));
137 fprintf(logfile, "Adding match entry %s:%s\n", match, mirror_path);
139 entry->match = match;
140 entry->mirror_path = mirror_path;
142 if (!match_list_head)
144 match_list_head = entry;
145 match_list_tail = entry;
149 match_list_tail->next = entry;
150 match_list_tail = entry;
156 static int read_data_block(FILE *template_file)
160 static INT64 template_offset = -1;
161 INT64 compressed_len = 0;
162 INT64 uncompressed_len = 0;
163 char *comp_buf = NULL;
167 if (-1 == template_offset)
169 fseek(template_file, 0, SEEK_SET);
170 fread(inbuf, sizeof(inbuf), 1, template_file);
171 for (i = 0; i < sizeof(inbuf); i++)
173 if (!strncmp(&inbuf[i], "DATA", 4))
175 zip_state.algorithm = CT_GZIP;
179 if (!strncmp(&inbuf[i], "BZIP", 4))
181 zip_state.algorithm = CT_BZIP2;
186 if (-1 == template_offset)
188 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
194 fseek(template_file, template_offset, SEEK_SET);
195 fread(inbuf, 16, 1, template_file);
196 if (strncmp(inbuf, "DATA", 4) && strncmp(inbuf, "BZIP", 4))
198 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
203 compressed_len = read_le48((unsigned char *)&inbuf[4]);
204 uncompressed_len = read_le48((unsigned char *)&inbuf[10]);
206 comp_buf = calloc(1, compressed_len);
209 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
214 zip_state.data_buf = calloc(1, uncompressed_len);
215 if (!zip_state.data_buf)
217 fprintf(logfile, "Unable to allocate %lld bytes for decompression\n",
222 read_num = fread(comp_buf, compressed_len, 1, template_file);
225 fprintf(logfile, "Unable to read %lld bytes for decompression\n",
230 error = decompress_data_block(comp_buf, compressed_len,
231 zip_state.data_buf, uncompressed_len, zip_state.algorithm);
234 fprintf(logfile, "Unable to decompress data block, error %d\n", error);
238 template_offset += compressed_len;
239 zip_state.buf_size = uncompressed_len;
240 zip_state.offset_in_curr_buf = 0;
245 static int skip_data_block(INT64 data_size, FILE *template_file)
248 INT64 remaining = data_size;
251 /* If we're coming in in the middle of the image, we'll need to
252 skip through some compressed data */
255 if (!zip_state.data_buf)
257 error = read_data_block(template_file);
260 fprintf(logfile, "Unable to decompress template data, error %d\n",
265 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
266 zip_state.offset_in_curr_buf += size;
269 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
271 free(zip_state.data_buf);
272 zip_state.data_buf = NULL;
276 fprintf(logfile, "skip_data_block: skipped %lld bytes of unmatched data\n", data_size);
280 static int parse_data_block(INT64 data_size, FILE *template_file, struct mk_MD5Context *context)
283 INT64 remaining = data_size;
289 if (!zip_state.data_buf)
291 error = read_data_block(template_file);
294 fprintf(logfile, "Unable to decompress template data, error %d\n",
299 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
300 out_size = fwrite(&zip_state.data_buf[zip_state.offset_in_curr_buf], size, 1, outfile);
303 fprintf(logfile, "parse_data_block: fwrite %lld failed with error %d; aborting\n", size, ferror(outfile));
304 return ferror(outfile);
308 display_progress(outfile, "template data");
311 mk_MD5Update(context,
312 (unsigned char *)&zip_state.data_buf[zip_state.offset_in_curr_buf],
314 zip_state.offset_in_curr_buf += size;
317 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
319 free(zip_state.data_buf);
320 zip_state.data_buf = NULL;
324 fprintf(logfile, "parse_data_block: wrote %lld bytes of unmatched data\n", data_size);
328 static int read_file_data(char *filename, char *missing, INT64 offset, INT64 data_size,
329 struct mk_MD5Context *file_context, struct mk_MD5Context *image_context)
331 FILE *input_file = NULL;
332 INT64 remaining = data_size;
337 input_file = fopen(filename, "rb");
340 fprintf(logfile, "Unable to open mirror file %s, error %d\n",
351 fseek(input_file, offset, SEEK_SET);
354 int size = MIN(BUF_SIZE, remaining);
355 memset(buf, 0, BUF_SIZE);
357 num_read = fread(buf, size, 1, input_file);
360 fprintf(logfile, "Unable to read from mirror file %s, error %d (offset %ld, length %d)\n",
361 filename, errno, ftell(input_file), size);
367 mk_MD5Update(image_context, (unsigned char *)buf, size);
368 mk_MD5Update(file_context, (unsigned char *)buf, size);
371 out_size = fwrite(buf, size, 1, outfile);
374 fprintf(logfile, "read_file_data: fwrite %d failed with error %d; aborting\n", size, ferror(outfile));
375 return ferror(outfile);
379 display_progress(outfile, file_base_name(filename));
384 fprintf(logfile, "read_file_data: wrote %lld bytes of data from %s\n",
385 data_size, filename);
390 static int parse_file_block(INT64 offset, INT64 data_size, INT64 file_size, JIGDB *dbp,
391 unsigned char *md5, struct mk_MD5Context *image_context,
394 char *base64_md5 = base64_dump(md5, 16);
395 struct mk_MD5Context file_context;
396 struct mk_MD5Context *use_context = NULL;
397 unsigned char file_md5[16];
398 md5_list_t *md5_list_entry = NULL;
399 db_file_entry_t *db_entry = NULL;
401 char *filename = NULL;
405 use_context = &file_context;
406 mk_MD5Init(use_context);
409 /* Try the DB first if we have one */
412 error = db_lookup_file_by_md5(dbp, base64_md5, &db_entry);
414 filename = db_entry->filename;
417 /* No joy; fall back to the MD5 list */
420 md5_list_entry = find_file_in_md5_list(base64_md5);
421 if (md5_list_entry && file_size == md5_list_entry->file_size)
422 filename = md5_list_entry->full_path;
427 error = read_file_data(filename, missing, offset, data_size,
428 use_context, image_context);
430 if (error && (ENOENT != error))
432 fprintf(logfile, "Failed to read file %s, error %d\n", filename, error);
438 mk_MD5Final(file_md5, &file_context);
440 if (memcmp(file_md5, md5, 16))
442 fprintf(logfile, "MD5 MISMATCH for file %s\n", filename);
443 fprintf(logfile, " template looking for %s\n", md5);
444 fprintf(logfile, " file %s is %s\n", filename, file_md5);
451 /* No file found. Add it to the list of missing files, or complain */
453 (MISSING == md5_list_entry->file_size) &&
454 (!memcmp(md5_list_entry->md5, base64_md5, 16) ) )
456 write_missing_entry(missing, md5_list_entry->full_path);
463 static int parse_template_file(char *filename, int sizeonly,
464 char *missing, char *output_name, char *db_filename)
466 INT64 template_offset = 0;
471 INT64 desc_start = 0;
472 INT64 written_length = 0;
473 INT64 output_offset = 0;
476 struct mk_MD5Context template_context;
477 unsigned char image_md5sum[16];
480 zip_state.total_offset = 0;
482 file = fopen(filename, "rb");
485 fprintf(logfile, "Failed to open template file %s, error %d!\n", filename, errno);
491 dbp = db_open(db_filename);
494 fprintf(logfile, "Failed to open DB file %s, error %d\n", db_filename, errno);
497 /* If we have a DB, then we should cache the template
498 * information in it too. Check and see if there is
499 * information about this template file in the database
503 buf = malloc(BUF_SIZE);
506 fprintf(logfile, "Failed to malloc %d bytes. Abort!\n", BUF_SIZE);
511 /* Find the beginning of the desc block */
512 file_size = get_file_size(filename);
513 fseek(file, file_size - 6, SEEK_SET);
514 fread(buf, 6, 1, file);
515 desc_start = file_size - read_le48((unsigned char *)buf);
517 /* Now seek back to the beginning image desc block to grab the MD5
519 fseek(file, file_size - 33, SEEK_SET);
520 fread(buf, BUF_SIZE, 1, file);
521 if (buf[0] != 5) /* image data */
523 fprintf(logfile, "Failed to find image desc in the template file\n");
531 printf("%lld\n", read_le48((unsigned char *)&buf[1]));
537 fprintf(logfile, "Image MD5 should be ");
538 for (i = 0; i < 16; i++)
539 fprintf(logfile, "%2.2x", (unsigned char)buf[i+7]);
540 fprintf(logfile, "\n");
541 fprintf(logfile, "Image size should be %lld bytes\n", read_le48((unsigned char *)&buf[1]));
544 out_size = read_le48((unsigned char *)&buf[1]);
546 /* Now seek back to the start of the desc block */
547 fseek(file, desc_start, SEEK_SET);
548 fread(buf, 10, 1, file);
549 if (strncmp(buf, "DESC", 4))
551 fprintf(logfile, "Failed to find desc start in the template file\n");
555 if ((file_size - desc_start) != read_le48((unsigned char *)&buf[4]))
557 fprintf(logfile, "Inconsistent desc length in the template file!\n");
558 fprintf(logfile, "Final chunk says %lld, first chunk says %lld\n",
559 file_size - desc_start, read_le48((unsigned char *)&buf[4]));
565 mk_MD5Init(&template_context);
566 template_offset = desc_start + 10;
569 fprintf(logfile, "Creating ISO image %s\n", output_name);
571 /* Main loop - walk through the template file and expand each entry we find */
576 INT64 read_length = 0;
578 if (template_offset >= (file_size - 33))
581 fprintf(logfile, "Reached end of template file\n");
582 break; /* Finished! */
585 if (output_offset > end_offset) /* Past the range we were asked for */
587 fprintf(logfile, "Reached end of range requested\n");
591 fseek(file, template_offset, SEEK_SET);
592 bytes = fread(buf, (MIN (BUF_SIZE, file_size - template_offset)), 1, file);
595 fprintf(logfile, "Failed to read template file!\n");
600 extent_size = read_le48((unsigned char *)&buf[1]);
601 read_length = extent_size;
603 if (start_offset > output_offset)
604 skip = start_offset - output_offset;
605 if ((output_offset + extent_size) > end_offset)
606 read_length -= (output_offset + extent_size - end_offset - 1);
612 case 2: /* unmatched data */
613 template_offset += 7;
616 if ((output_offset + extent_size) >= start_offset)
619 error = skip_data_block(skip, file);
622 fprintf(logfile, "Unable to read data block to skip, error %d\n", error);
626 error = parse_data_block(read_length, file, &template_context);
629 fprintf(logfile, "Unable to read data block, error %d\n", error);
633 written_length += read_length;
636 error = skip_data_block(extent_size, file);
639 template_offset += 31;
640 if ((output_offset + extent_size) >= start_offset)
642 error = parse_file_block(skip, read_length, extent_size, dbp,
643 (unsigned char *)&buf[15], &template_context, missing);
646 fprintf(logfile, "Unable to read file block, error %d\n", error);
650 written_length += read_length;
654 fprintf(logfile, "Unknown block type %d!\n", buf[0]);
658 output_offset += extent_size;
661 if (missing && missing_file)
667 fprintf(logfile, "\n");
670 mk_MD5Final (image_md5sum, &template_context);
671 fprintf(logfile, "Output image MD5 is ");
672 for (i = 0; i < 16; i++)
673 fprintf(logfile, "%2.2x", image_md5sum[i]);
674 fprintf(logfile, "\n");
676 fprintf(logfile, "Output image length is %lld bytes\n", written_length);
682 static void usage(char *progname)
684 printf("%s [OPTIONS]\n\n", progname);
685 printf(" Options:\n");
686 printf(" -M <missing name> Rather than try to build the image, just check that\n");
687 printf(" all the needed files are available. If any are missing,\n");
688 printf(" list them in this file.\n");
689 printf(" -d <DB name> Specify an input MD5 database file, as created by jigsum\n");
690 printf(" -e <bytenum> End byte number; will end at EOF if not specified\n");
691 printf(" -f <MD5 name> Specify an input MD5 file. MD5s must be in jigdo's\n");
692 printf(" pseudo-base64 format\n");
693 printf(" -j <jigdo name> Specify the input jigdo file\n");
694 printf(" -l <logfile> Specify a logfile to append to.\n");
695 printf(" If not specified, will log to stderr\n");
696 printf(" -m <item=path> Map <item> to <path> to find the files in the mirror\n");
697 printf(" -o <outfile> Specify a file to write the ISO image to.\n");
698 printf(" If not specified, will write to stdout\n");
699 printf(" -q Quick mode. Don't check MD5sums. Dangerous!\n");
700 printf(" -s <bytenum> Start byte number; will start at 0 if not specified\n");
701 printf(" -t <template name> Specify the input template file\n");
702 printf(" -v Make the output logging more verbose\n");
703 printf(" -z Don't attempt to rebuild the image; simply print its\n");
704 printf(" size in bytes\n");
707 int main(int argc, char **argv)
709 char *template_filename = NULL;
710 char *jigdo_filename = NULL;
711 char *md5_filename = NULL;
712 char *output_name = NULL;
713 char *db_filename = NULL;
721 bzero(&zip_state, sizeof(zip_state));
725 c = getopt(argc, argv, ":?M:d:e:f:h:j:l:m:o:qs:t:vz");
738 logfile = fopen(optarg, "ab");
741 fprintf(stderr, "Unable to open log file %s\n", optarg);
747 output_name = optarg;
748 outfile = fopen(output_name, "wb");
751 fprintf(stderr, "Unable to open output file %s\n", optarg);
758 fprintf(logfile, "Can only specify one jigdo file!\n");
762 jigdo_filename = optarg;
765 if (template_filename)
767 fprintf(logfile, "Can only specify one template file!\n");
771 template_filename = optarg;
776 fprintf(logfile, "Can only specify one MD5 file!\n");
780 md5_filename = optarg;
785 fprintf(logfile, "Can only specify one db file!\n");
789 db_filename = optarg;
792 error = add_match_entry(strdup(optarg));
797 missing_filename = optarg;
800 fprintf(logfile, "Missing argument!\n");
809 start_offset = strtoull(optarg, NULL, 10);
810 if (start_offset != 0)
814 end_offset = strtoull(optarg, NULL, 10);
822 fprintf(logfile, "Unknown option!\n");
828 end_offset = LLONG_MAX;
830 if ((NULL == jigdo_filename) &&
831 (NULL == md5_filename) &&
832 (NULL == db_filename) &&
835 fprintf(logfile, "No jigdo file, DB file or MD5 file specified!\n");
840 if (NULL == template_filename)
842 fprintf(logfile, "No template file specified!\n");
849 /* Build up a list of the files we've been fed */
850 error = parse_md5_file(md5_filename);
853 fprintf(logfile, "Unable to parse the MD5 file %s\n", md5_filename);
860 /* Build up a list of file mappings */
861 error = parse_jigdo_file(jigdo_filename);
864 fprintf(logfile, "Unable to parse the jigdo file %s\n", jigdo_filename);
870 output_name = "to stdout";
871 /* Read the template file and actually build the image to <outfile> */
872 error = parse_template_file(template_filename, sizeonly,
873 missing_filename, output_name, db_filename);
876 fprintf(logfile, "Unable to recreate image from template file %s\n", template_filename);
877 if (missing_filename)
878 fprintf(logfile, "%s contains the list of missing files\n", missing_filename);