4 * Tool to create an ISO image from jigdo files
6 * Copyright (c) 2004 Steve McIntyre <steve@einval.com>
23 #include <sys/types.h>
32 typedef long long INT64;
33 typedef unsigned long long UINT64;
34 typedef unsigned long UINT32;
37 # define LLONG_MAX (INT64)INT_MAX * INT_MAX
40 #define BUF_SIZE 65536
44 #define MIN(x,y) ( ((x) < (y)) ? (x) : (y))
52 FILE *missing_file = NULL;
53 long long start_offset = 0;
54 long long end_offset = 0;
58 char *missing_filename = NULL;
70 typedef struct match_list_
72 struct match_list_ *next;
77 match_list_t *match_list_head = NULL;
78 match_list_t *match_list_tail = NULL;
80 typedef struct md5_list_
82 struct md5_list_ *next;
88 md5_list_t *md5_list_head = NULL;
89 md5_list_t *md5_list_tail = NULL;
95 INT64 offset_in_curr_buf;
99 /* Grab the file component from a full path */
100 static char *file_base_name(char *path)
115 static void write_missing_entry(char *missing, char *filename)
119 missing_file = fopen(missing, "wb");
122 fprintf(logfile, "write_missing_entry: Unable to open missing log %s; error %d\n", missing, errno);
126 fprintf(missing_file, "%s\n", filename);
129 static INT64 get_file_size(char *filename)
134 error = stat(filename, &sb);
141 static void display_progress(FILE *file, char *text)
143 INT64 written = ftello(file);
145 fprintf(logfile, "\r %5.2f%% %-60.60s",
146 100.0 * written / out_size, text);
149 static int add_match_entry(char *match)
151 match_list_t *entry = NULL;
152 char *mirror_path = NULL;
155 /* Split "Foo=/mirror/foo" into its components */
170 fprintf(logfile, "Could not parse malformed match entry \"%s\"\n", match);
174 entry = calloc(1, sizeof(*entry));
178 fprintf(logfile, "Adding match entry %s:%s\n", match, mirror_path);
180 entry->match = match;
181 entry->mirror_path = mirror_path;
183 if (!match_list_head)
185 match_list_head = entry;
186 match_list_tail = entry;
190 match_list_tail->next = entry;
191 match_list_tail = entry;
197 static int file_exists(char *path, INT64 *size)
202 error = stat(path, &sb);
203 if (!error && S_ISREG(sb.st_mode))
213 static md5_list_t *find_file_in_md5_list(unsigned char *base64_md5)
215 md5_list_t *md5_list_entry = md5_list_head;
217 while (md5_list_entry)
219 if (!memcmp(md5_list_entry->md5, base64_md5, 16))
220 return md5_list_entry;
222 md5_list_entry = md5_list_entry->next;
224 return NULL; /* Not found */
227 static int find_file_in_mirror(char *jigdo_match, char *jigdo_name,
228 char *match, INT64 *file_size, char **mirror_path)
230 match_list_t *entry = match_list_head;
235 if (!strcmp(entry->match, match))
237 sprintf(path, "%s/%s", entry->mirror_path, jigdo_name);
238 if (file_exists(path, file_size))
240 *mirror_path = strdup(path);
247 *mirror_path = jigdo_name;
252 static int add_md5_entry(INT64 size, char *md5, char *path)
254 md5_list_t *new = NULL;
255 new = calloc(1, sizeof(*new));
260 new->full_path = path;
261 new->file_size = size;
270 md5_list_tail->next = new;
277 static int parse_md5_entry(char *md5_entry)
280 char *file_name = NULL;
288 file_name = &md5_entry[24];
290 if ('\n' == file_name[strlen(file_name) -1])
291 file_name[strlen(file_name) - 1] = 0;
293 file_size = get_file_size(file_name);
295 error = add_md5_entry(file_size, md5, file_name);
299 static int parse_md5_file(char *filename)
306 file = fopen(filename, "rb");
309 fprintf(logfile, "Failed to open MD5 file %s, error %d!\n", filename, errno);
315 ret = fgets(buf, sizeof(buf), file);
318 error = parse_md5_entry(strdup(buf));
323 /* DELIBERATELY do not sort these, or do anything clever with
324 insertion. The entries in the jigdo file should be in the same
325 order as the ones we'll want from the template. Simply add to the
326 end of the singly-linked list each time! */
327 static int add_file_entry(char *jigdo_entry)
330 char *file_name = NULL;
332 char *ptr = jigdo_entry;
333 char *base64_md5 = NULL;
335 char *jigdo_name = NULL;
337 /* Grab out the component strings from the entry in the jigdo file */
338 base64_md5 = jigdo_entry;
347 else if (':' == *ptr)
353 else if ('\n' == *ptr)
359 if (find_file_in_md5_list(base64_md5))
360 return 0; /* We already have an entry for this file; don't
361 * waste any more time on it */
363 /* else look for the file in the filesystem */
364 if (NULL == match || NULL == jigdo_name)
366 fprintf(logfile, "Could not parse malformed jigdo entry \"%s\"\n", jigdo_entry);
369 error = find_file_in_mirror(match, jigdo_name, match, &file_size, &file_name);
373 if (missing_filename)
374 add_md5_entry(MISSING, base64_md5, file_name);
377 fprintf(logfile, "Unable to find a file to match %s\n", file_name);
378 fprintf(logfile, "Abort!\n");
383 add_md5_entry(file_size, base64_md5, file_name);
387 static int parse_jigdo_file(char *filename)
394 file = gzopen(filename, "rb");
397 fprintf(logfile, "Failed to open jigdo file %s, error %d!\n", filename, errno);
401 /* Find the [Parts] section of the jigdo file */
404 ret = gzgets(file, buf, sizeof(buf));
407 if (!strncmp(buf, "[Parts]", 7))
411 /* Now grab the individual file entries and build a list */
414 ret = gzgets(file, buf, sizeof(buf));
415 if (NULL == ret || !strcmp(buf, "\n"))
417 if (!strcmp(buf, "[") || !strcmp(buf, "#"))
419 error = add_file_entry(strdup(buf));
428 static int ungzip_data_block(char *in_buf, INT64 in_len, char *out_buf, INT64 out_len)
433 uc_stream.zalloc = NULL;
434 uc_stream.zfree = NULL;
435 uc_stream.opaque = NULL;
436 uc_stream.next_in = (unsigned char *)in_buf;
437 uc_stream.avail_in = in_len;
439 error = inflateInit(&uc_stream);
442 fprintf(logfile, "ungzip_data_block: failed to init, error %d\n", error);
446 uc_stream.next_out = (unsigned char *)out_buf;
447 uc_stream.avail_out = out_len;
449 error = inflate(&uc_stream, Z_FINISH);
450 if (Z_OK != error && Z_STREAM_END != error)
452 fprintf(logfile, "ungzip_data_block: failed to decompress, error %d\n", error);
456 error = inflateEnd(&uc_stream);
459 fprintf(logfile, "ungzip_data_block: failed to end, error %d\n", error);
467 static int unbzip_data_block(char *in_buf, INT64 in_len, char *out_buf, INT64 out_len)
472 uc_stream.bzalloc = NULL;
473 uc_stream.bzfree = NULL;
474 uc_stream.opaque = NULL;
475 uc_stream.next_in = in_buf;
476 uc_stream.avail_in = in_len;
478 error = BZ2_bzDecompressInit(&uc_stream, 0, 0);
481 fprintf(logfile, "unbzip_data_block: failed to init, error %d\n", error);
485 uc_stream.next_out = out_buf;
486 uc_stream.avail_out = out_len;
488 error = BZ2_bzDecompress(&uc_stream);
489 if (BZ_OK != error && BZ_STREAM_END != error)
491 fprintf(logfile, "unbzip_data_block: failed to decompress, error %d\n", error);
495 error = BZ2_bzDecompressEnd(&uc_stream);
498 fprintf(logfile, "unbzip_data_block: failed to end, error %d\n", error);
506 static int decompress_data_block(char *in_buf, INT64 in_len, char *out_buf,
507 INT64 out_len, int compress_type)
510 if (COMP_BZIP == compress_type)
511 return unbzip_data_block(in_buf, in_len, out_buf, out_len);
514 return ungzip_data_block(in_buf, in_len, out_buf, out_len);
517 static int read_data_block(FILE *template_file, int compress_type)
521 static INT64 template_offset = -1;
522 INT64 compressed_len = 0;
523 INT64 uncompressed_len = 0;
524 char *comp_buf = NULL;
528 if (-1 == template_offset)
530 fseek(template_file, 0, SEEK_SET);
531 fread(inbuf, sizeof(inbuf), 1, template_file);
532 for (i = 0; i < sizeof(inbuf); i++)
534 if (!strncmp(&inbuf[i], "DATA", 4))
540 if (-1 == template_offset)
542 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
548 fseek(template_file, template_offset, SEEK_SET);
549 fread(inbuf, 16, 1, template_file);
550 if (strncmp(inbuf, "DATA", 4))
552 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
557 compressed_len = read_le48((unsigned char *)&inbuf[4]);
558 uncompressed_len = read_le48((unsigned char *)&inbuf[10]);
560 comp_buf = calloc(1, compressed_len);
563 fprintf(logfile, "Unable to locate DATA block in template (offset %lld)\n",
568 zip_state.data_buf = calloc(1, uncompressed_len);
569 if (!zip_state.data_buf)
571 fprintf(logfile, "Unable to allocate %lld bytes for decompression\n",
576 read_num = fread(comp_buf, compressed_len, 1, template_file);
579 fprintf(logfile, "Unable to read %lld bytes for decompression\n",
584 error = decompress_data_block(comp_buf, compressed_len,
585 zip_state.data_buf, uncompressed_len, compress_type);
588 fprintf(logfile, "Unable to decompress data block, error %d\n", error);
592 template_offset += compressed_len;
593 zip_state.buf_size = uncompressed_len;
594 zip_state.offset_in_curr_buf = 0;
599 static int skip_data_block(INT64 data_size, FILE *template_file, int compress_type)
602 INT64 remaining = data_size;
605 /* If we're coming in in the middle of the image, we'll need to
606 skip through some compressed data */
609 if (!zip_state.data_buf)
611 error = read_data_block(template_file, compress_type);
614 fprintf(logfile, "Unable to decompress template data, error %d\n",
619 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
620 zip_state.offset_in_curr_buf += size;
623 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
625 free(zip_state.data_buf);
626 zip_state.data_buf = NULL;
630 fprintf(logfile, "skip_data_block: skipped %lld bytes of unmatched data\n", data_size);
634 static int parse_data_block(INT64 data_size, FILE *template_file,
635 struct mk_MD5Context *context, int compress_type)
638 INT64 remaining = data_size;
644 if (!zip_state.data_buf)
646 error = read_data_block(template_file, compress_type);
649 fprintf(logfile, "Unable to decompress template data, error %d\n",
654 size = MIN((zip_state.buf_size - zip_state.offset_in_curr_buf), remaining);
655 out_size = fwrite(&zip_state.data_buf[zip_state.offset_in_curr_buf], size, 1, outfile);
658 fprintf(logfile, "parse_data_block: fwrite %lld failed with error %d; aborting\n", size, ferror(outfile));
659 return ferror(outfile);
663 display_progress(outfile, "template data");
666 mk_MD5Update(context,
667 (unsigned char *)&zip_state.data_buf[zip_state.offset_in_curr_buf],
669 zip_state.offset_in_curr_buf += size;
672 if (zip_state.offset_in_curr_buf == zip_state.buf_size)
674 free(zip_state.data_buf);
675 zip_state.data_buf = NULL;
679 fprintf(logfile, "parse_data_block: wrote %lld bytes of unmatched data\n", data_size);
683 static int read_file_data(char *filename, char *missing, INT64 offset, INT64 data_size,
684 struct mk_MD5Context *file_context, struct mk_MD5Context *image_context)
686 FILE *input_file = NULL;
687 INT64 remaining = data_size;
692 input_file = fopen(filename, "rb");
695 fprintf(logfile, "Unable to open mirror file %s, error %d\n",
706 fseek(input_file, offset, SEEK_SET);
709 int size = MIN(BUF_SIZE, remaining);
710 memset(buf, 0, BUF_SIZE);
712 num_read = fread(buf, size, 1, input_file);
715 fprintf(logfile, "Unable to read from mirror file %s, error %d (offset %ld, length %d)\n",
716 filename, errno, ftell(input_file), size);
722 mk_MD5Update(image_context, (unsigned char *)buf, size);
723 mk_MD5Update(file_context, (unsigned char *)buf, size);
726 out_size = fwrite(buf, size, 1, outfile);
729 fprintf(logfile, "read_file_data: fwrite %d failed with error %d; aborting\n", size, ferror(outfile));
730 return ferror(outfile);
734 display_progress(outfile, file_base_name(filename));
739 fprintf(logfile, "read_file_data: wrote %lld bytes of data from %s\n",
740 data_size, filename);
745 static int parse_file_block(INT64 offset, INT64 data_size, INT64 file_size, JIGDB *dbp,
746 unsigned char *md5, struct mk_MD5Context *image_context,
749 char *base64_md5 = base64_dump(md5, 16);
750 struct mk_MD5Context file_context;
751 struct mk_MD5Context *use_context = NULL;
752 unsigned char file_md5[16];
753 md5_list_t *md5_list_entry = NULL;
754 db_entry_t *db_entry = NULL;
756 char *filename = NULL;
760 use_context = &file_context;
761 mk_MD5Init(use_context);
764 /* Try the DB first if we have one */
767 error = db_lookup_by_md5(dbp, base64_md5, &db_entry);
769 filename = db_entry->filename;
772 /* No joy; fall back to the MD5 list */
775 md5_list_entry = find_file_in_md5_list(base64_md5);
776 if (md5_list_entry && file_size == md5_list_entry->file_size)
777 filename = md5_list_entry->full_path;
782 error = read_file_data(filename, missing, offset, data_size,
783 use_context, image_context);
785 if (error && (ENOENT != error))
787 fprintf(logfile, "Failed to read file %s, error %d\n", filename, error);
793 mk_MD5Final(file_md5, &file_context);
795 if (memcmp(file_md5, md5, 16))
797 fprintf(logfile, "MD5 MISMATCH for file %s\n", filename);
798 fprintf(logfile, " template looking for %s\n", md5);
799 fprintf(logfile, " file %s is %s\n", filename, file_md5);
806 /* No file found. Add it to the list of missing files, or complain */
808 (MISSING == md5_list_entry->file_size) &&
809 (!memcmp(md5_list_entry->md5, base64_md5, 16) ) )
811 write_missing_entry(missing, md5_list_entry->full_path);
818 static int parse_template_file(char *filename, int sizeonly,
819 char *missing, char *output_name, char *db_filename)
821 INT64 template_offset = 0;
826 INT64 desc_start = 0;
827 INT64 written_length = 0;
828 INT64 output_offset = 0;
831 struct mk_MD5Context template_context;
832 unsigned char image_md5sum[16];
835 zip_state.total_offset = 0;
837 file = fopen(filename, "rb");
840 fprintf(logfile, "Failed to open template file %s, error %d!\n", filename, errno);
846 dbp = db_open(db_filename);
849 fprintf(logfile, "Failed to open DB file %s, error %d\n", db_filename, errno);
854 buf = malloc(BUF_SIZE);
857 fprintf(logfile, "Failed to malloc %d bytes. Abort!\n", BUF_SIZE);
862 /* Find the beginning of the desc block */
863 file_size = get_file_size(filename);
864 fseek(file, file_size - 6, SEEK_SET);
865 fread(buf, 6, 1, file);
866 desc_start = file_size - read_le48((unsigned char *)buf);
868 /* Now seek back to the beginning image desc block to grab the MD5
870 fseek(file, file_size - 33, SEEK_SET);
871 fread(buf, BUF_SIZE, 1, file);
872 if (buf[0] != 5) /* image data */
874 fprintf(logfile, "Failed to find image desc in the template file\n");
882 printf("%lld\n", read_le48((unsigned char *)&buf[1]));
888 fprintf(logfile, "Image MD5 should be ");
889 for (i = 0; i < 16; i++)
890 fprintf(logfile, "%2.2x", (unsigned char)buf[i+7]);
891 fprintf(logfile, "\n");
892 fprintf(logfile, "Image size should be %lld bytes\n", read_le48((unsigned char *)&buf[1]));
895 out_size = read_le48((unsigned char *)&buf[1]);
897 /* Now seek back to the start of the desc block */
898 fseek(file, desc_start, SEEK_SET);
899 fread(buf, 10, 1, file);
900 if (strncmp(buf, "DESC", 4))
902 fprintf(logfile, "Failed to find desc start in the template file\n");
906 if ((file_size - desc_start) != read_le48((unsigned char *)&buf[4]))
908 fprintf(logfile, "Inconsistent desc length in the template file!\n");
909 fprintf(logfile, "Final chunk says %lld, first chunk says %lld\n",
910 file_size - desc_start, read_le48((unsigned char *)&buf[4]));
916 mk_MD5Init(&template_context);
917 template_offset = desc_start + 10;
920 fprintf(logfile, "Creating ISO image %s\n", output_name);
922 /* Main loop - walk through the template file and expand each entry we find */
927 INT64 read_length = 0;
929 if (template_offset >= (file_size - 33))
932 fprintf(logfile, "Reached end of template file\n");
933 break; /* Finished! */
936 if (output_offset > end_offset) /* Past the range we were asked for */
938 fprintf(logfile, "Reached end of range requested\n");
942 fseek(file, template_offset, SEEK_SET);
943 bytes = fread(buf, (MIN (BUF_SIZE, file_size - template_offset)), 1, file);
946 fprintf(logfile, "Failed to read template file!\n");
951 extent_size = read_le48((unsigned char *)&buf[1]);
952 read_length = extent_size;
954 if (start_offset > output_offset)
955 skip = start_offset - output_offset;
956 if ((output_offset + extent_size) > end_offset)
957 read_length -= (output_offset + extent_size - end_offset - 1);
963 case 2: /* unmatched data, gzip */
964 case 8: /* unmatched data, bzip2 */
965 template_offset += 7;
968 if ((output_offset + extent_size) >= start_offset)
971 error = skip_data_block(skip, file, buf[0]);
974 fprintf(logfile, "Unable to read data block to skip, error %d\n", error);
978 error = parse_data_block(read_length, file, &template_context, buf[0]);
981 fprintf(logfile, "Unable to read data block, error %d\n", error);
985 written_length += read_length;
988 error = skip_data_block(extent_size, file, buf[0]);
991 template_offset += 31;
992 if ((output_offset + extent_size) >= start_offset)
994 error = parse_file_block(skip, read_length, extent_size, dbp,
995 (unsigned char *)&buf[15], &template_context, missing);
998 fprintf(logfile, "Unable to read file block, error %d\n", error);
1002 written_length += read_length;
1006 fprintf(logfile, "Unknown block type %d!\n", buf[0]);
1010 output_offset += extent_size;
1013 if (missing && missing_file)
1019 fprintf(logfile, "\n");
1022 mk_MD5Final (image_md5sum, &template_context);
1023 fprintf(logfile, "Output image MD5 is ");
1024 for (i = 0; i < 16; i++)
1025 fprintf(logfile, "%2.2x", image_md5sum[i]);
1026 fprintf(logfile, "\n");
1028 fprintf(logfile, "Output image length is %lld bytes\n", written_length);
1034 static void usage(char *progname)
1036 printf("%s [OPTIONS]\n\n", progname);
1037 printf(" Options:\n");
1038 printf(" -M <missing name> Rather than try to build the image, just check that\n");
1039 printf(" all the needed files are available. If any are missing,\n");
1040 printf(" list them in this file.\n");
1041 printf(" -d <DB name> Specify an input MD5 database file, as created by jigsum\n");
1042 printf(" -e <bytenum> End byte number; will end at EOF if not specified\n");
1043 printf(" -f <MD5 name> Specify an input MD5 file. MD5s must be in jigdo's\n");
1044 printf(" pseudo-base64 format\n");
1045 printf(" -j <jigdo name> Specify the input jigdo file\n");
1046 printf(" -l <logfile> Specify a logfile to append to.\n");
1047 printf(" If not specified, will log to stderr\n");
1048 printf(" -m <item=path> Map <item> to <path> to find the files in the mirror\n");
1049 printf(" -o <outfile> Specify a file to write the ISO image to.\n");
1050 printf(" If not specified, will write to stdout\n");
1051 printf(" -q Quick mode. Don't check MD5sums. Dangerous!\n");
1052 printf(" -s <bytenum> Start byte number; will start at 0 if not specified\n");
1053 printf(" -t <template name> Specify the input template file\n");
1054 printf(" -v Make the output logging more verbose\n");
1055 printf(" -z Don't attempt to rebuild the image; simply print its\n");
1056 printf(" size in bytes\n");
1059 int main(int argc, char **argv)
1061 char *template_filename = NULL;
1062 char *jigdo_filename = NULL;
1063 char *md5_filename = NULL;
1064 char *output_name = NULL;
1065 char *db_filename = NULL;
1073 bzero(&zip_state, sizeof(zip_state));
1077 c = getopt(argc, argv, ":?M:d:e:f:h:j:l:m:o:qs:t:vz");
1090 logfile = fopen(optarg, "ab");
1093 fprintf(stderr, "Unable to open log file %s\n", optarg);
1096 setlinebuf(logfile);
1099 output_name = optarg;
1100 outfile = fopen(output_name, "wb");
1103 fprintf(stderr, "Unable to open output file %s\n", optarg);
1110 fprintf(logfile, "Can only specify one jigdo file!\n");
1114 jigdo_filename = optarg;
1117 if (template_filename)
1119 fprintf(logfile, "Can only specify one template file!\n");
1123 template_filename = optarg;
1128 fprintf(logfile, "Can only specify one MD5 file!\n");
1132 md5_filename = optarg;
1137 fprintf(logfile, "Can only specify one db file!\n");
1141 db_filename = optarg;
1144 error = add_match_entry(strdup(optarg));
1149 missing_filename = optarg;
1152 fprintf(logfile, "Missing argument!\n");
1161 start_offset = strtoull(optarg, NULL, 10);
1162 if (start_offset != 0)
1166 end_offset = strtoull(optarg, NULL, 10);
1167 if (end_offset != 0)
1174 fprintf(logfile, "Unknown option!\n");
1179 if (0 == end_offset)
1180 end_offset = LLONG_MAX;
1182 if ((NULL == jigdo_filename) &&
1183 (NULL == md5_filename) &&
1184 (NULL == db_filename) &&
1187 fprintf(logfile, "No jigdo file, DB file or MD5 file specified!\n");
1192 if (NULL == template_filename)
1194 fprintf(logfile, "No template file specified!\n");
1201 /* Build up a list of the files we've been fed */
1202 error = parse_md5_file(md5_filename);
1205 fprintf(logfile, "Unable to parse the MD5 file %s\n", md5_filename);
1212 /* Build up a list of file mappings */
1213 error = parse_jigdo_file(jigdo_filename);
1216 fprintf(logfile, "Unable to parse the jigdo file %s\n", jigdo_filename);
1222 output_name = "to stdout";
1223 /* Read the template file and actually build the image to <outfile> */
1224 error = parse_template_file(template_filename, sizeonly,
1225 missing_filename, output_name, db_filename);
1228 fprintf(logfile, "Unable to recreate image from template file %s\n", template_filename);
1229 if (missing_filename)
1230 fprintf(logfile, "%s contains the list of missing files\n", missing_filename);