+/*
+ * mkimage
+ *
+ * Tool to create an ISO image from jigdo files
+ *
+ * Copyright (c) 2004 Steve McIntyre <steve@einval.com>
+ *
+ * GPL v2 - see COPYING
+ */
+
#undef BZ2_SUPPORT
#include <errno.h>
#ifdef BZ2_SUPPORT
# include <bzlib.h>
#endif
-#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "endian.h"
#include "md5.h"
+typedef long long INT64;
typedef unsigned long long UINT64;
typedef unsigned long UINT32;
+#ifndef LLONG_MAX
+# define LLONG_MAX (INT64)INT_MAX * INT_MAX
+#endif
+
#define BUF_SIZE 65536
+#define MISSING -1
#ifndef MIN
#define MIN(x,y) ( ((x) < (y)) ? (x) : (y))
FILE *logfile = NULL;
FILE *outfile = NULL;
-unsigned long long start_offset = 0;
-unsigned long long end_offset = 0;
+FILE *missing_file = NULL;
+long long start_offset = 0;
+long long end_offset = 0;
int quick = 0;
+int verbose = 0;
+UINT64 out_size = 0;
typedef enum state_
{
match_list_t *match_list_head = NULL;
match_list_t *match_list_tail = NULL;
-typedef struct jigdo_list_
+typedef struct md5_list_
{
- struct jigdo_list_ *next;
- off_t file_size;
+ struct md5_list_ *next;
+ INT64 file_size;
char *md5;
char *full_path;
-} jigdo_list_t;
+} md5_list_t;
-jigdo_list_t *jigdo_list_head = NULL;
-jigdo_list_t *jigdo_list_tail = NULL;
+md5_list_t *md5_list_head = NULL;
+md5_list_t *md5_list_tail = NULL;
struct
{
char *data_buf;
- size_t buf_size;
- off_t offset_in_curr_buf;
- off_t total_offset;
+ INT64 buf_size;
+ INT64 offset_in_curr_buf;
+ INT64 total_offset;
} zip_state;
+/* Grab the file component from a full path */
+static char *file_base_name(char *path)
+{
+ char *endptr = path;
+ char *ptr = path;
+
+ while (*ptr != '\0')
+ {
+ if ('/' == *ptr)
+ endptr = ++ptr;
+ else
+ ++ptr;
+ }
+ return endptr;
+}
+
+static void write_missing_entry(char *missing, char *filename)
+{
+ if (!missing_file)
+ {
+ missing_file = fopen(missing, "wb");
+ if (!missing_file)
+ {
+ fprintf(logfile, "write_missing_entry: Unable to open missing log %s; error %d\n", missing, errno);
+ exit(1);
+ }
+ }
+ fprintf(missing_file, "%s\n", filename);
+}
+
+static INT64 get_file_size(char *filename)
+{
+ struct stat sb;
+ int error = 0;
+
+ error = stat(filename, &sb);
+ if (error)
+ return MISSING;
+ else
+ return sb.st_size;
+}
+
+static void display_progress(FILE *file, char *text)
+{
+ INT64 written = ftello(file);
+ if (out_size > 0)
+ fprintf(logfile, "\r %5.2f%% %-60.60s",
+ 100.0 * written / out_size, text);
+}
+
static int add_match_entry(char *match)
{
match_list_t *entry = NULL;
return 0;
}
-static int file_exists(char *path, off_t *size)
+static int file_exists(char *path, INT64 *size)
{
struct stat sb;
int error = 0;
return 0;
}
-static char *base64_dump(unsigned char *buf, size_t buf_size)
-{
- const char *b64_enc = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
- int value = 0;
- unsigned int i;
- int bits = 0;
- char *out = calloc(1, 30);
- unsigned int out_pos = 0;
-
- if (!out)
- return NULL;
-
- for (i = 0; i < buf_size ; i++)
- {
- value = (value << 8) | buf[i];
- bits += 2;
- out[out_pos++] = b64_enc[(value >> bits) & 63U];
- if (bits >= 8) {
- bits -= 6;
- out[out_pos++] = b64_enc[(value >> bits) & 63U];
- }
- }
- if (bits > 0)
- {
- value <<= 8 - bits;
- out[out_pos++] = b64_enc[(value >> bits) & 63U];
- }
- return out;
-}
-
-static int find_file_in_mirror(char *jigdo_entry, char **mirror_path, char **md5sum, off_t *file_size)
+static int find_file_in_mirror(char *jigdo_entry, char **mirror_path, char **md5sum, INT64 *file_size)
{
match_list_t *entry = match_list_head;
char path[PATH_MAX];
entry = entry->next;
}
- fprintf(logfile, "Could not find file %s:%s in any path\n", match, jigdo_name);
+ *mirror_path = jigdo_name;
return ENOENT;
}
-/* DELIBERATELY do not sort these, or do anything clever with
- insertion. The entries in the jigdo file should be in the same
- order as the ones we'll want from the template. Simply add to the
- end of the singly-linked list each time! */
-static int add_file_entry(char *jigdo_entry)
+
+static int add_md5_entry(INT64 size, char *md5, char *path)
{
- int error = 0;
- char *file_name = NULL;
- char *md5 = NULL;
- jigdo_list_t *new = NULL;
- off_t file_size = 0;
-
- error = find_file_in_mirror(jigdo_entry, &file_name, &md5, &file_size);
- if (error)
- return error;
-
+ md5_list_t *new = NULL;
new = calloc(1, sizeof(*new));
if (!new)
return ENOMEM;
new->md5 = md5;
- new->full_path = file_name;
- new->file_size = file_size;
+ new->full_path = path;
+ new->file_size = size;
- if (!jigdo_list_head)
+ if (!md5_list_head)
{
- jigdo_list_head = new;
- jigdo_list_tail = new;
+ md5_list_head = new;
+ md5_list_tail = new;
}
else
{
- jigdo_list_tail->next = new;
- jigdo_list_tail = new;
+ md5_list_tail->next = new;
+ md5_list_tail = new;
}
return 0;
}
+static int parse_md5_entry(char *md5_entry)
+{
+ int error = 0;
+ char *file_name = NULL;
+ char *md5 = NULL;
+ INT64 file_size = 0;
+
+ md5_entry[22] = 0;
+ md5_entry[23] = 0;
+
+ md5 = md5_entry;
+ file_name = &md5_entry[24];
+
+ if ('\n' == file_name[strlen(file_name) -1])
+ file_name[strlen(file_name) - 1] = 0;
+
+ file_size = get_file_size(file_name);
+
+ error = add_md5_entry(file_size, md5, file_name);
+ return 0;
+}
+
+static int parse_md5_file(char *filename)
+{
+ unsigned char buf[2048];
+ FILE *file = NULL;
+ char *ret = NULL;
+ int error = 0;
+
+ file = fopen(filename, "rb");
+ if (!file)
+ {
+ fprintf(logfile, "Failed to open MD5 file %s, error %d!\n", filename, errno);
+ return errno;
+ }
+
+ while(1)
+ {
+ ret = fgets(buf, sizeof(buf), file);
+ if (NULL == ret)
+ break;
+ error = parse_md5_entry(strdup(buf));
+ }
+ return 0;
+}
+
+/* DELIBERATELY do not sort these, or do anything clever with
+ insertion. The entries in the jigdo file should be in the same
+ order as the ones we'll want from the template. Simply add to the
+ end of the singly-linked list each time! */
+static int add_file_entry(char *jigdo_entry)
+{
+ int error = 0;
+ char *file_name = NULL;
+ char *md5 = NULL;
+ INT64 file_size = 0;
+
+ error = find_file_in_mirror(jigdo_entry, &file_name, &md5, &file_size);
+
+ if (error)
+ add_md5_entry(MISSING, md5, file_name);
+ else
+ add_md5_entry(file_size, md5, file_name);
+ return 0;
+}
+
static int parse_jigdo_file(char *filename)
{
unsigned char buf[2048];
return error;
}
-static off_t get_file_size(char *filename)
-{
- struct stat sb;
- int error = 0;
-
- error = stat(filename, &sb);
- if (error)
- return -1;
- else
- return sb.st_size;
-}
-
-static int ungzip_data_block(char *in_buf, size_t in_len, char *out_buf, size_t out_len)
+static int ungzip_data_block(char *in_buf, INT64 in_len, char *out_buf, INT64 out_len)
{
int error = 0;
z_stream uc_stream;
}
#ifdef BZ2_SUPPORT
-static int unbzip_data_block(char *in_buf, size_t in_len, char *out_buf, size_t out_len)
+static int unbzip_data_block(char *in_buf, INT64 in_len, char *out_buf, INT64 out_len)
{
int error = 0;
bz_stream uc_stream;
}
#endif
-static int decompress_data_block(char *in_buf, size_t in_len, char *out_buf,
- size_t out_len, int compress_type)
+static int decompress_data_block(char *in_buf, INT64 in_len, char *out_buf,
+ INT64 out_len, int compress_type)
{
#ifdef BZ2_SUPPORT
if (COMP_BZIP == compress_type)
static int read_data_block(FILE *template_file, int compress_type)
{
char inbuf[1024];
- off_t i = 0;
- static off_t template_offset = -1;
- off_t compressed_len = 0;
- off_t uncompressed_len = 0;
+ INT64 i = 0;
+ static INT64 template_offset = -1;
+ INT64 compressed_len = 0;
+ INT64 uncompressed_len = 0;
char *comp_buf = NULL;
int read_num = 0;
int error = 0;
return 0;
}
-static int skip_data_block(size_t data_size, FILE *template_file, int compress_type)
+static int skip_data_block(INT64 data_size, FILE *template_file, int compress_type)
{
int error = 0;
- size_t remaining = data_size;
- size_t size = 0;
+ INT64 remaining = data_size;
+ INT64 size = 0;
/* If we're coming in in the middle of the image, we'll need to
skip through some compressed data */
}
}
- fprintf(logfile, "skip_data_block: skipped %d bytes of unmatched data\n", data_size);
+ fprintf(logfile, "skip_data_block: skipped %lld bytes of unmatched data\n", data_size);
return error;
}
-static int parse_data_block(size_t data_size, FILE *template_file,
+static int parse_data_block(INT64 data_size, FILE *template_file,
struct mk_MD5Context *context, int compress_type)
{
int error = 0;
- size_t remaining = data_size;
- size_t size = 0;
+ INT64 remaining = data_size;
+ INT64 size = 0;
int out_size = 0;
while (remaining)
out_size = fwrite(&zip_state.data_buf[zip_state.offset_in_curr_buf], size, 1, outfile);
if (!out_size)
{
- fprintf(logfile, "parse_data_block: fwrite %d failed with error %d; aborting\n", size, ferror(outfile));
+ fprintf(logfile, "parse_data_block: fwrite %lld failed with error %d; aborting\n", size, ferror(outfile));
return ferror(outfile);
}
+ if (verbose)
+ display_progress(outfile, "template data");
+
if (!quick)
mk_MD5Update(context, &zip_state.data_buf[zip_state.offset_in_curr_buf], size);
zip_state.offset_in_curr_buf += size;
zip_state.data_buf = NULL;
}
}
-
- fprintf(logfile, "parse_data_block: wrote %d bytes of unmatched data\n", data_size);
+ if (verbose > 1)
+ fprintf(logfile, "parse_data_block: wrote %lld bytes of unmatched data\n", data_size);
return error;
}
-static int parse_file_block(off_t offset, size_t data_size, off_t file_size, char *md5, struct mk_MD5Context *image_context)
+static int parse_file_block(INT64 offset, INT64 data_size, INT64 file_size,
+ char *md5, struct mk_MD5Context *image_context,
+ char *missing)
{
char *base64_md5 = base64_dump(md5, 16);
FILE *input_file = NULL;
char buf[BUF_SIZE];
- size_t remaining = data_size;
+ INT64 remaining = data_size;
int num_read = 0;
struct mk_MD5Context file_context;
char file_md5[16];
- jigdo_list_t *jigdo_list_current = jigdo_list_head;
+ md5_list_t *md5_list_current = md5_list_head;
int out_size = 0;
if (!quick)
mk_MD5Init(&file_context);
- while (jigdo_list_current)
+ while (md5_list_current)
{
- if ( (jigdo_list_current->file_size == file_size) &&
- (!memcmp(jigdo_list_current->md5, base64_md5, 16) ) )
+ if ( (md5_list_current->file_size == file_size) &&
+ (!memcmp(md5_list_current->md5, base64_md5, 16) ) )
{
- input_file = fopen(jigdo_list_current->full_path, "rb");
+ input_file = fopen(md5_list_current->full_path, "rb");
if (!input_file)
{
fprintf(logfile, "Unable to open mirror file %s, error %d\n",
- jigdo_list_current->full_path, errno);
+ md5_list_current->full_path, errno);
return errno;
}
+
+ if (missing)
+ {
+ fclose(input_file);
+ return 0;
+ }
+
fseek(input_file, offset, SEEK_SET);
while (remaining)
{
num_read = fread(buf, size, 1, input_file);
if (!num_read)
{
- fprintf(logfile, "Unable to open mirror file %s, error %d\n",
- jigdo_list_current->full_path, errno);
+ fprintf(logfile, "Unable to read from mirror file %s, error %d (offset %ld, length %d)\n",
+ md5_list_current->full_path, errno, ftell(input_file), size);
fclose(input_file);
return errno;
}
return ferror(outfile);
}
+ if (verbose)
+ display_progress(outfile, file_base_name(md5_list_current->full_path));
+
remaining -= size;
}
- fprintf(logfile, "parse_file_block: wrote %lld bytes of data from %s\n",
- file_size, jigdo_list_current->full_path);
+ if (verbose > 1)
+ fprintf(logfile, "parse_file_block: wrote %lld bytes of data from %s\n",
+ file_size, md5_list_current->full_path);
fclose(input_file);
if (!quick)
if (memcmp(file_md5, md5, 16))
{
- fprintf(logfile, "MD5 MISMATCH for file %s\n", jigdo_list_current->full_path);
+ fprintf(logfile, "MD5 MISMATCH for file %s\n", md5_list_current->full_path);
fprintf(logfile, " template looking for %s\n", md5);
fprintf(logfile, " file in mirror is %s\n", file_md5);
return EINVAL;
}
return 0;
}
- jigdo_list_current = jigdo_list_current->next;
+ if ( missing &&
+ (MISSING == md5_list_current->file_size) &&
+ (!memcmp(md5_list_current->md5, base64_md5, 16) ) )
+ {
+ write_missing_entry(missing, md5_list_current->full_path);
+ return 0;
+ }
+ md5_list_current = md5_list_current->next;
}
return ENOENT;
}
-static int parse_template_file(char *filename, int sizeonly)
+static int parse_template_file(char *filename, int sizeonly, char *missing, char *output_name)
{
- off_t template_offset = 0;
- off_t bytes = 0;
+ INT64 template_offset = 0;
+ INT64 bytes = 0;
unsigned char *buf = NULL;
FILE *file = NULL;
- off_t file_size = 0;
- off_t desc_start = 0;
- off_t written_length = 0;
- off_t output_offset = 0;
+ INT64 file_size = 0;
+ INT64 desc_start = 0;
+ INT64 written_length = 0;
+ INT64 output_offset = 0;
int i = 0;
int error = 0;
struct mk_MD5Context template_context;
return 0;
}
- fprintf(logfile, "Image is %lld bytes\n", read_le48(&buf[1]));
- fprintf(logfile, "Image MD5 is ");
- for (i = 0; i < 16; i++)
- fprintf(logfile, "%2.2x", buf[i+7]);
- fprintf(logfile, "\n");
+ if (verbose)
+ {
+ fprintf(logfile, "Image should be %lld bytes\n", read_le48(&buf[1]));
+ fprintf(logfile, "Image MD5 should be ");
+ for (i = 0; i < 16; i++)
+ fprintf(logfile, "%2.2x", buf[i+7]);
+ fprintf(logfile, "\n");
+ }
+ out_size = read_le48(&buf[1]);
+
/* Now seek back to the start of the desc block */
fseek(file, desc_start, SEEK_SET);
fread(buf, 10, 1, file);
mk_MD5Init(&template_context);
template_offset = desc_start + 10;
+ if (1 == verbose)
+ fprintf(logfile, "Creating ISO image %s\n", output_name);
+
/* Main loop - walk through the template file and expand each entry we find */
while (1)
{
- off_t extent_size;
- off_t skip = 0;
- off_t read_length = 0;
+ INT64 extent_size;
+ INT64 skip = 0;
+ INT64 read_length = 0;
if (template_offset >= (file_size - 33))
{
- fprintf(logfile, "Reached end of template file\n");
+ if (verbose > 1)
+ fprintf(logfile, "Reached end of template file\n");
break; /* Finished! */
}
case 2: /* unmatched data, gzip */
case 8: /* unmatched data, bzip2 */
+ template_offset += 7;
+ if (missing)
+ break;
if ((output_offset + extent_size) >= start_offset)
{
if (skip)
}
else
error = skip_data_block(extent_size, file, buf[0]);
- template_offset += 7;
break;
case 6:
+ template_offset += 31;
if ((output_offset + extent_size) >= start_offset)
{
- error = parse_file_block(skip, read_length, extent_size, &buf[15], &template_context);
+ error = parse_file_block(skip, read_length, extent_size, &buf[15], &template_context, missing);
if (error)
{
fprintf(logfile, "Unable to read file block, error %d\n", error);
}
written_length += read_length;
}
- template_offset += 31;
break;
default:
fprintf(logfile, "Unknown block type %d!\n", buf[0]);
}
output_offset += extent_size;
}
+
+ if (missing && missing_file)
+ return ENOENT;
fclose(file);
- if (!quick)
+ if (verbose)
{
- mk_MD5Final (image_md5sum, &template_context);
- fprintf(logfile, "Output image MD5 is ");
- for (i = 0; i < 16; i++)
- fprintf(logfile, "%2.2x", image_md5sum[i]);
fprintf(logfile, "\n");
+ if (!quick)
+ {
+ mk_MD5Final (image_md5sum, &template_context);
+ fprintf(logfile, "Output image MD5 is ");
+ for (i = 0; i < 16; i++)
+ fprintf(logfile, "%2.2x", image_md5sum[i]);
+ fprintf(logfile, "\n");
+ }
+ fprintf(logfile, "Output image length is %lld\n", written_length);
}
- fprintf(logfile, "Output image length is %lld\n", written_length);
-
+
return 0;
}
{
printf("%s [OPTIONS]\n\n", progname);
printf(" Options:\n");
+ printf(" -f <MD5 name> Specify an input MD5 file. MD5s must be in jigdo's\n");
+ printf(" pseudo-base64 format\n");
printf(" -j <jigdo name> Specify the input jigdo file\n");
printf(" -t <template name> Specify the input template file\n");
printf(" -m <item=path> Map <item> to <path> to find the files in the mirror\n");
+ printf(" -M <missing name> Rather than try to build the image, just check that\n");
+ printf(" all the needed files are available. If any are missing,\n");
+ printf(" list them in this file.\n");
+ printf(" -v Make the output logging more verbose\n");
printf(" -l <logfile> Specify a logfile to append to.\n");
printf(" If not specified, will log to stderr\n");
printf(" -o <outfile> Specify a file to write the ISO image to.\n");
{
char *template_filename = NULL;
char *jigdo_filename = NULL;
+ char *md5_filename = NULL;
+ char *missing_filename = NULL;
+ char *output_name = NULL;
int c = -1;
int error = 0;
int sizeonly = 0;
while(1)
{
- c = getopt(argc, argv, ":ql:o:j:t:m:h?s:e:z");
+ c = getopt(argc, argv, ":ql:o:j:t:f:m:M:h?s:e:zv");
if (-1 == c)
break;
switch(c)
{
+ case 'v':
+ verbose++;
+ break;
case 'q':
quick = 1;
break;
setlinebuf(logfile);
break;
case 'o':
- outfile = fopen(optarg, "wb");
+ output_name = optarg;
+ outfile = fopen(output_name, "wb");
if (!outfile)
{
fprintf(stderr, "Unable to open output file %s\n", optarg);
/* else */
template_filename = optarg;
break;
+ case 'f':
+ if (md5_filename)
+ {
+ fprintf(logfile, "Can only specify one MD5 file!\n");
+ return EINVAL;
+ }
+ /* else */
+ md5_filename = optarg;
+ break;
case 'm':
error = add_match_entry(strdup(optarg));
if (error)
return error;
break;
+ case 'M':
+ missing_filename = optarg;
+ break;
case ':':
fprintf(logfile, "Missing argument!\n");
return EINVAL;
}
if (0 == end_offset)
- end_offset = (unsigned long long)LONG_MAX * LONG_MAX;
+ end_offset = LLONG_MAX;
- if ((NULL == jigdo_filename) && !sizeonly)
+ if ((NULL == jigdo_filename) &&
+ (NULL == md5_filename) &&
+ !sizeonly)
{
- fprintf(logfile, "No jigdo file specified!\n");
+ fprintf(logfile, "No jigdo file or MD5 file specified!\n");
usage(argv[0]);
return EINVAL;
}
return EINVAL;
}
- if (!sizeonly)
+ if (md5_filename)
+ {
+ /* Build up a list of the files we've been fed */
+ error = parse_md5_file(md5_filename);
+ if (error)
+ {
+ fprintf(logfile, "Unable to parse the MD5 file %s\n", md5_filename);
+ return error;
+ }
+ }
+
+ if (jigdo_filename)
{
/* Build up a list of file mappings */
error = parse_jigdo_file(jigdo_filename);
return error;
}
}
-
+
+ if (!output_name)
+ output_name = "to stdout";
/* Read the template file and actually build the image to <outfile> */
- error = parse_template_file(template_filename, sizeonly);
+ error = parse_template_file(template_filename, sizeonly, missing_filename, output_name);
if (error)
{
fprintf(logfile, "Unable to recreate image from template file %s\n", template_filename);
+ if (missing_filename)
+ fprintf(logfile, "%s contains the list of missing files\n", missing_filename);
return error;
}
fclose(logfile);
-
return 0;
}
+
projects / jigit.git / blobdiff