mkimage: Support placing data outside the FIT

One limitation of FIT is that all the data is 'inline' within it, using a
'data' property in each image node. This means that to find out what is in
the FIT it is necessary to scan the entire file. Once loaded it can be
scanned and then the images can be copied to the correct place in memory.

In SPL it can take a significant amount of time to copy images around in
memory. Also loading data that does not end up being used is wasteful. It
would be useful if the FIT were small, acting as a directory, with the
actual data stored elsewhere.

This allows SPL to load the entire FIT, without the images, then load the
images it wants later.

Add a -E option to mkimage to request that it output an 'external' FIT.

Signed-off-by: Simon Glass <sjg@chromium.org>

doc/mkimage.1

@@ -111,6 +111,15 @@ information.
 Provide special options to the device tree compiler that is used to
 create the image.
 
+.TP
+.BI "\-E
+After processing, move the image data outside the FIT and store a data offset
+in the FIT. Images will be placed one after the other immediately after the
+FIT, with each one aligned to a 4-byte boundary. The existing 'data' property
+in each image will be replaced with 'data-offset' and 'data-size' properties.
+A 'data-offset' of 0 indicates that it starts in the first (4-byte aligned)
+byte after the FIT.
+
 .TP
 .BI "\-f [" "image tree source file" " | " "auto" "]"
 Image tree source file that describes the structure and contents of the

doc/uImage.FIT/source_file_format.txt

@@ -2,6 +2,7 @@ U-Boot new uImage source file format (bindings definition)
 ==========================================================
 
 Author: Marian Balakowicz <m8@semihalf.com>
+External data additions, 25/1/16 Simon Glass <sjg@chromium.org>
 
 1) Introduction
 ---------------
@@ -262,7 +263,24 @@ Older, 2.4 kernel and 2.6 non-FDT kernel do not use FDT blob, in such cases
 not* be specified in a configuration node.
 
 
-8) Examples
+8) External data
+----------------
+
+The above format shows a 'data' property which holds the data for each image.
+It is also possible for this data to reside outside the FIT itself. This
+allows the FIT to be quite small, so that it can be loaded and scanned
+without loading a large amount of data. Then when an image is needed it can
+be loaded from an external source.
+
+In this case the 'data' property is omitted. Instead you can use:
+
+  - data-offset : offset of the data in a separate image store. The image
+    store is placed immediately after the last byte of the device tree binary,
+    aligned to a 4-byte boundary.
+  - data-size : size of the data in bytes
+
+
+9) Examples
 -----------
 
 Please see doc/uImage.FIT/*.its for actual image source files.

tools/fit_image.c

@@ -353,6 +353,108 @@ err:
 	return -1;
 }
 
+/**
+ * fit_extract_data() - Move all data outside the FIT
+ *
+ * This takes a normal FIT file and removes all the 'data' properties from it.
+ * The data is placed in an area after the FIT so that it can be accessed
+ * using an offset into that area. The 'data' properties turn into
+ * 'data-offset' properties.
+ *
+ * This function cannot cope with FITs with 'data-offset' properties. All
+ * data must be in 'data' properties on entry.
+ */
+static int fit_extract_data(struct image_tool_params *params, const char *fname)
+{
+	void *buf;
+	int buf_ptr;
+	int fit_size, new_size;
+	int fd;
+	struct stat sbuf;
+	void *fdt;
+	int ret;
+	int images;
+	int node;
+
+	fd = mmap_fdt(params->cmdname, fname, 0, &fdt, &sbuf, false);
+	if (fd < 0)
+		return -EIO;
+	fit_size = fdt_totalsize(fdt);
+
+	/* Allocate space to hold the image data we will extract */
+	buf = malloc(fit_size);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	buf_ptr = 0;
+
+	images = fdt_path_offset(fdt, FIT_IMAGES_PATH);
+	if (images < 0) {
+		debug("%s: Cannot find /images node: %d\n", __func__, images);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	for (node = fdt_first_subnode(fdt, images);
+	     node >= 0;
+	     node = fdt_next_subnode(fdt, node)) {
+		const char *data;
+		int len;
+
+		data = fdt_getprop(fdt, node, "data", &len);
+		if (!data)
+			continue;
+		memcpy(buf + buf_ptr, data, len);
+		debug("Extracting data size %x\n", len);
+
+		ret = fdt_delprop(fdt, node, "data");
+		if (ret) {
+			ret = -EPERM;
+			goto err;
+		}
+		fdt_setprop_u32(fdt, node, "data-offset", buf_ptr);
+		fdt_setprop_u32(fdt, node, "data-size", len);
+
+		buf_ptr += (len + 3) & ~3;
+	}
+
+	/* Pack the FDT and place the data after it */
+	fdt_pack(fdt);
+
+	debug("Size reduced from %x to %x\n", fit_size, fdt_totalsize(fdt));
+	debug("External data size %x\n", buf_ptr);
+	new_size = fdt_totalsize(fdt);
+	new_size = (new_size + 3) & ~3;
+	munmap(fdt, sbuf.st_size);
+
+	if (ftruncate(fd, new_size)) {
+		debug("%s: Failed to truncate file: %s\n", __func__,
+		      strerror(errno));
+		ret = -EIO;
+		goto err;
+	}
+	if (lseek(fd, new_size, SEEK_SET) < 0) {
+		debug("%s: Failed to seek to end of file: %s\n", __func__,
+		      strerror(errno));
+		ret = -EIO;
+		goto err;
+	}
+	if (write(fd, buf, buf_ptr) != buf_ptr) {
+		debug("%s: Failed to write external data to file %s\n",
+		      __func__, strerror(errno));
+		ret = -EIO;
+		goto err;
+	}
+	close(fd);
+
+	ret = 0;
+
+err:
+	close(fd);
+	return ret;
+}
+
 /**
  * fit_handle_file - main FIT file processing function
  *
@@ -430,6 +532,13 @@ static int fit_handle_file(struct image_tool_params *params)
 		goto err_system;
 	}
 
+	/* Move the data so it is external to the FIT, if requested */
+	if (params->external_data) {
+		ret = fit_extract_data(params, tmpfile);
+		if (ret)
+			goto err_system;
+	}
+
 	if (rename (tmpfile, params->imagefile) == -1) {
 		fprintf (stderr, "%s: Can't rename %s to %s: %s\n",
 				params->cmdname, tmpfile, params->imagefile,

tools/imagetool.h

@@ -72,6 +72,7 @@ struct image_tool_params {
 	int fit_image_type;	/* Image type to put into the FIT */
 	struct content_info *content_head;	/* List of files to include */
 	struct content_info *content_tail;
+	bool external_data;	/* Store data outside the FIT */
 };
 
 /*

tools/mkimage.c

@@ -138,7 +138,7 @@ static void process_args(int argc, char **argv)
 
 	expecting = IH_TYPE_COUNT;	/* Unknown */
 	while ((opt = getopt(argc, argv,
-			     "-a:A:bcC:d:D:e:f:Fk:K:ln:O:rR:sT:vVx")) != -1) {
+			     "-a:A:bcC:d:D:e:Ef:Fk:K:ln:O:rR:sT:vVx")) != -1) {
 		switch (opt) {
 		case 'a':
 			params.addr = strtoull(optarg, &ptr, 16);
@@ -180,6 +180,9 @@ static void process_args(int argc, char **argv)
 			}
 			params.eflag = 1;
 			break;
+		case 'E':
+			params.external_data = true;
+			break;
 		case 'f':
 			datafile = optarg;
 			params.auto_its = !strcmp(datafile, "auto");