binman: Add support for CBFS entries

Add support for putting CBFSs (Coreboot Filesystems) in an image. This
allows binman to produce firmware images used by coreboot to boot.

Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/tools/binman/README.entries b/tools/binman/README.entries
index 9a31676..2e6aea1 100644
--- a/tools/binman/README.entries
+++ b/tools/binman/README.entries
@@ -60,6 +60,148 @@
 
 
 
+Entry: cbfs: Entry containing a Coreboot Filesystem (CBFS)
+----------------------------------------------------------
+
+A CBFS provides a way to group files into a group. It has a simple directory
+structure and allows the position of individual files to be set, since it is
+designed to support execute-in-place in an x86 SPI-flash device. Where XIP
+is not used, it supports compression and storing ELF files.
+
+CBFS is used by coreboot as its way of orgnanising SPI-flash contents.
+
+The contents of the CBFS are defined by subnodes of the cbfs entry, e.g.:
+
+    cbfs {
+        size = <0x100000>;
+        u-boot {
+            cbfs-type = "raw";
+        };
+        u-boot-dtb {
+            cbfs-type = "raw";
+        };
+    };
+
+This creates a CBFS 1MB in size two files in it: u-boot.bin and u-boot.dtb.
+Note that the size is required since binman does not support calculating it.
+The contents of each entry is just what binman would normally provide if it
+were not a CBFS node. A blob type can be used to import arbitrary files as
+with the second subnode below:
+
+    cbfs {
+        size = <0x100000>;
+        u-boot {
+            cbfs-name = "BOOT";
+            cbfs-type = "raw";
+        };
+
+        dtb {
+            type = "blob";
+            filename = "u-boot.dtb";
+            cbfs-type = "raw";
+            cbfs-compress = "lz4";
+        };
+    };
+
+This creates a CBFS 1MB in size with u-boot.bin (named "BOOT") and
+u-boot.dtb (named "dtb") and compressed with the lz4 algorithm.
+
+
+Properties supported in the top-level CBFS node:
+
+cbfs-arch:
+    Defaults to "x86", but you can specify the architecture if needed.
+
+
+Properties supported in the CBFS entry subnodes:
+
+cbfs-name:
+    This is the name of the file created in CBFS. It defaults to the entry
+    name (which is the node name), but you can override it with this
+    property.
+
+cbfs-type:
+    This is the CBFS file type. The following are supported:
+
+    raw:
+        This is a 'raw' file, although compression is supported. It can be
+        used to store any file in CBFS.
+
+    stage:
+        This is an ELF file that has been loaded (i.e. mapped to memory), so
+        appears in the CBFS as a flat binary. The input file must be an ELF
+        image, for example this puts "u-boot" (the ELF image) into a 'stage'
+        entry:
+
+            cbfs {
+                size = <0x100000>;
+                u-boot-elf {
+                    cbfs-name = "BOOT";
+                    cbfs-type = "stage";
+                };
+            };
+
+        You can use your own ELF file with something like:
+
+            cbfs {
+                size = <0x100000>;
+                something {
+                    type = "blob";
+                    filename = "cbfs-stage.elf";
+                    cbfs-type = "stage";
+                };
+            };
+
+        As mentioned, the file is converted to a flat binary, so it is
+        equivalent to adding "u-boot.bin", for example, but with the load and
+        start addresses specified by the ELF. At present there is no option
+        to add a flat binary with a load/start address, similar to the
+        'add-flat-binary' option in cbfstool.
+
+
+The current implementation supports only a subset of CBFS features. It does
+not support other file types (e.g. payload), adding multiple files (like the
+'files' entry with a pattern supported by binman), putting files at a
+particular offset in the CBFS and a few other things.
+
+Of course binman can create images containing multiple CBFSs, simply by
+defining these in the binman config:
+
+
+    binman {
+        size = <0x800000>;
+        cbfs {
+            offset = <0x100000>;
+            size = <0x100000>;
+            u-boot {
+                cbfs-type = "raw";
+            };
+            u-boot-dtb {
+                cbfs-type = "raw";
+            };
+        };
+
+        cbfs2 {
+            offset = <0x700000>;
+            size = <0x100000>;
+            u-boot {
+                cbfs-type = "raw";
+            };
+            u-boot-dtb {
+                cbfs-type = "raw";
+            };
+            image {
+                type = "blob";
+                filename = "image.jpg";
+            };
+        };
+    };
+
+This creates an 8MB image with two CBFSs, one at offset 1MB, one at 7MB,
+both of size 1MB.
+
+
+
 Entry: cros-ec-rw: A blob entry which contains a Chromium OS read-write EC image
 --------------------------------------------------------------------------------
 
diff --git a/tools/binman/control.py b/tools/binman/control.py
index df78848..4a94afc 100644
--- a/tools/binman/control.py
+++ b/tools/binman/control.py
@@ -12,6 +12,7 @@
 import sys
 import tools
 
+import cbfs_util
 import command
 import elf
 from image import Image
@@ -108,6 +109,7 @@
 
         tout.Init(options.verbosity)
         elf.debug = options.debug
+        cbfs_util.VERBOSE = options.verbosity > 2
         state.use_fake_dtb = options.fake_dtb
         try:
             tools.SetInputDirs(options.indir)
diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py
new file mode 100644
index 0000000..513df21
--- /dev/null
+++ b/tools/binman/etype/cbfs.py
@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2019 Google LLC
+# Written by Simon Glass <sjg@chromium.org>
+#
+# Entry-type module for a Coreboot Filesystem (CBFS)
+#
+
+from collections import OrderedDict
+
+import cbfs_util
+from cbfs_util import CbfsWriter
+from entry import Entry
+import fdt_util
+
+class Entry_cbfs(Entry):
+    """Entry containing a Coreboot Filesystem (CBFS)
+
+    A CBFS provides a way to group files into a group. It has a simple directory
+    structure and allows the position of individual files to be set, since it is
+    designed to support execute-in-place in an x86 SPI-flash device. Where XIP
+    is not used, it supports compression and storing ELF files.
+
+    CBFS is used by coreboot as its way of orgnanising SPI-flash contents.
+
+    The contents of the CBFS are defined by subnodes of the cbfs entry, e.g.:
+
+        cbfs {
+            size = <0x100000>;
+            u-boot {
+                cbfs-type = "raw";
+            };
+            u-boot-dtb {
+                cbfs-type = "raw";
+            };
+        };
+
+    This creates a CBFS 1MB in size two files in it: u-boot.bin and u-boot.dtb.
+    Note that the size is required since binman does not support calculating it.
+    The contents of each entry is just what binman would normally provide if it
+    were not a CBFS node. A blob type can be used to import arbitrary files as
+    with the second subnode below:
+
+        cbfs {
+            size = <0x100000>;
+            u-boot {
+                cbfs-name = "BOOT";
+                cbfs-type = "raw";
+            };
+
+            dtb {
+                type = "blob";
+                filename = "u-boot.dtb";
+                cbfs-type = "raw";
+                cbfs-compress = "lz4";
+            };
+        };
+
+    This creates a CBFS 1MB in size with u-boot.bin (named "BOOT") and
+    u-boot.dtb (named "dtb") and compressed with the lz4 algorithm.
+
+
+    Properties supported in the top-level CBFS node:
+
+    cbfs-arch:
+        Defaults to "x86", but you can specify the architecture if needed.
+
+
+    Properties supported in the CBFS entry subnodes:
+
+    cbfs-name:
+        This is the name of the file created in CBFS. It defaults to the entry
+        name (which is the node name), but you can override it with this
+        property.
+
+    cbfs-type:
+        This is the CBFS file type. The following are supported:
+
+        raw:
+            This is a 'raw' file, although compression is supported. It can be
+            used to store any file in CBFS.
+
+        stage:
+            This is an ELF file that has been loaded (i.e. mapped to memory), so
+            appears in the CBFS as a flat binary. The input file must be an ELF
+            image, for example this puts "u-boot" (the ELF image) into a 'stage'
+            entry:
+
+                cbfs {
+                    size = <0x100000>;
+                    u-boot-elf {
+                        cbfs-name = "BOOT";
+                        cbfs-type = "stage";
+                    };
+                };
+
+            You can use your own ELF file with something like:
+
+                cbfs {
+                    size = <0x100000>;
+                    something {
+                        type = "blob";
+                        filename = "cbfs-stage.elf";
+                        cbfs-type = "stage";
+                    };
+                };
+
+            As mentioned, the file is converted to a flat binary, so it is
+            equivalent to adding "u-boot.bin", for example, but with the load and
+            start addresses specified by the ELF. At present there is no option
+            to add a flat binary with a load/start address, similar to the
+            'add-flat-binary' option in cbfstool.
+
+
+    The current implementation supports only a subset of CBFS features. It does
+    not support other file types (e.g. payload), adding multiple files (like the
+    'files' entry with a pattern supported by binman), putting files at a
+    particular offset in the CBFS and a few other things.
+
+    Of course binman can create images containing multiple CBFSs, simply by
+    defining these in the binman config:
+
+
+        binman {
+            size = <0x800000>;
+            cbfs {
+                offset = <0x100000>;
+                size = <0x100000>;
+                u-boot {
+                    cbfs-type = "raw";
+                };
+                u-boot-dtb {
+                    cbfs-type = "raw";
+                };
+            };
+
+            cbfs2 {
+                offset = <0x700000>;
+                size = <0x100000>;
+                u-boot {
+                    cbfs-type = "raw";
+                };
+                u-boot-dtb {
+                    cbfs-type = "raw";
+                };
+                image {
+                    type = "blob";
+                    filename = "image.jpg";
+                };
+            };
+        };
+
+    This creates an 8MB image with two CBFSs, one at offset 1MB, one at 7MB,
+    both of size 1MB.
+    """
+    def __init__(self, section, etype, node):
+        Entry.__init__(self, section, etype, node)
+        self._cbfs_arg = fdt_util.GetString(node, 'cbfs-arch', 'x86')
+        self._cbfs_entries = OrderedDict()
+        self._ReadSubnodes()
+
+    def ObtainContents(self):
+        arch = cbfs_util.find_arch(self._cbfs_arg)
+        if arch is None:
+            self.Raise("Invalid architecture '%s'" % self._cbfs_arg)
+        if self.size is None:
+            self.Raise("'cbfs' entry must have a size property")
+        cbfs = CbfsWriter(self.size, arch)
+        for entry in self._cbfs_entries.values():
+            # First get the input data and put it in a file. If not available,
+            # try later.
+            if not entry.ObtainContents():
+                return False
+            data = entry.GetData()
+            if entry._type == 'raw':
+                cbfs.add_file_raw(entry._cbfs_name, data, entry._cbfs_compress)
+            elif entry._type == 'stage':
+                cbfs.add_file_stage(entry._cbfs_name, data)
+        data = cbfs.get_data()
+        self.SetContents(data)
+        return True
+
+    def _ReadSubnodes(self):
+        """Read the subnodes to find out what should go in this IFWI"""
+        for node in self._node.subnodes:
+            entry = Entry.Create(self.section, node)
+            entry._cbfs_name = fdt_util.GetString(node, 'cbfs-name', entry.name)
+            entry._type = fdt_util.GetString(node, 'cbfs-type')
+            compress = fdt_util.GetString(node, 'cbfs-compress', 'none')
+            entry._cbfs_compress = cbfs_util.find_compress(compress)
+            if entry._cbfs_compress is None:
+                self.Raise("Invalid compression in '%s': '%s'" %
+                           (node.name, compress))
+            self._cbfs_entries[entry._cbfs_name] = entry
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index 3455b8c..14abfbf 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -18,6 +18,7 @@
 import unittest
 
 import binman
+import cbfs_util
 import cmdline
 import command
 import control
@@ -133,6 +134,14 @@
 
         TestFunctional._MakeInputFile('compress', COMPRESS_DATA)
 
+        # Travis-CI may have an old lz4
+        self.have_lz4 = True
+        try:
+            tools.Run('lz4', '--no-frame-crc', '-c',
+                      os.path.join(self._indir, 'u-boot.bin'))
+        except:
+            self.have_lz4 = False
+
     @classmethod
     def tearDownClass(self):
         """Remove the temporary input directory and its contents"""
@@ -160,6 +169,10 @@
         cls.preserve_outdirs = preserve_outdirs
         cls.toolpath = toolpath
 
+    def _CheckLz4(self):
+        if not self.have_lz4:
+            self.skipTest('lz4 --no-frame-crc not available')
+
     def setUp(self):
         # Enable this to turn on debugging output
         # tout.Init(tout.DEBUG)
@@ -1607,6 +1620,7 @@
 
     def testCompress(self):
         """Test compression of blobs"""
+        self._CheckLz4()
         data, _, _, out_dtb_fname = self._DoReadFileDtb('083_compress.dts',
                                             use_real_dtb=True, update_dtb=True)
         dtb = fdt.Fdt(out_dtb_fname)
@@ -1628,6 +1642,7 @@
 
     def testFilesCompress(self):
         """Test bringing in multiple files and compressing them"""
+        self._CheckLz4()
         data = self._DoReadFile('085_files_compress.dts')
 
         image = control.images['image']
@@ -1846,6 +1861,101 @@
                          tools.GetBytes(0x26, 4) + U_BOOT_DATA +
                              tools.GetBytes(0x26, 8))
 
+    def testCbfsRaw(self):
+        """Test base handling of a Coreboot Filesystem (CBFS)
+
+        The exact contents of the CBFS is verified by similar tests in
+        cbfs_util_test.py. The tests here merely check that the files added to
+        the CBFS can be found in the final image.
+        """
+        data = self._DoReadFile('102_cbfs_raw.dts')
+        size = 0xb0
+
+        cbfs = cbfs_util.CbfsReader(data)
+        self.assertEqual(size, cbfs.rom_size)
+
+        self.assertIn('u-boot-dtb', cbfs.files)
+        cfile = cbfs.files['u-boot-dtb']
+        self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
+
+    def testCbfsArch(self):
+        """Test on non-x86 architecture"""
+        data = self._DoReadFile('103_cbfs_raw_ppc.dts')
+        size = 0x100
+
+        cbfs = cbfs_util.CbfsReader(data)
+        self.assertEqual(size, cbfs.rom_size)
+
+        self.assertIn('u-boot-dtb', cbfs.files)
+        cfile = cbfs.files['u-boot-dtb']
+        self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
+
+    def testCbfsStage(self):
+        """Tests handling of a Coreboot Filesystem (CBFS)"""
+        if not elf.ELF_TOOLS:
+            self.skipTest('Python elftools not available')
+        elf_fname = os.path.join(self._indir, 'cbfs-stage.elf')
+        elf.MakeElf(elf_fname, U_BOOT_DATA, U_BOOT_DTB_DATA)
+        size = 0xb0
+
+        data = self._DoReadFile('104_cbfs_stage.dts')
+        cbfs = cbfs_util.CbfsReader(data)
+        self.assertEqual(size, cbfs.rom_size)
+
+        self.assertIn('u-boot', cbfs.files)
+        cfile = cbfs.files['u-boot']
+        self.assertEqual(U_BOOT_DATA + U_BOOT_DTB_DATA, cfile.data)
+
+    def testCbfsRawCompress(self):
+        """Test handling of compressing raw files"""
+        self._CheckLz4()
+        data = self._DoReadFile('105_cbfs_raw_compress.dts')
+        size = 0x140
+
+        cbfs = cbfs_util.CbfsReader(data)
+        self.assertIn('u-boot', cbfs.files)
+        cfile = cbfs.files['u-boot']
+        self.assertEqual(COMPRESS_DATA, cfile.data)
+
+    def testCbfsBadArch(self):
+        """Test handling of a bad architecture"""
+        with self.assertRaises(ValueError) as e:
+            self._DoReadFile('106_cbfs_bad_arch.dts')
+        self.assertIn("Invalid architecture 'bad-arch'", str(e.exception))
+
+    def testCbfsNoSize(self):
+        """Test handling of a missing size property"""
+        with self.assertRaises(ValueError) as e:
+            self._DoReadFile('107_cbfs_no_size.dts')
+        self.assertIn('entry must have a size property', str(e.exception))
+
+    def testCbfsNoCOntents(self):
+        """Test handling of a CBFS entry which does not provide contentsy"""
+        with self.assertRaises(ValueError) as e:
+            self._DoReadFile('108_cbfs_no_contents.dts')
+        self.assertIn('Could not complete processing of contents',
+                      str(e.exception))
+
+    def testCbfsBadCompress(self):
+        """Test handling of a bad architecture"""
+        with self.assertRaises(ValueError) as e:
+            self._DoReadFile('109_cbfs_bad_compress.dts')
+        self.assertIn("Invalid compression in 'u-boot': 'invalid-algo'",
+                      str(e.exception))
+
+    def testCbfsNamedEntries(self):
+        """Test handling of named entries"""
+        data = self._DoReadFile('110_cbfs_name.dts')
+
+        cbfs = cbfs_util.CbfsReader(data)
+        self.assertIn('FRED', cbfs.files)
+        cfile1 = cbfs.files['FRED']
+        self.assertEqual(U_BOOT_DATA, cfile1.data)
+
+        self.assertIn('hello', cbfs.files)
+        cfile2 = cbfs.files['hello']
+        self.assertEqual(U_BOOT_DTB_DATA, cfile2.data)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tools/binman/test/102_cbfs_raw.dts b/tools/binman/test/102_cbfs_raw.dts
new file mode 100644
index 0000000..779cbc1
--- /dev/null
+++ b/tools/binman/test/102_cbfs_raw.dts
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0xb0>;
+			u-boot {
+				cbfs-type = "raw";
+			};
+			u-boot-dtb {
+				cbfs-type = "raw";
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/103_cbfs_raw_ppc.dts b/tools/binman/test/103_cbfs_raw_ppc.dts
new file mode 100644
index 0000000..df1caf0
--- /dev/null
+++ b/tools/binman/test/103_cbfs_raw_ppc.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0x100>;
+			cbfs-arch = "ppc64";
+			u-boot {
+				cbfs-type = "raw";
+			};
+			u-boot-dtb {
+				cbfs-type = "raw";
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/104_cbfs_stage.dts b/tools/binman/test/104_cbfs_stage.dts
new file mode 100644
index 0000000..215e2f2
--- /dev/null
+++ b/tools/binman/test/104_cbfs_stage.dts
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0xb0>;
+			u-boot {
+				type = "blob";
+				filename = "cbfs-stage.elf";
+				cbfs-type = "stage";
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/105_cbfs_raw_compress.dts b/tools/binman/test/105_cbfs_raw_compress.dts
new file mode 100644
index 0000000..646168d
--- /dev/null
+++ b/tools/binman/test/105_cbfs_raw_compress.dts
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0x140>;
+			u-boot {
+				type = "text";
+				text = "compress xxxxxxxxxxxxxxxxxxxxxx data";
+				cbfs-type = "raw";
+				cbfs-compress = "lz4";
+			};
+			u-boot-dtb {
+				type = "text";
+				text = "compress xxxxxxxxxxxxxxxxxxxxxx data";
+				cbfs-type = "raw";
+				cbfs-compress = "lzma";
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/106_cbfs_bad_arch.dts b/tools/binman/test/106_cbfs_bad_arch.dts
new file mode 100644
index 0000000..4318d45
--- /dev/null
+++ b/tools/binman/test/106_cbfs_bad_arch.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0x100>;
+			cbfs-arch = "bad-arch";
+		};
+	};
+};
diff --git a/tools/binman/test/107_cbfs_no_size.dts b/tools/binman/test/107_cbfs_no_size.dts
new file mode 100644
index 0000000..3592f62
--- /dev/null
+++ b/tools/binman/test/107_cbfs_no_size.dts
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+		};
+	};
+};
diff --git a/tools/binman/test/108_cbfs_no_contents.dts b/tools/binman/test/108_cbfs_no_contents.dts
new file mode 100644
index 0000000..6233467
--- /dev/null
+++ b/tools/binman/test/108_cbfs_no_contents.dts
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0x100>;
+			_testing {
+				return-unknown-contents;
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/109_cbfs_bad_compress.dts b/tools/binman/test/109_cbfs_bad_compress.dts
new file mode 100644
index 0000000..9695024
--- /dev/null
+++ b/tools/binman/test/109_cbfs_bad_compress.dts
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0xb0>;
+			u-boot {
+				cbfs-type = "raw";
+				cbfs-compress = "invalid-algo";
+			};
+		};
+	};
+};
diff --git a/tools/binman/test/110_cbfs_name.dts b/tools/binman/test/110_cbfs_name.dts
new file mode 100644
index 0000000..98c16f3
--- /dev/null
+++ b/tools/binman/test/110_cbfs_name.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		cbfs {
+			size = <0x100>;
+			u-boot {
+				cbfs-name = "FRED";
+				cbfs-type = "raw";
+			};
+
+			hello {
+				type = "blob";
+				filename = "u-boot.dtb";
+				cbfs-type = "raw";
+			};
+		};
+	};
+};