summaryrefslogtreecommitdiff
path: root/tools/binman/control.py
blob: 343b0a0c35b4f90d64efaf5b654c259bad53dd90 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2016 Google, Inc
# Written by Simon Glass <sjg@chromium.org>
#
# Creates binary images from input files controlled by a description
#

from collections import OrderedDict
import os
import sys
from patman import tools

from binman import cbfs_util
from binman import elf
from patman import command
from patman import tout

# List of images we plan to create
# Make this global so that it can be referenced from tests
images = OrderedDict()

def _ReadImageDesc(binman_node):
    """Read the image descriptions from the /binman node

    This normally produces a single Image object called 'image'. But if
    multiple images are present, they will all be returned.

    Args:
        binman_node: Node object of the /binman node
    Returns:
        OrderedDict of Image objects, each of which describes an image
    """
    images = OrderedDict()
    if 'multiple-images' in binman_node.props:
        for node in binman_node.subnodes:
            images[node.name] = Image(node.name, node)
    else:
        images['image'] = Image('image', binman_node)
    return images

def _FindBinmanNode(dtb):
    """Find the 'binman' node in the device tree

    Args:
        dtb: Fdt object to scan
    Returns:
        Node object of /binman node, or None if not found
    """
    for node in dtb.GetRoot().subnodes:
        if node.name == 'binman':
            return node
    return None

def WriteEntryDocs(modules, test_missing=None):
    """Write out documentation for all entries

    Args:
        modules: List of Module objects to get docs for
        test_missing: Used for testing only, to force an entry's documeentation
            to show as missing even if it is present. Should be set to None in
            normal use.
    """
    from binman.entry import Entry
    Entry.WriteDocs(modules, test_missing)


def ListEntries(image_fname, entry_paths):
    """List the entries in an image

    This decodes the supplied image and displays a table of entries from that
    image, preceded by a header.

    Args:
        image_fname: Image filename to process
        entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*',
                                                     'section/u-boot'])
    """
    image = Image.FromFile(image_fname)

    entries, lines, widths = image.GetListEntries(entry_paths)

    num_columns = len(widths)
    for linenum, line in enumerate(lines):
        if linenum == 1:
            # Print header line
            print('-' * (sum(widths) + num_columns * 2))
        out = ''
        for i, item in enumerate(line):
            width = -widths[i]
            if item.startswith('>'):
                width = -width
                item = item[1:]
            txt = '%*s  ' % (width, item)
            out += txt
        print(out.rstrip())


def ReadEntry(image_fname, entry_path, decomp=True):
    """Extract an entry from an image

    This extracts the data from a particular entry in an image

    Args:
        image_fname: Image filename to process
        entry_path: Path to entry to extract
        decomp: True to return uncompressed data, if the data is compress
            False to return the raw data

    Returns:
        data extracted from the entry
    """
    global Image
    from image import Image

    image = Image.FromFile(image_fname)
    entry = image.FindEntryPath(entry_path)
    return entry.ReadData(decomp)


def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
                   decomp=True):
    """Extract the data from one or more entries and write it to files

    Args:
        image_fname: Image filename to process
        output_fname: Single output filename to use if extracting one file, None
            otherwise
        outdir: Output directory to use (for any number of files), else None
        entry_paths: List of entry paths to extract
        decomp: True to decompress the entry data

    Returns:
        List of EntryInfo records that were written
    """
    image = Image.FromFile(image_fname)

    # Output an entry to a single file, as a special case
    if output_fname:
        if not entry_paths:
            raise ValueError('Must specify an entry path to write with -f')
        if len(entry_paths) != 1:
            raise ValueError('Must specify exactly one entry path to write with -f')
        entry = image.FindEntryPath(entry_paths[0])
        data = entry.ReadData(decomp)
        tools.WriteFile(output_fname, data)
        tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname))
        return

    # Otherwise we will output to a path given by the entry path of each entry.
    # This means that entries will appear in subdirectories if they are part of
    # a sub-section.
    einfos = image.GetListEntries(entry_paths)[0]
    tout.Notice('%d entries match and will be written' % len(einfos))
    for einfo in einfos:
        entry = einfo.entry
        data = entry.ReadData(decomp)
        path = entry.GetPath()[1:]
        fname = os.path.join(outdir, path)

        # If this entry has children, create a directory for it and put its
        # data in a file called 'root' in that directory
        if entry.GetEntries():
            if not os.path.exists(fname):
                os.makedirs(fname)
            fname = os.path.join(fname, 'root')
        tout.Notice("Write entry '%s' to '%s'" % (entry.GetPath(), fname))
        tools.WriteFile(fname, data)
    return einfos


def BeforeReplace(image, allow_resize):
    """Handle getting an image ready for replacing entries in it

    Args:
        image: Image to prepare
    """
    state.PrepareFromLoadedData(image)
    image.LoadData()

    # If repacking, drop the old offset/size values except for the original
    # ones, so we are only left with the constraints.
    if allow_resize:
        image.ResetForPack()


def ReplaceOneEntry(image, entry, data, do_compress, allow_resize):
    """Handle replacing a single entry an an image

    Args:
        image: Image to update
        entry: Entry to write
        data: Data to replace with
        do_compress: True to compress the data if needed, False if data is
            already compressed so should be used as is
        allow_resize: True to allow entries to change size (this does a re-pack
            of the entries), False to raise an exception
    """
    if not entry.WriteData(data, do_compress):
        if not image.allow_repack:
            entry.Raise('Entry data size does not match, but allow-repack is not present for this image')
        if not allow_resize:
            entry.Raise('Entry data size does not match, but resize is disabled')


def AfterReplace(image, allow_resize, write_map):
    """Handle write out an image after replacing entries in it

    Args:
        image: Image to write
        allow_resize: True to allow entries to change size (this does a re-pack
            of the entries), False to raise an exception
        write_map: True to write a map file
    """
    tout.Info('Processing image')
    ProcessImage(image, update_fdt=True, write_map=write_map,
                 get_contents=False, allow_resize=allow_resize)


def WriteEntryToImage(image, entry, data, do_compress=True, allow_resize=True,
                      write_map=False):
    BeforeReplace(image, allow_resize)
    tout.Info('Writing data to %s' % entry.GetPath())
    ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
    AfterReplace(image, allow_resize=allow_resize, write_map=write_map)


def WriteEntry(image_fname, entry_path, data, do_compress=True,
               allow_resize=True, write_map=False):
    """Replace an entry in an image

    This replaces the data in a particular entry in an image. This size of the
    new data must match the size of the old data unless allow_resize is True.

    Args:
        image_fname: Image filename to process
        entry_path: Path to entry to extract
        data: Data to replace with
        do_compress: True to compress the data if needed, False if data is
            already compressed so should be used as is
        allow_resize: True to allow entries to change size (this does a re-pack
            of the entries), False to raise an exception
        write_map: True to write a map file

    Returns:
        Image object that was updated
    """
    tout.Info("Write entry '%s', file '%s'" % (entry_path, image_fname))
    image = Image.FromFile(image_fname)
    entry = image.FindEntryPath(entry_path)
    WriteEntryToImage(image, entry, data, do_compress=do_compress,
                      allow_resize=allow_resize, write_map=write_map)

    return image


def ReplaceEntries(image_fname, input_fname, indir, entry_paths,
                   do_compress=True, allow_resize=True, write_map=False):
    """Replace the data from one or more entries from input files

    Args:
        image_fname: Image filename to process
        input_fname: Single input ilename to use if replacing one file, None
            otherwise
        indir: Input directory to use (for any number of files), else None
        entry_paths: List of entry paths to extract
        do_compress: True if the input data is uncompressed and may need to be
            compressed if the entry requires it, False if the data is already
            compressed.
        write_map: True to write a map file

    Returns:
        List of EntryInfo records that were written
    """
    image = Image.FromFile(image_fname)

    # Replace an entry from a single file, as a special case
    if input_fname:
        if not entry_paths:
            raise ValueError('Must specify an entry path to read with -f')
        if len(entry_paths) != 1:
            raise ValueError('Must specify exactly one entry path to write with -f')
        entry = image.FindEntryPath(entry_paths[0])
        data = tools.ReadFile(input_fname)
        tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname))
        WriteEntryToImage(image, entry, data, do_compress=do_compress,
                          allow_resize=allow_resize, write_map=write_map)
        return

    # Otherwise we will input from a path given by the entry path of each entry.
    # This means that files must appear in subdirectories if they are part of
    # a sub-section.
    einfos = image.GetListEntries(entry_paths)[0]
    tout.Notice("Replacing %d matching entries in image '%s'" %
                (len(einfos), image_fname))

    BeforeReplace(image, allow_resize)

    for einfo in einfos:
        entry = einfo.entry
        if entry.GetEntries():
            tout.Info("Skipping section entry '%s'" % entry.GetPath())
            continue

        path = entry.GetPath()[1:]
        fname = os.path.join(indir, path)

        if os.path.exists(fname):
            tout.Notice("Write entry '%s' from file '%s'" %
                        (entry.GetPath(), fname))
            data = tools.ReadFile(fname)
            ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
        else:
            tout.Warning("Skipping entry '%s' from missing file '%s'" %
                         (entry.GetPath(), fname))

    AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
    return image


def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt):
    """Prepare the images to be processed and select the device tree

    This function:
    - reads in the device tree
    - finds and scans the binman node to create all entries
    - selects which images to build
    - Updates the device tress with placeholder properties for offset,
        image-pos, etc.

    Args:
        dtb_fname: Filename of the device tree file to use (.dts or .dtb)
        selected_images: List of images to output, or None for all
        update_fdt: True to update the FDT wth entry offsets, etc.
    """
    # Import these here in case libfdt.py is not available, in which case
    # the above help option still works.
    from dtoc import fdt
    from dtoc import fdt_util
    global images

    # Get the device tree ready by compiling it and copying the compiled
    # output into a file in our output directly. Then scan it for use
    # in binman.
    dtb_fname = fdt_util.EnsureCompiled(dtb_fname)
    fname = tools.GetOutputFilename('u-boot.dtb.out')
    tools.WriteFile(fname, tools.ReadFile(dtb_fname))
    dtb = fdt.FdtScan(fname)

    node = _FindBinmanNode(dtb)
    if not node:
        raise ValueError("Device tree '%s' does not have a 'binman' "
                            "node" % dtb_fname)

    images = _ReadImageDesc(node)

    if select_images:
        skip = []
        new_images = OrderedDict()
        for name, image in images.items():
            if name in select_images:
                new_images[name] = image
            else:
                skip.append(name)
        images = new_images
        tout.Notice('Skipping images: %s' % ', '.join(skip))

    state.Prepare(images, dtb)

    # Prepare the device tree by making sure that any missing
    # properties are added (e.g. 'pos' and 'size'). The values of these
    # may not be correct yet, but we add placeholders so that the
    # size of the device tree is correct. Later, in
    # SetCalculatedProperties() we will insert the correct values
    # without changing the device-tree size, thus ensuring that our
    # entry offsets remain the same.
    for image in images.values():
        image.ExpandEntries()
        if update_fdt:
            image.AddMissingProperties()
        image.ProcessFdt(dtb)

    for dtb_item in state.GetAllFdts():
        dtb_item.Sync(auto_resize=True)
        dtb_item.Pack()
        dtb_item.Flush()
    return images


def ProcessImage(image, update_fdt, write_map, get_contents=True,
                 allow_resize=True, allow_missing=False):
    """Perform all steps for this image, including checking and # writing it.

    This means that errors found with a later image will be reported after
    earlier images are already completed and written, but that does not seem
    important.

    Args:
        image: Image to process
        update_fdt: True to update the FDT wth entry offsets, etc.
        write_map: True to write a map file
        get_contents: True to get the image contents from files, etc., False if
            the contents is already present
        allow_resize: True to allow entries to change size (this does a re-pack
            of the entries), False to raise an exception
        allow_missing: Allow blob_ext objects to be missing

    Returns:
        True if one or more external blobs are missing, False if all are present
    """
    if get_contents:
        image.SetAllowMissing(allow_missing)
        image.GetEntryContents()
    image.GetEntryOffsets()

    # We need to pack the entries to figure out where everything
    # should be placed. This sets the offset/size of each entry.
    # However, after packing we call ProcessEntryContents() which
    # may result in an entry changing size. In that case we need to
    # do another pass. Since the device tree often contains the
    # final offset/size information we try to make space for this in
    # AddMissingProperties() above. However, if the device is
    # compressed we cannot know this compressed size in advance,
    # since changing an offset from 0x100 to 0x104 (for example) can
    # alter the compressed size of the device tree. So we need a
    # third pass for this.
    passes = 5
    for pack_pass in range(passes):
        try:
            image.PackEntries()
            image.CheckSize()
            image.CheckEntries()
        except Exception as e:
            if write_map:
                fname = image.WriteMap()
                print("Wrote map file '%s' to show errors"  % fname)
            raise
        image.SetImagePos()
        if update_fdt:
            image.SetCalculatedProperties()
            for dtb_item in state.GetAllFdts():
                dtb_item.Sync()
                dtb_item.Flush()
        image.WriteSymbols()
        sizes_ok = image.ProcessEntryContents()
        if sizes_ok:
            break
        image.ResetForPack()
    tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1))
    if not sizes_ok:
        image.Raise('Entries changed size after packing (tried %s passes)' %
                    passes)

    image.BuildImage()
    if write_map:
        image.WriteMap()
    missing_list = []
    image.CheckMissing(missing_list)
    if missing_list:
        tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
                     (image.name, ' '.join([e.name for e in missing_list])))
    return bool(missing_list)


def Binman(args):
    """The main control code for binman

    This assumes that help and test options have already been dealt with. It
    deals with the core task of building images.

    Args:
        args: Command line arguments Namespace object
    """
    global Image
    global state

    if args.full_help:
        pager = os.getenv('PAGER')
        if not pager:
            pager = 'more'
        fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
                            'README')
        command.Run(pager, fname)
        return 0

    # Put these here so that we can import this module without libfdt
    from image import Image
    from binman import state

    if args.cmd in ['ls', 'extract', 'replace']:
        try:
            tout.Init(args.verbosity)
            tools.PrepareOutputDir(None)
            if args.cmd == 'ls':
                ListEntries(args.image, args.paths)

            if args.cmd == 'extract':
                ExtractEntries(args.image, args.filename, args.outdir, args.paths,
                               not args.uncompressed)

            if args.cmd == 'replace':
                ReplaceEntries(args.image, args.filename, args.indir, args.paths,
                               do_compress=not args.compressed,
                               allow_resize=not args.fix_size, write_map=args.map)
        except:
            raise
        finally:
            tools.FinaliseOutputDir()
        return 0

    # Try to figure out which device tree contains our image description
    if args.dt:
        dtb_fname = args.dt
    else:
        board = args.board
        if not board:
            raise ValueError('Must provide a board to process (use -b <board>)')
        board_pathname = os.path.join(args.build_dir, board)
        dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
        if not args.indir:
            args.indir = ['.']
        args.indir.append(board_pathname)

    try:
        tout.Init(args.verbosity)
        elf.debug = args.debug
        cbfs_util.VERBOSE = args.verbosity > 2
        state.use_fake_dtb = args.fake_dtb
        try:
            tools.SetInputDirs(args.indir)
            tools.PrepareOutputDir(args.outdir, args.preserve)
            tools.SetToolPaths(args.toolpath)
            state.SetEntryArgs(args.entry_arg)

            images = PrepareImagesAndDtbs(dtb_fname, args.image,
                                          args.update_fdt)
            missing = False
            for image in images.values():
                missing |= ProcessImage(image, args.update_fdt, args.map,
                                        allow_missing=args.allow_missing)

            # Write the updated FDTs to our output files
            for dtb_item in state.GetAllFdts():
                tools.WriteFile(dtb_item._fname, dtb_item.GetContents())

            if missing:
                tout.Warning("Some images are invalid")
        finally:
            tools.FinaliseOutputDir()
    finally:
        tout.Uninit()

    return 0