diff options
Diffstat (limited to 'poky/scripts/lib/wic/engine.py')
-rw-r--r-- | poky/scripts/lib/wic/engine.py | 565 |
1 files changed, 565 insertions, 0 deletions
diff --git a/poky/scripts/lib/wic/engine.py b/poky/scripts/lib/wic/engine.py new file mode 100644 index 000000000..edcfab39e --- /dev/null +++ b/poky/scripts/lib/wic/engine.py @@ -0,0 +1,565 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (c) 2013, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION + +# This module implements the image creation engine used by 'wic' to +# create images. The engine parses through the OpenEmbedded kickstart +# (wks) file specified and generates images that can then be directly +# written onto media. +# +# AUTHORS +# Tom Zanussi <tom.zanussi (at] linux.intel.com> +# + +import logging +import os +import tempfile +import json +import subprocess + +from collections import namedtuple, OrderedDict +from distutils.spawn import find_executable + +from wic import WicError +from wic.filemap import sparse_copy +from wic.pluginbase import PluginMgr +from wic.misc import get_bitbake_var, exec_cmd + +logger = logging.getLogger('wic') + +def verify_build_env(): + """ + Verify that the build environment is sane. + + Returns True if it is, false otherwise + """ + if not os.environ.get("BUILDDIR"): + raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") + + return True + + +CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts +SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR +WIC_DIR = "wic" + +def build_canned_image_list(path): + layers_path = get_bitbake_var("BBLAYERS") + canned_wks_layer_dirs = [] + + if layers_path is not None: + for layer_path in layers_path.split(): + for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR): + cpath = os.path.join(layer_path, wks_path) + if os.path.isdir(cpath): + canned_wks_layer_dirs.append(cpath) + + cpath = os.path.join(path, CANNED_IMAGE_DIR) + canned_wks_layer_dirs.append(cpath) + + return canned_wks_layer_dirs + +def find_canned_image(scripts_path, wks_file): + """ + Find a .wks file with the given name in the canned files dir. + + Return False if not found + """ + layers_canned_wks_dir = build_canned_image_list(scripts_path) + + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname.endswith("~") or fname.endswith("#"): + continue + if fname.endswith(".wks") and wks_file + ".wks" == fname: + fullpath = os.path.join(canned_wks_dir, fname) + return fullpath + return None + + +def list_canned_images(scripts_path): + """ + List the .wks files in the canned image dir, minus the extension. + """ + layers_canned_wks_dir = build_canned_image_list(scripts_path) + + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname.endswith("~") or fname.endswith("#"): + continue + if fname.endswith(".wks"): + fullpath = os.path.join(canned_wks_dir, fname) + with open(fullpath) as wks: + for line in wks: + desc = "" + idx = line.find("short-description:") + if idx != -1: + desc = line[idx + len("short-description:"):].strip() + break + basename = os.path.splitext(fname)[0] + print(" %s\t\t%s" % (basename.ljust(30), desc)) + + +def list_canned_image_help(scripts_path, fullpath): + """ + List the help and params in the specified canned image. + """ + found = False + with open(fullpath) as wks: + for line in wks: + if not found: + idx = line.find("long-description:") + if idx != -1: + print() + print(line[idx + len("long-description:"):].strip()) + found = True + continue + if not line.strip(): + break + idx = line.find("#") + if idx != -1: + print(line[idx + len("#:"):].rstrip()) + else: + break + + +def list_source_plugins(): + """ + List the available source plugins i.e. plugins available for --source. + """ + plugins = PluginMgr.get_plugins('source') + + for plugin in plugins: + print(" %s" % plugin) + + +def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, options): + """ + Create image + + wks_file - user-defined OE kickstart file + rootfs_dir - absolute path to the build's /rootfs dir + bootimg_dir - absolute path to the build's boot artifacts directory + kernel_dir - absolute path to the build's kernel directory + native_sysroot - absolute path to the build's native sysroots dir + image_output_dir - dirname to create for image + options - wic command line options (debug, bmap, etc) + + Normally, the values for the build artifacts values are determined + by 'wic -e' from the output of the 'bitbake -e' command given an + image name e.g. 'core-image-minimal' and a given machine set in + local.conf. If that's the case, the variables get the following + values from the output of 'bitbake -e': + + rootfs_dir: IMAGE_ROOTFS + kernel_dir: DEPLOY_DIR_IMAGE + native_sysroot: STAGING_DIR_NATIVE + + In the above case, bootimg_dir remains unset and the + plugin-specific image creation code is responsible for finding the + bootimg artifacts. + + In the case where the values are passed in explicitly i.e 'wic -e' + is not used but rather the individual 'wic' options are used to + explicitly specify these values. + """ + try: + oe_builddir = os.environ["BUILDDIR"] + except KeyError: + raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)") + + if not os.path.exists(options.outdir): + os.makedirs(options.outdir) + + pname = 'direct' + plugin_class = PluginMgr.get_plugins('imager').get(pname) + if not plugin_class: + raise WicError('Unknown plugin: %s' % pname) + + plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, oe_builddir, options) + + plugin.do_create() + + logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file) + + +def wic_list(args, scripts_path): + """ + Print the list of images or source plugins. + """ + if args.list_type is None: + return False + + if args.list_type == "images": + + list_canned_images(scripts_path) + return True + elif args.list_type == "source-plugins": + list_source_plugins() + return True + elif len(args.help_for) == 1 and args.help_for[0] == 'help': + wks_file = args.list_type + fullpath = find_canned_image(scripts_path, wks_file) + if not fullpath: + raise WicError("No image named %s found, exiting. " + "(Use 'wic list images' to list available images, " + "or specify a fully-qualified OE kickstart (.wks) " + "filename)" % wks_file) + + list_canned_image_help(scripts_path, fullpath) + return True + + return False + + +class Disk: + def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')): + self.imagepath = imagepath + self.native_sysroot = native_sysroot + self.fstypes = fstypes + self._partitions = None + self._partimages = {} + self._lsector_size = None + self._psector_size = None + self._ptable_format = None + + # find parted + self.paths = "/bin:/usr/bin:/usr/sbin:/sbin/" + if native_sysroot: + for path in self.paths.split(':'): + self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) + + self.parted = find_executable("parted", self.paths) + if not self.parted: + raise WicError("Can't find executable parted") + + self.partitions = self.get_partitions() + + def __del__(self): + for path in self._partimages.values(): + os.unlink(path) + + def get_partitions(self): + if self._partitions is None: + self._partitions = OrderedDict() + out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) + parttype = namedtuple("Part", "pnum start end size fstype") + splitted = out.splitlines() + lsector_size, psector_size, self._ptable_format = splitted[1].split(":")[3:6] + self._lsector_size = int(lsector_size) + self._psector_size = int(psector_size) + for line in splitted[2:]: + pnum, start, end, size, fstype = line.split(':')[:5] + partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), + int(size[:-1]), fstype) + self._partitions[pnum] = partition + + return self._partitions + + def __getattr__(self, name): + """Get path to the executable in a lazy way.""" + if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck", + "resize2fs", "mkswap", "mkdosfs", "debugfs"): + aname = "_%s" % name + if aname not in self.__dict__: + setattr(self, aname, find_executable(name, self.paths)) + if aname not in self.__dict__: + raise WicError("Can't find executable {}".format(name)) + return self.__dict__[aname] + return self.__dict__[name] + + def _get_part_image(self, pnum): + if pnum not in self.partitions: + raise WicError("Partition %s is not in the image") + part = self.partitions[pnum] + # check if fstype is supported + for fstype in self.fstypes: + if part.fstype.startswith(fstype): + break + else: + raise WicError("Not supported fstype: {}".format(part.fstype)) + if pnum not in self._partimages: + tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") + dst_fname = tmpf.name + tmpf.close() + sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) + self._partimages[pnum] = dst_fname + + return self._partimages[pnum] + + def _put_part_image(self, pnum): + """Put partition image into partitioned image.""" + sparse_copy(self._partimages[pnum], self.imagepath, + seek=self.partitions[pnum].start) + + def dir(self, pnum, path): + if self.partitions[pnum].fstype.startswith('ext'): + return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs, + self._get_part_image(pnum), + path), as_shell=True) + else: # fat + return exec_cmd("{} -i {} ::{}".format(self.mdir, + self._get_part_image(pnum), + path)) + + def copy(self, src, pnum, path): + """Copy partition image into wic image.""" + if self.partitions[pnum].fstype.startswith('ext'): + cmd = "echo -e 'cd {}\nwrite {} {}' | {} -w {}".\ + format(path, src, os.path.basename(src), + self.debugfs, self._get_part_image(pnum)) + else: # fat + cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, + self._get_part_image(pnum), + src, path) + exec_cmd(cmd, as_shell=True) + self._put_part_image(pnum) + + def remove(self, pnum, path): + """Remove files/dirs from the partition.""" + partimg = self._get_part_image(pnum) + if self.partitions[pnum].fstype.startswith('ext'): + exec_cmd("{} {} -wR 'rm {}'".format(self.debugfs, + self._get_part_image(pnum), + path), as_shell=True) + else: # fat + cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) + try: + exec_cmd(cmd) + except WicError as err: + if "not found" in str(err) or "non empty" in str(err): + # mdel outputs 'File ... not found' or 'directory .. non empty" + # try to use mdeltree as path could be a directory + cmd = "{} -i {} ::{}".format(self.mdeltree, + partimg, path) + exec_cmd(cmd) + else: + raise err + self._put_part_image(pnum) + + def write(self, target, expand): + """Write disk image to the media or file.""" + def write_sfdisk_script(outf, parts): + for key, val in parts['partitiontable'].items(): + if key in ("partitions", "device", "firstlba", "lastlba"): + continue + if key == "id": + key = "label-id" + outf.write("{}: {}\n".format(key, val)) + outf.write("\n") + for part in parts['partitiontable']['partitions']: + line = '' + for name in ('attrs', 'name', 'size', 'type', 'uuid'): + if name == 'size' and part['type'] == 'f': + # don't write size for extended partition + continue + val = part.get(name) + if val: + line += '{}={}, '.format(name, val) + if line: + line = line[:-2] # strip ', ' + if part.get('bootable'): + line += ' ,bootable' + outf.write("{}\n".format(line)) + outf.flush() + + def read_ptable(path): + out = exec_cmd("{} -dJ {}".format(self.sfdisk, path)) + return json.loads(out) + + def write_ptable(parts, target): + with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf: + write_sfdisk_script(outf, parts) + cmd = "{} --no-reread {} < {} 2>/dev/null".format(self.sfdisk, target, outf.name) + try: + subprocess.check_output(cmd, shell=True) + except subprocess.CalledProcessError as err: + raise WicError("Can't run '{}' command: {}".format(cmd, err)) + + if expand is None: + sparse_copy(self.imagepath, target) + else: + # copy first sectors that may contain bootloader + sparse_copy(self.imagepath, target, length=2048 * self._lsector_size) + + # copy source partition table to the target + parts = read_ptable(self.imagepath) + write_ptable(parts, target) + + # get size of unpartitioned space + free = None + for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines(): + if line.startswith("Unpartitioned space ") and line.endswith("sectors"): + free = int(line.split()[-2]) + if free is None: + raise WicError("Can't get size of unpartitioned space") + + # calculate expanded partitions sizes + sizes = {} + for num, part in enumerate(parts['partitiontable']['partitions'], 1): + if num in expand: + if expand[num] != 0: # don't resize partition if size is set to 0 + sectors = expand[num] // self._lsector_size + free -= sectors - part['size'] + part['size'] = sectors + sizes[num] = sectors + elif part['type'] != 'f': + sizes[num] = -1 + + for num, part in enumerate(parts['partitiontable']['partitions'], 1): + if sizes.get(num) == -1: + part['size'] += free // len(sizes) + + # write resized partition table to the target + write_ptable(parts, target) + + # read resized partition table + parts = read_ptable(target) + + # copy partitions content + for num, part in enumerate(parts['partitiontable']['partitions'], 1): + pnum = str(num) + fstype = self.partitions[pnum].fstype + + # copy unchanged partition + if part['size'] == self.partitions[pnum].size // self._lsector_size: + logger.info("copying unchanged partition {}".format(pnum)) + sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size) + continue + + # resize or re-create partitions + if fstype.startswith('ext') or fstype.startswith('fat') or \ + fstype.startswith('linux-swap'): + + partfname = None + with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf: + partfname = partf.name + + if fstype.startswith('ext'): + logger.info("resizing ext partition {}".format(pnum)) + partimg = self._get_part_image(pnum) + sparse_copy(partimg, partfname) + exec_cmd("{} -pf {}".format(self.e2fsck, partfname)) + exec_cmd("{} {} {}s".format(\ + self.resize2fs, partfname, part['size'])) + elif fstype.startswith('fat'): + logger.info("copying content of the fat partition {}".format(pnum)) + with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir: + # copy content to the temporary directory + cmd = "{} -snompi {} :: {}".format(self.mcopy, + self._get_part_image(pnum), + tmpdir) + exec_cmd(cmd) + # create new msdos partition + label = part.get("name") + label_str = "-n {}".format(label) if label else '' + + cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname, + part['size']) + exec_cmd(cmd) + # copy content from the temporary directory to the new partition + cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir) + exec_cmd(cmd, as_shell=True) + elif fstype.startswith('linux-swap'): + logger.info("creating swap partition {}".format(pnum)) + label = part.get("name") + label_str = "-L {}".format(label) if label else '' + uuid = part.get("uuid") + uuid_str = "-U {}".format(uuid) if uuid else '' + with open(partfname, 'w') as sparse: + os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size) + exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname)) + sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) + os.unlink(partfname) + elif part['type'] != 'f': + logger.warn("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) + +def wic_ls(args, native_sysroot): + """List contents of partitioned image or vfat partition.""" + disk = Disk(args.path.image, native_sysroot) + if not args.path.part: + if disk.partitions: + print('Num Start End Size Fstype') + for part in disk.partitions.values(): + print("{:2d} {:12d} {:12d} {:12d} {}".format(\ + part.pnum, part.start, part.end, + part.size, part.fstype)) + else: + path = args.path.path or '/' + print(disk.dir(args.path.part, path)) + +def wic_cp(args, native_sysroot): + """ + Copy local file or directory to the vfat partition of + partitioned image. + """ + disk = Disk(args.dest.image, native_sysroot) + disk.copy(args.src, args.dest.part, args.dest.path) + +def wic_rm(args, native_sysroot): + """ + Remove files or directories from the vfat partition of + partitioned image. + """ + disk = Disk(args.path.image, native_sysroot) + disk.remove(args.path.part, args.path.path) + +def wic_write(args, native_sysroot): + """ + Write image to a target device. + """ + disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'swap')) + disk.write(args.target, args.expand) + +def find_canned(scripts_path, file_name): + """ + Find a file either by its path or by name in the canned files dir. + + Return None if not found + """ + if os.path.exists(file_name): + return file_name + + layers_canned_wks_dir = build_canned_image_list(scripts_path) + for canned_wks_dir in layers_canned_wks_dir: + for root, dirs, files in os.walk(canned_wks_dir): + for fname in files: + if fname == file_name: + fullpath = os.path.join(canned_wks_dir, fname) + return fullpath + +def get_custom_config(boot_file): + """ + Get the custom configuration to be used for the bootloader. + + Return None if the file can't be found. + """ + # Get the scripts path of poky + scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__)) + + cfg_file = find_canned(scripts_path, boot_file) + if cfg_file: + with open(cfg_file, "r") as f: + config = f.read() + return config |