From 1a4b7ee28bf7413af6513fb45ad0d0736048f866 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Sun, 16 Dec 2018 17:11:34 -0800 Subject: reset upstream subtrees to yocto 2.6 Reset the following subtrees on thud HEAD: poky: 87e3a9739d meta-openembedded: 6094ae18c8 meta-security: 31dc4e7532 meta-raspberrypi: a48743dc36 meta-xilinx: c42016e2e6 Also re-apply backports that didn't make it into thud: poky: 17726d0 systemd-systemctl-native: handle Install wildcards meta-openembedded: 4321a5d libtinyxml2: update to 7.0.1 042f0a3 libcereal: Add native and nativesdk classes e23284f libcereal: Allow empty package 030e8d4 rsyslog: curl-less build with fmhttp PACKAGECONFIG 179a1b9 gtest: update to 1.8.1 Squashed OpenBMC subtree compatibility updates: meta-aspeed: Brad Bishop (1): aspeed: add yocto 2.6 compatibility meta-ibm: Brad Bishop (1): ibm: prepare for yocto 2.6 meta-ingrasys: Brad Bishop (1): ingrasys: set layer compatibility to yocto 2.6 meta-openpower: Brad Bishop (1): openpower: set layer compatibility to yocto 2.6 meta-phosphor: Brad Bishop (3): phosphor: set layer compatibility to thud phosphor: libgpg-error: drop patches phosphor: react to fitimage artifact rename Ed Tanous (4): Dropbear: upgrade options for latest upgrade yocto2.6: update openssl options busybox: remove upstream watchdog patch systemd: Rebase CONFIG_CGROUP_BPF patch Change-Id: I7b1fe71cca880d0372a82d94b5fd785323e3a9e7 Signed-off-by: Brad Bishop --- poky/scripts/lib/wic/canned-wks/mkhybridiso.wks | 2 +- poky/scripts/lib/wic/engine.py | 29 ++++- poky/scripts/lib/wic/filemap.py | 9 +- poky/scripts/lib/wic/help.py | 18 ++- poky/scripts/lib/wic/ksparser.py | 16 ++- poky/scripts/lib/wic/partition.py | 4 +- poky/scripts/lib/wic/plugins/imager/direct.py | 6 + .../lib/wic/plugins/source/bootimg-partition.py | 121 +++++++++++++++++---- .../lib/wic/plugins/source/bootimg-pcbios.py | 10 +- .../lib/wic/plugins/source/isoimage-isohybrid.py | 36 +----- 10 files changed, 175 insertions(+), 76 deletions(-) (limited to 'poky/scripts/lib/wic') diff --git a/poky/scripts/lib/wic/canned-wks/mkhybridiso.wks b/poky/scripts/lib/wic/canned-wks/mkhybridiso.wks index 9d34e9b47..48c5ac479 100644 --- a/poky/scripts/lib/wic/canned-wks/mkhybridiso.wks +++ b/poky/scripts/lib/wic/canned-wks/mkhybridiso.wks @@ -2,6 +2,6 @@ # long-description: Creates an EFI and legacy bootable hybrid ISO image # which can be used on optical media as well as USB media. -part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO --fstype=ext4 +part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO bootloader --timeout=15 --append="" diff --git a/poky/scripts/lib/wic/engine.py b/poky/scripts/lib/wic/engine.py index f0c5ff0aa..4662c665c 100644 --- a/poky/scripts/lib/wic/engine.py +++ b/poky/scripts/lib/wic/engine.py @@ -191,7 +191,7 @@ def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, if not os.path.exists(options.outdir): os.makedirs(options.outdir) - pname = 'direct' + pname = options.imager plugin_class = PluginMgr.get_plugins('imager').get(pname) if not plugin_class: raise WicError('Unknown plugin: %s' % pname) @@ -266,10 +266,15 @@ class Disk: out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) parttype = namedtuple("Part", "pnum start end size fstype") splitted = out.splitlines() - lsector_size, psector_size, self._ptable_format = splitted[1].split(":")[3:6] + # skip over possible errors in exec_cmd output + try: + idx =splitted.index("BYT;") + except ValueError: + raise WicError("Error getting partition information from %s" % (self.parted)) + lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6] self._lsector_size = int(lsector_size) self._psector_size = int(psector_size) - for line in splitted[2:]: + for line in splitted[idx + 2:]: pnum, start, end, size, fstype = line.split(':')[:5] partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]), int(size[:-1]), fstype) @@ -340,9 +345,21 @@ class Disk: """Remove files/dirs from the partition.""" partimg = self._get_part_image(pnum) if self.partitions[pnum].fstype.startswith('ext'): - exec_cmd("{} {} -wR 'rm {}'".format(self.debugfs, + cmd = "{} {} -wR 'rm {}'".format(self.debugfs, self._get_part_image(pnum), - path), as_shell=True) + path) + out = exec_cmd(cmd , as_shell=True) + for line in out.splitlines(): + if line.startswith("rm:"): + if "file is a directory" in line: + # Try rmdir to see if this is an empty directory. This won't delete + # any non empty directory so let user know about any error that this might + # generate. + print(exec_cmd("{} {} -wR 'rmdir {}'".format(self.debugfs, + self._get_part_image(pnum), + path), as_shell=True)) + else: + raise WicError("Could not complete operation: wic %s" % str(line)) else: # fat cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) try: @@ -494,7 +511,7 @@ class Disk: sparse_copy(partfname, target, seek=part['start'] * self._lsector_size) os.unlink(partfname) elif part['type'] != 'f': - logger.warn("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) + logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype)) def wic_ls(args, native_sysroot): """List contents of partitioned image or vfat partition.""" diff --git a/poky/scripts/lib/wic/filemap.py b/poky/scripts/lib/wic/filemap.py index a72fa09ef..abbf958b8 100644 --- a/poky/scripts/lib/wic/filemap.py +++ b/poky/scripts/lib/wic/filemap.py @@ -22,6 +22,7 @@ and returns an instance of the class. # * Too many instance attributes (R0902) # pylint: disable=R0902 +import errno import os import struct import array @@ -189,9 +190,9 @@ def _lseek(file_obj, offset, whence): except OSError as err: # The 'lseek' system call returns the ENXIO if there is no data or # hole starting from the specified offset. - if err.errno == os.errno.ENXIO: + if err.errno == errno.ENXIO: return -1 - elif err.errno == os.errno.EINVAL: + elif err.errno == errno.EINVAL: raise ErrorNotSupp("the kernel or file-system does not support " "\"SEEK_HOLE\" and \"SEEK_DATA\"") else: @@ -394,12 +395,12 @@ class FilemapFiemap(_FilemapBase): except IOError as err: # Note, the FIEMAP ioctl is supported by the Linux kernel starting # from version 2.6.28 (year 2008). - if err.errno == os.errno.EOPNOTSUPP: + if err.errno == errno.EOPNOTSUPP: errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ "by the file-system" self._log.debug(errstr) raise ErrorNotSupp(errstr) - if err.errno == os.errno.ENOTTY: + if err.errno == errno.ENOTTY: errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ "by the kernel" self._log.debug(errstr) diff --git a/poky/scripts/lib/wic/help.py b/poky/scripts/lib/wic/help.py index 842b868a5..64f08052c 100644 --- a/poky/scripts/lib/wic/help.py +++ b/poky/scripts/lib/wic/help.py @@ -866,11 +866,11 @@ DESCRIPTION Partitions with a specified will be automatically mounted. This is achieved by wic adding entries to the fstab during image generation. In order for a valid fstab to be generated one of the - --ondrive, --ondisk or --use-uuid partition options must be used for - each partition that specifies a mountpoint. Note that with --use-uuid - and non-root , including swap, the mount program must - understand the PARTUUID syntax. This currently excludes the busybox - versions of these applications. + --ondrive, --ondisk, --use-uuid or --use-label partition options must + be used for each partition that specifies a mountpoint. Note that with + --use-{uuid,label} and non-root , including swap, the mount + program must understand the PARTUUID or LABEL syntax. This currently + excludes the busybox versions of these applications. The following are supported 'part' options: @@ -945,6 +945,14 @@ DESCRIPTION label is already in use by another filesystem, a new label is created for the partition. + --use-label: This option is specific to wic. It makes wic to use the + label in /etc/fstab to specify a partition. If the + --use-label and --use-uuid are used at the same time, + we prefer the uuid because it is less likely to cause + name confliction. We don't support using this parameter + on the root partition since it requires an initramfs to + parse this value and we do not currently support that. + --active: Marks the partition as active. --align (in KBytes): This option is specific to wic and says diff --git a/poky/scripts/lib/wic/ksparser.py b/poky/scripts/lib/wic/ksparser.py index e590b2fe3..7e5a9c509 100644 --- a/poky/scripts/lib/wic/ksparser.py +++ b/poky/scripts/lib/wic/ksparser.py @@ -141,6 +141,7 @@ class KickStart(): 'squashfs', 'vfat', 'msdos', 'swap')) part.add_argument('--mkfs-extraopts', default='') part.add_argument('--label') + part.add_argument('--use-label', action='store_true') part.add_argument('--no-table', action='store_true') part.add_argument('--ondisk', '--ondrive', dest='disk', default='sda') part.add_argument("--overhead-factor", type=overheadtype) @@ -196,9 +197,18 @@ class KickStart(): raise KickStartError('%s:%d: %s' % \ (confpath, lineno, err)) if line.startswith('part'): - # SquashFS does not support UUID - if parsed.fstype == 'squashfs' and parsed.use_uuid: - err = "%s:%d: SquashFS does not support UUID" \ + # SquashFS does not support filesystem UUID + if parsed.fstype == 'squashfs': + if parsed.fsuuid: + err = "%s:%d: SquashFS does not support UUID" \ + % (confpath, lineno) + raise KickStartError(err) + if parsed.label: + err = "%s:%d: SquashFS does not support LABEL" \ + % (confpath, lineno) + raise KickStartError(err) + if parsed.use_label and not parsed.label: + err = "%s:%d: Must set the label with --label" \ % (confpath, lineno) raise KickStartError(err) # using ArgumentParser one cannot easily tell if option diff --git a/poky/scripts/lib/wic/partition.py b/poky/scripts/lib/wic/partition.py index 3fe5c4e26..3da7e23e6 100644 --- a/poky/scripts/lib/wic/partition.py +++ b/poky/scripts/lib/wic/partition.py @@ -47,6 +47,7 @@ class Partition(): self.fsopts = args.fsopts self.fstype = args.fstype self.label = args.label + self.use_label = args.use_label self.mkfs_extraopts = args.mkfs_extraopts self.mountpoint = args.mountpoint self.no_table = args.no_table @@ -66,7 +67,6 @@ class Partition(): self.lineno = lineno self.source_file = "" - self.sourceparams_dict = {} def get_extra_block_count(self, current_blocks): """ @@ -211,7 +211,7 @@ class Partition(): """ p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR", - "%s/../pseudo" % get_bitbake_var("IMAGE_ROOTFS")) + "%s/../pseudo" % rootfs_dir) p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir) p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1") pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix diff --git a/poky/scripts/lib/wic/plugins/imager/direct.py b/poky/scripts/lib/wic/plugins/imager/direct.py index 1fa6b917e..bb14a334b 100644 --- a/poky/scripts/lib/wic/plugins/imager/direct.py +++ b/poky/scripts/lib/wic/plugins/imager/direct.py @@ -122,6 +122,10 @@ class DirectPlugin(ImagerPlugin): if self._update_fstab(fstab_lines, self.parts): # copy rootfs dir to workdir to update fstab # as rootfs can be used by other tasks and can't be modified + new_pseudo = os.path.realpath(os.path.join(self.workdir, "pseudo")) + from_dir = os.path.join(os.path.join(image_rootfs, ".."), "pseudo") + from_dir = os.path.realpath(from_dir) + copyhardlinktree(from_dir, new_pseudo) new_rootfs = os.path.realpath(os.path.join(self.workdir, "rootfs_copy")) copyhardlinktree(image_rootfs, new_rootfs) fstab_path = os.path.join(new_rootfs, 'etc/fstab') @@ -151,6 +155,8 @@ class DirectPlugin(ImagerPlugin): device_name = "UUID=%s" % part.fsuuid else: device_name = "PARTUUID=%s" % part.uuid + elif part.use_label: + device_name = "LABEL=%s" % part.label else: # mmc device partitions are named mmcblk0p1, mmcblk0p2.. prefix = 'p' if part.disk.startswith('mmcblk') else '' diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py index b239fc0b4..ddc880be3 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -30,6 +30,7 @@ import re from glob import glob from wic import WicError +from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin from wic.misc import exec_cmd, get_bitbake_var @@ -44,15 +45,11 @@ class BootimgPartitionPlugin(SourcePlugin): name = 'bootimg-partition' @classmethod - def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + def do_configure_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, - rootfs_dir, native_sysroot): + native_sysroot): """ - Called to do the actual content population for a partition i.e. it - 'prepares' the partition to be incorporated into the image. - In this case, does the following: - - sets up a vfat partition - - copies all files listed in IMAGE_BOOT_FILES variable + Called before do_prepare_partition(), create u-boot specific boot config """ hdddir = "%s/boot.%d" % (cr_workdir, part.lineno) install_cmd = "install -d %s" % hdddir @@ -63,8 +60,6 @@ class BootimgPartitionPlugin(SourcePlugin): if not kernel_dir: raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") - logger.debug('Kernel dir: %s', bootimg_dir) - boot_files = None for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)): if fmt: @@ -94,9 +89,9 @@ class BootimgPartitionPlugin(SourcePlugin): logger.debug('Destination entry: %r', dst_entry) deploy_files.append(dst_entry) + cls.install_task = []; for deploy_entry in deploy_files: src, dst = deploy_entry - install_task = [] if '*' in src: # by default install files under their basename entry_name_fn = os.path.basename @@ -111,21 +106,101 @@ class BootimgPartitionPlugin(SourcePlugin): logger.debug('Globbed sources: %s', ', '.join(srcs)) for entry in srcs: + src = os.path.relpath(entry, kernel_dir) entry_dst_name = entry_name_fn(entry) - install_task.append((entry, - os.path.join(hdddir, - entry_dst_name))) + cls.install_task.append((src, entry_dst_name)) else: - install_task = [(os.path.join(kernel_dir, src), - os.path.join(hdddir, dst))] - - for task in install_task: - src_path, dst_path = task - logger.debug('Install %s as %s', - os.path.basename(src_path), dst_path) - install_cmd = "install -m 0644 -D %s %s" \ - % (src_path, dst_path) - exec_cmd(install_cmd) + cls.install_task.append((src, dst)) + + if source_params.get('loader') != "u-boot": + return + + configfile = cr.ks.bootloader.configfile + custom_cfg = None + if configfile: + custom_cfg = get_custom_config(configfile) + if custom_cfg: + # Use a custom configuration for extlinux.conf + extlinux_conf = custom_cfg + logger.debug("Using custom configuration file " + "%s for extlinux.cfg", configfile) + else: + raise WicError("configfile is specified but failed to " + "get it from %s." % configfile) + + if not custom_cfg: + # The kernel types supported by the sysboot of u-boot + kernel_types = ["zImage", "Image", "fitImage", "uImage", "vmlinux"] + has_dtb = False + fdt_dir = '/' + kernel_name = None + + # Find the kernel image name, from the highest precedence to lowest + for image in kernel_types: + for task in cls.install_task: + src, dst = task + if re.match(image, src): + kernel_name = os.path.join('/', dst) + break + if kernel_name: + break + + for task in cls.install_task: + src, dst = task + # We suppose that all the dtb are in the same directory + if re.search(r'\.dtb', src) and fdt_dir == '/': + has_dtb = True + fdt_dir = os.path.join(fdt_dir, os.path.dirname(dst)) + break + + if not kernel_name: + raise WicError('No kernel file founded') + + # Compose the extlinux.conf + extlinux_conf = "default Yocto\n" + extlinux_conf += "label Yocto\n" + extlinux_conf += " kernel %s\n" % kernel_name + if has_dtb: + extlinux_conf += " fdtdir %s\n" % fdt_dir + bootloader = cr.ks.bootloader + extlinux_conf += "append root=%s rootwait %s\n" \ + % (cr.rootdev, bootloader.append if bootloader.append else '') + + install_cmd = "install -d %s/extlinux/" % hdddir + exec_cmd(install_cmd) + cfg = open("%s/extlinux/extlinux.conf" % hdddir, "w") + cfg.write(extlinux_conf) + cfg.close() + + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + In this case, does the following: + - sets up a vfat partition + - copies all files listed in IMAGE_BOOT_FILES variable + """ + hdddir = "%s/boot.%d" % (cr_workdir, part.lineno) + + if not kernel_dir: + kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + if not kernel_dir: + raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting") + + logger.debug('Kernel dir: %s', bootimg_dir) + + + for task in cls.install_task: + src_path, dst_path = task + logger.debug('Install %s as %s', src_path, dst_path) + install_cmd = "install -m 0644 -D %s %s" \ + % (os.path.join(kernel_dir, src_path), + os.path.join(hdddir, dst_path)) + exec_cmd(install_cmd) logger.debug('Prepare boot partition using rootfs in %s', hdddir) part.prepare_rootfs(cr_workdir, oe_builddir, hdddir, diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py index d599112dd..9347aa7fc 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -26,6 +26,7 @@ import logging import os +import re from wic import WicError from wic.engine import get_custom_config @@ -47,10 +48,17 @@ class BootimgPcbiosPlugin(SourcePlugin): """ Check if dirname exists in default bootimg_dir or in STAGING_DIR. """ - for result in (bootimg_dir, get_bitbake_var("STAGING_DATADIR")): + staging_datadir = get_bitbake_var("STAGING_DATADIR") + for result in (bootimg_dir, staging_datadir): if os.path.exists("%s/%s" % (result, dirname)): return result + # STAGING_DATADIR is expanded with MLPREFIX if multilib is enabled + # but dependency syslinux is still populated to original STAGING_DATADIR + nonarch_datadir = re.sub('/[^/]*recipe-sysroot', '/recipe-sysroot', staging_datadir) + if os.path.exists(os.path.join(nonarch_datadir, dirname)): + return nonarch_datadir + raise WicError("Couldn't find correct bootimg_dir, exiting") @classmethod diff --git a/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index b119c9c2f..170077c22 100644 --- a/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -47,7 +47,7 @@ class IsoImagePlugin(SourcePlugin): Example kickstart file: part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\ - image_name= IsoImage" --ondisk cd --label LIVECD --fstype=ext2 + image_name= IsoImage" --ondisk cd --label LIVECD bootloader --timeout=10 --append=" " In --sourceparams "loader" specifies the bootloader used for booting in EFI @@ -191,10 +191,9 @@ class IsoImagePlugin(SourcePlugin): else: raise WicError("Couldn't find or build initrd, exiting.") - exec_cmd("cd %s && find . | cpio -o -H newc -R +0:+0 >./initrd.cpio " \ - % initrd_dir, as_shell=True) - exec_cmd("gzip -f -9 -c %s/initrd.cpio > %s" \ - % (cr_workdir, initrd), as_shell=True) + exec_cmd("cd %s && find . | cpio -o -H newc -R root:root >%s/initrd.cpio " \ + % (initrd_dir, cr_workdir), as_shell=True) + exec_cmd("gzip -f -9 %s/initrd.cpio" % cr_workdir, as_shell=True) shutil.rmtree(initrd_dir) return initrd @@ -253,33 +252,8 @@ class IsoImagePlugin(SourcePlugin): raise WicError("Couldn't find IMAGE_ROOTFS, exiting.") part.rootfs_dir = rootfs_dir - - # Prepare rootfs.img deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") img_iso_dir = get_bitbake_var("ISODIR") - rootfs_img = "%s/rootfs.img" % img_iso_dir - if not os.path.isfile(rootfs_img): - # check if rootfs.img is in deploydir - deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") - image_name = get_bitbake_var("IMAGE_LINK_NAME") - rootfs_img = "%s/%s.%s" \ - % (deploy_dir, image_name, part.fstype) - - if not os.path.isfile(rootfs_img): - # create image file with type specified by --fstype - # which contains rootfs - du_cmd = "du -bks %s" % rootfs_dir - out = exec_cmd(du_cmd) - part.size = int(out.split()[0]) - part.extra_space = 0 - part.overhead_factor = 1.2 - part.prepare_rootfs(cr_workdir, oe_builddir, rootfs_dir, \ - native_sysroot) - rootfs_img = part.source_file - - install_cmd = "install -m 0644 %s %s/rootfs.img" \ - % (rootfs_img, isodir) - exec_cmd(install_cmd) # Remove the temporary file created by part.prepare_rootfs() if os.path.isfile(part.source_file): @@ -342,7 +316,7 @@ class IsoImagePlugin(SourcePlugin): grub_src = os.path.join(deploy_dir, grub_src_image) if not os.path.exists(grub_src): raise WicError("Grub loader %s is not found in %s. " - "Please build grub-efi first" % (grub_image, deploy_dir)) + "Please build grub-efi first" % (grub_src_image, deploy_dir)) shutil.copy(grub_src, grub_target) if not os.path.isfile(os.path.join(target_dir, "boot.cfg")): -- cgit v1.2.3