From 1a4b7ee28bf7413af6513fb45ad0d0736048f866 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Sun, 16 Dec 2018 17:11:34 -0800 Subject: reset upstream subtrees to yocto 2.6 Reset the following subtrees on thud HEAD: poky: 87e3a9739d meta-openembedded: 6094ae18c8 meta-security: 31dc4e7532 meta-raspberrypi: a48743dc36 meta-xilinx: c42016e2e6 Also re-apply backports that didn't make it into thud: poky: 17726d0 systemd-systemctl-native: handle Install wildcards meta-openembedded: 4321a5d libtinyxml2: update to 7.0.1 042f0a3 libcereal: Add native and nativesdk classes e23284f libcereal: Allow empty package 030e8d4 rsyslog: curl-less build with fmhttp PACKAGECONFIG 179a1b9 gtest: update to 1.8.1 Squashed OpenBMC subtree compatibility updates: meta-aspeed: Brad Bishop (1): aspeed: add yocto 2.6 compatibility meta-ibm: Brad Bishop (1): ibm: prepare for yocto 2.6 meta-ingrasys: Brad Bishop (1): ingrasys: set layer compatibility to yocto 2.6 meta-openpower: Brad Bishop (1): openpower: set layer compatibility to yocto 2.6 meta-phosphor: Brad Bishop (3): phosphor: set layer compatibility to thud phosphor: libgpg-error: drop patches phosphor: react to fitimage artifact rename Ed Tanous (4): Dropbear: upgrade options for latest upgrade yocto2.6: update openssl options busybox: remove upstream watchdog patch systemd: Rebase CONFIG_CGROUP_BPF patch Change-Id: I7b1fe71cca880d0372a82d94b5fd785323e3a9e7 Signed-off-by: Brad Bishop --- poky/meta/lib/oe/buildhistory_analysis.py | 42 ++----- poky/meta/lib/oe/copy_buildsystem.py | 26 +++- poky/meta/lib/oe/elf.py | 128 +++++++++++++++++++ poky/meta/lib/oe/gpg_sign.py | 15 +-- poky/meta/lib/oe/maketype.py | 9 +- poky/meta/lib/oe/package.py | 126 +++++++++++-------- poky/meta/lib/oe/package_manager.py | 200 +++++++++++++++++++++--------- poky/meta/lib/oe/patch.py | 3 +- poky/meta/lib/oe/path.py | 38 +++++- poky/meta/lib/oe/qa.py | 3 +- poky/meta/lib/oe/rootfs.py | 22 +++- poky/meta/lib/oe/sdk.py | 83 ++++--------- poky/meta/lib/oe/sstatesig.py | 17 ++- poky/meta/lib/oe/terminal.py | 6 +- poky/meta/lib/oe/types.py | 7 +- poky/meta/lib/oe/utils.py | 133 ++++++++++++++------ 16 files changed, 600 insertions(+), 258 deletions(-) create mode 100644 poky/meta/lib/oe/elf.py (limited to 'poky/meta/lib/oe') diff --git a/poky/meta/lib/oe/buildhistory_analysis.py b/poky/meta/lib/oe/buildhistory_analysis.py index b0365abced..ad7fceb8bb 100644 --- a/poky/meta/lib/oe/buildhistory_analysis.py +++ b/poky/meta/lib/oe/buildhistory_analysis.py @@ -13,6 +13,7 @@ import os.path import difflib import git import re +import shlex import hashlib import collections import bb.utils @@ -31,13 +32,6 @@ ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR'] monitor_numeric_threshold = 10 # Image files to monitor (note that image-info.txt is handled separately) img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] -# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields) -related_fields = {} -related_fields['RDEPENDS'] = ['DEPENDS'] -related_fields['RRECOMMENDS'] = ['DEPENDS'] -related_fields['FILELIST'] = ['FILES'] -related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] -related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] colours = { 'colour_default': '', @@ -67,7 +61,6 @@ class ChangeRecord: self.oldvalue = oldvalue self.newvalue = newvalue self.monitored = monitored - self.related = [] self.filechanges = None def __str__(self): @@ -123,10 +116,13 @@ class ChangeRecord: aitems = pkglist_combine(depvera) bitems = pkglist_combine(depverb) else: - aitems = self.oldvalue.split() - bitems = self.newvalue.split() if self.fieldname == 'FILELIST': + aitems = shlex.split(self.oldvalue) + bitems = shlex.split(self.newvalue) renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + else: + aitems = self.oldvalue.split() + bitems = self.newvalue.split() removed = list(set(aitems) - set(bitems)) added = list(set(bitems) - set(aitems)) @@ -206,13 +202,6 @@ class ChangeRecord: else: out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours) - if self.related: - for chg in self.related: - if not outer and chg.fieldname in ['PE', 'PV', 'PR']: - continue - for line in chg._str_internal(False).splitlines(): - out += '\n * %s' % line - return '%s%s' % (prefix, out) if out else '' class FileChange: @@ -424,9 +413,13 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): (depvera, depverb) = compare_pkg_lists(astr, bstr) if depvera == depverb: continue - alist = astr.split() + if key == 'FILELIST': + alist = shlex.split(astr) + blist = shlex.split(bstr) + else: + alist = astr.split() + blist = bstr.split() alist.sort() - blist = bstr.split() blist.sort() # We don't care about the removal of self-dependencies if pkgname in alist and not pkgname in blist: @@ -635,17 +628,6 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) - # Link related changes - for chg in changes: - if chg.monitored: - for chg2 in changes: - # (Check dirname in the case of fields from recipe info files) - if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path: - if chg2.fieldname in related_fields.get(chg.fieldname, []): - chg.related.append(chg2) - elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: - chg.related.append(chg2) - # filter out unwanted paths if exclude_path: for chg in changes: diff --git a/poky/meta/lib/oe/copy_buildsystem.py b/poky/meta/lib/oe/copy_buildsystem.py index 4b94806c73..7cb784cf8c 100644 --- a/poky/meta/lib/oe/copy_buildsystem.py +++ b/poky/meta/lib/oe/copy_buildsystem.py @@ -1,5 +1,12 @@ # This class should provide easy access to the different aspects of the # buildsystem such as layers, bitbake location, etc. +# +# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers. +# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports +# python regular expression, use space as separator, +# e.g.: ".*-downloads closed-.*" +# + import stat import shutil @@ -23,9 +30,12 @@ class BuildSystem(object): self.context = context self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() + self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN') def copy_bitbake_and_layers(self, destdir, workspace_name=None): + import re # Copy in all metadata layers + bitbake (as repositories) + copied_corebase = None layers_copied = [] bb.utils.mkdirhier(destdir) layers = list(self.layerdirs) @@ -40,8 +50,17 @@ class BuildSystem(object): # Exclude layers for layer_exclude in self.layers_exclude: if layer_exclude in layers: + bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude) layers.remove(layer_exclude) + if self.layers_exclude_pattern: + layers_cp = layers[:] + for pattern in self.layers_exclude_pattern.split(): + for layer in layers_cp: + if re.match(pattern, layer): + bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer) + layers.remove(layer) + workspace_newname = workspace_name if workspace_newname: layernames = [os.path.basename(layer) for layer in layers] @@ -84,17 +103,18 @@ class BuildSystem(object): layer_relative = os.path.relpath(layerdestpath, destdir) - layers_copied.append(layer_relative) - # Treat corebase as special since it typically will contain # build directories or other custom items. if corebase == layer: + copied_corebase = layer_relative bb.utils.mkdirhier(layerdestpath) for f in corebase_files: f_basename = os.path.basename(f) destname = os.path.join(layerdestpath, f_basename) _smart_copy(f, destname) else: + layers_copied.append(layer_relative) + if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')): bb.note("Skipping layer %s, already handled" % layer) else: @@ -140,7 +160,7 @@ class BuildSystem(object): layers_copied.remove(layer) break - return layers_copied + return copied_corebase, layers_copied def generate_locked_sigs(sigfile, d): bb.utils.mkdirhier(os.path.dirname(sigfile)) diff --git a/poky/meta/lib/oe/elf.py b/poky/meta/lib/oe/elf.py new file mode 100644 index 0000000000..0ed59ae05c --- /dev/null +++ b/poky/meta/lib/oe/elf.py @@ -0,0 +1,128 @@ +def machine_dict(d): +# TARGET_OS TARGET_ARCH MACHINE, OSABI, ABIVERSION, Little Endian, 32bit? + machdata = { + "darwin9" : { + "arm" : (40, 0, 0, True, 32), + }, + "eabi" : { + "arm" : (40, 0, 0, True, 32), + }, + "elf" : { + "aarch64" : (183, 0, 0, True, 64), + "aarch64_be" :(183, 0, 0, False, 64), + "i586" : (3, 0, 0, True, 32), + "x86_64": (62, 0, 0, True, 64), + "epiphany": (4643, 0, 0, True, 32), + "lm32": (138, 0, 0, False, 32), + "mips": ( 8, 0, 0, False, 32), + "mipsel": ( 8, 0, 0, True, 32), + "microblaze": (189, 0, 0, False, 32), + "microblazeeb":(189, 0, 0, False, 32), + "microblazeel":(189, 0, 0, True, 32), + "powerpc": (20, 0, 0, False, 32), + "riscv32": (243, 0, 0, True, 32), + "riscv64": (243, 0, 0, True, 64), + }, + "linux" : { + "aarch64" : (183, 0, 0, True, 64), + "aarch64_be" :(183, 0, 0, False, 64), + "arm" : (40, 97, 0, True, 32), + "armeb": (40, 97, 0, False, 32), + "powerpc": (20, 0, 0, False, 32), + "powerpc64": (21, 0, 0, False, 64), + "i386": ( 3, 0, 0, True, 32), + "i486": ( 3, 0, 0, True, 32), + "i586": ( 3, 0, 0, True, 32), + "i686": ( 3, 0, 0, True, 32), + "x86_64": (62, 0, 0, True, 64), + "ia64": (50, 0, 0, True, 64), + "alpha": (36902, 0, 0, True, 64), + "hppa": (15, 3, 0, False, 32), + "m68k": ( 4, 0, 0, False, 32), + "mips": ( 8, 0, 0, False, 32), + "mipsel": ( 8, 0, 0, True, 32), + "mips64": ( 8, 0, 0, False, 64), + "mips64el": ( 8, 0, 0, True, 64), + "mipsisa32r6": ( 8, 0, 0, False, 32), + "mipsisa32r6el": ( 8, 0, 0, True, 32), + "mipsisa64r6": ( 8, 0, 0, False, 64), + "mipsisa64r6el": ( 8, 0, 0, True, 64), + "nios2": (113, 0, 0, True, 32), + "riscv32": (243, 0, 0, True, 32), + "riscv64": (243, 0, 0, True, 64), + "s390": (22, 0, 0, False, 32), + "sh4": (42, 0, 0, True, 32), + "sparc": ( 2, 0, 0, False, 32), + "microblaze": (189, 0, 0, False, 32), + "microblazeeb":(189, 0, 0, False, 32), + "microblazeel":(189, 0, 0, True, 32), + }, + "linux-musl" : { + "aarch64" : (183, 0, 0, True, 64), + "aarch64_be" :(183, 0, 0, False, 64), + "arm" : ( 40, 97, 0, True, 32), + "armeb": ( 40, 97, 0, False, 32), + "powerpc": ( 20, 0, 0, False, 32), + "i386": ( 3, 0, 0, True, 32), + "i486": ( 3, 0, 0, True, 32), + "i586": ( 3, 0, 0, True, 32), + "i686": ( 3, 0, 0, True, 32), + "x86_64": ( 62, 0, 0, True, 64), + "mips": ( 8, 0, 0, False, 32), + "mipsel": ( 8, 0, 0, True, 32), + "mips64": ( 8, 0, 0, False, 64), + "mips64el": ( 8, 0, 0, True, 64), + "microblaze": (189, 0, 0, False, 32), + "microblazeeb":(189, 0, 0, False, 32), + "microblazeel":(189, 0, 0, True, 32), + "riscv32": (243, 0, 0, True, 32), + "riscv64": (243, 0, 0, True, 64), + "sh4": ( 42, 0, 0, True, 32), + }, + "uclinux-uclibc" : { + "bfin": ( 106, 0, 0, True, 32), + }, + "linux-gnueabi" : { + "arm" : (40, 0, 0, True, 32), + "armeb" : (40, 0, 0, False, 32), + }, + "linux-musleabi" : { + "arm" : (40, 0, 0, True, 32), + "armeb" : (40, 0, 0, False, 32), + }, + "linux-gnuspe" : { + "powerpc": (20, 0, 0, False, 32), + }, + "linux-muslspe" : { + "powerpc": (20, 0, 0, False, 32), + }, + "linux-gnu" : { + "powerpc": (20, 0, 0, False, 32), + "sh4": (42, 0, 0, True, 32), + }, + "linux-gnu_ilp32" : { + "aarch64" : (183, 0, 0, True, 32), + }, + "linux-gnux32" : { + "x86_64": (62, 0, 0, True, 32), + }, + "linux-muslx32" : { + "x86_64": (62, 0, 0, True, 32), + }, + "linux-gnun32" : { + "mips64": ( 8, 0, 0, False, 32), + "mips64el": ( 8, 0, 0, True, 32), + "mipsisa64r6": ( 8, 0, 0, False, 32), + "mipsisa64r6el":( 8, 0, 0, True, 32), + }, + } + + # Add in any extra user supplied data which may come from a BSP layer, removing the + # need to always change this class directly + extra_machdata = (d and d.getVar("PACKAGEQA_EXTRA_MACHDEFFUNCS" or None) or "").split() + for m in extra_machdata: + call = m + "(machdata, d)" + locs = { "machdata" : machdata, "d" : d} + machdata = bb.utils.better_eval(call, locs) + + return machdata diff --git a/poky/meta/lib/oe/gpg_sign.py b/poky/meta/lib/oe/gpg_sign.py index b17272928f..ccd5aee420 100644 --- a/poky/meta/lib/oe/gpg_sign.py +++ b/poky/meta/lib/oe/gpg_sign.py @@ -3,6 +3,8 @@ import os import bb import oe.utils +import subprocess +import shlex class LocalSigner(object): """Class for handling local (on the build host) signing""" @@ -23,10 +25,7 @@ class LocalSigner(object): if armor: cmd += "--armor " cmd += keyid - status, output = oe.utils.getstatusoutput(cmd) - if status: - raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % - (keyid, output)) + subprocess.check_output(shlex.split(cmd), stderr=subprocess.STDOUT) def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None): """Sign RPM files""" @@ -48,13 +47,10 @@ class LocalSigner(object): # Sign in chunks for i in range(0, len(files), sign_chunk): - status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+sign_chunk])) - if status: - raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + subprocess.check_output(shlex.split(cmd + ' '.join(files[i:i+sign_chunk])), stderr=subprocess.STDOUT) def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): """Create a detached signature of a file""" - import subprocess if passphrase_file and passphrase: raise Exception("You should use either passphrase_file of passphrase, not both") @@ -100,7 +96,6 @@ class LocalSigner(object): def get_gpg_version(self): """Return the gpg version as a tuple of ints""" - import subprocess try: ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8") return tuple([int(i) for i in ver_str.split("-")[0].split('.')]) @@ -114,7 +109,7 @@ class LocalSigner(object): if self.gpg_path: cmd += "--homedir %s " % self.gpg_path cmd += sig_file - status, _ = oe.utils.getstatusoutput(cmd) + status = subprocess.call(shlex.split(cmd)) ret = False if status else True return ret diff --git a/poky/meta/lib/oe/maketype.py b/poky/meta/lib/oe/maketype.py index f88981dd90..c36e7b5604 100644 --- a/poky/meta/lib/oe/maketype.py +++ b/poky/meta/lib/oe/maketype.py @@ -7,7 +7,12 @@ the arguments of the type's factory for details. import inspect import oe.types as types -import collections +try: + # Python 3.7+ + from collections.abc import Callable +except ImportError: + # Python < 3.7 + from collections import Callable available_types = {} @@ -96,7 +101,7 @@ for name in dir(types): continue obj = getattr(types, name) - if not isinstance(obj, collections.Callable): + if not isinstance(obj, Callable): continue register(name, obj) diff --git a/poky/meta/lib/oe/package.py b/poky/meta/lib/oe/package.py index 4f3e21ad40..efd36b3758 100644 --- a/poky/meta/lib/oe/package.py +++ b/poky/meta/lib/oe/package.py @@ -1,15 +1,17 @@ +import stat +import mmap +import subprocess + def runstrip(arg): # Function to strip a single file, called from split_and_strip_files below # A working 'file' (one which works on the target architecture) # - # The elftype is a bit pattern (explained in split_and_strip_files) to tell + # The elftype is a bit pattern (explained in is_elf below) to tell # us what type of file we're processing... # 4 - executable # 8 - shared library # 16 - kernel module - import stat, subprocess - (file, elftype, strip) = arg newmode = None @@ -19,11 +21,15 @@ def runstrip(arg): os.chmod(file, newmode) stripcmd = [strip] - + skip_strip = False # kernel module if elftype & 16: - stripcmd.extend(["--strip-debug", "--remove-section=.comment", - "--remove-section=.note", "--preserve-dates"]) + if is_kernel_module_signed(file): + bb.debug(1, "Skip strip on signed module %s" % file) + skip_strip = True + else: + stripcmd.extend(["--strip-debug", "--remove-section=.comment", + "--remove-section=.note", "--preserve-dates"]) # .so and shared library elif ".so" in file and elftype & 8: stripcmd.extend(["--remove-section=.comment", "--remove-section=.note", "--strip-unneeded"]) @@ -34,18 +40,59 @@ def runstrip(arg): stripcmd.append(file) bb.debug(1, "runstrip: %s" % stripcmd) - try: + if not skip_strip: output = subprocess.check_output(stripcmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - bb.error("runstrip: '%s' strip command failed with %s (%s)" % (stripcmd, e.returncode, e.output)) if newmode: os.chmod(file, origmode) - return - - -def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False): +# Detect .ko module by searching for "vermagic=" string +def is_kernel_module(path): + with open(path) as f: + return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0 + +# Detect if .ko module is signed +def is_kernel_module_signed(path): + with open(path, "rb") as f: + f.seek(-28, 2) + module_tail = f.read() + return "Module signature appended" in "".join(chr(c) for c in bytearray(module_tail)) + +# Return type (bits): +# 0 - not elf +# 1 - ELF +# 2 - stripped +# 4 - executable +# 8 - shared library +# 16 - kernel module +def is_elf(path): + exec_type = 0 + result = subprocess.check_output(["file", "-b", path], stderr=subprocess.STDOUT).decode("utf-8") + + if "ELF" in result: + exec_type |= 1 + if "not stripped" not in result: + exec_type |= 2 + if "executable" in result: + exec_type |= 4 + if "shared" in result: + exec_type |= 8 + if "relocatable" in result: + if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path): + exec_type |= 16 + return (path, exec_type) + +def is_static_lib(path): + if path.endswith('.a') and not os.path.islink(path): + with open(path, 'rb') as fh: + # The magic must include the first slash to avoid + # matching golang static libraries + magic = b'!\x0a/' + start = fh.read(len(magic)) + return start == magic + return False + +def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False): """ Strip executable code (like executables, shared libraries) _in_place_ - Based on sysroot_strip in staging.bbclass @@ -56,39 +103,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped= :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP} This is for proper logging and messages only. """ - import stat, errno, oe.path, oe.utils, mmap - - # Detect .ko module by searching for "vermagic=" string - def is_kernel_module(path): - with open(path) as f: - return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0 - - # Return type (bits): - # 0 - not elf - # 1 - ELF - # 2 - stripped - # 4 - executable - # 8 - shared library - # 16 - kernel module - def is_elf(path): - exec_type = 0 - ret, result = oe.utils.getstatusoutput("file -b '%s'" % path) - - if ret: - bb.error("split_and_strip_files: 'file %s' failed" % path) - return exec_type - - if "ELF" in result: - exec_type |= 1 - if "not stripped" not in result: - exec_type |= 2 - if "executable" in result: - exec_type |= 4 - if "shared" in result: - exec_type |= 8 - if "relocatable" in result and is_kernel_module(path): - exec_type |= 16 - return exec_type + import stat, errno, oe.path, oe.utils elffiles = {} inodes = {} @@ -98,6 +113,8 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped= # # First lets figure out all of the files we may have to process # + checkelf = [] + inodecache = {} for root, dirs, files in os.walk(dstdir): for f in files: file = os.path.join(root, f) @@ -123,7 +140,11 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped= # It's a file (or hardlink), not a link # ...but is it ELF, and is it already stripped? - elf_file = is_elf(file) + checkelf.append(file) + inodecache[file] = s.st_ino + results = oe.utils.multiprocess_launch(is_elf, checkelf, d) + for (file, elf_file) in results: + #elf_file = is_elf(file) if elf_file & 1: if elf_file & 2: if qa_already_stripped: @@ -132,13 +153,13 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped= bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn)) continue - if s.st_ino in inodes: + if inodecache[file] in inodes: os.unlink(file) - os.link(inodes[s.st_ino], file) + os.link(inodes[inodecache[file]], file) else: # break hardlinks so that we do not strip the original. - inodes[s.st_ino] = file - bb.utils.copyfile(file, file) + inodes[inodecache[file]] = file + bb.utils.break_hardlinks(file) elffiles[file] = elf_file # @@ -149,8 +170,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped= elf_file = int(elffiles[file]) sfiles.append((file, elf_file, strip_cmd)) - oe.utils.multiprocess_exec(sfiles, runstrip) - + oe.utils.multiprocess_launch(runstrip, sfiles, d) def file_translate(file): diff --git a/poky/meta/lib/oe/package_manager.py b/poky/meta/lib/oe/package_manager.py index 2d8aeba037..882e7c429f 100644 --- a/poky/meta/lib/oe/package_manager.py +++ b/poky/meta/lib/oe/package_manager.py @@ -3,7 +3,6 @@ import os import glob import subprocess import shutil -import multiprocessing import re import collections import bb @@ -13,6 +12,7 @@ import oe.path import string from oe.gpg_sign import get_signer import hashlib +import fnmatch # this can be used by all PM backends to create the index files in parallel def create_index(arg): @@ -84,12 +84,55 @@ def opkg_query(cmd_output): return output -# Note: this should be bb.fatal in the future. -def failed_postinsts_warn(pkgs, log_path): - bb.warn("""Intentionally failing postinstall scriptlets of %s to defer them to first boot is deprecated. Please place them into pkg_postinst_ontarget_${PN} (). -If deferring to first boot wasn't the intent, then scriptlet failure may mean an issue in the recipe, or a regression elsewhere. +def failed_postinsts_abort(pkgs, log_path): + bb.fatal("""Postinstall scriptlets of %s have failed. If the intention is to defer them to first boot, +then please place them into pkg_postinst_ontarget_${PN} (). +Deferring to first boot via 'exit 1' is no longer supported. Details of the failure are in %s.""" %(pkgs, log_path)) +def generate_locale_archive(d, rootfs, target_arch, localedir): + # Pretty sure we don't need this for locale archive generation but + # keeping it to be safe... + locale_arch_options = { \ + "arm": ["--uint32-align=4", "--little-endian"], + "armeb": ["--uint32-align=4", "--big-endian"], + "aarch64": ["--uint32-align=4", "--little-endian"], + "aarch64_be": ["--uint32-align=4", "--big-endian"], + "sh4": ["--uint32-align=4", "--big-endian"], + "powerpc": ["--uint32-align=4", "--big-endian"], + "powerpc64": ["--uint32-align=4", "--big-endian"], + "mips": ["--uint32-align=4", "--big-endian"], + "mipsisa32r6": ["--uint32-align=4", "--big-endian"], + "mips64": ["--uint32-align=4", "--big-endian"], + "mipsisa64r6": ["--uint32-align=4", "--big-endian"], + "mipsel": ["--uint32-align=4", "--little-endian"], + "mipsisa32r6el": ["--uint32-align=4", "--little-endian"], + "mips64el": ["--uint32-align=4", "--little-endian"], + "mipsisa64r6el": ["--uint32-align=4", "--little-endian"], + "riscv64": ["--uint32-align=4", "--little-endian"], + "riscv32": ["--uint32-align=4", "--little-endian"], + "i586": ["--uint32-align=4", "--little-endian"], + "i686": ["--uint32-align=4", "--little-endian"], + "x86_64": ["--uint32-align=4", "--little-endian"] + } + if target_arch in locale_arch_options: + arch_options = locale_arch_options[target_arch] + else: + bb.error("locale_arch_options not found for target_arch=" + target_arch) + bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") + + # Need to set this so cross-localedef knows where the archive is + env = dict(os.environ) + env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive") + + for name in os.listdir(localedir): + path = os.path.join(localedir, name) + if os.path.isdir(path): + cmd = ["cross-localedef", "--verbose"] + cmd += arch_options + cmd += ["--add-to-archive", path] + subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT) + class Indexer(object, metaclass=ABCMeta): def __init__(self, d, deploy_dir): self.d = d @@ -177,7 +220,7 @@ class OpkgIndexer(Indexer): bb.note("There are no packages in %s!" % self.deploy_dir) return - oe.utils.multiprocess_exec(index_cmds, create_index) + oe.utils.multiprocess_launch(create_index, index_cmds, self.d) if signer: feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') @@ -258,7 +301,7 @@ class DpkgIndexer(Indexer): bb.note("There are no packages in %s" % self.deploy_dir) return - oe.utils.multiprocess_exec(index_cmds, create_index) + oe.utils.multiprocess_launch(create_index, index_cmds, self.d) if self.d.getVar('PACKAGE_FEED_SIGN') == '1': raise NotImplementedError('Package feed signing not implementd for dpkg') @@ -336,17 +379,24 @@ class PackageManager(object, metaclass=ABCMeta): def _initialize_intercepts(self): bb.note("Initializing intercept dir for %s" % self.target_rootfs) - postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") - if not postinst_intercepts_dir: - postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") # As there might be more than one instance of PackageManager operating at the same time # we need to isolate the intercept_scripts directories from each other, # hence the ugly hash digest in dir name. - self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), - "intercept_scripts-%s" %(hashlib.sha256(self.target_rootfs.encode()).hexdigest()) ) + self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts-%s" % + (hashlib.sha256(self.target_rootfs.encode()).hexdigest())) + postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS") or "").split() + if not postinst_intercepts: + postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_PATH") + if not postinst_intercepts_path: + postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_DIR") or self.d.expand("${COREBASE}/scripts/postinst-intercepts") + postinst_intercepts = oe.path.which_wild('*', postinst_intercepts_path) + + bb.debug(1, 'Collected intercepts:\n%s' % ''.join(' %s\n' % i for i in postinst_intercepts)) bb.utils.remove(self.intercepts_dir, True) - shutil.copytree(postinst_intercepts_dir, self.intercepts_dir) + bb.utils.mkdirhier(self.intercepts_dir) + for intercept in postinst_intercepts: + bb.utils.copyfile(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept))) @abstractmethod def _handle_intercept_failure(self, failed_script): @@ -370,7 +420,7 @@ class PackageManager(object, metaclass=ABCMeta): self._handle_intercept_failure(registered_pkgs) - def run_intercepts(self): + def run_intercepts(self, populate_sdk=None): intercepts_dir = self.intercepts_dir bb.note("Running intercept scripts:") @@ -382,7 +432,8 @@ class PackageManager(object, metaclass=ABCMeta): if script == "postinst_intercept" or not os.access(script_full, os.X_OK): continue - if script == "delay_to_first_boot": + # we do not want to run any multilib variant of this + if script.startswith("delay_to_first_boot"): self._postpone_to_first_boot(script_full) continue @@ -392,9 +443,22 @@ class PackageManager(object, metaclass=ABCMeta): output = subprocess.check_output(script_full, stderr=subprocess.STDOUT) if output: bb.note(output.decode("utf-8")) except subprocess.CalledProcessError as e: - bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8"))) - self._postpone_to_first_boot(script_full) + if populate_sdk == 'host': + bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + elif populate_sdk == 'target': + if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"): + bb.warn("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" + % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + else: + bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + else: + if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"): + bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" + % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + self._postpone_to_first_boot(script_full) + else: + bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) @abstractmethod def update(self): @@ -523,6 +587,13 @@ class PackageManager(object, metaclass=ABCMeta): "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + target_arch = self.d.getVar('TARGET_ARCH') + localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale") + if os.path.exists(localedir) and os.listdir(localedir): + generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir) + # And now delete the binary locales + self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False) + def deploy_dir_lock(self): if self.deploy_dir is None: raise RuntimeError("deploy_dir is not set!") @@ -559,7 +630,7 @@ class PackageManager(object, metaclass=ABCMeta): return res return _append(uris, base_paths) -def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependencies): +def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies): """ Go through our do_package_write_X dependencies and hardlink the packages we depend upon into the repo directory. This prevents us seeing other packages that may @@ -574,15 +645,13 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci seendirs = set() multilibs = {} - rpm_subrepo_dir = oe.path.join(rpm_repo_dir, "rpm") - - bb.utils.remove(rpm_subrepo_dir, recurse=True) - bb.utils.mkdirhier(rpm_subrepo_dir) + bb.utils.remove(subrepo_dir, recurse=True) + bb.utils.mkdirhier(subrepo_dir) # Detect bitbake -b usage nodeps = d.getVar("BB_LIMITEDDEPS") or False if nodeps or not filterbydependencies: - oe.path.symlink(deploydir, rpm_subrepo_dir, True) + oe.path.symlink(deploydir, subrepo_dir, True) return start = None @@ -593,24 +662,24 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci break if start is None: bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") - rpmdeps = set() + pkgdeps = set() start = [start] seen = set(start) - # Support direct dependencies (do_rootfs -> rpms) - # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> rpms) + # Support direct dependencies (do_rootfs -> do_package_write_X) + # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> do_package_write_X) while start: next = [] for dep2 in start: for dep in taskdepdata[dep2][3]: if taskdepdata[dep][0] != pn: if "do_" + taskname in dep: - rpmdeps.add(dep) + pkgdeps.add(dep) elif dep not in seen: next.append(dep) seen.add(dep) start = next - for dep in rpmdeps: + for dep in pkgdeps: c = taskdepdata[dep][0] manifest, d2 = oe.sstatesig.find_sstate_manifest(c, taskdepdata[dep][2], taskname, d, multilibs) if not manifest: @@ -620,8 +689,12 @@ def create_packages_dir(d, rpm_repo_dir, deploydir, taskname, filterbydependenci with open(manifest, "r") as f: for l in f: l = l.strip() - dest = l.replace(deploydir, "") - dest = rpm_subrepo_dir + dest + deploydir = os.path.normpath(deploydir) + if bb.data.inherits_class('packagefeed-stability', d): + dest = l.replace(deploydir + "-prediff", "") + else: + dest = l.replace(deploydir, "") + dest = subrepo_dir + dest if l.endswith("/"): if dest not in seendirs: bb.utils.mkdirhier(dest) @@ -663,12 +736,12 @@ class RpmPM(PackageManager): self.primary_arch = self.d.getVar('MACHINE_ARCH') self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir) - create_packages_dir(self.d, self.rpm_repo_dir, d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies) + create_packages_dir(self.d, oe.path.join(self.rpm_repo_dir, "rpm"), d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies) self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) if not os.path.exists(self.d.expand('${T}/saved_packaging_data')): bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data')) - self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf'] + self.packaging_data_dirs = ['etc/rpm', 'etc/rpmrc', 'etc/dnf', 'var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf'] self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % self.task_name) if not os.path.exists(self.d.expand('${T}/saved')): @@ -697,7 +770,9 @@ class RpmPM(PackageManager): rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") bb.utils.mkdirhier(platformconfdir) open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch) - open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) + with open(rpmrcconfdir + "rpmrc", 'w') as f: + f.write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch)) + f.write("buildarch_compat: %s: noarch\n" % self.primary_arch) open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n") if self.d.getVar('RPM_PREFER_ELF_ARCH'): @@ -789,13 +864,12 @@ class RpmPM(PackageManager): failed_scriptlets_pkgnames[line.split()[-1]] = True if len(failed_scriptlets_pkgnames) > 0: - failed_postinsts_warn(list(failed_scriptlets_pkgnames.keys()), self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) - for pkg in failed_scriptlets_pkgnames.keys(): - self.save_rpmpostinst(pkg) + failed_postinsts_abort(list(failed_scriptlets_pkgnames.keys()), self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) def remove(self, pkgs, with_dependencies = True): - if len(pkgs) == 0: + if not pkgs: return + self._prepare_pkg_transaction() if with_dependencies: @@ -832,7 +906,10 @@ class RpmPM(PackageManager): for i in self.packaging_data_dirs: source_dir = oe.path.join(self.target_rootfs, i) target_dir = oe.path.join(self.saved_packaging_data, i) - shutil.copytree(source_dir, target_dir, symlinks=True) + if os.path.isdir(source_dir): + shutil.copytree(source_dir, target_dir, symlinks=True) + elif os.path.isfile(source_dir): + shutil.copy2(source_dir, target_dir) def recovery_packaging_data(self): # Move the rpmlib back @@ -842,9 +919,10 @@ class RpmPM(PackageManager): if os.path.exists(target_dir): bb.utils.remove(target_dir, True) source_dir = oe.path.join(self.saved_packaging_data, i) - shutil.copytree(source_dir, - target_dir, - symlinks=True) + if os.path.isdir(source_dir): + shutil.copytree(source_dir, target_dir, symlinks=True) + elif os.path.isfile(source_dir): + shutil.copy2(source_dir, target_dir) def list_installed(self): output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{name}-%{version}-%{release}.%{arch}.rpm\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"], @@ -884,7 +962,7 @@ class RpmPM(PackageManager): os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") - standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", + standard_dnf_args = ["-v", "--rpmverbosity=debug", "-y", "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")), "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir), @@ -896,7 +974,7 @@ class RpmPM(PackageManager): try: output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") if print_output: - bb.note(output) + bb.debug(1, output) return output except subprocess.CalledProcessError as e: if print_output: @@ -1060,19 +1138,22 @@ class OpkgDpkgPM(PackageManager): self.mark_packages("unpacked", registered_pkgs.split()) class OpkgPM(OpkgDpkgPM): - def __init__(self, d, target_rootfs, config_file, archs, task_name='target'): + def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True): super(OpkgPM, self).__init__(d, target_rootfs) self.config_file = config_file self.pkg_archs = archs self.task_name = task_name - self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK") + self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) self.opkg_args += self.d.getVar("OPKG_ARGS") + if prepare_index: + create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies) + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') if opkg_lib_dir[0] == "/": opkg_lib_dir = opkg_lib_dir[1:] @@ -1245,7 +1326,11 @@ class OpkgPM(OpkgDpkgPM): if not pkgs: return - cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) + cmd = "%s %s" % (self.opkg_cmd, self.opkg_args) + for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split(): + cmd += " --add-exclude %s" % exclude + cmd += " install " + cmd += " ".join(pkgs) os.environ['D'] = self.target_rootfs os.environ['OFFLINE_ROOT'] = self.target_rootfs @@ -1265,13 +1350,16 @@ class OpkgPM(OpkgDpkgPM): bb.warn(line) failed_pkgs.append(line.split(".")[0]) if failed_pkgs: - failed_postinsts_warn(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) + failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) except subprocess.CalledProcessError as e: (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) def remove(self, pkgs, with_dependencies=True): + if not pkgs: + return + if with_dependencies: cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) @@ -1437,9 +1525,12 @@ class OpkgPM(OpkgDpkgPM): return tmp_dir class DpkgPM(OpkgDpkgPM): - def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None): + def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True): super(DpkgPM, self).__init__(d, target_rootfs) - self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB') + self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), deb_repo_workdir) + + create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_DEB"), "package_write_deb", filterbydependencies) + if apt_conf_dir is None: self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt") else: @@ -1515,7 +1606,6 @@ class DpkgPM(OpkgDpkgPM): os.environ['INTERCEPT_DIR'] = self.intercepts_dir os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') - failed_pkgs = [] for pkg_name in installed_pkgs: for control_script in control_scripts: p_full = os.path.join(info_dir, pkg_name + control_script.suffix) @@ -1530,12 +1620,7 @@ class DpkgPM(OpkgDpkgPM): bb.warn("%s for package %s failed with %d:\n%s" % (control_script.name, pkg_name, e.returncode, e.output.decode("utf-8"))) - failed_postinsts_warn([pkg_name], self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) - failed_pkgs.append(pkg_name) - break - - if len(failed_pkgs): - self.mark_packages("unpacked", failed_pkgs) + failed_postinsts_abort([pkg_name], self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) def update(self): os.environ['APT_CONFIG'] = self.apt_conf_file @@ -1585,6 +1670,9 @@ class DpkgPM(OpkgDpkgPM): def remove(self, pkgs, with_dependencies=True): + if not pkgs: + return + if with_dependencies: os.environ['APT_CONFIG'] = self.apt_conf_file cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs)) diff --git a/poky/meta/lib/oe/patch.py b/poky/meta/lib/oe/patch.py index af7aa52351..e0f0604251 100644 --- a/poky/meta/lib/oe/patch.py +++ b/poky/meta/lib/oe/patch.py @@ -21,6 +21,7 @@ class CmdError(bb.BBHandledException): def runcmd(args, dir = None): import pipes + import subprocess if dir: olddir = os.path.abspath(os.curdir) @@ -33,7 +34,7 @@ def runcmd(args, dir = None): args = [ pipes.quote(str(arg)) for arg in args ] cmd = " ".join(args) # print("cmd: %s" % cmd) - (exitstatus, output) = oe.utils.getstatusoutput(cmd) + (exitstatus, output) = subprocess.getstatusoutput(cmd) if exitstatus != 0: raise CmdError(cmd, exitstatus >> 8, output) if " fuzz " in output: diff --git a/poky/meta/lib/oe/path.py b/poky/meta/lib/oe/path.py index 76c58fa760..1e24d0586b 100644 --- a/poky/meta/lib/oe/path.py +++ b/poky/meta/lib/oe/path.py @@ -86,7 +86,7 @@ def copytree(src, dst): # This way we also preserve hardlinks between files in the tree. bb.utils.mkdirhier(dst) - cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) def copyhardlinktree(src, dst): @@ -98,7 +98,7 @@ def copyhardlinktree(src, dst): if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) source = '' if os.path.isdir(src): @@ -259,3 +259,37 @@ def is_path_parent(possible_parent, *paths): if not path_abs.startswith(possible_parent_abs): return False return True + +def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=False): + """Search a search path for pathname, supporting wildcards. + + Return all paths in the specific search path matching the wildcard pattern + in pathname, returning only the first encountered for each file. If + candidates is True, information on all potential candidate paths are + included. + """ + paths = (path or os.environ.get('PATH', os.defpath)).split(':') + if reverse: + paths.reverse() + + seen, files = set(), [] + for index, element in enumerate(paths): + if not os.path.isabs(element): + element = os.path.abspath(element) + + candidate = os.path.join(element, pathname) + globbed = glob.glob(candidate) + if globbed: + for found_path in sorted(globbed): + if not os.access(found_path, mode): + continue + rel = os.path.relpath(found_path, element) + if rel not in seen: + seen.add(rel) + if candidates: + files.append((found_path, [os.path.join(p, rel) for p in paths[:index+1]])) + else: + files.append(found_path) + + return files + diff --git a/poky/meta/lib/oe/qa.py b/poky/meta/lib/oe/qa.py index 3231e60cea..59c72ce580 100644 --- a/poky/meta/lib/oe/qa.py +++ b/poky/meta/lib/oe/qa.py @@ -158,7 +158,8 @@ def elf_machine_to_string(machine): 0x2A: "SuperH", 0x32: "IA-64", 0x3E: "x86-64", - 0xB7: "AArch64" + 0xB7: "AArch64", + 0xF7: "BPF" }[machine] except: return "Unknown (%s)" % repr(machine) diff --git a/poky/meta/lib/oe/rootfs.py b/poky/meta/lib/oe/rootfs.py index c156607bfa..e5512d09ef 100644 --- a/poky/meta/lib/oe/rootfs.py +++ b/poky/meta/lib/oe/rootfs.py @@ -144,6 +144,20 @@ class Rootfs(object, metaclass=ABCMeta): bb.note(" Install complementary '*-dbg' packages...") self.pm.install_complementary('*-dbg') + if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': + bb.note(" Install complementary '*-src' packages...") + self.pm.install_complementary('*-src') + + """ + Install additional debug packages. Possibility to install additional packages, + which are not automatically installed as complementary package of + standard one, e.g. debug package of static libraries. + """ + extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS') + if extra_debug_pkgs: + bb.note(" Install extra debug packages...") + self.pm.install(extra_debug_pkgs.split(), True) + bb.note(" Rename debug rootfs...") try: shutil.rmtree(self.image_rootfs + '-dbg') @@ -472,7 +486,8 @@ class RpmRootfs(Rootfs): self._log_check_error() def _cleanup(self): - self.pm._invoke_dnf(["clean", "all"]) + if bb.utils.contains("IMAGE_FEATURES", "package-management", True, False, self.d): + self.pm._invoke_dnf(["clean", "all"]) class DpkgOpkgRootfs(Rootfs): @@ -560,6 +575,9 @@ class DpkgOpkgRootfs(Rootfs): return pkg_list def _save_postinsts_common(self, dst_postinst_dir, src_postinst_dir): + if bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d): + return num = 0 for p in self._get_delayed_postinsts(): bb.utils.mkdirhier(dst_postinst_dir) @@ -782,7 +800,7 @@ class OpkgRootfs(DpkgOpkgRootfs): ml_opkg_conf = os.path.join(ml_temp, variant + "-" + os.path.basename(self.opkg_conf)) - ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs) + ml_pm = OpkgPM(self.d, ml_target_rootfs, ml_opkg_conf, self.pkg_archs, prepare_index=False) ml_pm.update() ml_pm.install(pkgs) diff --git a/poky/meta/lib/oe/sdk.py b/poky/meta/lib/oe/sdk.py index d6a503372a..153b07d76b 100644 --- a/poky/meta/lib/oe/sdk.py +++ b/poky/meta/lib/oe/sdk.py @@ -7,52 +7,6 @@ import shutil import glob import traceback -def generate_locale_archive(d, rootfs): - # Pretty sure we don't need this for SDK archive generation but - # keeping it to be safe... - target_arch = d.getVar('SDK_ARCH') - locale_arch_options = { \ - "arm": ["--uint32-align=4", "--little-endian"], - "armeb": ["--uint32-align=4", "--big-endian"], - "aarch64": ["--uint32-align=4", "--little-endian"], - "aarch64_be": ["--uint32-align=4", "--big-endian"], - "sh4": ["--uint32-align=4", "--big-endian"], - "powerpc": ["--uint32-align=4", "--big-endian"], - "powerpc64": ["--uint32-align=4", "--big-endian"], - "mips": ["--uint32-align=4", "--big-endian"], - "mipsisa32r6": ["--uint32-align=4", "--big-endian"], - "mips64": ["--uint32-align=4", "--big-endian"], - "mipsisa64r6": ["--uint32-align=4", "--big-endian"], - "mipsel": ["--uint32-align=4", "--little-endian"], - "mipsisa32r6el": ["--uint32-align=4", "--little-endian"], - "mips64el": ["--uint32-align=4", "--little-endian"], - "mipsisa64r6el": ["--uint32-align=4", "--little-endian"], - "i586": ["--uint32-align=4", "--little-endian"], - "i686": ["--uint32-align=4", "--little-endian"], - "x86_64": ["--uint32-align=4", "--little-endian"] - } - if target_arch in locale_arch_options: - arch_options = locale_arch_options[target_arch] - else: - bb.error("locale_arch_options not found for target_arch=" + target_arch) - bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") - - localedir = oe.path.join(rootfs, d.getVar("libdir_nativesdk"), "locale") - # Need to set this so cross-localedef knows where the archive is - env = dict(os.environ) - env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive") - - for name in os.listdir(localedir): - path = os.path.join(localedir, name) - if os.path.isdir(path): - try: - cmd = ["cross-localedef", "--verbose"] - cmd += arch_options - cmd += ["--add-to-archive", path] - subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT) - except Exception as e: - bb.fatal("Cannot create locale archive: %s" % e.output) - class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): self.d = d @@ -144,7 +98,10 @@ class Sdk(object, metaclass=ABCMeta): for lang in linguas.split(): pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang) # Generate a locale archive of them - generate_locale_archive(self.d, oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)) + target_arch = self.d.getVar('SDK_ARCH') + rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path) + localedir = oe.path.join(rootfs, self.d.getVar("libdir_nativesdk"), "locale") + generate_locale_archive(self.d, rootfs, target_arch, localedir) # And now delete the binary locales pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8") pm.remove(pkgs) @@ -209,7 +166,7 @@ class RpmSdk(Sdk): self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - self.target_pm.run_intercepts() + self.target_pm.run_intercepts(populate_sdk='target') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) @@ -220,7 +177,7 @@ class RpmSdk(Sdk): self._populate_sysroot(self.host_pm, self.host_manifest) self.install_locales(self.host_pm) - self.host_pm.run_intercepts() + self.host_pm.run_intercepts(populate_sdk='host') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) @@ -270,11 +227,17 @@ class OpkgSdk(Sdk): self.host_manifest = OpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_HOST) + ipk_repo_workdir = "oe-sdk-repo" + if "sdk_ext" in d.getVar("BB_RUNTASK"): + ipk_repo_workdir = "oe-sdk-ext-repo" + self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf, - self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")) + self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"), + ipk_repo_workdir=ipk_repo_workdir) self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf, - self.d.getVar("SDK_PACKAGE_ARCHS")) + self.d.getVar("SDK_PACKAGE_ARCHS"), + ipk_repo_workdir=ipk_repo_workdir) def _populate_sysroot(self, pm, manifest): pkgs_to_install = manifest.parse_initial_manifest() @@ -297,7 +260,7 @@ class OpkgSdk(Sdk): self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - self.target_pm.run_intercepts() + self.target_pm.run_intercepts(populate_sdk='target') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) @@ -308,7 +271,7 @@ class OpkgSdk(Sdk): self._populate_sysroot(self.host_pm, self.host_manifest) self.install_locales(self.host_pm) - self.host_pm.run_intercepts() + self.host_pm.run_intercepts(populate_sdk='host') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) @@ -350,15 +313,21 @@ class DpkgSdk(Sdk): self.host_manifest = DpkgManifest(d, self.manifest_dir, Manifest.MANIFEST_TYPE_SDK_HOST) + deb_repo_workdir = "oe-sdk-repo" + if "sdk_ext" in d.getVar("BB_RUNTASK"): + deb_repo_workdir = "oe-sdk-ext-repo" + self.target_pm = DpkgPM(d, self.sdk_target_sysroot, self.d.getVar("PACKAGE_ARCHS"), self.d.getVar("DPKG_ARCH"), - self.target_conf_dir) + self.target_conf_dir, + deb_repo_workdir=deb_repo_workdir) self.host_pm = DpkgPM(d, self.sdk_host_sysroot, self.d.getVar("SDK_PACKAGE_ARCHS"), self.d.getVar("DEB_SDK_ARCH"), - self.host_conf_dir) + self.host_conf_dir, + deb_repo_workdir=deb_repo_workdir) def _copy_apt_dir_to(self, dst_dir): staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE") @@ -386,7 +355,7 @@ class DpkgSdk(Sdk): self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY')) - self.target_pm.run_intercepts() + self.target_pm.run_intercepts(populate_sdk='target') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND")) @@ -399,7 +368,7 @@ class DpkgSdk(Sdk): self._populate_sysroot(self.host_pm, self.host_manifest) self.install_locales(self.host_pm) - self.host_pm.run_intercepts() + self.host_pm.run_intercepts(populate_sdk='host') execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index b82e0f422e..18c5a353a2 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -45,7 +45,7 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes if isPackageGroup(fn) and isAllArch(fn) and not isNative(depname): - return False + return False # Exclude well defined machine specific configurations which don't change ABI if depname in siggen.abisaferecipes and not isImage(fn): @@ -150,16 +150,23 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): if recipename in self.unlockedrecipes: unlocked = True else: + def get_mc(tid): + tid = tid.rsplit('.', 1)[0] + if tid.startswith('multiconfig:'): + elems = tid.split(':') + return elems[1] def recipename_from_dep(dep): # The dep entry will look something like # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, # ... + fn = dep.rsplit('.', 1)[0] return dataCache.pkg_fn[fn] + mc = get_mc(fn) # If any unlocked recipe is in the direct dependencies then the # current recipe should be unlocked as well. - depnames = [ recipename_from_dep(x) for x in deps ] + depnames = [ recipename_from_dep(x) for x in deps if mc == get_mc(x)] if any(x in y for y in depnames for x in self.unlockedrecipes): self.unlockedrecipes[recipename] = '' unlocked = True @@ -372,8 +379,14 @@ def sstate_get_manifest_filename(task, d): def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): d2 = d variant = '' + curr_variant = '' + if d.getVar("BBEXTENDCURR") == "multilib": + curr_variant = d.getVar("BBEXTENDVARIANT") + if "virtclass-multilib" not in d.getVar("OVERRIDES"): + curr_variant = "invalid" if taskdata2.startswith("virtual:multilib"): variant = taskdata2.split(":")[2] + if curr_variant != variant: if variant not in multilibcache: multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d) d2 = multilibcache[variant] diff --git a/poky/meta/lib/oe/terminal.py b/poky/meta/lib/oe/terminal.py index 94afe394ed..caeb5e31ec 100644 --- a/poky/meta/lib/oe/terminal.py +++ b/poky/meta/lib/oe/terminal.py @@ -112,7 +112,7 @@ class Screen(Terminal): bb.event.fire(bb.event.LogExecTTY(msg, "screen -r %s" % s_id, 0.5, 10), d) else: - logger.warn(msg) + logger.warning(msg) class TmuxRunning(Terminal): """Open a new pane in the current running tmux window""" @@ -168,7 +168,7 @@ class Tmux(Terminal): if d: bb.event.fire(bb.event.LogExecTTY(msg, attach_cmd, 0.5, 10), d) else: - logger.warn(msg) + logger.warning(msg) class Custom(Terminal): command = 'false' # This is a placeholder @@ -180,7 +180,7 @@ class Custom(Terminal): if not '{command}' in self.command: self.command += ' {command}' Terminal.__init__(self, sh_cmd, title, env, d) - logger.warn('Custom terminal was started.') + logger.warning('Custom terminal was started.') else: logger.debug(1, 'No custom terminal (OE_TERMINAL_CUSTOMCMD) set') raise UnsupportedTerminal('OE_TERMINAL_CUSTOMCMD not set') diff --git a/poky/meta/lib/oe/types.py b/poky/meta/lib/oe/types.py index 4ae58acfac..f4017130df 100644 --- a/poky/meta/lib/oe/types.py +++ b/poky/meta/lib/oe/types.py @@ -103,8 +103,13 @@ def boolean(value): """OpenEmbedded 'boolean' type Valid values for true: 'yes', 'y', 'true', 't', '1' - Valid values for false: 'no', 'n', 'false', 'f', '0' + Valid values for false: 'no', 'n', 'false', 'f', '0', None """ + if value is None: + return False + + if isinstance(value, bool): + return value if not isinstance(value, str): raise TypeError("boolean accepts a string, not '%s'" % type(value)) diff --git a/poky/meta/lib/oe/utils.py b/poky/meta/lib/oe/utils.py index 80f0442d0a..8a584d6ddd 100644 --- a/poky/meta/lib/oe/utils.py +++ b/poky/meta/lib/oe/utils.py @@ -1,4 +1,6 @@ import subprocess +import multiprocessing +import traceback def read_file(filename): try: @@ -23,6 +25,13 @@ def conditional(variable, checkvalue, truevalue, falsevalue, d): else: return falsevalue +def vartrue(var, iftrue, iffalse, d): + import oe.types + if oe.types.boolean(d.getVar(var)): + return iftrue + else: + return iffalse + def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): if float(d.getVar(variable)) <= float(checkvalue): return truevalue @@ -246,38 +255,73 @@ def execute_pre_post_process(d, cmds): bb.note("Executing %s ..." % cmd) bb.build.exec_func(cmd, d) -def multiprocess_exec(commands, function): - import signal - import multiprocessing - - if not commands: - return [] +# For each item in items, call the function 'target' with item as the first +# argument, extraargs as the other arguments and handle any exceptions in the +# parent thread +def multiprocess_launch(target, items, d, extraargs=None): - def init_worker(): - signal.signal(signal.SIGINT, signal.SIG_IGN) - - fails = [] - - def failures(res): - fails.append(res) - - nproc = min(multiprocessing.cpu_count(), len(commands)) - pool = bb.utils.multiprocessingpool(nproc, init_worker) - - try: - mapresult = pool.map_async(function, commands, error_callback=failures) - - pool.close() - pool.join() - results = mapresult.get() - except KeyboardInterrupt: - pool.terminate() - pool.join() - raise - - if fails: - raise fails[0] + class ProcessLaunch(multiprocessing.Process): + def __init__(self, *args, **kwargs): + multiprocessing.Process.__init__(self, *args, **kwargs) + self._pconn, self._cconn = multiprocessing.Pipe() + self._exception = None + self._result = None + def run(self): + try: + ret = self._target(*self._args, **self._kwargs) + self._cconn.send((None, ret)) + except Exception as e: + tb = traceback.format_exc() + self._cconn.send((e, tb)) + + def update(self): + if self._pconn.poll(): + (e, tb) = self._pconn.recv() + if e is not None: + self._exception = (e, tb) + else: + self._result = tb + + @property + def exception(self): + self.update() + return self._exception + + @property + def result(self): + self.update() + return self._result + + max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) + launched = [] + errors = [] + results = [] + items = list(items) + while (items and not errors) or launched: + if not errors and items and len(launched) < max_process: + args = (items.pop(),) + if extraargs is not None: + args = args + extraargs + p = ProcessLaunch(target=target, args=args) + p.start() + launched.append(p) + for q in launched: + # The finished processes are joined when calling is_alive() + if not q.is_alive(): + if q.exception: + errors.append(q.exception) + if q.result: + results.append(q.result) + launched.remove(q) + # Paranoia doesn't hurt + for p in launched: + p.join() + if errors: + msg = "" + for (e, tb) in errors: + msg = msg + str(e) + ": " + str(tb) + "\n" + bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg) return results def squashspaces(string): @@ -304,11 +348,20 @@ def format_pkg_list(pkg_dict, ret_format=None): for pkg in sorted(pkg_dict): output.append(pkg) - return '\n'.join(output) + output_str = '\n'.join(output) + + if output_str: + # make sure last line is newline terminated + output_str += '\n' -def host_gcc_version(d): + return output_str + +def host_gcc_version(d, taskcontextonly=False): import re, subprocess + if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1': + return + compiler = d.getVar("BUILD_CC") try: env = os.environ.copy() @@ -327,9 +380,18 @@ def host_gcc_version(d): def get_multilib_datastore(variant, d): localdata = bb.data.createCopy(d) - overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant - localdata.setVar("OVERRIDES", overrides) - localdata.setVar("MLPREFIX", variant + "-") + if variant: + overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant + localdata.setVar("OVERRIDES", overrides) + localdata.setVar("MLPREFIX", variant + "-") + else: + origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL") + if origdefault: + localdata.setVar("DEFAULTTUNE", origdefault) + overrides = localdata.getVar("OVERRIDES", False).split(":") + overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")]) + localdata.setVar("OVERRIDES", overrides) + localdata.setVar("MLPREFIX", "") return localdata # @@ -419,3 +481,4 @@ class ImageQAFailed(bb.build.FuncFailed): msg = msg + ' (%s)' % self.description return msg + -- cgit v1.2.3