From 193236933b0f4ab91b1625b64e2187e2db4e0e8f Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Fri, 5 Apr 2019 15:28:33 -0400 Subject: reset upstream subtrees to HEAD Reset the following subtrees on HEAD: poky: 8217b477a1(master) meta-xilinx: 64aa3d35ae(master) meta-openembedded: 0435c9e193(master) meta-raspberrypi: 490a4441ac(master) meta-security: cb6d1c85ee(master) Squashed patches: meta-phosphor: drop systemd 239 patches meta-phosphor: mrw-api: use correct install path Change-Id: I268e2646d9174ad305630c6bbd3fbc1a6105f43d Signed-off-by: Brad Bishop --- poky/meta/lib/oe/classextend.py | 2 +- poky/meta/lib/oe/elf.py | 1 + poky/meta/lib/oe/license.py | 8 +- poky/meta/lib/oe/package.py | 2 +- poky/meta/lib/oe/package_manager.py | 100 +++++------- poky/meta/lib/oe/patch.py | 8 +- poky/meta/lib/oe/recipeutils.py | 209 +++++++++++++++++++++---- poky/meta/lib/oe/rootfs.py | 51 +++--- poky/meta/lib/oe/sdk.py | 4 +- poky/meta/lib/oe/sstatesig.py | 302 +++++++++++++++++++++++++++++++++++- poky/meta/lib/oe/terminal.py | 12 +- poky/meta/lib/oe/utils.py | 19 ++- 12 files changed, 578 insertions(+), 140 deletions(-) (limited to 'poky/meta/lib/oe') diff --git a/poky/meta/lib/oe/classextend.py b/poky/meta/lib/oe/classextend.py index d2eeaf0e5..662707b85 100644 --- a/poky/meta/lib/oe/classextend.py +++ b/poky/meta/lib/oe/classextend.py @@ -114,7 +114,7 @@ class NativesdkClassExtender(ClassExtender): def map_depends(self, dep): if dep.startswith(self.extname): return dep - if dep.endswith(("-gcc-initial", "-gcc", "-g++")): + if dep.endswith(("-gcc", "-g++")): return dep + "-crosssdk" elif dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): return dep diff --git a/poky/meta/lib/oe/elf.py b/poky/meta/lib/oe/elf.py index 0ed59ae05..4cc9a9a09 100644 --- a/poky/meta/lib/oe/elf.py +++ b/poky/meta/lib/oe/elf.py @@ -63,6 +63,7 @@ def machine_dict(d): "arm" : ( 40, 97, 0, True, 32), "armeb": ( 40, 97, 0, False, 32), "powerpc": ( 20, 0, 0, False, 32), + "powerpc64": ( 21, 0, 0, False, 64), "i386": ( 3, 0, 0, True, 32), "i486": ( 3, 0, 0, True, 32), "i586": ( 3, 0, 0, True, 32), diff --git a/poky/meta/lib/oe/license.py b/poky/meta/lib/oe/license.py index ca385d518..04f5b316a 100644 --- a/poky/meta/lib/oe/license.py +++ b/poky/meta/lib/oe/license.py @@ -13,8 +13,8 @@ def license_ok(license, dont_want_licenses): # will exclude a trailing '+' character from LICENSE in # case INCOMPATIBLE_LICENSE is not a 'X+' license. lic = license - if not re.search('\+$', dwl): - lic = re.sub('\+', '', license) + if not re.search(r'\+$', dwl): + lic = re.sub(r'\+', '', license) if fnmatch(lic, dwl): return False return True @@ -40,8 +40,8 @@ class InvalidLicense(LicenseError): return "invalid characters in license '%s'" % self.license license_operator_chars = '&|() ' -license_operator = re.compile('([' + license_operator_chars + '])') -license_pattern = re.compile('[a-zA-Z0-9.+_\-]+$') +license_operator = re.compile(r'([' + license_operator_chars + '])') +license_pattern = re.compile(r'[a-zA-Z0-9.+_\-]+$') class LicenseVisitor(ast.NodeVisitor): """Get elements based on OpenEmbedded license strings""" diff --git a/poky/meta/lib/oe/package.py b/poky/meta/lib/oe/package.py index efd36b375..6e83f01f1 100644 --- a/poky/meta/lib/oe/package.py +++ b/poky/meta/lib/oe/package.py @@ -255,7 +255,7 @@ def read_shlib_providers(d): shlib_provider = {} shlibs_dirs = d.getVar('SHLIBSDIRS').split() - list_re = re.compile('^(.*)\.list$') + list_re = re.compile(r'^(.*)\.list$') # Go from least to most specific since the last one found wins for dir in reversed(shlibs_dirs): bb.debug(2, "Reading shlib providers in %s" % (dir)) diff --git a/poky/meta/lib/oe/package_manager.py b/poky/meta/lib/oe/package_manager.py index 882e7c429..2835c1ddf 100644 --- a/poky/meta/lib/oe/package_manager.py +++ b/poky/meta/lib/oe/package_manager.py @@ -29,7 +29,7 @@ def opkg_query(cmd_output): a dictionary with the information of the packages. This is used when the packages are in deb or ipk format. """ - verregex = re.compile(' \([=<>]* [^ )]*\)') + verregex = re.compile(r' \([=<>]* [^ )]*\)') output = dict() pkg = "" arch = "" @@ -94,6 +94,8 @@ def generate_locale_archive(d, rootfs, target_arch, localedir): # Pretty sure we don't need this for locale archive generation but # keeping it to be safe... locale_arch_options = { \ + "arc": ["--uint32-align=4", "--little-endian"], + "arceb": ["--uint32-align=4", "--big-endian"], "arm": ["--uint32-align=4", "--little-endian"], "armeb": ["--uint32-align=4", "--big-endian"], "aarch64": ["--uint32-align=4", "--little-endian"], @@ -250,8 +252,8 @@ class DpkgIndexer(Indexer): with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"), "apt", "apt.conf.sample")) as apt_conf_sample: for line in apt_conf_sample.read().split("\n"): - line = re.sub("#ROOTFS#", "/dev/null", line) - line = re.sub("#APTCONF#", self.apt_conf_dir, line) + line = re.sub(r"#ROOTFS#", "/dev/null", line) + line = re.sub(r"#APTCONF#", self.apt_conf_dir, line) apt_conf.write(line + "\n") def write_index(self): @@ -318,7 +320,7 @@ class PkgsList(object, metaclass=ABCMeta): class RpmPkgsList(PkgsList): def list_pkgs(self): - return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR')).list_installed() + return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR'), needfeed=False).list_installed() class OpkgPkgsList(PkgsList): def __init__(self, d, rootfs_dir, config_file): @@ -406,7 +408,7 @@ class PackageManager(object, metaclass=ABCMeta): with open(postinst_intercept_hook) as intercept: registered_pkgs = None for line in intercept.read().split("\n"): - m = re.match("^##PKGS:(.*)", line) + m = re.match(r"^##PKGS:(.*)", line) if m is not None: registered_pkgs = m.group(1).strip() break @@ -437,6 +439,11 @@ class PackageManager(object, metaclass=ABCMeta): self._postpone_to_first_boot(script_full) continue + if populate_sdk == 'host' and self.d.getVar('SDK_OS') == 'mingw32': + bb.note("The postinstall intercept hook '%s' could not be executed due to missing wine support, details in %s/log.do_%s" + % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + continue + bb.note("> Executing %s intercept ..." % script) try: @@ -445,10 +452,10 @@ class PackageManager(object, metaclass=ABCMeta): except subprocess.CalledProcessError as e: bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8"))) if populate_sdk == 'host': - bb.warn("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) + bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) elif populate_sdk == 'target': if "qemuwrapper: qemu usermode is not supported" in e.output.decode("utf-8"): - bb.warn("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" + bb.note("The postinstall intercept hook '%s' could not be executed due to missing qemu usermode support, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) else: bb.fatal("The postinstall intercept hook '%s' failed, details in %s/log.do_%s" % (script, self.d.getVar('T'), self.d.getVar('BB_CURRENTTASK'))) @@ -722,7 +729,8 @@ class RpmPM(PackageManager): arch_var=None, os_var=None, rpm_repo_workdir="oe-rootfs-repo", - filterbydependencies=True): + filterbydependencies=True, + needfeed=True): super(RpmPM, self).__init__(d, target_rootfs) self.target_vendor = target_vendor self.task_name = task_name @@ -735,8 +743,9 @@ class RpmPM(PackageManager): else: self.primary_arch = self.d.getVar('MACHINE_ARCH') - self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir) - create_packages_dir(self.d, oe.path.join(self.rpm_repo_dir, "rpm"), d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies) + if needfeed: + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir) + create_packages_dir(self.d, oe.path.join(self.rpm_repo_dir, "rpm"), d.getVar("DEPLOY_DIR_RPM"), "package_write_rpm", filterbydependencies) self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) if not os.path.exists(self.d.expand('${T}/saved_packaging_data')): @@ -860,7 +869,7 @@ class RpmPM(PackageManager): failed_scriptlets_pkgnames = collections.OrderedDict() for line in output.splitlines(): - if line.startswith("Non-fatal POSTIN scriptlet failure in rpm package"): + if line.startswith("Error in POSTIN scriptlet in rpm package"): failed_scriptlets_pkgnames[line.split()[-1]] = True if len(failed_scriptlets_pkgnames) > 0: @@ -962,13 +971,14 @@ class RpmPM(PackageManager): os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") - standard_dnf_args = ["-v", "--rpmverbosity=debug", "-y", + standard_dnf_args = ["-v", "--rpmverbosity=info", "-y", "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")), - "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir), "--installroot=%s" % (self.target_rootfs), "--setopt=logdir=%s" % (self.d.getVar('T')) ] + if hasattr(self, "rpm_repo_dir"): + standard_dnf_args.append("--repofrompath=oe-repo,%s" % (self.rpm_repo_dir)) cmd = [dnf_cmd] + standard_dnf_args + dnf_args bb.note('Running %s' % ' '.join(cmd)) try: @@ -1108,10 +1118,7 @@ class OpkgDpkgPM(PackageManager): tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) - if self.d.getVar('IMAGE_PKGTYPE') == 'deb': - data_tar = 'data.tar.xz' - else: - data_tar = 'data.tar.gz' + data_tar = 'data.tar.xz' try: cmd = [ar_cmd, 'x', pkg_path] @@ -1212,7 +1219,7 @@ class OpkgPM(OpkgDpkgPM): priority += 5 for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): - feed_match = re.match("^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) + feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) if feed_match is not None: feed_name = feed_match.group(1) @@ -1329,6 +1336,8 @@ class OpkgPM(OpkgDpkgPM): cmd = "%s %s" % (self.opkg_cmd, self.opkg_args) for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split(): cmd += " --add-exclude %s" % exclude + for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split(): + cmd += " --add-ignore-recommends %s" % bad_recommendation cmd += " install " cmd += " ".join(pkgs) @@ -1397,45 +1406,6 @@ class OpkgPM(OpkgDpkgPM): def list_installed(self): return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs() - def handle_bad_recommendations(self): - bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS") or "" - if bad_recommendations.strip() == "": - return - - status_file = os.path.join(self.opkg_dir, "status") - - # If status file existed, it means the bad recommendations has already - # been handled - if os.path.exists(status_file): - return - - cmd = "%s %s info " % (self.opkg_cmd, self.opkg_args) - - with open(status_file, "w+") as status: - for pkg in bad_recommendations.split(): - pkg_info = cmd + pkg - - try: - output = subprocess.check_output(pkg_info.split(), stderr=subprocess.STDOUT).strip().decode("utf-8") - except subprocess.CalledProcessError as e: - bb.fatal("Cannot get package info. Command '%s' " - "returned %d:\n%s" % (pkg_info, e.returncode, e.output.decode("utf-8"))) - - if output == "": - bb.note("Ignored bad recommendation: '%s' is " - "not a package" % pkg) - continue - - for line in output.split('\n'): - if line.startswith("Status:"): - status.write("Status: deinstall hold not-installed\n") - else: - status.write(line + "\n") - - # Append a blank line after each package entry to ensure that it - # is separated from the following entry - status.write("\n") - def dummy_install(self, pkgs): """ The following function dummy installs pkgs and returns the log of output. @@ -1520,7 +1490,7 @@ class OpkgPM(OpkgDpkgPM): "trying to extract the package." % pkg) tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) - bb.utils.remove(os.path.join(tmp_dir, "data.tar.gz")) + bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) return tmp_dir @@ -1592,7 +1562,7 @@ class DpkgPM(OpkgDpkgPM): with open(status_file, "r") as status: for line in status.read().split('\n'): - m = re.match("^Package: (.*)", line) + m = re.match(r"^Package: (.*)", line) if m is not None: installed_pkgs.append(m.group(1)) @@ -1657,13 +1627,13 @@ class DpkgPM(OpkgDpkgPM): # rename *.dpkg-new files/dirs for root, dirs, files in os.walk(self.target_rootfs): for dir in dirs: - new_dir = re.sub("\.dpkg-new", "", dir) + new_dir = re.sub(r"\.dpkg-new", "", dir) if dir != new_dir: os.rename(os.path.join(root, dir), os.path.join(root, new_dir)) for file in files: - new_file = re.sub("\.dpkg-new", "", file) + new_file = re.sub(r"\.dpkg-new", "", file) if file != new_file: os.rename(os.path.join(root, file), os.path.join(root, new_file)) @@ -1728,7 +1698,7 @@ class DpkgPM(OpkgDpkgPM): sources_file.write("deb %s ./\n" % uri) def _create_configs(self, archs, base_archs): - base_archs = re.sub("_", "-", base_archs) + base_archs = re.sub(r"_", r"-", base_archs) if os.path.exists(self.apt_conf_dir): bb.utils.remove(self.apt_conf_dir, True) @@ -1782,7 +1752,7 @@ class DpkgPM(OpkgDpkgPM): with open(self.apt_conf_file, "w+") as apt_conf: with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample: for line in apt_conf_sample.read().split("\n"): - match_arch = re.match(" Architecture \".*\";$", line) + match_arch = re.match(r" Architecture \".*\";$", line) architectures = "" if match_arch: for base_arch in base_arch_list: @@ -1790,8 +1760,8 @@ class DpkgPM(OpkgDpkgPM): apt_conf.write(" Architectures {%s};\n" % architectures); apt_conf.write(" Architecture \"%s\";\n" % base_archs) else: - line = re.sub("#ROOTFS#", self.target_rootfs, line) - line = re.sub("#APTCONF#", self.apt_conf_dir, line) + line = re.sub(r"#ROOTFS#", self.target_rootfs, line) + line = re.sub(r"#APTCONF#", self.apt_conf_dir, line) apt_conf.write(line + "\n") target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs diff --git a/poky/meta/lib/oe/patch.py b/poky/meta/lib/oe/patch.py index e0f060425..7dd31d9d4 100644 --- a/poky/meta/lib/oe/patch.py +++ b/poky/meta/lib/oe/patch.py @@ -334,8 +334,8 @@ class GitApplyTree(PatchTree): @staticmethod def interpretPatchHeader(headerlines): import re - author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') - from_commit_re = re.compile('^From [a-z0-9]{40} .*') + author_re = re.compile(r'[\S ]+ <\S+@\S+\.\S+>') + from_commit_re = re.compile(r'^From [a-z0-9]{40} .*') outlines = [] author = None date = None @@ -790,9 +790,11 @@ class UserResolver(Resolver): def patch_path(url, fetch, workdir, expand=True): - """Return the local path of a patch, or None if this isn't a patch""" + """Return the local path of a patch, or return nothing if this isn't a patch""" local = fetch.localpath(url) + if os.path.isdir(local): + return base, ext = os.path.splitext(os.path.basename(local)) if ext in ('.gz', '.bz2', '.xz', '.Z'): if expand: diff --git a/poky/meta/lib/oe/recipeutils.py b/poky/meta/lib/oe/recipeutils.py index aa64553c0..4ca200d83 100644 --- a/poky/meta/lib/oe/recipeutils.py +++ b/poky/meta/lib/oe/recipeutils.py @@ -16,40 +16,40 @@ import shutil import re import fnmatch import glob -from collections import OrderedDict, defaultdict +import bb.tinfoil +from collections import OrderedDict, defaultdict +from bb.utils import vercmp_string # Help us to find places to insert values recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] # Variables that sometimes are a bit long but shouldn't be wrapped -nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI\[(.+\.)?md5sum\]', 'SRC_URI\[(.+\.)?sha256sum\]'] +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] -def pn_to_recipe(cooker, pn, mc=''): - """Convert a recipe name (PN) to the path to the recipe file""" - - best = cooker.findBestProvider(pn, mc) - return best[3] - - -def get_unavailable_reasons(cooker, pn): - """If a recipe could not be found, find out why if possible""" - import bb.taskdata - taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist) - return taskdata.get_reasons(pn) - - -def parse_recipe(cooker, fn, appendfiles): +def simplify_history(history, d): """ - Parse an individual recipe file, optionally with a list of - bbappend files. + Eliminate any irrelevant events from a variable history """ - import bb.cache - parser = bb.cache.NoCache(cooker.databuilder) - envdata = parser.loadDataFull(fn, appendfiles) - return envdata + ret_history = [] + has_set = False + # Go backwards through the history and remove any immediate operations + # before the most recent set + for event in reversed(history): + if 'flag' in event or not 'file' in event: + continue + if event['op'] == 'set': + if has_set: + continue + has_set = True + elif event['op'] in ('append', 'prepend', 'postdot', 'predot'): + # Reminder: "append" and "prepend" mean += and =+ respectively, NOT _append / _prepend + if has_set: + continue + ret_history.insert(0, event) + return ret_history def get_var_files(fn, varlist, d): @@ -58,11 +58,19 @@ def get_var_files(fn, varlist, d): """ varfiles = {} for v in varlist: - history = d.varhistory.variable(v) files = [] - for event in history: - if 'file' in event and not 'flag' in event: - files.append(event['file']) + if '[' in v: + varsplit = v.split('[') + varflag = varsplit[1].split(']')[0] + history = d.varhistory.variable(varsplit[0]) + for event in history: + if 'file' in event and event.get('flag', '') == varflag: + files.append(event['file']) + else: + history = d.varhistory.variable(v) + for event in history: + if 'file' in event and not 'flag' in event: + files.append(event['file']) if files: actualfile = files[-1] else: @@ -153,7 +161,7 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True): key = item[:-2] else: key = item - restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key + restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key if item.endswith('()'): recipe_progression_restrs.append(restr + '()') else: @@ -176,7 +184,14 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True): def outputvalue(name, lines, rewindcomments=False): if values[name] is None: return - rawtext = '%s = "%s"%s' % (name, values[name], newline) + if isinstance(values[name], tuple): + op, value = values[name] + if op == '+=' and value.strip() == '': + return + else: + value = values[name] + op = '=' + rawtext = '%s %s "%s"%s' % (name, op, value, newline) addlines = [] nowrap = False for nowrap_re in nowrap_vars_res: @@ -186,10 +201,10 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True): if nowrap: addlines.append(rawtext) elif name in list_vars: - splitvalue = split_var_value(values[name], assignment=False) + splitvalue = split_var_value(value, assignment=False) if len(splitvalue) > 1: linesplit = ' \\\n' + (' ' * (len(name) + 4)) - addlines.append('%s = "%s%s"%s' % (name, linesplit.join(splitvalue), linesplit, newline)) + addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline)) else: addlines.append(rawtext) else: @@ -321,12 +336,47 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None """Modify a list of variable values in the specified recipe. Handles inc files if used by the recipe. """ + overrides = d.getVar('OVERRIDES').split(':') + def override_applicable(hevent): + op = hevent['op'] + if '[' in op: + opoverrides = op.split('[')[1].split(']')[0].split('_') + for opoverride in opoverrides: + if not opoverride in overrides: + return False + return True + varlist = varvalues.keys() + fn = os.path.abspath(fn) varfiles = get_var_files(fn, varlist, d) locs = localise_file_vars(fn, varfiles, varlist) patches = [] for f,v in locs.items(): vals = {k: varvalues[k] for k in v} + f = os.path.abspath(f) + if f == fn: + extravals = {} + for var, value in vals.items(): + if var in list_vars: + history = simplify_history(d.varhistory.variable(var), d) + recipe_set = False + for event in history: + if os.path.abspath(event['file']) == fn: + if event['op'] == 'set': + recipe_set = True + if not recipe_set: + for event in history: + if event['op'].startswith('_remove'): + continue + if not override_applicable(event): + continue + newvalue = value.replace(event['detail'], '') + if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith('_'): + op = event['op'].replace('[', '_').replace(']', '') + extravals[var + op] = None + value = newvalue + vals[var] = ('+=', value) + vals.update(extravals) patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output) if patch: patches.append(patchdata) @@ -432,7 +482,14 @@ def get_recipe_local_files(d, patches=False, archives=False): unpack = fetch.ud[uri].parm.get('unpack', True) if unpack: continue - ret[fname] = localpath + if os.path.isdir(localpath): + for root, dirs, files in os.walk(localpath): + for fname in files: + fileabspath = os.path.join(root,fname) + srcdir = os.path.dirname(localpath) + ret[os.path.relpath(fileabspath,srcdir)] = fileabspath + else: + ret[fname] = localpath return ret @@ -875,7 +932,7 @@ def get_recipe_pv_without_srcpv(pv, uri_type): sfx = '' if uri_type == 'git': - git_regex = re.compile("(?Pv?)(?P[^\+]*)((?P\+(git)?r?(AUTOINC\+))(?P.*))?") + git_regex = re.compile(r"(?Pv?)(?P[^\+]*)((?P\+(git)?r?(AUTOINC\+))(?P.*))?") m = git_regex.match(pv) if m: @@ -883,7 +940,7 @@ def get_recipe_pv_without_srcpv(pv, uri_type): pfx = m.group('pfx') sfx = m.group('sfx') else: - regex = re.compile("(?P(v|r)?)(?P.*)") + regex = re.compile(r"(?P(v|r)?)(?P.*)") m = regex.match(pv) if m: pv = m.group('ver') @@ -969,3 +1026,87 @@ def get_recipe_upstream_version(rd): ru['datetime'] = datetime.now() return ru + +def _get_recipe_upgrade_status(data): + uv = get_recipe_upstream_version(data) + + pn = data.getVar('PN') + cur_ver = uv['current_version'] + + upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') + if not uv['version']: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + else: + cmp = vercmp_string(uv['current_version'], uv['version']) + if cmp == -1: + status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" + elif cmp == 0: + status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" + else: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + + next_ver = uv['version'] if uv['version'] else "N/A" + revision = uv['revision'] if uv['revision'] else "N/A" + maintainer = data.getVar('RECIPE_MAINTAINER') + no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') + + return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) + +def get_recipe_upgrade_status(recipes=None): + pkgs_list = [] + data_copy_list = [] + copy_vars = ('SRC_URI', + 'PV', + 'GITDIR', + 'DL_DIR', + 'PN', + 'CACHE', + 'PERSISTENT_DIR', + 'BB_URI_HEADREVS', + 'UPSTREAM_CHECK_COMMITS', + 'UPSTREAM_CHECK_GITTAGREGEX', + 'UPSTREAM_CHECK_REGEX', + 'UPSTREAM_CHECK_URI', + 'UPSTREAM_VERSION_UNKNOWN', + 'RECIPE_MAINTAINER', + 'RECIPE_NO_UPDATE_REASON', + 'RECIPE_UPSTREAM_VERSION', + 'RECIPE_UPSTREAM_DATE', + 'CHECK_DATE', + ) + + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=False) + + if not recipes: + recipes = tinfoil.all_recipe_files(variants=False) + + for fn in recipes: + try: + if fn.startswith("/"): + data = tinfoil.parse_recipe_file(fn) + else: + data = tinfoil.parse_recipe(fn) + except bb.providers.NoProvider: + bb.note(" No provider for %s" % fn) + continue + + unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE') + if unreliable == "1": + bb.note(" Skip package %s as upstream check unreliable" % pn) + continue + + data_copy = bb.data.init() + for var in copy_vars: + data_copy.setVar(var, data.getVar(var)) + for k in data: + if k.startswith('SRCREV'): + data_copy.setVar(k, data.getVar(k)) + + data_copy_list.append(data_copy) + + from concurrent.futures import ProcessPoolExecutor + with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: + pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) + + return pkgs_list diff --git a/poky/meta/lib/oe/rootfs.py b/poky/meta/lib/oe/rootfs.py index e5512d09e..b7c0b9c24 100644 --- a/poky/meta/lib/oe/rootfs.py +++ b/poky/meta/lib/oe/rootfs.py @@ -354,9 +354,9 @@ class Rootfs(object, metaclass=ABCMeta): class RpmRootfs(Rootfs): def __init__(self, d, manifest_dir, progress_reporter=None, logcatcher=None): super(RpmRootfs, self).__init__(d, progress_reporter, logcatcher) - self.log_check_regex = '(unpacking of archive failed|Cannot find package'\ - '|exit 1|ERROR: |Error: |Error |ERROR '\ - '|Failed |Failed: |Failed$|Failed\(\d+\):)' + self.log_check_regex = r'(unpacking of archive failed|Cannot find package'\ + r'|exit 1|ERROR: |Error: |Error |ERROR '\ + r'|Failed |Failed: |Failed$|Failed\(\d+\):)' self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d, @@ -499,7 +499,7 @@ class DpkgOpkgRootfs(Rootfs): pkg_depends_list = [] # filter version requirements like libc (>= 1.1) for dep in pkg_depends.split(', '): - m_dep = re.match("^(.*) \(.*\)$", dep) + m_dep = re.match(r"^(.*) \(.*\)$", dep) if m_dep: dep = m_dep.group(1) pkg_depends_list.append(dep) @@ -515,21 +515,33 @@ class DpkgOpkgRootfs(Rootfs): data = status.read() status.close() for line in data.split('\n'): - m_pkg = re.match("^Package: (.*)", line) - m_status = re.match("^Status:.*unpacked", line) - m_depends = re.match("^Depends: (.*)", line) + m_pkg = re.match(r"^Package: (.*)", line) + m_status = re.match(r"^Status:.*unpacked", line) + m_depends = re.match(r"^Depends: (.*)", line) + #Only one of m_pkg, m_status or m_depends is not None at time + #If m_pkg is not None, we started a new package if m_pkg is not None: - if pkg_name and pkg_status_match: - pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) - + #Get Package name pkg_name = m_pkg.group(1) + #Make sure we reset other variables pkg_status_match = False pkg_depends = "" elif m_status is not None: + #New status matched pkg_status_match = True elif m_depends is not None: + #New depends macthed pkg_depends = m_depends.group(1) + else: + pass + + #Now check if we can process package depends and postinst + if "" != pkg_name and pkg_status_match: + pkgs[pkg_name] = _get_pkg_depends_list(pkg_depends) + else: + #Not enough information + pass # remove package dependencies not in postinsts pkg_names = list(pkgs.keys()) @@ -735,15 +747,16 @@ class OpkgRootfs(DpkgOpkgRootfs): if filecmp.cmp(f1, f2): return True - if self.image_rootfs not in f1: - self._prelink_file(f1.replace(key, ''), f1) + if bb.data.inherits_class('image-prelink', self.d): + if self.image_rootfs not in f1: + self._prelink_file(f1.replace(key, ''), f1) - if self.image_rootfs not in f2: - self._prelink_file(f2.replace(key, ''), f2) + if self.image_rootfs not in f2: + self._prelink_file(f2.replace(key, ''), f2) - # Both of them are prelinked - if filecmp.cmp(f1, f2): - return True + # Both of them are prelinked + if filecmp.cmp(f1, f2): + return True # Not equal return False @@ -759,7 +772,7 @@ class OpkgRootfs(DpkgOpkgRootfs): if allow_replace is None: allow_replace = "" - allow_rep = re.compile(re.sub("\|$", "", allow_replace)) + allow_rep = re.compile(re.sub(r"\|$", r"", allow_replace)) error_prompt = "Multilib check error:" files = {} @@ -879,8 +892,6 @@ class OpkgRootfs(DpkgOpkgRootfs): self.pm.update() - self.pm.handle_bad_recommendations() - if self.progress_reporter: self.progress_reporter.next_stage() diff --git a/poky/meta/lib/oe/sdk.py b/poky/meta/lib/oe/sdk.py index 153b07d76..878ee1647 100644 --- a/poky/meta/lib/oe/sdk.py +++ b/poky/meta/lib/oe/sdk.py @@ -95,8 +95,8 @@ class Sdk(object, metaclass=ABCMeta): if linguas == "all": pm.install_glob("nativesdk-glibc-binary-localedata-*.utf-8", sdk=True) else: - for lang in linguas.split(): - pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang) + pm.install(["nativesdk-glibc-binary-localedata-%s.utf-8" % \ + lang for lang in linguas.split()]) # Generate a locale archive of them target_arch = self.d.getVar('SDK_ARCH') rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path) diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index 18c5a353a..a83af519e 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -263,10 +263,181 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): if error_msgs: bb.fatal("\n".join(error_msgs)) +class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): + name = "OEEquivHash" + + def init_rundepcheck(self, data): + super().init_rundepcheck(data) + self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') + self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') + self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) + + def get_taskdata(self): + return (self.server, self.method) + super().get_taskdata() + + def set_taskdata(self, data): + self.server, self.method = data[:2] + super().set_taskdata(data[2:]) + + def __get_task_unihash_key(self, task): + # TODO: The key only *needs* to be the taskhash, the task is just + # convenient + return '%s:%s' % (task, self.taskhash[task]) + + def get_stampfile_hash(self, task): + if task in self.taskhash: + # If a unique hash is reported, use it as the stampfile hash. This + # ensures that if a task won't be re-run if the taskhash changes, + # but it would result in the same output hash + unihash = self.unihashes.get(self.__get_task_unihash_key(task)) + if unihash is not None: + return unihash + + return super().get_stampfile_hash(task) + + def get_unihash(self, task): + import urllib + import json + + taskhash = self.taskhash[task] + + key = self.__get_task_unihash_key(task) + + # TODO: This cache can grow unbounded. It probably only needs to keep + # for each task + unihash = self.unihashes.get(key) + if unihash is not None: + return unihash + + # In the absence of being able to discover a unique hash from the + # server, make it be equivalent to the taskhash. The unique "hash" only + # really needs to be a unique string (not even necessarily a hash), but + # making it match the taskhash has a few advantages: + # + # 1) All of the sstate code that assumes hashes can be the same + # 2) It provides maximal compatibility with builders that don't use + # an equivalency server + # 3) The value is easy for multiple independent builders to derive the + # same unique hash from the same input. This means that if the + # independent builders find the same taskhash, but it isn't reported + # to the server, there is a better chance that they will agree on + # the unique hash. + unihash = taskhash + + try: + url = '%s/v1/equivalent?%s' % (self.server, + urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]})) + + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + data = response.read().decode('utf-8') + + json_data = json.loads(data) + + if json_data: + unihash = json_data['unihash'] + # A unique hash equal to the taskhash is not very interesting, + # so it is reported it at debug level 2. If they differ, that + # is much more interesting, so it is reported at debug level 1 + bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server)) + else: + bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server)) + except urllib.error.URLError as e: + bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) + except (KeyError, json.JSONDecodeError) as e: + bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) + + self.unihashes[key] = unihash + return unihash + + def report_unihash(self, path, task, d): + import urllib + import json + import tempfile + import base64 + import importlib + + taskhash = d.getVar('BB_TASKHASH') + unihash = d.getVar('BB_UNIHASH') + report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1' + tempdir = d.getVar('T') + fn = d.getVar('BB_FILENAME') + key = fn + '.do_' + task + ':' + taskhash + + # Sanity checks + cache_unihash = self.unihashes.get(key) + if cache_unihash is None: + bb.fatal('%s not in unihash cache. Please report this error' % key) + + if cache_unihash != unihash: + bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash)) + + sigfile = None + sigfile_name = "depsig.do_%s.%d" % (task, os.getpid()) + sigfile_link = "depsig.do_%s" % task + + try: + sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') + + locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} + + (module, method) = self.method.rsplit('.', 1) + locs['method'] = getattr(importlib.import_module(module), method) + + outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) + + try: + url = '%s/v1/equivalent' % self.server + task_data = { + 'taskhash': taskhash, + 'method': self.method, + 'outhash': outhash, + 'unihash': unihash, + 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER') + } + + if report_taskdata: + sigfile.seek(0) + + task_data['PN'] = d.getVar('PN') + task_data['PV'] = d.getVar('PV') + task_data['PR'] = d.getVar('PR') + task_data['task'] = task + task_data['outhash_siginfo'] = sigfile.read().decode('utf-8') + + headers = {'content-type': 'application/json'} + + request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers) + response = urllib.request.urlopen(request) + data = response.read().decode('utf-8') + + json_data = json.loads(data) + new_unihash = json_data['unihash'] + + if new_unihash != unihash: + bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) + else: + bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) + except urllib.error.URLError as e: + bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) + except (KeyError, json.JSONDecodeError) as e: + bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) + finally: + if sigfile: + sigfile.close() + + sigfile_link_path = os.path.join(tempdir, sigfile_link) + bb.utils.remove(sigfile_link_path) + + try: + os.symlink(sigfile_name, sigfile_link_path) + except OSError: + pass # Insert these classes into siggen's namespace so it can see and select them bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash +bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash def find_siginfo(pn, taskname, taskhashlist, d): @@ -327,7 +498,7 @@ def find_siginfo(pn, taskname, taskhashlist, d): if not taskhashlist or (len(filedates) < 2 and not foundall): # That didn't work, look in sstate-cache - hashes = taskhashlist or ['?' * 32] + hashes = taskhashlist or ['?' * 64] localdata = bb.data.createCopy(d) for hashval in hashes: localdata.setVar('PACKAGE_ARCH', '*') @@ -414,4 +585,133 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) return None, d2 +def OEOuthashBasic(path, sigfile, task, d): + """ + Basic output hash function + + Calculates the output hash of a task by hashing all output file metadata, + and file contents. + """ + import hashlib + import stat + import pwd + import grp + + def update_hash(s): + s = s.encode('utf-8') + h.update(s) + if sigfile: + sigfile.write(s) + + h = hashlib.sha256() + prev_dir = os.getcwd() + include_owners = os.environ.get('PSEUDO_DISABLED') == '0' + + try: + os.chdir(path) + + update_hash("OEOuthashBasic\n") + + # It is only currently useful to get equivalent hashes for things that + # can be restored from sstate. Since the sstate object is named using + # SSTATE_PKGSPEC and the task name, those should be included in the + # output hash calculation. + update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) + update_hash("task=%s\n" % task) + + for root, dirs, files in os.walk('.', topdown=True): + # Sort directories to ensure consistent ordering when recursing + dirs.sort() + files.sort() + + def process(path): + s = os.lstat(path) + + if stat.S_ISDIR(s.st_mode): + update_hash('d') + elif stat.S_ISCHR(s.st_mode): + update_hash('c') + elif stat.S_ISBLK(s.st_mode): + update_hash('b') + elif stat.S_ISSOCK(s.st_mode): + update_hash('s') + elif stat.S_ISLNK(s.st_mode): + update_hash('l') + elif stat.S_ISFIFO(s.st_mode): + update_hash('p') + else: + update_hash('-') + + def add_perm(mask, on, off='-'): + if mask & s.st_mode: + update_hash(on) + else: + update_hash(off) + + add_perm(stat.S_IRUSR, 'r') + add_perm(stat.S_IWUSR, 'w') + if stat.S_ISUID & s.st_mode: + add_perm(stat.S_IXUSR, 's', 'S') + else: + add_perm(stat.S_IXUSR, 'x') + + add_perm(stat.S_IRGRP, 'r') + add_perm(stat.S_IWGRP, 'w') + if stat.S_ISGID & s.st_mode: + add_perm(stat.S_IXGRP, 's', 'S') + else: + add_perm(stat.S_IXGRP, 'x') + + add_perm(stat.S_IROTH, 'r') + add_perm(stat.S_IWOTH, 'w') + if stat.S_ISVTX & s.st_mode: + update_hash('t') + else: + add_perm(stat.S_IXOTH, 'x') + + if include_owners: + update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) + update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) + + update_hash(" ") + if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): + update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) + else: + update_hash(" " * 9) + + update_hash(" ") + if stat.S_ISREG(s.st_mode): + update_hash("%10d" % s.st_size) + else: + update_hash(" " * 10) + + update_hash(" ") + fh = hashlib.sha256() + if stat.S_ISREG(s.st_mode): + # Hash file contents + with open(path, 'rb') as d: + for chunk in iter(lambda: d.read(4096), b""): + fh.update(chunk) + update_hash(fh.hexdigest()) + else: + update_hash(" " * len(fh.hexdigest())) + + update_hash(" %s" % path) + + if stat.S_ISLNK(s.st_mode): + update_hash(" -> %s" % os.readlink(path)) + + update_hash("\n") + + # Process this directory and all its child files + process(root) + for f in files: + if f == 'fixmepath': + continue + process(os.path.join(root, f)) + finally: + os.chdir(prev_dir) + + return h.hexdigest() + diff --git a/poky/meta/lib/oe/terminal.py b/poky/meta/lib/oe/terminal.py index caeb5e31e..e4045559f 100644 --- a/poky/meta/lib/oe/terminal.py +++ b/poky/meta/lib/oe/terminal.py @@ -39,7 +39,7 @@ class Terminal(Popen, metaclass=Registry): raise def format_command(self, sh_cmd, title): - fmt = {'title': title or 'Terminal', 'command': sh_cmd} + fmt = {'title': title or 'Terminal', 'command': sh_cmd, 'cwd': os.getcwd() } if isinstance(self.command, str): return shlex.split(self.command.format(**fmt)) else: @@ -117,7 +117,7 @@ class Screen(Terminal): class TmuxRunning(Terminal): """Open a new pane in the current running tmux window""" name = 'tmux-running' - command = 'tmux split-window "{command}"' + command = 'tmux split-window -c "{cwd}" "{command}"' priority = 2.75 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -135,7 +135,7 @@ class TmuxRunning(Terminal): class TmuxNewWindow(Terminal): """Open a new window in the current running tmux session""" name = 'tmux-new-window' - command = 'tmux new-window -n "{title}" "{command}"' + command = 'tmux new-window -c "{cwd}" -n "{title}" "{command}"' priority = 2.70 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -149,7 +149,7 @@ class TmuxNewWindow(Terminal): class Tmux(Terminal): """Start a new tmux session and window""" - command = 'tmux new -d -s devshell -n devshell "{command}"' + command = 'tmux new -c "{cwd}" -d -s devshell -n devshell "{command}"' priority = 0.75 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -160,7 +160,7 @@ class Tmux(Terminal): # devshells, if it's already there, add a new window to it. window_name = 'devshell-%i' % os.getpid() - self.command = 'tmux new -d -s {0} -n {0} "{{command}}"'.format(window_name) + self.command = 'tmux new -c "{{cwd}}" -d -s {0} -n {0} "{{command}}"'.format(window_name) Terminal.__init__(self, sh_cmd, title, env, d) attach_cmd = 'tmux att -t {0}'.format(window_name) @@ -296,6 +296,8 @@ def check_terminal_version(terminalName): vernum = ver.split(' ')[-1] if ver.startswith('tmux'): vernum = ver.split()[-1] + if ver.startswith('tmux next-'): + vernum = ver.split()[-1][5:] return vernum def distro_name(): diff --git a/poky/meta/lib/oe/utils.py b/poky/meta/lib/oe/utils.py index 8a584d6dd..cedd053d3 100644 --- a/poky/meta/lib/oe/utils.py +++ b/poky/meta/lib/oe/utils.py @@ -214,7 +214,7 @@ def packages_filter_out_system(d): PN-dbg PN-doc PN-locale-eb-gb removed. """ pn = d.getVar('PN') - blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] + blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')] localepkg = pn + "-locale-" pkgs = [] @@ -307,6 +307,10 @@ def multiprocess_launch(target, items, d, extraargs=None): p.start() launched.append(p) for q in launched: + # Have to manually call update() to avoid deadlocks. The pipe can be full and + # transfer stalled until we try and read the results object but the subprocess won't exit + # as it still has data to write (https://bugs.python.org/issue8426) + q.update() # The finished processes are joined when calling is_alive() if not q.is_alive(): if q.exception: @@ -326,7 +330,7 @@ def multiprocess_launch(target, items, d, extraargs=None): def squashspaces(string): import re - return re.sub("\s+", " ", string).strip() + return re.sub(r"\s+", " ", string).strip() def format_pkg_list(pkg_dict, ret_format=None): output = [] @@ -363,14 +367,18 @@ def host_gcc_version(d, taskcontextonly=False): return compiler = d.getVar("BUILD_CC") + # Get rid of ccache since it is not present when parsing. + if compiler.startswith('ccache '): + compiler = compiler[7:] try: env = os.environ.copy() env["PATH"] = d.getVar("PATH") - output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + output = subprocess.check_output("%s --version" % compiler, \ + shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) - match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + match = re.match(r".* (\d\.\d)\.\d.*", output.split('\n')[0]) if not match: bb.fatal("Can't get compiler version from %s --version output" % compiler) @@ -482,3 +490,6 @@ class ImageQAFailed(bb.build.FuncFailed): return msg +def sh_quote(string): + import shlex + return shlex.quote(string) -- cgit v1.2.3