diff options
Diffstat (limited to 'poky/meta/lib/oe')
-rw-r--r-- | poky/meta/lib/oe/cve_check.py | 60 | ||||
-rw-r--r-- | poky/meta/lib/oe/package_manager/__init__.py | 11 | ||||
-rw-r--r-- | poky/meta/lib/oe/package_manager/deb/__init__.py | 13 | ||||
-rw-r--r-- | poky/meta/lib/oe/package_manager/ipk/sdk.py | 6 | ||||
-rw-r--r-- | poky/meta/lib/oe/package_manager/rpm/__init__.py | 3 | ||||
-rw-r--r-- | poky/meta/lib/oe/patch.py | 16 | ||||
-rw-r--r-- | poky/meta/lib/oe/path.py | 21 | ||||
-rw-r--r-- | poky/meta/lib/oe/qa.py | 1 | ||||
-rw-r--r-- | poky/meta/lib/oe/rootfs.py | 3 | ||||
-rw-r--r-- | poky/meta/lib/oe/sstatesig.py | 6 | ||||
-rw-r--r-- | poky/meta/lib/oe/terminal.py | 4 | ||||
-rw-r--r-- | poky/meta/lib/oe/utils.py | 2 |
12 files changed, 129 insertions, 17 deletions
diff --git a/poky/meta/lib/oe/cve_check.py b/poky/meta/lib/oe/cve_check.py new file mode 100644 index 0000000000..ce755f940a --- /dev/null +++ b/poky/meta/lib/oe/cve_check.py @@ -0,0 +1,60 @@ +import collections +import re +import itertools +import functools + +_Version = collections.namedtuple( + "_Version", ["release", "patch_l", "pre_l", "pre_v"] +) + +@functools.total_ordering +class Version(): + + def __init__(self, version, suffix=None): + if str(suffix) == "alphabetical": + version_pattern = r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<patch>[-_\.]?(?P<patch_l>[a-z]))?(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?""" + else: + version_pattern = r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?""" + regex = re.compile(r"^\s*" + version_pattern + r"\s*$", re.VERBOSE | re.IGNORECASE) + + match = regex.search(version) + if not match: + raise Exception("Invalid version: '{0}'".format(version)) + + self._version = _Version( + release=tuple(int(i) for i in match.group("release").replace("-",".").split(".")), + patch_l=match.group("patch_l") if str(suffix) == "alphabetical" and match.group("patch_l") else "", + pre_l=match.group("pre_l"), + pre_v=match.group("pre_v") + ) + + self._key = _cmpkey( + self._version.release, + self._version.patch_l, + self._version.pre_l, + self._version.pre_v + ) + + def __eq__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key == other._key + + def __gt__(self, other): + if not isinstance(other, Version): + return NotImplemented + return self._key > other._key + +def _cmpkey(release, patch_l, pre_l, pre_v): + # remove leading 0 + _release = tuple( + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) + ) + + _patch = patch_l.upper() + + if pre_l is None and pre_v is None: + _pre = float('inf') + else: + _pre = float(pre_v) if pre_v else float('-inf') + return _release, _patch, _pre diff --git a/poky/meta/lib/oe/package_manager/__init__.py b/poky/meta/lib/oe/package_manager/__init__.py index 42225a3b2e..8e7128b195 100644 --- a/poky/meta/lib/oe/package_manager/__init__.py +++ b/poky/meta/lib/oe/package_manager/__init__.py @@ -328,7 +328,11 @@ class PackageManager(object, metaclass=ABCMeta): try: bb.note("Installing globbed packages...") cmd = ["oe-pkgdata-util", "-p", pkgdatadir, "list-pkgs", globs] - pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + bb.note('Running %s' % cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + if stderr: bb.note(stderr.decode("utf-8")) + pkgs = stdout.decode("utf-8") self.install(pkgs.split(), attempt_only=True) except subprocess.CalledProcessError as e: # Return code 1 means no packages matched @@ -384,7 +388,10 @@ class PackageManager(object, metaclass=ABCMeta): cmd.extend(['--exclude=' + '|'.join(exclude.split())]) try: bb.note('Running %s' % cmd) - complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + if stderr: bb.note(stderr.decode("utf-8")) + complementary_pkgs = stdout.decode("utf-8") complementary_pkgs = set(complementary_pkgs.split()) skip_pkgs = sorted(complementary_pkgs & provided_pkgs) install_pkgs = sorted(complementary_pkgs - provided_pkgs) diff --git a/poky/meta/lib/oe/package_manager/deb/__init__.py b/poky/meta/lib/oe/package_manager/deb/__init__.py index 10ad707c23..7fdfdaa4fa 100644 --- a/poky/meta/lib/oe/package_manager/deb/__init__.py +++ b/poky/meta/lib/oe/package_manager/deb/__init__.py @@ -287,7 +287,8 @@ class DpkgPM(OpkgDpkgPM): try: bb.note("Installing the following packages: %s" % ' '.join(pkgs)) - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + bb.note(output.decode("utf-8")) except subprocess.CalledProcessError as e: (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % @@ -343,8 +344,12 @@ class DpkgPM(OpkgDpkgPM): if feed_uris == "": return + sources_conf = os.path.join("%s/etc/apt/sources.list" % self.target_rootfs) + if not os.path.exists(os.path.dirname(sources_conf)): + return + arch_list = [] if feed_archs is None: @@ -362,11 +367,11 @@ class DpkgPM(OpkgDpkgPM): if arch_list: for arch in arch_list: bb.note('Adding dpkg channel at (%s)' % uri) - sources_file.write("deb %s/%s ./\n" % + sources_file.write("deb [trusted=yes] %s/%s ./\n" % (uri, arch)) else: bb.note('Adding dpkg channel at (%s)' % uri) - sources_file.write("deb %s ./\n" % uri) + sources_file.write("deb [trusted=yes] %s ./\n" % uri) def _create_configs(self, archs, base_archs): base_archs = re.sub(r"_", r"-", base_archs) @@ -406,7 +411,7 @@ class DpkgPM(OpkgDpkgPM): with open(os.path.join(self.apt_conf_dir, "sources.list"), "w+") as sources_file: for arch in arch_list: - sources_file.write("deb file:%s/ ./\n" % + sources_file.write("deb [trusted=yes] file:%s/ ./\n" % os.path.join(self.deploy_dir, arch)) base_arch_list = base_archs.split() diff --git a/poky/meta/lib/oe/package_manager/ipk/sdk.py b/poky/meta/lib/oe/package_manager/ipk/sdk.py index 37af0344eb..e2ca415c8e 100644 --- a/poky/meta/lib/oe/package_manager/ipk/sdk.py +++ b/poky/meta/lib/oe/package_manager/ipk/sdk.py @@ -14,6 +14,12 @@ class PkgSdk(Sdk): def __init__(self, d, manifest_dir=None): super(PkgSdk, self).__init__(d, manifest_dir) + # In sdk_list_installed_packages the call to opkg is hardcoded to + # always use IPKGCONF_TARGET and there's no exposed API to change this + # so simply override IPKGCONF_TARGET to use this separated config file. + ipkgconf_sdk_target = d.getVar("IPKGCONF_SDK_TARGET") + d.setVar("IPKGCONF_TARGET", ipkgconf_sdk_target) + self.target_conf = self.d.getVar("IPKGCONF_TARGET") self.host_conf = self.d.getVar("IPKGCONF_SDK") diff --git a/poky/meta/lib/oe/package_manager/rpm/__init__.py b/poky/meta/lib/oe/package_manager/rpm/__init__.py index 898184442f..6df0092281 100644 --- a/poky/meta/lib/oe/package_manager/rpm/__init__.py +++ b/poky/meta/lib/oe/package_manager/rpm/__init__.py @@ -33,6 +33,9 @@ class RpmIndexer(Indexer): class RpmSubdirIndexer(RpmIndexer): def write_index(self): bb.note("Generating package index for %s" %(self.deploy_dir)) + # Remove the existing repodata to ensure that we re-generate it no matter what + bb.utils.remove(os.path.join(self.deploy_dir, "repodata"), recurse=True) + self.do_write_index(self.deploy_dir) for entry in os.walk(self.deploy_dir): if os.path.samefile(self.deploy_dir, entry[0]): diff --git a/poky/meta/lib/oe/patch.py b/poky/meta/lib/oe/patch.py index 40755fbb03..fccbedb519 100644 --- a/poky/meta/lib/oe/patch.py +++ b/poky/meta/lib/oe/patch.py @@ -38,15 +38,19 @@ def runcmd(args, dir = None): args = [ pipes.quote(str(arg)) for arg in args ] cmd = " ".join(args) # print("cmd: %s" % cmd) - (exitstatus, output) = subprocess.getstatusoutput(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + stdout, stderr = proc.communicate() + stdout = stdout.decode('utf-8') + stderr = stderr.decode('utf-8') + exitstatus = proc.returncode if exitstatus != 0: - raise CmdError(cmd, exitstatus >> 8, output) - if " fuzz " in output and "Hunk " in output: + raise CmdError(cmd, exitstatus >> 8, "stdout: %s\nstderr: %s" % (stdout, stderr)) + if " fuzz " in stdout and "Hunk " in stdout: # Drop patch fuzz info with header and footer to log file so # insane.bbclass can handle to throw error/warning - bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(output)) + bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(stdout)) - return output + return stdout finally: if dir: @@ -512,7 +516,7 @@ class GitApplyTree(PatchTree): try: shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot] self.gitCommandUserOptions(shellcmd, self.commituser, self.commitemail) - shellcmd += ["am", "-3", "--keep-cr", "-p%s" % patch['strippath']] + shellcmd += ["am", "-3", "--keep-cr", "--no-scissors", "-p%s" % patch['strippath']] return _applypatchhelper(shellcmd, patch, force, reverse, run) except CmdError: # Need to abort the git am, or we'll still be within it at the end diff --git a/poky/meta/lib/oe/path.py b/poky/meta/lib/oe/path.py index 082972457b..c8d8ad05b9 100644 --- a/poky/meta/lib/oe/path.py +++ b/poky/meta/lib/oe/path.py @@ -320,3 +320,24 @@ def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=F return files +def canonicalize(paths, sep=','): + """Given a string with paths (separated by commas by default), expand + each path using os.path.realpath() and return the resulting paths as a + string (separated using the same separator a the original string). + """ + # Ignore paths containing "$" as they are assumed to be unexpanded bitbake + # variables. Normally they would be ignored, e.g., when passing the paths + # through the shell they would expand to empty strings. However, when they + # are passed through os.path.realpath(), it will cause them to be prefixed + # with the absolute path to the current directory and thus not be empty + # anymore. + # + # Also maintain trailing slashes, as the paths may actually be used as + # prefixes in sting compares later on, where the slashes then are important. + canonical_paths = [] + for path in (paths or '').split(sep): + if '$' not in path: + trailing_slash = path.endswith('/') and '/' or '' + canonical_paths.append(os.path.realpath(path) + trailing_slash) + + return sep.join(canonical_paths) diff --git a/poky/meta/lib/oe/qa.py b/poky/meta/lib/oe/qa.py index ea831b930a..e8a854a302 100644 --- a/poky/meta/lib/oe/qa.py +++ b/poky/meta/lib/oe/qa.py @@ -156,6 +156,7 @@ def elf_machine_to_string(machine): """ try: return { + 0x00: "Unset", 0x02: "SPARC", 0x03: "x86", 0x08: "MIPS", diff --git a/poky/meta/lib/oe/rootfs.py b/poky/meta/lib/oe/rootfs.py index 4b747dd0f4..249c685dcf 100644 --- a/poky/meta/lib/oe/rootfs.py +++ b/poky/meta/lib/oe/rootfs.py @@ -217,6 +217,9 @@ class Rootfs(object, metaclass=ABCMeta): self.progress_reporter.next_stage() if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", + True, False, self.d) and \ + not bb.utils.contains("IMAGE_FEATURES", + "read-only-rootfs-delayed-postinsts", True, False, self.d): delayed_postinsts = self._get_delayed_postinsts() if delayed_postinsts is not None: diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index adfe2e403b..84999ee94d 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -552,9 +552,11 @@ def OEOuthashBasic(path, sigfile, task, d): try: update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) - except KeyError: + except KeyError as e: bb.warn("KeyError in %s" % path) - raise + msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match " + "any user/group on target. This may be due to host contamination." % (e, path, s.st_uid, s.st_gid)) + raise Exception(msg).with_traceback(e.__traceback__) if include_timestamps: update_hash(" %10d" % s.st_mtime) diff --git a/poky/meta/lib/oe/terminal.py b/poky/meta/lib/oe/terminal.py index eb10a6e33e..61c2687ef4 100644 --- a/poky/meta/lib/oe/terminal.py +++ b/poky/meta/lib/oe/terminal.py @@ -185,7 +185,7 @@ class Custom(Terminal): Terminal.__init__(self, sh_cmd, title, env, d) logger.warning('Custom terminal was started.') else: - logger.debug(1, 'No custom terminal (OE_TERMINAL_CUSTOMCMD) set') + logger.debug('No custom terminal (OE_TERMINAL_CUSTOMCMD) set') raise UnsupportedTerminal('OE_TERMINAL_CUSTOMCMD not set') @@ -216,7 +216,7 @@ def spawn_preferred(sh_cmd, title=None, env=None, d=None): def spawn(name, sh_cmd, title=None, env=None, d=None): """Spawn the specified terminal, by name""" - logger.debug(1, 'Attempting to spawn terminal "%s"', name) + logger.debug('Attempting to spawn terminal "%s"', name) try: terminal = Registry.registry[name] except KeyError: diff --git a/poky/meta/lib/oe/utils.py b/poky/meta/lib/oe/utils.py index 468c76f30f..9a2187e36f 100644 --- a/poky/meta/lib/oe/utils.py +++ b/poky/meta/lib/oe/utils.py @@ -193,7 +193,7 @@ def parallel_make(d, makeinst=False): return int(v) - return None + return '' def parallel_make_argument(d, fmt, limit=None, makeinst=False): """ |