diff options
author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2017-12-04 09:01:44 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2017-12-14 20:17:23 +0300 |
commit | 37a0e4ddff58c0120cc5cfef104b60d0e180638c (patch) | |
tree | 1628857a2eb33ab517ba93d6a3ca25e55bd3e628 /import-layers/yocto-poky/meta/lib/oe | |
parent | 3c4c45d1e9a2324191a8640b22df1b71f15f3037 (diff) | |
download | openbmc-37a0e4ddff58c0120cc5cfef104b60d0e180638c.tar.xz |
Squashed 'import-layers/yocto-poky/' changes from dc8508f6099..67491b0c104
Yocto 2.2.2 (Morty)
Change-Id: Id9a452e28940d9f166957de243d9cb1d8818704e
git-subtree-dir: import-layers/yocto-poky
git-subtree-split: 67491b0c104101bb9f366d697edd23c895be4302
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'import-layers/yocto-poky/meta/lib/oe')
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py | 6 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/gpg_sign.py | 21 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/package_manager.py | 23 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/path.py | 10 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/qa.py | 82 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/rootfs.py | 10 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/terminal.py | 2 | ||||
-rw-r--r-- | import-layers/yocto-poky/meta/lib/oe/utils.py | 19 |
8 files changed, 103 insertions, 70 deletions
diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py index afaff6859..29ac6d418 100644 --- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -4,11 +4,15 @@ import stat import shutil def _smart_copy(src, dest): + import subprocess # smart_copy will choose the correct function depending on whether the # source is a file or a directory. mode = os.stat(src).st_mode if stat.S_ISDIR(mode): - shutil.copytree(src, dest, symlinks=True, ignore=shutil.ignore_patterns('.git')) + bb.utils.mkdirhier(dest) + cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ + | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) else: shutil.copyfile(src, dest) shutil.copymode(src, dest) diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py index 38eb0cb13..ba61f9890 100644 --- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py +++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py @@ -10,6 +10,7 @@ class LocalSigner(object): self.gpg_bin = d.getVar('GPG_BIN', True) or \ bb.utils.which(os.getenv('PATH'), 'gpg') self.gpg_path = d.getVar('GPG_PATH', True) + self.gpg_version = self.get_gpg_version() self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpm") def export_pubkey(self, output_file, keyid, armor=True): @@ -31,15 +32,18 @@ class LocalSigner(object): cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid cmd += "--define '_gpg_passphrase %s' " % passphrase + if self.gpg_version > (2,1,): + cmd += "--define '_gpg_sign_cmd_extra_args --pinentry-mode=loopback' " if self.gpg_bin: cmd += "--define '%%__gpg %s' " % self.gpg_bin if self.gpg_path: cmd += "--define '_gpg_path %s' " % self.gpg_path - cmd += ' '.join(files) - status, output = oe.utils.getstatusoutput(cmd) - if status: - raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + # Sign in chunks of 100 packages + for i in range(0, len(files), 100): + status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100])) + if status: + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): """Create a detached signature of a file""" @@ -58,9 +62,7 @@ class LocalSigner(object): #gpg > 2.1 supports password pipes only through the loopback interface #gpg < 2.1 errors out if given unknown parameters - dots = self.get_gpg_version().split('.') - assert len(dots) >= 2 - if int(dots[0]) >= 2 and int(dots[1]) >= 1: + if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback'] cmd += [input_file] @@ -87,10 +89,11 @@ class LocalSigner(object): def get_gpg_version(self): - """Return the gpg version""" + """Return the gpg version as a tuple of ints""" import subprocess try: - return subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + return tuple([int(i) for i in ver_str.split('.')]) except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py index 3cee9730a..13577b18b 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py +++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py @@ -1673,13 +1673,15 @@ class OpkgPM(OpkgDpkgPM): self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True), arch)) - if self.opkg_dir != '/var/lib/opkg': + if self.d.getVar('OPKGLIBDIR', True) != '/var/lib': # There is no command line option for this anymore, we need to add # info_dir and status_file to config file, if OPKGLIBDIR doesn't have # the default value of "/var/lib" as defined in opkg: - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) + cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'lists')) cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) @@ -1698,13 +1700,15 @@ class OpkgPM(OpkgDpkgPM): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) - if self.opkg_dir != '/var/lib/opkg': + if self.d.getVar('OPKGLIBDIR', True) != '/var/lib': # There is no command line option for this anymore, we need to add # info_dir and status_file to config file, if OPKGLIBDIR doesn't have # the default value of "/var/lib" as defined in opkg: - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) + config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'lists')) config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) def insert_feeds_uris(self): @@ -1776,7 +1780,7 @@ class OpkgPM(OpkgDpkgPM): def remove(self, pkgs, with_dependencies=True): if with_dependencies: - cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \ + cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) else: cmd = "%s %s --force-depends remove %s" % \ @@ -1860,7 +1864,10 @@ class OpkgPM(OpkgDpkgPM): # Create an temp dir as opkg root for dummy installation temp_rootfs = self.d.expand('${T}/opkg') - temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg') + opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + if opkg_lib_dir[0] == "/": + opkg_lib_dir = opkg_lib_dir[1:] + temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') bb.utils.mkdirhier(temp_opkg_dir) opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py index 06a5af265..ed7fd1eef 100644 --- a/import-layers/yocto-poky/meta/lib/oe/path.py +++ b/import-layers/yocto-poky/meta/lib/oe/path.py @@ -83,12 +83,14 @@ def copyhardlinktree(src, dst): if os.path.isdir(src): import glob if len(glob.glob('%s/.??*' % src)) > 0: - source = '%s/.??* ' % src - source = source + '%s/*' % src + source = './.??* ' + source += './*' + s_dir = src else: source = src - cmd = 'cp -afl --preserve=xattr %s %s' % (source, dst) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + s_dir = os.getcwd() + cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) + subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) else: copytree(src, dst) diff --git a/import-layers/yocto-poky/meta/lib/oe/qa.py b/import-layers/yocto-poky/meta/lib/oe/qa.py index fbe719d8e..22d76dcbc 100644 --- a/import-layers/yocto-poky/meta/lib/oe/qa.py +++ b/import-layers/yocto-poky/meta/lib/oe/qa.py @@ -1,4 +1,4 @@ -import os, struct +import os, struct, mmap class NotELFFileError(Exception): pass @@ -23,9 +23,9 @@ class ELFFile: EV_CURRENT = 1 # possible values for EI_DATA - ELFDATANONE = 0 - ELFDATA2LSB = 1 - ELFDATA2MSB = 2 + EI_DATA_NONE = 0 + EI_DATA_LSB = 1 + EI_DATA_MSB = 2 PT_INTERP = 3 @@ -34,51 +34,46 @@ class ELFFile: #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name) raise NotELFFileError("%s is not an ELF" % self.name) - def __init__(self, name, bits = 0): + def __init__(self, name): self.name = name - self.bits = bits self.objdump_output = {} - def open(self): - if not os.path.isfile(self.name): - raise NotELFFileError("%s is not a normal file" % self.name) + # Context Manager functions to close the mmap explicitly + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.data.close() + def open(self): with open(self.name, "rb") as f: - # Read 4k which should cover most of the headers we're after - self.data = f.read(4096) + try: + self.data = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + except ValueError: + # This means the file is empty + raise NotELFFileError("%s is empty" % self.name) + # Check the file has the minimum number of ELF table entries if len(self.data) < ELFFile.EI_NIDENT + 4: raise NotELFFileError("%s is not an ELF" % self.name) + # ELF header self.my_assert(self.data[0], 0x7f) self.my_assert(self.data[1], ord('E')) self.my_assert(self.data[2], ord('L')) self.my_assert(self.data[3], ord('F')) - if self.bits == 0: - if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: - self.bits = 32 - elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: - self.bits = 64 - else: - # Not 32-bit or 64.. lets assert - raise NotELFFileError("ELF but not 32 or 64 bit.") - elif self.bits == 32: - self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS32) - elif self.bits == 64: - self.my_assert(self.data[ELFFile.EI_CLASS], ELFFile.ELFCLASS64) + if self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS32: + self.bits = 32 + elif self.data[ELFFile.EI_CLASS] == ELFFile.ELFCLASS64: + self.bits = 64 else: - raise NotELFFileError("Must specify unknown, 32 or 64 bit size.") + # Not 32-bit or 64.. lets assert + raise NotELFFileError("ELF but not 32 or 64 bit.") self.my_assert(self.data[ELFFile.EI_VERSION], ELFFile.EV_CURRENT) - self.sex = self.data[ELFFile.EI_DATA] - if self.sex == ELFFile.ELFDATANONE: - raise NotELFFileError("self.sex == ELFDATANONE") - elif self.sex == ELFFile.ELFDATA2LSB: - self.sex = "<" - elif self.sex == ELFFile.ELFDATA2MSB: - self.sex = ">" - else: - raise NotELFFileError("Unknown self.sex") + self.endian = self.data[ELFFile.EI_DATA] + if self.endian not in (ELFFile.EI_DATA_LSB, ELFFile.EI_DATA_MSB): + raise NotELFFileError("Unexpected EI_DATA %x" % self.endian) def osAbi(self): return self.data[ELFFile.EI_OSABI] @@ -90,16 +85,20 @@ class ELFFile: return self.bits def isLittleEndian(self): - return self.sex == "<" + return self.endian == ELFFile.EI_DATA_LSB def isBigEndian(self): - return self.sex == ">" + return self.endian == ELFFile.EI_DATA_MSB + + def getStructEndian(self): + return {ELFFile.EI_DATA_LSB: "<", + ELFFile.EI_DATA_MSB: ">"}[self.endian] def getShort(self, offset): - return struct.unpack_from(self.sex+"H", self.data, offset)[0] + return struct.unpack_from(self.getStructEndian() + "H", self.data, offset)[0] def getWord(self, offset): - return struct.unpack_from(self.sex+"i", self.data, offset)[0] + return struct.unpack_from(self.getStructEndian() + "i", self.data, offset)[0] def isDynamic(self): """ @@ -118,7 +117,7 @@ class ELFFile: def machine(self): """ - We know the sex stored in self.sex and we + We know the endian stored in self.endian and we know the position """ return self.getShort(ELFFile.E_MACHINE) @@ -166,6 +165,7 @@ def elf_machine_to_string(machine): if __name__ == "__main__": import sys - elf = ELFFile(sys.argv[1]) - elf.open() - print(elf.isDynamic()) + + with ELFFile(sys.argv[1]) as elf: + elf.open() + print(elf.isDynamic()) diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py index a348b975c..f96788399 100644 --- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py +++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py @@ -186,10 +186,6 @@ class Rootfs(object, metaclass=ABCMeta): shutil.copytree(postinst_intercepts_dir, intercepts_dir) - shutil.copy(self.d.expand("${COREBASE}/meta/files/deploydir_readme.txt"), - self.deploydir + - "/README_-_DO_NOT_DELETE_FILES_IN_THIS_DIRECTORY.txt") - execute_pre_post_process(self.d, pre_process_cmds) if self.progress_reporter: @@ -477,8 +473,6 @@ class RpmRootfs(Rootfs): execute_pre_post_process(self.d, rpm_post_process_cmds) - self._log_check() - if self.inc_rpm_image_gen == "1": self.pm.backup_packaging_data() @@ -951,7 +945,9 @@ class OpkgRootfs(DpkgOpkgRootfs): if self.progress_reporter: self.progress_reporter.next_stage() - self._setup_dbg_rootfs(['/etc', '/var/lib/opkg', '/usr/lib/ssl']) + opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') + self._setup_dbg_rootfs(['/etc', opkg_dir, '/usr/lib/ssl']) execute_pre_post_process(self.d, opkg_post_process_cmds) diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py index 3901ad3f2..3c8ef59a4 100644 --- a/import-layers/yocto-poky/meta/lib/oe/terminal.py +++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py @@ -227,6 +227,8 @@ def spawn(name, sh_cmd, title=None, env=None, d=None): pipe = terminal(sh_cmd, title, env, d) output = pipe.communicate()[0] + if output: + output = output.decode("utf-8") if pipe.returncode != 0: raise ExecutionError(sh_cmd, pipe.returncode, output) diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py index d6545b197..36cf74f29 100644 --- a/import-layers/yocto-poky/meta/lib/oe/utils.py +++ b/import-layers/yocto-poky/meta/lib/oe/utils.py @@ -230,6 +230,25 @@ def format_pkg_list(pkg_dict, ret_format=None): return '\n'.join(output) +def host_gcc_version(d): + import re, subprocess + + compiler = d.getVar("BUILD_CC", True) + + try: + env = os.environ.copy() + env["PATH"] = d.getVar("PATH", True) + output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) + + match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) + + version = match.group(1) + return "-%s" % version if version in ("4.8", "4.9") else "" + # # Python 2.7 doesn't have threaded pools (just multiprocessing) # so implement a version here |