diff options
Diffstat (limited to 'import-layers/yocto-poky/meta/lib/oe')
17 files changed, 460 insertions, 120 deletions
diff --git a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py index 3a5b7b6b44..3e86a46a3f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py +++ b/import-layers/yocto-poky/meta/lib/oe/buildhistory_analysis.py @@ -143,22 +143,25 @@ class ChangeRecord: out += '\n '.join(list(diff)[2:]) out += '\n --' elif self.fieldname in img_monitor_files or '/image-files/' in self.path: - fieldname = self.fieldname - if '/image-files/' in self.path: - fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname) - out = 'Changes to %s:\n ' % fieldname - else: - if outer: - prefix = 'Changes to %s ' % self.path - out = '(%s):\n ' % self.fieldname - if self.filechanges: - out += '\n '.join(['%s' % i for i in self.filechanges]) + if self.filechanges or (self.oldvalue and self.newvalue): + fieldname = self.fieldname + if '/image-files/' in self.path: + fieldname = os.path.join('/' + self.path.split('/image-files/')[1], self.fieldname) + out = 'Changes to %s:\n ' % fieldname + else: + if outer: + prefix = 'Changes to %s ' % self.path + out = '(%s):\n ' % self.fieldname + if self.filechanges: + out += '\n '.join(['%s' % i for i in self.filechanges]) + else: + alines = self.oldvalue.splitlines() + blines = self.newvalue.splitlines() + diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='') + out += '\n '.join(list(diff)) + out += '\n --' else: - alines = self.oldvalue.splitlines() - blines = self.newvalue.splitlines() - diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='') - out += '\n '.join(list(diff)) - out += '\n --' + out = '' else: out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue) @@ -169,7 +172,7 @@ class ChangeRecord: for line in chg._str_internal(False).splitlines(): out += '\n * %s' % line - return '%s%s' % (prefix, out) + return '%s%s' % (prefix, out) if out else '' class FileChange: changetype_add = 'A' @@ -508,7 +511,8 @@ def compare_siglists(a_blob, b_blob, taskdiff=False): return '\n'.join(out) -def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, sigs=False, sigsdiff=False): +def process_changes(repopath, revision1, revision2='HEAD', report_all=False, report_ver=False, + sigs=False, sigsdiff=False, exclude_path=None): repo = git.Repo(repopath) assert repo.bare == False commit = repo.commit(revision1) @@ -601,6 +605,19 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: chg.related.append(chg2) + # filter out unwanted paths + if exclude_path: + for chg in changes: + if chg.filechanges: + fchgs = [] + for fchg in chg.filechanges: + for epath in exclude_path: + if fchg.path.startswith(epath): + break + else: + fchgs.append(fchg) + chg.filechanges = fchgs + if report_all: return changes else: diff --git a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py index a372904183..ac2fae1ed1 100644 --- a/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/import-layers/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -32,6 +32,10 @@ class BuildSystem(object): corebase = os.path.abspath(self.d.getVar('COREBASE')) layers.append(corebase) + # The bitbake build system uses the meta-skeleton layer as a layout + # for common recipies, e.g: the recipetool script to create kernel recipies + # Add the meta-skeleton layer to be included as part of the eSDK installation + layers.append(os.path.join(corebase, 'meta-skeleton')) # Exclude layers for layer_exclude in self.layers_exclude: @@ -71,6 +75,11 @@ class BuildSystem(object): layerdestpath = destdir if corebase == os.path.dirname(layer): layerdestpath += '/' + os.path.basename(corebase) + else: + layer_relative = os.path.basename(corebase) + '/' + os.path.relpath(layer, corebase) + if os.path.dirname(layer_relative) != layernewname: + layerdestpath += '/' + os.path.dirname(layer_relative) + layerdestpath += '/' + layernewname layer_relative = os.path.relpath(layerdestpath, @@ -123,6 +132,14 @@ class BuildSystem(object): line = line.replace('workspacelayer', workspace_newname) f.write(line) + # meta-skeleton layer is added as part of the build system + # but not as a layer included in the build, therefore it is + # not reported to the function caller. + for layer in layers_copied: + if layer.endswith('/meta-skeleton'): + layers_copied.remove(layer) + break + return layers_copied def generate_locked_sigs(sigfile, d): @@ -239,6 +256,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) env = dict(d.getVar('BB_ORIGENV', False)) env.pop('BUILDDIR', '') + env.pop('BBPATH', '') pathitems = env['PATH'].split(':') env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') diff --git a/import-layers/yocto-poky/meta/lib/oe/distro_check.py b/import-layers/yocto-poky/meta/lib/oe/distro_check.py index 37f04ed359..e775c3a6ec 100644 --- a/import-layers/yocto-poky/meta/lib/oe/distro_check.py +++ b/import-layers/yocto-poky/meta/lib/oe/distro_check.py @@ -77,17 +77,10 @@ def get_latest_released_fedora_source_package_list(d): def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) - package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/%s/src/" % latest, "updates", d) - return latest, package_names - -def get_latest_released_mandriva_source_package_list(d): - "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) - package_names |= get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) return latest, package_names def get_latest_released_clear_source_package_list(d): @@ -161,8 +154,7 @@ def create_distro_packages_list(distro_check_dir, d): ("Debian", get_latest_released_debian_source_package_list), ("Ubuntu", get_latest_released_ubuntu_source_package_list), ("Fedora", get_latest_released_fedora_source_package_list), - ("OpenSuSE", get_latest_released_opensuse_source_package_list), - ("Mandriva", get_latest_released_mandriva_source_package_list), + ("openSUSE", get_latest_released_opensuse_source_package_list), ("Clear", get_latest_released_clear_source_package_list), ) diff --git a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py index 7ce767ee0a..9cc88f020c 100644 --- a/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py +++ b/import-layers/yocto-poky/meta/lib/oe/gpg_sign.py @@ -15,7 +15,7 @@ class LocalSigner(object): def export_pubkey(self, output_file, keyid, armor=True): """Export GPG public key to a file""" - cmd = '%s --batch --yes --export -o %s ' % \ + cmd = '%s --no-permission-warning --batch --yes --export -o %s ' % \ (self.gpg_bin, output_file) if self.gpg_path: cmd += "--homedir %s " % self.gpg_path @@ -27,22 +27,27 @@ class LocalSigner(object): raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % (keyid, output)) - def sign_rpms(self, files, keyid, passphrase): + def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None): """Sign RPM files""" cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid - gpg_args = '--batch --passphrase=%s' % passphrase + gpg_args = '--no-permission-warning --batch --passphrase=%s' % passphrase if self.gpg_version > (2,1,): gpg_args += ' --pinentry-mode=loopback' cmd += "--define '_gpg_sign_cmd_extra_args %s' " % gpg_args + cmd += "--define '_binary_filedigest_algorithm %s' " % digest if self.gpg_bin: - cmd += "--define '%%__gpg %s' " % self.gpg_bin + cmd += "--define '__gpg %s' " % self.gpg_bin if self.gpg_path: cmd += "--define '_gpg_path %s' " % self.gpg_path - - # Sign in chunks of 100 packages - for i in range(0, len(files), 100): - status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+100])) + if fsk: + cmd += "--signfiles --fskpath %s " % fsk + if fsk_password: + cmd += "--define '_file_signing_key_password %s' " % fsk_password + + # Sign in chunks + for i in range(0, len(files), sign_chunk): + status, output = oe.utils.getstatusoutput(cmd + ' '.join(files[i:i+sign_chunk])) if status: raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) @@ -53,8 +58,8 @@ class LocalSigner(object): if passphrase_file and passphrase: raise Exception("You should use either passphrase_file of passphrase, not both") - cmd = [self.gpg_bin, '--detach-sign', '--batch', '--no-tty', '--yes', - '--passphrase-fd', '0', '-u', keyid] + cmd = [self.gpg_bin, '--detach-sign', '--no-permission-warning', '--batch', + '--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid] if self.gpg_path: cmd += ['--homedir', self.gpg_path] @@ -93,7 +98,7 @@ class LocalSigner(object): """Return the gpg version as a tuple of ints""" import subprocess try: - ver_str = subprocess.check_output((self.gpg_bin, "--version")).split()[2].decode("utf-8") + ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8") return tuple([int(i) for i in ver_str.split('.')]) except subprocess.CalledProcessError as e: raise bb.build.FuncFailed("Could not get gpg version: %s" % e) @@ -101,7 +106,7 @@ class LocalSigner(object): def verify(self, sig_file): """Verify signature""" - cmd = self.gpg_bin + " --verify " + cmd = self.gpg_bin + " --verify --no-permission-warning " if self.gpg_path: cmd += "--homedir %s " % self.gpg_path cmd += sig_file diff --git a/import-layers/yocto-poky/meta/lib/oe/license.py b/import-layers/yocto-poky/meta/lib/oe/license.py index 8d2fd1709c..ca385d5187 100644 --- a/import-layers/yocto-poky/meta/lib/oe/license.py +++ b/import-layers/yocto-poky/meta/lib/oe/license.py @@ -106,7 +106,8 @@ def is_included(licensestr, whitelist=None, blacklist=None): license string matches the whitelist and does not match the blacklist. Returns a tuple holding the boolean state and a list of the applicable - licenses which were excluded (or None, if the state is True) + licenses that were excluded if state is False, or the licenses that were + included if the state is True. """ def include_license(license): @@ -117,10 +118,17 @@ def is_included(licensestr, whitelist=None, blacklist=None): def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most - included licenses).""" - alpha_weight = len(list(filter(include_license, alpha))) - beta_weight = len(list(filter(include_license, beta))) - if alpha_weight > beta_weight: + included licenses and no excluded licenses).""" + # The factor 1000 below is arbitrary, just expected to be much larger + # that the number of licenses actually specified. That way the weight + # will be negative if the list of licenses contains an excluded license, + # but still gives a higher weight to the list with the most included + # licenses. + alpha_weight = (len(list(filter(include_license, alpha))) - + 1000 * (len(list(filter(exclude_license, alpha))) > 0)) + beta_weight = (len(list(filter(include_license, beta))) - + 1000 * (len(list(filter(exclude_license, beta))) > 0)) + if alpha_weight >= beta_weight: return alpha else: return beta diff --git a/import-layers/yocto-poky/meta/lib/oe/lsb.py b/import-layers/yocto-poky/meta/lib/oe/lsb.py index 3a945e0fce..71c0992c5d 100644 --- a/import-layers/yocto-poky/meta/lib/oe/lsb.py +++ b/import-layers/yocto-poky/meta/lib/oe/lsb.py @@ -1,19 +1,26 @@ +def get_os_release(): + """Get all key-value pairs from /etc/os-release as a dict""" + from collections import OrderedDict + + data = OrderedDict() + if os.path.exists('/etc/os-release'): + with open('/etc/os-release') as f: + for line in f: + try: + key, val = line.rstrip().split('=', 1) + except ValueError: + continue + data[key.strip()] = val.strip('"') + return data + def release_dict_osr(): """ Populate a dict with pertinent values from /etc/os-release """ - if not os.path.exists('/etc/os-release'): - return None - data = {} - with open('/etc/os-release') as f: - for line in f: - try: - key, val = line.rstrip().split('=', 1) - except ValueError: - continue - if key == 'ID': - data['DISTRIB_ID'] = val.strip('"') - if key == 'VERSION_ID': - data['DISTRIB_RELEASE'] = val.strip('"') + os_release = get_os_release() + if 'ID' in os_release: + data['DISTRIB_ID'] = os_release['ID'] + if 'VERSION_ID' in os_release: + data['DISTRIB_RELEASE'] = os_release['VERSION_ID'] return data diff --git a/import-layers/yocto-poky/meta/lib/oe/package.py b/import-layers/yocto-poky/meta/lib/oe/package.py index 4797e7d65a..1e5c3aa8e1 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package.py +++ b/import-layers/yocto-poky/meta/lib/oe/package.py @@ -45,6 +45,115 @@ def runstrip(arg): return +def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False): + """ + Strip executable code (like executables, shared libraries) _in_place_ + - Based on sysroot_strip in staging.bbclass + :param dstdir: directory in which to strip files + :param strip_cmd: Strip command (usually ${STRIP}) + :param libdir: ${libdir} - strip .so files in this directory + :param base_libdir: ${base_libdir} - strip .so files in this directory + :param qa_already_stripped: Set to True if already-stripped' in ${INSANE_SKIP} + This is for proper logging and messages only. + """ + import stat, errno, oe.path, oe.utils, mmap + + # Detect .ko module by searching for "vermagic=" string + def is_kernel_module(path): + with open(path) as f: + return mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ).find(b"vermagic=") >= 0 + + # Return type (bits): + # 0 - not elf + # 1 - ELF + # 2 - stripped + # 4 - executable + # 8 - shared library + # 16 - kernel module + def is_elf(path): + exec_type = 0 + ret, result = oe.utils.getstatusoutput( + "file \"%s\"" % path.replace("\"", "\\\"")) + + if ret: + bb.error("split_and_strip_files: 'file %s' failed" % path) + return exec_type + + if "ELF" in result: + exec_type |= 1 + if "not stripped" not in result: + exec_type |= 2 + if "executable" in result: + exec_type |= 4 + if "shared" in result: + exec_type |= 8 + if "relocatable" in result and is_kernel_module(path): + exec_type |= 16 + return exec_type + + elffiles = {} + inodes = {} + libdir = os.path.abspath(dstdir + os.sep + libdir) + base_libdir = os.path.abspath(dstdir + os.sep + base_libdir) + exec_mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + # + # First lets figure out all of the files we may have to process + # + for root, dirs, files in os.walk(dstdir): + for f in files: + file = os.path.join(root, f) + + try: + ltarget = oe.path.realpath(file, dstdir, False) + s = os.lstat(ltarget) + except OSError as e: + (err, strerror) = e.args + if err != errno.ENOENT: + raise + # Skip broken symlinks + continue + if not s: + continue + # Check its an excutable + if s[stat.ST_MODE] & exec_mask \ + or ((file.startswith(libdir) or file.startswith(base_libdir)) and ".so" in f) \ + or file.endswith('.ko'): + # If it's a symlink, and points to an ELF file, we capture the readlink target + if os.path.islink(file): + continue + + # It's a file (or hardlink), not a link + # ...but is it ELF, and is it already stripped? + elf_file = is_elf(file) + if elf_file & 1: + if elf_file & 2: + if qa_already_stripped: + bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dstdir):], pn)) + else: + bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn)) + continue + + if s.st_ino in inodes: + os.unlink(file) + os.link(inodes[s.st_ino], file) + else: + # break hardlinks so that we do not strip the original. + inodes[s.st_ino] = file + bb.utils.copyfile(file, file) + elffiles[file] = elf_file + + # + # Now strip them (in parallel) + # + sfiles = [] + for file in elffiles: + elf_file = int(elffiles[file]) + sfiles.append((file, elf_file, strip_cmd)) + + oe.utils.multiprocess_exec(sfiles, runstrip) + + + def file_translate(file): ft = file.replace("@", "@at@") ft = ft.replace(" ", "@space@") @@ -67,8 +176,7 @@ def filedeprunner(arg): def process_deps(pipe, pkg, pkgdest, provides, requires): file = None - for line in pipe: - line = line.decode("utf-8") + for line in pipe.split("\n"): m = file_re.match(line) if m: @@ -117,12 +225,8 @@ def filedeprunner(arg): return provides, requires - try: - dep_popen = subprocess.Popen(shlex.split(rpmdeps) + pkgfiles, stdout=subprocess.PIPE) - provides, requires = process_deps(dep_popen.stdout, pkg, pkgdest, provides, requires) - except OSError as e: - bb.error("rpmdeps: '%s' command failed, '%s'" % (shlex.split(rpmdeps) + pkgfiles, e)) - raise e + output = subprocess.check_output(shlex.split(rpmdeps) + pkgfiles, stderr=subprocess.STDOUT).decode("utf-8") + provides, requires = process_deps(output, pkg, pkgdest, provides, requires) return (pkg, provides, requires) diff --git a/import-layers/yocto-poky/meta/lib/oe/package_manager.py b/import-layers/yocto-poky/meta/lib/oe/package_manager.py index 3a2daadafa..0c5d907ff1 100644 --- a/import-layers/yocto-poky/meta/lib/oe/package_manager.py +++ b/import-layers/yocto-poky/meta/lib/oe/package_manager.py @@ -17,18 +17,11 @@ from oe.gpg_sign import get_signer def create_index(arg): index_cmd = arg - try: - bb.note("Executing '%s' ..." % index_cmd) - result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") - except subprocess.CalledProcessError as e: - return("Index creation command '%s' failed with return code %d:\n%s" % - (e.cmd, e.returncode, e.output.decode("utf-8"))) - + bb.note("Executing '%s' ..." % index_cmd) + result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") if result: bb.note(result) - return None - """ This method parse the output from the package managerand return a dictionary with the information of the packages. This is used @@ -104,13 +97,25 @@ class Indexer(object, metaclass=ABCMeta): class RpmIndexer(Indexer): def write_index(self): if self.d.getVar('PACKAGE_FEED_SIGN') == '1': - raise NotImplementedError('Package feed signing not yet implementd for rpm') + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) + else: + signer = None createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c") result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir)) if result: bb.fatal(result) + # Sign repomd + if signer: + sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') + is_ascii_sig = (sig_type.upper() != "BIN") + signer.detach_sign(os.path.join(self.deploy_dir, 'repodata', 'repomd.xml'), + self.d.getVar('PACKAGE_FEED_GPG_NAME'), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), + armor=is_ascii_sig) + + class OpkgIndexer(Indexer): def write_index(self): arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", @@ -152,9 +157,7 @@ class OpkgIndexer(Indexer): bb.note("There are no packages in %s!" % self.deploy_dir) return - result = oe.utils.multiprocess_exec(index_cmds, create_index) - if result: - bb.fatal('%s' % ('\n'.join(result))) + oe.utils.multiprocess_exec(index_cmds, create_index) if signer: feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') @@ -220,7 +223,7 @@ class DpkgIndexer(Indexer): cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive) - cmd += "%s -fc Packages > Packages.gz;" % gzip + cmd += "%s -fcn Packages > Packages.gz;" % gzip with open(os.path.join(arch_dir, "Release"), "w+") as release: release.write("Label: %s\n" % arch) @@ -235,9 +238,7 @@ class DpkgIndexer(Indexer): bb.note("There are no packages in %s" % self.deploy_dir) return - result = oe.utils.multiprocess_exec(index_cmds, create_index) - if result: - bb.fatal('%s' % ('\n'.join(result))) + oe.utils.multiprocess_exec(index_cmds, create_index) if self.d.getVar('PACKAGE_FEED_SIGN') == '1': raise NotImplementedError('Package feed signing not implementd for dpkg') @@ -548,6 +549,14 @@ class RpmPM(PackageManager): if feed_uris == "": return + gpg_opts = '' + if self.d.getVar('PACKAGE_FEED_SIGN') == '1': + gpg_opts += 'repo_gpgcheck=1\n' + gpg_opts += 'gpgkey=file://%s/pki/packagefeed-gpg/PACKAGEFEED-GPG-KEY-%s-%s\n' % (self.d.getVar('sysconfdir'), self.d.getVar('DISTRO'), self.d.getVar('DISTRO_CODENAME')) + + if self.d.getVar('RPM_SIGN_PACKAGES') == '0': + gpg_opts += 'gpgcheck=0\n' + bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d")) remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) for uri in remote_uris: @@ -558,12 +567,12 @@ class RpmPM(PackageManager): repo_id = "oe-remote-repo" + "-".join(urlparse(repo_uri).path.split("/")) repo_name = "OE Remote Repo:" + " ".join(urlparse(repo_uri).path.split("/")) open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'a').write( - "[%s]\nname=%s\nbaseurl=%s\n\n" % (repo_id, repo_name, repo_uri)) + "[%s]\nname=%s\nbaseurl=%s\n%s\n" % (repo_id, repo_name, repo_uri, gpg_opts)) else: repo_name = "OE Remote Repo:" + " ".join(urlparse(uri).path.split("/")) repo_uri = uri open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_base + ".repo"), 'w').write( - "[%s]\nname=%s\nbaseurl=%s\n" % (repo_base, repo_name, repo_uri)) + "[%s]\nname=%s\nbaseurl=%s\n%s" % (repo_base, repo_name, repo_uri, gpg_opts)) def _prepare_pkg_transaction(self): os.environ['D'] = self.target_rootfs @@ -608,10 +617,12 @@ class RpmPM(PackageManager): self._invoke_dnf(["remove"] + pkgs) else: cmd = bb.utils.which(os.getenv('PATH'), "rpm") - args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs] + args = ["-e", "-v", "--nodeps", "--root=%s" %self.target_rootfs] try: + bb.note("Running %s" % ' '.join([cmd] + args + pkgs)) output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8") + bb.note(output) except subprocess.CalledProcessError as e: bb.fatal("Could not invoke rpm. Command " "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8"))) @@ -682,7 +693,7 @@ class RpmPM(PackageManager): return packages def update(self): - self._invoke_dnf(["makecache"]) + self._invoke_dnf(["makecache", "--refresh"]) def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ): os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs @@ -1145,7 +1156,7 @@ class OpkgPM(OpkgDpkgPM): # Create an temp dir as opkg root for dummy installation temp_rootfs = self.d.expand('${T}/opkg') - opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True) + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') if opkg_lib_dir[0] == "/": opkg_lib_dir = opkg_lib_dir[1:] temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') diff --git a/import-layers/yocto-poky/meta/lib/oe/patch.py b/import-layers/yocto-poky/meta/lib/oe/patch.py index f1ab3dd809..584bf6c05f 100644 --- a/import-layers/yocto-poky/meta/lib/oe/patch.py +++ b/import-layers/yocto-poky/meta/lib/oe/patch.py @@ -1,4 +1,5 @@ import oe.path +import oe.types class NotFoundError(bb.BBHandledException): def __init__(self, path): diff --git a/import-layers/yocto-poky/meta/lib/oe/path.py b/import-layers/yocto-poky/meta/lib/oe/path.py index 448a2b944e..1ea03d5d56 100644 --- a/import-layers/yocto-poky/meta/lib/oe/path.py +++ b/import-layers/yocto-poky/meta/lib/oe/path.py @@ -98,7 +98,7 @@ def copyhardlinktree(src, dst): if (os.stat(src).st_dev == os.stat(dst).st_dev): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time - cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, src, dst) + cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) source = '' if os.path.isdir(src): diff --git a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py index a7fdd36e40..cab8e40152 100644 --- a/import-layers/yocto-poky/meta/lib/oe/recipeutils.py +++ b/import-layers/yocto-poky/meta/lib/oe/recipeutils.py @@ -2,7 +2,7 @@ # # Some code borrowed from the OE layer index # -# Copyright (C) 2013-2016 Intel Corporation +# Copyright (C) 2013-2017 Intel Corporation # import sys @@ -188,6 +188,11 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True): for wrapline in wrapped[:-1]: addlines.append('%s \\%s' % (wrapline, newline)) addlines.append('%s%s' % (wrapped[-1], newline)) + + # Split on newlines - this isn't strictly necessary if you are only + # going to write the output to disk, but if you want to compare it + # (as patch_recipe_file() will do if patch=True) then it's important. + addlines = [line for l in addlines for line in l.splitlines(True)] if rewindcomments: # Ensure we insert the lines before any leading comments # (that we'd want to ensure remain leading the next value) @@ -320,7 +325,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath=''): -def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): +def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False): """Copy (local) recipe files, including both files included via include/require, and files referred to in the SRC_URI variable.""" import bb.fetch2 @@ -328,18 +333,41 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): # FIXME need a warning if the unexpanded SRC_URI value contains variable references - uris = (d.getVar('SRC_URI') or "").split() - fetch = bb.fetch2.Fetch(uris, d) - if download: - fetch.download() + uri_values = [] + localpaths = [] + def fetch_urls(rdata): + # Collect the local paths from SRC_URI + srcuri = rdata.getVar('SRC_URI') or "" + if srcuri not in uri_values: + fetch = bb.fetch2.Fetch(srcuri.split(), rdata) + if download: + fetch.download() + for pth in fetch.localpaths(): + if pth not in localpaths: + localpaths.append(pth) + uri_values.append(srcuri) + + fetch_urls(d) + if all_variants: + # Get files for other variants e.g. in the case of a SRC_URI_append + localdata = bb.data.createCopy(d) + variants = (localdata.getVar('BBCLASSEXTEND') or '').split() + if variants: + # Ensure we handle class-target if we're dealing with one of the variants + variants.append('target') + for variant in variants: + localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant) + fetch_urls(localdata) # Copy local files to target directory and gather any remote files - bb_dir = os.path.dirname(d.getVar('FILE')) + os.sep + bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep remotes = [] copied = [] - includes = [path for path in d.getVar('BBINCLUDED').split() if - path.startswith(bb_dir) and os.path.exists(path)] - for path in fetch.localpaths() + includes: + # Need to do this in two steps since we want to check against the absolute path + includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)] + # We also check this below, but we don't want any items in this list being considered remotes + includes = [path for path in includes if path.startswith(bb_dir)] + for path in localpaths + includes: # Only import files that are under the meta directory if path.startswith(bb_dir): if not whole_dir: @@ -778,7 +806,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, def find_layerdir(fn): """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" - pth = fn + pth = os.path.abspath(fn) layerdir = '' while pth: if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): diff --git a/import-layers/yocto-poky/meta/lib/oe/rootfs.py b/import-layers/yocto-poky/meta/lib/oe/rootfs.py index 96591f370e..754ef563ab 100644 --- a/import-layers/yocto-poky/meta/lib/oe/rootfs.py +++ b/import-layers/yocto-poky/meta/lib/oe/rootfs.py @@ -261,15 +261,22 @@ class Rootfs(object, metaclass=ABCMeta): # Remove components that we don't need if it's a read-only rootfs unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() pkgs_installed = image_list_installed_packages(self.d) - # Make sure update-alternatives is last on the command line, so - # that it is removed last. This makes sure that its database is - # available while uninstalling packages, allowing alternative - # symlinks of packages to be uninstalled to be managed correctly. + # Make sure update-alternatives is removed last. This is + # because its database has to available while uninstalling + # other packages, allowing alternative symlinks of packages + # to be uninstalled or to be managed correctly otherwise. provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) + # update-alternatives provider is removed in its own remove() + # call because all package managers do not guarantee the packages + # are removed in the order they given in the list (which is + # passed to the command line). The sorting done earlier is + # utilized to implement the 2-stage removal. + if len(pkgs_to_remove) > 1: + self.pm.remove(pkgs_to_remove[:-1], False) if len(pkgs_to_remove) > 0: - self.pm.remove(pkgs_to_remove, False) + self.pm.remove([pkgs_to_remove[-1]], False) if delayed_postinsts: self._save_postinsts() @@ -302,10 +309,11 @@ class Rootfs(object, metaclass=ABCMeta): bb.note("> Executing %s intercept ..." % script) try: - subprocess.check_output(script_full) + output = subprocess.check_output(script_full, stderr=subprocess.STDOUT) + if output: bb.note(output.decode("utf-8")) except subprocess.CalledProcessError as e: - bb.warn("The postinstall intercept hook '%s' failed (exit code: %d)! See log for details! (Output: %s)" % - (script, e.returncode, e.output)) + bb.warn("The postinstall intercept hook '%s' failed, details in log.do_rootfs" % script) + bb.note("Exit code %d. Output:\n%s" % (e.returncode, e.output.decode("utf-8"))) with open(script_full) as intercept: registered_pkgs = None @@ -524,7 +532,8 @@ class RpmRootfs(Rootfs): self.pm.save_rpmpostinst(pkg) def _cleanup(self): - pass + self.pm._invoke_dnf(["clean", "all"]) + class DpkgOpkgRootfs(Rootfs): def __init__(self, d, progress_reporter=None, logcatcher=None): diff --git a/import-layers/yocto-poky/meta/lib/oe/sdk.py b/import-layers/yocto-poky/meta/lib/oe/sdk.py index 9fe1687ac3..a3a6c39172 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sdk.py +++ b/import-layers/yocto-poky/meta/lib/oe/sdk.py @@ -152,6 +152,8 @@ class RpmSdk(Sdk): pm.install(pkgs_attempt, True) def _populate(self): + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND")) + bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) @@ -233,6 +235,8 @@ class OpkgSdk(Sdk): [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) def _populate(self): + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND")) + bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) @@ -315,6 +319,8 @@ class DpkgSdk(Sdk): [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) def _populate(self): + execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND")) + bb.note("Installing TARGET packages") self._populate_sysroot(self.target_pm, self.target_manifest) @@ -379,5 +385,24 @@ def populate_sdk(d, manifest_dir=None): os.environ.clear() os.environ.update(env_bkp) +def get_extra_sdkinfo(sstate_dir): + """ + This function is going to be used for generating the target and host manifest files packages of eSDK. + """ + import math + + extra_info = {} + extra_info['tasksizes'] = {} + extra_info['filesizes'] = {} + for root, _, files in os.walk(sstate_dir): + for fn in files: + if fn.endswith('.tgz'): + fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024)) + task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0] + origtotal = extra_info['tasksizes'].get(task, 0) + extra_info['tasksizes'][task] = origtotal + fsize + extra_info['filesizes'][fn] = fsize + return extra_info + if __name__ == "__main__": pass diff --git a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py index b8dd4c869e..3a8778eae0 100644 --- a/import-layers/yocto-poky/meta/lib/oe/sstatesig.py +++ b/import-layers/yocto-poky/meta/lib/oe/sstatesig.py @@ -29,7 +29,7 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): return True # Quilt (patch application) changing isn't likely to affect anything - excludelist = ['quilt-native', 'subversion-native', 'git-native'] + excludelist = ['quilt-native', 'subversion-native', 'git-native', 'ccache-native'] if depname in excludelist and recipename != depname: return False @@ -320,7 +320,7 @@ def find_siginfo(pn, taskname, taskhashlist, d): if not taskhashlist or (len(filedates) < 2 and not foundall): # That didn't work, look in sstate-cache - hashes = taskhashlist or ['*'] + hashes = taskhashlist or ['?' * 32] localdata = bb.data.createCopy(d) for hashval in hashes: localdata.setVar('PACKAGE_ARCH', '*') diff --git a/import-layers/yocto-poky/meta/lib/oe/terminal.py b/import-layers/yocto-poky/meta/lib/oe/terminal.py index 2f18ec0287..94afe394ed 100644 --- a/import-layers/yocto-poky/meta/lib/oe/terminal.py +++ b/import-layers/yocto-poky/meta/lib/oe/terminal.py @@ -292,6 +292,8 @@ def check_terminal_version(terminalName): vernum = ver.split(' ')[-1] if ver.startswith('GNOME Terminal'): vernum = ver.split(' ')[-1] + if ver.startswith('MATE Terminal'): + vernum = ver.split(' ')[-1] if ver.startswith('tmux'): vernum = ver.split()[-1] return vernum diff --git a/import-layers/yocto-poky/meta/lib/oe/useradd.py b/import-layers/yocto-poky/meta/lib/oe/useradd.py new file mode 100644 index 0000000000..179ac76b5e --- /dev/null +++ b/import-layers/yocto-poky/meta/lib/oe/useradd.py @@ -0,0 +1,68 @@ +import argparse +import re + +class myArgumentParser(argparse.ArgumentParser): + def _print_message(self, message, file=None): + bb.warn("%s - %s: %s" % (d.getVar('PN'), pkg, message)) + + # This should never be called... + def exit(self, status=0, message=None): + message = message or ("%s - %s: useradd.bbclass: Argument parsing exited" % (d.getVar('PN'), pkg)) + error(message) + + def error(self, message): + raise bb.build.FuncFailed(message) + +def split_commands(params): + params = re.split('''[ \t]*;[ \t]*(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip()) + # Remove any empty items + return [x for x in params if x] + +def split_args(params): + params = re.split('''[ \t]+(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''', params.strip()) + # Remove any empty items + return [x for x in params if x] + +def build_useradd_parser(): + # The following comes from --help on useradd from shadow + parser = myArgumentParser(prog='useradd') + parser.add_argument("-b", "--base-dir", metavar="BASE_DIR", help="base directory for the home directory of the new account") + parser.add_argument("-c", "--comment", metavar="COMMENT", help="GECOS field of the new account") + parser.add_argument("-d", "--home-dir", metavar="HOME_DIR", help="home directory of the new account") + parser.add_argument("-D", "--defaults", help="print or change default useradd configuration", action="store_true") + parser.add_argument("-e", "--expiredate", metavar="EXPIRE_DATE", help="expiration date of the new account") + parser.add_argument("-f", "--inactive", metavar="INACTIVE", help="password inactivity period of the new account") + parser.add_argument("-g", "--gid", metavar="GROUP", help="name or ID of the primary group of the new account") + parser.add_argument("-G", "--groups", metavar="GROUPS", help="list of supplementary groups of the new account") + parser.add_argument("-k", "--skel", metavar="SKEL_DIR", help="use this alternative skeleton directory") + parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults") + parser.add_argument("-l", "--no-log-init", help="do not add the user to the lastlog and faillog databases", action="store_true") + parser.add_argument("-m", "--create-home", help="create the user's home directory", action="store_const", const=True) + parser.add_argument("-M", "--no-create-home", dest="create_home", help="do not create the user's home directory", action="store_const", const=False) + parser.add_argument("-N", "--no-user-group", dest="user_group", help="do not create a group with the same name as the user", action="store_const", const=False) + parser.add_argument("-o", "--non-unique", help="allow to create users with duplicate (non-unique UID)", action="store_true") + parser.add_argument("-p", "--password", metavar="PASSWORD", help="encrypted password of the new account") + parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new account") + parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into") + parser.add_argument("-r", "--system", help="create a system account", action="store_true") + parser.add_argument("-s", "--shell", metavar="SHELL", help="login shell of the new account") + parser.add_argument("-u", "--uid", metavar="UID", help="user ID of the new account") + parser.add_argument("-U", "--user-group", help="create a group with the same name as the user", action="store_const", const=True) + parser.add_argument("LOGIN", help="Login name of the new user") + + return parser + +def build_groupadd_parser(): + # The following comes from --help on groupadd from shadow + parser = myArgumentParser(prog='groupadd') + parser.add_argument("-f", "--force", help="exit successfully if the group already exists, and cancel -g if the GID is already used", action="store_true") + parser.add_argument("-g", "--gid", metavar="GID", help="use GID for the new group") + parser.add_argument("-K", "--key", metavar="KEY=VALUE", help="override /etc/login.defs defaults") + parser.add_argument("-o", "--non-unique", help="allow to create groups with duplicate (non-unique) GID", action="store_true") + parser.add_argument("-p", "--password", metavar="PASSWORD", help="use this encrypted password for the new group") + parser.add_argument("-P", "--clear-password", metavar="CLEAR_PASSWORD", help="use this clear password for the new group") + parser.add_argument("-R", "--root", metavar="CHROOT_DIR", help="directory to chroot into") + parser.add_argument("-r", "--system", help="create a system account", action="store_true") + parser.add_argument("GROUP", help="Group name of the new group") + + return parser diff --git a/import-layers/yocto-poky/meta/lib/oe/utils.py b/import-layers/yocto-poky/meta/lib/oe/utils.py index 330a5ff94a..643ab78df7 100644 --- a/import-layers/yocto-poky/meta/lib/oe/utils.py +++ b/import-layers/yocto-poky/meta/lib/oe/utils.py @@ -126,6 +126,46 @@ def features_backfill(var,d): if addfeatures: d.appendVar(var, " " + " ".join(addfeatures)) +def all_distro_features(d, features, truevalue="1", falsevalue=""): + """ + Returns truevalue if *all* given features are set in DISTRO_FEATURES, + else falsevalue. The features can be given as single string or anything + that can be turned into a set. + + This is a shorter, more flexible version of + bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d). + + Without explicit true/false values it can be used directly where + Python expects a boolean: + if oe.utils.all_distro_features(d, "foo bar"): + bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES") + + With just a truevalue, it can be used to include files that are meant to be + used only when requested via DISTRO_FEATURES: + require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc") + """ + return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d) + +def any_distro_features(d, features, truevalue="1", falsevalue=""): + """ + Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES, + else falsevalue. The features can be given as single string or anything + that can be turned into a set. + + This is a shorter, more flexible version of + bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d). + + Without explicit true/false values it can be used directly where + Python expects a boolean: + if not oe.utils.any_distro_features(d, "foo bar"): + bb.fatal("foo, bar or both must be set in DISTRO_FEATURES") + + With just a truevalue, it can be used to include files that are meant to be + used only when requested via DISTRO_FEATURES: + require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc") + + """ + return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d) def packages_filter_out_system(d): """ @@ -184,25 +224,30 @@ def multiprocess_exec(commands, function): def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) + fails = [] + + def failures(res): + fails.append(res) + nproc = min(multiprocessing.cpu_count(), len(commands)) pool = bb.utils.multiprocessingpool(nproc, init_worker) - imap = pool.imap(function, commands) try: - res = list(imap) + mapresult = pool.map_async(function, commands, error_callback=failures) + pool.close() pool.join() - results = [] - for result in res: - if result is not None: - results.append(result) - return results - + results = mapresult.get() except KeyboardInterrupt: pool.terminate() pool.join() raise + if fails: + raise fails[0] + + return results + def squashspaces(string): import re return re.sub("\s+", " ", string).strip() |