diff options
Diffstat (limited to 'poky/meta/lib/oe')
| -rw-r--r-- | poky/meta/lib/oe/gpg_sign.py | 27 | ||||
| -rw-r--r-- | poky/meta/lib/oe/license.py | 6 | ||||
| -rw-r--r-- | poky/meta/lib/oe/package_manager/__init__.py | 2 | ||||
| -rw-r--r-- | poky/meta/lib/oe/packagedata.py | 2 | ||||
| -rw-r--r-- | poky/meta/lib/oe/qa.py | 34 | ||||
| -rw-r--r-- | poky/meta/lib/oe/reproducible.py | 84 | ||||
| -rw-r--r-- | poky/meta/lib/oe/spdx.py | 6 | ||||
| -rw-r--r-- | poky/meta/lib/oe/sstatesig.py | 29 | ||||
| -rw-r--r-- | poky/meta/lib/oe/utils.py | 3 |
9 files changed, 170 insertions, 23 deletions
diff --git a/poky/meta/lib/oe/gpg_sign.py b/poky/meta/lib/oe/gpg_sign.py index 492f096eaa..1bce6cb792 100644 --- a/poky/meta/lib/oe/gpg_sign.py +++ b/poky/meta/lib/oe/gpg_sign.py @@ -109,16 +109,33 @@ class LocalSigner(object): bb.fatal("Could not get gpg version: %s" % e) - def verify(self, sig_file): + def verify(self, sig_file, valid_sigs = ''): """Verify signature""" - cmd = self.gpg_cmd + ["--verify", "--no-permission-warning"] + cmd = self.gpg_cmd + ["--verify", "--no-permission-warning", "--status-fd", "1"] if self.gpg_path: cmd += ["--homedir", self.gpg_path] cmd += [sig_file] - status = subprocess.call(cmd) - ret = False if status else True - return ret + status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Valid if any key matches if unspecified + if not valid_sigs: + ret = False if status.returncode else True + return ret + + import re + goodsigs = [] + sigre = re.compile(r'^\[GNUPG:\] GOODSIG (\S+)\s(.*)$') + for l in status.stdout.decode("utf-8").splitlines(): + s = sigre.match(l) + if s: + goodsigs += [s.group(1)] + + for sig in valid_sigs.split(): + if sig in goodsigs: + return True + if len(goodsigs): + bb.warn('No accepted signatures found. Good signatures found: %s.' % ' '.join(goodsigs)) + return False def get_signer(d, backend): diff --git a/poky/meta/lib/oe/license.py b/poky/meta/lib/oe/license.py index 665d32ecbb..b5d378a549 100644 --- a/poky/meta/lib/oe/license.py +++ b/poky/meta/lib/oe/license.py @@ -74,6 +74,9 @@ class FlattenVisitor(LicenseVisitor): def visit_Str(self, node): self.licenses.append(node.s) + def visit_Constant(self, node): + self.licenses.append(node.value) + def visit_BinOp(self, node): if isinstance(node.op, ast.BitOr): left = FlattenVisitor(self.choose_licenses) @@ -227,6 +230,9 @@ class ListVisitor(LicenseVisitor): def visit_Str(self, node): self.licenses.add(node.s) + def visit_Constant(self, node): + self.licenses.add(node.value) + def list_licenses(licensestr): """Simply get a list of all licenses mentioned in a license string. Binary operators are not applied or taken into account in any way""" diff --git a/poky/meta/lib/oe/package_manager/__init__.py b/poky/meta/lib/oe/package_manager/__init__.py index 8f7b60e077..80bc1a6bc6 100644 --- a/poky/meta/lib/oe/package_manager/__init__.py +++ b/poky/meta/lib/oe/package_manager/__init__.py @@ -321,7 +321,7 @@ class PackageManager(object, metaclass=ABCMeta): # TODO don't have sdk here but have a property on the superclass # (and respect in install_complementary) if sdk: - pkgdatadir = self.d.expand("${TMPDIR}/pkgdata/${SDK_SYS}") + pkgdatadir = self.d.getVar("PKGDATA_DIR_SDK") else: pkgdatadir = self.d.getVar("PKGDATA_DIR") diff --git a/poky/meta/lib/oe/packagedata.py b/poky/meta/lib/oe/packagedata.py index 02c81e5a52..212f048bc6 100644 --- a/poky/meta/lib/oe/packagedata.py +++ b/poky/meta/lib/oe/packagedata.py @@ -19,7 +19,7 @@ def read_pkgdatafile(fn): import re with open(fn, 'r') as f: lines = f.readlines() - r = re.compile("(^.+?):\s+(.*)") + r = re.compile(r"(^.+?):\s+(.*)") for l in lines: m = r.match(l) if m: diff --git a/poky/meta/lib/oe/qa.py b/poky/meta/lib/oe/qa.py index e8a854a302..efab7e8564 100644 --- a/poky/meta/lib/oe/qa.py +++ b/poky/meta/lib/oe/qa.py @@ -171,6 +171,40 @@ def elf_machine_to_string(machine): except: return "Unknown (%s)" % repr(machine) +def write_error(type, error, d): + logfile = d.getVar('QA_LOGFILE') + if logfile: + p = d.getVar('P') + with open(logfile, "a+") as f: + f.write("%s: %s [%s]\n" % (p, error, type)) + +def handle_error(error_class, error_msg, d): + if error_class in (d.getVar("ERROR_QA") or "").split(): + write_error(error_class, error_msg, d) + bb.error("QA Issue: %s [%s]" % (error_msg, error_class)) + d.setVar("QA_ERRORS_FOUND", "True") + return False + elif error_class in (d.getVar("WARN_QA") or "").split(): + write_error(error_class, error_msg, d) + bb.warn("QA Issue: %s [%s]" % (error_msg, error_class)) + else: + bb.note("QA Issue: %s [%s]" % (error_msg, error_class)) + return True + +def add_message(messages, section, new_msg): + if section not in messages: + messages[section] = new_msg + else: + messages[section] = messages[section] + "\n" + new_msg + +def exit_with_message_if_errors(message, d): + qa_fatal_errors = bb.utils.to_boolean(d.getVar("QA_ERRORS_FOUND"), False) + if qa_fatal_errors: + bb.fatal(message) + +def exit_if_errors(d): + exit_with_message_if_errors("Fatal QA errors were found, failing task.", d) + if __name__ == "__main__": import sys diff --git a/poky/meta/lib/oe/reproducible.py b/poky/meta/lib/oe/reproducible.py index 204b9bd734..4fb99d963c 100644 --- a/poky/meta/lib/oe/reproducible.py +++ b/poky/meta/lib/oe/reproducible.py @@ -5,6 +5,57 @@ import os import subprocess import bb +# For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each +# component's build environment. The format is number of seconds since the +# system epoch. +# +# Upstream components (generally) respect this environment variable, +# using it in place of the "current" date and time. +# See https://reproducible-builds.org/specs/source-date-epoch/ +# +# The default value of SOURCE_DATE_EPOCH comes from the function +# get_source_date_epoch_value which reads from the SDE_FILE, or if the file +# is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. +# +# The SDE_FILE is normally constructed from the function +# create_source_date_epoch_stamp which is typically added as a postfuncs to +# the do_unpack task. If a recipe does NOT have do_unpack, it should be added +# to a task that runs after the source is available and before the +# do_deploy_source_date_epoch task is executed. +# +# If a recipe wishes to override the default behavior it should set it's own +# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task +# with recipe-specific functionality to write the appropriate +# SOURCE_DATE_EPOCH into the SDE_FILE. +# +# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should +# be reproducible for anyone who builds the same revision from the same +# sources. +# +# There are 4 ways the create_source_date_epoch_stamp function determines what +# becomes SOURCE_DATE_EPOCH: +# +# 1. Use the value from __source_date_epoch.txt file if this file exists. +# This file was most likely created in the previous build by one of the +# following methods 2,3,4. +# Alternatively, it can be provided by a recipe via SRC_URI. +# +# If the file does not exist: +# +# 2. If there is a git checkout, use the last git commit timestamp. +# Git does not preserve file timestamps on checkout. +# +# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... +# This works for well-kept repositories distributed via tarball. +# +# 4. Use the modification time of the youngest file in the source tree, if +# there is one. +# This will be the newest file from the distribution tarball, if any. +# +# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). +# +# Once the value is determined, it is stored in the recipe's SDE_FILE. + def get_source_date_epoch_from_known_files(d, sourcedir): source_date_epoch = None newest_file = None @@ -106,3 +157,36 @@ def get_source_date_epoch(d, sourcedir): fixed_source_date_epoch(d) # Last resort ) +def epochfile_read(epochfile, d): + cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None) + if cached and efile == epochfile: + return cached + + if cached and epochfile != efile: + bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile)) + + source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) + try: + with open(epochfile, 'r') as f: + s = f.read() + try: + source_date_epoch = int(s) + except ValueError: + bb.warn("SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK" % s) + source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) + bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) + except FileNotFoundError: + bb.debug(1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" % (epochfile, source_date_epoch)) + + d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile)) + return str(source_date_epoch) + +def epochfile_write(source_date_epoch, epochfile, d): + + bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) + bb.utils.mkdirhier(os.path.dirname(epochfile)) + + tmp_file = "%s.new" % epochfile + with open(tmp_file, 'w') as f: + f.write(str(source_date_epoch)) + os.rename(tmp_file, epochfile) diff --git a/poky/meta/lib/oe/spdx.py b/poky/meta/lib/oe/spdx.py index 4416194e06..9e7ced5a15 100644 --- a/poky/meta/lib/oe/spdx.py +++ b/poky/meta/lib/oe/spdx.py @@ -196,6 +196,7 @@ class SPDXRelationship(SPDXObject): relatedSpdxElement = _String() relationshipType = _String() comment = _String() + annotations = _ObjectList(SPDXAnnotation) class SPDXExternalReference(SPDXObject): @@ -300,7 +301,7 @@ class SPDXDocument(SPDXObject): def from_json(cls, f): return cls(**json.load(f)) - def add_relationship(self, _from, relationship, _to, *, comment=None): + def add_relationship(self, _from, relationship, _to, *, comment=None, annotation=None): if isinstance(_from, SPDXObject): from_spdxid = _from.SPDXID else: @@ -320,6 +321,9 @@ class SPDXDocument(SPDXObject): if comment is not None: r.comment = comment + if annotation is not None: + r.annotations.append(annotation) + self.relationships.append(r) def find_by_spdxid(self, spdxid): diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index 0c3b4589c5..038404e377 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -489,7 +489,7 @@ def OEOuthashBasic(path, sigfile, task, d): include_timestamps = False include_root = True if task == "package": - include_timestamps = d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1' + include_timestamps = True include_root = False extra_content = d.getVar('HASHEQUIV_HASH_VERSION') @@ -552,21 +552,22 @@ def OEOuthashBasic(path, sigfile, task, d): else: add_perm(stat.S_IXUSR, 'x') - add_perm(stat.S_IRGRP, 'r') - add_perm(stat.S_IWGRP, 'w') - if stat.S_ISGID & s.st_mode: - add_perm(stat.S_IXGRP, 's', 'S') - else: - add_perm(stat.S_IXGRP, 'x') + if include_owners: + # Group/other permissions are only relevant in pseudo context + add_perm(stat.S_IRGRP, 'r') + add_perm(stat.S_IWGRP, 'w') + if stat.S_ISGID & s.st_mode: + add_perm(stat.S_IXGRP, 's', 'S') + else: + add_perm(stat.S_IXGRP, 'x') - add_perm(stat.S_IROTH, 'r') - add_perm(stat.S_IWOTH, 'w') - if stat.S_ISVTX & s.st_mode: - update_hash('t') - else: - add_perm(stat.S_IXOTH, 'x') + add_perm(stat.S_IROTH, 'r') + add_perm(stat.S_IWOTH, 'w') + if stat.S_ISVTX & s.st_mode: + update_hash('t') + else: + add_perm(stat.S_IXOTH, 'x') - if include_owners: try: update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) diff --git a/poky/meta/lib/oe/utils.py b/poky/meta/lib/oe/utils.py index 238af314d1..cf65639647 100644 --- a/poky/meta/lib/oe/utils.py +++ b/poky/meta/lib/oe/utils.py @@ -508,7 +508,8 @@ class ThreadedWorker(Thread): try: func(self, *args, **kargs) except Exception as e: - print(e) + # Eat all exceptions + bb.mainlogger.debug("Worker task raised %s" % e, exc_info=e) finally: self.tasks.task_done() |