diff options
Diffstat (limited to 'poky/meta/lib')
-rw-r--r-- | poky/meta/lib/buildstats.py | 4 | ||||
-rw-r--r-- | poky/meta/lib/oe/spdx.py | 64 | ||||
-rw-r--r-- | poky/meta/lib/oe/sstatesig.py | 47 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/bbtests.py | 4 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/gotoolchain.py | 6 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/tinfoil.py | 6 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/wic.py | 29 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/utils/buildproject.py | 3 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/utils/qemurunner.py | 2 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/utils/targetbuild.py | 4 |
10 files changed, 150 insertions, 19 deletions
diff --git a/poky/meta/lib/buildstats.py b/poky/meta/lib/buildstats.py index 8627ed3c3..c52b6c3b7 100644 --- a/poky/meta/lib/buildstats.py +++ b/poky/meta/lib/buildstats.py @@ -43,8 +43,8 @@ class SystemStats: # depends on the heartbeat event, which fires less often. self.min_seconds = 1 - self.meminfo_regex = re.compile(b'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)') - self.diskstats_regex = re.compile(b'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') + self.meminfo_regex = re.compile(rb'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)') + self.diskstats_regex = re.compile(rb'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$') self.diskstats_ltime = None self.diskstats_data = None self.stat_ltimes = None diff --git a/poky/meta/lib/oe/spdx.py b/poky/meta/lib/oe/spdx.py index 9814fbfd6..4416194e0 100644 --- a/poky/meta/lib/oe/spdx.py +++ b/poky/meta/lib/oe/spdx.py @@ -2,6 +2,18 @@ # SPDX-License-Identifier: GPL-2.0-only # +# +# This library is intended to capture the JSON SPDX specification in a type +# safe manner. It is not intended to encode any particular OE specific +# behaviors, see the sbom.py for that. +# +# The documented SPDX spec document doesn't cover the JSON syntax for +# particular configuration, which can make it hard to determine what the JSON +# syntax should be. I've found it is actually much simpler to read the official +# SPDX JSON schema which can be found here: https://github.com/spdx/spdx-spec +# in schemas/spdx-schema.json +# + import hashlib import itertools import json @@ -9,7 +21,16 @@ import json SPDX_VERSION = "2.2" +# +# The following are the support classes that are used to implement SPDX object +# + class _Property(object): + """ + A generic SPDX object property. The different types will derive from this + class + """ + def __init__(self, *, default=None): self.default = default @@ -19,6 +40,10 @@ class _Property(object): class _String(_Property): + """ + A scalar string property for an SPDX object + """ + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -39,6 +64,10 @@ class _String(_Property): class _Object(_Property): + """ + A scalar SPDX object property of a SPDX object + """ + def __init__(self, cls, **kwargs): super().__init__(**kwargs) self.cls = cls @@ -62,6 +91,10 @@ class _Object(_Property): class _ListProperty(_Property): + """ + A list of SPDX properties + """ + def __init__(self, prop, **kwargs): super().__init__(**kwargs) self.prop = prop @@ -82,16 +115,28 @@ class _ListProperty(_Property): class _StringList(_ListProperty): + """ + A list of strings as a property for an SPDX object + """ + def __init__(self, **kwargs): super().__init__(_String(), **kwargs) class _ObjectList(_ListProperty): + """ + A list of SPDX objects as a property for an SPDX object + """ + def __init__(self, cls, **kwargs): super().__init__(_Object(cls), **kwargs) class MetaSPDXObject(type): + """ + A metaclass that allows properties (anything derived from a _Property + class) to be defined for a SPDX object + """ def __new__(mcls, name, bases, attrs): attrs["_properties"] = {} @@ -105,6 +150,9 @@ class MetaSPDXObject(type): class SPDXObject(metaclass=MetaSPDXObject): + """ + The base SPDX object; all SPDX spec classes must derive from this class + """ def __init__(self, **d): self._spdx = {} @@ -122,6 +170,21 @@ class SPDXObject(metaclass=MetaSPDXObject): return raise KeyError("%r is not a valid SPDX property" % name) +# +# These are the SPDX objects implemented from the spec. The *only* properties +# that can be added to these objects are ones directly specified in the SPDX +# spec, however you may add helper functions to make operations easier. +# +# Defaults should *only* be specified if the SPDX spec says there is a certain +# required value for a field (e.g. dataLicense), or if the field is mandatory +# and has some sane "this field is unknown" (e.g. "NOASSERTION") +# + +class SPDXAnnotation(SPDXObject): + annotationDate = _String() + annotationType = _String() + annotator = _String() + comment = _String() class SPDXChecksum(SPDXObject): algorithm = _String() @@ -164,6 +227,7 @@ class SPDXPackage(SPDXObject): packageVerificationCode = _Object(SPDXPackageVerificationCode) hasFiles = _StringList() packageFileName = _String() + annotations = _ObjectList(SPDXAnnotation) class SPDXFile(SPDXObject): diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index dd6b9de7b..0c3b4589c 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -108,7 +108,6 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or "").split() self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } - self.buildarch = data.getVar('BUILD_ARCH') self._internal = False pass @@ -147,13 +146,6 @@ class SignatureGeneratorOEBasicHashMixIn(object): self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) - def prep_taskhash(self, tid, deps, dataCaches): - super().prep_taskhash(tid, deps, dataCaches) - if hasattr(self, "extramethod"): - (mc, _, _, fn) = bb.runqueue.split_tid_mcfn(tid) - inherits = " ".join(dataCaches[mc].inherits[fn]) - if inherits.find("/native.bbclass") != -1 or inherits.find("/cross.bbclass") != -1: - self.extramethod[tid] = ":" + self.buildarch def get_taskhash(self, tid, deps, dataCaches): if tid in self.lockedhashes: @@ -478,6 +470,8 @@ def OEOuthashBasic(path, sigfile, task, d): import stat import pwd import grp + import re + import fnmatch def update_hash(s): s = s.encode('utf-8') @@ -487,6 +481,8 @@ def OEOuthashBasic(path, sigfile, task, d): h = hashlib.sha256() prev_dir = os.getcwd() + corebase = d.getVar("COREBASE") + tmpdir = d.getVar("TMPDIR") include_owners = os.environ.get('PSEUDO_DISABLED') == '0' if "package_write_" in task or task == "package_qa": include_owners = False @@ -497,8 +493,17 @@ def OEOuthashBasic(path, sigfile, task, d): include_root = False extra_content = d.getVar('HASHEQUIV_HASH_VERSION') + filemaps = {} + for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split(): + entry = m.split(":") + if len(entry) != 3 or entry[0] != task: + continue + filemaps.setdefault(entry[1], []) + filemaps[entry[1]].append(entry[2]) + try: os.chdir(path) + basepath = os.path.normpath(path) update_hash("OEOuthashBasic\n") if extra_content: @@ -580,8 +585,13 @@ def OEOuthashBasic(path, sigfile, task, d): else: update_hash(" " * 9) + filterfile = False + for entry in filemaps: + if fnmatch.fnmatch(path, entry): + filterfile = True + update_hash(" ") - if stat.S_ISREG(s.st_mode): + if stat.S_ISREG(s.st_mode) and not filterfile: update_hash("%10d" % s.st_size) else: update_hash(" " * 10) @@ -590,9 +600,24 @@ def OEOuthashBasic(path, sigfile, task, d): fh = hashlib.sha256() if stat.S_ISREG(s.st_mode): # Hash file contents - with open(path, 'rb') as d: - for chunk in iter(lambda: d.read(4096), b""): + if filterfile: + # Need to ignore paths in crossscripts and postinst-useradd files. + with open(path, 'rb') as d: + chunk = d.read() + chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'') + for entry in filemaps: + if not fnmatch.fnmatch(path, entry): + continue + for r in filemaps[entry]: + if r.startswith("regex-"): + chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk) + else: + chunk = chunk.replace(bytes(r, encoding='utf8'), b'') fh.update(chunk) + else: + with open(path, 'rb') as d: + for chunk in iter(lambda: d.read(4096), b""): + fh.update(chunk) update_hash(fh.hexdigest()) else: update_hash(" " * len(fh.hexdigest())) diff --git a/poky/meta/lib/oeqa/selftest/cases/bbtests.py b/poky/meta/lib/oeqa/selftest/cases/bbtests.py index 8831de606..656236407 100644 --- a/poky/meta/lib/oeqa/selftest/cases/bbtests.py +++ b/poky/meta/lib/oeqa/selftest/cases/bbtests.py @@ -83,8 +83,10 @@ class BitbakeTests(OESelftestTestCase): def test_force_task_1(self): # test 1 from bug 5875 + import uuid test_recipe = 'zlib' - test_data = "Microsoft Made No Profit From Anyone's Zunes Yo" + # Need to use uuid otherwise hash equivlance would change the workflow + test_data = "Microsoft Made No Profit From Anyone's Zunes Yo %s" % uuid.uuid1() bb_vars = get_bb_vars(['D', 'PKGDEST', 'mandir'], test_recipe) image_dir = bb_vars['D'] pkgsplit_dir = bb_vars['PKGDEST'] diff --git a/poky/meta/lib/oeqa/selftest/cases/gotoolchain.py b/poky/meta/lib/oeqa/selftest/cases/gotoolchain.py index 4fc3605f4..c809d7c9b 100644 --- a/poky/meta/lib/oeqa/selftest/cases/gotoolchain.py +++ b/poky/meta/lib/oeqa/selftest/cases/gotoolchain.py @@ -43,6 +43,12 @@ class oeGoToolchainSelfTest(OESelftestTestCase): @classmethod def tearDownClass(cls): + # Go creates file which are readonly + for dirpath, dirnames, filenames in os.walk(cls.tmpdir_SDKQA): + for filename in filenames + dirnames: + f = os.path.join(dirpath, filename) + if not os.path.islink(f): + os.chmod(f, 0o775) shutil.rmtree(cls.tmpdir_SDKQA, ignore_errors=True) super(oeGoToolchainSelfTest, cls).tearDownClass() diff --git a/poky/meta/lib/oeqa/selftest/cases/tinfoil.py b/poky/meta/lib/oeqa/selftest/cases/tinfoil.py index 51092805d..8fd48bb05 100644 --- a/poky/meta/lib/oeqa/selftest/cases/tinfoil.py +++ b/poky/meta/lib/oeqa/selftest/cases/tinfoil.py @@ -94,14 +94,13 @@ class TinfoilTests(OESelftestTestCase): pass pattern = 'conf' - res = tinfoil.run_command('findFilesMatchingInDir', pattern, 'conf/machine') + res = tinfoil.run_command('testCookerCommandEvent', pattern) self.assertTrue(res) eventreceived = False commandcomplete = False start = time.time() # Wait for maximum 60s in total so we'd detect spurious heartbeat events for example - # The test is IO load sensitive too while (not (eventreceived == True and commandcomplete == True) and (time.time() - start < 60)): # if we received both events (on let's say a good day), we are done @@ -111,7 +110,8 @@ class TinfoilTests(OESelftestTestCase): commandcomplete = True elif isinstance(event, bb.event.FilesMatchingFound): self.assertEqual(pattern, event._pattern) - self.assertIn('qemuarm.conf', event._matches) + self.assertIn('A', event._matches) + self.assertIn('B', event._matches) eventreceived = True elif isinstance(event, logging.LogRecord): continue diff --git a/poky/meta/lib/oeqa/selftest/cases/wic.py b/poky/meta/lib/oeqa/selftest/cases/wic.py index dc7b9e637..5fc8e6514 100644 --- a/poky/meta/lib/oeqa/selftest/cases/wic.py +++ b/poky/meta/lib/oeqa/selftest/cases/wic.py @@ -1158,6 +1158,35 @@ class Wic2(WicTestCase): out = glob(self.resultdir + "%s-*.direct" % wksname) self.assertEqual(1, len(out)) + @only_for_arch(['i586', 'i686', 'x86_64']) + def test_efi_plugin_unified_kernel_image_qemu(self): + """Test efi plugin's Unified Kernel Image feature in qemu""" + config = 'IMAGE_FSTYPES = "wic"\n'\ + 'INITRAMFS_IMAGE = "core-image-minimal-initramfs"\n'\ + 'WKS_FILE = "test_efi_plugin.wks"\n'\ + 'MACHINE_FEATURES:append = " efi"\n' + self.append_config(config) + self.assertEqual(0, bitbake('core-image-minimal core-image-minimal-initramfs ovmf').status) + self.remove_config(config) + + with runqemu('core-image-minimal', ssh=False, + runqemuparams='ovmf', image_fstype='wic') as qemu: + # Check that /boot has EFI bootx64.efi (required for EFI) + cmd = "ls /boot/EFI/BOOT/bootx64.efi | wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '1') + # Check that /boot has EFI/Linux/linux.efi (required for Unified Kernel Images auto detection) + cmd = "ls /boot/EFI/Linux/linux.efi | wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '1') + # Check that /boot doesn't have loader/entries/boot.conf (Unified Kernel Images are auto detected by the bootloader) + cmd = "ls /boot/loader/entries/boot.conf 2&>/dev/null | wc -l" + status, output = qemu.run_serial(cmd) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) + self.assertEqual(output, '0') + def test_fs_types(self): """Test filesystem types for empty and not empty partitions""" img = 'core-image-minimal' diff --git a/poky/meta/lib/oeqa/utils/buildproject.py b/poky/meta/lib/oeqa/utils/buildproject.py index e6d80cc8d..dfb966186 100644 --- a/poky/meta/lib/oeqa/utils/buildproject.py +++ b/poky/meta/lib/oeqa/utils/buildproject.py @@ -18,6 +18,7 @@ class BuildProject(metaclass=ABCMeta): def __init__(self, uri, foldername=None, tmpdir=None, dl_dir=None): self.uri = uri self.archive = os.path.basename(uri) + self.tempdirobj = None if not tmpdir: self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') tmpdir = self.tempdirobj.name @@ -57,6 +58,8 @@ class BuildProject(metaclass=ABCMeta): return self._run('cd %s; make install %s' % (self.targetdir, install_args)) def clean(self): + if self.tempdirobj: + self.tempdirobj.cleanup() if not self.needclean: return self._run('rm -rf %s' % self.targetdir) diff --git a/poky/meta/lib/oeqa/utils/qemurunner.py b/poky/meta/lib/oeqa/utils/qemurunner.py index d55248c49..d961a9a21 100644 --- a/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/poky/meta/lib/oeqa/utils/qemurunner.py @@ -265,7 +265,7 @@ class QemuRunner: r = os.fdopen(r) x = r.read() os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) - sys.exit(0) + os._exit(0) self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid) self.logger.debug("waiting at most %s seconds for qemu pid (%s)" % diff --git a/poky/meta/lib/oeqa/utils/targetbuild.py b/poky/meta/lib/oeqa/utils/targetbuild.py index 1055810ca..09738add1 100644 --- a/poky/meta/lib/oeqa/utils/targetbuild.py +++ b/poky/meta/lib/oeqa/utils/targetbuild.py @@ -19,6 +19,7 @@ class BuildProject(metaclass=ABCMeta): self.d = d self.uri = uri self.archive = os.path.basename(uri) + self.tempdirobj = None if not tmpdir: tmpdir = self.d.getVar('WORKDIR') if not tmpdir: @@ -71,9 +72,10 @@ class BuildProject(metaclass=ABCMeta): return self._run('cd %s; make install %s' % (self.targetdir, install_args)) def clean(self): + if self.tempdirobj: + self.tempdirobj.cleanup() self._run('rm -rf %s' % self.targetdir) subprocess.check_call('rm -f %s' % self.localarchive, shell=True) - pass class TargetBuildProject(BuildProject): |