diff options
Diffstat (limited to 'poky/meta/lib/oeqa/selftest/cases')
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/bbtests.py | 2 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/binutils.py | 49 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/gcc.py | 139 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/glibc.py | 88 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/kerneldevelopment.py | 67 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/meta_ide.py | 2 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/reproducible.py | 80 | ||||
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/runqemu.py | 2 |
8 files changed, 399 insertions, 30 deletions
diff --git a/poky/meta/lib/oeqa/selftest/cases/bbtests.py b/poky/meta/lib/oeqa/selftest/cases/bbtests.py index 0693ba8cf8..8e59bafae3 100644 --- a/poky/meta/lib/oeqa/selftest/cases/bbtests.py +++ b/poky/meta/lib/oeqa/selftest/cases/bbtests.py @@ -119,7 +119,7 @@ class BitbakeTests(OESelftestTestCase): def test_bitbake_g(self): result = bitbake('-g core-image-minimal') - for f in ['pn-buildlist', 'recipe-depends.dot', 'task-depends.dot']: + for f in ['pn-buildlist', 'task-depends.dot']: self.addCleanup(os.remove, f) self.assertTrue('Task dependencies saved to \'task-depends.dot\'' in result.output, msg = "No task dependency \"task-depends.dot\" file was generated for the given task target. bitbake output: %s" % result.output) self.assertTrue('busybox' in ftools.read_file(os.path.join(self.builddir, 'task-depends.dot')), msg = "No \"busybox\" dependency found in task-depends.dot file.") diff --git a/poky/meta/lib/oeqa/selftest/cases/binutils.py b/poky/meta/lib/oeqa/selftest/cases/binutils.py new file mode 100644 index 0000000000..9bc752040f --- /dev/null +++ b/poky/meta/lib/oeqa/selftest/cases/binutils.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: MIT +import os +import sys +import re +import logging +from oeqa.core.decorator import OETestTag +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars + +def parse_values(content): + for i in content: + for v in ["PASS", "FAIL", "XPASS", "XFAIL", "UNRESOLVED", "UNSUPPORTED", "UNTESTED", "ERROR", "WARNING"]: + if i.startswith(v + ": "): + yield i[len(v) + 2:].strip(), v + break + +@OETestTag("toolchain-user", "toolchain-system") +class BinutilsCrossSelfTest(OESelftestTestCase): + def test_binutils(self): + self.run_binutils("binutils") + + def test_gas(self): + self.run_binutils("gas") + + def test_ld(self): + self.run_binutils("ld") + + def run_binutils(self, suite): + features = [] + features.append('CHECK_TARGETS = "{0}"'.format(suite)) + self.write_config("\n".join(features)) + + recipe = "binutils-cross-testsuite" + bb_vars = get_bb_vars(["B", "TARGET_SYS", "T"], recipe) + builddir, target_sys, tdir = bb_vars["B"], bb_vars["TARGET_SYS"], bb_vars["T"] + + bitbake("{0} -c check".format(recipe)) + + ptestsuite = "binutils-{}".format(suite) if suite != "binutils" else suite + self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} + + sumspath = os.path.join(builddir, suite, "{0}.sum".format(suite)) + if not os.path.exists(sumspath): + sumspath = os.path.join(builddir, suite, "testsuite", "{0}.sum".format(suite)) + + with open(sumspath, "r") as f: + for test, result in parse_values(f): + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + diff --git a/poky/meta/lib/oeqa/selftest/cases/gcc.py b/poky/meta/lib/oeqa/selftest/cases/gcc.py new file mode 100644 index 0000000000..2c25b5904c --- /dev/null +++ b/poky/meta/lib/oeqa/selftest/cases/gcc.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: MIT +import os +from oeqa.core.decorator import OETestTag +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command + +def parse_values(content): + for i in content: + for v in ["PASS", "FAIL", "XPASS", "XFAIL", "UNRESOLVED", "UNSUPPORTED", "UNTESTED", "ERROR", "WARNING"]: + if i.startswith(v + ": "): + yield i[len(v) + 2:].strip(), v + break + +class GccSelfTestBase(OESelftestTestCase): + def check_skip(self, suite): + targets = get_bb_var("RUNTIMETARGET", "gcc-runtime").split() + if suite not in targets: + self.skipTest("Target does not use {0}".format(suite)) + + def run_check(self, *suites, ssh = None): + targets = set() + for s in suites: + if s in ["gcc", "g++"]: + targets.add("check-gcc") + else: + targets.add("check-target-{}".format(s)) + + # configure ssh target + features = [] + features.append('MAKE_CHECK_TARGETS = "{0}"'.format(" ".join(targets))) + if ssh is not None: + features.append('TOOLCHAIN_TEST_TARGET = "ssh"') + features.append('TOOLCHAIN_TEST_HOST = "{0}"'.format(ssh)) + features.append('TOOLCHAIN_TEST_HOST_USER = "root"') + features.append('TOOLCHAIN_TEST_HOST_PORT = "22"') + self.write_config("\n".join(features)) + + recipe = "gcc-runtime" + bitbake("{} -c check".format(recipe)) + + bb_vars = get_bb_vars(["B", "TARGET_SYS"], recipe) + builddir, target_sys = bb_vars["B"], bb_vars["TARGET_SYS"] + + self.extraresults = {"ptestresult.sections" : {}} + for suite in suites: + sumspath = os.path.join(builddir, "gcc", "testsuite", suite, "{0}.sum".format(suite)) + if not os.path.exists(sumspath): # check in target dirs + sumspath = os.path.join(builddir, target_sys, suite, "testsuite", "{0}.sum".format(suite)) + if not os.path.exists(sumspath): # handle libstdc++-v3 -> libstdc++ + sumspath = os.path.join(builddir, target_sys, suite, "testsuite", "{0}.sum".format(suite.split("-")[0])) + + ptestsuite = "gcc-{}".format(suite) if suite != "gcc" else suite + ptestsuite = ptestsuite + "-user" if ssh is None else ptestsuite + self.extraresults["ptestresult.sections"][ptestsuite] = {} + with open(sumspath, "r") as f: + for test, result in parse_values(f): + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + + def run_check_emulated(self, *args, **kwargs): + # build core-image-minimal with required packages + default_installed_packages = ["libgcc", "libstdc++", "libatomic", "libgomp"] + features = [] + features.append('IMAGE_FEATURES += "ssh-server-openssh"') + features.append('CORE_IMAGE_EXTRA_INSTALL += "{0}"'.format(" ".join(default_installed_packages))) + self.write_config("\n".join(features)) + bitbake("core-image-minimal") + + # wrap the execution with a qemu instance + with runqemu("core-image-minimal", runqemuparams = "nographic") as qemu: + # validate that SSH is working + status, _ = qemu.run("uname") + self.assertEqual(status, 0) + + return self.run_check(*args, ssh=qemu.ip, **kwargs) + +@OETestTag("toolchain-user") +class GccCrossSelfTest(GccSelfTestBase): + def test_cross_gcc(self): + self.run_check("gcc", "g++") + +@OETestTag("toolchain-user") +class GccLibAtomicSelfTest(GccSelfTestBase): + def test_libatomic(self): + self.run_check("libatomic") + +@OETestTag("toolchain-user") +class GccLibGompSelfTest(GccSelfTestBase): + def test_libgomp(self): + self.run_check("libgomp") + +@OETestTag("toolchain-user") +class GccLibStdCxxSelfTest(GccSelfTestBase): + def test_libstdcxx(self): + self.run_check("libstdc++-v3") + +@OETestTag("toolchain-user") +class GccLibSspSelfTest(GccSelfTestBase): + def test_libssp(self): + self.check_skip("libssp") + self.run_check("libssp") + +@OETestTag("toolchain-user") +class GccLibItmSelfTest(GccSelfTestBase): + def test_libitm(self): + self.check_skip("libitm") + self.run_check("libitm") + +@OETestTag("toolchain-system") +class GccCrossSelfTestSystemEmulated(GccSelfTestBase): + def test_cross_gcc(self): + self.run_check_emulated("gcc", "g++") + +@OETestTag("toolchain-system") +class GccLibAtomicSelfTestSystemEmulated(GccSelfTestBase): + def test_libatomic(self): + self.run_check_emulated("libatomic") + +@OETestTag("toolchain-system") +class GccLibGompSelfTestSystemEmulated(GccSelfTestBase): + def test_libgomp(self): + self.run_check_emulated("libgomp") + +@OETestTag("toolchain-system") +class GccLibStdCxxSelfTestSystemEmulated(GccSelfTestBase): + def test_libstdcxx(self): + self.run_check_emulated("libstdc++-v3") + +@OETestTag("toolchain-system") +class GccLibSspSelfTestSystemEmulated(GccSelfTestBase): + def test_libssp(self): + self.check_skip("libssp") + self.run_check_emulated("libssp") + +@OETestTag("toolchain-system") +class GccLibItmSelfTestSystemEmulated(GccSelfTestBase): + def test_libitm(self): + self.check_skip("libitm") + self.run_check_emulated("libitm") + diff --git a/poky/meta/lib/oeqa/selftest/cases/glibc.py b/poky/meta/lib/oeqa/selftest/cases/glibc.py new file mode 100644 index 0000000000..2e42485dd6 --- /dev/null +++ b/poky/meta/lib/oeqa/selftest/cases/glibc.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: MIT +import os +import contextlib +from oeqa.core.decorator import OETestTag +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runqemu, Command +from oeqa.utils.nfs import unfs_server + +def parse_values(content): + for i in content: + for v in ["PASS", "FAIL", "XPASS", "XFAIL", "UNRESOLVED", "UNSUPPORTED", "UNTESTED", "ERROR", "WARNING"]: + if i.startswith(v + ": "): + yield i[len(v) + 2:].strip(), v + break + +class GlibcSelfTestBase(OESelftestTestCase): + def run_check(self, ssh = None): + # configure ssh target + features = [] + if ssh is not None: + features.append('TOOLCHAIN_TEST_TARGET = "ssh"') + features.append('TOOLCHAIN_TEST_HOST = "{0}"'.format(ssh)) + features.append('TOOLCHAIN_TEST_HOST_USER = "root"') + features.append('TOOLCHAIN_TEST_HOST_PORT = "22"') + # force single threaded test execution + features.append('EGLIBCPARALLELISM_task-check_pn-glibc-testsuite = "PARALLELMFLAGS="-j1""') + self.write_config("\n".join(features)) + + bitbake("glibc-testsuite -c check") + + builddir = get_bb_var("B", "glibc-testsuite") + + ptestsuite = "glibc-user" if ssh is None else "glibc" + self.extraresults = {"ptestresult.sections" : {ptestsuite : {}}} + with open(os.path.join(builddir, "tests.sum"), "r") as f: + for test, result in parse_values(f): + self.extraresults["ptestresult.{}.{}".format(ptestsuite, test)] = {"status" : result} + + def run_check_emulated(self): + with contextlib.ExitStack() as s: + # use the base work dir, as the nfs mount, since the recipe directory may not exist + tmpdir = get_bb_var("BASE_WORKDIR") + nfsport, mountport = s.enter_context(unfs_server(tmpdir)) + + # build core-image-minimal with required packages + default_installed_packages = [ + "glibc-charmaps", + "libgcc", + "libstdc++", + "libatomic", + "libgomp", + # "python3", + # "python3-pexpect", + "nfs-utils", + ] + features = [] + features.append('IMAGE_FEATURES += "ssh-server-openssh"') + features.append('CORE_IMAGE_EXTRA_INSTALL += "{0}"'.format(" ".join(default_installed_packages))) + self.write_config("\n".join(features)) + bitbake("core-image-minimal") + + # start runqemu + qemu = s.enter_context(runqemu("core-image-minimal", runqemuparams = "nographic")) + + # validate that SSH is working + status, _ = qemu.run("uname") + self.assertEqual(status, 0) + + # setup nfs mount + if qemu.run("mkdir -p \"{0}\"".format(tmpdir))[0] != 0: + raise Exception("Failed to setup NFS mount directory on target") + mountcmd = "mount -o noac,nfsvers=3,port={0},udp,mountport={1} \"{2}:{3}\" \"{3}\"".format(nfsport, mountport, qemu.server_ip, tmpdir) + status, output = qemu.run(mountcmd) + if status != 0: + raise Exception("Failed to setup NFS mount on target ({})".format(repr(output))) + + self.run_check(ssh = qemu.ip) + +@OETestTag("toolchain-user") +class GlibcSelfTest(GlibcSelfTestBase): + def test_glibc(self): + self.run_check() + +@OETestTag("toolchain-system") +class GlibcSelfTestSystemEmulated(GlibcSelfTestBase): + def test_glibc(self): + self.run_check_emulated() + diff --git a/poky/meta/lib/oeqa/selftest/cases/kerneldevelopment.py b/poky/meta/lib/oeqa/selftest/cases/kerneldevelopment.py new file mode 100644 index 0000000000..a61876ee61 --- /dev/null +++ b/poky/meta/lib/oeqa/selftest/cases/kerneldevelopment.py @@ -0,0 +1,67 @@ +import os +from oeqa.selftest.case import OESelftestTestCase +from oeqa.utils.commands import runCmd, get_bb_var +from oeqa.utils.git import GitRepo + +class KernelDev(OESelftestTestCase): + + @classmethod + def setUpClass(cls): + super(KernelDev, cls).setUpClass() + # Create the recipe directory structure inside the created layer + cls.layername = 'meta-kerneltest' + runCmd('bitbake-layers create-layer %s' % cls.layername) + runCmd('mkdir -p %s/recipes-kernel/linux/linux-yocto' % cls.layername) + cls.recipes_linuxyocto_dir = os.path.join \ + (cls.builddir, cls.layername, 'recipes-kernel', 'linux', 'linux-yocto') + cls.recipeskernel_dir = os.path.dirname(cls.recipes_linuxyocto_dir) + runCmd('bitbake-layers add-layer %s' % cls.layername) + + @classmethod + def tearDownClass(cls): + runCmd('bitbake-layers remove-layer %s' % cls.layername, ignore_status=True) + runCmd('rm -rf %s' % cls.layername) + super(KernelDev, cls).tearDownClass() + + def setUp(self): + super(KernelDev, self).setUp() + self.set_machine_config('MACHINE = "qemux86-64"\n') + + def test_apply_patches(self): + """ + Summary: Able to apply a single patch to the Linux kernel source + Expected: The README file should exist and the patch changes should be + displayed at the end of the file. + Product: Kernel Development + Author: Yeoh Ee Peng <ee.peng.yeoh@intel.com> + AutomatedBy: Mazliana Mohamad <mazliana.mohamad@intel.com> + """ + runCmd('bitbake virtual/kernel -c patch') + kernel_source = get_bb_var('STAGING_KERNEL_DIR') + readme = os.path.join(kernel_source, 'README') + + # This test step adds modified file 'README' to git and creates a + # patch file '0001-KERNEL_DEV_TEST_CASE.patch' at the same location as file + patch_content = 'This is a test to apply a patch to the kernel' + with open(readme, 'a+') as f: + f.write(patch_content) + repo = GitRepo('%s' % kernel_source, is_topdir=True) + repo.run_cmd('add %s' % readme) + repo.run_cmd(['commit', '-m', 'KERNEL_DEV_TEST_CASE']) + repo.run_cmd(['format-patch', '-1']) + patch_name = '0001-KERNEL_DEV_TEST_CASE.patch' + patchpath = os.path.join(kernel_source, patch_name) + runCmd('mv %s %s' % (patchpath, self.recipes_linuxyocto_dir)) + runCmd('rm %s ' % readme) + self.assertFalse(os.path.exists(readme)) + + recipe_append = os.path.join(self.recipeskernel_dir, 'linux-yocto_%.bbappend') + with open(recipe_append, 'w+') as fh: + fh.write('SRC_URI += "file://%s"\n' % patch_name) + fh.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"') + + runCmd('bitbake virtual/kernel -c clean') + runCmd('bitbake virtual/kernel -c patch') + self.assertTrue(os.path.exists(readme)) + result = runCmd('tail -n 1 %s' % readme) + self.assertEqual(result.output, patch_content) diff --git a/poky/meta/lib/oeqa/selftest/cases/meta_ide.py b/poky/meta/lib/oeqa/selftest/cases/meta_ide.py index f47bc70917..03901a2f32 100644 --- a/poky/meta/lib/oeqa/selftest/cases/meta_ide.py +++ b/poky/meta/lib/oeqa/selftest/cases/meta_ide.py @@ -5,9 +5,11 @@ from oeqa.selftest.case import OESelftestTestCase from oeqa.sdk.utils.sdkbuildproject import SDKBuildProject from oeqa.utils.commands import bitbake, get_bb_vars, runCmd +from oeqa.core.decorator import OETestTag import tempfile import shutil +@OETestTag("machine") class MetaIDE(OESelftestTestCase): @classmethod diff --git a/poky/meta/lib/oeqa/selftest/cases/reproducible.py b/poky/meta/lib/oeqa/selftest/cases/reproducible.py index 6dc83d2847..eee09d3fb2 100644 --- a/poky/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/poky/meta/lib/oeqa/selftest/cases/reproducible.py @@ -8,6 +8,7 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import functools import multiprocessing import textwrap +import json import unittest MISSING = 'MISSING' @@ -81,14 +82,12 @@ class ReproducibleTests(OESelftestTestCase): for v in needed_vars: setattr(self, v.lower(), bb_vars[v]) - if not hasattr(self.tc, "extraresults"): - self.tc.extraresults = {} - self.extras = self.tc.extraresults - - self.extras.setdefault('reproducible.rawlogs', {})['log'] = '' + self.extrasresults = {} + self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = '' + self.extrasresults.setdefault('reproducible', {}).setdefault('files', {}) def append_to_log(self, msg): - self.extras['reproducible.rawlogs']['log'] += msg + self.extrasresults['reproducible.rawlogs']['log'] += msg def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): result = PackageCompareResults() @@ -114,47 +113,70 @@ class ReproducibleTests(OESelftestTestCase): result.sort() return result - @unittest.skip("Reproducible builds do not yet pass") + def write_package_list(self, package_class, name, packages): + self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [ + {'reference': p.reference, 'test': p.test} for p in packages] + def test_reproducible_builds(self): capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes] + # Build native utilities + self.write_config('') + bitbake("diffutils-native -c addto_recipe_sysroot") + diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") + + # Reproducible builds should not pull from sstate or mirrors, but + # sharing DL_DIR is fine common_config = textwrap.dedent('''\ INHERIT += "reproducible_build" PACKAGE_CLASSES = "%s" + SSTATE_DIR = "${TMPDIR}/sstate" ''') % (' '.join('package_%s' % c for c in self.package_classes)) - # Do an initial build. It's acceptable for this build to use sstate - self.write_config(common_config) - vars_reference = get_bb_vars(capture_vars) + # Perform a build. + reproducibleA_tmp = os.path.join(self.topdir, 'reproducibleA', 'tmp') + if os.path.exists(reproducibleA_tmp): + bb.utils.remove(reproducibleA_tmp, recurse=True) + + self.write_config((textwrap.dedent('''\ + TMPDIR = "%s" + ''') % reproducibleA_tmp) + common_config) + vars_A = get_bb_vars(capture_vars) bitbake(' '.join(self.images)) - # Build native utilities - bitbake("diffutils-native -c addto_recipe_sysroot") - diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") + # Perform another build. + reproducibleB_tmp = os.path.join(self.topdir, 'reproducibleB', 'tmp') + if os.path.exists(reproducibleB_tmp): + bb.utils.remove(reproducibleB_tmp, recurse=True) - # Perform another build. This build should *not* share sstate or pull - # from any mirrors, but sharing a DL_DIR is fine - self.write_config(textwrap.dedent('''\ - TMPDIR = "${TOPDIR}/reproducible/tmp" - SSTATE_DIR = "${TMPDIR}/sstate" + self.write_config((textwrap.dedent('''\ SSTATE_MIRROR = "" - ''') + common_config) - vars_test = get_bb_vars(capture_vars) + TMPDIR = "%s" + ''') % reproducibleB_tmp) + common_config) + vars_B = get_bb_vars(capture_vars) bitbake(' '.join(self.images)) + # NOTE: The temp directories from the reproducible build are purposely + # kept after the build so it can be diffed for debugging. + for c in self.package_classes: - package_class = 'package_' + c + with self.subTest(package_class=c): + package_class = 'package_' + c + + deploy_A = vars_A['DEPLOY_DIR_' + c.upper()] + deploy_B = vars_B['DEPLOY_DIR_' + c.upper()] - deploy_reference = vars_reference['DEPLOY_DIR_' + c.upper()] - deploy_test = vars_test['DEPLOY_DIR_' + c.upper()] + result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot) - result = self.compare_packages(deploy_reference, deploy_test, diffutils_sysroot) + self.logger.info('Reproducibility summary for %s: %s' % (c, result)) - self.logger.info('Reproducibility summary for %s: %s' % (c, result)) + self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) - self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) + self.write_package_list(package_class, 'missing', result.missing) + self.write_package_list(package_class, 'different', result.different) + self.write_package_list(package_class, 'same', result.same) - if result.missing or result.different: - self.fail("The following %s packages are missing or different: %s" % - (c, ' '.join(r.test for r in (result.missing + result.different)))) + if result.missing or result.different: + self.fail("The following %s packages are missing or different: %s" % + (c, ' '.join(r.test for r in (result.missing + result.different)))) diff --git a/poky/meta/lib/oeqa/selftest/cases/runqemu.py b/poky/meta/lib/oeqa/selftest/cases/runqemu.py index b88ae306c0..7e676bcb41 100644 --- a/poky/meta/lib/oeqa/selftest/cases/runqemu.py +++ b/poky/meta/lib/oeqa/selftest/cases/runqemu.py @@ -8,6 +8,7 @@ import re import tempfile import time import oe.types +from oeqa.core.decorator import OETestTag from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd @@ -147,6 +148,7 @@ SYSLINUX_TIMEOUT = "10" # dedicated for MACHINE=qemux86-64 where it test that qemux86-64 will # bootup various filesystem types, including live image(iso and hddimg) # where live image was not supported on all qemu architecture. +@OETestTag("machine") class QemuTest(OESelftestTestCase): @classmethod |