summaryrefslogtreecommitdiff
path: root/poky/meta/lib/oeqa/selftest/cases
diff options
context:
space:
mode:
Diffstat (limited to 'poky/meta/lib/oeqa/selftest/cases')
-rw-r--r--poky/meta/lib/oeqa/selftest/cases/prservice.py2
-rw-r--r--poky/meta/lib/oeqa/selftest/cases/resulttooltests.py245
-rw-r--r--poky/meta/lib/oeqa/selftest/cases/runtime_test.py11
-rw-r--r--poky/meta/lib/oeqa/selftest/cases/wic.py20
-rw-r--r--poky/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py39
5 files changed, 304 insertions, 13 deletions
diff --git a/poky/meta/lib/oeqa/selftest/cases/prservice.py b/poky/meta/lib/oeqa/selftest/cases/prservice.py
index cb95503c2c..9fe3b80a31 100644
--- a/poky/meta/lib/oeqa/selftest/cases/prservice.py
+++ b/poky/meta/lib/oeqa/selftest/cases/prservice.py
@@ -77,7 +77,7 @@ class BitbakePrTests(OESelftestTestCase):
exported_db_path = os.path.join(self.builddir, 'export.inc')
export_result = runCmd("bitbake-prserv-tool export %s" % exported_db_path, ignore_status=True)
self.assertEqual(export_result.status, 0, msg="PR Service database export failed: %s" % export_result.output)
- self.assertTrue(os.path.exists(exported_db_path))
+ self.assertTrue(os.path.exists(exported_db_path), msg="%s didn't exist, tool output %s" % (exported_db_path, export_result.output))
if replace_current_db:
current_db_path = os.path.join(get_bb_var('PERSISTENT_DIR'), 'prserv.sqlite3')
diff --git a/poky/meta/lib/oeqa/selftest/cases/resulttooltests.py b/poky/meta/lib/oeqa/selftest/cases/resulttooltests.py
index c2e76f1a44..f059991c19 100644
--- a/poky/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/poky/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -71,7 +71,7 @@ class ResultToolTests(OESelftestTestCase):
self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results)
self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results)
- def test_regrresion_can_get_regression_result(self):
+ def test_regression_can_get_regression_result(self):
base_result_data = {'result': {'test1': {'status': 'PASSED'},
'test2': {'status': 'PASSED'},
'test3': {'status': 'FAILED'},
@@ -98,3 +98,246 @@ class ResultToolTests(OESelftestTestCase):
resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
+ def test_results_without_metadata_can_be_compared(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600
+ }, "result": {}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600
+ }, "result": {}}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests without metadata should be compared")
+
+ def test_target_result_with_missing_metadata_can_not_be_compared(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "OESELFTEST_METADATA": {
+ "run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None
+ }}, "result": {}}
+ target_configuration = {"configuration": {"TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600
+ }, "result": {}}
+ self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests should not be compared")
+
+ def test_results_with_matching_metadata_can_be_compared(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}
+ }, "result": {}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}
+ }, "result": {}}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests with matching metadata should be compared")
+
+ def test_results_with_mismatching_metadata_can_not_be_compared(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}
+ }, "result": {}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "oeselftest",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["machine"],
+ "exclude_tags": None}
+ }, "result": {}}
+ self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect metadata filtering, tests with mismatching metadata should not be compared")
+
+ def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
+ base_configuration = {"configuration": {"TEST_TYPE": "runtime",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["toolchain-user", "toolchain-system"],
+ "exclude_tags": None}}, "result": {}}
+ target_configuration = {"configuration": {"TEST_TYPE": "runtime",
+ "TESTSERIES": "series1",
+ "IMAGE_BASENAME": "image",
+ "IMAGE_PKGTYPE": "ipk",
+ "DISTRO": "mydistro",
+ "MACHINE": "qemux86",
+ "STARTTIME": 1672527600,
+ "OESELFTEST_METADATA": {"run_all_tests": True,
+ "run_tests": None,
+ "skips": None,
+ "machine": None,
+ "select_tags": ["machine"],
+ "exclude_tags": None}}, "result": {}}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['configuration']['TEST_TYPE'])
+
+ def test_machine_matches(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"}, "result": {}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {}}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect machine filtering, identical machine tests should be compared")
+
+ def test_machine_mismatches(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86_64"
+ }, "result": {}}
+ self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect machine filtering, mismatching machine tests should not be compared")
+
+ def test_can_not_compare_non_ltp_tests(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {
+ "ltpresult_foo": {
+ "STATUS": "PASSED"
+ }}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86_64"
+ }, "result": {
+ "bar": {
+ "STATUS": "PASSED"
+ }}}
+ self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared")
+
+ def test_can_compare_ltp_tests(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {
+ "ltpresult_foo": {
+ "STATUS": "PASSED"
+ }}}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {
+ "ltpresult_foo": {
+ "STATUS": "PASSED"
+ }}}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect ltpresult filtering, matching ltpresult content should be compared")
+
+ def test_can_match_non_static_ptest_names(self):
+ base_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {
+ "ptestresult.lttng-tools.foo_-_bar_-_moo": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.babeltrace.bar_-_moo_-_foo": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.babletrace2.moo_-_foo_-_bar": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.curl.test_0000__foo_out_of_bar": {
+ "STATUS": "PASSED"
+ }
+ }}
+ target_configuration = {"configuration": {
+ "TEST_TYPE": "runtime",
+ "MACHINE": "qemux86"
+ }, "result": {
+ "ptestresult.lttng-tools.xxx_-_yyy_-_zzz": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.babeltrace.yyy_-_zzz_-_xxx": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.babletrace2.zzz_-_xxx_-_yyy": {
+ "STATUS": "PASSED"
+ },
+ "ptestresult.curl.test_0000__xxx_out_of_yyy": {
+ "STATUS": "PASSED"
+ }
+ }}
+ self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
+ msg="incorrect ptests filtering, tests shoould be compared if prefixes match")
diff --git a/poky/meta/lib/oeqa/selftest/cases/runtime_test.py b/poky/meta/lib/oeqa/selftest/cases/runtime_test.py
index 7d99c158e5..e32c4aff85 100644
--- a/poky/meta/lib/oeqa/selftest/cases/runtime_test.py
+++ b/poky/meta/lib/oeqa/selftest/cases/runtime_test.py
@@ -260,17 +260,6 @@ class TestImage(OESelftestTestCase):
if distro and (distro in ['debian-9', 'debian-10', 'centos-7', 'centos-8', 'ubuntu-16.04', 'ubuntu-18.04'] or distro.startswith('almalinux')):
self.skipTest('virgl headless cannot be tested with %s' %(distro))
- render_hint = """If /dev/dri/renderD* is absent due to lack of suitable GPU, 'modprobe vgem' will create one suitable for mesa llvmpipe software renderer."""
- try:
- content = os.listdir("/dev/dri")
- if len([i for i in content if i.startswith('render')]) == 0:
- self.fail("No render nodes found in /dev/dri: %s. %s" %(content, render_hint))
- except FileNotFoundError:
- self.fail("/dev/dri directory does not exist; no render nodes available on this machine. %s" %(render_hint))
- try:
- dripath = subprocess.check_output("PATH=/bin:/usr/bin:$PATH pkg-config --variable=dridriverdir dri", shell=True)
- except subprocess.CalledProcessError as e:
- self.fail("Could not determine the path to dri drivers on the host via pkg-config.\nPlease install Mesa development files (particularly, dri.pc) on the host machine.")
qemu_distrofeatures = get_bb_var('DISTRO_FEATURES', 'qemu-system-native')
features = 'IMAGE_CLASSES += "testimage"\n'
if 'opengl' not in qemu_distrofeatures:
diff --git a/poky/meta/lib/oeqa/selftest/cases/wic.py b/poky/meta/lib/oeqa/selftest/cases/wic.py
index ca1abb970a..b9430cdb3b 100644
--- a/poky/meta/lib/oeqa/selftest/cases/wic.py
+++ b/poky/meta/lib/oeqa/selftest/cases/wic.py
@@ -1151,6 +1151,26 @@ class Wic2(WicTestCase):
out = glob(os.path.join(self.resultdir, "%s-*.direct" % wksname))
self.assertEqual(1, len(out))
+ @skipIfNotArch(['i586', 'i686', 'x86_64', 'aarch64'])
+ def test_uefi_kernel(self):
+ """ Test uefi-kernel in wic """
+ config = 'IMAGE_EFI_BOOT_FILES="/etc/fstab;testfile"\nIMAGE_FSTYPES = "wic"\nWKS_FILE = "test_uefikernel.wks"\nMACHINE_FEATURES:append = " efi"\n'
+ self.append_config(config)
+ bitbake('core-image-minimal')
+ self.remove_config(config)
+
+ img = 'core-image-minimal'
+ with NamedTemporaryFile("w", suffix=".wks") as wks:
+ wks.writelines(['part /boot --source bootimg-efi --sourceparams="loader=uefi-kernel"\n'
+ 'part / --source rootfs --fstype=ext4 --align 1024 --use-uuid\n'\
+ 'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n'])
+ wks.flush()
+ cmd = "wic create %s -e %s -o %s" % (wks.name, img, self.resultdir)
+ runCmd(cmd)
+ wksname = os.path.splitext(os.path.basename(wks.name))[0]
+ out = glob(os.path.join(self.resultdir, "%s-*.direct" % wksname))
+ self.assertEqual(1, len(out))
+
# TODO this test could also work on aarch64
@skipIfNotArch(['i586', 'i686', 'x86_64'])
@OETestTag("runqemu")
diff --git a/poky/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py b/poky/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py
new file mode 100644
index 0000000000..312edb6431
--- /dev/null
+++ b/poky/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py
@@ -0,0 +1,39 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import os
+import sys
+import subprocess
+import shutil
+from oeqa.selftest.case import OESelftestTestCase
+from yocto_testresults_query import get_sha1, create_workdir
+basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../')
+lib_path = basepath + '/scripts/lib'
+sys.path = sys.path + [lib_path]
+
+
+class TestResultsQueryTests(OESelftestTestCase):
+ def test_get_sha1(self):
+ test_data_get_sha1 = [
+ {"input": "yocto-4.0", "expected": "00cfdde791a0176c134f31e5a09eff725e75b905"},
+ {"input": "4.1_M1", "expected": "95066dde6861ee08fdb505ab3e0422156cc24fae"},
+ ]
+ for data in test_data_get_sha1:
+ test_name = data["input"]
+ with self.subTest(f"Test SHA1 from {test_name}"):
+ self.assertEqual(
+ get_sha1(basepath, data["input"]), data["expected"])
+
+ def test_create_workdir(self):
+ workdir = create_workdir()
+ try:
+ url = subprocess.check_output(
+ ["git", "-C", workdir, "remote", "get-url", "origin"]).strip().decode("utf-8")
+ except:
+ shutil.rmtree(workdir, ignore_errors=True)
+ self.fail(f"Can not execute git commands in {workdir}")
+ shutil.rmtree(workdir)
+ self.assertEqual(url, "git://git.yoctoproject.org/yocto-testresults")
generated by cgit v1.2.3 (git 2.39.0) at 2024-09-29 09:39:52 +0300