From f86d0556ed6040140f3d635b0bb2eeb1c507f818 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Tue, 4 Dec 2018 14:18:15 -0800 Subject: poky: sumo refresh a4c7d28688..78020fb639 Update poky to sumo HEAD. Michael Halstead (1): scripts/runqemu: Replace subprocess.run() for compatibilty Richard Purdie (35): scripts/runqemu: Tidy up lock handling code scripts/runqemu: Improve lockfile handling for python with close_fd=True oeqa/selftest/signing: Skip tests if gpg isn't found oeqa/selftest/signing: Allow tests not to need gpg on the host oeqa/selftest/signing: Use do_populate_lic target instead of do_package oeqa/selftest/case: Use bb.utils.remove() instead of shutil.remove() oeqa/selftest/buildoptions: Improve ccache test failure output oeqa/utils/commands: Add extra qemu failure logging oeqa/utils/qemurunner: Fix python ResourceWarning for unclosed file oeqa/utils/commands: Avoid log message duplication oeqa/qemurunner: Remove resource python warnings oeqa/selftest/buildoptions: Improve ccache test oeqa/selftest/buildoptions: Ensure diskmon tests run consistently oeqa/selftest/runqemu: Improve testcase failure handling oeqa/utils/qemurunner: Avoid tracebacks on closed files oeqa/loader: Fix deprecation warning oeqa/utils/commands: Avoid unclosed file warnings oeqa/selftest/context: Replace deprecated imp module usage oeqa/utils/qemurunner.py: Fix python regex warnings oeqa/selftest/context: Improve log file handling oeqa/core/runner: Improve test case comparision oeqa/runner: Ensure we don't print misleading results output oeqa/core/threaded: Remove in favour of using concurrenttests oeqa/runner: Simplify code oeqa: Remove xmlrunner oeqa/runtime/ptest: Inject results+logs into stored json results file oeqa/runner: Always show a summary of success/fail/error/skip counts oeqa/runner: Sort the test result output by result class oeqa/selftest: Improvements to the json logging oeqa/utils/metadata: Allow to function without the git module image-buildinfo,oeqa/selftest/containerimage: Ensure image-buildinfo doesn't break tests oeqa/selftest/esdk: Ensure parent directory exists oeqa/selftest/esdk: Fix typo causing test failure testimage: Improvements to the json logging testsdk: Improvements to the json logging Ross Burton (3): oeqa/oelib/path: don't leak temporary directories oeqa: don't litter /tmp with temporary directories oeqa/selftest/esdk: run selftest inside workdir not /tmp Scott Rifenbark (1): documentation: Prepared for 2.5.2 document release Stefan Lendl (1): default-versions.inc: Make PREFERRED_VERSION_openssl* overwritable Yeoh Ee Peng (6): oeqa/core/runner: refactor for OEQA to write json testresult oeqa/core/runner: write testresult to json files oeqa/selftest/context: write testresult to json files testimage.bbclass: write testresult to json files testsdk.bbclass: write testresult to json files oeqa/selftest: Standardize json logging output directory Change-Id: I3c8123930c8c3441126c1e81b3bbbde9f2b126f2 Signed-off-by: Brad Bishop --- poky/meta/lib/oeqa/core/context.py | 1 - poky/meta/lib/oeqa/core/decorator/depends.py | 9 +- poky/meta/lib/oeqa/core/decorator/oetimeout.py | 40 +-- poky/meta/lib/oeqa/core/loader.py | 2 +- poky/meta/lib/oeqa/core/runner.py | 186 +++++++------- .../core/tests/cases/loader/threaded/threaded.py | 12 - .../tests/cases/loader/threaded/threaded_alone.py | 8 - .../cases/loader/threaded/threaded_depends.py | 10 - .../tests/cases/loader/threaded/threaded_module.py | 12 - poky/meta/lib/oeqa/core/tests/common.py | 10 - poky/meta/lib/oeqa/core/tests/test_data.py | 4 +- poky/meta/lib/oeqa/core/tests/test_decorators.py | 12 - poky/meta/lib/oeqa/core/tests/test_loader.py | 30 +-- poky/meta/lib/oeqa/core/threaded.py | 275 --------------------- poky/meta/lib/oeqa/runtime/cases/ptest.py | 21 +- poky/meta/lib/oeqa/sdk/context.py | 5 +- poky/meta/lib/oeqa/selftest/case.py | 4 +- poky/meta/lib/oeqa/selftest/cases/buildoptions.py | 12 +- .../meta/lib/oeqa/selftest/cases/containerimage.py | 1 + poky/meta/lib/oeqa/selftest/cases/eSDK.py | 14 +- poky/meta/lib/oeqa/selftest/cases/oelib/path.py | 10 +- poky/meta/lib/oeqa/selftest/cases/runqemu.py | 32 ++- poky/meta/lib/oeqa/selftest/cases/signing.py | 80 ++++-- poky/meta/lib/oeqa/selftest/context.py | 45 +++- poky/meta/lib/oeqa/utils/buildproject.py | 3 +- poky/meta/lib/oeqa/utils/commands.py | 11 +- poky/meta/lib/oeqa/utils/metadata.py | 17 +- poky/meta/lib/oeqa/utils/qemurunner.py | 26 +- poky/meta/lib/oeqa/utils/targetbuild.py | 5 +- 29 files changed, 312 insertions(+), 585 deletions(-) delete mode 100644 poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py delete mode 100644 poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py delete mode 100644 poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py delete mode 100644 poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py delete mode 100644 poky/meta/lib/oeqa/core/threaded.py (limited to 'poky/meta/lib/oeqa') diff --git a/poky/meta/lib/oeqa/core/context.py b/poky/meta/lib/oeqa/core/context.py index acd547416f..ef008454ff 100644 --- a/poky/meta/lib/oeqa/core/context.py +++ b/poky/meta/lib/oeqa/core/context.py @@ -27,7 +27,6 @@ class OETestContext(object): self.logger = logger self._registry = {} self._registry['cases'] = collections.OrderedDict() - self._results = {} def _read_modules_from_manifest(self, manifest): if not os.path.exists(manifest): diff --git a/poky/meta/lib/oeqa/core/decorator/depends.py b/poky/meta/lib/oeqa/core/decorator/depends.py index baa04341c7..69c604d8f4 100644 --- a/poky/meta/lib/oeqa/core/decorator/depends.py +++ b/poky/meta/lib/oeqa/core/decorator/depends.py @@ -3,7 +3,6 @@ from unittest import SkipTest -from oeqa.core.threaded import OETestRunnerThreaded from oeqa.core.exception import OEQADependency from . import OETestDiscover, registerDecorator @@ -64,16 +63,10 @@ def _order_test_case_by_depends(cases, depends): return [cases[case_id] for case_id in cases_ordered] def _skipTestDependency(case, depends): - if isinstance(case.tc.runner, OETestRunnerThreaded): - import threading - results = case.tc._results[threading.get_ident()] - else: - results = case.tc._results - skipReasons = ['errors', 'failures', 'skipped'] for reason in skipReasons: - for test, _ in results[reason]: + for test, _ in getattr(case.tc.results, reason): if test.id() in depends: raise SkipTest("Test case %s depends on %s and was in %s." \ % (case.id(), test.id(), reason)) diff --git a/poky/meta/lib/oeqa/core/decorator/oetimeout.py b/poky/meta/lib/oeqa/core/decorator/oetimeout.py index f85e7d9792..a247583f7f 100644 --- a/poky/meta/lib/oeqa/core/decorator/oetimeout.py +++ b/poky/meta/lib/oeqa/core/decorator/oetimeout.py @@ -1,12 +1,8 @@ # Copyright (C) 2016 Intel Corporation # Released under the MIT license (see COPYING.MIT) -from . import OETestDecorator, registerDecorator - import signal -from threading import Timer - -from oeqa.core.threaded import OETestRunnerThreaded +from . import OETestDecorator, registerDecorator from oeqa.core.exception import OEQATimeoutError @registerDecorator @@ -14,32 +10,16 @@ class OETimeout(OETestDecorator): attrs = ('oetimeout',) def setUpDecorator(self): - self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) - - if isinstance(self.case.tc.runner, OETestRunnerThreaded): - self.timeouted = False - def _timeoutHandler(): - self.timeouted = True - - self.timer = Timer(self.oetimeout, _timeoutHandler) - self.timer.start() - else: - timeout = self.oetimeout - def _timeoutHandler(signum, frame): - raise OEQATimeoutError("Timed out after %s " + timeout = self.oetimeout + def _timeoutHandler(signum, frame): + raise OEQATimeoutError("Timed out after %s " "seconds of execution" % timeout) - self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) - signal.alarm(self.oetimeout) + self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) + self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) + signal.alarm(self.oetimeout) def tearDownDecorator(self): - if isinstance(self.case.tc.runner, OETestRunnerThreaded): - self.timer.cancel() - self.logger.debug("Removed Timer handler") - if self.timeouted: - raise OEQATimeoutError("Timed out after %s " - "seconds of execution" % self.oetimeout) - else: - signal.alarm(0) - signal.signal(signal.SIGALRM, self.alarmSignal) - self.logger.debug("Removed SIGALRM handler") + signal.alarm(0) + signal.signal(signal.SIGALRM, self.alarmSignal) + self.logger.debug("Removed SIGALRM handler") diff --git a/poky/meta/lib/oeqa/core/loader.py b/poky/meta/lib/oeqa/core/loader.py index 98fc0f696a..2255cf1dc5 100644 --- a/poky/meta/lib/oeqa/core/loader.py +++ b/poky/meta/lib/oeqa/core/loader.py @@ -24,7 +24,7 @@ from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ # Generate the function definition because this differ across python versions # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 # ueses four parameters so isn't incremental. -_failed_test_args = inspect.getargspec(unittest.loader._make_failed_test).args +_failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) unittest.loader._make_failed_test = _make_failed_test diff --git a/poky/meta/lib/oeqa/core/runner.py b/poky/meta/lib/oeqa/core/runner.py index 13cdf5ba52..f8bb23f344 100644 --- a/poky/meta/lib/oeqa/core/runner.py +++ b/poky/meta/lib/oeqa/core/runner.py @@ -6,17 +6,10 @@ import time import unittest import logging import re +import json -xmlEnabled = False -try: - import xmlrunner - from xmlrunner.result import _XMLTestResult as _TestResult - from xmlrunner.runner import XMLTestRunner as _TestRunner - xmlEnabled = True -except ImportError: - # use the base runner instead - from unittest import TextTestResult as _TestResult - from unittest import TextTestRunner as _TestRunner +from unittest import TextTestResult as _TestResult +from unittest import TextTestRunner as _TestRunner class OEStreamLogger(object): def __init__(self, logger): @@ -42,8 +35,12 @@ class OETestResult(_TestResult): def __init__(self, tc, *args, **kwargs): super(OETestResult, self).__init__(*args, **kwargs) + self.successes = [] + + # Inject into tc so that TestDepends decorator can see results + tc.results = self + self.tc = tc - self._tc_map_results() def startTest(self, test): # Allow us to trigger the testcase buffer mode on a per test basis @@ -53,12 +50,6 @@ class OETestResult(_TestResult): self.buffer = test.buffer super(OETestResult, self).startTest(test) - def _tc_map_results(self): - self.tc._results['failures'] = self.failures - self.tc._results['errors'] = self.errors - self.tc._results['skipped'] = self.skipped - self.tc._results['expectedFailures'] = self.expectedFailures - def logSummary(self, component, context_msg=''): elapsed_time = self.tc._run_end_time - self.tc._run_start_time self.tc.logger.info("SUMMARY:") @@ -70,67 +61,60 @@ class OETestResult(_TestResult): msg = "%s - OK - All required tests passed" % component else: msg = "%s - FAIL - Required tests failed" % component - skipped = len(self.tc._results['skipped']) - if skipped: - msg += " (skipped=%d)" % skipped + msg += " (successes=%d, skipped=%d, failures=%d, errors=%d)" % (len(self.successes), len(self.skipped), len(self.failures), len(self.errors)) self.tc.logger.info(msg) - def _getDetailsNotPassed(self, case, type, desc): - found = False + def _getTestResultDetails(self, case): + result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED', + 'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED'} - for (scase, msg) in self.tc._results[type]: - # XXX: When XML reporting is enabled scase is - # xmlrunner.result._TestInfo instance instead of - # string. - if xmlEnabled: - if case.id() == scase.test_id: - found = True - break - scase_str = scase.test_id - else: - if case == scase: + for rtype in result_types: + found = False + for (scase, msg) in getattr(self, rtype): + if case.id() == scase.id(): found = True break - scase_str = str(scase) + scase_str = str(scase.id()) - # When fails at module or class level the class name is passed as string - # so figure out to see if match - m = re.search("^setUpModule \((?P.*)\)$", scase_str) - if m: - if case.__class__.__module__ == m.group('module_name'): - found = True - break + # When fails at module or class level the class name is passed as string + # so figure out to see if match + m = re.search("^setUpModule \((?P.*)\)$", scase_str) + if m: + if case.__class__.__module__ == m.group('module_name'): + found = True + break - m = re.search("^setUpClass \((?P.*)\)$", scase_str) - if m: - class_name = "%s.%s" % (case.__class__.__module__, - case.__class__.__name__) + m = re.search("^setUpClass \((?P.*)\)$", scase_str) + if m: + class_name = "%s.%s" % (case.__class__.__module__, + case.__class__.__name__) - if class_name == m.group('class_name'): - found = True - break + if class_name == m.group('class_name'): + found = True + break + + if found: + return result_types[rtype], msg - if found: - return (found, msg) + return 'UNKNOWN', None - return (found, None) + def addSuccess(self, test): + #Added so we can keep track of successes too + self.successes.append((test, None)) + super(OETestResult, self).addSuccess(test) - def logDetails(self): + def logDetails(self, json_file_dir=None, configuration=None, result_id=None): self.tc.logger.info("RESULTS:") + + result = {} + logs = {} + if hasattr(self.tc, "extraresults"): + result = self.tc.extraresults + for case_name in self.tc._registry['cases']: case = self.tc._registry['cases'][case_name] - result_types = ['failures', 'errors', 'skipped', 'expectedFailures'] - result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL'] - - fail = False - desc = None - for idx, name in enumerate(result_types): - (fail, msg) = self._getDetailsNotPassed(case, result_types[idx], - result_desc[idx]) - if fail: - desc = result_desc[idx] - break + (status, log) = self._getTestResultDetails(case) oeid = -1 if hasattr(case, 'decorators'): @@ -138,12 +122,27 @@ class OETestResult(_TestResult): if hasattr(d, 'oeid'): oeid = d.oeid - if fail: - self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), - oeid, desc)) + if status not in logs: + logs[status] = [] + logs[status].append("RESULTS - %s - Testcase %s: %s" % (case.id(), oeid, status)) + if log: + result[case.id()] = {'status': status, 'log': log} else: - self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), - oeid, 'PASSED')) + result[case.id()] = {'status': status} + + for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: + if i not in logs: + continue + for l in logs[i]: + self.tc.logger.info(l) + + if json_file_dir: + tresultjsonhelper = OETestResultJSONHelper() + tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result) + + def wasSuccessful(self): + # Override as we unexpected successes aren't failures for us + return (len(self.failures) == len(self.errors) == 0) class OEListTestsResult(object): def wasSuccessful(self): @@ -153,33 +152,14 @@ class OETestRunner(_TestRunner): streamLoggerClass = OEStreamLogger def __init__(self, tc, *args, **kwargs): - if xmlEnabled: - if not kwargs.get('output'): - kwargs['output'] = os.path.join(os.getcwd(), - 'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid())) - kwargs['stream'] = self.streamLoggerClass(tc.logger) super(OETestRunner, self).__init__(*args, **kwargs) self.tc = tc self.resultclass = OETestResult - # XXX: The unittest-xml-reporting package defines _make_result method instead - # of _makeResult standard on unittest. - if xmlEnabled: - def _make_result(self): - """ - Creates a TestResult object which will be used to store - information about the executed tests. - """ - # override in subclasses if necessary. - return self.resultclass(self.tc, - self.stream, self.descriptions, self.verbosity, self.elapsed_times - ) - else: - def _makeResult(self): - return self.resultclass(self.tc, self.stream, self.descriptions, - self.verbosity) - + def _makeResult(self): + return self.resultclass(self.tc, self.stream, self.descriptions, + self.verbosity) def _walk_suite(self, suite, func): for obj in suite: @@ -275,3 +255,29 @@ class OETestRunner(_TestRunner): self._list_tests_module(suite) return OEListTestsResult() + +class OETestResultJSONHelper(object): + + testresult_filename = 'testresults.json' + + def _get_existing_testresults_if_available(self, write_dir): + testresults = {} + file = os.path.join(write_dir, self.testresult_filename) + if os.path.exists(file): + with open(file, "r") as f: + testresults = json.load(f) + return testresults + + def _write_file(self, write_dir, file_name, file_content): + file_path = os.path.join(write_dir, file_name) + with open(file_path, 'w') as the_file: + the_file.write(file_content) + + def dump_testresult_file(self, write_dir, configuration, result_id, test_result): + bb.utils.mkdirhier(write_dir) + lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock')) + test_results = self._get_existing_testresults_if_available(write_dir) + test_results[result_id] = {'configuration': configuration, 'result': test_result} + json_testresults = json.dumps(test_results, sort_keys=True, indent=4) + self._write_file(write_dir, self.testresult_filename, json_testresults) + bb.utils.unlockfile(lf) diff --git a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py b/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py deleted file mode 100644 index 0fe4cb3f11..0000000000 --- a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTest(OETestCase): - def test_threaded_no_depends(self): - self.assertTrue(True, msg='How is this possible?') - -class ThreadedTest2(OETestCase): - def test_threaded_same_module(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py b/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py deleted file mode 100644 index 905f397846..0000000000 --- a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTestAlone(OETestCase): - def test_threaded_alone(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py b/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py deleted file mode 100644 index 0c158d3bac..0000000000 --- a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase -from oeqa.core.decorator.depends import OETestDepends - -class ThreadedTest3(OETestCase): - @OETestDepends(['threaded.ThreadedTest.test_threaded_no_depends']) - def test_threaded_depends(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py b/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py deleted file mode 100644 index 63d17e0401..0000000000 --- a/poky/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTestModule(OETestCase): - def test_threaded_module(self): - self.assertTrue(True, msg='How is this possible?') - -class ThreadedTestModule2(OETestCase): - def test_threaded_module2(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/poky/meta/lib/oeqa/core/tests/common.py b/poky/meta/lib/oeqa/core/tests/common.py index 1932323409..52b18a1c3e 100644 --- a/poky/meta/lib/oeqa/core/tests/common.py +++ b/poky/meta/lib/oeqa/core/tests/common.py @@ -33,13 +33,3 @@ class TestBase(unittest.TestCase): tc.loadTests(self.cases_path, modules=modules, tests=tests, filters=filters) return tc - - def _testLoaderThreaded(self, d={}, modules=[], - tests=[], filters={}): - from oeqa.core.threaded import OETestContextThreaded - - tc = OETestContextThreaded(d, self.logger) - tc.loadTests(self.cases_path, modules=modules, tests=tests, - filters=filters) - - return tc diff --git a/poky/meta/lib/oeqa/core/tests/test_data.py b/poky/meta/lib/oeqa/core/tests/test_data.py index 320468cbe4..21b6c68b8a 100755 --- a/poky/meta/lib/oeqa/core/tests/test_data.py +++ b/poky/meta/lib/oeqa/core/tests/test_data.py @@ -21,7 +21,7 @@ class TestData(TestBase): tc = self._testLoader(modules=self.modules) self.assertEqual(False, tc.runTests().wasSuccessful()) - for test, data in tc._results['errors']: + for test, data in tc.errors: expect = False if expectedException in data: expect = True @@ -34,7 +34,7 @@ class TestData(TestBase): tc = self._testLoader(d=d, modules=self.modules) self.assertEqual(False, tc.runTests().wasSuccessful()) - for test, data in tc._results['failures']: + for test, data in tc.failures: expect = False if expectedError in data: expect = True diff --git a/poky/meta/lib/oeqa/core/tests/test_decorators.py b/poky/meta/lib/oeqa/core/tests/test_decorators.py index cf99e0d72d..f7d11e885a 100755 --- a/poky/meta/lib/oeqa/core/tests/test_decorators.py +++ b/poky/meta/lib/oeqa/core/tests/test_decorators.py @@ -131,17 +131,5 @@ class TestTimeoutDecorator(TestBase): msg = "OETestTimeout didn't restore SIGALRM" self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) - def test_timeout_thread(self): - tests = ['timeout.TimeoutTest.testTimeoutPass'] - msg = 'Failed to run test using OETestTimeout' - tc = self._testLoaderThreaded(modules=self.modules, tests=tests) - self.assertTrue(tc.runTests().wasSuccessful(), msg=msg) - - def test_timeout_threaded_fail(self): - tests = ['timeout.TimeoutTest.testTimeoutFail'] - msg = "OETestTimeout test didn't timeout as expected" - tc = self._testLoaderThreaded(modules=self.modules, tests=tests) - self.assertFalse(tc.runTests().wasSuccessful(), msg=msg) - if __name__ == '__main__': unittest.main() diff --git a/poky/meta/lib/oeqa/core/tests/test_loader.py b/poky/meta/lib/oeqa/core/tests/test_loader.py index e0d917d317..b79b8bad4d 100755 --- a/poky/meta/lib/oeqa/core/tests/test_loader.py +++ b/poky/meta/lib/oeqa/core/tests/test_loader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (C) 2016-2017 Intel Corporation +# Copyright (C) 2016 Intel Corporation # Released under the MIT license (see COPYING.MIT) import os @@ -82,33 +82,5 @@ class TestLoader(TestBase): msg = 'Expected modules from two different paths' self.assertEqual(modules, expected_modules, msg=msg) - def test_loader_threaded(self): - cases_path = self.cases_path - - self.cases_path = [os.path.join(self.cases_path, 'loader', 'threaded')] - - tc = self._testLoaderThreaded() - self.assertEqual(len(tc.suites), 3, "Expected to be 3 suites") - - case_ids = ['threaded.ThreadedTest.test_threaded_no_depends', - 'threaded.ThreadedTest2.test_threaded_same_module', - 'threaded_depends.ThreadedTest3.test_threaded_depends'] - for case in tc.suites[0]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[0]._tests.index(case)]) - - case_ids = ['threaded_alone.ThreadedTestAlone.test_threaded_alone'] - for case in tc.suites[1]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[1]._tests.index(case)]) - - case_ids = ['threaded_module.ThreadedTestModule.test_threaded_module', - 'threaded_module.ThreadedTestModule2.test_threaded_module2'] - for case in tc.suites[2]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[2]._tests.index(case)]) - - self.cases_path = cases_path - if __name__ == '__main__': unittest.main() diff --git a/poky/meta/lib/oeqa/core/threaded.py b/poky/meta/lib/oeqa/core/threaded.py deleted file mode 100644 index 2cafe03a21..0000000000 --- a/poky/meta/lib/oeqa/core/threaded.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -import threading -import multiprocessing -import queue -import time - -from unittest.suite import TestSuite - -from oeqa.core.loader import OETestLoader -from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner -from oeqa.core.context import OETestContext - -class OETestLoaderThreaded(OETestLoader): - def __init__(self, tc, module_paths, modules, tests, modules_required, - filters, process_num=0, *args, **kwargs): - super(OETestLoaderThreaded, self).__init__(tc, module_paths, modules, - tests, modules_required, filters, *args, **kwargs) - - self.process_num = process_num - - def discover(self): - suite = super(OETestLoaderThreaded, self).discover() - - if self.process_num <= 0: - self.process_num = min(multiprocessing.cpu_count(), - len(suite._tests)) - - suites = [] - for _ in range(self.process_num): - suites.append(self.suiteClass()) - - def _search_for_module_idx(suites, case): - """ - Cases in the same module needs to be run - in the same thread because PyUnit keeps track - of setUp{Module, Class,} and tearDown{Module, Class,}. - """ - - for idx in range(self.process_num): - suite = suites[idx] - for c in suite._tests: - if case.__module__ == c.__module__: - return idx - - return -1 - - def _search_for_depend_idx(suites, depends): - """ - Dependency cases needs to be run in the same - thread, because OEQA framework look at the state - of dependant test to figure out if skip or not. - """ - - for idx in range(self.process_num): - suite = suites[idx] - - for case in suite._tests: - if case.id() in depends: - return idx - return -1 - - def _get_best_idx(suites): - sizes = [len(suite._tests) for suite in suites] - return sizes.index(min(sizes)) - - def _fill_suites(suite): - idx = -1 - for case in suite: - if isinstance(case, TestSuite): - _fill_suites(case) - else: - idx = _search_for_module_idx(suites, case) - - depends = {} - if 'depends' in self.tc._registry: - depends = self.tc._registry['depends'] - - if idx == -1 and case.id() in depends: - case_depends = depends[case.id()] - idx = _search_for_depend_idx(suites, case_depends) - - if idx == -1: - idx = _get_best_idx(suites) - - suites[idx].addTest(case) - _fill_suites(suite) - - suites_tmp = suites - suites = [] - for suite in suites_tmp: - if len(suite._tests) > 0: - suites.append(suite) - - return suites - -class OEStreamLoggerThreaded(OEStreamLogger): - _lock = threading.Lock() - buffers = {} - - def write(self, msg): - tid = threading.get_ident() - - if not tid in self.buffers: - self.buffers[tid] = "" - - if msg: - self.buffers[tid] += msg - - def finish(self): - tid = threading.get_ident() - - self._lock.acquire() - self.logger.info('THREAD: %d' % tid) - self.logger.info('-' * 70) - for line in self.buffers[tid].split('\n'): - self.logger.info(line) - self._lock.release() - -class OETestResultThreadedInternal(OETestResult): - def _tc_map_results(self): - tid = threading.get_ident() - - # PyUnit generates a result for every test module run, test - # if the thread already has an entry to avoid lose the previous - # test module results. - if not tid in self.tc._results: - self.tc._results[tid] = {} - self.tc._results[tid]['failures'] = self.failures - self.tc._results[tid]['errors'] = self.errors - self.tc._results[tid]['skipped'] = self.skipped - self.tc._results[tid]['expectedFailures'] = self.expectedFailures - -class OETestResultThreaded(object): - _results = {} - _lock = threading.Lock() - - def __init__(self, tc): - self.tc = tc - - def _fill_tc_results(self): - tids = list(self.tc._results.keys()) - fields = ['failures', 'errors', 'skipped', 'expectedFailures'] - - for tid in tids: - result = self.tc._results[tid] - for field in fields: - if not field in self.tc._results: - self.tc._results[field] = [] - self.tc._results[field].extend(result[field]) - - def addResult(self, result, run_start_time, run_end_time): - tid = threading.get_ident() - - self._lock.acquire() - self._results[tid] = {} - self._results[tid]['result'] = result - self._results[tid]['run_start_time'] = run_start_time - self._results[tid]['run_end_time'] = run_end_time - self._results[tid]['result'] = result - self._lock.release() - - def wasSuccessful(self): - wasSuccessful = True - for tid in self._results.keys(): - wasSuccessful = wasSuccessful and \ - self._results[tid]['result'].wasSuccessful() - return wasSuccessful - - def stop(self): - for tid in self._results.keys(): - self._results[tid]['result'].stop() - - def logSummary(self, component, context_msg=''): - elapsed_time = (self.tc._run_end_time - self.tc._run_start_time) - - self.tc.logger.info("SUMMARY:") - self.tc.logger.info("%s (%s) - Ran %d tests in %.3fs" % (component, - context_msg, len(self.tc._registry['cases']), elapsed_time)) - if self.wasSuccessful(): - msg = "%s - OK - All required tests passed" % component - else: - msg = "%s - FAIL - Required tests failed" % component - self.tc.logger.info(msg) - - def logDetails(self): - if list(self._results): - tid = list(self._results)[0] - result = self._results[tid]['result'] - result.logDetails() - -class _Worker(threading.Thread): - """Thread executing tasks from a given tasks queue""" - def __init__(self, tasks, result, stream): - threading.Thread.__init__(self) - self.tasks = tasks - - self.result = result - self.stream = stream - - def run(self): - while True: - try: - func, args, kargs = self.tasks.get(block=False) - except queue.Empty: - break - - try: - run_start_time = time.time() - rc = func(*args, **kargs) - run_end_time = time.time() - self.result.addResult(rc, run_start_time, run_end_time) - self.stream.finish() - except Exception as e: - print(e) - finally: - self.tasks.task_done() - -class _ThreadedPool: - """Pool of threads consuming tasks from a queue""" - def __init__(self, num_workers, num_tasks, stream=None, result=None): - self.tasks = queue.Queue(num_tasks) - self.workers = [] - - for _ in range(num_workers): - worker = _Worker(self.tasks, result, stream) - self.workers.append(worker) - - def start(self): - for worker in self.workers: - worker.start() - - def add_task(self, func, *args, **kargs): - """Add a task to the queue""" - self.tasks.put((func, args, kargs)) - - def wait_completion(self): - """Wait for completion of all the tasks in the queue""" - self.tasks.join() - for worker in self.workers: - worker.join() - -class OETestRunnerThreaded(OETestRunner): - streamLoggerClass = OEStreamLoggerThreaded - - def __init__(self, tc, *args, **kwargs): - super(OETestRunnerThreaded, self).__init__(tc, *args, **kwargs) - self.resultclass = OETestResultThreadedInternal # XXX: XML reporting overrides at __init__ - - def run(self, suites): - result = OETestResultThreaded(self.tc) - - pool = _ThreadedPool(len(suites), len(suites), stream=self.stream, - result=result) - for s in suites: - pool.add_task(super(OETestRunnerThreaded, self).run, s) - pool.start() - pool.wait_completion() - result._fill_tc_results() - - return result - -class OETestContextThreaded(OETestContext): - loaderClass = OETestLoaderThreaded - runnerClass = OETestRunnerThreaded - - def loadTests(self, module_paths, modules=[], tests=[], - modules_manifest="", modules_required=[], filters={}, process_num=0): - if modules_manifest: - modules = self._read_modules_from_manifest(modules_manifest) - - self.loader = self.loaderClass(self, module_paths, modules, tests, - modules_required, filters, process_num) - self.suites = self.loader.discover() diff --git a/poky/meta/lib/oeqa/runtime/cases/ptest.py b/poky/meta/lib/oeqa/runtime/cases/ptest.py index f60a433d59..77ae7b6b86 100644 --- a/poky/meta/lib/oeqa/runtime/cases/ptest.py +++ b/poky/meta/lib/oeqa/runtime/cases/ptest.py @@ -1,3 +1,6 @@ +import unittest +import pprint + from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.core.decorator.oeid import OETestID @@ -49,6 +52,7 @@ class PtestRunnerTest(OERuntimeTestCase): @OETestID(1600) @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES') @OETestDepends(['ssh.SSHTest.test_ssh']) + @unittest.expectedFailure def test_ptestrunner(self): status, output = self.target.run('which ptest-runner', 0) if status != 0: @@ -76,6 +80,11 @@ class PtestRunnerTest(OERuntimeTestCase): # status != 0 is OK since some ptest tests may fail self.assertTrue(status != 127, msg="Cannot execute ptest-runner!") + if not hasattr(self.tc, "extraresults"): + self.tc.extraresults = {} + extras = self.tc.extraresults + extras['ptestresult.rawlogs'] = {'log': output} + # Parse and save results parse_result = self.parse_ptest(ptest_runner_log) parse_result.log_as_files(ptest_log_dir, test_status = ['pass','fail', 'skip']) @@ -84,10 +93,18 @@ class PtestRunnerTest(OERuntimeTestCase): os.remove(ptest_log_dir_link) os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link) + trans = str.maketrans("()", "__") + resmap = {'pass': 'PASSED', 'skip': 'SKIPPED', 'fail': 'FAILED'} + for section in parse_result.result_dict: + for test, result in parse_result.result_dict[section]: + testname = "ptestresult." + section + "." + "_".join(test.translate(trans).split()) + extras[testname] = {'status': resmap[result]} + failed_tests = {} for section in parse_result.result_dict: - failed_testcases = [ test for test, result in parse_result.result_dict[section] if result == 'fail' ] + failed_testcases = [ "_".join(test.translate(trans).split()) for test, result in parse_result.result_dict[section] if result == 'fail' ] if failed_testcases: failed_tests[section] = failed_testcases - self.assertFalse(failed_tests, msg = "Failed ptests: %s" %(str(failed_tests))) + if failed_tests: + self.fail("Failed ptests:\n%s" % pprint.pformat(failed_tests)) diff --git a/poky/meta/lib/oeqa/sdk/context.py b/poky/meta/lib/oeqa/sdk/context.py index b3d7c75183..82e4c19bfc 100644 --- a/poky/meta/lib/oeqa/sdk/context.py +++ b/poky/meta/lib/oeqa/sdk/context.py @@ -6,10 +6,9 @@ import sys import glob import re -from oeqa.core.context import OETestContextExecutor -from oeqa.core.threaded import OETestContextThreaded +from oeqa.core.context import OETestContext, OETestContextExecutor -class OESDKTestContext(OETestContextThreaded): +class OESDKTestContext(OETestContext): sdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files") def __init__(self, td=None, logger=None, sdk_dir=None, sdk_env=None, diff --git a/poky/meta/lib/oeqa/selftest/case.py b/poky/meta/lib/oeqa/selftest/case.py index e09915b495..2e446b0a64 100644 --- a/poky/meta/lib/oeqa/selftest/case.py +++ b/poky/meta/lib/oeqa/selftest/case.py @@ -12,6 +12,8 @@ import oeqa.utils.ftools as ftools from oeqa.utils.commands import runCmd, bitbake, get_bb_var from oeqa.core.case import OETestCase +import bb.utils + class OESelftestTestCase(OETestCase): def __init__(self, methodName="runTest"): self._extra_tear_down_commands = [] @@ -167,7 +169,7 @@ to ensure accurate results.") if self._track_for_cleanup: for path in self._track_for_cleanup: if os.path.isdir(path): - shutil.rmtree(path) + bb.utils.remove(path, recurse=True) if os.path.isfile(path): os.remove(path) self._track_for_cleanup = [] diff --git a/poky/meta/lib/oeqa/selftest/cases/buildoptions.py b/poky/meta/lib/oeqa/selftest/cases/buildoptions.py index e60e32dadf..24597ac192 100644 --- a/poky/meta/lib/oeqa/selftest/cases/buildoptions.py +++ b/poky/meta/lib/oeqa/selftest/cases/buildoptions.py @@ -38,10 +38,12 @@ class ImageOptionsTests(OESelftestTestCase): self.assertTrue(os.path.isfile(p), msg = "No ccache found (%s)" % p) self.write_config('INHERIT += "ccache"') self.add_command_to_tearDown('bitbake -c clean m4') + bitbake("m4 -c clean") bitbake("m4 -f -c compile") log_compile = os.path.join(get_bb_var("WORKDIR","m4"), "temp/log.do_compile") - res = runCmd("grep ccache %s" % log_compile, ignore_status=True) - self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile) + with open(log_compile, "r") as f: + loglines = "".join(f.readlines()) + self.assertIn("ccache", loglines, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile) @OETestID(1435) def test_read_only_image(self): @@ -57,15 +59,15 @@ class DiskMonTest(OESelftestTestCase): @OETestID(277) def test_stoptask_behavior(self): self.write_config('BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},100000G,100K"') - res = bitbake("m4", ignore_status = True) + res = bitbake("delay -c delay", ignore_status = True) self.assertTrue('ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"!' in res.output, msg = "Tasks should have stopped. Disk monitor is set to STOPTASK: %s" % res.output) self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_config('BB_DISKMON_DIRS = "ABORT,${TMPDIR},100000G,100K"') - res = bitbake("m4", ignore_status = True) + res = bitbake("delay -c delay", ignore_status = True) self.assertTrue('ERROR: Immediately abort since the disk space monitor action is "ABORT"!' in res.output, "Tasks should have been aborted immediatelly. Disk monitor is set to ABORT: %s" % res.output) self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_config('BB_DISKMON_DIRS = "WARN,${TMPDIR},100000G,100K"') - res = bitbake("m4") + res = bitbake("delay -c delay") self.assertTrue('WARNING: The free space' in res.output, msg = "A warning should have been displayed for disk monitor is set to WARN: %s" %res.output) class SanityOptionsTest(OESelftestTestCase): diff --git a/poky/meta/lib/oeqa/selftest/cases/containerimage.py b/poky/meta/lib/oeqa/selftest/cases/containerimage.py index 99a5cc9e57..8deaae75d8 100644 --- a/poky/meta/lib/oeqa/selftest/cases/containerimage.py +++ b/poky/meta/lib/oeqa/selftest/cases/containerimage.py @@ -39,6 +39,7 @@ class ContainerImageTests(OESelftestTestCase): IMAGE_FSTYPES = "container" PACKAGE_CLASSES = "package_ipk" IMAGE_FEATURES = "" +IMAGE_BUILDINFO_FILE = "" """) bbvars = get_bb_vars(['bindir', 'sysconfdir', 'localstatedir', diff --git a/poky/meta/lib/oeqa/selftest/cases/eSDK.py b/poky/meta/lib/oeqa/selftest/cases/eSDK.py index d03188f2f7..d7aef93991 100644 --- a/poky/meta/lib/oeqa/selftest/cases/eSDK.py +++ b/poky/meta/lib/oeqa/selftest/cases/eSDK.py @@ -70,11 +70,13 @@ CORE_IMAGE_EXTRA_INSTALL = "perl" @classmethod def setUpClass(cls): super(oeSDKExtSelfTest, cls).setUpClass() - cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA') + cls.image = 'core-image-minimal' - sstate_dir = get_bb_var('SSTATE_DIR') + bb_vars = get_bb_vars(['SSTATE_DIR', 'WORKDIR'], cls.image) + bb.utils.mkdirhier(bb_vars["WORKDIR"]) + cls.tmpdirobj = tempfile.TemporaryDirectory(prefix="selftest-esdk-", dir=bb_vars["WORKDIR"]) + cls.tmpdir_eSDKQA = cls.tmpdirobj.name - cls.image = 'core-image-minimal' oeSDKExtSelfTest.generate_eSDK(cls.image) # Install eSDK @@ -87,14 +89,14 @@ CORE_IMAGE_EXTRA_INSTALL = "perl" sstate_config=""" SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS" SSTATE_MIRRORS = "file://.* file://%s/PATH" - """ % sstate_dir + """ % bb_vars["SSTATE_DIR"] with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f: f.write(sstate_config) @classmethod def tearDownClass(cls): - shutil.rmtree(cls.tmpdir_eSDKQA, ignore_errors=True) - super(oeSDKExtSelfTest, cls).tearDownClass() + cls.tmpdirobj.cleanup() + super().tearDownClass() @OETestID(1602) def test_install_libraries_headers(self): diff --git a/poky/meta/lib/oeqa/selftest/cases/oelib/path.py b/poky/meta/lib/oeqa/selftest/cases/oelib/path.py index 75a27c06f7..e0eb8134a9 100644 --- a/poky/meta/lib/oeqa/selftest/cases/oelib/path.py +++ b/poky/meta/lib/oeqa/selftest/cases/oelib/path.py @@ -38,13 +38,6 @@ class TestRealPath(TestCase): ( "b/test", errno.ENOENT ), ] - def __del__(self): - try: - #os.system("tree -F %s" % self.tmpdir) - shutil.rmtree(self.tmpdir) - except: - pass - def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix = "oe-test_path") self.root = os.path.join(self.tmpdir, "R") @@ -59,6 +52,9 @@ class TestRealPath(TestCase): for l in self.LINKS: os.symlink(l[1], os.path.join(self.root, l[0])) + def tearDown(self): + shutil.rmtree(self.tmpdir) + def __realpath(self, file, use_physdir, assume_dir = True): return oe.path.realpath(os.path.join(self.root, file), self.root, use_physdir, assume_dir = assume_dir) diff --git a/poky/meta/lib/oeqa/selftest/cases/runqemu.py b/poky/meta/lib/oeqa/selftest/cases/runqemu.py index 5ebdd57a41..a758aafbdd 100644 --- a/poky/meta/lib/oeqa/selftest/cases/runqemu.py +++ b/poky/meta/lib/oeqa/selftest/cases/runqemu.py @@ -44,7 +44,8 @@ SYSLINUX_TIMEOUT = "10" """Test runqemu machine""" cmd = "%s %s" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: - self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + with open(qemu.qemurunnerlog) as f: + self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) @OETestID(2002) def test_boot_machine_ext4(self): @@ -52,7 +53,7 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s ext4" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue('rootfs.ext4' in f.read(), "Failed: %s" % cmd) + self.assertIn('rootfs.ext4', f.read(), "Failed: %s" % cmd) @OETestID(2003) def test_boot_machine_iso(self): @@ -60,14 +61,16 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s iso" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue('media=cdrom' in f.read(), "Failed: %s" % cmd) + self.assertIn('media=cdrom', f.read(), "Failed: %s" % cmd) @OETestID(2004) def test_boot_recipe_image(self): """Test runqemu recipe-image""" cmd = "%s %s" % (self.cmd_common, self.recipe) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: - self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + with open(qemu.qemurunnerlog) as f: + self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) + @OETestID(2005) def test_boot_recipe_image_vmdk(self): @@ -75,7 +78,7 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s wic.vmdk" % (self.cmd_common, self.recipe) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd) + self.assertIn('format=vmdk', f.read(), "Failed: %s" % cmd) @OETestID(2006) def test_boot_recipe_image_vdi(self): @@ -83,14 +86,16 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s wic.vdi" % (self.cmd_common, self.recipe) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd) + self.assertIn('format=vdi', f.read(), "Failed: %s" % cmd) @OETestID(2007) def test_boot_deploy(self): """Test runqemu deploy_dir_image""" cmd = "%s %s" % (self.cmd_common, self.deploy_dir_image) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: - self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + with open(qemu.qemurunnerlog) as f: + self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) + @OETestID(2008) def test_boot_deploy_hddimg(self): @@ -98,7 +103,7 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s" % cmd) + self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s, %s" % (cmd, f.read())) @OETestID(2009) def test_boot_machine_slirp(self): @@ -106,7 +111,7 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s slirp %s" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue(' -netdev user' in f.read(), "Failed: %s" % cmd) + self.assertIn(' -netdev user', f.read(), "Failed: %s" % cmd) @OETestID(2009) def test_boot_machine_slirp_qcow2(self): @@ -114,7 +119,7 @@ SYSLINUX_TIMEOUT = "10" cmd = "%s slirp wic.qcow2 %s" % (self.cmd_common, self.machine) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: with open(qemu.qemurunnerlog) as f: - self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd) + self.assertIn('format=qcow2', f.read(), "Failed: %s" % cmd) @OETestID(2010) def test_boot_qemu_boot(self): @@ -125,7 +130,8 @@ SYSLINUX_TIMEOUT = "10" self.skipTest("%s not found" % qemuboot_conf) cmd = "%s %s" % (self.cmd_common, qemuboot_conf) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: - self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + with open(qemu.qemurunnerlog) as f: + self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) @OETestID(2011) def test_boot_rootfs(self): @@ -136,7 +142,9 @@ SYSLINUX_TIMEOUT = "10" self.skipTest("%s not found" % rootfs) cmd = "%s %s" % (self.cmd_common, rootfs) with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu: - self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd) + with open(qemu.qemurunnerlog) as f: + self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read())) + # This test was designed as a separate class to test that shutdown # command will shutdown qemu as expected on each qemu architecture diff --git a/poky/meta/lib/oeqa/selftest/cases/signing.py b/poky/meta/lib/oeqa/selftest/cases/signing.py index a750cfc7b2..0edaf400bb 100644 --- a/poky/meta/lib/oeqa/selftest/cases/signing.py +++ b/poky/meta/lib/oeqa/selftest/cases/signing.py @@ -1,10 +1,12 @@ from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars import os +import oe import glob import re import shutil import tempfile +from contextlib import contextmanager from oeqa.core.decorator.oeid import OETestID from oeqa.utils.ftools import write_file @@ -15,23 +17,39 @@ class Signing(OESelftestTestCase): pub_key_path = "" secret_key_path = "" - @classmethod - def setUpClass(cls): - super(Signing, cls).setUpClass() - # Check that we can find the gpg binary and fail early if we can't - if not shutil.which("gpg"): - raise AssertionError("This test needs GnuPG") + def setup_gpg(self): + bitbake('gnupg-native -c addto_recipe_sysroot') - cls.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-") + self.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-") + self.track_for_cleanup(self.gpg_dir) - cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub") - cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret") + self.pub_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.pub") + self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret") - runCmd('gpg --batch --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path)) + nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native") + runCmd('gpg --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot) + return nsysroot + get_bb_var("bindir_native") - @classmethod - def tearDownClass(cls): - shutil.rmtree(cls.gpg_dir, ignore_errors=True) + + @contextmanager + def create_new_builddir(self, builddir, newbuilddir): + bb.utils.mkdirhier(newbuilddir) + oe.path.copytree(builddir + "/conf", newbuilddir + "/conf") + oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") + + origenv = os.environ.copy() + + for e in os.environ: + if builddir in os.environ[e]: + os.environ[e] = os.environ[e].replace(builddir, newbuilddir) + + os.chdir(newbuilddir) + try: + yield + finally: + for e in origenv: + os.environ[e] = origenv[e] + os.chdir(builddir) @OETestID(1362) def test_signing_packages(self): @@ -46,6 +64,8 @@ class Signing(OESelftestTestCase): """ import oe.packagedata + self.setup_gpg() + package_classes = get_bb_var('PACKAGE_CLASSES') if 'package_rpm' not in package_classes: self.skipTest('This test requires RPM Packaging.') @@ -108,11 +128,12 @@ class Signing(OESelftestTestCase): test_recipe = 'ed' - builddir = os.environ.get('BUILDDIR') + # Since we need gpg but we can't use gpg-native for sstate signatures, we + # build gpg-native in our original builddir then run the tests in a second one. + builddir = os.environ.get('BUILDDIR') + "-testsign" sstatedir = os.path.join(builddir, 'test-sstate') - self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) - self.add_command_to_tearDown('rm -rf %s' % sstatedir) + nsysroot = self.setup_gpg() feature = 'SSTATE_SIG_KEY ?= "testuser"\n' feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n' @@ -124,19 +145,26 @@ class Signing(OESelftestTestCase): self.write_config(feature) - bitbake('-c clean %s' % test_recipe) - bitbake(test_recipe) + with self.create_new_builddir(os.environ['BUILDDIR'], builddir): + + os.environ["PATH"] = nsysroot + ":" + os.environ["PATH"] + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + self.add_command_to_tearDown('rm -rf %s' % sstatedir) + self.add_command_to_tearDown('rm -rf %s' % builddir) + + bitbake('-c clean %s' % test_recipe) + bitbake('-c populate_lic %s' % test_recipe) - recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig') - recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_package.tgz') + recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig') + recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz') - self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') - self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') + self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') + self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') - ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0])) - # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30 - # gpg: Good signature from "testuser (nocomment) " - self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.') + ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0])) + # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30 + # gpg: Good signature from "testuser (nocomment) " + self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.') class LockedSignatures(OESelftestTestCase): diff --git a/poky/meta/lib/oeqa/selftest/context.py b/poky/meta/lib/oeqa/selftest/context.py index 9e90d3c256..9a56888c2f 100644 --- a/poky/meta/lib/oeqa/selftest/context.py +++ b/poky/meta/lib/oeqa/selftest/context.py @@ -5,7 +5,7 @@ import os import time import glob import sys -import imp +import importlib import signal from shutil import copyfile from random import choice @@ -96,11 +96,17 @@ class OESelftestTestContextExecutor(OETestContextExecutor): return cases_paths def _process_args(self, logger, args): - args.output_log = '%s-results-%s.log' % (self.name, - time.strftime("%Y%m%d%H%M%S")) + + args.test_start_time = time.strftime("%Y%m%d%H%M%S") args.test_data_file = None args.CASES_PATHS = None + bbvars = get_bb_vars() + logdir = os.environ.get("BUILDDIR") + if 'LOG_DIR' in bbvars: + logdir = bbvars['LOG_DIR'] + args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time) + super(OESelftestTestContextExecutor, self)._process_args(logger, args) if args.list_modules: @@ -110,7 +116,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor): elif args.list_tests: args.list_tests = 'name' - self.tc_kwargs['init']['td'] = get_bb_vars() + self.tc_kwargs['init']['td'] = bbvars self.tc_kwargs['init']['machines'] = self._get_available_machines() builddir = os.environ.get("BUILDDIR") @@ -174,7 +180,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor): self.tc.logger.info("\t%s" % l) sys.path.extend(layer_libdirs) - imp.reload(oeqa.selftest) + importlib.reload(oeqa.selftest) _check_required_env_variables(["BUILDDIR"]) _check_presence_meta_selftest() @@ -196,6 +202,28 @@ class OESelftestTestContextExecutor(OETestContextExecutor): self.tc.logger.info("Running bitbake -p") runCmd("bitbake -p") + def get_json_result_dir(self, args): + json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') + if "OEQA_JSON_RESULT_DIR" in self.tc.td: + json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"] + + return json_result_dir + + def get_configuration(self, args): + import platform + from oeqa.utils.metadata import metadata_from_bb + metadata = metadata_from_bb() + configuration = {'TEST_TYPE': 'oeselftest', + 'STARTTIME': args.test_start_time, + 'MACHINE': self.tc.td["MACHINE"], + 'HOST_DISTRO': ('-'.join(platform.linux_distribution())).replace(' ', '-'), + 'HOST_NAME': metadata['hostname'], + 'LAYERS': metadata['layers']} + return configuration + + def get_result_id(self, configuration): + return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME']) + def _internal_run(self, logger, args): self.module_paths = self._get_cases_paths( self.tc_kwargs['init']['td']['BBPATH'].split(':')) @@ -212,7 +240,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor): else: self._pre_run() rc = self.tc.runTests(**self.tc_kwargs['run']) - rc.logDetails() + configuration = self.get_configuration(args) + rc.logDetails(self.get_json_result_dir(args), + configuration, + self.get_result_id(configuration)) rc.logSummary(self.name) return rc @@ -270,7 +301,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor): output_link = os.path.join(os.path.dirname(args.output_log), "%s-results.log" % self.name) - if os.path.exists(output_link): + if os.path.lexists(output_link): os.remove(output_link) os.symlink(args.output_log, output_link) diff --git a/poky/meta/lib/oeqa/utils/buildproject.py b/poky/meta/lib/oeqa/utils/buildproject.py index 721f35d996..88e7b7fe27 100644 --- a/poky/meta/lib/oeqa/utils/buildproject.py +++ b/poky/meta/lib/oeqa/utils/buildproject.py @@ -17,7 +17,8 @@ class BuildProject(metaclass=ABCMeta): self.uri = uri self.archive = os.path.basename(uri) if not tmpdir: - tmpdir = tempfile.mkdtemp(prefix='buildproject') + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name self.localarchive = os.path.join(tmpdir, self.archive) self.dl_dir = dl_dir if foldername: diff --git a/poky/meta/lib/oeqa/utils/commands.py b/poky/meta/lib/oeqa/utils/commands.py index 0d9cf23fe4..2e6a2289cd 100644 --- a/poky/meta/lib/oeqa/utils/commands.py +++ b/poky/meta/lib/oeqa/utils/commands.py @@ -146,6 +146,9 @@ class Command(object): # At this point we know that the process has closed stdout/stderr, so # it is safe and necessary to wait for the actual process completion. self.status = self.process.wait() + self.process.stdout.close() + if self.process.stderr: + self.process.stderr.close() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) # logging the complete output is insane @@ -333,16 +336,20 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, try: qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) except bb.build.FuncFailed: - raise Exception('Failed to start QEMU - see the logs in %s' % logdir) + msg = 'Failed to start QEMU - see the logs in %s' % logdir + if os.path.exists(qemu.qemurunnerlog): + with open(qemu.qemurunnerlog, 'r') as f: + msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read()) + raise Exception(msg) yield qemu finally: + targetlogger.removeHandler(handler) try: qemu.stop() except: pass - targetlogger.removeHandler(handler) def updateEnv(env_file): """ diff --git a/poky/meta/lib/oeqa/utils/metadata.py b/poky/meta/lib/oeqa/utils/metadata.py index 65bbdc61f4..b7def77224 100644 --- a/poky/meta/lib/oeqa/utils/metadata.py +++ b/poky/meta/lib/oeqa/utils/metadata.py @@ -58,9 +58,22 @@ def metadata_from_data_store(d): def git_rev_info(path): """Get git revision information as a dict""" - from git import Repo, InvalidGitRepositoryError, NoSuchPathError - info = OrderedDict() + + try: + from git import Repo, InvalidGitRepositoryError, NoSuchPathError + except ImportError: + import subprocess + try: + info['branch'] = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + try: + info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + return info + try: repo = Repo(path, search_parent_directories=True) except (InvalidGitRepositoryError, NoSuchPathError): diff --git a/poky/meta/lib/oeqa/utils/qemurunner.py b/poky/meta/lib/oeqa/utils/qemurunner.py index c962602a63..b87d7765e7 100644 --- a/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/poky/meta/lib/oeqa/utils/qemurunner.py @@ -135,7 +135,7 @@ class QemuRunner: def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): try: - threadsock, threadport = self.create_socket() + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) @@ -203,8 +203,8 @@ class QemuRunner: # No point waiting any longer self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() - self.stop() self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) + self.stop() return False time.sleep(0.5) @@ -216,8 +216,8 @@ class QemuRunner: processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() - self.stop() op = self.getOutput(output) + self.stop() if op: self.logger.error("Output from runqemu:\n%s" % op) else: @@ -238,13 +238,13 @@ class QemuRunner: # because is possible to have control characters cmdline = re_control_char.sub(' ', cmdline) try: - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) self.ip = ips[0] self.server_ip = ips[1] self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output - match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE|re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() @@ -263,7 +263,7 @@ class QemuRunner: self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, threadsock, self.logger) + self.thread = LoggingThread(self.log, self.threadsock, self.logger) self.thread.start() if not self.thread.connection_established.wait(self.boottime): self.logger.error("Didn't receive a console connection from qemu. " @@ -331,14 +331,14 @@ class QemuRunner: # If we are not able to login the tests can continue try: (status, output) = self.run_serial("root\n", raw=True) - if re.search("root@[a-zA-Z0-9\-]+:~#", output): + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: # configure guest networking cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) output = self.run_serial(cmd, raw=True)[1] - if re.search("root@[a-zA-Z0-9\-]+:~#", output): + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): self.logger.debug("configured ip address %s", self.ip) else: self.logger.debug("Couldn't configure guest networking") @@ -369,14 +369,22 @@ class QemuRunner: if self.runqemu.poll() is None: self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) + self.runqemu.stdin.close() + self.runqemu.stdout.close() self.runqemu = None + if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None + if hasattr(self, 'threadsock') and self.threadsock: + self.threadsock.close() + self.threadsock = None self.qemupid = None self.ip = None if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) + if self.monitorpipe: + self.monitorpipe.close() def stop_qemu_system(self): if self.qemupid: @@ -436,7 +444,7 @@ class QemuRunner: if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): break else: raise Exception("No data on serial console socket") diff --git a/poky/meta/lib/oeqa/utils/targetbuild.py b/poky/meta/lib/oeqa/utils/targetbuild.py index 1202d579fb..b8db7b2aca 100644 --- a/poky/meta/lib/oeqa/utils/targetbuild.py +++ b/poky/meta/lib/oeqa/utils/targetbuild.py @@ -20,8 +20,9 @@ class BuildProject(metaclass=ABCMeta): if not tmpdir: tmpdir = self.d.getVar('WORKDIR') if not tmpdir: - tmpdir = tempfile.mkdtemp(prefix='buildproject') - self.localarchive = os.path.join(tmpdir,self.archive) + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name + self.localarchive = os.path.join(tmpdir, self.archive) if foldername: self.fname = foldername else: -- cgit v1.2.3