poky: sumo refresh a4c7d28688..78020fb639

Update poky to sumo HEAD.

Michael Halstead (1):
      scripts/runqemu: Replace subprocess.run() for compatibilty

Richard Purdie (35):
      scripts/runqemu: Tidy up lock handling code
      scripts/runqemu: Improve lockfile handling for python with close_fd=True
      oeqa/selftest/signing: Skip tests if gpg isn't found
      oeqa/selftest/signing: Allow tests not to need gpg on the host
      oeqa/selftest/signing: Use do_populate_lic target instead of do_package
      oeqa/selftest/case: Use bb.utils.remove() instead of shutil.remove()
      oeqa/selftest/buildoptions: Improve ccache test failure output
      oeqa/utils/commands: Add extra qemu failure logging
      oeqa/utils/qemurunner: Fix python ResourceWarning for unclosed file
      oeqa/utils/commands: Avoid log message duplication
      oeqa/qemurunner: Remove resource python warnings
      oeqa/selftest/buildoptions: Improve ccache test
      oeqa/selftest/buildoptions: Ensure diskmon tests run consistently
      oeqa/selftest/runqemu: Improve testcase failure handling
      oeqa/utils/qemurunner: Avoid tracebacks on closed files
      oeqa/loader: Fix deprecation warning
      oeqa/utils/commands: Avoid unclosed file warnings
      oeqa/selftest/context: Replace deprecated imp module usage
      oeqa/utils/qemurunner.py: Fix python regex warnings
      oeqa/selftest/context: Improve log file handling
      oeqa/core/runner: Improve test case comparision
      oeqa/runner: Ensure we don't print misleading results output
      oeqa/core/threaded: Remove in favour of using concurrenttests
      oeqa/runner: Simplify code
      oeqa: Remove xmlrunner
      oeqa/runtime/ptest: Inject results+logs into stored json results file
      oeqa/runner: Always show a summary of success/fail/error/skip counts
      oeqa/runner: Sort the test result output by result class
      oeqa/selftest: Improvements to the json logging
      oeqa/utils/metadata: Allow to function without the git module
      image-buildinfo,oeqa/selftest/containerimage: Ensure image-buildinfo doesn't break tests
      oeqa/selftest/esdk: Ensure parent directory exists
      oeqa/selftest/esdk: Fix typo causing test failure
      testimage: Improvements to the json logging
      testsdk: Improvements to the json logging

Ross Burton (3):
      oeqa/oelib/path: don't leak temporary directories
      oeqa: don't litter /tmp with temporary directories
      oeqa/selftest/esdk: run selftest inside workdir not /tmp

Scott Rifenbark (1):
      documentation: Prepared for 2.5.2 document release

Stefan Lendl (1):
      default-versions.inc: Make PREFERRED_VERSION_openssl* overwritable

Yeoh Ee Peng (6):
      oeqa/core/runner: refactor for OEQA to write json testresult
      oeqa/core/runner: write testresult to json files
      oeqa/selftest/context: write testresult to json files
      testimage.bbclass: write testresult to json files
      testsdk.bbclass: write testresult to json files
      oeqa/selftest: Standardize json logging output directory

Change-Id: I3c8123930c8c3441126c1e81b3bbbde9f2b126f2
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>

6 files changed, 93 insertions, 56 deletions

diff --git a/poky/meta/lib/oeqa/selftest/cases/buildoptions.py b/poky/meta/lib/oeqa/selftest/cases/buildoptions.py
index e60e32dadf..24597ac192 100644
--- a/poky/meta/lib/oeqa/selftest/cases/buildoptions.py
+++ b/poky/meta/lib/oeqa/selftest/cases/buildoptions.py
@@ -38,10 +38,12 @@ class ImageOptionsTests(OESelftestTestCase):
         self.assertTrue(os.path.isfile(p), msg = "No ccache found (%s)" % p)
         self.write_config('INHERIT += "ccache"')
         self.add_command_to_tearDown('bitbake -c clean m4')
+        bitbake("m4 -c clean")
         bitbake("m4 -f -c compile")
         log_compile = os.path.join(get_bb_var("WORKDIR","m4"), "temp/log.do_compile")
-        res = runCmd("grep ccache %s" % log_compile, ignore_status=True)
-        self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile)
+        with open(log_compile, "r") as f:
+            loglines = "".join(f.readlines())
+        self.assertIn("ccache", loglines, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile)
 
     @OETestID(1435)
     def test_read_only_image(self):
@@ -57,15 +59,15 @@ class DiskMonTest(OESelftestTestCase):
     @OETestID(277)
     def test_stoptask_behavior(self):
         self.write_config('BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},100000G,100K"')
-        res = bitbake("m4", ignore_status = True)
+        res = bitbake("delay -c delay", ignore_status = True)
         self.assertTrue('ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"!' in res.output, msg = "Tasks should have stopped. Disk monitor is set to STOPTASK: %s" % res.output)
         self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output))
         self.write_config('BB_DISKMON_DIRS = "ABORT,${TMPDIR},100000G,100K"')
-        res = bitbake("m4", ignore_status = True)
+        res = bitbake("delay -c delay", ignore_status = True)
         self.assertTrue('ERROR: Immediately abort since the disk space monitor action is "ABORT"!' in res.output, "Tasks should have been aborted immediatelly. Disk monitor is set to ABORT: %s" % res.output)
         self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output))
         self.write_config('BB_DISKMON_DIRS = "WARN,${TMPDIR},100000G,100K"')
-        res = bitbake("m4")
+        res = bitbake("delay -c delay")
         self.assertTrue('WARNING: The free space' in res.output, msg = "A warning should have been displayed for disk monitor is set to WARN: %s" %res.output)
 
 class SanityOptionsTest(OESelftestTestCase):
diff --git a/poky/meta/lib/oeqa/selftest/cases/containerimage.py b/poky/meta/lib/oeqa/selftest/cases/containerimage.py
index 99a5cc9e57..8deaae75d8 100644
--- a/poky/meta/lib/oeqa/selftest/cases/containerimage.py
+++ b/poky/meta/lib/oeqa/selftest/cases/containerimage.py
@@ -39,6 +39,7 @@ class ContainerImageTests(OESelftestTestCase):
 IMAGE_FSTYPES = "container"
 PACKAGE_CLASSES = "package_ipk"
 IMAGE_FEATURES = ""
+IMAGE_BUILDINFO_FILE = ""
 """)
 
         bbvars = get_bb_vars(['bindir', 'sysconfdir', 'localstatedir',
diff --git a/poky/meta/lib/oeqa/selftest/cases/eSDK.py b/poky/meta/lib/oeqa/selftest/cases/eSDK.py
index d03188f2f7..d7aef93991 100644
--- a/poky/meta/lib/oeqa/selftest/cases/eSDK.py
+++ b/poky/meta/lib/oeqa/selftest/cases/eSDK.py
@@ -70,11 +70,13 @@ CORE_IMAGE_EXTRA_INSTALL = "perl"
     @classmethod
     def setUpClass(cls):
         super(oeSDKExtSelfTest, cls).setUpClass()
-        cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA')
+        cls.image = 'core-image-minimal'
 
-        sstate_dir = get_bb_var('SSTATE_DIR')
+        bb_vars = get_bb_vars(['SSTATE_DIR', 'WORKDIR'], cls.image)
+        bb.utils.mkdirhier(bb_vars["WORKDIR"])
+        cls.tmpdirobj = tempfile.TemporaryDirectory(prefix="selftest-esdk-", dir=bb_vars["WORKDIR"])
+        cls.tmpdir_eSDKQA = cls.tmpdirobj.name
 
-        cls.image = 'core-image-minimal'
         oeSDKExtSelfTest.generate_eSDK(cls.image)
 
         # Install eSDK
@@ -87,14 +89,14 @@ CORE_IMAGE_EXTRA_INSTALL = "perl"
         sstate_config="""
 SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS"
 SSTATE_MIRRORS =  "file://.* file://%s/PATH"
-            """ % sstate_dir
+            """ % bb_vars["SSTATE_DIR"]
         with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f:
             f.write(sstate_config)
 
     @classmethod
     def tearDownClass(cls):
-        shutil.rmtree(cls.tmpdir_eSDKQA, ignore_errors=True)
-        super(oeSDKExtSelfTest, cls).tearDownClass()
+        cls.tmpdirobj.cleanup()
+        super().tearDownClass()
 
     @OETestID(1602)
     def test_install_libraries_headers(self):
diff --git a/poky/meta/lib/oeqa/selftest/cases/oelib/path.py b/poky/meta/lib/oeqa/selftest/cases/oelib/path.py
index 75a27c06f7..e0eb8134a9 100644
--- a/poky/meta/lib/oeqa/selftest/cases/oelib/path.py
+++ b/poky/meta/lib/oeqa/selftest/cases/oelib/path.py
@@ -38,13 +38,6 @@ class TestRealPath(TestCase):
         ( "b/test", errno.ENOENT ),
     ]
 
-    def __del__(self):
-        try:
-            #os.system("tree -F %s" % self.tmpdir)
-            shutil.rmtree(self.tmpdir)
-        except:
-            pass
-
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix = "oe-test_path")
         self.root = os.path.join(self.tmpdir, "R")
@@ -59,6 +52,9 @@ class TestRealPath(TestCase):
         for l in self.LINKS:
             os.symlink(l[1], os.path.join(self.root, l[0]))
 
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
     def __realpath(self, file, use_physdir, assume_dir = True):
         return oe.path.realpath(os.path.join(self.root, file), self.root,
                                 use_physdir, assume_dir = assume_dir)
diff --git a/poky/meta/lib/oeqa/selftest/cases/runqemu.py b/poky/meta/lib/oeqa/selftest/cases/runqemu.py
index 5ebdd57a41..a758aafbdd 100644
--- a/poky/meta/lib/oeqa/selftest/cases/runqemu.py
+++ b/poky/meta/lib/oeqa/selftest/cases/runqemu.py
@@ -44,7 +44,8 @@ SYSLINUX_TIMEOUT = "10"
         """Test runqemu machine"""
         cmd = "%s %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2002)
     def test_boot_machine_ext4(self):
@@ -52,7 +53,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s ext4" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('rootfs.ext4' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('rootfs.ext4', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2003)
     def test_boot_machine_iso(self):
@@ -60,14 +61,16 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s iso" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('media=cdrom' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('media=cdrom', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2004)
     def test_boot_recipe_image(self):
         """Test runqemu recipe-image"""
         cmd = "%s %s" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
     @OETestID(2005)
     def test_boot_recipe_image_vmdk(self):
@@ -75,7 +78,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s wic.vmdk" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=vmdk', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2006)
     def test_boot_recipe_image_vdi(self):
@@ -83,14 +86,16 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s wic.vdi" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=vdi', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2007)
     def test_boot_deploy(self):
         """Test runqemu deploy_dir_image"""
         cmd = "%s %s" % (self.cmd_common, self.deploy_dir_image)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
     @OETestID(2008)
     def test_boot_deploy_hddimg(self):
@@ -98,7 +103,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s" % cmd)
+                self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2009)
     def test_boot_machine_slirp(self):
@@ -106,7 +111,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s slirp %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue(' -netdev user' in f.read(), "Failed: %s" % cmd)
+                self.assertIn(' -netdev user', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2009)
     def test_boot_machine_slirp_qcow2(self):
@@ -114,7 +119,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s slirp wic.qcow2 %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=qcow2', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2010)
     def test_boot_qemu_boot(self):
@@ -125,7 +130,8 @@ SYSLINUX_TIMEOUT = "10"
             self.skipTest("%s not found" % qemuboot_conf)
         cmd = "%s %s" % (self.cmd_common, qemuboot_conf)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2011)
     def test_boot_rootfs(self):
@@ -136,7 +142,9 @@ SYSLINUX_TIMEOUT = "10"
             self.skipTest("%s not found" % rootfs)
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
 # This test was designed as a separate class to test that shutdown
 # command will shutdown qemu as expected on each qemu architecture
diff --git a/poky/meta/lib/oeqa/selftest/cases/signing.py b/poky/meta/lib/oeqa/selftest/cases/signing.py
index a750cfc7b2..0edaf400bb 100644
--- a/poky/meta/lib/oeqa/selftest/cases/signing.py
+++ b/poky/meta/lib/oeqa/selftest/cases/signing.py
@@ -1,10 +1,12 @@
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
 import os
+import oe
 import glob
 import re
 import shutil
 import tempfile
+from contextlib import contextmanager
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.utils.ftools import write_file
 
@@ -15,23 +17,39 @@ class Signing(OESelftestTestCase):
     pub_key_path = ""
     secret_key_path = ""
 
-    @classmethod
-    def setUpClass(cls):
-        super(Signing, cls).setUpClass()
-        # Check that we can find the gpg binary and fail early if we can't
-        if not shutil.which("gpg"):
-            raise AssertionError("This test needs GnuPG")
+    def setup_gpg(self):
+        bitbake('gnupg-native -c addto_recipe_sysroot')
 
-        cls.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-")
+        self.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-")
+        self.track_for_cleanup(self.gpg_dir)
 
-        cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub")
-        cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret")
+        self.pub_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.pub")
+        self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret")
 
-        runCmd('gpg --batch --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path))
+        nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native")
+        runCmd('gpg --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot)
+        return nsysroot + get_bb_var("bindir_native")
 
-    @classmethod
-    def tearDownClass(cls):
-        shutil.rmtree(cls.gpg_dir, ignore_errors=True)
+
+    @contextmanager
+    def create_new_builddir(self, builddir, newbuilddir):
+        bb.utils.mkdirhier(newbuilddir)
+        oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
+        oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
+
+        origenv = os.environ.copy()
+
+        for e in os.environ:
+            if builddir in os.environ[e]:
+                os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
+
+        os.chdir(newbuilddir)
+        try:
+            yield
+        finally:
+            for e in origenv:
+                os.environ[e] = origenv[e]
+            os.chdir(builddir)
 
     @OETestID(1362)
     def test_signing_packages(self):
@@ -46,6 +64,8 @@ class Signing(OESelftestTestCase):
         """
         import oe.packagedata
 
+        self.setup_gpg()
+
         package_classes = get_bb_var('PACKAGE_CLASSES')
         if 'package_rpm' not in package_classes:
             self.skipTest('This test requires RPM Packaging.')
@@ -108,11 +128,12 @@ class Signing(OESelftestTestCase):
 
         test_recipe = 'ed'
 
-        builddir = os.environ.get('BUILDDIR')
+        # Since we need gpg but we can't use gpg-native for sstate signatures, we 
+        # build gpg-native in our original builddir then run the tests in a second one.
+        builddir = os.environ.get('BUILDDIR') + "-testsign"
         sstatedir = os.path.join(builddir, 'test-sstate')
 
-        self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
-        self.add_command_to_tearDown('rm -rf %s' % sstatedir)
+        nsysroot = self.setup_gpg()
 
         feature = 'SSTATE_SIG_KEY ?= "testuser"\n'
         feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n'
@@ -124,19 +145,26 @@ class Signing(OESelftestTestCase):
 
         self.write_config(feature)
 
-        bitbake('-c clean %s' % test_recipe)
-        bitbake(test_recipe)
+        with self.create_new_builddir(os.environ['BUILDDIR'], builddir):
+
+            os.environ["PATH"] = nsysroot + ":" + os.environ["PATH"]
+            self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
+            self.add_command_to_tearDown('rm -rf %s' % sstatedir)
+            self.add_command_to_tearDown('rm -rf %s' % builddir)
+
+            bitbake('-c clean %s' % test_recipe)
+            bitbake('-c populate_lic %s' % test_recipe)
 
-        recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig')
-        recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_package.tgz')
+            recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig')
+            recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz')
 
-        self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
-        self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
+            self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
+            self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
 
-        ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0]))
-        # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30
-        # gpg: Good signature from "testuser (nocomment) <testuser@email.com>"
-        self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.')
+            ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0]))
+            # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30
+            # gpg: Good signature from "testuser (nocomment) <testuser@email.com>"
+            self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.')
 
 
 class LockedSignatures(OESelftestTestCase):