From f86d0556ed6040140f3d635b0bb2eeb1c507f818 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Tue, 4 Dec 2018 14:18:15 -0800 Subject: poky: sumo refresh a4c7d28688..78020fb639 Update poky to sumo HEAD. Michael Halstead (1): scripts/runqemu: Replace subprocess.run() for compatibilty Richard Purdie (35): scripts/runqemu: Tidy up lock handling code scripts/runqemu: Improve lockfile handling for python with close_fd=True oeqa/selftest/signing: Skip tests if gpg isn't found oeqa/selftest/signing: Allow tests not to need gpg on the host oeqa/selftest/signing: Use do_populate_lic target instead of do_package oeqa/selftest/case: Use bb.utils.remove() instead of shutil.remove() oeqa/selftest/buildoptions: Improve ccache test failure output oeqa/utils/commands: Add extra qemu failure logging oeqa/utils/qemurunner: Fix python ResourceWarning for unclosed file oeqa/utils/commands: Avoid log message duplication oeqa/qemurunner: Remove resource python warnings oeqa/selftest/buildoptions: Improve ccache test oeqa/selftest/buildoptions: Ensure diskmon tests run consistently oeqa/selftest/runqemu: Improve testcase failure handling oeqa/utils/qemurunner: Avoid tracebacks on closed files oeqa/loader: Fix deprecation warning oeqa/utils/commands: Avoid unclosed file warnings oeqa/selftest/context: Replace deprecated imp module usage oeqa/utils/qemurunner.py: Fix python regex warnings oeqa/selftest/context: Improve log file handling oeqa/core/runner: Improve test case comparision oeqa/runner: Ensure we don't print misleading results output oeqa/core/threaded: Remove in favour of using concurrenttests oeqa/runner: Simplify code oeqa: Remove xmlrunner oeqa/runtime/ptest: Inject results+logs into stored json results file oeqa/runner: Always show a summary of success/fail/error/skip counts oeqa/runner: Sort the test result output by result class oeqa/selftest: Improvements to the json logging oeqa/utils/metadata: Allow to function without the git module image-buildinfo,oeqa/selftest/containerimage: Ensure image-buildinfo doesn't break tests oeqa/selftest/esdk: Ensure parent directory exists oeqa/selftest/esdk: Fix typo causing test failure testimage: Improvements to the json logging testsdk: Improvements to the json logging Ross Burton (3): oeqa/oelib/path: don't leak temporary directories oeqa: don't litter /tmp with temporary directories oeqa/selftest/esdk: run selftest inside workdir not /tmp Scott Rifenbark (1): documentation: Prepared for 2.5.2 document release Stefan Lendl (1): default-versions.inc: Make PREFERRED_VERSION_openssl* overwritable Yeoh Ee Peng (6): oeqa/core/runner: refactor for OEQA to write json testresult oeqa/core/runner: write testresult to json files oeqa/selftest/context: write testresult to json files testimage.bbclass: write testresult to json files testsdk.bbclass: write testresult to json files oeqa/selftest: Standardize json logging output directory Change-Id: I3c8123930c8c3441126c1e81b3bbbde9f2b126f2 Signed-off-by: Brad Bishop --- poky/meta/lib/oeqa/utils/buildproject.py | 3 ++- poky/meta/lib/oeqa/utils/commands.py | 11 +++++++++-- poky/meta/lib/oeqa/utils/metadata.py | 17 +++++++++++++++-- poky/meta/lib/oeqa/utils/qemurunner.py | 26 +++++++++++++++++--------- poky/meta/lib/oeqa/utils/targetbuild.py | 5 +++-- 5 files changed, 46 insertions(+), 16 deletions(-) (limited to 'poky/meta/lib/oeqa/utils') diff --git a/poky/meta/lib/oeqa/utils/buildproject.py b/poky/meta/lib/oeqa/utils/buildproject.py index 721f35d996..88e7b7fe27 100644 --- a/poky/meta/lib/oeqa/utils/buildproject.py +++ b/poky/meta/lib/oeqa/utils/buildproject.py @@ -17,7 +17,8 @@ class BuildProject(metaclass=ABCMeta): self.uri = uri self.archive = os.path.basename(uri) if not tmpdir: - tmpdir = tempfile.mkdtemp(prefix='buildproject') + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name self.localarchive = os.path.join(tmpdir, self.archive) self.dl_dir = dl_dir if foldername: diff --git a/poky/meta/lib/oeqa/utils/commands.py b/poky/meta/lib/oeqa/utils/commands.py index 0d9cf23fe4..2e6a2289cd 100644 --- a/poky/meta/lib/oeqa/utils/commands.py +++ b/poky/meta/lib/oeqa/utils/commands.py @@ -146,6 +146,9 @@ class Command(object): # At this point we know that the process has closed stdout/stderr, so # it is safe and necessary to wait for the actual process completion. self.status = self.process.wait() + self.process.stdout.close() + if self.process.stderr: + self.process.stderr.close() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) # logging the complete output is insane @@ -333,16 +336,20 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, try: qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) except bb.build.FuncFailed: - raise Exception('Failed to start QEMU - see the logs in %s' % logdir) + msg = 'Failed to start QEMU - see the logs in %s' % logdir + if os.path.exists(qemu.qemurunnerlog): + with open(qemu.qemurunnerlog, 'r') as f: + msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read()) + raise Exception(msg) yield qemu finally: + targetlogger.removeHandler(handler) try: qemu.stop() except: pass - targetlogger.removeHandler(handler) def updateEnv(env_file): """ diff --git a/poky/meta/lib/oeqa/utils/metadata.py b/poky/meta/lib/oeqa/utils/metadata.py index 65bbdc61f4..b7def77224 100644 --- a/poky/meta/lib/oeqa/utils/metadata.py +++ b/poky/meta/lib/oeqa/utils/metadata.py @@ -58,9 +58,22 @@ def metadata_from_data_store(d): def git_rev_info(path): """Get git revision information as a dict""" - from git import Repo, InvalidGitRepositoryError, NoSuchPathError - info = OrderedDict() + + try: + from git import Repo, InvalidGitRepositoryError, NoSuchPathError + except ImportError: + import subprocess + try: + info['branch'] = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + try: + info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + return info + try: repo = Repo(path, search_parent_directories=True) except (InvalidGitRepositoryError, NoSuchPathError): diff --git a/poky/meta/lib/oeqa/utils/qemurunner.py b/poky/meta/lib/oeqa/utils/qemurunner.py index c962602a63..b87d7765e7 100644 --- a/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/poky/meta/lib/oeqa/utils/qemurunner.py @@ -135,7 +135,7 @@ class QemuRunner: def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): try: - threadsock, threadport = self.create_socket() + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) @@ -203,8 +203,8 @@ class QemuRunner: # No point waiting any longer self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() - self.stop() self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) + self.stop() return False time.sleep(0.5) @@ -216,8 +216,8 @@ class QemuRunner: processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() - self.stop() op = self.getOutput(output) + self.stop() if op: self.logger.error("Output from runqemu:\n%s" % op) else: @@ -238,13 +238,13 @@ class QemuRunner: # because is possible to have control characters cmdline = re_control_char.sub(' ', cmdline) try: - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) self.ip = ips[0] self.server_ip = ips[1] self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output - match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE|re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() @@ -263,7 +263,7 @@ class QemuRunner: self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, threadsock, self.logger) + self.thread = LoggingThread(self.log, self.threadsock, self.logger) self.thread.start() if not self.thread.connection_established.wait(self.boottime): self.logger.error("Didn't receive a console connection from qemu. " @@ -331,14 +331,14 @@ class QemuRunner: # If we are not able to login the tests can continue try: (status, output) = self.run_serial("root\n", raw=True) - if re.search("root@[a-zA-Z0-9\-]+:~#", output): + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: # configure guest networking cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) output = self.run_serial(cmd, raw=True)[1] - if re.search("root@[a-zA-Z0-9\-]+:~#", output): + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): self.logger.debug("configured ip address %s", self.ip) else: self.logger.debug("Couldn't configure guest networking") @@ -369,14 +369,22 @@ class QemuRunner: if self.runqemu.poll() is None: self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) + self.runqemu.stdin.close() + self.runqemu.stdout.close() self.runqemu = None + if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None + if hasattr(self, 'threadsock') and self.threadsock: + self.threadsock.close() + self.threadsock = None self.qemupid = None self.ip = None if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) + if self.monitorpipe: + self.monitorpipe.close() def stop_qemu_system(self): if self.qemupid: @@ -436,7 +444,7 @@ class QemuRunner: if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): break else: raise Exception("No data on serial console socket") diff --git a/poky/meta/lib/oeqa/utils/targetbuild.py b/poky/meta/lib/oeqa/utils/targetbuild.py index 1202d579fb..b8db7b2aca 100644 --- a/poky/meta/lib/oeqa/utils/targetbuild.py +++ b/poky/meta/lib/oeqa/utils/targetbuild.py @@ -20,8 +20,9 @@ class BuildProject(metaclass=ABCMeta): if not tmpdir: tmpdir = self.d.getVar('WORKDIR') if not tmpdir: - tmpdir = tempfile.mkdtemp(prefix='buildproject') - self.localarchive = os.path.join(tmpdir,self.archive) + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name + self.localarchive = os.path.join(tmpdir, self.archive) if foldername: self.fname = foldername else: -- cgit v1.2.3