meta-openembedded and poky: subtree updates

Squash of the following due to dependencies among them
and OpenBMC changes:

meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7

The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:

meta-openembedded:d0748372d2
      cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
      mpv: Remove X11 dependency
poky:9052e5b32a
      package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
      pbzip2: Fix license warning

Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>

6 files changed, 91 insertions, 33 deletions

diff --git a/poky/meta/lib/oeqa/utils/commands.py b/poky/meta/lib/oeqa/utils/commands.py
index dc1e286da..f1679875d 100644
--- a/poky/meta/lib/oeqa/utils/commands.py
+++ b/poky/meta/lib/oeqa/utils/commands.py
@@ -315,15 +315,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
     try:
         tinfoil.logger.setLevel(logging.WARNING)
         import oeqa.targetcontrol
-        tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
-        tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
+        recipedata = tinfoil.parse_recipe(pn)
+        recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
+        recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
         # Tell QemuTarget() whether need find rootfs/kernel or not
         if launch_cmd:
-            tinfoil.config_data.setVar("FIND_ROOTFS", '0')
+            recipedata.setVar("FIND_ROOTFS", '0')
         else:
-            tinfoil.config_data.setVar("FIND_ROOTFS", '1')
+            recipedata.setVar("FIND_ROOTFS", '1')
 
-        recipedata = tinfoil.parse_recipe(pn)
         for key, value in overrides.items():
             recipedata.setVar(key, value)
 
diff --git a/poky/meta/lib/oeqa/utils/dump.py b/poky/meta/lib/oeqa/utils/dump.py
index d34e05e2b..09a44329e 100644
--- a/poky/meta/lib/oeqa/utils/dump.py
+++ b/poky/meta/lib/oeqa/utils/dump.py
@@ -71,8 +71,11 @@ class HostDumper(BaseDumper):
     def dump_host(self, dump_dir=""):
         if dump_dir:
             self.dump_dir = dump_dir
+        env = os.environ.copy()
+        env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin'
+        env['COLUMNS'] = '9999'
         for cmd in self.cmds:
-            result = runCmd(cmd, ignore_status=True)
+            result = runCmd(cmd, ignore_status=True, env=env)
             self._write_dump(cmd.split()[0], result.output)
 
 class TargetDumper(BaseDumper):
diff --git a/poky/meta/lib/oeqa/utils/httpserver.py b/poky/meta/lib/oeqa/utils/httpserver.py
index aa435590f..58d3c3b3f 100644
--- a/poky/meta/lib/oeqa/utils/httpserver.py
+++ b/poky/meta/lib/oeqa/utils/httpserver.py
@@ -22,10 +22,10 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
 
 class HTTPService(object):
 
-    def __init__(self, root_dir, host='', logger=None):
+    def __init__(self, root_dir, host='', port=0, logger=None):
         self.root_dir = root_dir
         self.host = host
-        self.port = 0
+        self.port = port
         self.logger = logger
 
     def start(self):
diff --git a/poky/meta/lib/oeqa/utils/logparser.py b/poky/meta/lib/oeqa/utils/logparser.py
index 7313df8ec..60e16d500 100644
--- a/poky/meta/lib/oeqa/utils/logparser.py
+++ b/poky/meta/lib/oeqa/utils/logparser.py
@@ -25,13 +25,20 @@ class PtestParser(object):
         section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)")
         section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest")
 
+        # Cache markers so we don't take the re.search() hit all the time.
+        markers = ("PASS:", "FAIL:", "SKIP:", "BEGIN:", "END:", "DURATION:", "ERROR: Exit", "TIMEOUT:")
+
         def newsection():
-            return { 'name': "No-section", 'log': "" }
+            return { 'name': "No-section", 'log': [] }
 
         current_section = newsection()
 
         with open(logfile, errors='replace') as f:
             for line in f:
+                if not line.startswith(markers):
+                    current_section['log'].append(line)
+                    continue
+
                 result = section_regex['begin'].search(line)
                 if result:
                     current_section['name'] = result.group(1)
@@ -61,7 +68,7 @@ class PtestParser(object):
                         current_section[t] = result.group(1)
                         continue
 
-                current_section['log'] = current_section['log'] + line 
+                current_section['log'].append(line)
 
                 for t in test_regex:
                     result = test_regex[t].search(line)
@@ -70,6 +77,11 @@ class PtestParser(object):
                             self.results[current_section['name']] = {}
                         self.results[current_section['name']][result.group(1).strip()] = t
 
+        # Python performance for repeatedly joining long strings is poor, do it all at once at the end.
+        # For 2.1 million lines in a log this reduces 18 hours to 12s.
+        for section in self.sections:
+            self.sections[section]['log'] = "".join(self.sections[section]['log'])
+
         return self.results, self.sections
 
     # Log the results as files. The file name is the section name and the contents are the tests in that section.
diff --git a/poky/meta/lib/oeqa/utils/qemurunner.py b/poky/meta/lib/oeqa/utils/qemurunner.py
index fe8b77d97..4b7433765 100644
--- a/poky/meta/lib/oeqa/utils/qemurunner.py
+++ b/poky/meta/lib/oeqa/utils/qemurunner.py
@@ -21,6 +21,7 @@ import threading
 import codecs
 import logging
 from oeqa.utils.dump import HostDumper
+from collections import defaultdict
 
 # Get Unicode non printable control chars
 control_range = list(range(0,32))+list(range(127,160))
@@ -31,10 +32,11 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
 class QemuRunner:
 
     def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds,
-                 use_kvm, logger, use_slirp=False):
+                 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False):
 
         # Popen object for runqemu
         self.runqemu = None
+        self.runqemu_exited = False
         # pid of the qemu process that runqemu will start
         self.qemupid = None
         # target ip - from the command line or runqemu output
@@ -54,8 +56,11 @@ class QemuRunner:
         self.logged = False
         self.thread = None
         self.use_kvm = use_kvm
+        self.use_ovmf = use_ovmf
         self.use_slirp = use_slirp
+        self.serial_ports = serial_ports
         self.msg = ''
+        self.boot_patterns = boot_patterns
 
         self.runqemutime = 120
         self.qemu_pidfile = 'pidfile_'+str(os.getpid())
@@ -64,6 +69,25 @@ class QemuRunner:
 
         self.logger = logger
 
+        # Enable testing other OS's
+        # Set commands for target communication, and default to Linux ALWAYS
+        # Other OS's or baremetal applications need to provide their
+        # own implementation passing it through QemuRunner's constructor
+        # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag]
+        # provided variables, where <flag> is one of the mentioned below.
+        accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished']
+        default_boot_patterns = defaultdict(str)
+        # Default to the usual paterns used to communicate with the target
+        default_boot_patterns['search_reached_prompt'] = b' login:'
+        default_boot_patterns['send_login_user'] = 'root\n'
+        default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#"
+        default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#"
+
+        # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n"
+        for pattern in accepted_patterns:
+            if not self.boot_patterns[pattern]:
+                self.boot_patterns[pattern] = default_boot_patterns[pattern]
+
     def create_socket(self):
         try:
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -98,11 +122,10 @@ class QemuRunner:
     def handleSIGCHLD(self, signum, frame):
         if self.runqemu and self.runqemu.poll():
             if self.runqemu.returncode:
-                self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
-                self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout))
+                self.logger.error('runqemu exited with code %d' % self.runqemu.returncode)
+                self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout))
                 self.stop()
                 self._dump_host()
-                raise SystemExit
 
     def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True):
         env = os.environ.copy()
@@ -136,13 +159,16 @@ class QemuRunner:
                 launch_cmd += ' nographic'
             if self.use_slirp:
                 launch_cmd += ' slirp'
+            if self.use_ovmf:
+                launch_cmd += ' ovmf'
             launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs)
 
         return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env)
 
     def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None):
         try:
-            self.threadsock, threadport = self.create_socket()
+            if self.serial_ports >= 2:
+                self.threadsock, threadport = self.create_socket()
             self.server_socket, self.serverport = self.create_socket()
         except socket.error as msg:
             self.logger.error("Failed to create listening socket: %s" % msg[1])
@@ -160,7 +186,10 @@ class QemuRunner:
         if qemuparams:
             self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
 
-        launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams)
+        if self.serial_ports >= 2:
+            launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams)
+        else:
+            launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)
 
         self.origchldhandler = signal.getsignal(signal.SIGCHLD)
         signal.signal(signal.SIGCHLD, self.handleSIGCHLD)
@@ -206,6 +235,8 @@ class QemuRunner:
         endtime = time.time() + self.runqemutime
         while not self.is_alive() and time.time() < endtime:
             if self.runqemu.poll():
+                if self.runqemu_exited:
+                    return False
                 if self.runqemu.returncode:
                     # No point waiting any longer
                     self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
@@ -215,6 +246,9 @@ class QemuRunner:
                     return False
             time.sleep(0.5)
 
+        if self.runqemu_exited:
+            return False
+
         if not self.is_alive():
             self.logger.error("Qemu pid didn't appear in %s seconds (%s)" %
                               (self.runqemutime, time.strftime("%D %H:%M:%S")))
@@ -237,8 +271,8 @@ class QemuRunner:
         self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" %
                           (time.time() - (endtime - self.runqemutime),
                            self.qemupid, time.strftime("%D %H:%M:%S")))
+        cmdline = ''
         if get_ip:
-            cmdline = ''
             with open('/proc/%s/cmdline' % self.qemupid) as p:
                 cmdline = p.read()
                 # It is needed to sanitize the data received
@@ -275,14 +309,15 @@ class QemuRunner:
         self.logger.debug("Target IP: %s" % self.ip)
         self.logger.debug("Server IP: %s" % self.server_ip)
 
-        self.thread = LoggingThread(self.log, self.threadsock, self.logger)
-        self.thread.start()
-        if not self.thread.connection_established.wait(self.boottime):
-            self.logger.error("Didn't receive a console connection from qemu. "
-                         "Here is the qemu command line used:\n%s\nand "
-                         "output from runqemu:\n%s" % (cmdline, out))
-            self.stop_thread()
-            return False
+        if self.serial_ports >= 2:
+            self.thread = LoggingThread(self.log, self.threadsock, self.logger)
+            self.thread.start()
+            if not self.thread.connection_established.wait(self.boottime):
+                self.logger.error("Didn't receive a console connection from qemu. "
+                             "Here is the qemu command line used:\n%s\nand "
+                             "output from runqemu:\n%s" % (cmdline, out))
+                self.stop_thread()
+                return False
 
         self.logger.debug("Output from runqemu:\n%s", out)
         self.logger.debug("Waiting at most %d seconds for login banner (%s)" %
@@ -310,8 +345,12 @@ class QemuRunner:
                     data = data + sock.recv(1024)
                     if data:
                         bootlog += data
+                        if self.serial_ports < 2:
+                            # this socket has mixed console/kernel data, log it to logfile
+                            self.log(data)
+
                         data = b''
-                        if b' login:' in bootlog:
+                        if self.boot_patterns['search_reached_prompt'] in bootlog:
                             self.server_socket = qemusock
                             stopread = True
                             reachedlogin = True
@@ -343,8 +382,8 @@ class QemuRunner:
 
         # If we are not able to login the tests can continue
         try:
-            (status, output) = self.run_serial("root\n", raw=True)
-            if re.search(r"root@[a-zA-Z0-9\-]+:~#", output):
+            (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True)
+            if re.search(self.boot_patterns['search_login_succeeded'], output):
                 self.logged = True
                 self.logger.debug("Logged as root in serial console")
                 if netconf:
@@ -385,7 +424,7 @@ class QemuRunner:
                 os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL)
             self.runqemu.stdin.close()
             self.runqemu.stdout.close()
-            self.runqemu = None
+            self.runqemu_exited = True
 
         if hasattr(self, 'server_socket') and self.server_socket:
             self.server_socket.close()
@@ -396,7 +435,11 @@ class QemuRunner:
         self.qemupid = None
         self.ip = None
         if os.path.exists(self.qemu_pidfile):
-            os.remove(self.qemu_pidfile)
+            try:
+                os.remove(self.qemu_pidfile)
+            except FileNotFoundError as e:
+                # We raced, ignore
+                pass
         if self.monitorpipe:
             self.monitorpipe.close()
 
@@ -422,7 +465,7 @@ class QemuRunner:
         return False
 
     def is_alive(self):
-        if not self.runqemu or self.runqemu.poll() is not None:
+        if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited:
             return False
         if os.path.isfile(self.qemu_pidfile):
             # when handling pidfile, qemu creates the file, stat it, lock it and then write to it
@@ -465,7 +508,7 @@ class QemuRunner:
                 if answer:
                     data += answer.decode('utf-8')
                     # Search the prompt to stop
-                    if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data):
+                    if re.search(self.boot_patterns['search_cmd_finished'], data):
                         break
                 else:
                     raise Exception("No data on serial console socket")
diff --git a/poky/meta/lib/oeqa/utils/sshcontrol.py b/poky/meta/lib/oeqa/utils/sshcontrol.py
index 49a07264c..36c2ecb3d 100644
--- a/poky/meta/lib/oeqa/utils/sshcontrol.py
+++ b/poky/meta/lib/oeqa/utils/sshcontrol.py
@@ -23,7 +23,7 @@ class SSHProcess(object):
             "stdin": None,
             "shell": False,
             "bufsize": -1,
-            "preexec_fn": os.setsid,
+            "start_new_session": True,
         }
         self.options = dict(self.defaultopts)
         self.options.update(options)