poky: refresh thud: 1cab405d88..3541f019a5

Update poky to thud HEAD.

Alejandro Enedino Hernandez Samaniego (1):
      bitbake: cooker: fix indirect multiconfig dependencies

Richard Purdie (8):
      bitbake: server/process: Increase server startup time delay
      bitbake: cooker: Add some timing debug messages to the server startup
      bitbake: server/process: Handle short reads
      bitbake: lib/bb/server: Avoid UnboundLocalError traceback
      bitbake: server/process: Ensure socket has a timeout set
      bitbake: process.py: Set socket timeout to 10 seconds
      bitbake: process: Handle EWOULDBLOCK in socket connect
      bitbake: process: Rewrite multiple connection handling

Scott Rifenbark (3):
      poky.ent: Updated release date to "January 2019"
      brief-yoctoprojectqs, dev-manual: Update tag checkout examples
      dev-manual, brief-yoctoprojectqs: Added "2.6.1" tag to examples

Change-Id: Ifd3211fa9513bc5c05cf202d11d65c5c4b48f533
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>

5 files changed, 174 insertions, 65 deletions

diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py
index 16681ba244..db52964c3a 100644
--- a/poky/bitbake/lib/bb/cooker.py
+++ b/poky/bitbake/lib/bb/cooker.py
@@ -175,18 +175,31 @@ class BBCooker:
 
         self.configuration = configuration
 
+        bb.debug(1, "BBCooker starting %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher.bbseen = []
         self.configwatcher.bbwatchedfiles = []
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+        sys.stdout.flush()
         self.watcher.bbseen = []
         self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
+        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+        sys.stdout.flush()
+
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.clear_cache()
@@ -196,6 +209,9 @@ class BBCooker:
 
         self.initConfigurationData()
 
+        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+        sys.stdout.flush()
+
         # we log all events to a file if so directed
         if self.configuration.writeeventlog:
             # register the log file writer as UI Handler
@@ -233,6 +249,9 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        bb.debug(1, "BBCooker startup complete %s" % time.time())
+        sys.stdout.flush()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):
@@ -620,27 +639,38 @@ class BBCooker:
             runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
 
-        mcdeps = taskdata[mc].get_mcdepends()
+
         # No need to do check providers if there are no mcdeps or not an mc build
-        if mcdeps and mc:
-            # Make sure we can provide the multiconfig dependency
-            seen = set()
-            new = True
-            while new:
-                new = False
-                for mc in self.multiconfigs:
-                    for k in mcdeps:
-                        if k in seen:
-                            continue
-                        l = k.split(':')
-                        depmc = l[2]
-                        if depmc not in self.multiconfigs:
-                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
-                        else:
-                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
-                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
-                            seen.add(k)
-                            new = True
+        if mc:
+            # Add unresolved first, so we can get multiconfig indirect dependencies on time
+            for mcavailable in self.multiconfigs:
+                # The first element is empty
+                if mcavailable:
+                    taskdata[mcavailable].add_unresolved(localdata[mcavailable], self.recipecaches[mcavailable])
+
+
+            mcdeps = taskdata[mc].get_mcdepends()
+
+            if mcdeps:
+                # Make sure we can provide the multiconfig dependency
+                seen = set()
+                new = True
+                while new:
+                    new = False
+                    for mc in self.multiconfigs:
+                        for k in mcdeps:
+                            if k in seen:
+                                continue
+                            l = k.split(':')
+                            depmc = l[2]
+                            if depmc not in self.multiconfigs:
+                                bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                            else:
+                                logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                                taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                                seen.add(k)
+                                new = True
+
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
 
diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py
index 070da4fe7a..28b8eb9b86 100644
--- a/poky/bitbake/lib/bb/server/process.py
+++ b/poky/bitbake/lib/bb/server/process.py
@@ -130,6 +130,7 @@ class ProcessServer(multiprocessing.Process):
         bb.utils.set_process_name("Cooker")
 
         ready = []
+        newconnections = []
 
         self.controllersock = False
         fds = [self.sock]
@@ -138,37 +139,48 @@ class ProcessServer(multiprocessing.Process):
         print("Entering server connection loop")
 
         def disconnect_client(self, fds):
-            if not self.haveui:
-                return
             print("Disconnecting Client")
-            fds.remove(self.controllersock)
-            fds.remove(self.command_channel)
-            bb.event.unregister_UIHhandler(self.event_handle, True)
-            self.command_channel_reply.writer.close()
-            self.event_writer.writer.close()
-            del self.event_writer
-            self.controllersock.close()
-            self.controllersock = False
-            self.haveui = False
-            self.lastui = time.time()
-            self.cooker.clientComplete()
-            if self.timeout is None:
+            if self.controllersock:
+                fds.remove(self.controllersock)
+                self.controllersock.close()
+                self.controllersock = False
+            if self.haveui:
+                fds.remove(self.command_channel)
+                bb.event.unregister_UIHhandler(self.event_handle, True)
+                self.command_channel_reply.writer.close()
+                self.event_writer.writer.close()
+                self.command_channel.close()
+                self.command_channel = False
+                del self.event_writer
+                self.lastui = time.time()
+                self.cooker.clientComplete()
+                self.haveui = False
+            ready = select.select(fds,[],[],0)[0]
+            if newconnections:
+                print("Starting new client")
+                conn = newconnections.pop(-1)
+                fds.append(conn)
+                self.controllersock = conn
+            elif self.timeout is None and not ready:
                 print("No timeout, exiting.")
                 self.quit = True
 
         while not self.quit:
             if self.sock in ready:
-                self.controllersock, address = self.sock.accept()
-                if self.haveui:
-                    print("Dropping connection attempt as we have a UI %s" % (str(ready)))
-                    self.controllersock.close()
-                else:
-                    print("Accepting %s" % (str(ready)))
-                    fds.append(self.controllersock)
+                while select.select([self.sock],[],[],0)[0]:
+                    controllersock, address = self.sock.accept()
+                    if self.controllersock:
+                        print("Queuing %s (%s)" % (str(ready), str(newconnections)))
+                        newconnections.append(controllersock)
+                    else:
+                        print("Accepting %s (%s)" % (str(ready), str(newconnections)))
+                        self.controllersock = controllersock
+                        fds.append(controllersock)
             if self.controllersock in ready:
                 try:
-                    print("Connecting Client")
+                    print("Processing Client")
                     ui_fds = recvfds(self.controllersock, 3)
+                    print("Connecting Client")
 
                     # Where to write events to
                     writer = ConnectionWriter(ui_fds[0])
@@ -400,16 +412,19 @@ class BitBakeServer(object):
         os.close(self.readypipein)
 
         ready = ConnectionReader(self.readypipe)
-        r = ready.poll(30)
+        r = ready.poll(5)
+        if not r:
+            bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+            r = ready.poll(90)
         if r:
             try:
                 r = ready.get()
             except EOFError:
                 # Trap the child exitting/closing the pipe and error out
                 r = None
-        if not r or r != "ready":
+        if not r or r[0] != "r":
             ready.close()
-            bb.error("Unable to start bitbake server")
+            bb.error("Unable to start bitbake server (%s)" % str(r))
             if os.path.exists(logfile):
                 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
@@ -452,7 +467,7 @@ class BitBakeServer(object):
         os.close(self.readypipe)
         writer = ConnectionWriter(self.readypipein)
         self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
-        writer.send("ready")
+        writer.send("r")
         writer.close()
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
@@ -468,16 +483,25 @@ def connectProcessServer(sockname, featureset):
     # AF_UNIX has path length issues so chdir here to workaround
     cwd = os.getcwd()
 
+    readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
+    eq = command_chan_recv = command_chan = None
+
+    sock.settimeout(10)
+
     try:
         try:
             os.chdir(os.path.dirname(sockname))
-            sock.connect(os.path.basename(sockname))
+            finished = False
+            while not finished:
+                try:
+                    sock.connect(os.path.basename(sockname))
+                    finished = True
+                except IOError as e:
+                    if e.errno == errno.EWOULDBLOCK:
+                        pass
         finally:
             os.chdir(cwd)
 
-        readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
-        eq = command_chan_recv = command_chan = None
-
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
         eq = BBUIEventQueue(readfd)
diff --git a/poky/documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml b/poky/documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml
index f8b01ecc4c..5da03a890d 100644
--- a/poky/documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml
+++ b/poky/documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml
@@ -144,23 +144,49 @@
             you need to get a copy of the Poky repository on your build
             host.
             Use the following commands to clone the Poky
-            repository and then checkout the &DISTRO_REL_TAG; release:
+            repository.
             <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 361782, done.
-     remote: Compressing objects: 100% (87100/87100), done.
-     remote: Total 361782 (delta 268619), reused 361439 (delta 268277)
-     Receiving objects: 100% (361782/361782), 131.94 MiB | 6.88 MiB/s, done.
-     Resolving deltas: 100% (268619/268619), done.
+     remote: Counting objects: 428632, done.
+     remote: Compressing objects: 100% (101203/101203), done.
+     remote: Total 428632 (delta 320463), reused 428532 (delta 320363)
+     Receiving objects: 100% (428632/428632), 153.01 MiB | 12.40 MiB/s, done.
+     Resolving deltas: 100% (320463/320463), done.
      Checking connectivity... done.
-     $ git checkout tags/yocto-2.5 -b my-yocto-2.5
+            </literallayout>
+            Move to the <filename>poky</filename> directory and take a look
+            at the tags:
+            <literallayout class='monospaced'>
+     $ cd poky
+     $ git fetch --tags
+     $ git tag
+     1.1_M1.final
+     1.1_M1.rc1
+     1.1_M1.rc2
+     1.1_M2.final
+     1.1_M2.rc1
+        .
+        .
+        .
+     yocto-2.5
+     yocto-2.5.1
+     yocto-2.5.2
+     yocto-2.6
+     yocto-2.6.1
+     yocto_1.5_M5.rc8
+            </literallayout>
+            For this example, check out the branch based on the
+            &DISTRO_REL_TAG; release:
+            <literallayout class='monospaced'>
+     $ git checkout tags/&DISTRO_REL_TAG; -b my-&DISTRO_REL_TAG;
+     Switched to a new branch 'my-&DISTRO_REL_TAG;'
             </literallayout>
             The previous Git checkout command creates a local branch
             named my-&DISTRO_REL_TAG;. The files available to you in that
             branch exactly match the repository's files in the
             "&DISTRO_NAME_NO_CAP;" development branch at the time of the
-            Yocto Project &DISTRO; release.
+            Yocto Project &DISTRO_REL_TAG; release.
         </para>
 
         <para>
@@ -204,12 +230,41 @@
             <orderedlist>
                 <listitem><para>
                     <emphasis>Initialize the Build Environment:</emphasis>
-                    Run the
+                    From within the <filename>poky</filename> directory, run the
                     <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
                     environment setup script to define Yocto Project's
                     build environment on your build host.
                     <literallayout class='monospaced'>
+     $ cd ~/poky
      $ source &OE_INIT_FILE;
+     You had no conf/local.conf file. This configuration file has therefore been
+     created for you with some default values. You may wish to edit it to, for
+     example, select a different MACHINE (target hardware). See conf/local.conf
+     for more information as common configuration options are commented.
+
+     You had no conf/bblayers.conf file. This configuration file has therefore been
+     created for you with some default values. To add additional metadata layers
+     into your configuration please add entries to conf/bblayers.conf.
+
+     The Yocto Project has extensive documentation about OE including a reference
+     manual which can be found at:
+         http://yoctoproject.org/documentation
+
+     For more information about OpenEmbedded see their website:
+         http://www.openembedded.org/
+
+
+     ### Shell environment set up for builds. ###
+
+     You can now run 'bitbake &lt;target&gt;'
+
+     Common targets are:
+         core-image-minimal
+         core-image-sato
+         meta-toolchain
+         meta-ide-support
+
+     You can also run generated qemu images with a command like 'runqemu qemux86'
                     </literallayout>
                     Among other things, the script creates the
                     <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
diff --git a/poky/documentation/dev-manual/dev-manual-start.xml b/poky/documentation/dev-manual/dev-manual-start.xml
index 3f971ba4b0..d37c0303f3 100644
--- a/poky/documentation/dev-manual/dev-manual-start.xml
+++ b/poky/documentation/dev-manual/dev-manual-start.xml
@@ -1122,11 +1122,11 @@
         .
         .
         .
-     yocto-2.2
-     yocto-2.2.1
-     yocto-2.3
-     yocto-2.3.1
-     yocto-2.4
+     yocto-2.5
+     yocto-2.5.1
+     yocto-2.5.2
+     yocto-2.6
+     yocto-2.6.1
      yocto_1.5_M5.rc8
                     </literallayout>
                     </para></listitem>
diff --git a/poky/documentation/poky.ent b/poky/documentation/poky.ent
index c76fe7b2d3..f951798105 100644
--- a/poky/documentation/poky.ent
+++ b/poky/documentation/poky.ent
@@ -8,7 +8,7 @@
 <!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.5.2">
 <!ENTITY DISTRO_REL_TAG "yocto-2.6.1">
 <!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "December 2018">
+<!ENTITY REL_MONTH_YEAR "January 2019">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
 <!ENTITY POKYVERSION "21.0.1">
 <!ENTITY POKYVERSION_COMPRESSED "2101">