diff options
author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2019-01-29 07:50:33 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2019-01-29 07:50:34 +0300 |
commit | f058f4906e5eebc8753404c606393eb5a0999cec (patch) | |
tree | 149ba67b2af6bc169282594f388d155f3e748406 /poky/bitbake/lib/bb | |
parent | 3cce45a96f0416b4c3d8f2b698cb830662a29227 (diff) | |
download | openbmc-f058f4906e5eebc8753404c606393eb5a0999cec.tar.xz |
poky: refresh thud: cc73390a75..3541f019a5
Update poky to thud HEAD:
Alejandro Enedino Hernandez Samaniego (1):
bitbake: cooker: fix indirect multiconfig dependencies
Richard Purdie (8):
bitbake: server/process: Increase server startup time delay
bitbake: cooker: Add some timing debug messages to the server startup
bitbake: server/process: Handle short reads
bitbake: lib/bb/server: Avoid UnboundLocalError traceback
bitbake: server/process: Ensure socket has a timeout set
bitbake: process.py: Set socket timeout to 10 seconds
bitbake: process: Handle EWOULDBLOCK in socket connect
bitbake: process: Rewrite multiple connection handling
Change-Id: I175bb3a51cc19f3aef5ffb6e09b04cf0bf882835
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/bitbake/lib/bb')
-rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 70 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 84 |
2 files changed, 104 insertions, 50 deletions
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index 16681ba24..db52964c3 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -175,18 +175,31 @@ class BBCooker: self.configuration = configuration + bb.debug(1, "BBCooker starting %s" % time.time()) + sys.stdout.flush() + self.configwatcher = pyinotify.WatchManager() + bb.debug(1, "BBCooker pyinotify1 %s" % time.time()) + sys.stdout.flush() + self.configwatcher.bbseen = [] self.configwatcher.bbwatchedfiles = [] self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) + bb.debug(1, "BBCooker pyinotify2 %s" % time.time()) + sys.stdout.flush() self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \ pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \ pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO self.watcher = pyinotify.WatchManager() + bb.debug(1, "BBCooker pyinotify3 %s" % time.time()) + sys.stdout.flush() self.watcher.bbseen = [] self.watcher.bbwatchedfiles = [] self.notifier = pyinotify.Notifier(self.watcher, self.notifications) + bb.debug(1, "BBCooker pyinotify complete %s" % time.time()) + sys.stdout.flush() + # If being called by something like tinfoil, we need to clean cached data # which may now be invalid bb.parse.clear_cache() @@ -196,6 +209,9 @@ class BBCooker: self.initConfigurationData() + bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) + sys.stdout.flush() + # we log all events to a file if so directed if self.configuration.writeeventlog: # register the log file writer as UI Handler @@ -233,6 +249,9 @@ class BBCooker: # Let SIGHUP exit as SIGTERM signal.signal(signal.SIGHUP, self.sigterm_exception) + bb.debug(1, "BBCooker startup complete %s" % time.time()) + sys.stdout.flush() + def process_inotify_updates(self): for n in [self.confignotifier, self.notifier]: if n.check_events(timeout=0): @@ -620,27 +639,38 @@ class BBCooker: runlist.append([mc, k, ktask, fn]) bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) - mcdeps = taskdata[mc].get_mcdepends() + # No need to do check providers if there are no mcdeps or not an mc build - if mcdeps and mc: - # Make sure we can provide the multiconfig dependency - seen = set() - new = True - while new: - new = False - for mc in self.multiconfigs: - for k in mcdeps: - if k in seen: - continue - l = k.split(':') - depmc = l[2] - if depmc not in self.multiconfigs: - bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc)) - else: - logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3]) - taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) - seen.add(k) - new = True + if mc: + # Add unresolved first, so we can get multiconfig indirect dependencies on time + for mcavailable in self.multiconfigs: + # The first element is empty + if mcavailable: + taskdata[mcavailable].add_unresolved(localdata[mcavailable], self.recipecaches[mcavailable]) + + + mcdeps = taskdata[mc].get_mcdepends() + + if mcdeps: + # Make sure we can provide the multiconfig dependency + seen = set() + new = True + while new: + new = False + for mc in self.multiconfigs: + for k in mcdeps: + if k in seen: + continue + l = k.split(':') + depmc = l[2] + if depmc not in self.multiconfigs: + bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc)) + else: + logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3]) + taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) + seen.add(k) + new = True + for mc in self.multiconfigs: taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 070da4fe7..28b8eb9b8 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -130,6 +130,7 @@ class ProcessServer(multiprocessing.Process): bb.utils.set_process_name("Cooker") ready = [] + newconnections = [] self.controllersock = False fds = [self.sock] @@ -138,37 +139,48 @@ class ProcessServer(multiprocessing.Process): print("Entering server connection loop") def disconnect_client(self, fds): - if not self.haveui: - return print("Disconnecting Client") - fds.remove(self.controllersock) - fds.remove(self.command_channel) - bb.event.unregister_UIHhandler(self.event_handle, True) - self.command_channel_reply.writer.close() - self.event_writer.writer.close() - del self.event_writer - self.controllersock.close() - self.controllersock = False - self.haveui = False - self.lastui = time.time() - self.cooker.clientComplete() - if self.timeout is None: + if self.controllersock: + fds.remove(self.controllersock) + self.controllersock.close() + self.controllersock = False + if self.haveui: + fds.remove(self.command_channel) + bb.event.unregister_UIHhandler(self.event_handle, True) + self.command_channel_reply.writer.close() + self.event_writer.writer.close() + self.command_channel.close() + self.command_channel = False + del self.event_writer + self.lastui = time.time() + self.cooker.clientComplete() + self.haveui = False + ready = select.select(fds,[],[],0)[0] + if newconnections: + print("Starting new client") + conn = newconnections.pop(-1) + fds.append(conn) + self.controllersock = conn + elif self.timeout is None and not ready: print("No timeout, exiting.") self.quit = True while not self.quit: if self.sock in ready: - self.controllersock, address = self.sock.accept() - if self.haveui: - print("Dropping connection attempt as we have a UI %s" % (str(ready))) - self.controllersock.close() - else: - print("Accepting %s" % (str(ready))) - fds.append(self.controllersock) + while select.select([self.sock],[],[],0)[0]: + controllersock, address = self.sock.accept() + if self.controllersock: + print("Queuing %s (%s)" % (str(ready), str(newconnections))) + newconnections.append(controllersock) + else: + print("Accepting %s (%s)" % (str(ready), str(newconnections))) + self.controllersock = controllersock + fds.append(controllersock) if self.controllersock in ready: try: - print("Connecting Client") + print("Processing Client") ui_fds = recvfds(self.controllersock, 3) + print("Connecting Client") # Where to write events to writer = ConnectionWriter(ui_fds[0]) @@ -400,16 +412,19 @@ class BitBakeServer(object): os.close(self.readypipein) ready = ConnectionReader(self.readypipe) - r = ready.poll(30) + r = ready.poll(5) + if not r: + bb.note("Bitbake server didn't start within 5 seconds, waiting for 90") + r = ready.poll(90) if r: try: r = ready.get() except EOFError: # Trap the child exitting/closing the pipe and error out r = None - if not r or r != "ready": + if not r or r[0] != "r": ready.close() - bb.error("Unable to start bitbake server") + bb.error("Unable to start bitbake server (%s)" % str(r)) if os.path.exists(logfile): logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False @@ -452,7 +467,7 @@ class BitBakeServer(object): os.close(self.readypipe) writer = ConnectionWriter(self.readypipein) self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset) - writer.send("ready") + writer.send("r") writer.close() server.cooker = self.cooker server.server_timeout = self.configuration.server_timeout @@ -468,16 +483,25 @@ def connectProcessServer(sockname, featureset): # AF_UNIX has path length issues so chdir here to workaround cwd = os.getcwd() + readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None + eq = command_chan_recv = command_chan = None + + sock.settimeout(10) + try: try: os.chdir(os.path.dirname(sockname)) - sock.connect(os.path.basename(sockname)) + finished = False + while not finished: + try: + sock.connect(os.path.basename(sockname)) + finished = True + except IOError as e: + if e.errno == errno.EWOULDBLOCK: + pass finally: os.chdir(cwd) - readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None - eq = command_chan_recv = command_chan = None - # Send an fd for the remote to write events to readfd, writefd = os.pipe() eq = BBUIEventQueue(readfd) |