diff options
Diffstat (limited to 'poky/bitbake/lib/bb/server/process.py')
| -rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 259 |
1 files changed, 159 insertions, 100 deletions
diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 65e1eab52..c7cb34f0c 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -25,6 +25,7 @@ import subprocess import errno import re import datetime +import pickle import bb.server.xmlrpcserver from bb import daemonize from multiprocessing import queues @@ -34,11 +35,15 @@ logger = logging.getLogger('BitBake') class ProcessTimeout(SystemExit): pass +def serverlog(msg): + print(str(os.getpid()) + " " + datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg) + sys.stdout.flush() + class ProcessServer(): profile_filename = "profile.log" profile_processed_filename = "profile.log.processed" - def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface): + def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface): self.command_channel = False self.command_channel_reply = False self.quit = False @@ -54,10 +59,12 @@ class ProcessServer(): self._idlefuns = {} self.bitbake_lock = lock + self.bitbake_lock_name = lockname self.sock = sock self.sockname = sockname self.server_timeout = server_timeout + self.timeout = self.server_timeout self.xmlrpcinterface = xmlrpcinterface def register_idle_function(self, function, data): @@ -70,22 +77,7 @@ class ProcessServer(): if self.xmlrpcinterface[0]: self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self) - print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port)) - - heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') - if heartbeat_event: - try: - self.heartbeat_seconds = float(heartbeat_event) - except: - bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) - - self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT') - try: - if self.timeout: - self.timeout = float(self.timeout) - except: - bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout) - + serverlog("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port)) try: self.bitbake_lock.seek(0) @@ -96,7 +88,7 @@ class ProcessServer(): self.bitbake_lock.write("%s\n" % (os.getpid())) self.bitbake_lock.flush() except Exception as e: - print("Error writing to lock file: %s" % str(e)) + serverlog("Error writing to lock file: %s" % str(e)) pass if self.cooker.configuration.profile: @@ -110,7 +102,7 @@ class ProcessServer(): prof.dump_stats("profile.log") bb.utils.process_profilelog("profile.log") - print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") + serverlog("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") else: ret = self.main() @@ -129,10 +121,11 @@ class ProcessServer(): fds = [self.sock] if self.xmlrpc: fds.append(self.xmlrpc) - print("Entering server connection loop") + seendata = False + serverlog("Entering server connection loop") def disconnect_client(self, fds): - print("Disconnecting Client") + serverlog("Disconnecting Client") if self.controllersock: fds.remove(self.controllersock) self.controllersock.close() @@ -150,12 +143,12 @@ class ProcessServer(): self.haveui = False ready = select.select(fds,[],[],0)[0] if newconnections: - print("Starting new client") + serverlog("Starting new client") conn = newconnections.pop(-1) fds.append(conn) self.controllersock = conn elif self.timeout is None and not ready: - print("No timeout, exiting.") + serverlog("No timeout, exiting.") self.quit = True self.lastui = time.time() @@ -164,17 +157,17 @@ class ProcessServer(): while select.select([self.sock],[],[],0)[0]: controllersock, address = self.sock.accept() if self.controllersock: - print("Queuing %s (%s)" % (str(ready), str(newconnections))) + serverlog("Queuing %s (%s)" % (str(ready), str(newconnections))) newconnections.append(controllersock) else: - print("Accepting %s (%s)" % (str(ready), str(newconnections))) + serverlog("Accepting %s (%s)" % (str(ready), str(newconnections))) self.controllersock = controllersock fds.append(controllersock) if self.controllersock in ready: try: - print("Processing Client") + serverlog("Processing Client") ui_fds = recvfds(self.controllersock, 3) - print("Connecting Client") + serverlog("Connecting Client") # Where to write events to writer = ConnectionWriter(ui_fds[0]) @@ -198,14 +191,14 @@ class ProcessServer(): if not self.timeout == -1.0 and not self.haveui and self.timeout and \ (self.lastui + self.timeout) < time.time(): - print("Server timeout, exiting.") + serverlog("Server timeout, exiting.") self.quit = True # If we don't see a UI connection within maxuiwait, its unlikely we're going to see # one. We have had issue with processes hanging indefinitely so timing out UI-less # servers is useful. if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): - print("No UI connection within max timeout, exiting to avoid infinite loop.") + serverlog("No UI connection within max timeout, exiting to avoid infinite loop.") self.quit = True if self.command_channel in ready: @@ -220,17 +213,37 @@ class ProcessServer(): self.quit = True continue try: - print("Running command %s" % command) + serverlog("Running command %s" % command) self.command_channel_reply.send(self.cooker.command.runCommand(command)) + serverlog("Command Completed") except Exception as e: logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e))) if self.xmlrpc in ready: self.xmlrpc.handle_requests() + if not seendata and hasattr(self.cooker, "data"): + heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') + if heartbeat_event: + try: + self.heartbeat_seconds = float(heartbeat_event) + except: + bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) + + self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT') + try: + if self.timeout: + self.timeout = float(self.timeout) + except: + bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout) + seendata = True + ready = self.idle_commands(.1, fds) - print("Exiting") + if len(threading.enumerate()) != 1: + serverlog("More than one thread left?: " + str(threading.enumerate())) + + serverlog("Exiting") # Remove the socket file so we don't get any more connections to avoid races try: os.unlink(self.sockname) @@ -253,39 +266,67 @@ class ProcessServer(): # Finally release the lockfile but warn about other processes holding it open lock = self.bitbake_lock - lockfile = lock.name + lockfile = self.bitbake_lock_name + + def get_lock_contents(lockfile): + try: + with open(lockfile, "r") as f: + return f.readlines() + except FileNotFoundError: + return None + + lockcontents = get_lock_contents(lockfile) + serverlog("Original lockfile contents: " + str(lockcontents)) + lock.close() lock = None while not lock: - with bb.utils.timeout(3): - lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) - if lock: - # We hold the lock so we can remove the file (hide stale pid data) - # via unlockfile. - bb.utils.unlockfile(lock) - return - + i = 0 + lock = None + while not lock and i < 30: + lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False) if not lock: - # Some systems may not have lsof available - procs = None + newlockcontents = get_lock_contents(lockfile) + if newlockcontents != lockcontents: + # A new server was started, the lockfile contents changed, we can exit + serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents)) + return + time.sleep(0.1) + i += 1 + if lock: + # We hold the lock so we can remove the file (hide stale pid data) + # via unlockfile. + bb.utils.unlockfile(lock) + serverlog("Exiting as we could obtain the lock") + return + + if not lock: + # Some systems may not have lsof available + procs = None + try: + procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # File was deleted? + continue + except OSError as e: + if e.errno != errno.ENOENT: + raise + if procs is None: + # Fall back to fuser if lsof is unavailable try: - procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) + procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # File was deleted? + continue except OSError as e: if e.errno != errno.ENOENT: raise - if procs is None: - # Fall back to fuser if lsof is unavailable - try: - procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" - if procs: - msg += ":\n%s" % str(procs) - print(msg) + + msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" + if procs: + msg += ":\n%s" % str(procs.decode("utf-8")) + serverlog(msg) def idle_commands(self, delay, fds=None): nextsleep = delay @@ -323,8 +364,9 @@ class ProcessServer(): self.next_heartbeat += self.heartbeat_seconds if self.next_heartbeat <= now: self.next_heartbeat = now + self.heartbeat_seconds - heartbeat = bb.event.HeartbeatEvent(now) - bb.event.fire(heartbeat, self.cooker.data) + if hasattr(self.cooker, "data"): + heartbeat = bb.event.HeartbeatEvent(now) + bb.event.fire(heartbeat, self.cooker.data) if nextsleep and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for # the heartbeat. @@ -353,7 +395,12 @@ class ServerCommunicator(): logger.info("No reply from server in 30s") if not self.recv.poll(30): raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)") - return self.recv.get() + ret, exc = self.recv.get() + # Should probably turn all exceptions in exc back into exceptions? + # For now, at least handle BBHandledException + if exc and ("BBHandledException" in exc or "SystemExit" in exc): + raise bb.BBHandledException() + return ret, exc def updateFeatureSet(self, featureset): _, error = self.runCommand(["setFeatures", featureset]) @@ -386,39 +433,26 @@ class BitBakeProcessServerConnection(object): self.connection.recv.close() return +start_log_format = '--- Starting bitbake server pid %s at %s ---' +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' + class BitBakeServer(object): - start_log_format = '--- Starting bitbake server pid %s at %s ---' - start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' - def __init__(self, lock, sockname, configuration, featureset): + def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): - self.configuration = configuration + self.server_timeout = server_timeout + self.xmlrpcinterface = xmlrpcinterface self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock self.readypipe, self.readypipein = os.pipe() - # Create server control socket - if os.path.exists(sockname): - os.unlink(sockname) - # Place the log in the builddirectory alongside the lock file logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") + self.logfile = logfile - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - # AF_UNIX has path length issues so chdir here to workaround - cwd = os.getcwd() - try: - os.chdir(os.path.dirname(sockname)) - self.sock.bind(os.path.basename(sockname)) - finally: - os.chdir(cwd) - self.sock.listen(1) - - os.set_inheritable(self.sock.fileno(), True) startdatetime = datetime.datetime.now() bb.daemonize.createDaemon(self._startServer, logfile) - self.sock.close() self.bitbake_lock.close() os.close(self.readypipein) @@ -437,7 +471,7 @@ class BitBakeServer(object): ready.close() bb.error("Unable to start bitbake server (%s)" % str(r)) if os.path.exists(logfile): - logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) + logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] lastlines = [] @@ -447,9 +481,9 @@ class BitBakeServer(object): lines.append(line) else: lastlines.append(line) - res = logstart_re.match(line.rstrip()) + res = logstart_re.search(line.rstrip()) if res: - ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) + ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) @@ -470,28 +504,53 @@ class BitBakeServer(object): ready.close() def _startServer(self): - print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format))) - sys.stdout.flush() + os.close(self.readypipe) + os.set_inheritable(self.bitbake_lock.fileno(), True) + os.set_inheritable(self.readypipein, True) + serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") + os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) - try: - server = ProcessServer(self.bitbake_lock, self.sock, self.sockname, self.configuration.server_timeout, self.configuration.xmlrpcinterface) - os.close(self.readypipe) - writer = ConnectionWriter(self.readypipein) - try: - self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, server.register_idle_function) - except bb.BBHandledException: - return None - writer.send("r") - writer.close() - server.cooker = self.cooker - print("Started bitbake server pid %d" % os.getpid()) - sys.stdout.flush() +def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface): + + import bb.cookerdata + import bb.cooker + + serverlog(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format))) + + try: + bitbake_lock = os.fdopen(lockfd, "w") - server.run() + # Create server control socket + if os.path.exists(sockname): + os.unlink(sockname) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # AF_UNIX has path length issues so chdir here to workaround + cwd = os.getcwd() + try: + os.chdir(os.path.dirname(sockname)) + sock.bind(os.path.basename(sockname)) finally: - # Flush any ,essages/errors to the logfile before exit - sys.stdout.flush() - sys.stderr.flush() + os.chdir(cwd) + sock.listen(1) + + server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface) + writer = ConnectionWriter(readypipeinfd) + try: + featureset = [] + cooker = bb.cooker.BBCooker(featureset, server.register_idle_function) + except bb.BBHandledException: + return None + writer.send("r") + writer.close() + server.cooker = cooker + serverlog("Started bitbake server pid %d" % os.getpid()) + + server.run() + finally: + # Flush any ,essages/errors to the logfile before exit + sys.stdout.flush() + sys.stderr.flush() def connectProcessServer(sockname, featureset): # Connect to socket |