diff options
Diffstat (limited to 'poky/bitbake/lib')
21 files changed, 258 insertions, 182 deletions
diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py index a144bd609..c1e30697b 100644 --- a/poky/bitbake/lib/bb/__init__.py +++ b/poky/bitbake/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "1.51.0" +__version__ = "1.51.1" import sys if sys.version_info < (3, 5, 0): diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py index 79919c5be..3eb4fdde8 100644 --- a/poky/bitbake/lib/bb/asyncrpc/client.py +++ b/poky/bitbake/lib/bb/asyncrpc/client.py @@ -11,13 +11,14 @@ from . import chunkify, DEFAULT_MAX_CHUNK class AsyncClient(object): - def __init__(self, proto_name, proto_version, logger): + def __init__(self, proto_name, proto_version, logger, timeout=30): self.reader = None self.writer = None self.max_chunk = DEFAULT_MAX_CHUNK self.proto_name = proto_name self.proto_version = proto_version self.logger = logger + self.timeout = timeout async def connect_tcp(self, address, port): async def connect_sock(): @@ -70,14 +71,18 @@ class AsyncClient(object): async def send_message(self, msg): async def get_line(): - line = await self.reader.readline() + try: + line = await asyncio.wait_for(self.reader.readline(), self.timeout) + except asyncio.TimeoutError: + raise ConnectionError("Timed out waiting for server") + if not line: raise ConnectionError("Connection closed") line = line.decode("utf-8") if not line.endswith("\n"): - raise ConnectionError("Bad message %r" % msg) + raise ConnectionError("Bad message %r" % (line)) return line diff --git a/poky/bitbake/lib/bb/asyncrpc/serv.py b/poky/bitbake/lib/bb/asyncrpc/serv.py index ef20cb71d..4084f300d 100644 --- a/poky/bitbake/lib/bb/asyncrpc/serv.py +++ b/poky/bitbake/lib/bb/asyncrpc/serv.py @@ -9,6 +9,7 @@ import os import signal import socket import sys +import multiprocessing from . import chunkify, DEFAULT_MAX_CHUNK @@ -201,12 +202,14 @@ class AsyncServer(object): pass def signal_handler(self): + self.logger.debug("Got exit signal") self.loop.stop() def serve_forever(self): asyncio.set_event_loop(self.loop) try: self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler) + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM]) self.run_loop_forever() self.server.close() @@ -221,3 +224,21 @@ class AsyncServer(object): if self._cleanup_socket is not None: self._cleanup_socket() + + def serve_as_process(self, *, prefunc=None, args=()): + def run(): + if prefunc is not None: + prefunc(self, *args) + self.serve_forever() + + # Temporarily block SIGTERM. The server process will inherit this + # block which will ensure it doesn't receive the SIGTERM until the + # handler is ready for it + mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTERM]) + try: + self.process = multiprocessing.Process(target=run) + self.process.start() + + return self.process + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, mask) diff --git a/poky/bitbake/lib/bb/cache.py b/poky/bitbake/lib/bb/cache.py index 27eb27179..73bc6e966 100644 --- a/poky/bitbake/lib/bb/cache.py +++ b/poky/bitbake/lib/bb/cache.py @@ -53,12 +53,12 @@ class RecipeInfoCommon(object): @classmethod def pkgvar(cls, var, packages, metadata): - return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata)) + return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata)) for pkg in packages) @classmethod def taskvar(cls, var, tasks, metadata): - return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) + return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata)) for task in tasks) @classmethod diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index f530cf844..a81dcb136 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -65,9 +65,17 @@ class Command: # Ensure cooker is ready for commands if command != "updateConfig" and command != "setFeatures": - self.cooker.init_configdata() - if not self.remotedatastores: - self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) + try: + self.cooker.init_configdata() + if not self.remotedatastores: + self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) + except (Exception, SystemExit) as exc: + import traceback + if isinstance(exc, bb.BBHandledException): + # We need to start returning real exceptions here. Until we do, we can't + # tell if an exception is an instance of bb.BBHandledException + return None, "bb.BBHandledException()\n" + traceback.format_exc() + return None, traceback.format_exc() if hasattr(CommandsSync, command): # Can run synchronous commands straight away diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index 39e10e613..b2d69c28c 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -390,8 +390,7 @@ class BBCooker: dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False) - self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever) - self.hashserv.process.start() + self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr) self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr) diff --git a/poky/bitbake/lib/bb/data_smart.py b/poky/bitbake/lib/bb/data_smart.py index f48726a34..65528c6ae 100644 --- a/poky/bitbake/lib/bb/data_smart.py +++ b/poky/bitbake/lib/bb/data_smart.py @@ -26,9 +26,9 @@ from bb.COW import COWDictBase logger = logging.getLogger("BitBake.Data") -__setvar_keyword__ = ["_append", "_prepend", "_remove"] -__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$') -__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}") +__setvar_keyword__ = [":append", ":prepend", ":remove"] +__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') +__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") __expand_python_regexp__ = re.compile(r"\${@.+?}") __whitespace_split__ = re.compile(r'(\s)') __override_regexp__ = re.compile(r'[a-z0-9]+') @@ -277,7 +277,7 @@ class VariableHistory(object): for (r, override) in d.overridedata[var]: for event in self.variable(r): loginfo = event.copy() - if 'flag' in loginfo and not loginfo['flag'].startswith("_"): + if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): continue loginfo['variable'] = var loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) @@ -342,7 +342,7 @@ class VariableHistory(object): for event in history: if 'flag' in event: continue - if event['op'] == '_remove': + if event['op'] == ':remove': continue if isset and event['op'] == 'set?': continue @@ -481,7 +481,15 @@ class DataSmart(MutableMapping): def setVar(self, var, value, **loginfo): #print("var=" + str(var) + " val=" + str(value)) - var = var.replace(":", "_") + + if "_append" in var or "_prepend" in var or "_remove" in var: + info = "%s" % var + if "filename" in loginfo: + info += " file: %s" % loginfo[filename] + if "lineno" in loginfo: + info += " line: %s" % loginfo[lineno] + bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) + self.expand_cache = {} parsing=False if 'parsing' in loginfo: @@ -510,7 +518,7 @@ class DataSmart(MutableMapping): # pay the cookie monster # more cookies for the cookie monster - if '_' in var: + if ':' in var: self._setvar_update_overrides(base, **loginfo) if base in self.overridevars: @@ -521,27 +529,27 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) if not parsing: - if "_append" in self.dict[var]: - del self.dict[var]["_append"] - if "_prepend" in self.dict[var]: - del self.dict[var]["_prepend"] - if "_remove" in self.dict[var]: - del self.dict[var]["_remove"] + if ":append" in self.dict[var]: + del self.dict[var][":append"] + if ":prepend" in self.dict[var]: + del self.dict[var][":prepend"] + if ":remove" in self.dict[var]: + del self.dict[var][":remove"] if var in self.overridedata: active = [] self.need_overrides() for (r, o) in self.overridedata[var]: if o in self.overridesset: active.append(r) - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): active.append(r) for a in active: self.delVar(a) del self.overridedata[var] # more cookies for the cookie monster - if '_' in var: + if ':' in var: self._setvar_update_overrides(var, **loginfo) # setting var @@ -567,8 +575,8 @@ class DataSmart(MutableMapping): def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster - override = var[var.rfind('_')+1:] - shortvar = var[:var.rfind('_')] + override = var[var.rfind(':')+1:] + shortvar = var[:var.rfind(':')] while override and __override_regexp__.match(override): if shortvar not in self.overridedata: self.overridedata[shortvar] = [] @@ -577,9 +585,9 @@ class DataSmart(MutableMapping): self.overridedata[shortvar] = list(self.overridedata[shortvar]) self.overridedata[shortvar].append([var, override]) override = None - if "_" in shortvar: - override = var[shortvar.rfind('_')+1:] - shortvar = var[:shortvar.rfind('_')] + if ":" in shortvar: + override = var[shortvar.rfind(':')+1:] + shortvar = var[:shortvar.rfind(':')] if len(shortvar) == 0: override = None @@ -590,8 +598,6 @@ class DataSmart(MutableMapping): """ Rename the variable key to newkey """ - key = key.replace(":", "_") - newkey = newkey.replace(":", "_") if key == newkey: bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) return @@ -620,7 +626,7 @@ class DataSmart(MutableMapping): self.overridedata[newkey].append([v.replace(key, newkey), o]) self.renameVar(v, v.replace(key, newkey)) - if '_' in newkey and val is None: + if ':' in newkey and val is None: self._setvar_update_overrides(newkey, **loginfo) loginfo['variable'] = key @@ -632,15 +638,14 @@ class DataSmart(MutableMapping): def appendVar(self, var, value, **loginfo): loginfo['op'] = 'append' self.varhistory.record(**loginfo) - self.setVar(var + "_append", value, ignore=True, parsing=True) + self.setVar(var + ":append", value, ignore=True, parsing=True) def prependVar(self, var, value, **loginfo): loginfo['op'] = 'prepend' self.varhistory.record(**loginfo) - self.setVar(var + "_prepend", value, ignore=True, parsing=True) + self.setVar(var + ":prepend", value, ignore=True, parsing=True) def delVar(self, var, **loginfo): - var = var.replace(":", "_") self.expand_cache = {} loginfo['detail'] = "" @@ -649,9 +654,9 @@ class DataSmart(MutableMapping): self.dict[var] = {} if var in self.overridedata: del self.overridedata[var] - if '_' in var: - override = var[var.rfind('_')+1:] - shortvar = var[:var.rfind('_')] + if ':' in var: + override = var[var.rfind(':')+1:] + shortvar = var[:var.rfind(':')] while override and override.islower(): try: if shortvar in self.overridedata: @@ -661,14 +666,13 @@ class DataSmart(MutableMapping): except ValueError as e: pass override = None - if "_" in shortvar: - override = var[shortvar.rfind('_')+1:] - shortvar = var[:shortvar.rfind('_')] + if ":" in shortvar: + override = var[shortvar.rfind(':')+1:] + shortvar = var[:shortvar.rfind(':')] if len(shortvar) == 0: override = None def setVarFlag(self, var, flag, value, **loginfo): - var = var.replace(":", "_") self.expand_cache = {} if 'op' not in loginfo: @@ -679,7 +683,7 @@ class DataSmart(MutableMapping): self._makeShadowCopy(var) self.dict[var][flag] = value - if flag == "_defaultval" and '_' in var: + if flag == "_defaultval" and ':' in var: self._setvar_update_overrides(var, **loginfo) if flag == "_defaultval" and var in self.overridevars: self._setvar_update_overridevars(var, value) @@ -692,7 +696,6 @@ class DataSmart(MutableMapping): self.dict["__exportlist"]["_content"].add(var) def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): - var = var.replace(":", "_") if flag == "_content": cachename = var else: @@ -712,11 +715,11 @@ class DataSmart(MutableMapping): active = {} self.need_overrides() for (r, o) in overridedata: - # What about double overrides both with "_" in the name? + # FIXME What about double overrides both with "_" in the name? if o in self.overridesset: active[o] = r - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): active[o] = r mod = True @@ -724,10 +727,10 @@ class DataSmart(MutableMapping): mod = False for o in self.overrides: for a in active.copy(): - if a.endswith("_" + o): + if a.endswith(":" + o): t = active[a] del active[a] - active[a.replace("_" + o, "")] = t + active[a.replace(":" + o, "")] = t mod = True elif a == o: match = active[a] @@ -746,31 +749,31 @@ class DataSmart(MutableMapping): value = copy.copy(local_var["_defaultval"]) - if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: - if not value: - value = "" + if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_append"]: + for (r, o) in local_var[":append"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: + if value is None: + value = "" value = value + r - if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: - if not value: - value = "" + if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_prepend"]: + for (r, o) in local_var[":prepend"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: + if value is None: + value = "" value = r + value parser = None @@ -779,12 +782,12 @@ class DataSmart(MutableMapping): if expand: value = parser.value - if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing: + if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: self.need_overrides() - for (r, o) in local_var["_remove"]: + for (r, o) in local_var[":remove"]: match = True if o: - for o2 in o.split("_"): + for o2 in o.split(":"): if not o2 in self.overrides: match = False if match: @@ -820,7 +823,6 @@ class DataSmart(MutableMapping): return value def delVarFlag(self, var, flag, **loginfo): - var = var.replace(":", "_") self.expand_cache = {} local_var, _ = self._findVar(var) @@ -838,7 +840,6 @@ class DataSmart(MutableMapping): del self.dict[var][flag] def appendVarFlag(self, var, flag, value, **loginfo): - var = var.replace(":", "_") loginfo['op'] = 'append' loginfo['flag'] = flag self.varhistory.record(**loginfo) @@ -846,7 +847,6 @@ class DataSmart(MutableMapping): self.setVarFlag(var, flag, newvalue, ignore=True) def prependVarFlag(self, var, flag, value, **loginfo): - var = var.replace(":", "_") loginfo['op'] = 'prepend' loginfo['flag'] = flag self.varhistory.record(**loginfo) @@ -854,7 +854,6 @@ class DataSmart(MutableMapping): self.setVarFlag(var, flag, newvalue, ignore=True) def setVarFlags(self, var, flags, **loginfo): - var = var.replace(":", "_") self.expand_cache = {} infer_caller_details(loginfo) if not var in self.dict: @@ -869,13 +868,12 @@ class DataSmart(MutableMapping): self.dict[var][i] = flags[i] def getVarFlags(self, var, expand = False, internalflags=False): - var = var.replace(":", "_") local_var, _ = self._findVar(var) flags = {} if local_var: for i in local_var: - if i.startswith("_") and not internalflags: + if i.startswith(("_", ":")) and not internalflags: continue flags[i] = local_var[i] if expand and i in expand: @@ -886,7 +884,6 @@ class DataSmart(MutableMapping): def delVarFlags(self, var, **loginfo): - var = var.replace(":", "_") self.expand_cache = {} if not var in self.dict: self._makeShadowCopy(var) @@ -974,8 +971,8 @@ class DataSmart(MutableMapping): for (r, o) in self.overridedata[var]: if o in self.overridesset: overrides.add(var) - elif "_" in o: - if set(o.split("_")).issubset(self.overridesset): + elif ":" in o: + if set(o.split(":")).issubset(self.overridesset): overrides.add(var) for k in keylist(self.dict): diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 0d49e1da3..ad898680f 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1146,11 +1146,11 @@ def srcrev_internal_helper(ud, d, name): pn = d.getVar("PN") attempts = [] if name != '' and pn: - attempts.append("SRCREV_%s_pn-%s" % (name, pn)) + attempts.append("SRCREV_%s:pn-%s" % (name, pn)) if name != '': attempts.append("SRCREV_%s" % name) if pn: - attempts.append("SRCREV_pn-%s" % pn) + attempts.append("SRCREV:pn-%s" % pn) attempts.append("SRCREV") for a in attempts: diff --git a/poky/bitbake/lib/bb/parse/ast.py b/poky/bitbake/lib/bb/parse/ast.py index db2bdc35e..743ea0dfc 100644 --- a/poky/bitbake/lib/bb/parse/ast.py +++ b/poky/bitbake/lib/bb/parse/ast.py @@ -97,7 +97,6 @@ class DataNode(AstNode): def eval(self, data): groupd = self.groupd key = groupd["var"] - key = key.replace(":", "_") loginfo = { 'variable': key, 'file': self.filename, @@ -146,7 +145,7 @@ class DataNode(AstNode): data.setVar(key, val, parsing=True, **loginfo) class MethodNode(AstNode): - tr_tbl = str.maketrans('/.+-@%&', '_______') + tr_tbl = str.maketrans('/.+-@%&~', '________') def __init__(self, filename, lineno, func_name, body, python, fakeroot): AstNode.__init__(self, filename, lineno) @@ -208,7 +207,6 @@ class ExportFuncsNode(AstNode): def eval(self, data): for func in self.n: - func = func.replace(":", "_") calledfunc = self.classname + "_" + func if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 6c41fe6d4..25e012125 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -2443,6 +2443,11 @@ class RunQueueExecute: if update_tasks: self.sqdone = False + for tid in [t[0] for t in update_tasks]: + h = pending_hash_index(tid, self.rqdata) + if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]: + self.sq_deferred[tid] = self.sqdata.hashes[h] + bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h])) update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) for (tid, harddepfail, origvalid) in update_tasks: @@ -2786,6 +2791,19 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): sqdata.stamppresent = set() sqdata.valid = set() + sqdata.hashes = {} + sqrq.sq_deferred = {} + for mc in sorted(sqdata.multiconfigs): + for tid in sorted(sqdata.sq_revdeps): + if mc_from_tid(tid) != mc: + continue + h = pending_hash_index(tid, rqdata) + if h not in sqdata.hashes: + sqdata.hashes[h] = tid + else: + sqrq.sq_deferred[tid] = sqdata.hashes[h] + bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h])) + update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True) # Compute a list of 'stale' sstate tasks where the current hash does not match the one @@ -2850,32 +2868,20 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary) - sqdata.hashes = {} - sqrq.sq_deferred = {} - for mc in sorted(sqdata.multiconfigs): - for tid in sorted(sqdata.sq_revdeps): - if mc_from_tid(tid) != mc: - continue - if tid in sqdata.stamppresent: - continue - if tid in sqdata.valid: - continue - if tid in sqdata.noexec: - continue - if tid in sqrq.scenequeue_notcovered: - continue - if tid in sqrq.scenequeue_covered: - continue - - h = pending_hash_index(tid, rqdata) - if h not in sqdata.hashes: - if tid in tids: - sqdata.outrightfail.add(tid) - sqdata.hashes[h] = tid - else: - sqrq.sq_deferred[tid] = sqdata.hashes[h] - bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h])) - + for tid in tids: + if tid in sqdata.stamppresent: + continue + if tid in sqdata.valid: + continue + if tid in sqdata.noexec: + continue + if tid in sqrq.scenequeue_covered: + continue + if tid in sqrq.scenequeue_notcovered: + continue + if tid in sqrq.sq_deferred: + continue + sqdata.outrightfail.add(tid) class TaskFailure(Exception): """ diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index a0955722e..6127fd40e 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -26,6 +26,7 @@ import errno import re import datetime import pickle +import traceback import bb.server.xmlrpcserver from bb import daemonize from multiprocessing import queues @@ -217,8 +218,9 @@ class ProcessServer(): self.command_channel_reply.send(self.cooker.command.runCommand(command)) serverlog("Command Completed") except Exception as e: - serverlog('Exception in server main event loop running command %s (%s)' % (command, str(e))) - logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e))) + stack = traceback.format_exc() + serverlog('Exception in server main event loop running command %s (%s)' % (command, stack)) + logger.exception('Exception in server main event loop running command %s (%s)' % (command, stack)) if self.xmlrpc in ready: self.xmlrpc.handle_requests() diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 07692e673..3f9fe5064 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -228,7 +228,7 @@ class SignatureGeneratorBasic(SignatureGenerator): # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: - d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task]) + d.setVar("BB_BASEHASH:task-%s" % task, self.basehash[fn + ":" + task]) def postparsing_clean_cache(self): # @@ -325,7 +325,7 @@ class SignatureGeneratorBasic(SignatureGenerator): h = hashlib.sha256(data.encode("utf-8")).hexdigest() self.taskhash[tid] = h - #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) + #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task]) return h def writeout_file_checksum_cache(self): diff --git a/poky/bitbake/lib/bb/tests/codeparser.py b/poky/bitbake/lib/bb/tests/codeparser.py index 826a2d2f6..f48520479 100644 --- a/poky/bitbake/lib/bb/tests/codeparser.py +++ b/poky/bitbake/lib/bb/tests/codeparser.py @@ -111,9 +111,9 @@ ${D}${libdir}/pkgconfig/*.pc self.assertExecs(set(["sed"])) def test_parameter_expansion_modifiers(self): - # - and + are also valid modifiers for parameter expansion, but are + # -,+ and : are also valid modifiers for parameter expansion, but are # valid characters in bitbake variable names, so are not included here - for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'): + for i in ('=', '?', '#', '%', '##', '%%'): name = "foo%sbar" % i self.parseExpression("${%s}" % name) self.assertNotIn(name, self.references) diff --git a/poky/bitbake/lib/bb/tests/data.py b/poky/bitbake/lib/bb/tests/data.py index 1d4a64b10..e667c7c7d 100644 --- a/poky/bitbake/lib/bb/tests/data.py +++ b/poky/bitbake/lib/bb/tests/data.py @@ -245,35 +245,35 @@ class TestConcatOverride(unittest.TestCase): def test_prepend(self): self.d.setVar("TEST", "${VAL}") - self.d.setVar("TEST_prepend", "${FOO}:") + self.d.setVar("TEST:prepend", "${FOO}:") self.assertEqual(self.d.getVar("TEST"), "foo:val") def test_append(self): self.d.setVar("TEST", "${VAL}") - self.d.setVar("TEST_append", ":${BAR}") + self.d.setVar("TEST:append", ":${BAR}") self.assertEqual(self.d.getVar("TEST"), "val:bar") def test_multiple_append(self): self.d.setVar("TEST", "${VAL}") - self.d.setVar("TEST_prepend", "${FOO}:") - self.d.setVar("TEST_append", ":val2") - self.d.setVar("TEST_append", ":${BAR}") + self.d.setVar("TEST:prepend", "${FOO}:") + self.d.setVar("TEST:append", ":val2") + self.d.setVar("TEST:append", ":${BAR}") self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar") def test_append_unset(self): - self.d.setVar("TEST_prepend", "${FOO}:") - self.d.setVar("TEST_append", ":val2") - self.d.setVar("TEST_append", ":${BAR}") + self.d.setVar("TEST:prepend", "${FOO}:") + self.d.setVar("TEST:append", ":val2") + self.d.setVar("TEST:append", ":${BAR}") self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar") def test_remove(self): self.d.setVar("TEST", "${VAL} ${BAR}") - self.d.setVar("TEST_remove", "val") + self.d.setVar("TEST:remove", "val") self.assertEqual(self.d.getVar("TEST"), " bar") def test_remove_cleared(self): self.d.setVar("TEST", "${VAL} ${BAR}") - self.d.setVar("TEST_remove", "val") + self.d.setVar("TEST:remove", "val") self.d.setVar("TEST", "${VAL} ${BAR}") self.assertEqual(self.d.getVar("TEST"), "val bar") @@ -281,42 +281,42 @@ class TestConcatOverride(unittest.TestCase): # (including that whitespace is preserved) def test_remove_inactive_override(self): self.d.setVar("TEST", "${VAL} ${BAR} 123") - self.d.setVar("TEST_remove_inactiveoverride", "val") + self.d.setVar("TEST:remove:inactiveoverride", "val") self.assertEqual(self.d.getVar("TEST"), "val bar 123") def test_doubleref_remove(self): self.d.setVar("TEST", "${VAL} ${BAR}") - self.d.setVar("TEST_remove", "val") + self.d.setVar("TEST:remove", "val") self.d.setVar("TEST_TEST", "${TEST} ${TEST}") self.assertEqual(self.d.getVar("TEST_TEST"), " bar bar") def test_empty_remove(self): self.d.setVar("TEST", "") - self.d.setVar("TEST_remove", "val") + self.d.setVar("TEST:remove", "val") self.assertEqual(self.d.getVar("TEST"), "") def test_remove_expansion(self): self.d.setVar("BAR", "Z") self.d.setVar("TEST", "${BAR}/X Y") - self.d.setVar("TEST_remove", "${BAR}/X") + self.d.setVar("TEST:remove", "${BAR}/X") self.assertEqual(self.d.getVar("TEST"), " Y") def test_remove_expansion_items(self): self.d.setVar("TEST", "A B C D") self.d.setVar("BAR", "B D") - self.d.setVar("TEST_remove", "${BAR}") + self.d.setVar("TEST:remove", "${BAR}") self.assertEqual(self.d.getVar("TEST"), "A C ") def test_remove_preserve_whitespace(self): # When the removal isn't active, the original value should be preserved self.d.setVar("TEST", " A B") - self.d.setVar("TEST_remove", "C") + self.d.setVar("TEST:remove", "C") self.assertEqual(self.d.getVar("TEST"), " A B") def test_remove_preserve_whitespace2(self): # When the removal is active preserve the whitespace self.d.setVar("TEST", " A B") - self.d.setVar("TEST_remove", "B") + self.d.setVar("TEST:remove", "B") self.assertEqual(self.d.getVar("TEST"), " A ") class TestOverrides(unittest.TestCase): @@ -329,70 +329,70 @@ class TestOverrides(unittest.TestCase): self.assertEqual(self.d.getVar("TEST"), "testvalue") def test_one_override(self): - self.d.setVar("TEST_bar", "testvalue2") + self.d.setVar("TEST:bar", "testvalue2") self.assertEqual(self.d.getVar("TEST"), "testvalue2") def test_one_override_unset(self): - self.d.setVar("TEST2_bar", "testvalue2") + self.d.setVar("TEST2:bar", "testvalue2") self.assertEqual(self.d.getVar("TEST2"), "testvalue2") - self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar']) + self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2:bar']) def test_multiple_override(self): - self.d.setVar("TEST_bar", "testvalue2") - self.d.setVar("TEST_local", "testvalue3") - self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("TEST:bar", "testvalue2") + self.d.setVar("TEST:local", "testvalue3") + self.d.setVar("TEST:foo", "testvalue4") self.assertEqual(self.d.getVar("TEST"), "testvalue3") - self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local']) + self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST:foo', 'OVERRIDES', 'TEST:bar', 'TEST:local']) def test_multiple_combined_overrides(self): - self.d.setVar("TEST_local_foo_bar", "testvalue3") + self.d.setVar("TEST:local:foo:bar", "testvalue3") self.assertEqual(self.d.getVar("TEST"), "testvalue3") def test_multiple_overrides_unset(self): - self.d.setVar("TEST2_local_foo_bar", "testvalue3") + self.d.setVar("TEST2:local:foo:bar", "testvalue3") self.assertEqual(self.d.getVar("TEST2"), "testvalue3") def test_keyexpansion_override(self): self.d.setVar("LOCAL", "local") - self.d.setVar("TEST_bar", "testvalue2") - self.d.setVar("TEST_${LOCAL}", "testvalue3") - self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("TEST:bar", "testvalue2") + self.d.setVar("TEST:${LOCAL}", "testvalue3") + self.d.setVar("TEST:foo", "testvalue4") bb.data.expandKeys(self.d) self.assertEqual(self.d.getVar("TEST"), "testvalue3") def test_rename_override(self): - self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a") + self.d.setVar("ALTERNATIVE:ncurses-tools:class-target", "a") self.d.setVar("OVERRIDES", "class-target") - self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools") - self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a") + self.d.renameVar("ALTERNATIVE:ncurses-tools", "ALTERNATIVE:lib32-ncurses-tools") + self.assertEqual(self.d.getVar("ALTERNATIVE:lib32-ncurses-tools"), "a") def test_underscore_override(self): - self.d.setVar("TEST_bar", "testvalue2") - self.d.setVar("TEST_some_val", "testvalue3") - self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("TEST:bar", "testvalue2") + self.d.setVar("TEST:some_val", "testvalue3") + self.d.setVar("TEST:foo", "testvalue4") self.d.setVar("OVERRIDES", "foo:bar:some_val") self.assertEqual(self.d.getVar("TEST"), "testvalue3") def test_remove_with_override(self): - self.d.setVar("TEST_bar", "testvalue2") - self.d.setVar("TEST_some_val", "testvalue3 testvalue5") - self.d.setVar("TEST_some_val_remove", "testvalue3") - self.d.setVar("TEST_foo", "testvalue4") + self.d.setVar("TEST:bar", "testvalue2") + self.d.setVar("TEST:some_val", "testvalue3 testvalue5") + self.d.setVar("TEST:some_val:remove", "testvalue3") + self.d.setVar("TEST:foo", "testvalue4") self.d.setVar("OVERRIDES", "foo:bar:some_val") self.assertEqual(self.d.getVar("TEST"), " testvalue5") def test_append_and_override_1(self): - self.d.setVar("TEST_append", "testvalue2") - self.d.setVar("TEST_bar", "testvalue3") + self.d.setVar("TEST:append", "testvalue2") + self.d.setVar("TEST:bar", "testvalue3") self.assertEqual(self.d.getVar("TEST"), "testvalue3testvalue2") def test_append_and_override_2(self): - self.d.setVar("TEST_append_bar", "testvalue2") + self.d.setVar("TEST:append:bar", "testvalue2") self.assertEqual(self.d.getVar("TEST"), "testvaluetestvalue2") def test_append_and_override_3(self): - self.d.setVar("TEST_bar_append", "testvalue2") + self.d.setVar("TEST:bar:append", "testvalue2") self.assertEqual(self.d.getVar("TEST"), "testvalue2") # Test an override with _<numeric> in it based on a real world OE issue @@ -400,11 +400,16 @@ class TestOverrides(unittest.TestCase): self.d.setVar("TARGET_ARCH", "x86_64") self.d.setVar("PN", "test-${TARGET_ARCH}") self.d.setVar("VERSION", "1") - self.d.setVar("VERSION_pn-test-${TARGET_ARCH}", "2") + self.d.setVar("VERSION:pn-test-${TARGET_ARCH}", "2") self.d.setVar("OVERRIDES", "pn-${PN}") bb.data.expandKeys(self.d) self.assertEqual(self.d.getVar("VERSION"), "2") + def test_append_and_unused_override(self): + # Had a bug where an unused override append could return "" instead of None + self.d.setVar("BAR:append:unusedoverride", "testvalue2") + self.assertEqual(self.d.getVar("BAR"), None) + class TestKeyExpansion(unittest.TestCase): def setUp(self): self.d = bb.data.init() @@ -498,7 +503,7 @@ class TaskHash(unittest.TestCase): d.setVar("VAR", "val") # Adding an inactive removal shouldn't change the hash d.setVar("BAR", "notbar") - d.setVar("MYCOMMAND_remove", "${BAR}") + d.setVar("MYCOMMAND:remove", "${BAR}") nexthash = gettask_bashhash("mytask", d) self.assertEqual(orighash, nexthash) diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 9e21e1842..4d17f82ed 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -98,8 +98,8 @@ exportD = "d" overridetest = """ -RRECOMMENDS_${PN} = "a" -RRECOMMENDS_${PN}_libc = "b" +RRECOMMENDS:${PN} = "a" +RRECOMMENDS:${PN}:libc = "b" OVERRIDES = "libc:${PN}" PN = "gtk+" """ @@ -110,13 +110,13 @@ PN = "gtk+" self.assertEqual(d.getVar("RRECOMMENDS"), "b") bb.data.expandKeys(d) self.assertEqual(d.getVar("RRECOMMENDS"), "b") - d.setVar("RRECOMMENDS_gtk+", "c") + d.setVar("RRECOMMENDS:gtk+", "c") self.assertEqual(d.getVar("RRECOMMENDS"), "c") overridetest2 = """ EXTRA_OECONF = "" -EXTRA_OECONF_class-target = "b" -EXTRA_OECONF_append = " c" +EXTRA_OECONF:class-target = "b" +EXTRA_OECONF:append = " c" """ def test_parse_overrides(self): @@ -128,7 +128,7 @@ EXTRA_OECONF_append = " c" overridetest3 = """ DESCRIPTION = "A" -DESCRIPTION_${PN}-dev = "${DESCRIPTION} B" +DESCRIPTION:${PN}-dev = "${DESCRIPTION} B" PN = "bc" """ @@ -136,15 +136,15 @@ PN = "bc" f = self.parsehelper(self.overridetest3) d = bb.parse.handle(f.name, self.d)[''] bb.data.expandKeys(d) - self.assertEqual(d.getVar("DESCRIPTION_bc-dev"), "A B") + self.assertEqual(d.getVar("DESCRIPTION:bc-dev"), "A B") d.setVar("DESCRIPTION", "E") - d.setVar("DESCRIPTION_bc-dev", "C D") + d.setVar("DESCRIPTION:bc-dev", "C D") d.setVar("OVERRIDES", "bc-dev") self.assertEqual(d.getVar("DESCRIPTION"), "C D") classextend = """ -VAR_var_override1 = "B" +VAR_var:override1 = "B" EXTRA = ":override1" OVERRIDES = "nothing${EXTRA}" diff --git a/poky/bitbake/lib/bb/ui/taskexp.py b/poky/bitbake/lib/bb/ui/taskexp.py index 2b246710c..c00eaf663 100644 --- a/poky/bitbake/lib/bb/ui/taskexp.py +++ b/poky/bitbake/lib/bb/ui/taskexp.py @@ -8,6 +8,7 @@ # import sys +import traceback try: import gi @@ -196,6 +197,7 @@ def main(server, eventHandler, params): gtkgui.start() try: + params.updateToServer(server, os.environ.copy()) params.updateFromServer(server) cmdline = params.parseActions() if not cmdline: @@ -218,6 +220,9 @@ def main(server, eventHandler, params): except client.Fault as x: print("XMLRPC Fault getting commandline:\n %s" % x) return + except Exception as e: + print("Exception in startup:\n %s" % traceback.format_exc()) + return if gtkthread.quit.isSet(): return diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index 6ba1d2a37..e6e82d111 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -1178,7 +1178,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): variables: a list of variable names to look for. Functions may also be specified, but must be specified with '()' at the end of the name. Note that the function doesn't have - any intrinsic understanding of _append, _prepend, _remove, + any intrinsic understanding of :append, :prepend, :remove, or overrides, so these are considered as part of the name. These values go into a regular expression, so regular expression syntax is allowed. diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py index e2b762dbf..e851535c5 100644 --- a/poky/bitbake/lib/hashserv/tests.py +++ b/poky/bitbake/lib/hashserv/tests.py @@ -15,28 +15,32 @@ import tempfile import threading import unittest import socket +import time +import signal -def _run_server(server, idx): - # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', - # format='%(levelname)s %(filename)s:%(lineno)d %(message)s') +def server_prefunc(server, idx): + logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', + format='%(levelname)s %(filename)s:%(lineno)d %(message)s') + server.logger.debug("Running server %d" % idx) sys.stdout = open('bbhashserv-%d.log' % idx, 'w') sys.stderr = sys.stdout - server.serve_forever() - class HashEquivalenceTestSetup(object): METHOD = 'TestMethod' server_index = 0 - def start_server(self, dbpath=None, upstream=None, read_only=False): + def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc): self.server_index += 1 if dbpath is None: dbpath = os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index) - def cleanup_thread(thread): - thread.terminate() - thread.join() + def cleanup_server(server): + if server.process.exitcode is not None: + return + + server.process.terminate() + server.process.join() server = create_server(self.get_server_addr(self.server_index), dbpath, @@ -44,9 +48,8 @@ class HashEquivalenceTestSetup(object): read_only=read_only) server.dbpath = dbpath - server.thread = multiprocessing.Process(target=_run_server, args=(server, self.server_index)) - server.thread.start() - self.addCleanup(cleanup_thread, server.thread) + server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) + self.addCleanup(cleanup_server, server) def cleanup_client(client): client.close() @@ -283,6 +286,33 @@ class HashEquivalenceCommonTests(object): self.assertClientGetHash(self.client, taskhash2, None) + def test_slow_server_start(self): + """ + Ensures that the server will exit correctly even if it gets a SIGTERM + before entering the main loop + """ + + event = multiprocessing.Event() + + def prefunc(server, idx): + nonlocal event + server_prefunc(server, idx) + event.wait() + + def do_nothing(signum, frame): + pass + + old_signal = signal.signal(signal.SIGTERM, do_nothing) + self.addCleanup(signal.signal, signal.SIGTERM, old_signal) + + _, server = self.start_server(prefunc=prefunc) + server.process.terminate() + time.sleep(30) + event.set() + server.process.join(300) + self.assertIsNotNone(server.process.exitcode, "Server did not exit in a timely manner!") + + class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): def get_server_addr(self, server_idx): return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx) diff --git a/poky/bitbake/lib/toaster/orm/models.py b/poky/bitbake/lib/toaster/orm/models.py index 7f7e922ad..4c94b407d 100644 --- a/poky/bitbake/lib/toaster/orm/models.py +++ b/poky/bitbake/lib/toaster/orm/models.py @@ -1719,7 +1719,7 @@ class CustomImageRecipe(Recipe): """Generate the contents for the recipe file.""" # If we have no excluded packages we only need to _append if self.excludes_set.count() == 0: - packages_conf = "IMAGE_INSTALL_append = \" " + packages_conf = "IMAGE_INSTALL:append = \" " for pkg in self.appends_set.all(): packages_conf += pkg.name+' ' diff --git a/poky/bitbake/lib/toaster/toastergui/views.py b/poky/bitbake/lib/toaster/toastergui/views.py index 9a5e48e3b..74f9d5696 100644 --- a/poky/bitbake/lib/toaster/toastergui/views.py +++ b/poky/bitbake/lib/toaster/toastergui/views.py @@ -1708,7 +1708,7 @@ if True: except ProjectVariable.DoesNotExist: pass try: - return_data['image_install_append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value, + return_data['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value, except ProjectVariable.DoesNotExist: pass try: @@ -1839,7 +1839,7 @@ if True: except ProjectVariable.DoesNotExist: pass try: - context['image_install_append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value + context['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value context['image_install_append_defined'] = "1" except ProjectVariable.DoesNotExist: pass diff --git a/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py index 59da6ff7a..e25b55e5a 100644 --- a/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py +++ b/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py @@ -451,7 +451,7 @@ class Command(BaseCommand): # Catch vars relevant to Toaster (in case no Toaster section) self.update_project_vars(project,'DISTRO') self.update_project_vars(project,'MACHINE') - self.update_project_vars(project,'IMAGE_INSTALL_append') + self.update_project_vars(project,'IMAGE_INSTALL:append') self.update_project_vars(project,'IMAGE_FSTYPES') self.update_project_vars(project,'PACKAGE_CLASSES') # These vars are typically only assigned by Toaster |