diff options
Diffstat (limited to 'poky/bitbake/lib/bb')
| -rw-r--r-- | poky/bitbake/lib/bb/command.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 89 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/parse/ast.py | 10 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/parse/parse_py/BBHandler.py | 5 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/pysh/pyshyacc.py | 1 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 23 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/cooker.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/parse.py | 6 |
9 files changed, 89 insertions, 51 deletions
diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index 3902ccca71..805ed9216c 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -396,7 +396,7 @@ class CommandsSync: def sortkey(x): vfn, _ = x realfn, _, mc = bb.cache.virtualfn2realfn(vfn) - return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn) + return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn) skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) return list(skipdict.items()) diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index effd02442c..f6abc63487 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -626,6 +626,7 @@ class BBCooker: current = 0 runlist = [] for k in fulltargetlist: + origk = k mc = "" if k.startswith("mc:"): mc = k.split(":")[1] @@ -635,6 +636,10 @@ class BBCooker: k2 = k.split(":do_") k = k2[0] ktask = k2[1] + + if mc not in self.multiconfigs: + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc)) + taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) current += 1 if not ktask.startswith("do_"): @@ -670,7 +675,7 @@ class BBCooker: l = k.split(':') depmc = l[2] if depmc not in self.multiconfigs: - bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc)) + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) else: logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3]) taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) @@ -1588,7 +1593,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1609,7 +1614,7 @@ class BBCooker: raise NothingToBuild ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() - for pkg in pkgs_to_build: + for pkg in pkgs_to_build.copy(): if pkg in ignore: parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) if pkg.startswith("multiconfig:"): @@ -1704,14 +1709,11 @@ class CookerCollectFiles(object): # the shortest. This allows nested layers to be properly evaluated. self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) - def calc_bbfile_priority( self, filename, matched = None ): + def calc_bbfile_priority(self, filename): for _, _, regex, pri in self.bbfile_config_priorities: if regex.match(filename): - if matched is not None: - if not regex in matched: - matched.add(regex) - return pri - return 0 + return pri, regex + return 0, None def get_bbfiles(self): """Get list of default .bb files by reading out the current directory""" @@ -1744,7 +1746,7 @@ class CookerCollectFiles(object): config.setVar("BBFILES", " ".join(files)) # Sort files by priority - files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) + files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) if not len(files): files = self.get_bbfiles() @@ -1866,39 +1868,62 @@ class CookerCollectFiles(object): filelist.append(filename) return tuple(filelist) - def collection_priorities(self, pkgfns, d): + def collection_priorities(self, pkgfns, fns, d): + # Return the priorities of the entries in pkgfns + # Also check that all the regexes in self.bbfile_config_priorities are used + # (but to do that we need to ensure skipped recipes aren't counted, nor + # collections in BBFILE_PATTERN_IGNORE_EMPTY) priorities = {} + seen = set() + matched = set() + + matched_regex = set() + unmatched_regex = set() + for _, _, regex, _ in self.bbfile_config_priorities: + unmatched_regex.add(regex) # Calculate priorities for each file - matched = set() for p in pkgfns: realfn, cls, mc = bb.cache.virtualfn2realfn(p) - priorities[p] = self.calc_bbfile_priority(realfn, matched) - - unmatched = set() - for _, _, regex, pri in self.bbfile_config_priorities: - if not regex in matched: - unmatched.add(regex) - - # Don't show the warning if the BBFILE_PATTERN did match .bbappend files - def find_bbappend_match(regex): + priorities[p], regex = self.calc_bbfile_priority(realfn) + if regex in unmatched_regex: + matched_regex.add(regex) + unmatched_regex.remove(regex) + seen.add(realfn) + if regex: + matched.add(realfn) + + if unmatched_regex: + # Account for bbappend files for b in self.bbappends: (bbfile, append) = b - if regex.match(append): - # If the bbappend is matched by already "matched set", return False - for matched_regex in matched: - if matched_regex.match(append): - return False - return True - return False + seen.add(append) + + # Account for skipped recipes + seen.update(fns) + + seen.difference_update(matched) - for unmatch in unmatched.copy(): - if find_bbappend_match(unmatch): - unmatched.remove(unmatch) + def already_matched(fn): + for regex in matched_regex: + if regex.match(fn): + return True + return False + + for unmatch in unmatched_regex.copy(): + for fn in seen: + if unmatch.match(fn): + # If the bbappend or file was already matched by another regex, skip it + # e.g. for a layer within a layer, the outer regex could match, the inner + # regex may match nothing and we should warn about that + if already_matched(fn): + continue + unmatched_regex.remove(unmatch) + break for collection, pattern, regex, _ in self.bbfile_config_priorities: - if regex in unmatched: + if regex in unmatched_regex: if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', collection, pattern)) diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index dbf8715674..644ba9238b 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -355,7 +355,7 @@ class Git(FetchMethod): runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) diff --git a/poky/bitbake/lib/bb/parse/ast.py b/poky/bitbake/lib/bb/parse/ast.py index 785aa974eb..0714296af2 100644 --- a/poky/bitbake/lib/bb/parse/ast.py +++ b/poky/bitbake/lib/bb/parse/ast.py @@ -244,12 +244,14 @@ class AddTaskNode(AstNode): bb.build.addtask(self.func, self.before, self.after, data) class DelTaskNode(AstNode): - def __init__(self, filename, lineno, func): + def __init__(self, filename, lineno, tasks): AstNode.__init__(self, filename, lineno) - self.func = func + self.tasks = tasks def eval(self, data): - bb.build.deltask(self.func, data) + tasks = data.expand(self.tasks).split() + for task in tasks: + bb.build.deltask(task, data) class BBHandlerNode(AstNode): def __init__(self, filename, lineno, fns): @@ -305,7 +307,7 @@ def handleAddTask(statements, filename, lineno, m): statements.append(AddTaskNode(filename, lineno, func, before, after)) def handleDelTask(statements, filename, lineno, m): - func = m.group("func") + func = m.group(1) if func is None: return diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index 6e216effb8..215f940b60 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -26,7 +26,7 @@ __func_start_regexp__ = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*( __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) __addtask_regexp__ = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*") -__deltask_regexp__ = re.compile(r"deltask\s+(?P<func>\w+)(?P<ignores>.*)") +__deltask_regexp__ = re.compile(r"deltask\s+(.+)") __addhandler_regexp__ = re.compile(r"addhandler\s+(.+)" ) __def_regexp__ = re.compile(r"def\s+(\w+).*:" ) __python_func_regexp__ = re.compile(r"(\s+.*)|(^$)|(^#)" ) @@ -238,9 +238,6 @@ def feeder(lineno, s, fn, root, statements, eof=False): m = __deltask_regexp__.match(s) if m: - # Check and warn "for deltask task1 task2" - if m.group('ignores'): - logger.warning('deltask ignored: "%s"' % m.group('ignores')) ast.handleDelTask(statements, fn, lineno, m) return diff --git a/poky/bitbake/lib/bb/pysh/pyshyacc.py b/poky/bitbake/lib/bb/pysh/pyshyacc.py index de565dc9af..924860a6f3 100644 --- a/poky/bitbake/lib/bb/pysh/pyshyacc.py +++ b/poky/bitbake/lib/bb/pysh/pyshyacc.py @@ -570,6 +570,7 @@ def p_linebreak(p): def p_separator_op(p): """separator_op : COMMA + | COMMA COMMA | AMP""" p[0] = p[1] diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 83385baf60..9ec79f5b64 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -48,7 +48,7 @@ class ProcessServer(multiprocessing.Process): self.event_handle = None self.haveui = False - self.lastui = False + self.maxuiwait = 30 self.xmlrpc = False self._idlefuns = {} @@ -155,6 +155,7 @@ class ProcessServer(multiprocessing.Process): print("No timeout, exiting.") self.quit = True + self.lastui = time.time() while not self.quit: if self.sock in ready: while select.select([self.sock],[],[],0)[0]: @@ -191,11 +192,18 @@ class ProcessServer(multiprocessing.Process): except (EOFError, OSError): disconnect_client(self, fds) - if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \ + if not self.timeout == -1.0 and not self.haveui and self.timeout and \ (self.lastui + self.timeout) < time.time(): print("Server timeout, exiting.") self.quit = True + # If we don't see a UI connection within maxuiwait, its unlikely we're going to see + # one. We have had issue with processes hanging indefinitely so timing out UI-less + # servers is useful. + if not self.haveui and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): + print("No UI connection within max timeout, exiting to avoid infinite loop.") + self.quit = True + if self.command_channel in ready: try: command = self.command_channel.get() @@ -220,10 +228,13 @@ class ProcessServer(multiprocessing.Process): print("Exiting") # Remove the socket file so we don't get any more connections to avoid races - os.unlink(self.sockname) + try: + os.unlink(self.sockname) + except: + pass self.sock.close() - try: + try: self.cooker.shutdown(True) self.cooker.notifier.stop() self.cooker.confignotifier.stop() @@ -243,7 +254,7 @@ class ProcessServer(multiprocessing.Process): lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) if lock: # We hold the lock so we can remove the file (hide stale pid data) - bb.utils.remove(lockfile) + # via unlockfile. bb.utils.unlockfile(lock) return @@ -331,7 +342,7 @@ class ServerCommunicator(): def runCommand(self, command): self.connection.send(command) if not self.recv.poll(30): - logger.note("No reply from server in 30s") + logger.info("No reply from server in 30s") if not self.recv.poll(30): raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)") return self.recv.get() diff --git a/poky/bitbake/lib/bb/tests/cooker.py b/poky/bitbake/lib/bb/tests/cooker.py index 74c903f010..c82d4b7b81 100644 --- a/poky/bitbake/lib/bb/tests/cooker.py +++ b/poky/bitbake/lib/bb/tests/cooker.py @@ -60,7 +60,7 @@ class CookerTest(unittest.TestCase): log_handler = LogHandler() logger.addHandler(log_handler) collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities) - collection.collection_priorities(pkgfns, self.d) + collection.collection_priorities(pkgfns, pkgfns, self.d) logger.removeHandler(log_handler) # Should be empty (no generated messages) diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 9afd1b208e..9e21e18425 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -178,7 +178,10 @@ python () { addtask do_patch after do_foo after do_unpack before do_configure before do_compile addtask do_fetch do_patch -deltask do_fetch do_patch +MYVAR = "do_patch" +EMPTYVAR = "" +deltask do_fetch ${MYVAR} ${EMPTYVAR} +deltask ${EMPTYVAR} """ def test_parse_addtask_deltask(self): import sys @@ -189,6 +192,5 @@ deltask do_fetch do_patch self.assertTrue("addtask contained multiple 'before' keywords" in stdout) self.assertTrue("addtask contained multiple 'after' keywords" in stdout) self.assertTrue('addtask ignored: " do_patch"' in stdout) - self.assertTrue('deltask ignored: " do_patch"' in stdout) #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout) |