diff options
Diffstat (limited to 'poky/bitbake')
| -rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 54 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/data.py | 19 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/parse/__init__.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 3 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/codeparser.py | 28 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/ui/knotty.py | 22 | ||||
| -rw-r--r-- | poky/bitbake/lib/pyinotify.py | 11 |
7 files changed, 90 insertions, 49 deletions
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index df4acca18a..08593d114a 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -169,27 +169,15 @@ class BBCooker: bb.debug(1, "BBCooker starting %s" % time.time()) sys.stdout.flush() - self.configwatcher = pyinotify.WatchManager() - bb.debug(1, "BBCooker pyinotify1 %s" % time.time()) - sys.stdout.flush() + self.configwatcher = None + self.confignotifier = None - self.configwatcher.bbseen = set() - self.configwatcher.bbwatchedfiles = set() - self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) - bb.debug(1, "BBCooker pyinotify2 %s" % time.time()) - sys.stdout.flush() self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \ pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \ pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO - self.watcher = pyinotify.WatchManager() - bb.debug(1, "BBCooker pyinotify3 %s" % time.time()) - sys.stdout.flush() - self.watcher.bbseen = set() - self.watcher.bbwatchedfiles = set() - self.notifier = pyinotify.Notifier(self.watcher, self.notifications) - bb.debug(1, "BBCooker pyinotify complete %s" % time.time()) - sys.stdout.flush() + self.watcher = None + self.notifier = None # If being called by something like tinfoil, we need to clean cached data # which may now be invalid @@ -240,9 +228,29 @@ class BBCooker: sys.stdout.flush() self.handlePRServ() + def setupConfigWatcher(self): + if self.configwatcher: + self.configwatcher.close() + self.confignotifier = None + self.configwatcher = None + self.configwatcher = pyinotify.WatchManager() + self.configwatcher.bbseen = set() + self.configwatcher.bbwatchedfiles = set() + self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) + + def setupParserWatcher(self): + if self.watcher: + self.watcher.close() + self.notifier = None + self.watcher = None + self.watcher = pyinotify.WatchManager() + self.watcher.bbseen = set() + self.watcher.bbwatchedfiles = set() + self.notifier = pyinotify.Notifier(self.watcher, self.notifications) + def process_inotify_updates(self): for n in [self.confignotifier, self.notifier]: - if n.check_events(timeout=0): + if n and n.check_events(timeout=0): # read notified events and enqeue them n.read_events() n.process_events() @@ -262,10 +270,6 @@ class BBCooker: self.configwatcher.bbseen.remove(event.pathname) # Could remove all entries starting with the directory but for now... bb.parse.clear_cache() - if "IN_CREATE" in event.maskname: - self.add_filewatch([[event.pathname]], watcher=self.configwatcher, dirs=True) - elif "IN_DELETE" in event.maskname and event.pathname in self.configwatcher.bbseen: - self.configwatcher.bbseen.remove(event.pathname) if not event.pathname in self.inotify_modified_files: self.inotify_modified_files.append(event.pathname) self.baseconfig_valid = False @@ -285,10 +289,6 @@ class BBCooker: self.watcher.bbseen.remove(event.pathname) # Could remove all entries starting with the directory but for now... bb.parse.clear_cache() - if "IN_CREATE" in event.maskname: - self.add_filewatch([[event.pathname]], dirs=True) - elif "IN_DELETE" in event.maskname and event.pathname in self.watcher.bbseen: - self.watcher.bbseen.remove(event.pathname) if not event.pathname in self.inotify_modified_files: self.inotify_modified_files.append(event.pathname) self.parsecache_valid = False @@ -358,6 +358,8 @@ class BBCooker: if mod not in self.orig_sysmodules: del sys.modules[mod] + self.setupConfigWatcher() + # Need to preserve BB_CONSOLELOG over resets consolelog = None if hasattr(self, "data"): @@ -1639,6 +1641,8 @@ class BBCooker: self.updateCacheSync() if self.state != state.parsing and not self.parsecache_valid: + self.setupParserWatcher() + bb.parse.siggen.reset(self.data) self.parseConfiguration () if CookerFeatures.SEND_SANITYEVENTS in self.featureset: diff --git a/poky/bitbake/lib/bb/data.py b/poky/bitbake/lib/bb/data.py index 5d02bab99b..62ec214762 100644 --- a/poky/bitbake/lib/bb/data.py +++ b/poky/bitbake/lib/bb/data.py @@ -272,7 +272,7 @@ def update_data(d): """Performs final steps upon the datastore, including application of overrides""" d.finalize(parent = True) -def build_dependencies(key, keys, shelldeps, varflagsexcl, d): +def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): deps = set() try: if key[-1] == ']': @@ -283,12 +283,15 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): return deps, value varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {} vardeps = varflags.get("vardeps") + exclusions = varflags.get("vardepsexclude", "").split() - def handle_contains(value, contains, d): + def handle_contains(value, contains, exclusions, d): newvalue = [] if value: newvalue.append(str(value)) for k in sorted(contains): + if k in exclusions or k in ignored_vars: + continue l = (d.getVar(k) or "").split() for item in sorted(contains[k]): for word in item.split(): @@ -316,7 +319,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno")) deps = deps | parser.references deps = deps | (keys & parser.execs) - value = handle_contains(value, parser.contains, d) + value = handle_contains(value, parser.contains, exclusions, d) else: value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True) parser = bb.codeparser.ShellParser(key, logger) @@ -324,7 +327,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): deps = deps | shelldeps deps = deps | parsedvar.references deps = deps | (keys & parser.execs) | (keys & parsedvar.execs) - value = handle_contains(value, parsedvar.contains, d) + value = handle_contains(value, parsedvar.contains, exclusions, d) if hasattr(parsedvar, "removes"): value = handle_remove(value, deps, parsedvar.removes, d) if vardeps is None: @@ -339,7 +342,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): value, parser = d.getVarFlag(key, "_content", False, retparser=True) deps |= parser.references deps = deps | (keys & parser.execs) - value = handle_contains(value, parser.contains, d) + value = handle_contains(value, parser.contains, exclusions, d) if hasattr(parser, "removes"): value = handle_remove(value, deps, parser.removes, d) @@ -359,7 +362,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): deps |= set(varfdeps) deps |= set((vardeps or "").split()) - deps -= set(varflags.get("vardepsexclude", "").split()) + deps -= set(exclusions) except bb.parse.SkipRecipe: raise except Exception as e: @@ -380,7 +383,7 @@ def generate_dependencies(d, ignored_vars): tasklist = d.getVar('__BBTASKS', False) or [] for task in tasklist: - deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d) + deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, ignored_vars, d) newdeps = deps[task] seen = set() while newdeps: @@ -389,7 +392,7 @@ def generate_dependencies(d, ignored_vars): newdeps = set() for dep in nextdeps: if dep not in deps: - deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d) + deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, ignored_vars, d) newdeps |= deps[dep] newdeps -= seen #print "For %s: %s" % (task, str(deps[task])) diff --git a/poky/bitbake/lib/bb/parse/__init__.py b/poky/bitbake/lib/bb/parse/__init__.py index c01807ba87..347609513b 100644 --- a/poky/bitbake/lib/bb/parse/__init__.py +++ b/poky/bitbake/lib/bb/parse/__init__.py @@ -113,6 +113,8 @@ def init(fn, data): return h['init'](data) def init_parser(d): + if hasattr(bb.parse, "siggen"): + bb.parse.siggen.exit() bb.parse.siggen = bb.siggen.init(d) def resolve_file(fn, d): diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index ce53fdc678..19ef83980f 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -28,6 +28,7 @@ import re import datetime import pickle import traceback +import gc import bb.server.xmlrpcserver from bb import daemonize from multiprocessing import queues @@ -739,8 +740,10 @@ class ConnectionWriter(object): self.event = self def _send(self, obj): + gc.disable() with self.wlock: self.writer.send_bytes(obj) + gc.enable() def send(self, obj): obj = multiprocessing.reduction.ForkingPickler.dumps(obj) diff --git a/poky/bitbake/lib/bb/tests/codeparser.py b/poky/bitbake/lib/bb/tests/codeparser.py index f485204791..71ed382ab8 100644 --- a/poky/bitbake/lib/bb/tests/codeparser.py +++ b/poky/bitbake/lib/bb/tests/codeparser.py @@ -318,7 +318,7 @@ d.getVar(a(), False) "filename": "example.bb", }) - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) @@ -365,7 +365,7 @@ esac self.d.setVarFlags("FOO", {"func": True}) self.setEmptyVars(execs) - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) self.assertEqual(deps, set(["somevar", "inverted"] + execs)) @@ -375,7 +375,7 @@ esac self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) self.assertEqual(deps, set(["oe_libinstall"])) @@ -384,7 +384,7 @@ esac self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") - deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d) + deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) self.assertEqual(deps, set(["oe_libinstall"])) @@ -399,7 +399,7 @@ esac # Check dependencies self.d.setVar('ANOTHERVAR', expr) self.d.setVar('TESTVAR', 'anothervalue testval testval2') - deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d) + deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) self.assertEqual(sorted(values.splitlines()), sorted([expr, 'TESTVAR{anothervalue} = Set', @@ -412,6 +412,24 @@ esac # Check final value self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone']) + def test_contains_vardeps_excluded(self): + # Check the ignored_vars option to build_dependencies is handled by contains functionality + varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}' + self.d.setVar('ANOTHERVAR', varval) + self.d.setVar('TESTVAR', 'anothervalue testval testval2') + self.d.setVar('TESTVAR2', 'testval3') + deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(["TESTVAR"]), self.d) + self.assertEqual(sorted(values.splitlines()), sorted([varval])) + self.assertEqual(deps, set(["TESTVAR2"])) + self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) + + # Check the vardepsexclude flag is handled by contains functionality + self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') + deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) + self.assertEqual(sorted(values.splitlines()), sorted([varval])) + self.assertEqual(deps, set(["TESTVAR2"])) + self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) + #Currently no wildcard support #def test_vardeps_wildcards(self): # self.d.setVar("oe_libinstall", "echo test") diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index b2e7520ee7..3f410fd525 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -252,26 +252,26 @@ class TerminalFilter(object): return tasks = [] for t in runningpids: + start_time = activetasks[t].get("starttime", None) + if start_time: + msg = "%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"]) + else: + msg = "%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]) progress = activetasks[t].get("progress", None) if progress is not None: pbar = activetasks[t].get("progressbar", None) rate = activetasks[t].get("rate", None) - start_time = activetasks[t].get("starttime", None) if not pbar or pbar.bouncing != (progress < 0): if progress < 0: - pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle) + pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle) pbar.bouncing = True else: - pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle) + pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle) pbar.bouncing = False activetasks[t]["progressbar"] = pbar - tasks.append((pbar, progress, rate, start_time)) + tasks.append((pbar, msg, progress, rate, start_time)) else: - start_time = activetasks[t].get("starttime", None) - if start_time: - tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"])) - else: - tasks.append("%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"])) + tasks.append(msg) if self.main.shutdown: content = pluralise("Waiting for %s running task to finish", @@ -308,12 +308,12 @@ class TerminalFilter(object): if not self.quiet: for tasknum, task in enumerate(tasks[:(self.rows - 1 - lines)]): if isinstance(task, tuple): - pbar, progress, rate, start_time = task + pbar, msg, progress, rate, start_time = task if not pbar.start_time: pbar.start(False) if start_time: pbar.start_time = start_time - pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1])) + pbar.setmessage('%s: %s' % (tasknum, msg)) pbar.setextra(rate) if progress > -1: content = pbar.update(progress) diff --git a/poky/bitbake/lib/pyinotify.py b/poky/bitbake/lib/pyinotify.py index 8c94b3e334..5c9b6d0fe2 100644 --- a/poky/bitbake/lib/pyinotify.py +++ b/poky/bitbake/lib/pyinotify.py @@ -603,6 +603,17 @@ class _ProcessEvent: unknown event. """ stripped_mask = event.mask - (event.mask & IN_ISDIR) + # Bitbake hack - we see event masks of 0x6, IN_MODIFY & IN_ATTRIB + # The kernel inotify code can set more than one of the bits in the mask, + # fsnotify_change() in linux/fsnotify.h is quite clear that IN_ATTRIB, + # IN_MODIFY and IN_ACCESS can arrive together. + # This breaks the code below which assume only one mask bit is ever + # set in an event. We don't care about attrib or access in bitbake so drop those + if (stripped_mask & IN_MODIFY) and (stripped_mask & IN_ATTRIB): + stripped_mask = stripped_mask - (stripped_mask & IN_ATTRIB) + if (stripped_mask & IN_MODIFY) and (stripped_mask & IN_ACCESS): + stripped_mask = stripped_mask - (stripped_mask & IN_ACCESS) + maskname = EventsCodes.ALL_VALUES.get(stripped_mask) if maskname is None: raise ProcessEventError("Unknown mask 0x%08x" % stripped_mask) |