diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/bb/cooker.py')
| -rw-r--r-- | yocto-poky/bitbake/lib/bb/cooker.py | 141 |
1 files changed, 84 insertions, 57 deletions
diff --git a/yocto-poky/bitbake/lib/bb/cooker.py b/yocto-poky/bitbake/lib/bb/cooker.py index 4df88818f..9b565fc37 100644 --- a/yocto-poky/bitbake/lib/bb/cooker.py +++ b/yocto-poky/bitbake/lib/bb/cooker.py @@ -67,6 +67,14 @@ class CollectionError(bb.BBHandledException): class state: initial, parsing, running, shutdown, forceshutdown, stopped, error = range(7) + @classmethod + def get_name(cls, code): + for name in dir(cls): + value = getattr(cls, name) + if type(value) == type(cls.initial) and value == code: + return name + raise ValueError("Invalid status code: %s" % code) + class SkippedPackage: def __init__(self, info = None, reason = None): @@ -136,6 +144,10 @@ class BBCooker: self.watcher.bbwatchedfiles = [] self.notifier = pyinotify.Notifier(self.watcher, self.notifications) + # If being called by something like tinfoil, we need to clean cached data + # which may now be invalid + bb.parse.__mtime_cache = {} + bb.parse.BBHandler.cached_statements = {} self.initConfigurationData() @@ -189,13 +201,13 @@ class BBCooker: def config_notifications(self, event): if not event.pathname in self.configwatcher.bbwatchedfiles: return - if not event.path in self.inotify_modified_files: - self.inotify_modified_files.append(event.path) + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) self.baseconfig_valid = False def notifications(self, event): - if not event.path in self.inotify_modified_files: - self.inotify_modified_files.append(event.path) + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) self.parsecache_valid = False def add_filewatch(self, deps, watcher=None): @@ -234,9 +246,9 @@ class BBCooker: def sigterm_exception(self, signum, stackframe): if signum == signal.SIGTERM: - bb.warn("Cooker recieved SIGTERM, shutting down...") + bb.warn("Cooker received SIGTERM, shutting down...") elif signum == signal.SIGHUP: - bb.warn("Cooker recieved SIGHUP, shutting down...") + bb.warn("Cooker received SIGHUP, shutting down...") self.state = state.forceshutdown def setFeatures(self, features): @@ -645,7 +657,7 @@ class BBCooker: data.expandKeys(envdata) for e in envdata.keys(): if data.getVarFlag( e, 'python', envdata ): - logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, True)) + logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False): @@ -717,8 +729,15 @@ class BBCooker: depend_tree["packages"] = {} depend_tree["rdepends-pkg"] = {} depend_tree["rrecs-pkg"] = {} + depend_tree['providermap'] = {} depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities + for name, fn in taskdata.get_providermap().iteritems(): + pn = self.recipecache.pkg_fn[fn] + if name != pn: + version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] + depend_tree['providermap'][name] = (pn, version) + for task in xrange(len(rq.rqdata.runq_fnid)): taskname = rq.rqdata.runq_task[task] fnid = rq.rqdata.runq_fnid[task] @@ -893,7 +912,7 @@ class BBCooker: logger.info("PN build list saved to 'pn-buildlist'") for pn in depgraph["depends"]: for depend in depgraph["depends"][pn]: - print('"%s" -> "%s"' % (pn, depend), file=depends_file) + print('"%s" -> "%s" [style=solid]' % (pn, depend), file=depends_file) for pn in depgraph["rdepends-pn"]: for rdepend in depgraph["rdepends-pn"][pn]: print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file) @@ -911,13 +930,13 @@ class BBCooker: else: print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file) for depend in depgraph["depends"][pn]: - print('"%s" -> "%s"' % (package, depend), file=depends_file) + print('"%s" -> "%s" [style=solid]' % (package, depend), file=depends_file) for package in depgraph["rdepends-pkg"]: for rdepend in depgraph["rdepends-pkg"][package]: print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) for package in depgraph["rrecs-pkg"]: for rdepend in depgraph["rrecs-pkg"][package]: - print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file) + print('"%s" -> "%s" [style=dotted]' % (package, rdepend), file=depends_file) print("}", file=depends_file) logger.info("Package dependencies saved to 'package-depends.dot'") @@ -1019,22 +1038,21 @@ class BBCooker: def findFilesMatchingInDir(self, filepattern, directory): """ - Searches for files matching the regex 'pattern' which are children of + Searches for files containing the substring 'filepattern' which are children of 'directory' in each BBPATH. i.e. to find all rootfs package classes available to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes') or to find all machine configuration files one could call: - findFilesMatchingInDir(self, 'conf/machines', 'conf') + findFilesMatchingInDir(self, '.conf', 'conf/machine') """ matches = [] - p = re.compile(re.escape(filepattern)) bbpaths = self.data.getVar('BBPATH', True).split(':') for path in bbpaths: dirpath = os.path.join(path, directory) if os.path.exists(dirpath): for root, dirs, files in os.walk(dirpath): for f in files: - if p.search(f): + if filepattern in f: matches.append(f) if matches: @@ -1072,7 +1090,7 @@ class BBCooker: for pfn in self.recipecache.pkg_fn: inherits = self.recipecache.inherits.get(pfn, None) - if inherits and inherits.count(klass) > 0: + if inherits and klass in inherits: pkg_list.append(self.recipecache.pkg_fn[pfn]) return pkg_list @@ -1095,28 +1113,6 @@ class BBCooker: tree = self.generatePkgDepTreeData(pkgs, 'build') bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data) - def buildWorldTargetList(self): - """ - Build package list for "bitbake world" - """ - parselog.debug(1, "collating packages for \"world\"") - for f in self.recipecache.possible_world: - terminal = True - pn = self.recipecache.pkg_fn[f] - - for p in self.recipecache.pn_provides[pn]: - if p.startswith('virtual/'): - parselog.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p) - terminal = False - break - for pf in self.recipecache.providers[p]: - if self.recipecache.pkg_fn[pf] != pn: - parselog.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p) - terminal = False - break - if terminal: - self.recipecache.world_target.add(pn) - def interactiveMode( self ): """Drop off into a shell""" try: @@ -1353,7 +1349,7 @@ class BBCooker: failures += len(exc.args) retval = False except SystemExit as exc: - self.command.finishAsyncCommand() + self.command.finishAsyncCommand(str(exc)) return False if not retval: @@ -1389,7 +1385,7 @@ class BBCooker: failures += len(exc.args) retval = False except SystemExit as exc: - self.command.finishAsyncCommand() + self.command.finishAsyncCommand(str(exc)) return False if not retval: @@ -1437,14 +1433,21 @@ class BBCooker: dump = {} for k in self.data.keys(): try: - v = self.data.getVar(k, True) + expand = True + flags = self.data.getVarFlags(k) + if flags and "func" in flags and "python" in flags: + expand = False + v = self.data.getVar(k, expand) if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart): dump[k] = { 'v' : v , 'history' : self.data.varhistory.variable(k), } for d in flaglist: - dump[k][d] = self.data.getVarFlag(k, d) + if flags and d in flags: + dump[k][d] = flags[d] + else: + dump[k][d] = None except Exception as e: print(e) return dump @@ -1506,6 +1509,8 @@ class BBCooker: # reload files for which we got notifications for p in self.inotify_modified_files: bb.parse.update_cache(p) + if p in bb.parse.BBHandler.cached_statements: + del bb.parse.BBHandler.cached_statements[p] self.inotify_modified_files = [] if not self.baseconfig_valid: @@ -1577,7 +1582,7 @@ class BBCooker: parselog.warn("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) if 'world' in pkgs_to_build: - self.buildWorldTargetList() + bb.providers.buildWorldTargetList(self.recipecache) pkgs_to_build.remove('world') for t in self.recipecache.world_target: pkgs_to_build.append(t) @@ -1643,6 +1648,9 @@ class BBCooker: else: self.state = state.shutdown + if self.parser: + self.parser.shutdown(clean=not force, force=force) + def finishcommand(self): self.state = state.initial @@ -1760,18 +1768,32 @@ class CookerCollectFiles(object): globbed = glob.glob(f) if not globbed and os.path.exists(f): globbed = [f] - for g in globbed: + # glob gives files in order on disk. Sort to be deterministic. + for g in sorted(globbed): if g not in newfiles: newfiles.append(g) bbmask = config.getVar('BBMASK', True) if bbmask: + # First validate the individual regular expressions and ignore any + # that do not compile + bbmasks = [] + for mask in bbmask.split(): + try: + re.compile(mask) + bbmasks.append(mask) + except sre_constants.error: + collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) + + # Then validate the combined regular expressions. This should never + # fail, but better safe than sorry... + bbmask = "|".join(bbmasks) try: bbmask_compiled = re.compile(bbmask) except sre_constants.error: - collectlog.critical("BBMASK is not a valid regular expression, ignoring.") - return list(newfiles), 0 + collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) + bbmask = None bbfiles = [] bbappend = [] @@ -1948,9 +1970,17 @@ class Parser(multiprocessing.Process): def parse(self, filename, appends, caches_array): try: + # Record the filename we're parsing into any events generated + def parse_filter(self, record): + record.taskpid = bb.event.worker_pid + record.fn = filename + return True + # Reset our environment and handlers to the original settings bb.utils.set_context(self.context.copy()) bb.event.set_class_handlers(self.handlers.copy()) + bb.event.LogHandler.filter = parse_filter + return True, bb.cache.Cache.parse(filename, appends, self.cfg, caches_array) except Exception as exc: tb = sys.exc_info()[2] @@ -1981,8 +2011,6 @@ class CookerParser(object): self.total = len(filelist) self.current = 0 - self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or - multiprocessing.cpu_count()) self.process_names = [] self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) @@ -1997,6 +2025,9 @@ class CookerParser(object): self.toparse = self.total - len(self.fromcache) self.progress_chunk = max(self.toparse / 100, 1) + self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or + multiprocessing.cpu_count()), len(self.willparse)) + self.start() self.haveshutdown = False @@ -2007,8 +2038,9 @@ class CookerParser(object): bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) def init(): Parser.cfg = self.cfgdata - multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cfgdata,), exitpriority=1) - multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, args=(self.cfgdata,), exitpriority=1) + bb.utils.set_process_name(multiprocessing.current_process().name) + multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) self.feeder_quit = multiprocessing.Queue(maxsize=1) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) @@ -2040,7 +2072,7 @@ class CookerParser(object): bb.event.fire(event, self.cfgdata) self.feeder_quit.put(None) for process in self.processes: - self.jobs.put(None) + self.parser_quit.put(None) else: self.feeder_quit.put('cancel') @@ -2061,8 +2093,8 @@ class CookerParser(object): sync = threading.Thread(target=self.bb_cache.sync) sync.start() multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) - bb.codeparser.parser_cache_savemerge(self.cooker.data) - bb.fetch.fetcher_parse_done(self.cooker.data) + bb.codeparser.parser_cache_savemerge() + bb.fetch.fetcher_parse_done() if self.cooker.configuration.profile: profiles = [] for i in self.process_names: @@ -2125,11 +2157,6 @@ class CookerParser(object): logger.error('ExpansionError during parsing %s: %s', value.recipe, str(exc)) self.shutdown(clean=False) return False - except SyntaxError as exc: - self.error += 1 - logger.error('Unable to parse %s', exc.recipe) - self.shutdown(clean=False) - return False except Exception as exc: self.error += 1 etype, value, tb = sys.exc_info() |