From eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 Mon Sep 17 00:00:00 2001 From: Dave Cobbley Date: Tue, 14 Aug 2018 10:05:37 -0700 Subject: [Subtree] Removing import-layers directory As part of the move to subtrees, need to bring all the import layers content to the top level. Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f Signed-off-by: Dave Cobbley Signed-off-by: Brad Bishop --- poky/bitbake/lib/bb/cooker.py | 2161 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2161 insertions(+) create mode 100644 poky/bitbake/lib/bb/cooker.py (limited to 'poky/bitbake/lib/bb/cooker.py') diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py new file mode 100644 index 000000000..1fda40dd4 --- /dev/null +++ b/poky/bitbake/lib/bb/cooker.py @@ -0,0 +1,2161 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 - 2007 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import sys, os, glob, os.path, re, time +import atexit +import itertools +import logging +import multiprocessing +import sre_constants +import threading +from io import StringIO, UnsupportedOperation +from contextlib import closing +from functools import wraps +from collections import defaultdict, namedtuple +import bb, bb.exceptions, bb.command +from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build +import queue +import signal +import subprocess +import errno +import prserv.serv +import pyinotify +import json +import pickle +import codecs + +logger = logging.getLogger("BitBake") +collectlog = logging.getLogger("BitBake.Collection") +buildlog = logging.getLogger("BitBake.Build") +parselog = logging.getLogger("BitBake.Parsing") +providerlog = logging.getLogger("BitBake.Provider") + +class NoSpecificMatch(bb.BBHandledException): + """ + Exception raised when no or multiple file matches are found + """ + +class NothingToBuild(Exception): + """ + Exception raised when there is nothing to build + """ + +class CollectionError(bb.BBHandledException): + """ + Exception raised when layer configuration is incorrect + """ + +class state: + initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7)) + + @classmethod + def get_name(cls, code): + for name in dir(cls): + value = getattr(cls, name) + if type(value) == type(cls.initial) and value == code: + return name + raise ValueError("Invalid status code: %s" % code) + + +class SkippedPackage: + def __init__(self, info = None, reason = None): + self.pn = None + self.skipreason = None + self.provides = None + self.rprovides = None + + if info: + self.pn = info.pn + self.skipreason = info.skipreason + self.provides = info.provides + self.rprovides = info.rprovides + elif reason: + self.skipreason = reason + + +class CookerFeatures(object): + _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3)) + + def __init__(self): + self._features=set() + + def setFeature(self, f): + # validate we got a request for a feature we support + if f not in CookerFeatures._feature_list: + return + self._features.add(f) + + def __contains__(self, f): + return f in self._features + + def __iter__(self): + return self._features.__iter__() + + def __next__(self): + return next(self._features) + + +class EventWriter: + def __init__(self, cooker, eventfile): + self.file_inited = None + self.cooker = cooker + self.eventfile = eventfile + self.event_queue = [] + + def write_event(self, event): + with open(self.eventfile, "a") as f: + try: + str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') + f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, + "vars": str_event})) + except Exception as err: + import traceback + print(err, traceback.format_exc()) + + def send(self, event): + if self.file_inited: + # we have the file, just write the event + self.write_event(event) + else: + # init on bb.event.BuildStarted + name = "%s.%s" % (event.__module__, event.__class__.__name__) + if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"): + with open(self.eventfile, "w") as f: + f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) + + self.file_inited = True + + # write pending events + for evt in self.event_queue: + self.write_event(evt) + + # also write the current event + self.write_event(event) + else: + # queue all events until the file is inited + self.event_queue.append(event) + +#============================================================================# +# BBCooker +#============================================================================# +class BBCooker: + """ + Manages one bitbake build run + """ + + def __init__(self, configuration, featureSet=None): + self.recipecaches = None + self.skiplist = {} + self.featureset = CookerFeatures() + if featureSet: + for f in featureSet: + self.featureset.setFeature(f) + + self.configuration = configuration + + self.configwatcher = pyinotify.WatchManager() + self.configwatcher.bbseen = [] + self.configwatcher.bbwatchedfiles = [] + self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) + self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \ + pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \ + pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO + self.watcher = pyinotify.WatchManager() + self.watcher.bbseen = [] + self.watcher.bbwatchedfiles = [] + self.notifier = pyinotify.Notifier(self.watcher, self.notifications) + + # If being called by something like tinfoil, we need to clean cached data + # which may now be invalid + bb.parse.clear_cache() + bb.parse.BBHandler.cached_statements = {} + + self.ui_cmdline = None + + self.initConfigurationData() + + # we log all events to a file if so directed + if self.configuration.writeeventlog: + # register the log file writer as UI Handler + writer = EventWriter(self, self.configuration.writeeventlog) + EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) + bb.event.register_UIHhandler(EventLogWriteHandler(writer)) + + self.inotify_modified_files = [] + + def _process_inotify_updates(server, cooker, abort): + cooker.process_inotify_updates() + return 1.0 + + self.configuration.server_register_idlecallback(_process_inotify_updates, self) + + # TOSTOP must not be set or our children will hang when they output + try: + fd = sys.stdout.fileno() + if os.isatty(fd): + import termios + tcattr = termios.tcgetattr(fd) + if tcattr[3] & termios.TOSTOP: + buildlog.info("The terminal had the TOSTOP bit set, clearing...") + tcattr[3] = tcattr[3] & ~termios.TOSTOP + termios.tcsetattr(fd, termios.TCSANOW, tcattr) + except UnsupportedOperation: + pass + + self.command = bb.command.Command(self) + self.state = state.initial + + self.parser = None + + signal.signal(signal.SIGTERM, self.sigterm_exception) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, self.sigterm_exception) + + def process_inotify_updates(self): + for n in [self.confignotifier, self.notifier]: + if n.check_events(timeout=0): + # read notified events and enqeue them + n.read_events() + n.process_events() + + def config_notifications(self, event): + if event.maskname == "IN_Q_OVERFLOW": + bb.warn("inotify event queue overflowed, invalidating caches.") + self.parsecache_valid = False + self.baseconfig_valid = False + bb.parse.clear_cache() + return + if not event.pathname in self.configwatcher.bbwatchedfiles: + return + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) + self.baseconfig_valid = False + + def notifications(self, event): + if event.maskname == "IN_Q_OVERFLOW": + bb.warn("inotify event queue overflowed, invalidating caches.") + self.parsecache_valid = False + bb.parse.clear_cache() + return + if event.pathname.endswith("bitbake-cookerdaemon.log") \ + or event.pathname.endswith("bitbake.lock"): + return + if not event.pathname in self.inotify_modified_files: + self.inotify_modified_files.append(event.pathname) + self.parsecache_valid = False + + def add_filewatch(self, deps, watcher=None, dirs=False): + if not watcher: + watcher = self.watcher + for i in deps: + watcher.bbwatchedfiles.append(i[0]) + if dirs: + f = i[0] + else: + f = os.path.dirname(i[0]) + if f in watcher.bbseen: + continue + watcher.bbseen.append(f) + watchtarget = None + while True: + # We try and add watches for files that don't exist but if they did, would influence + # the parser. The parent directory of these files may not exist, in which case we need + # to watch any parent that does exist for changes. + try: + watcher.add_watch(f, self.watchmask, quiet=False) + if watchtarget: + watcher.bbwatchedfiles.append(watchtarget) + break + except pyinotify.WatchManagerError as e: + if 'ENOENT' in str(e): + watchtarget = f + f = os.path.dirname(f) + if f in watcher.bbseen: + break + watcher.bbseen.append(f) + continue + if 'ENOSPC' in str(e): + providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?") + providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.") + providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=.") + providerlog.error("Root privilege is required to modify max_user_watches.") + raise + + def sigterm_exception(self, signum, stackframe): + if signum == signal.SIGTERM: + bb.warn("Cooker received SIGTERM, shutting down...") + elif signum == signal.SIGHUP: + bb.warn("Cooker received SIGHUP, shutting down...") + self.state = state.forceshutdown + + def setFeatures(self, features): + # we only accept a new feature set if we're in state initial, so we can reset without problems + if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]: + raise Exception("Illegal state for feature set change") + original_featureset = list(self.featureset) + for feature in features: + self.featureset.setFeature(feature) + bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset))) + if (original_featureset != list(self.featureset)) and self.state != state.error: + self.reset() + + def initConfigurationData(self): + + self.state = state.initial + self.caches_array = [] + + # Need to preserve BB_CONSOLELOG over resets + consolelog = None + if hasattr(self, "data"): + consolelog = self.data.getVar("BB_CONSOLELOG") + + if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: + self.enableDataTracking() + + all_extra_cache_names = [] + # We hardcode all known cache types in a single place, here. + if CookerFeatures.HOB_EXTRA_CACHES in self.featureset: + all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo") + + caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names + + # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! + # This is the entry point, no further check needed! + for var in caches_name_array: + try: + module_name, cache_name = var.split(':') + module = __import__(module_name, fromlist=(cache_name,)) + self.caches_array.append(getattr(module, cache_name)) + except ImportError as exc: + logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) + sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) + + self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False) + self.databuilder.parseBaseConfiguration() + self.data = self.databuilder.data + self.data_hash = self.databuilder.data_hash + self.extraconfigdata = {} + + if consolelog: + self.data.setVar("BB_CONSOLELOG", consolelog) + + self.data.setVar('BB_CMDLINE', self.ui_cmdline) + + # + # Copy of the data store which has been expanded. + # Used for firing events and accessing variables where expansion needs to be accounted for + # + bb.parse.init_parser(self.data) + + if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: + self.disableDataTracking() + + self.data.renameVar("__depends", "__base_depends") + self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher) + + self.baseconfig_valid = True + self.parsecache_valid = False + + def handlePRServ(self): + # Setup a PR Server based on the new configuration + try: + self.prhost = prserv.serv.auto_start(self.data) + except prserv.serv.PRServiceConfigError as e: + bb.fatal("Unable to start PR Server, exitting") + + def enableDataTracking(self): + self.configuration.tracking = True + if hasattr(self, "data"): + self.data.enableTracking() + + def disableDataTracking(self): + self.configuration.tracking = False + if hasattr(self, "data"): + self.data.disableTracking() + + def parseConfiguration(self): + # Set log file verbosity + verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", False)) + if verboselogs: + bb.msg.loggerVerboseLogs = True + + # Change nice level if we're asked to + nice = self.data.getVar("BB_NICE_LEVEL") + if nice: + curnice = os.nice(0) + nice = int(nice) - curnice + buildlog.verbose("Renice to %s " % os.nice(nice)) + + if self.recipecaches: + del self.recipecaches + self.multiconfigs = self.databuilder.mcdata.keys() + self.recipecaches = {} + for mc in self.multiconfigs: + self.recipecaches[mc] = bb.cache.CacheData(self.caches_array) + + self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS")) + + self.parsecache_valid = False + + def updateConfigOpts(self, options, environment, cmdline): + self.ui_cmdline = cmdline + clean = True + for o in options: + if o in ['prefile', 'postfile']: + # Only these options may require a reparse + try: + if getattr(self.configuration, o) == options[o]: + # Value is the same, no need to mark dirty + continue + except AttributeError: + pass + logger.debug(1, "Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) + print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) + clean = False + setattr(self.configuration, o, options[o]) + for k in bb.utils.approved_variables(): + if k in environment and k not in self.configuration.env: + logger.debug(1, "Updating new environment variable %s to %s" % (k, environment[k])) + self.configuration.env[k] = environment[k] + clean = False + if k in self.configuration.env and k not in environment: + logger.debug(1, "Updating environment variable %s (deleted)" % (k)) + del self.configuration.env[k] + clean = False + if k not in self.configuration.env and k not in environment: + continue + if environment[k] != self.configuration.env[k]: + logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k])) + self.configuration.env[k] = environment[k] + clean = False + if not clean: + logger.debug(1, "Base environment change, triggering reparse") + self.reset() + + def runCommands(self, server, data, abort): + """ + Run any queued asynchronous command + This is done by the idle handler so it runs in true context rather than + tied to any UI. + """ + + return self.command.runAsyncCommand() + + def showVersions(self): + + (latest_versions, preferred_versions) = self.findProviders() + + logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") + logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") + + for p in sorted(self.recipecaches[''].pkg_pn): + pref = preferred_versions[p] + latest = latest_versions[p] + + prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2] + lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] + + if pref == latest: + prefstr = "" + + logger.plain("%-35s %25s %25s", p, lateststr, prefstr) + + def showEnvironment(self, buildfile=None, pkgs_to_build=None): + """ + Show the outer or per-recipe environment + """ + fn = None + envdata = None + if not pkgs_to_build: + pkgs_to_build = [] + + orig_tracking = self.configuration.tracking + if not orig_tracking: + self.enableDataTracking() + self.reset() + + + if buildfile: + # Parse the configuration here. We need to do it explicitly here since + # this showEnvironment() code path doesn't use the cache + self.parseConfiguration() + + fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) + fn = self.matchFile(fn) + fn = bb.cache.realfn2virtual(fn, cls, mc) + elif len(pkgs_to_build) == 1: + ignore = self.data.getVar("ASSUME_PROVIDED") or "" + if pkgs_to_build[0] in set(ignore.split()): + bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) + + taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) + + mc = runlist[0][0] + fn = runlist[0][3] + else: + envdata = self.data + data.expandKeys(envdata) + parse.ast.runAnonFuncs(envdata) + + if fn: + try: + bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) + envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) + except Exception as e: + parselog.exception("Unable to read %s", fn) + raise + + # Display history + with closing(StringIO()) as env: + self.data.inchistory.emit(env) + logger.plain(env.getvalue()) + + # emit variables and shell functions + with closing(StringIO()) as env: + data.emit_env(env, envdata, True) + logger.plain(env.getvalue()) + + # emit the metadata which isnt valid shell + for e in sorted(envdata.keys()): + if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False): + logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) + + if not orig_tracking: + self.disableDataTracking() + self.reset() + + def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False): + """ + Prepare a runqueue and taskdata object for iteration over pkgs_to_build + """ + bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data) + + # A task of None means use the default task + if task is None: + task = self.configuration.cmd + if not task.startswith("do_"): + task = "do_%s" % task + + targetlist = self.checkPackages(pkgs_to_build, task) + fulltargetlist = [] + defaulttask_implicit = '' + defaulttask_explicit = False + wildcard = False + + # Wild card expansion: + # Replace string such as "multiconfig:*:bash" + # into "multiconfig:A:bash multiconfig:B:bash bash" + for k in targetlist: + if k.startswith("multiconfig:"): + if wildcard: + bb.fatal('multiconfig conflict') + if k.split(":")[1] == "*": + wildcard = True + for mc in self.multiconfigs: + if mc: + fulltargetlist.append(k.replace('*', mc)) + # implicit default task + else: + defaulttask_implicit = k.split(":")[2] + else: + fulltargetlist.append(k) + else: + defaulttask_explicit = True + fulltargetlist.append(k) + + if not defaulttask_explicit and defaulttask_implicit != '': + fulltargetlist.append(defaulttask_implicit) + + bb.debug(1,"Target list: %s" % (str(fulltargetlist))) + taskdata = {} + localdata = {} + + for mc in self.multiconfigs: + taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete) + localdata[mc] = data.createCopy(self.databuilder.mcdata[mc]) + bb.data.expandKeys(localdata[mc]) + + current = 0 + runlist = [] + for k in fulltargetlist: + mc = "" + if k.startswith("multiconfig:"): + mc = k.split(":")[1] + k = ":".join(k.split(":")[2:]) + ktask = task + if ":do_" in k: + k2 = k.split(":do_") + k = k2[0] + ktask = k2[1] + taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) + current += 1 + if not ktask.startswith("do_"): + ktask = "do_%s" % ktask + if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]: + # e.g. in ASSUME_PROVIDED + continue + fn = taskdata[mc].build_targets[k][0] + runlist.append([mc, k, ktask, fn]) + bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) + + for mc in self.multiconfigs: + taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) + + bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data) + return taskdata, runlist + + def prepareTreeData(self, pkgs_to_build, task): + """ + Prepare a runqueue and taskdata object for iteration over pkgs_to_build + """ + + # We set abort to False here to prevent unbuildable targets raising + # an exception when we're just generating data + taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) + + return runlist, taskdata + + ######## WARNING : this function requires cache_extra to be enabled ######## + + def generateTaskDepTreeData(self, pkgs_to_build, task): + """ + Create a dependency graph of pkgs_to_build including reverse dependency + information. + """ + if not task.startswith("do_"): + task = "do_%s" % task + + runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) + rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) + rq.rqdata.prepare() + return self.buildDependTree(rq, taskdata) + + @staticmethod + def add_mc_prefix(mc, pn): + if mc: + return "multiconfig:%s:%s" % (mc, pn) + return pn + + def buildDependTree(self, rq, taskdata): + seen_fns = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["tdepends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["packages"] = {} + depend_tree["rdepends-pkg"] = {} + depend_tree["rrecs-pkg"] = {} + depend_tree['providermap'] = {} + depend_tree["layer-priorities"] = self.bbfile_config_priorities + + for mc in taskdata: + for name, fn in list(taskdata[mc].get_providermap().items()): + pn = self.recipecaches[mc].pkg_fn[fn] + pn = self.add_mc_prefix(mc, pn) + if name != pn: + version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn] + depend_tree['providermap'][name] = (pn, version) + + for tid in rq.rqdata.runtaskentries: + (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) + pn = self.recipecaches[mc].pkg_fn[taskfn] + pn = self.add_mc_prefix(mc, pn) + version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = taskfn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) + + # if we have extra caches, list all attributes they bring in + extra_info = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): + cachefields = getattr(cache_class, 'cachefields', []) + extra_info = extra_info + cachefields + + # for all attributes stored, add them to the dependency tree + for ei in extra_info: + depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] + + + dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid)) + if not dotname in depend_tree["tdepends"]: + depend_tree["tdepends"][dotname] = [] + for dep in rq.rqdata.runtaskentries[tid].depends: + (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep) + deppn = self.recipecaches[mc].pkg_fn[deptaskfn] + depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) + if taskfn not in seen_fns: + seen_fns.append(taskfn) + packages = [] + + depend_tree["depends"][pn] = [] + for dep in taskdata[mc].depids[taskfn]: + depend_tree["depends"][pn].append(dep) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata[mc].rdepids[taskfn]: + depend_tree["rdepends-pn"][pn].append(rdep) + + rdepends = self.recipecaches[mc].rundeps[taskfn] + for package in rdepends: + depend_tree["rdepends-pkg"][package] = [] + for rdepend in rdepends[package]: + depend_tree["rdepends-pkg"][package].append(rdepend) + packages.append(package) + + rrecs = self.recipecaches[mc].runrecs[taskfn] + for package in rrecs: + depend_tree["rrecs-pkg"][package] = [] + for rdepend in rrecs[package]: + depend_tree["rrecs-pkg"][package].append(rdepend) + if not package in packages: + packages.append(package) + + for package in packages: + if package not in depend_tree["packages"]: + depend_tree["packages"][package] = {} + depend_tree["packages"][package]["pn"] = pn + depend_tree["packages"][package]["filename"] = taskfn + depend_tree["packages"][package]["version"] = version + + return depend_tree + + ######## WARNING : this function requires cache_extra to be enabled ######## + def generatePkgDepTreeData(self, pkgs_to_build, task): + """ + Create a dependency tree of pkgs_to_build, returning the data. + """ + if not task.startswith("do_"): + task = "do_%s" % task + + _, taskdata = self.prepareTreeData(pkgs_to_build, task) + + seen_fns = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["rdepends-pkg"] = {} + depend_tree["rrecs-pkg"] = {} + + # if we have extra caches, list all attributes they bring in + extra_info = [] + for cache_class in self.caches_array: + if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): + cachefields = getattr(cache_class, 'cachefields', []) + extra_info = extra_info + cachefields + + tids = [] + for mc in taskdata: + for tid in taskdata[mc].taskentries: + tids.append(tid) + + for tid in tids: + (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) + + pn = self.recipecaches[mc].pkg_fn[taskfn] + pn = self.add_mc_prefix(mc, pn) + + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = taskfn + version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] + depend_tree["pn"][pn]["version"] = version + rdepends = self.recipecaches[mc].rundeps[taskfn] + rrecs = self.recipecaches[mc].runrecs[taskfn] + depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) + + # for all extra attributes stored, add them to the dependency tree + for ei in extra_info: + depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] + + if taskfn not in seen_fns: + seen_fns.append(taskfn) + + depend_tree["depends"][pn] = [] + for dep in taskdata[mc].depids[taskfn]: + pn_provider = "" + if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]: + fn_provider = taskdata[mc].build_targets[dep][0] + pn_provider = self.recipecaches[mc].pkg_fn[fn_provider] + else: + pn_provider = dep + pn_provider = self.add_mc_prefix(mc, pn_provider) + depend_tree["depends"][pn].append(pn_provider) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata[mc].rdepids[taskfn]: + pn_rprovider = "" + if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]: + fn_rprovider = taskdata[mc].run_targets[rdep][0] + pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider] + else: + pn_rprovider = rdep + pn_rprovider = self.add_mc_prefix(mc, pn_rprovider) + depend_tree["rdepends-pn"][pn].append(pn_rprovider) + + depend_tree["rdepends-pkg"].update(rdepends) + depend_tree["rrecs-pkg"].update(rrecs) + + return depend_tree + + def generateDepTreeEvent(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Generate an event with the result + """ + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) + bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data) + + def generateDotGraphFiles(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Save the result to a set of .dot files. + """ + + depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) + + with open('pn-buildlist', 'w') as f: + for pn in depgraph["pn"]: + f.write(pn + "\n") + logger.info("PN build list saved to 'pn-buildlist'") + + # Remove old format output files to ensure no confusion with stale data + try: + os.unlink('pn-depends.dot') + except FileNotFoundError: + pass + try: + os.unlink('package-depends.dot') + except FileNotFoundError: + pass + + with open('task-depends.dot', 'w') as f: + f.write("digraph depends {\n") + for task in sorted(depgraph["tdepends"]): + (pn, taskname) = task.rsplit(".", 1) + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn)) + for dep in sorted(depgraph["tdepends"][task]): + f.write('"%s" -> "%s"\n' % (task, dep)) + f.write("}\n") + logger.info("Task dependencies saved to 'task-depends.dot'") + + with open('recipe-depends.dot', 'w') as f: + f.write("digraph depends {\n") + pndeps = {} + for task in sorted(depgraph["tdepends"]): + (pn, taskname) = task.rsplit(".", 1) + if pn not in pndeps: + pndeps[pn] = set() + for dep in sorted(depgraph["tdepends"][task]): + (deppn, deptaskname) = dep.rsplit(".", 1) + pndeps[pn].add(deppn) + for pn in sorted(pndeps): + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn)) + for dep in sorted(pndeps[pn]): + if dep == pn: + continue + f.write('"%s" -> "%s"\n' % (pn, dep)) + f.write("}\n") + logger.info("Flattened recipe dependencies saved to 'recipe-depends.dot'") + + def show_appends_with_no_recipes(self): + # Determine which bbappends haven't been applied + + # First get list of recipes, including skipped + recipefns = list(self.recipecaches[''].pkg_fn.keys()) + recipefns.extend(self.skiplist.keys()) + + # Work out list of bbappends that have been applied + applied_appends = [] + for fn in recipefns: + applied_appends.extend(self.collection.get_file_appends(fn)) + + appends_without_recipes = [] + for _, appendfn in self.collection.bbappends: + if not appendfn in applied_appends: + appends_without_recipes.append(appendfn) + + if appends_without_recipes: + msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) + warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ + False) or "no" + if warn_only.lower() in ("1", "yes", "true"): + bb.warn(msg) + else: + bb.fatal(msg) + + def handlePrefProviders(self): + + for mc in self.multiconfigs: + localdata = data.createCopy(self.databuilder.mcdata[mc]) + bb.data.expandKeys(localdata) + + # Handle PREFERRED_PROVIDERS + for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split(): + try: + (providee, provider) = p.split(':') + except: + providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) + continue + if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider: + providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee]) + self.recipecaches[mc].preferred[providee] = provider + + def findConfigFilePath(self, configfile): + """ + Find the location on disk of configfile and if it exists and was parsed by BitBake + emit the ConfigFilePathFound event with the path to the file. + """ + path = bb.cookerdata.findConfigFile(configfile, self.data) + if not path: + return + + # Generate a list of parsed configuration files by searching the files + # listed in the __depends and __base_depends variables with a .conf suffix. + conffiles = [] + dep_files = self.data.getVar('__base_depends', False) or [] + dep_files = dep_files + (self.data.getVar('__depends', False) or []) + + for f in dep_files: + if f[0].endswith(".conf"): + conffiles.append(f[0]) + + _, conf, conffile = path.rpartition("conf/") + match = os.path.join(conf, conffile) + # Try and find matches for conf/conffilename.conf as we don't always + # have the full path to the file. + for cfg in conffiles: + if cfg.endswith(match): + bb.event.fire(bb.event.ConfigFilePathFound(path), + self.data) + break + + def findFilesMatchingInDir(self, filepattern, directory): + """ + Searches for files containing the substring 'filepattern' which are children of + 'directory' in each BBPATH. i.e. to find all rootfs package classes available + to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes') + or to find all machine configuration files one could call: + findFilesMatchingInDir(self, '.conf', 'conf/machine') + """ + + matches = [] + bbpaths = self.data.getVar('BBPATH').split(':') + for path in bbpaths: + dirpath = os.path.join(path, directory) + if os.path.exists(dirpath): + for root, dirs, files in os.walk(dirpath): + for f in files: + if filepattern in f: + matches.append(f) + + if matches: + bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) + + def findProviders(self, mc=''): + return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + + def findBestProvider(self, pn, mc=''): + if pn in self.recipecaches[mc].providers: + filenames = self.recipecaches[mc].providers[pn] + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc]) + filename = eligible[0] + return None, None, None, filename + elif pn in self.recipecaches[mc].pkg_pn: + return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn) + else: + return None, None, None, None + + def findConfigFiles(self, varname): + """ + Find config files which are appropriate values for varname. + i.e. MACHINE, DISTRO + """ + possible = [] + var = varname.lower() + + data = self.data + # iterate configs + bbpaths = data.getVar('BBPATH').split(':') + for path in bbpaths: + confpath = os.path.join(path, "conf", var) + if os.path.exists(confpath): + for root, dirs, files in os.walk(confpath): + # get all child files, these are appropriate values + for f in files: + val, sep, end = f.rpartition('.') + if end == 'conf': + possible.append(val) + + if possible: + bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data) + + def findInheritsClass(self, klass): + """ + Find all recipes which inherit the specified class + """ + pkg_list = [] + + for pfn in self.recipecaches[''].pkg_fn: + inherits = self.recipecaches[''].inherits.get(pfn, None) + if inherits and klass in inherits: + pkg_list.append(self.recipecaches[''].pkg_fn[pfn]) + + return pkg_list + + def generateTargetsTree(self, klass=None, pkgs=None): + """ + Generate a dependency tree of buildable targets + Generate an event with the result + """ + # if the caller hasn't specified a pkgs list default to universe + if not pkgs: + pkgs = ['universe'] + # if inherited_class passed ensure all recipes which inherit the + # specified class are included in pkgs + if klass: + extra_pkgs = self.findInheritsClass(klass) + pkgs = pkgs + extra_pkgs + + # generate a dependency tree for all our packages + tree = self.generatePkgDepTreeData(pkgs, 'build') + bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data) + + def interactiveMode( self ): + """Drop off into a shell""" + try: + from bb import shell + except ImportError: + parselog.exception("Interactive mode not available") + sys.exit(1) + else: + shell.start( self ) + + + def handleCollections(self, collections): + """Handle collections""" + errors = False + self.bbfile_config_priorities = [] + if collections: + collection_priorities = {} + collection_depends = {} + collection_list = collections.split() + min_prio = 0 + for c in collection_list: + bb.debug(1,'Processing %s in collection list' % (c)) + + # Get collection priority if defined explicitly + priority = self.data.getVar("BBFILE_PRIORITY_%s" % c) + if priority: + try: + prio = int(priority) + except ValueError: + parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) + errors = True + if min_prio == 0 or prio < min_prio: + min_prio = prio + collection_priorities[c] = prio + else: + collection_priorities[c] = None + + # Check dependencies and store information for priority calculation + deps = self.data.getVar("LAYERDEPENDS_%s" % c) + if deps: + try: + depDict = bb.utils.explode_dep_versions2(deps) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) + for dep, oplist in list(depDict.items()): + if dep in collection_list: + for opstr in oplist: + layerver = self.data.getVar("LAYERVERSION_%s" % dep) + (op, depver) = opstr.split() + if layerver: + try: + res = bb.utils.vercmp_string_op(layerver, depver, op) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) + if not res: + parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver) + errors = True + else: + parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep) + errors = True + else: + parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep) + errors = True + collection_depends[c] = list(depDict.keys()) + else: + collection_depends[c] = [] + + # Check recommends and store information for priority calculation + recs = self.data.getVar("LAYERRECOMMENDS_%s" % c) + if recs: + try: + recDict = bb.utils.explode_dep_versions2(recs) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) + for rec, oplist in list(recDict.items()): + if rec in collection_list: + if oplist: + opstr = oplist[0] + layerver = self.data.getVar("LAYERVERSION_%s" % rec) + if layerver: + (op, recver) = opstr.split() + try: + res = bb.utils.vercmp_string_op(layerver, recver, op) + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) + if not res: + parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver) + continue + else: + parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec) + continue + parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec) + collection_depends[c].append(rec) + else: + parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec) + + # Recursively work out collection priorities based on dependencies + def calc_layer_priority(collection): + if not collection_priorities[collection]: + max_depprio = min_prio + for dep in collection_depends[collection]: + calc_layer_priority(dep) + depprio = collection_priorities[dep] + if depprio > max_depprio: + max_depprio = depprio + max_depprio += 1 + parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio) + collection_priorities[collection] = max_depprio + + # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities + for c in collection_list: + calc_layer_priority(c) + regex = self.data.getVar("BBFILE_PATTERN_%s" % c) + if regex == None: + parselog.error("BBFILE_PATTERN_%s not defined" % c) + errors = True + continue + elif regex == "": + parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c) + errors = False + continue + else: + try: + cre = re.compile(regex) + except re.error: + parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) + errors = True + continue + self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) + if errors: + # We've already printed the actual error(s) + raise CollectionError("Errors during parsing layer configuration") + + def buildSetVars(self): + """ + Setup any variables needed before starting a build + """ + t = time.gmtime() + for mc in self.databuilder.mcdata: + ds = self.databuilder.mcdata[mc] + if not ds.getVar("BUILDNAME", False): + ds.setVar("BUILDNAME", "${DATE}${TIME}") + ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t)) + ds.setVar("DATE", time.strftime('%Y%m%d', t)) + ds.setVar("TIME", time.strftime('%H%M%S', t)) + + def reset_mtime_caches(self): + """ + Reset mtime caches - this is particularly important when memory resident as something + which is cached is not unlikely to have changed since the last invocation (e.g. a + file associated with a recipe might have been modified by the user). + """ + build.reset_cache() + bb.fetch._checksum_cache.mtime_cache.clear() + siggen_cache = getattr(bb.parse.siggen, 'checksum_cache', None) + if siggen_cache: + bb.parse.siggen.checksum_cache.mtime_cache.clear() + + def matchFiles(self, bf): + """ + Find the .bb files which match the expression in 'buildfile'. + """ + if bf.startswith("/") or bf.startswith("../"): + bf = os.path.abspath(bf) + + self.collection = CookerCollectFiles(self.bbfile_config_priorities) + filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) + try: + os.stat(bf) + bf = os.path.abspath(bf) + return [bf] + except OSError: + regexp = re.compile(bf) + matches = [] + for f in filelist: + if regexp.search(f) and os.path.isfile(f): + matches.append(f) + return matches + + def matchFile(self, buildfile): + """ + Find the .bb file which matches the expression in 'buildfile'. + Raise an error if multiple files + """ + matches = self.matchFiles(buildfile) + if len(matches) != 1: + if matches: + msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) + if matches: + for f in matches: + msg += "\n %s" % f + parselog.error(msg) + else: + parselog.error("Unable to find any recipe file matching '%s'" % buildfile) + raise NoSpecificMatch + return matches[0] + + def buildFile(self, buildfile, task): + """ + Build the file matching regexp buildfile + """ + bb.event.fire(bb.event.BuildInit(), self.data) + + # Too many people use -b because they think it's how you normally + # specify a target to be built, so show a warning + bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.") + + self.buildFileInternal(buildfile, task) + + def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False): + """ + Build the file matching regexp buildfile + """ + + # Parse the configuration here. We need to do it explicitly here since + # buildFile() doesn't use the cache + self.parseConfiguration() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + if not task.startswith("do_"): + task = "do_%s" % task + + fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) + fn = self.matchFile(fn) + + self.buildSetVars() + self.reset_mtime_caches() + + bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) + + infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) + infos = dict(infos) + + fn = bb.cache.realfn2virtual(fn, cls, mc) + try: + info_array = infos[fn] + except KeyError: + bb.fatal("%s does not exist" % fn) + + if info_array[0].skipped: + bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason)) + + self.recipecaches[mc].add_from_recipeinfo(fn, info_array) + + # Tweak some variables + item = info_array[0].pn + self.recipecaches[mc].ignored_dependencies = set() + self.recipecaches[mc].bbfile_priority[fn] = 1 + self.configuration.limited_deps = True + + # Remove external dependencies + self.recipecaches[mc].task_deps[fn]['depends'] = {} + self.recipecaches[mc].deps[fn] = [] + self.recipecaches[mc].rundeps[fn] = defaultdict(list) + self.recipecaches[mc].runrecs[fn] = defaultdict(list) + + # Invalidate task for target if force mode active + if self.configuration.force: + logger.verbose("Invalidate task %s, %s", task, fn) + bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn) + + # Setup taskdata structure + taskdata = {} + taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort) + taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item) + + if quietlog: + rqloglevel = bb.runqueue.logger.getEffectiveLevel() + bb.runqueue.logger.setLevel(logging.WARNING) + + buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME") + if fireevents: + bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc]) + + # Execute the runqueue + runlist = [[mc, item, task, fn]] + + rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) + + def buildFileIdle(server, rq, abort): + + msg = None + interrupted = 0 + if abort or self.state == state.forceshutdown: + rq.finish_runqueue(True) + msg = "Forced shutdown" + interrupted = 2 + elif self.state == state.shutdown: + rq.finish_runqueue(False) + msg = "Stopped build" + interrupted = 1 + failures = 0 + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure as exc: + failures += len(exc.args) + retval = False + except SystemExit as exc: + self.command.finishAsyncCommand(str(exc)) + if quietlog: + bb.runqueue.logger.setLevel(rqloglevel) + return False + + if not retval: + if fireevents: + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc]) + self.command.finishAsyncCommand(msg) + # We trashed self.recipecaches above + self.parsecache_valid = False + self.configuration.limited_deps = False + bb.parse.siggen.reset(self.data) + if quietlog: + bb.runqueue.logger.setLevel(rqloglevel) + return False + if retval is True: + return True + return retval + + self.configuration.server_register_idlecallback(buildFileIdle, rq) + + def buildTargets(self, targets, task): + """ + Attempt to build the targets specified + """ + + def buildTargetsIdle(server, rq, abort): + msg = None + interrupted = 0 + if abort or self.state == state.forceshutdown: + rq.finish_runqueue(True) + msg = "Forced shutdown" + interrupted = 2 + elif self.state == state.shutdown: + rq.finish_runqueue(False) + msg = "Stopped build" + interrupted = 1 + failures = 0 + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure as exc: + failures += len(exc.args) + retval = False + except SystemExit as exc: + self.command.finishAsyncCommand(str(exc)) + return False + + if not retval: + try: + for mc in self.multiconfigs: + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc]) + finally: + self.command.finishAsyncCommand(msg) + return False + if retval is True: + return True + return retval + + self.reset_mtime_caches() + self.buildSetVars() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + + if not task.startswith("do_"): + task = "do_%s" % task + + packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets] + + bb.event.fire(bb.event.BuildInit(packages), self.data) + + taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort) + + buildname = self.data.getVar("BUILDNAME", False) + + # make targets to always look as :do_ + ntargets = [] + for target in runlist: + if target[0]: + ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2])) + ntargets.append("%s:%s" % (target[1], target[2])) + + for mc in self.multiconfigs: + bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc]) + + rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) + if 'universe' in targets: + rq.rqdata.warn_multi_bb = True + + self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + + + def getAllKeysWithFlags(self, flaglist): + dump = {} + for k in self.data.keys(): + try: + expand = True + flags = self.data.getVarFlags(k) + if flags and "func" in flags and "python" in flags: + expand = False + v = self.data.getVar(k, expand) + if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart): + dump[k] = { + 'v' : str(v) , + 'history' : self.data.varhistory.variable(k), + } + for d in flaglist: + if flags and d in flags: + dump[k][d] = flags[d] + else: + dump[k][d] = None + except Exception as e: + print(e) + return dump + + + def updateCacheSync(self): + if self.state == state.running: + return + + # reload files for which we got notifications + for p in self.inotify_modified_files: + bb.parse.update_cache(p) + if p in bb.parse.BBHandler.cached_statements: + del bb.parse.BBHandler.cached_statements[p] + self.inotify_modified_files = [] + + if not self.baseconfig_valid: + logger.debug(1, "Reloading base configuration data") + self.initConfigurationData() + self.handlePRServ() + + # This is called for all async commands when self.state != running + def updateCache(self): + if self.state == state.running: + return + + if self.state in (state.shutdown, state.forceshutdown, state.error): + if hasattr(self.parser, 'shutdown'): + self.parser.shutdown(clean=False, force = True) + raise bb.BBHandledException() + + if self.state != state.parsing: + self.updateCacheSync() + + if self.state != state.parsing and not self.parsecache_valid: + bb.parse.siggen.reset(self.data) + self.parseConfiguration () + if CookerFeatures.SEND_SANITYEVENTS in self.featureset: + for mc in self.multiconfigs: + bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc]) + + for mc in self.multiconfigs: + ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or "" + self.recipecaches[mc].ignored_dependencies = set(ignore.split()) + + for dep in self.configuration.extra_assume_provided: + self.recipecaches[mc].ignored_dependencies.add(dep) + + self.collection = CookerCollectFiles(self.bbfile_config_priorities) + (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) + + # Add inotify watches for directories searched for bb/bbappend files + for dirent in searchdirs: + self.add_filewatch([[dirent]], dirs=True) + + self.parser = CookerParser(self, filelist, masked) + self.parsecache_valid = True + + self.state = state.parsing + + if not self.parser.parse_next(): + collectlog.debug(1, "parsing complete") + if self.parser.error: + raise bb.BBHandledException() + self.show_appends_with_no_recipes() + self.handlePrefProviders() + for mc in self.multiconfigs: + self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.state = state.running + + # Send an event listing all stamps reachable after parsing + # which the metadata may use to clean up stale data + for mc in self.multiconfigs: + event = bb.event.ReachableStamps(self.recipecaches[mc].stamp) + bb.event.fire(event, self.databuilder.mcdata[mc]) + return None + + return True + + def checkPackages(self, pkgs_to_build, task=None): + + # Return a copy, don't modify the original + pkgs_to_build = pkgs_to_build[:] + + if len(pkgs_to_build) == 0: + raise NothingToBuild + + ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() + for pkg in pkgs_to_build: + if pkg in ignore: + parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) + + if 'world' in pkgs_to_build: + pkgs_to_build.remove('world') + for mc in self.multiconfigs: + bb.providers.buildWorldTargetList(self.recipecaches[mc], task) + for t in self.recipecaches[mc].world_target: + if mc: + t = "multiconfig:" + mc + ":" + t + pkgs_to_build.append(t) + + if 'universe' in pkgs_to_build: + parselog.warning("The \"universe\" target is only intended for testing and may produce errors.") + parselog.debug(1, "collating packages for \"universe\"") + pkgs_to_build.remove('universe') + for mc in self.multiconfigs: + for t in self.recipecaches[mc].universe_target: + if task: + foundtask = False + for provider_fn in self.recipecaches[mc].providers[t]: + if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']: + foundtask = True + break + if not foundtask: + bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task)) + continue + if mc: + t = "multiconfig:" + mc + ":" + t + pkgs_to_build.append(t) + + return pkgs_to_build + + def pre_serve(self): + # We now are in our own process so we can call this here. + # PRServ exits if its parent process exits + self.handlePRServ() + return + + def post_serve(self): + prserv.serv.auto_shutdown() + bb.event.fire(CookerExit(), self.data) + + + def shutdown(self, force = False): + if force: + self.state = state.forceshutdown + else: + self.state = state.shutdown + + if self.parser: + self.parser.shutdown(clean=not force, force=force) + + def finishcommand(self): + self.state = state.initial + + def reset(self): + self.initConfigurationData() + + def clientComplete(self): + """Called when the client is done using the server""" + self.finishcommand() + self.extraconfigdata = {} + self.command.reset() + self.databuilder.reset() + self.data = self.databuilder.data + + +class CookerExit(bb.event.Event): + """ + Notify clients of the Cooker shutdown + """ + + def __init__(self): + bb.event.Event.__init__(self) + + +class CookerCollectFiles(object): + def __init__(self, priorities): + self.bbappends = [] + self.bbfile_config_priorities = priorities + + def calc_bbfile_priority( self, filename, matched = None ): + for _, _, regex, pri in self.bbfile_config_priorities: + if regex.match(filename): + if matched != None: + if not regex in matched: + matched.add(regex) + return pri + return 0 + + def get_bbfiles(self): + """Get list of default .bb files by reading out the current directory""" + path = os.getcwd() + contents = os.listdir(path) + bbfiles = [] + for f in contents: + if f.endswith(".bb"): + bbfiles.append(os.path.abspath(os.path.join(path, f))) + return bbfiles + + def find_bbfiles(self, path): + """Find all the .bb and .bbappend files in a directory""" + found = [] + for dir, dirs, files in os.walk(path): + for ignored in ('SCCS', 'CVS', '.svn'): + if ignored in dirs: + dirs.remove(ignored) + found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))] + + return found + + def collect_bbfiles(self, config, eventdata): + """Collect all available .bb build files""" + masked = 0 + + collectlog.debug(1, "collecting .bb files") + + files = (config.getVar( "BBFILES") or "").split() + config.setVar("BBFILES", " ".join(files)) + + # Sort files by priority + files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) + + if not len(files): + files = self.get_bbfiles() + + if not len(files): + collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") + bb.event.fire(CookerExit(), eventdata) + + # We need to track where we look so that we can add inotify watches. There + # is no nice way to do this, this is horrid. We intercept the os.listdir() + # (or os.scandir() for python 3.6+) calls while we run glob(). + origlistdir = os.listdir + if hasattr(os, 'scandir'): + origscandir = os.scandir + searchdirs = [] + + def ourlistdir(d): + searchdirs.append(d) + return origlistdir(d) + + def ourscandir(d): + searchdirs.append(d) + return origscandir(d) + + os.listdir = ourlistdir + if hasattr(os, 'scandir'): + os.scandir = ourscandir + try: + # Can't use set here as order is important + newfiles = [] + for f in files: + if os.path.isdir(f): + dirfiles = self.find_bbfiles(f) + for g in dirfiles: + if g not in newfiles: + newfiles.append(g) + else: + globbed = glob.glob(f) + if not globbed and os.path.exists(f): + globbed = [f] + # glob gives files in order on disk. Sort to be deterministic. + for g in sorted(globbed): + if g not in newfiles: + newfiles.append(g) + finally: + os.listdir = origlistdir + if hasattr(os, 'scandir'): + os.scandir = origscandir + + bbmask = config.getVar('BBMASK') + + if bbmask: + # First validate the individual regular expressions and ignore any + # that do not compile + bbmasks = [] + for mask in bbmask.split(): + # When constructing an older style single regex, it's possible for BBMASK + # to end up beginning with '|', which matches and masks _everything_. + if mask.startswith("|"): + collectlog.warn("BBMASK contains regular expression beginning with '|', fixing: %s" % mask) + mask = mask[1:] + try: + re.compile(mask) + bbmasks.append(mask) + except sre_constants.error: + collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) + + # Then validate the combined regular expressions. This should never + # fail, but better safe than sorry... + bbmask = "|".join(bbmasks) + try: + bbmask_compiled = re.compile(bbmask) + except sre_constants.error: + collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) + bbmask = None + + bbfiles = [] + bbappend = [] + for f in newfiles: + if bbmask and bbmask_compiled.search(f): + collectlog.debug(1, "skipping masked file %s", f) + masked += 1 + continue + if f.endswith('.bb'): + bbfiles.append(f) + elif f.endswith('.bbappend'): + bbappend.append(f) + else: + collectlog.debug(1, "skipping %s: unknown file extension", f) + + # Build a list of .bbappend files for each .bb file + for f in bbappend: + base = os.path.basename(f).replace('.bbappend', '.bb') + self.bbappends.append((base, f)) + + # Find overlayed recipes + # bbfiles will be in priority order which makes this easy + bbfile_seen = dict() + self.overlayed = defaultdict(list) + for f in reversed(bbfiles): + base = os.path.basename(f) + if base not in bbfile_seen: + bbfile_seen[base] = f + else: + topfile = bbfile_seen[base] + self.overlayed[topfile].append(f) + + return (bbfiles, masked, searchdirs) + + def get_file_appends(self, fn): + """ + Returns a list of .bbappend files to apply to fn + """ + filelist = [] + f = os.path.basename(fn) + for b in self.bbappends: + (bbappend, filename) = b + if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): + filelist.append(filename) + return filelist + + def collection_priorities(self, pkgfns, d): + + priorities = {} + + # Calculate priorities for each file + matched = set() + for p in pkgfns: + realfn, cls, mc = bb.cache.virtualfn2realfn(p) + priorities[p] = self.calc_bbfile_priority(realfn, matched) + + unmatched = set() + for _, _, regex, pri in self.bbfile_config_priorities: + if not regex in matched: + unmatched.add(regex) + + # Don't show the warning if the BBFILE_PATTERN did match .bbappend files + def find_bbappend_match(regex): + for b in self.bbappends: + (bbfile, append) = b + if regex.match(append): + # If the bbappend is matched by already "matched set", return False + for matched_regex in matched: + if matched_regex.match(append): + return False + return True + return False + + for unmatch in unmatched.copy(): + if find_bbappend_match(unmatch): + unmatched.remove(unmatch) + + for collection, pattern, regex, _ in self.bbfile_config_priorities: + if regex in unmatched: + if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': + collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + + return priorities + +class ParsingFailure(Exception): + def __init__(self, realexception, recipe): + self.realexception = realexception + self.recipe = recipe + Exception.__init__(self, realexception, recipe) + +class Feeder(multiprocessing.Process): + def __init__(self, jobs, to_parsers, quit): + self.quit = quit + self.jobs = jobs + self.to_parsers = to_parsers + multiprocessing.Process.__init__(self) + + def run(self): + while True: + try: + quit = self.quit.get_nowait() + except queue.Empty: + pass + else: + if quit == 'cancel': + self.to_parsers.cancel_join_thread() + break + + try: + job = self.jobs.pop() + except IndexError: + break + + try: + self.to_parsers.put(job, timeout=0.5) + except queue.Full: + self.jobs.insert(0, job) + continue + +class Parser(multiprocessing.Process): + def __init__(self, jobs, results, quit, init, profile): + self.jobs = jobs + self.results = results + self.quit = quit + self.init = init + multiprocessing.Process.__init__(self) + self.context = bb.utils.get_context().copy() + self.handlers = bb.event.get_class_handlers().copy() + self.profile = profile + + def run(self): + + if not self.profile: + self.realrun() + return + + try: + import cProfile as profile + except: + import profile + prof = profile.Profile() + try: + profile.Profile.runcall(prof, self.realrun) + finally: + logfile = "profile-parse-%s.log" % multiprocessing.current_process().name + prof.dump_stats(logfile) + + def realrun(self): + if self.init: + self.init() + + pending = [] + while True: + try: + self.quit.get_nowait() + except queue.Empty: + pass + else: + self.results.cancel_join_thread() + break + + if pending: + result = pending.pop() + else: + try: + job = self.jobs.get(timeout=0.25) + except queue.Empty: + continue + + if job is None: + break + result = self.parse(*job) + + try: + self.results.put(result, timeout=0.25) + except queue.Full: + pending.append(result) + + def parse(self, filename, appends): + try: + # Record the filename we're parsing into any events generated + def parse_filter(self, record): + record.taskpid = bb.event.worker_pid + record.fn = filename + return True + + # Reset our environment and handlers to the original settings + bb.utils.set_context(self.context.copy()) + bb.event.set_class_handlers(self.handlers.copy()) + bb.event.LogHandler.filter = parse_filter + + return True, self.bb_cache.parse(filename, appends) + except Exception as exc: + tb = sys.exc_info()[2] + exc.recipe = filename + exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) + return True, exc + # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown + # and for example a worker thread doesn't just exit on its own in response to + # a SystemExit event for example. + except BaseException as exc: + return True, ParsingFailure(exc, filename) + +class CookerParser(object): + def __init__(self, cooker, filelist, masked): + self.filelist = filelist + self.cooker = cooker + self.cfgdata = cooker.data + self.cfghash = cooker.data_hash + self.cfgbuilder = cooker.databuilder + + # Accounting statistics + self.parsed = 0 + self.cached = 0 + self.error = 0 + self.masked = masked + + self.skipped = 0 + self.virtuals = 0 + self.total = len(filelist) + + self.current = 0 + self.process_names = [] + + self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) + self.fromcache = [] + self.willparse = [] + for filename in self.filelist: + appends = self.cooker.collection.get_file_appends(filename) + if not self.bb_cache.cacheValid(filename, appends): + self.willparse.append((filename, appends)) + else: + self.fromcache.append((filename, appends)) + self.toparse = self.total - len(self.fromcache) + self.progress_chunk = int(max(self.toparse / 100, 1)) + + self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or + multiprocessing.cpu_count()), len(self.willparse)) + + self.start() + self.haveshutdown = False + + def start(self): + self.results = self.load_cached() + self.processes = [] + if self.toparse: + bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) + def init(): + Parser.bb_cache = self.bb_cache + bb.utils.set_process_name(multiprocessing.current_process().name) + multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) + + self.feeder_quit = multiprocessing.Queue(maxsize=1) + self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) + self.jobs = multiprocessing.Queue(maxsize=self.num_processes) + self.result_queue = multiprocessing.Queue() + self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit) + self.feeder.start() + for i in range(0, self.num_processes): + parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser.start() + self.process_names.append(parser.name) + self.processes.append(parser) + + self.results = itertools.chain(self.results, self.parse_generator()) + + def shutdown(self, clean=True, force=False): + if not self.toparse: + return + if self.haveshutdown: + return + self.haveshutdown = True + + if clean: + event = bb.event.ParseCompleted(self.cached, self.parsed, + self.skipped, self.masked, + self.virtuals, self.error, + self.total) + + bb.event.fire(event, self.cfgdata) + self.feeder_quit.put(None) + for process in self.processes: + self.parser_quit.put(None) + else: + self.feeder_quit.put('cancel') + + self.parser_quit.cancel_join_thread() + for process in self.processes: + self.parser_quit.put(None) + + self.jobs.cancel_join_thread() + + for process in self.processes: + if force: + process.join(.1) + process.terminate() + else: + process.join() + self.feeder.join() + + sync = threading.Thread(target=self.bb_cache.sync) + sync.start() + multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) + bb.codeparser.parser_cache_savemerge() + bb.fetch.fetcher_parse_done() + if self.cooker.configuration.profile: + profiles = [] + for i in self.process_names: + logfile = "profile-parse-%s.log" % i + if os.path.exists(logfile): + profiles.append(logfile) + + pout = "profile-parse.log.processed" + bb.utils.process_profilelog(profiles, pout = pout) + print("Processed parsing statistics saved to %s" % (pout)) + + def load_cached(self): + for filename, appends in self.fromcache: + cached, infos = self.bb_cache.load(filename, appends) + yield not cached, infos + + def parse_generator(self): + while True: + if self.parsed >= self.toparse: + break + + try: + result = self.result_queue.get(timeout=0.25) + except queue.Empty: + pass + else: + value = result[1] + if isinstance(value, BaseException): + raise value + else: + yield result + + def parse_next(self): + result = [] + parsed = None + try: + parsed, result = next(self.results) + except StopIteration: + self.shutdown() + return False + except bb.BBHandledException as exc: + self.error += 1 + logger.error('Failed to parse recipe: %s' % exc.recipe) + self.shutdown(clean=False) + return False + except ParsingFailure as exc: + self.error += 1 + logger.error('Unable to parse %s: %s' % + (exc.recipe, bb.exceptions.to_string(exc.realexception))) + self.shutdown(clean=False) + return False + except bb.parse.ParseError as exc: + self.error += 1 + logger.error(str(exc)) + self.shutdown(clean=False) + return False + except bb.data_smart.ExpansionError as exc: + self.error += 1 + bbdir = os.path.dirname(__file__) + os.sep + etype, value, _ = sys.exc_info() + tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback)) + logger.error('ExpansionError during parsing %s', value.recipe, + exc_info=(etype, value, tb)) + self.shutdown(clean=False) + return False + except Exception as exc: + self.error += 1 + etype, value, tb = sys.exc_info() + if hasattr(value, "recipe"): + logger.error('Unable to parse %s' % value.recipe, + exc_info=(etype, value, exc.traceback)) + else: + # Most likely, an exception occurred during raising an exception + import traceback + logger.error('Exception during parse: %s' % traceback.format_exc()) + self.shutdown(clean=False) + return False + + self.current += 1 + self.virtuals += len(result) + if parsed: + self.parsed += 1 + if self.parsed % self.progress_chunk == 0: + bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse), + self.cfgdata) + else: + self.cached += 1 + + for virtualfn, info_array in result: + if info_array[0].skipped: + self.skipped += 1 + self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) + (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn) + self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc], + parsed=parsed, watcher = self.cooker.add_filewatch) + return True + + def reparse(self, filename): + infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) + for vfn, info_array in infos: + (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) + self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) -- cgit v1.2.3