diff options
author | Andrew Geissler <geissonator@yahoo.com> | 2020-06-13 18:46:56 +0300 |
---|---|---|
committer | Andrew Geissler <geissonator@yahoo.com> | 2020-06-13 18:49:06 +0300 |
commit | 5a43b434bfdf1be10819414433f28984670a6549 (patch) | |
tree | b9c9bcff18ecf82739c258dbf73c23f23ba545e4 /poky/bitbake/lib/bb/cooker.py | |
parent | 89770b09490d616883ab728cc628a844ca0e1150 (diff) | |
download | openbmc-5a43b434bfdf1be10819414433f28984670a6549.tar.xz |
poky: subtree update:b66b9f7548..26ae42ded7
Adrian Bunk (1):
dpkg: Remove workaound patch for host tar < 1.27
Alexander Kanavin (39):
linux-yocto: exclude from version checks/automated version updates
pciutils: upgrade 3.6.4 -> 3.7.0
createrepo-c: upgrade 0.15.10 -> 0.15.11
librepo: upgrade 1.11.3 -> 1.12.0
pkgconf: upgrade 1.6.3 -> 1.7.3
python3-numpy: upgrade 1.18.4 -> 1.18.5
python3-git: upgrade 3.1.2 -> 3.1.3
strace: upgrade 5.6 -> 5.7
acpica: upgrade 20200430 -> 20200528
man-db: upgrade 2.9.1 -> 2.9.2
msmtp: upgrade 1.8.10 -> 1.8.11
epiphany: upgrade 3.36.1 -> 3.36.2
cogl-1.0: upgrade 1.22.6 -> 1.22.8
libdrm: upgrade 2.4.101 -> 2.4.102
vulkan-demos: upgrade to latest revision
xkeyboard-config: upgrade 2.29 -> 2.30
linux-firmware: upgrade 20200421 -> 20200519
babeltrace2: upgrade 2.0.2 -> 2.0.3
lttng-tools: upgrade 2.12.0 -> 2.12.1
ffmpeg: upgrade 4.2.2 -> 4.2.3
wpebackend-fdo: upgrade 1.6.0 -> 1.6.1
gnutls: upgrade 3.6.13 -> 3.6.14
libcap: upgrade 2.34 -> 2.36
bison: upgrade 3.6.2 -> 3.6.3
asciidoc: 8.6.10 -> 9.0.0
debianutils: 4.9.1 -> 4.11
git: upgrade 2.26.2 -> 2.27.0
go: 1.14.3 -> 1.14.4
iproute2: upgrade 5.6.0 -> 5.7.0
libksba: 1.3.5 -> 1.4.0
lttng-modules: update to 2.12.1
mpg123: update to 1.26.1
ovmf: update to 202005
shared-mime-info: upgrade 1.15 -> 2.0
subversion: upgrade 1.13.0 -> 1.14.0
xinetd: 2.3.15 -> 2.3.15.4
init-system-helpers: use https for fetching
ca-certificates: correct upstream version check
build-sysroots: add sysroot paths with native binaries to PATH
Andreas Müller (4):
vte: tiny cleanup / renumber patch
vte: upgrade 0.60.2 -> 0.60.3
harfbuzz: upgrade 2.6.4 -> 2.6.7
sqlite3: upgrade 3.32.1 -> 3.32.2
Changqing Li (1):
cups.inc: remove template service from SYSTEMD_SERVICE
Chen Qi (2):
db: do not install db_verify if 'verify' is not enabled
vim: restore the 'chmod -x' workaround in do_install
Hongxu Jia (1):
glib-networking/btrfs-tools/dosfstools/parted/bmap-tools/libsoup-2.4: add nativesdk support
Jacob Kroon (4):
features_check: Factorize code for checking features
meta: Don't inherit 'features_check' in recipes that don't utilize it
features_check: Warn if not used
insane: Check for feature check variables not being used
Joe Slater (2):
qemu: force build type to production
vim: _FORTIFY_SOURCE=2 be gone
Joshua Watt (12):
bitbake: bitbake: cooker: Split file collections per multiconfig
bitbake: bitbake: cache: Use multiconfig aware caches
bitbake: bitbake: lib: Add support for Logging Adapters
bitbake: bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: bitbake: cache: Improve logging
bitbake: bitbake: cache: Cache size optimization
bitbake: bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: bitbake: command: Move split_mc_pn to runqueue
bitbake: bitbake: cache: Fix error when cache is rebuilt
wic: Fix --extra-space argument handling
bitbake: bitbake: siggen: Pass all data caches to hash functions
bitbake: bitbake: tests: Add mcdepends test
Kai Kang (4):
mdadm: remove service template from SYSTEMD_SERVICE
wpa-supplicant: remove service templates from SYSTEMD_SERVICE
encodings: clear postinst script
avahi-dnsconfd: rdepends on avahi-daemon
Khem Raj (2):
libunwind: Fix build on aarch64/musl
stress-ng: Fix build on musl
Lee Chee Yang (1):
qemu: fix CVE-2020-13361
Ming Liu (1):
u-boot: support merging .cfg files for UBOOT_CONFIG
Mingli Yu (2):
python3-magic: add the missing rdepends
python3-setuptools: add missing rdepends for python3-pkg-resources
Paul Barker (5):
selftest: git-submodule-test: New recipe for testing a gitsm SRC_URI
archiver: Capture git submodules in mirror archiver
selftest-ed: Support native builds
selftest-nopackages: New recipe in meta-selftest
archiver: Speed up tests
Pierre-Jean Texier (2):
libarchive: upgrade 3.4.2 -> 3.4.3
iptables: upgrade 1.8.4 -> 1.8.5
Rasmus Villemoes (1):
glibc: move ld.so.conf back to main package
Richard Purdie (1):
Revert "bitbake.conf: Remove unused DEPLOY_DIR_TOOLS variable"
Stefan Agner (1):
initramfs-framework: check successful mount using mountpoint
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
Change-Id: I047d0fa664dcc2864fd7c1a09d124e3d8c197e9f
Diffstat (limited to 'poky/bitbake/lib/bb/cooker.py')
-rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 159 |
1 files changed, 93 insertions, 66 deletions
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index e527e2311..effd02442 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -525,7 +525,7 @@ class BBCooker: self.parseConfiguration() fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) fn = bb.cache.realfn2virtual(fn, cls, mc) elif len(pkgs_to_build) == 1: mc = mc_base(pkgs_to_build[0]) @@ -541,8 +541,8 @@ class BBCooker: if fn: try: - bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) - envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) + bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) + envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn)) except Exception as e: parselog.exception("Unable to read %s", fn) raise @@ -929,26 +929,33 @@ class BBCooker: logger.info("Task dependencies saved to 'task-depends.dot'") def show_appends_with_no_recipes(self): + appends_without_recipes = {} # Determine which bbappends haven't been applied - - # First get list of recipes, including skipped - recipefns = list(self.recipecaches[''].pkg_fn.keys()) - recipefns.extend(self.skiplist.keys()) - - # Work out list of bbappends that have been applied - applied_appends = [] - for fn in recipefns: - applied_appends.extend(self.collection.get_file_appends(fn)) - - appends_without_recipes = [] - for _, appendfn in self.collection.bbappends: - if not appendfn in applied_appends: - appends_without_recipes.append(appendfn) - - if appends_without_recipes: - msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) - warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ - False) or "no" + for mc in self.multiconfigs: + # First get list of recipes, including skipped + recipefns = list(self.recipecaches[mc].pkg_fn.keys()) + recipefns.extend(self.skiplist.keys()) + + # Work out list of bbappends that have been applied + applied_appends = [] + for fn in recipefns: + applied_appends.extend(self.collections[mc].get_file_appends(fn)) + + appends_without_recipes[mc] = [] + for _, appendfn in self.collections[mc].bbappends: + if not appendfn in applied_appends: + appends_without_recipes[mc].append(appendfn) + + msgs = [] + for mc in sorted(appends_without_recipes.keys()): + if appends_without_recipes[mc]: + msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', + '\n '.join(appends_without_recipes[mc]))) + + if msgs: + msg = "\n".join(msgs) + warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ + False) or "no" if warn_only.lower() in ("1", "yes", "true"): bb.warn(msg) else: @@ -1249,15 +1256,15 @@ class BBCooker: if siggen_cache: bb.parse.siggen.checksum_cache.mtime_cache.clear() - def matchFiles(self, bf): + def matchFiles(self, bf, mc=''): """ Find the .bb files which match the expression in 'buildfile'. """ if bf.startswith("/") or bf.startswith("../"): bf = os.path.abspath(bf) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} + filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) try: os.stat(bf) bf = os.path.abspath(bf) @@ -1270,12 +1277,12 @@ class BBCooker: matches.append(f) return matches - def matchFile(self, buildfile): + def matchFile(self, buildfile, mc=''): """ Find the .bb file which matches the expression in 'buildfile'. Raise an error if multiple files """ - matches = self.matchFiles(buildfile) + matches = self.matchFiles(buildfile, mc) if len(matches) != 1: if matches: msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) @@ -1316,14 +1323,14 @@ class BBCooker: task = "do_%s" % task fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) self.buildSetVars() self.reset_mtime_caches() - bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) + bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) - infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) + infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn)) infos = dict(infos) fn = bb.cache.realfn2virtual(fn, cls, mc) @@ -1552,14 +1559,24 @@ class BBCooker: for dep in self.configuration.extra_assume_provided: self.recipecaches[mc].ignored_dependencies.add(dep) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {} + + mcfilelist = {} + total_masked = 0 + searchdirs = set() + for mc in self.multiconfigs: + self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) + (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) + + mcfilelist[mc] = filelist + total_masked += masked + searchdirs |= set(search) # Add inotify watches for directories searched for bb/bbappend files for dirent in searchdirs: self.add_filewatch([[dirent]], dirs=True) - self.parser = CookerParser(self, filelist, masked) + self.parser = CookerParser(self, mcfilelist, total_masked) self.parsecache_valid = True self.state = state.parsing @@ -1571,7 +1588,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event): class CookerCollectFiles(object): - def __init__(self, priorities): + def __init__(self, priorities, mc=''): + self.mc = mc self.bbappends = [] # Priorities is a list of tupples, with the second element as the pattern. # We need to sort the list with the longest pattern first, and so on to @@ -1846,7 +1864,7 @@ class CookerCollectFiles(object): (bbappend, filename) = b if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): filelist.append(filename) - return filelist + return tuple(filelist) def collection_priorities(self, pkgfns, d): @@ -1882,7 +1900,8 @@ class CookerCollectFiles(object): for collection, pattern, regex, _ in self.bbfile_config_priorities: if regex in unmatched: if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': - collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', + collection, pattern)) return priorities @@ -1949,7 +1968,7 @@ class Parser(multiprocessing.Process): except queue.Full: pending.append(result) - def parse(self, filename, appends): + def parse(self, mc, cache, filename, appends): try: origfilter = bb.event.LogHandler.filter # Record the filename we're parsing into any events generated @@ -1963,7 +1982,7 @@ class Parser(multiprocessing.Process): bb.event.set_class_handlers(self.handlers.copy()) bb.event.LogHandler.filter = parse_filter - return True, self.bb_cache.parse(filename, appends) + return True, mc, cache.parse(filename, appends) except Exception as exc: tb = sys.exc_info()[2] exc.recipe = filename @@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process): bb.event.LogHandler.filter = origfilter class CookerParser(object): - def __init__(self, cooker, filelist, masked): - self.filelist = filelist + def __init__(self, cooker, mcfilelist, masked): + self.mcfilelist = mcfilelist self.cooker = cooker self.cfgdata = cooker.data self.cfghash = cooker.data_hash @@ -1993,25 +2012,27 @@ class CookerParser(object): self.skipped = 0 self.virtuals = 0 - self.total = len(filelist) self.current = 0 self.process_names = [] - self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) - self.fromcache = [] - self.willparse = [] - for filename in self.filelist: - appends = self.cooker.collection.get_file_appends(filename) - if not self.bb_cache.cacheValid(filename, appends): - self.willparse.append((filename, appends)) - else: - self.fromcache.append((filename, appends)) - self.toparse = self.total - len(self.fromcache) + self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array) + self.fromcache = set() + self.willparse = set() + for mc in self.cooker.multiconfigs: + for filename in self.mcfilelist[mc]: + appends = self.cooker.collections[mc].get_file_appends(filename) + if not self.bb_caches[mc].cacheValid(filename, appends): + self.willparse.add((mc, self.bb_caches[mc], filename, appends)) + else: + self.fromcache.add((mc, self.bb_caches[mc], filename, appends)) + + self.total = len(self.fromcache) + len(self.willparse) + self.toparse = len(self.willparse) self.progress_chunk = int(max(self.toparse / 100, 1)) self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or - multiprocessing.cpu_count()), len(self.willparse)) + multiprocessing.cpu_count()), self.toparse) self.start() self.haveshutdown = False @@ -2022,7 +2043,6 @@ class CookerParser(object): if self.toparse: bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) def init(): - Parser.bb_cache = self.bb_cache bb.utils.set_process_name(multiprocessing.current_process().name) multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) @@ -2032,7 +2052,7 @@ class CookerParser(object): def chunkify(lst,n): return [lst[i::n] for i in range(n)] - self.jobs = chunkify(self.willparse, self.num_processes) + self.jobs = chunkify(list(self.willparse), self.num_processes) for i in range(0, self.num_processes): parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) @@ -2078,7 +2098,11 @@ class CookerParser(object): else: process.join() - sync = threading.Thread(target=self.bb_cache.sync) + def sync_caches(): + for c in self.bb_caches.values(): + c.sync() + + sync = threading.Thread(target=sync_caches) sync.start() multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) bb.codeparser.parser_cache_savemerge() @@ -2095,9 +2119,9 @@ class CookerParser(object): print("Processed parsing statistics saved to %s" % (pout)) def load_cached(self): - for filename, appends in self.fromcache: - cached, infos = self.bb_cache.load(filename, appends) - yield not cached, infos + for mc, cache, filename, appends in self.fromcache: + cached, infos = cache.load(filename, appends) + yield not cached, mc, infos def parse_generator(self): while True: @@ -2119,7 +2143,7 @@ class CookerParser(object): result = [] parsed = None try: - parsed, result = next(self.results) + parsed, mc, result = next(self.results) except StopIteration: self.shutdown() return False @@ -2175,13 +2199,16 @@ class CookerParser(object): if info_array[0].skipped: self.skipped += 1 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) - (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn) - self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc], + self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], parsed=parsed, watcher = self.cooker.add_filewatch) return True def reparse(self, filename): - infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) - for vfn, info_array in infos: - (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) - self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) + to_reparse = set() + for mc in self.cooker.multiconfigs: + to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) + + for mc, filename, appends in to_reparse: + infos = self.bb_caches[mc].parse(filename, appends) + for vfn, info_array in infos: + self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) |