poky: subtree update:b66b9f7548..26ae42ded7

Adrian Bunk (1):
      dpkg: Remove workaound patch for host tar < 1.27

Alexander Kanavin (39):
      linux-yocto: exclude from version checks/automated version updates
      pciutils: upgrade 3.6.4 -> 3.7.0
      createrepo-c: upgrade 0.15.10 -> 0.15.11
      librepo: upgrade 1.11.3 -> 1.12.0
      pkgconf: upgrade 1.6.3 -> 1.7.3
      python3-numpy: upgrade 1.18.4 -> 1.18.5
      python3-git: upgrade 3.1.2 -> 3.1.3
      strace: upgrade 5.6 -> 5.7
      acpica: upgrade 20200430 -> 20200528
      man-db: upgrade 2.9.1 -> 2.9.2
      msmtp: upgrade 1.8.10 -> 1.8.11
      epiphany: upgrade 3.36.1 -> 3.36.2
      cogl-1.0: upgrade 1.22.6 -> 1.22.8
      libdrm: upgrade 2.4.101 -> 2.4.102
      vulkan-demos: upgrade to latest revision
      xkeyboard-config: upgrade 2.29 -> 2.30
      linux-firmware: upgrade 20200421 -> 20200519
      babeltrace2: upgrade 2.0.2 -> 2.0.3
      lttng-tools: upgrade 2.12.0 -> 2.12.1
      ffmpeg: upgrade 4.2.2 -> 4.2.3
      wpebackend-fdo: upgrade 1.6.0 -> 1.6.1
      gnutls: upgrade 3.6.13 -> 3.6.14
      libcap: upgrade 2.34 -> 2.36
      bison: upgrade 3.6.2 -> 3.6.3
      asciidoc: 8.6.10 -> 9.0.0
      debianutils: 4.9.1 -> 4.11
      git: upgrade 2.26.2 -> 2.27.0
      go: 1.14.3 -> 1.14.4
      iproute2: upgrade 5.6.0 -> 5.7.0
      libksba: 1.3.5 -> 1.4.0
      lttng-modules: update to 2.12.1
      mpg123: update to 1.26.1
      ovmf: update to 202005
      shared-mime-info: upgrade 1.15 -> 2.0
      subversion: upgrade 1.13.0 -> 1.14.0
      xinetd: 2.3.15 -> 2.3.15.4
      init-system-helpers: use https for fetching
      ca-certificates: correct upstream version check
      build-sysroots: add sysroot paths with native binaries to PATH

Andreas Müller (4):
      vte: tiny cleanup / renumber patch
      vte: upgrade 0.60.2 -> 0.60.3
      harfbuzz: upgrade 2.6.4 -> 2.6.7
      sqlite3: upgrade 3.32.1 -> 3.32.2

Changqing Li (1):
      cups.inc: remove template service from SYSTEMD_SERVICE

Chen Qi (2):
      db: do not install db_verify if 'verify' is not enabled
      vim: restore the 'chmod -x' workaround in do_install

Hongxu Jia (1):
      glib-networking/btrfs-tools/dosfstools/parted/bmap-tools/libsoup-2.4: add nativesdk support

Jacob Kroon (4):
      features_check: Factorize code for checking features
      meta: Don't inherit 'features_check' in recipes that don't utilize it
      features_check: Warn if not used
      insane: Check for feature check variables not being used

Joe Slater (2):
      qemu: force build type to production
      vim: _FORTIFY_SOURCE=2 be gone

Joshua Watt (12):
      bitbake: bitbake: cooker: Split file collections per multiconfig
      bitbake: bitbake: cache: Use multiconfig aware caches
      bitbake: bitbake: lib: Add support for Logging Adapters
      bitbake: bitbake: lib: Add PrefixLoggerAdapter helper
      bitbake: bitbake: cache: Improve logging
      bitbake: bitbake: cache: Cache size optimization
      bitbake: bitbake: tests: Add tests for BBMASK in multiconfig
      bitbake: bitbake: command: Move split_mc_pn to runqueue
      bitbake: bitbake: cache: Fix error when cache is rebuilt
      wic: Fix --extra-space argument handling
      bitbake: bitbake: siggen: Pass all data caches to hash functions
      bitbake: bitbake: tests: Add mcdepends test

Kai Kang (4):
      mdadm: remove service template from SYSTEMD_SERVICE
      wpa-supplicant: remove service templates from SYSTEMD_SERVICE
      encodings: clear postinst script
      avahi-dnsconfd: rdepends on avahi-daemon

Khem Raj (2):
      libunwind: Fix build on aarch64/musl
      stress-ng: Fix build on musl

Lee Chee Yang (1):
      qemu: fix CVE-2020-13361

Ming Liu (1):
      u-boot: support merging .cfg files for UBOOT_CONFIG

Mingli Yu (2):
      python3-magic: add the missing rdepends
      python3-setuptools: add missing rdepends for python3-pkg-resources

Paul Barker (5):
      selftest: git-submodule-test: New recipe for testing a gitsm SRC_URI
      archiver: Capture git submodules in mirror archiver
      selftest-ed: Support native builds
      selftest-nopackages: New recipe in meta-selftest
      archiver: Speed up tests

Pierre-Jean Texier (2):
      libarchive: upgrade 3.4.2 -> 3.4.3
      iptables: upgrade 1.8.4 -> 1.8.5

Rasmus Villemoes (1):
      glibc: move ld.so.conf back to main package

Richard Purdie (1):
      Revert "bitbake.conf: Remove unused DEPLOY_DIR_TOOLS variable"

Stefan Agner (1):
      initramfs-framework: check successful mount using mountpoint

Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
Change-Id: I047d0fa664dcc2864fd7c1a09d124e3d8c197e9f

1 files changed, 93 insertions, 66 deletions

diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py
index e527e23114..effd02442c 100644
--- a/poky/bitbake/lib/bb/cooker.py
+++ b/poky/bitbake/lib/bb/cooker.py
@@ -525,7 +525,7 @@ class BBCooker:
             self.parseConfiguration()
 
             fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-            fn = self.matchFile(fn)
+            fn = self.matchFile(fn, mc)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
             mc = mc_base(pkgs_to_build[0])
@@ -541,8 +541,8 @@ class BBCooker:
 
         if fn:
             try:
-                bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
-                envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
+                bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+                envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
@@ -929,26 +929,33 @@ class BBCooker:
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
     def show_appends_with_no_recipes(self):
+        appends_without_recipes = {}
         # Determine which bbappends haven't been applied
-
-        # First get list of recipes, including skipped
-        recipefns = list(self.recipecaches[''].pkg_fn.keys())
-        recipefns.extend(self.skiplist.keys())
-
-        # Work out list of bbappends that have been applied
-        applied_appends = []
-        for fn in recipefns:
-            applied_appends.extend(self.collection.get_file_appends(fn))
-
-        appends_without_recipes = []
-        for _, appendfn in self.collection.bbappends:
-            if not appendfn in applied_appends:
-                appends_without_recipes.append(appendfn)
-
-        if appends_without_recipes:
-            msg = 'No recipes available for:\n  %s' % '\n  '.join(appends_without_recipes)
-            warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                 False) or "no"
+        for mc in self.multiconfigs:
+            # First get list of recipes, including skipped
+            recipefns = list(self.recipecaches[mc].pkg_fn.keys())
+            recipefns.extend(self.skiplist.keys())
+
+            # Work out list of bbappends that have been applied
+            applied_appends = []
+            for fn in recipefns:
+                applied_appends.extend(self.collections[mc].get_file_appends(fn))
+
+            appends_without_recipes[mc] = []
+            for _, appendfn in self.collections[mc].bbappends:
+                if not appendfn in applied_appends:
+                    appends_without_recipes[mc].append(appendfn)
+
+        msgs = []
+        for mc in sorted(appends_without_recipes.keys()):
+            if appends_without_recipes[mc]:
+                msgs.append('No recipes in %s available for:\n  %s' % (mc if mc else 'default',
+                                                                        '\n  '.join(appends_without_recipes[mc])))
+
+        if msgs:
+            msg = "\n".join(msgs)
+            warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+                False) or "no"
             if warn_only.lower() in ("1", "yes", "true"):
                 bb.warn(msg)
             else:
@@ -1249,15 +1256,15 @@ class BBCooker:
         if siggen_cache:
             bb.parse.siggen.checksum_cache.mtime_cache.clear()
 
-    def matchFiles(self, bf):
+    def matchFiles(self, bf, mc=''):
         """
         Find the .bb files which match the expression in 'buildfile'.
         """
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
+        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1270,12 +1277,12 @@ class BBCooker:
                     matches.append(f)
             return matches
 
-    def matchFile(self, buildfile):
+    def matchFile(self, buildfile, mc=''):
         """
         Find the .bb file which matches the expression in 'buildfile'.
         Raise an error if multiple files
         """
-        matches = self.matchFiles(buildfile)
+        matches = self.matchFiles(buildfile, mc)
         if len(matches) != 1:
             if matches:
                 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
@@ -1316,14 +1323,14 @@ class BBCooker:
             task = "do_%s" % task
 
         fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-        fn = self.matchFile(fn)
+        fn = self.matchFile(fn, mc)
 
         self.buildSetVars()
         self.reset_mtime_caches()
 
-        bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
 
-        infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
+        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
         infos = dict(infos)
 
         fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1552,14 +1559,24 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+            self.collections = {}
+
+            mcfilelist = {}
+            total_masked = 0
+            searchdirs = set()
+            for mc in self.multiconfigs:
+                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+                (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+
+                mcfilelist[mc] = filelist
+                total_masked += masked
+                searchdirs |= set(search)
 
             # Add inotify watches for directories searched for bb/bbappend files
             for dirent in searchdirs:
                 self.add_filewatch([[dirent]], dirs=True)
 
-            self.parser = CookerParser(self, filelist, masked)
+            self.parser = CookerParser(self, mcfilelist, total_masked)
             self.parsecache_valid = True
 
         self.state = state.parsing
@@ -1571,7 +1588,7 @@ class BBCooker:
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             for mc in self.multiconfigs:
-                self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+                self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
             self.state = state.running
 
             # Send an event listing all stamps reachable after parsing
@@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event):
 
 
 class CookerCollectFiles(object):
-    def __init__(self, priorities):
+    def __init__(self, priorities, mc=''):
+        self.mc = mc
         self.bbappends = []
         # Priorities is a list of tupples, with the second element as the pattern.
         # We need to sort the list with the longest pattern first, and so on to
@@ -1846,7 +1864,7 @@ class CookerCollectFiles(object):
             (bbappend, filename) = b
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
                 filelist.append(filename)
-        return filelist
+        return tuple(filelist)
 
     def collection_priorities(self, pkgfns, d):
 
@@ -1882,7 +1900,8 @@ class CookerCollectFiles(object):
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
             if regex in unmatched:
                 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
-                    collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+                    collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
+                                                                                             collection, pattern))
 
         return priorities
 
@@ -1949,7 +1968,7 @@ class Parser(multiprocessing.Process):
             except queue.Full:
                 pending.append(result)
 
-    def parse(self, filename, appends):
+    def parse(self, mc, cache, filename, appends):
         try:
             origfilter = bb.event.LogHandler.filter
             # Record the filename we're parsing into any events generated
@@ -1963,7 +1982,7 @@ class Parser(multiprocessing.Process):
             bb.event.set_class_handlers(self.handlers.copy())
             bb.event.LogHandler.filter = parse_filter
 
-            return True, self.bb_cache.parse(filename, appends)
+            return True, mc, cache.parse(filename, appends)
         except Exception as exc:
             tb = sys.exc_info()[2]
             exc.recipe = filename
@@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process):
             bb.event.LogHandler.filter = origfilter
 
 class CookerParser(object):
-    def __init__(self, cooker, filelist, masked):
-        self.filelist = filelist
+    def __init__(self, cooker, mcfilelist, masked):
+        self.mcfilelist = mcfilelist
         self.cooker = cooker
         self.cfgdata = cooker.data
         self.cfghash = cooker.data_hash
@@ -1993,25 +2012,27 @@ class CookerParser(object):
 
         self.skipped = 0
         self.virtuals = 0
-        self.total = len(filelist)
 
         self.current = 0
         self.process_names = []
 
-        self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
-        self.fromcache = []
-        self.willparse = []
-        for filename in self.filelist:
-            appends = self.cooker.collection.get_file_appends(filename)
-            if not self.bb_cache.cacheValid(filename, appends):
-                self.willparse.append((filename, appends))
-            else:
-                self.fromcache.append((filename, appends))
-        self.toparse = self.total - len(self.fromcache)
+        self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
+        self.fromcache = set()
+        self.willparse = set()
+        for mc in self.cooker.multiconfigs:
+            for filename in self.mcfilelist[mc]:
+                appends = self.cooker.collections[mc].get_file_appends(filename)
+                if not self.bb_caches[mc].cacheValid(filename, appends):
+                    self.willparse.add((mc, self.bb_caches[mc], filename, appends))
+                else:
+                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
+
+        self.total = len(self.fromcache) + len(self.willparse)
+        self.toparse = len(self.willparse)
         self.progress_chunk = int(max(self.toparse / 100, 1))
 
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
-                                 multiprocessing.cpu_count()), len(self.willparse))
+                                 multiprocessing.cpu_count()), self.toparse)
 
         self.start()
         self.haveshutdown = False
@@ -2022,7 +2043,6 @@ class CookerParser(object):
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
             def init():
-                Parser.bb_cache = self.bb_cache
                 bb.utils.set_process_name(multiprocessing.current_process().name)
                 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
                 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2032,7 +2052,7 @@ class CookerParser(object):
 
             def chunkify(lst,n):
                 return [lst[i::n] for i in range(n)]
-            self.jobs = chunkify(self.willparse, self.num_processes)
+            self.jobs = chunkify(list(self.willparse), self.num_processes)
 
             for i in range(0, self.num_processes):
                 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
@@ -2078,7 +2098,11 @@ class CookerParser(object):
             else:
                 process.join()
 
-        sync = threading.Thread(target=self.bb_cache.sync)
+        def sync_caches():
+            for c in self.bb_caches.values():
+                c.sync()
+
+        sync = threading.Thread(target=sync_caches)
         sync.start()
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
         bb.codeparser.parser_cache_savemerge()
@@ -2095,9 +2119,9 @@ class CookerParser(object):
             print("Processed parsing statistics saved to %s" % (pout))
 
     def load_cached(self):
-        for filename, appends in self.fromcache:
-            cached, infos = self.bb_cache.load(filename, appends)
-            yield not cached, infos
+        for mc, cache, filename, appends in self.fromcache:
+            cached, infos = cache.load(filename, appends)
+            yield not cached, mc, infos
 
     def parse_generator(self):
         while True:
@@ -2119,7 +2143,7 @@ class CookerParser(object):
         result = []
         parsed = None
         try:
-            parsed, result = next(self.results)
+            parsed, mc, result = next(self.results)
         except StopIteration:
             self.shutdown()
             return False
@@ -2175,13 +2199,16 @@ class CookerParser(object):
             if info_array[0].skipped:
                 self.skipped += 1
                 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
-            (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
-            self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
+            self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
                                         parsed=parsed, watcher = self.cooker.add_filewatch)
         return True
 
     def reparse(self, filename):
-        infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
-        for vfn, info_array in infos:
-            (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
-            self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
+        to_reparse = set()
+        for mc in self.cooker.multiconfigs:
+            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+
+        for mc, filename, appends in to_reparse:
+            infos = self.bb_caches[mc].parse(filename, appends)
+            for vfn, info_array in infos:
+                self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)