From b7d2861976669d4f6decc55762ba83fe0371d6d5 Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Fri, 24 Jul 2020 16:15:54 -0500 Subject: poky: subtree update:968fcf4989..23deb29c1b Arthur She (1): igt-gpu-tools: Add PACKAGECONFIG for Chamelium support Bruce Ashfield (4): linux-yocto/5.4: update to v5.4.51 linux-yocto-rt/5.4: fix mmdrop stress test issues kernel-yocto: account for extracted defconfig in elements check kernel-devsrc: fix on-target module build for v5.8+ Changqing Li (2): dpkg: change SRC_URI to take dpkg from git gtk-immodules-cache.bbclass: fix post install scriptlet error Charlie Davies (1): u-boot: fix condition to allow use of *.cfg Chen Qi (1): rpm: fix nativesdk's default var location Christian Eggers (2): avahi: Fix typo in recipe util-linux: Set license for library sub packages Daniel Ammann (1): image.bbclass: improve wording when image size exceeds the specified limit Dmitry Baryshkov (1): gcc-10.1: add fix for PR 96130 Douglas (2): nativesdk: clear MACHINE_FEATURES nativesdk: Set the CXXFLAGS to the BUILDSDK_CXXFLAGS He Zhe (1): cryptodev-module: Backport a patch to fix build failure with kernel v5.8 Hongxu Jia (1): e2fsprogs: fix up check for hardlinks always false if inode > 0xFFFFFFFF Jens Rehsack (3): subversion: extend for nativesdk serf: extend for nativesdk kmod: add packageconfig for xz and ssl Joshua Watt (8): virtual/libgbm is the provider of gbm.pc diffoscope: upgrade 150 -> 151 python3-pycryptodomex: upgrade 3.9.7 -> 3.9.8 python3-pycryptodome: upgrade 3.9.7 -> 3.9.8 classes/reproducible: Move to library code lib/oe/reproducible: Fix error when no git HEAD classes/cmake: Fix host detection classes/package: Use HOST_OS for runtime dependencies Kamil Dziezyk (1): qemu: fix for virtfs configuration error in qemu 5.0.0 Kevin Hao (3): wic/filemap: Drop the unused block_is_unmapped() wic/filemap: Drop the unused get_unmapped_ranges() wic/filemap: Fall back to standard copy when no way to get the block map Khem Raj (4): go: Disbale CGO for riscv64 go-dep: Fix build on riscv64 musl: Update to latest tip site: Make sys_siglist default to no Konrad Weihmann (2): bitbake: pyshyacc: allow double COMMA statements ptest: append to FILES Kurt Kiefer (1): linux-firmware: add ibt-20 package Lee Chee Yang (1): bison: fix Argument list too long error Mingli Yu (1): python3: define a profile directory path Naveen Saini (3): libva: upgrade 2.7.1 -> 2.8.0 libva-initial: upgrade 2.7.1 -> 2.8.0 libva-utils: upgrade 2.7.1 -> 2.8.0 Oleksandr (1): expat: Added ptest Pierre-Jean Texier (1): u-boot: upgrade 2020.04 -> 2020.07 Rasmus Villemoes (1): cml1: Move find_cfgs() helper to cml1.bbclass Ricardo Salveti (1): sudo: set with-rundir to /run/sudo Richard Purdie (34): bitbake: fetch2: Change git fetcher not to destroy old references oeqa/selftest/sstatetests: Avoid polluting DL_DIR bitbake: server/process: Fix a rare lockfile race qemurunner: Ensure pid location is deterministic qemurunner: Add extra debug info when qemu fails to start bitbake: server/process: Ensure UI-less servers don't sit in infinite loops oeqa/utils/qemurunner: Fix missing pid file tracebacks mpfr: upgrade 4.0.2 -> 4.1.0 libuv: upgrade 1.38.0 -> 1.38.1 btrfs-tools: upgrade 5.6.1 -> 5.7 init-system-helpers: upgrade 1.57 -> 1.58 createrepo-c: upgrade 0.15.11 -> 0.16.0 mtd-utils: upgrade 2.1.1 -> 2.1.2 dpkg: upgrade 1.20.0 -> 1.20.5 python3-cython: upgrade 0.29.20 -> 0.29.21 python3-git: upgrade 3.1.3 -> 3.1.7 asciidoc: upgrade 9.0.0 -> 9.0.1 libnsl2: upgrade 1.2.0 -> 1.3.0 rpcsvc-proto: upgrade 1.4.1 -> 1.4.2 stress-ng: upgrade 0.11.14 -> 0.11.15 epiphany: upgrade 3.36.2 -> 3.36.3 ffmpeg: upgrade 4.3 -> 4.3.1 gnupg: upgrade 2.2.20 -> 2.2.21 mpg123: upgrade 1.26.1 -> 1.26.2 libevent: upgrade 2.1.11 -> 2.1.12 webkitgtk: upgrade 2.28.2 -> 2.28.3 libgcrypt: upgrade 1.8.5 -> 1.8.6 bitbake: server/process: Fix note reference -> info bitbake: cooker: Fix unmatched files handling leading to misleading warnings bitbake: build: Allow deltask to take multiple tasknames pseudo: Update to add OFC fcntl lock updates oeqa/qemurunner: Add priority/nice information for running processes bitbake: cooker: Improve multiconfig configuration error reporting bitbake: cooker: Handle multiconfig name mappings correctly Robert Yang (1): openssl: openssl-bin requires openssl-conf to run Ross Burton (9): insane: consolidate skipping of temporary do_package files perf: add PACKAGECONFIG for CoreSight support autotools: don't special-case help2man-native for dependencies flex: fix build with autoconf 2.70 nasm: fix build with autoconf 2.70 init-ifupdown: always make machine-specific insane: improve arch test messages startup-notification: add time_t type mismatch patch from upstream gcc: mitigate the Straight-line Speculation attack Sakib Sajal (5): qemu: fix CVE-2020-13362 qemu: fix CVE-2020-13659 qemu: fix CVE-2020-13800 qemu: fix CVE-2020-13791 busybox: make hwclock compatible with glibc 2.31 Tanu Kaskinen (2): alsa-lib: upgrade 1.2.3.1 -> 1.2.3.2 pulseaudio: improve the Thumb frame pointer fix Taras Kondratiuk (1): nfs-utils: use rpcgen tool from HOSTTOOLS_DIR Tim Orling (2): lib/oe/recipeutils.py: add AUTHOR; BBCLASSEXTEND scripts/lib/recipetool/create.py: fix regex strings Wang Mingyu (4): dbus: upgrade 1.12.18 -> 1.12.20 fribidi: upgrade 1.0.9 -> 1.0.10 glib-2.0: upgrade 2.64.3 -> 2.64.4 libvorbis: upgrade 1.3.6 -> 1.3.7 Yi Zhao (1): bind: upgrade 9.11.19 -> 9.11.21 Yongxin Liu (2): linux-firmware: fix the wrong file path for ibt-misc linux-firmware: move ibt-misc to the end of ibt packages akuster (3): cve-check.bbclass: always save cve report ref-system-requirements: update supported hosts lists glibc: whitelist CVE-2010-10029 zhengruoqin (1): gnutls: Fix krb5 code license to GPLv2.1+ to match the LICENSE file. Signed-off-by: Andrew Geissler Change-Id: Iae9b13b7fe09bb3c0ab953a063793c95e8b17468 --- poky/bitbake/lib/bb/command.py | 2 +- poky/bitbake/lib/bb/cooker.py | 89 ++++++++++++++++--------- poky/bitbake/lib/bb/fetch2/git.py | 2 +- poky/bitbake/lib/bb/parse/ast.py | 10 +-- poky/bitbake/lib/bb/parse/parse_py/BBHandler.py | 5 +- poky/bitbake/lib/bb/pysh/pyshyacc.py | 1 + poky/bitbake/lib/bb/server/process.py | 23 +++++-- poky/bitbake/lib/bb/tests/cooker.py | 2 +- poky/bitbake/lib/bb/tests/parse.py | 6 +- 9 files changed, 89 insertions(+), 51 deletions(-) (limited to 'poky/bitbake') diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index 3902ccca71..805ed9216c 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -396,7 +396,7 @@ class CommandsSync: def sortkey(x): vfn, _ = x realfn, _, mc = bb.cache.virtualfn2realfn(vfn) - return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn) + return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn) skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) return list(skipdict.items()) diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index effd02442c..f6abc63487 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -626,6 +626,7 @@ class BBCooker: current = 0 runlist = [] for k in fulltargetlist: + origk = k mc = "" if k.startswith("mc:"): mc = k.split(":")[1] @@ -635,6 +636,10 @@ class BBCooker: k2 = k.split(":do_") k = k2[0] ktask = k2[1] + + if mc not in self.multiconfigs: + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc)) + taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) current += 1 if not ktask.startswith("do_"): @@ -670,7 +675,7 @@ class BBCooker: l = k.split(':') depmc = l[2] if depmc not in self.multiconfigs: - bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc)) + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) else: logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3]) taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) @@ -1588,7 +1593,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1609,7 +1614,7 @@ class BBCooker: raise NothingToBuild ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() - for pkg in pkgs_to_build: + for pkg in pkgs_to_build.copy(): if pkg in ignore: parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) if pkg.startswith("multiconfig:"): @@ -1704,14 +1709,11 @@ class CookerCollectFiles(object): # the shortest. This allows nested layers to be properly evaluated. self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) - def calc_bbfile_priority( self, filename, matched = None ): + def calc_bbfile_priority(self, filename): for _, _, regex, pri in self.bbfile_config_priorities: if regex.match(filename): - if matched is not None: - if not regex in matched: - matched.add(regex) - return pri - return 0 + return pri, regex + return 0, None def get_bbfiles(self): """Get list of default .bb files by reading out the current directory""" @@ -1744,7 +1746,7 @@ class CookerCollectFiles(object): config.setVar("BBFILES", " ".join(files)) # Sort files by priority - files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) + files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) if not len(files): files = self.get_bbfiles() @@ -1866,39 +1868,62 @@ class CookerCollectFiles(object): filelist.append(filename) return tuple(filelist) - def collection_priorities(self, pkgfns, d): + def collection_priorities(self, pkgfns, fns, d): + # Return the priorities of the entries in pkgfns + # Also check that all the regexes in self.bbfile_config_priorities are used + # (but to do that we need to ensure skipped recipes aren't counted, nor + # collections in BBFILE_PATTERN_IGNORE_EMPTY) priorities = {} + seen = set() + matched = set() + + matched_regex = set() + unmatched_regex = set() + for _, _, regex, _ in self.bbfile_config_priorities: + unmatched_regex.add(regex) # Calculate priorities for each file - matched = set() for p in pkgfns: realfn, cls, mc = bb.cache.virtualfn2realfn(p) - priorities[p] = self.calc_bbfile_priority(realfn, matched) - - unmatched = set() - for _, _, regex, pri in self.bbfile_config_priorities: - if not regex in matched: - unmatched.add(regex) - - # Don't show the warning if the BBFILE_PATTERN did match .bbappend files - def find_bbappend_match(regex): + priorities[p], regex = self.calc_bbfile_priority(realfn) + if regex in unmatched_regex: + matched_regex.add(regex) + unmatched_regex.remove(regex) + seen.add(realfn) + if regex: + matched.add(realfn) + + if unmatched_regex: + # Account for bbappend files for b in self.bbappends: (bbfile, append) = b - if regex.match(append): - # If the bbappend is matched by already "matched set", return False - for matched_regex in matched: - if matched_regex.match(append): - return False - return True - return False + seen.add(append) + + # Account for skipped recipes + seen.update(fns) + + seen.difference_update(matched) - for unmatch in unmatched.copy(): - if find_bbappend_match(unmatch): - unmatched.remove(unmatch) + def already_matched(fn): + for regex in matched_regex: + if regex.match(fn): + return True + return False + + for unmatch in unmatched_regex.copy(): + for fn in seen: + if unmatch.match(fn): + # If the bbappend or file was already matched by another regex, skip it + # e.g. for a layer within a layer, the outer regex could match, the inner + # regex may match nothing and we should warn about that + if already_matched(fn): + continue + unmatched_regex.remove(unmatch) + break for collection, pattern, regex, _ in self.bbfile_config_priorities: - if regex in unmatched: + if regex in unmatched_regex: if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', collection, pattern)) diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index dbf8715674..644ba9238b 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -355,7 +355,7 @@ class Git(FetchMethod): runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) diff --git a/poky/bitbake/lib/bb/parse/ast.py b/poky/bitbake/lib/bb/parse/ast.py index 785aa974eb..0714296af2 100644 --- a/poky/bitbake/lib/bb/parse/ast.py +++ b/poky/bitbake/lib/bb/parse/ast.py @@ -244,12 +244,14 @@ class AddTaskNode(AstNode): bb.build.addtask(self.func, self.before, self.after, data) class DelTaskNode(AstNode): - def __init__(self, filename, lineno, func): + def __init__(self, filename, lineno, tasks): AstNode.__init__(self, filename, lineno) - self.func = func + self.tasks = tasks def eval(self, data): - bb.build.deltask(self.func, data) + tasks = data.expand(self.tasks).split() + for task in tasks: + bb.build.deltask(task, data) class BBHandlerNode(AstNode): def __init__(self, filename, lineno, fns): @@ -305,7 +307,7 @@ def handleAddTask(statements, filename, lineno, m): statements.append(AddTaskNode(filename, lineno, func, before, after)) def handleDelTask(statements, filename, lineno, m): - func = m.group("func") + func = m.group(1) if func is None: return diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index 6e216effb8..215f940b60 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -26,7 +26,7 @@ __func_start_regexp__ = re.compile(r"(((?Ppython)|(?Pfakeroot))\s*)*( __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) __addtask_regexp__ = re.compile(r"addtask\s+(?P\w+)\s*((before\s*(?P((.*(?=after))|(.*))))|(after\s*(?P((.*(?=before))|(.*)))))*") -__deltask_regexp__ = re.compile(r"deltask\s+(?P\w+)(?P.*)") +__deltask_regexp__ = re.compile(r"deltask\s+(.+)") __addhandler_regexp__ = re.compile(r"addhandler\s+(.+)" ) __def_regexp__ = re.compile(r"def\s+(\w+).*:" ) __python_func_regexp__ = re.compile(r"(\s+.*)|(^$)|(^#)" ) @@ -238,9 +238,6 @@ def feeder(lineno, s, fn, root, statements, eof=False): m = __deltask_regexp__.match(s) if m: - # Check and warn "for deltask task1 task2" - if m.group('ignores'): - logger.warning('deltask ignored: "%s"' % m.group('ignores')) ast.handleDelTask(statements, fn, lineno, m) return diff --git a/poky/bitbake/lib/bb/pysh/pyshyacc.py b/poky/bitbake/lib/bb/pysh/pyshyacc.py index de565dc9af..924860a6f3 100644 --- a/poky/bitbake/lib/bb/pysh/pyshyacc.py +++ b/poky/bitbake/lib/bb/pysh/pyshyacc.py @@ -570,6 +570,7 @@ def p_linebreak(p): def p_separator_op(p): """separator_op : COMMA + | COMMA COMMA | AMP""" p[0] = p[1] diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 83385baf60..9ec79f5b64 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -48,7 +48,7 @@ class ProcessServer(multiprocessing.Process): self.event_handle = None self.haveui = False - self.lastui = False + self.maxuiwait = 30 self.xmlrpc = False self._idlefuns = {} @@ -155,6 +155,7 @@ class ProcessServer(multiprocessing.Process): print("No timeout, exiting.") self.quit = True + self.lastui = time.time() while not self.quit: if self.sock in ready: while select.select([self.sock],[],[],0)[0]: @@ -191,11 +192,18 @@ class ProcessServer(multiprocessing.Process): except (EOFError, OSError): disconnect_client(self, fds) - if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \ + if not self.timeout == -1.0 and not self.haveui and self.timeout and \ (self.lastui + self.timeout) < time.time(): print("Server timeout, exiting.") self.quit = True + # If we don't see a UI connection within maxuiwait, its unlikely we're going to see + # one. We have had issue with processes hanging indefinitely so timing out UI-less + # servers is useful. + if not self.haveui and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): + print("No UI connection within max timeout, exiting to avoid infinite loop.") + self.quit = True + if self.command_channel in ready: try: command = self.command_channel.get() @@ -220,10 +228,13 @@ class ProcessServer(multiprocessing.Process): print("Exiting") # Remove the socket file so we don't get any more connections to avoid races - os.unlink(self.sockname) + try: + os.unlink(self.sockname) + except: + pass self.sock.close() - try: + try: self.cooker.shutdown(True) self.cooker.notifier.stop() self.cooker.confignotifier.stop() @@ -243,7 +254,7 @@ class ProcessServer(multiprocessing.Process): lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) if lock: # We hold the lock so we can remove the file (hide stale pid data) - bb.utils.remove(lockfile) + # via unlockfile. bb.utils.unlockfile(lock) return @@ -331,7 +342,7 @@ class ServerCommunicator(): def runCommand(self, command): self.connection.send(command) if not self.recv.poll(30): - logger.note("No reply from server in 30s") + logger.info("No reply from server in 30s") if not self.recv.poll(30): raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)") return self.recv.get() diff --git a/poky/bitbake/lib/bb/tests/cooker.py b/poky/bitbake/lib/bb/tests/cooker.py index 74c903f010..c82d4b7b81 100644 --- a/poky/bitbake/lib/bb/tests/cooker.py +++ b/poky/bitbake/lib/bb/tests/cooker.py @@ -60,7 +60,7 @@ class CookerTest(unittest.TestCase): log_handler = LogHandler() logger.addHandler(log_handler) collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities) - collection.collection_priorities(pkgfns, self.d) + collection.collection_priorities(pkgfns, pkgfns, self.d) logger.removeHandler(log_handler) # Should be empty (no generated messages) diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 9afd1b208e..9e21e18425 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -178,7 +178,10 @@ python () { addtask do_patch after do_foo after do_unpack before do_configure before do_compile addtask do_fetch do_patch -deltask do_fetch do_patch +MYVAR = "do_patch" +EMPTYVAR = "" +deltask do_fetch ${MYVAR} ${EMPTYVAR} +deltask ${EMPTYVAR} """ def test_parse_addtask_deltask(self): import sys @@ -189,6 +192,5 @@ deltask do_fetch do_patch self.assertTrue("addtask contained multiple 'before' keywords" in stdout) self.assertTrue("addtask contained multiple 'after' keywords" in stdout) self.assertTrue('addtask ignored: " do_patch"' in stdout) - self.assertTrue('deltask ignored: " do_patch"' in stdout) #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout) -- cgit v1.2.3