From 635e0e4637e40ba03f69204265427550fd404f4c Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Fri, 21 Aug 2020 15:58:33 -0500 Subject: poky: subtree update:23deb29c1b..c67f57c09e Adrian Bunk (1): librsvg: Upgrade 2.40.20 -> 2.40.21 Alejandro Hernandez (1): musl: Upgrade to latest release 1.2.1 Alex Kiernan (8): systemd: Upgrade v245.6 -> v246 systemd: Move musl patches to SRC_URI_MUSL systemd: Fix path to modules-load.d et al nfs-utils: Drop StandardError=syslog from systemd unit openssh: Drop StandardError=syslog from systemd unit volatile-binds: Drop StandardOutput=syslog from systemd unit systemd: Upgrade v246 -> v246.1 systemd: Upgrade v246.1 -> v246.2 Alexander Kanavin (16): sysvinit: update 2.96 -> 2.97 kbd: update 2.2.0 -> 2.3.0 gnu-config: update to latest revision go: update 1.14.4 -> 1.14.6 meson: update 0.54.3 -> 0.55.0 nasm: update 2.14.02 -> 2.15.03 glib-2.0: correct build with latest meson rsync: update 3.2.1 -> 3.2.2 vala: update 0.48.6 -> 0.48.7 logrotate: update 3.16.0 -> 3.17.0 mesa: update 20.1.2 -> 20.1.4 libcap: update 2.36 -> 2.41 net-tools: fix upstream version check meson.bbclass: add a cups-config entry oeqa: write @OETestTag content into json test reports for each case libhandy: upstream has moved to gnome Alistair Francis (1): binutils: Remove RISC-V PIE patch Andrei Gherzan (2): initscripts: Fix various shellcheck warnings in populate-volatile.sh initscripts: Fix populate-volatile.sh bug when file/dir exists Anuj Mittal (4): harfbuzz: upgrade 2.6.8 -> 2.7.1 sqlite3: upgrade 3.32.3 -> 3.33.0 stress-ng: upgrade 0.11.17 -> 0.11.18 x264: upgrade to latest revision Armin Kuster (1): glibc: Secruity fix for CVE-2020-6096 Bruce Ashfield (25): linux-yocto/5.4: update to v5.4.53 linux-yocto/5.4: fix perf build with binutils 2.35 kernel/yocto: allow dangling KERNEL_FEATURES linux-yocto/5.4: update to v5.4.54 systemtap: update to 4.3 latest kernel-devsrc: fix x86 (32bit) on target module build lttng-modules: update to 2.12.2 (fixes v5.8+ builds) yocto-bsps: update reference BSPs to 5.4.54 kernel-yocto: enhance configuration queue analysis capabilities strace: update to 5.8 (fix build against v5.8 uapi headers) linux-yocto-rt/5.4: update to rt32 linux-yocto/5.4: update to v5.4.56 linux-yocto/5.4: update to v5.4.57 kernel-yocto: set cwd before querying the meta data dir kernel-yocto: make # is not set matching more precise kernel-yocto: split meta data gathering into patch and config phases make-mod-scripts: add HOSTCXX definitions and gmp-native dependency kernel-devsrc: fix on target modules prepare for ARM kernel-devsrc: 5.8 + gcc10 require gcc-plugins + libmpc-dev linux-yocto/5.4: update to v5.4.58 linux-yocto/5.4: perf cs-etm: Move definition of 'traceid_list' global variable from header file libc-headers: update to v5.8 linux-yocto: introduce 5.8 reference kernel kernel-yocto/5.8: add gmp-native dependency linux-yocto/5.8: update to v5.8.1 Chandana kalluri (1): qemu.inc: Use virtual/libgl instead of mesa Changhyeok Bae (2): iproute2: upgrade 5.7.0 -> 5.8.0 ethtool: upgrade 5.7 -> 5.8 Changqing Li (5): layer.conf: fix adwaita-icon-theme signature change problem gtk-icon-cache.bbclass: add features_check gcc-runtime.inc: fix m32 compile fail with x86-64 compiler libffi: fix multilib header conflict gpgme: fix multilib header conflict Chen Qi (3): grub: set CVE_PRODUCT to grub2 runqemu: fix permission check of /dev/vhost-net fribidi: extend CVE_PRODUCT to include fribidi Chris Laplante (11): lib/oe/log_colorizer.py: add LogColorizerProxyProgressHandler bitbake: build: print traceback if progress handler can't be created bitbake: build: create_progress_handler: fix calling 'get' on NoneType bitbake: progress: modernize syntax, format bitbake: progress: fix hypothetical NameError if 'progress' isn't set bitbake: progress: filter ANSI escape codes before looking for progress text bitbake: tests/color: add test suite for ANSI color code filtering bitbake: data: emit filename/lineno information for shell functions bitbake: build: print a backtrace when a Bash shell function fails bitbake: build: print a backtrace with the original metadata locations of Bash shell funcs bitbake: build: make shell traps less chatty when 'bitbake -v' is used Dan Callaghan (1): stress-ng: create a symlink for /usr/bin/stress Daniel Ammann (1): wic: fix typo Daniel Gomez (1): allarch: Add missing allarch ttf-bitstream-vera Diego Sueiro (1): cml1: Add the option to choose the .config root dir Dmitry Baryshkov (3): mesa: enable freedreno Vulkan driver if freedreno is enabled arch-armv8-2a.inc: add tune include for armv8.2a tune-cortexa55.inc: switch to using armv8.2a include file Fredrik Gustafsson (13): package_manager: Move to package_manager/__init__.py rpm: Move manifest to its own subdir ipk: Move ipk manifest to its own subdir deb: Move deb manifest to its own subdir rpm: Move rootfs to its own dir ipk: Move rootfs to its own dir deb: Move rootfs to its own dir rpm: Move sdk to its own dir ipk: Move sdk to its own dir deb: Move sdk to its own dir rpm: Move package manager to its own dir ipk: Move package manager to its own dir deb: Move package manager to its own dir Guillaume Champagne (1): weston: add missing packageconfigs Jeremy Puhlman (1): gobject-introspection: disable scanner caching in install Joe Slater (3): libdnf: allow reproducible binary builds gconf: use python3 gcr: make sure gcr-oids.h is generated Jonathan Richardson (1): cortex-m0plus.inc: Add tuning for cortex M0 plus Joshua Watt (3): bitbake: bitbake: command: Handle multiconfig in findSigInfo lib/oe/reproducible.py: Fix git HEAD check perl: Add check for non-arch Storable.pm file Khasim Mohammed (2): wic/bootimg-efi: Add support for IMAGE_BOOT_FILES wic/bootimg-efi: Update docs for IMAGE_BOOT_FILES support in bootimg-efi Khem Raj (23): qemumips: Use 34Kf CPU emulation libunwind: Backport a fix for -fno-common option to compile dhcp: Use -fcommon compiler option inetutils: Fix build with -fno-common libomxil: Use -fcommon compiler option kexec-tools: Fix build with -fno-common distcc: Fix build with -fno-common libacpi: Fix build with -fno-common minicom: Fix build when using -fno-common binutils: Upgrade to 2.35 release xf86-video-intel: Fix build with -fno-common glibc: Upgrade to 2.32 release go: Upgrade to 1.14.7 webkitgtk: Upgrade to 2.28.4 kexec-tools: Fix additional duplicate symbols on aarch64/x86_64 builds gcc: Upgrade to 10.2.0 buildcpio.py: Apply patch to fix build with -fno-common buildgalculator: Patch to fix build with -fno-common localedef: Update to include floatn.h fix xserver-xorg: Fix build with -fno-common/mips binutils: Let crosssdk gold linker generate 4096 btyes long .interp section gcc-cross-canadian: Correct the regexp to delete versioned gcc binary curl: Upgrade to 7.72.0 Konrad Weihmann (2): rootfs-post: remove traling blanks from tasks cve-update: handle baseMetricV2 as optional Lee Chee Yang (4): buildhistory: use pid for temporary txt file name checklayer: check layer in BBLAYERS before test ghostscript: fix CVE-2020-15900 qemu : fix CVE-2020-15863 Mark Hatle (1): package.bbclass: Sort shlib2 output for hash equivalency Martin Jansa (2): net-tools: upgrade to latest revision in upstream repo instead of old debian snapshot perf: backport a fix for confusing non-fatal error Matt Madison (1): cogl-1.0: correct X11 dependencies Matthew (3): ltp: remove --with-power-management-testsuite from EXTRA_OECONF ltp: remove OOM tests from runtest/mm ltp: make copyFrom scp command non-fatal Mikko Rapeli (2): alsa-topology-conf: use ${datadir} in do_install() alsa-ucm-conf: use ${datadir} in do_install() Ming Liu (3): conf/machine: set UBOOT_MACHINE for qemumips and qemumips64 multilib.conf: add u-boot to NON_MULTILIB_RECIPES libubootenv: uprev to v0.3 Mingli Yu (2): ccache: Upgrade to 3.7.11 Revert "python3: define a profile directory path" Naoto Yamaguchi (1): patch.py: Change to more strictly fuzz detection Nathan Rossi (4): libexif: Enable native and nativesdk cmake.bbclass: Rework compiler program variables for allarch python3: Improve handling of python3 manifest generation python3-manifest.json: Updates Oleksandr Kravchuk (9): python3-setuptools: update to 49.2.0 bash-completion: update to 2.11 python3: update to 3.8.5 re2c: update to 2.0 diffoscope: update to 153 json-c: update to 0.15 git: update 2.28.0 libwpe: update to 1.7.1 python3-setuptools: update to 49.3.1 Richard Purdie (20): perl: Avoid race continually rebuilding miniperl gcc: Fix mangled patch bitbake: server/process: Fix UI first connection tracking bitbake: server/process: Account for xmlrpc connections Revert "lib/oe/log_colorizer.py: add LogColorizerProxyProgressHandler" lib/package_manager: Fix missing imports populate_sdk_ext: Ensure buildtools doesn't corrupt OECORE_NATIVE_SYSROOT buildtools: Handle generic environment setup injection uninative: Handle PREMIRRORS generically maintainers: Update entries for Mark Hatle gcr: Fix patch Upstream-Status from v2 patch bitbake: server/process: Remove pointless process forking bitbake: server/process: Simplfy idle callback handler function bitbake: server/process: Pass timeout/xmlrpc parameters directly to the server bitbake: server/process: Add extra logfile flushing packagefeed-stability: Remove as obsolete build-compare: Drop recipe qemu: Upgrade 5.0.0 -> 5.1.0 selftest/tinfoil: Increase wait event timeout lttng-tools: upgrade 2.12.1 -> 2.12.2 Ross Burton (3): popt: upgrade to 1.18 conf/machine: set UBOOT_MACHINE for qemuarm and qemuarm64 gcc: backport a fix for out-of-line atomics on aarch64 TeohJayShen (2): oeqa/manual/bsp-hw.json : remove shutdown_system test oeqa/manual/bsp-hw.json : remove X_server_can_start_up_with_runlevel_5_boot test Trevor Gamblin (1): llvm: upgrade 9.0.1 -> 10.0.1 Tyler Hicks (1): kernel-devicetree: Fix intermittent build failures caused by DTB builds Usama Arif (3): kernel-fitimage: build configuration for image tree when dtb is not present oeqa/selftest/imagefeatures: Add testcase for fitImage ref-manual: Add documentation for kernel-fitimage Vasyl Vavrychuk (1): runqemu: Check gtk or sdl option is passed together with gl or gl-es options. Yi Zhao (1): pbzip2: extend for nativesdk Zhang Qiang (1): kernel.bbclass: Configuration for environment with HOSTCXX hongxu (1): nativesdk-rpm: adjust RPM_CONFIGDIR paths dynamically zangrc (8): libevdev:upgrade 1.9.0 -> 1.9.1 mpg123:upgrade 1.26.2 -> 1.26.3 flex: Refresh patch stress-ng:upgrade 0.11.15 -> 0.11.17 sudo:upgrade 1.9.1 -> 1.9.2 libcap: Upgrade 2.41 -> 2.42 libinput: Upgrade 1.15.6 -> 1.16.0 python3-setuptools: Upgrade 49.2.0 -> 49.2.1 Signed-off-by: Andrew Geissler Change-Id: Ic7fa1e8484c1c7722a70c75608aa4ab21fa7d755 --- poky/bitbake/bin/bitbake-selftest | 1 + poky/bitbake/lib/bb/build.py | 130 ++++++++++++++++++++++++++++++---- poky/bitbake/lib/bb/command.py | 8 +-- poky/bitbake/lib/bb/cooker.py | 10 +-- poky/bitbake/lib/bb/cookerdata.py | 8 +-- poky/bitbake/lib/bb/data.py | 6 ++ poky/bitbake/lib/bb/process.py | 3 +- poky/bitbake/lib/bb/progress.py | 60 +++++++++++----- poky/bitbake/lib/bb/server/process.py | 50 +++++++------ poky/bitbake/lib/bb/tests/color.py | 95 +++++++++++++++++++++++++ 10 files changed, 306 insertions(+), 65 deletions(-) create mode 100644 poky/bitbake/lib/bb/tests/color.py (limited to 'poky/bitbake') diff --git a/poky/bitbake/bin/bitbake-selftest b/poky/bitbake/bin/bitbake-selftest index 041a2719f..e84d6a559 100755 --- a/poky/bitbake/bin/bitbake-selftest +++ b/poky/bitbake/bin/bitbake-selftest @@ -18,6 +18,7 @@ except RuntimeError as exc: sys.exit(str(exc)) tests = ["bb.tests.codeparser", + "bb.tests.color", "bb.tests.cooker", "bb.tests.cow", "bb.tests.data", diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index 23b6ee455..94f9cb371 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -16,7 +16,9 @@ import os import sys import logging import glob +import itertools import time +import re import stat import bb import bb.msg @@ -303,20 +305,60 @@ def exec_func_python(func, d, runfile, cwd=None): def shell_trap_code(): return '''#!/bin/sh\n +__BITBAKE_LAST_LINE=0 + # Emit a useful diagnostic if something fails: -bb_exit_handler() { +bb_sh_exit_handler() { + ret=$? + if [ "$ret" != 0 ]; then + echo "WARNING: exit code $ret from a shell command." + fi + exit $ret +} + +bb_bash_exit_handler() { ret=$? - case $ret in - 0) ;; - *) case $BASH_VERSION in - "") echo "WARNING: exit code $ret from a shell command.";; - *) echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";; - esac - exit $ret - esac + { set +x; } > /dev/null + trap "" DEBUG + if [ "$ret" != 0 ]; then + echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" + + echo "WARNING: Backtrace (BB generated script): " + for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do + if [ "$i" -eq 1 ]; then + echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" + else + echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" + fi + done + fi + exit $ret +} + +bb_bash_debug_handler() { + local line=${BASH_LINENO[0]} + # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it + if [ "$line" -eq 1 ]; then + return + fi + + # Track the line number of commands as they execute. This is so we can have access to the failing line number + # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html + if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then + __BITBAKE_LAST_LINE=$line + fi } -trap 'bb_exit_handler' 0 -set -e + +case $BASH_VERSION in +"") trap 'bb_sh_exit_handler' 0 + set -e + ;; +*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 + trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG + set -e + shopt -s extdebug + ;; +esac ''' def create_progress_handler(func, progress, logfile, d): @@ -346,7 +388,7 @@ def create_progress_handler(func, progress, logfile, d): cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) if not cls_obj: # Fall-back on __builtins__ - cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__) + cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) if cls_obj: return cls_obj(d, outfile=logfile, otherargs=otherargs) bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) @@ -398,7 +440,13 @@ exit $ret progress = d.getVarFlag(func, 'progress') if progress: - logfile = create_progress_handler(func, progress, logfile, d) + try: + logfile = create_progress_handler(func, progress, logfile, d) + except: + from traceback import format_exc + logger.error("Failed to create progress handler") + logger.error(format_exc()) + raise fifobuffer = bytearray() def readfifo(data): @@ -450,6 +498,62 @@ exit $ret bb.debug(2, "Executing shell function %s" % func) with open(os.devnull, 'r+') as stdin, logfile: bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) + except bb.process.ExecutionError as exe: + # Find the backtrace that the shell trap generated + backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") + stdout_lines = (exe.stdout or "").split("\n") + backtrace_start_line = None + for i, line in enumerate(reversed(stdout_lines)): + if backtrace_marker_regex.search(line): + backtrace_start_line = len(stdout_lines) - i + break + + # Read the backtrace frames, starting at the location we just found + backtrace_entry_regex = re.compile(r"#(?P\d+): (?P[^\s]+), (?P.+?), line (" + r"?P\d+)") + backtrace_frames = [] + if backtrace_start_line: + for line in itertools.islice(stdout_lines, backtrace_start_line, None): + match = backtrace_entry_regex.search(line) + if match: + backtrace_frames.append(match.groupdict()) + + with open(runfile, "r") as script: + script_lines = [line.rstrip() for line in script.readlines()] + + # For each backtrace frame, search backwards in the script (from the line number called out by the frame), + # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata + # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. + script_metadata_comment_regex = re.compile(r"# line: (?P\d+), file: (?P.+)") + better_frames = [] + # Skip the very last frame since it's just the call to the shell task in the body of the script + for frame in backtrace_frames[:-1]: + # Check whether the frame corresponds to a function defined in the script vs external script. + if os.path.samefile(frame["file"], runfile): + # Search backwards from the frame lineno to locate the comment that BB injected + i = int(frame["lineno"]) - 1 + while i >= 0: + match = script_metadata_comment_regex.match(script_lines[i]) + if match: + # Calculate the relative line in the function itself + relative_line_in_function = int(frame["lineno"]) - i - 2 + # Calculate line in the function as declared in the metadata + metadata_function_line = relative_line_in_function + int(match["lineno"]) + better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( + frameno=frame["frameno"], + funcname=frame["funcname"], + file=match["file"], + lineno=metadata_function_line + )) + break + i -= 1 + else: + better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) + + if better_frames: + better_frames = ("\t{0}".format(frame) for frame in better_frames) + exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) + raise finally: os.unlink(fifopath) diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index 805ed9216..4d152ff4c 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -84,7 +84,7 @@ class Command: if command not in CommandsAsync.__dict__: return None, "No such command" self.currentAsyncCommand = (command, commandline) - self.cooker.configuration.server_register_idlecallback(self.cooker.runCommands, self.cooker) + self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker) return True, None def runAsyncCommand(self): @@ -723,10 +723,10 @@ class CommandsAsync: """ Find signature info files via the signature generator """ - pn = params[0] + (mc, pn) = bb.runqueue.split_mc(params[0]) taskname = params[1] sigs = params[2] - res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.data) - bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.data) + res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc]) + bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc]) command.finishAsyncCommand() findSigInfo.needcache = False diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index f6abc6348..912360546 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -148,7 +148,7 @@ class BBCooker: Manages one bitbake build run """ - def __init__(self, configuration, featureSet=None): + def __init__(self, configuration, featureSet=None, idleCallBackRegister=None): self.recipecaches = None self.skiplist = {} self.featureset = CookerFeatures() @@ -158,6 +158,8 @@ class BBCooker: self.configuration = configuration + self.idleCallBackRegister = idleCallBackRegister + bb.debug(1, "BBCooker starting %s" % time.time()) sys.stdout.flush() @@ -210,7 +212,7 @@ class BBCooker: cooker.process_inotify_updates() return 1.0 - self.configuration.server_register_idlecallback(_process_inotify_updates, self) + self.idleCallBackRegister(_process_inotify_updates, self) # TOSTOP must not be set or our children will hang when they output try: @@ -1423,7 +1425,7 @@ class BBCooker: return True return retval - self.configuration.server_register_idlecallback(buildFileIdle, rq) + self.idleCallBackRegister(buildFileIdle, rq) def buildTargets(self, targets, task): """ @@ -1494,7 +1496,7 @@ class BBCooker: if 'universe' in targets: rq.rqdata.warn_multi_bb = True - self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + self.idleCallBackRegister(buildTargetsIdle, rq) def getAllKeysWithFlags(self, flaglist): diff --git a/poky/bitbake/lib/bb/cookerdata.py b/poky/bitbake/lib/bb/cookerdata.py index 24bf09c56..b86e7d446 100644 --- a/poky/bitbake/lib/bb/cookerdata.py +++ b/poky/bitbake/lib/bb/cookerdata.py @@ -143,16 +143,10 @@ class CookerConfiguration(object): setattr(self, key, parameters.options.__dict__[key]) self.env = parameters.environment.copy() - def setServerRegIdleCallback(self, srcb): - self.server_register_idlecallback = srcb - def __getstate__(self): state = {} for key in self.__dict__.keys(): - if key == "server_register_idlecallback": - state[key] = None - else: - state[key] = getattr(self, key) + state[key] = getattr(self, key) return state def __setstate__(self,state): diff --git a/poky/bitbake/lib/bb/data.py b/poky/bitbake/lib/bb/data.py index b0683c518..97022853c 100644 --- a/poky/bitbake/lib/bb/data.py +++ b/poky/bitbake/lib/bb/data.py @@ -161,6 +161,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False): return True if func: + # Write a comment indicating where the shell function came from (line number and filename) to make it easier + # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata + # location of shell functions. + o.write("# line: {0}, file: {1}\n".format( + d.getVarFlag(var, "lineno", False), + d.getVarFlag(var, "filename", False))) # NOTE: should probably check for unbalanced {} within the var val = val.rstrip('\n') o.write("%s() {\n%s\n}\n" % (varExpanded, val)) diff --git a/poky/bitbake/lib/bb/process.py b/poky/bitbake/lib/bb/process.py index 2dc472a86..f36c929d2 100644 --- a/poky/bitbake/lib/bb/process.py +++ b/poky/bitbake/lib/bb/process.py @@ -41,6 +41,7 @@ class ExecutionError(CmdError): self.exitcode = exitcode self.stdout = stdout self.stderr = stderr + self.extra_message = None def __str__(self): message = "" @@ -51,7 +52,7 @@ class ExecutionError(CmdError): if message: message = ":\n" + message return (CmdError.__str__(self) + - " with exit code %s" % self.exitcode + message) + " with exit code %s" % self.exitcode + message + (self.extra_message or "")) class Popen(subprocess.Popen): defaults = { diff --git a/poky/bitbake/lib/bb/progress.py b/poky/bitbake/lib/bb/progress.py index 9c755b7f7..d051ba019 100644 --- a/poky/bitbake/lib/bb/progress.py +++ b/poky/bitbake/lib/bb/progress.py @@ -14,7 +14,27 @@ import bb.event import bb.build from bb.build import StdoutNoopContextManager -class ProgressHandler(object): + +# from https://stackoverflow.com/a/14693789/221061 +ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') + + +def filter_color(string): + """ + Filter ANSI escape codes out of |string|, return new string + """ + return ANSI_ESCAPE_REGEX.sub('', string) + + +def filter_color_n(string): + """ + Filter ANSI escape codes out of |string|, returns tuple of + (new string, # of ANSI codes removed) + """ + return ANSI_ESCAPE_REGEX.subn('', string) + + +class ProgressHandler: """ Base class that can pretend to be a file object well enough to be used to build objects to intercept console output and determine the @@ -55,6 +75,7 @@ class ProgressHandler(object): self._lastevent = ts self._progress = progress + class LineFilterProgressHandler(ProgressHandler): """ A ProgressHandler variant that provides the ability to filter out @@ -66,7 +87,7 @@ class LineFilterProgressHandler(ProgressHandler): """ def __init__(self, d, outfile=None): self._linebuffer = '' - super(LineFilterProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) def write(self, string): self._linebuffer += string @@ -80,41 +101,44 @@ class LineFilterProgressHandler(ProgressHandler): lbreakpos = line.rfind('\r') + 1 if lbreakpos: line = line[lbreakpos:] - if self.writeline(line): - super(LineFilterProgressHandler, self).write(line) + if self.writeline(filter_color(line)): + super().write(line) def writeline(self, line): return True + class BasicProgressHandler(ProgressHandler): def __init__(self, d, regex=r'(\d+)%', outfile=None): - super(BasicProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) self._regex = re.compile(regex) # Send an initial progress event so the bar gets shown self._fire_progress(0) def write(self, string): - percs = self._regex.findall(string) + percs = self._regex.findall(filter_color(string)) if percs: progress = int(percs[-1]) self.update(progress) - super(BasicProgressHandler, self).write(string) + super().write(string) + class OutOfProgressHandler(ProgressHandler): def __init__(self, d, regex, outfile=None): - super(OutOfProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) self._regex = re.compile(regex) # Send an initial progress event so the bar gets shown self._fire_progress(0) def write(self, string): - nums = self._regex.findall(string) + nums = self._regex.findall(filter_color(string)) if nums: progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100 self.update(progress) - super(OutOfProgressHandler, self).write(string) + super().write(string) + -class MultiStageProgressReporter(object): +class MultiStageProgressReporter: """ Class which allows reporting progress without the caller having to know where they are in the overall sequence. Useful @@ -199,6 +223,7 @@ class MultiStageProgressReporter(object): value is considered to be out of stage_total, otherwise it should be a percentage value from 0 to 100. """ + progress = None if self._stage_total: stage_progress = (float(stage_progress) / self._stage_total) * 100 if self._stage < 0: @@ -207,9 +232,10 @@ class MultiStageProgressReporter(object): progress = self._base_progress + (stage_progress * self._stage_weights[self._stage]) else: progress = self._base_progress - if progress > 100: - progress = 100 - self._fire_progress(progress) + if progress: + if progress > 100: + progress = 100 + self._fire_progress(progress) def finish(self): if self._finished: @@ -230,6 +256,7 @@ class MultiStageProgressReporter(object): out.append('Up to finish: %d' % stage_weight) bb.warn('Stage times:\n %s' % '\n '.join(out)) + class MultiStageProcessProgressReporter(MultiStageProgressReporter): """ Version of MultiStageProgressReporter intended for use with @@ -238,7 +265,7 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter): def __init__(self, d, processname, stage_weights, debug=False): self._processname = processname self._started = False - MultiStageProgressReporter.__init__(self, d, stage_weights, debug) + super().__init__(d, stage_weights, debug) def start(self): if not self._started: @@ -255,13 +282,14 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter): MultiStageProgressReporter.finish(self) bb.event.fire(bb.event.ProcessFinished(self._processname), self._data) + class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter): """ MultiStageProcessProgressReporter that takes the calls and does nothing with them (to avoid a bunch of "if progress_reporter:" checks) """ def __init__(self): - MultiStageProcessProgressReporter.__init__(self, "", None, []) + super().__init__(None, []) def _fire_progress(self, taskprogress, rate=None): pass diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 9ec79f5b6..65e1eab52 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -34,12 +34,11 @@ logger = logging.getLogger('BitBake') class ProcessTimeout(SystemExit): pass -class ProcessServer(multiprocessing.Process): +class ProcessServer(): profile_filename = "profile.log" profile_processed_filename = "profile.log.processed" - def __init__(self, lock, sock, sockname): - multiprocessing.Process.__init__(self) + def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface): self.command_channel = False self.command_channel_reply = False self.quit = False @@ -47,6 +46,7 @@ class ProcessServer(multiprocessing.Process): self.next_heartbeat = time.time() self.event_handle = None + self.hadanyui = False self.haveui = False self.maxuiwait = 30 self.xmlrpc = False @@ -57,6 +57,9 @@ class ProcessServer(multiprocessing.Process): self.sock = sock self.sockname = sockname + self.server_timeout = server_timeout + self.xmlrpcinterface = xmlrpcinterface + def register_idle_function(self, function, data): """Register a function to be called while the server is idle""" assert hasattr(function, '__call__') @@ -188,6 +191,7 @@ class ProcessServer(multiprocessing.Process): self.command_channel_reply = writer self.haveui = True + self.hadanyui = True except (EOFError, OSError): disconnect_client(self, fds) @@ -200,7 +204,7 @@ class ProcessServer(multiprocessing.Process): # If we don't see a UI connection within maxuiwait, its unlikely we're going to see # one. We have had issue with processes hanging indefinitely so timing out UI-less # servers is useful. - if not self.haveui and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): + if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): print("No UI connection within max timeout, exiting to avoid infinite loop.") self.quit = True @@ -243,6 +247,10 @@ class ProcessServer(multiprocessing.Process): self.cooker.post_serve() + # Flush logs before we release the lock + sys.stdout.flush() + sys.stderr.flush() + # Finally release the lockfile but warn about other processes holding it open lock = self.bitbake_lock lockfile = lock.name @@ -465,23 +473,25 @@ class BitBakeServer(object): print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format))) sys.stdout.flush() - server = ProcessServer(self.bitbake_lock, self.sock, self.sockname) - self.configuration.setServerRegIdleCallback(server.register_idle_function) - os.close(self.readypipe) - writer = ConnectionWriter(self.readypipein) try: - self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset) - except bb.BBHandledException: - return None - writer.send("r") - writer.close() - server.cooker = self.cooker - server.server_timeout = self.configuration.server_timeout - server.xmlrpcinterface = self.configuration.xmlrpcinterface - print("Started bitbake server pid %d" % os.getpid()) - sys.stdout.flush() - - server.start() + server = ProcessServer(self.bitbake_lock, self.sock, self.sockname, self.configuration.server_timeout, self.configuration.xmlrpcinterface) + os.close(self.readypipe) + writer = ConnectionWriter(self.readypipein) + try: + self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, server.register_idle_function) + except bb.BBHandledException: + return None + writer.send("r") + writer.close() + server.cooker = self.cooker + print("Started bitbake server pid %d" % os.getpid()) + sys.stdout.flush() + + server.run() + finally: + # Flush any ,essages/errors to the logfile before exit + sys.stdout.flush() + sys.stderr.flush() def connectProcessServer(sockname, featureset): # Connect to socket diff --git a/poky/bitbake/lib/bb/tests/color.py b/poky/bitbake/lib/bb/tests/color.py new file mode 100644 index 000000000..bf03750c6 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/color.py @@ -0,0 +1,95 @@ +# +# BitBake Test for ANSI color code filtering +# +# Copyright (C) 2020 Agilent Technologies, Inc. +# Author: Chris Laplante +# +# SPDX-License-Identifier: MIT +# + +import unittest +import bb.progress +import bb.data +import bb.event +from bb.progress import filter_color, filter_color_n +import io +import re + + +class ProgressWatcher: + def __init__(self): + self._reports = [] + + def handle_event(self, event): + self._reports.append((event.progress, event.rate)) + + def reports(self): + return self._reports + + +class ColorCodeTests(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self._progress_watcher = ProgressWatcher() + bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event) + + def tearDown(self): + bb.event.remove("bb.build.TaskProgress", None) + + def test_filter_color(self): + input_string = "~~~~~~~~~~~~^~~~~~~~" + filtered = filter_color(input_string) + self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~") + + def test_filter_color_n(self): + input_string = "~~~~~~~~~~~~^~~~~~~~" + filtered, code_count = filter_color_n(input_string) + self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~") + self.assertEqual(code_count, 4) + + def test_LineFilterProgressHandler_color_filtering(self): + class CustomProgressHandler(bb.progress.LineFilterProgressHandler): + PROGRESS_REGEX = re.compile(r"Progress: (?P\d+)%") + + def writeline(self, line): + match = self.PROGRESS_REGEX.match(line) + if match: + self.update(int(match.group("progress"))) + return False + return True + + buffer = io.StringIO() + handler = CustomProgressHandler(self.d, buffer) + handler.write("Program output!\n") + handler.write("More output!\n") + handler.write("Progress: 10%\n") # 10% + handler.write("Even more\n") + handler.write("Progress: 50%\n") # 50% + handler.write("Progress: 60%\n") # 60% + handler.write("Progress: 100%\n") # 100% + + expected = [(10, None), (50, None), (60, None), (100, None)] + self.assertEqual(self._progress_watcher.reports(), expected) + + self.assertEqual(buffer.getvalue(), "Program output!\nMore output!\nEven more\n") + + def test_BasicProgressHandler_color_filtering(self): + buffer = io.StringIO() + handler = bb.progress.BasicProgressHandler(self.d, outfile=buffer) + handler.write("1%\n") # 1% + handler.write("2%\n") # 2% + handler.write("10%\n") # 10% + handler.write("100%\n") # 100% + + expected = [(0, None), (1, None), (2, None), (10, None), (100, None)] + self.assertListEqual(self._progress_watcher.reports(), expected) + + def test_OutOfProgressHandler_color_filtering(self): + buffer = io.StringIO() + handler = bb.progress.OutOfProgressHandler(self.d, r'(\d+) of (\d+)', outfile=buffer) + handler.write("Text text 1 of 5") # 1/5 + handler.write("Text text 3 of 5") # 3/5 + handler.write("Text text 5 of 5") # 5/5 + + expected = [(0, None), (20.0, None), (60.0, None), (100.0, None)] + self.assertListEqual(self._progress_watcher.reports(), expected) -- cgit v1.2.3