From a5c52ff0bb57fccc2dcd3bd10647d2fa77ddf8c3 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Fri, 23 Nov 2018 10:55:50 +1300 Subject: poky: thud refresh 87e3a9739d..1cab405d88 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update poky to thud HEAD. Adrian Bunk (1): archiver.bbclass: Fix COPYLEFT_LICENSE_{IN, EX}CLUDE Adrian Freihofer (2): systemd: fix PN-container package splitting devtool: fix target-deploy --strip Alejandro Enedino Hernandez Samaniego (1): python: Adds instructions to the manifest file Alexander Kanavin (3): meson: do not manipulate the environment when looking for python via pkg-config openssl: update to 1.1.1a libc-package: fix postinst error when ENABLE_BINARY_LOCALE_GENERATION = "0" Alexey Brodkin (1): gdb: Remove long ago upstreamed patch André Draszik (1): linux-firmware: better packaging for TI wl12xx & wl18xx firmwares Anuj Mittal (13): maintainers.inc: update Intel owners gst-plugins-bad: add PACKAGECONFIG for msdk gstreamer1.0: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-plugins-base: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-plugins-good: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-plugins-bad: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-plugins-ugly: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-libav: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-vaapi: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-rtsp-server: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-omx: upgrade 1.14.3 -> 1.14.4 gstreamer1.0-python: upgrade 1.14.3 -> 1.14.4 gst-validate: upgrade 1.14.2 -> 1.14.4 Armin Kuster (1): gnutls: update to 3.6.4 Bruce Ashfield (13): linux-yocto: remove obselete options from lxc config linux-yocto/4.14: configuration cleanups linux-yocto/4.18: -rt sync and config cleanups linux-yocto/tiny: switch default branch to standard/tiny/base linux-yocto/tiny: restore qemuarm support linux-yocto/4.18: bug fixes and configuration tweaks linux-yocto/4.18: update to v4.18.17 linux-yocto/4.14: update to v4.14.79 linux-yocto/4.18: integrate CVE fixes linux-yocto/4.18: update to v4.18.20 linux-yocto/4.18: update to v4.18.21 linux-yocto: configuration updates (virtio and tpm) linux-yocto: correct qemumips64el definition Carlos Rafael Giani (10): gstreamer1.0: upgrade to version 1.14.3 gstreamer1.0-plugin-base: upgrade to version 1.14.3 gstreamer1.0-plugin-good: upgrade to version 1.14.3 gstreamer1.0-plugin-bad: upgrade to version 1.14.3 gstreamer1.0-plugin-ugly: upgrade to version 1.14.3 gstreamer1.0-libav: upgrade to version 1.14.3 gstreamer1.0-rtsp-server: upgrade to version 1.14.3 gstreamer1.0-vaapi: upgrade to version 1.14.3 gstreamer1.0-omx: upgrade to version 1.14.3 gstreamer1.0-python: upgrade to version 1.14.3 Changhyeok Bae (2): iproute2: 4.18.0 -> 4.19.0 ethtool: 4.17 -> 4.19 Christophe PRIOUZEAU (1): openssl: correct bad path on package preprocess Douglas Royds (4): boost-context: Reproducibility: Set .file section for all *_elf_gas.S files reproducible: Refactor: Break out fixed_source_date_epoch() function reproducible: Don't look for youngest file when no source tarball ptest: Reproducibility: Take control of umask Eric Chanudet (1): licence: Add license file CC-BY-SA-4.0 Hongxu Jia (6): elfutils: 0.174 -> 0.175 gnupg: upgrade 2.2.9 -> 2.2.10 gnupg: upgrade 2.2.10 -> 2.2.11 libgcrypt: upgrade 1.8.3 -> 1.8.4 ghostscript: 9.25 -> 9.26 go 1.9/1.11: fix textrel qa warning for non mips arch Joshua Watt (3): meta/icecc.bbclass: Move system blacklist to variables meta/icecc.bbclass: Update system blacklists classes/icecc.bbclass: Fix ccache disable Kai Kang (1): multilib_script: fix packages split Khem Raj (3): valgrind: Skip vgpreload_memcheck shared object from stripping populate_sdk_ext.bbclass: Include site.conf in parsing for contents for local.conf local.conf.sample: Update the sample config as per new migration manual Ming Liu (1): image.bbclass: fix a wrong position blank Mingli Yu (2): mdadm: improve the run-ptest nspr: improve reproducibility Niko Mauno (1): opkg-utils: Fix update-alternatives link relocation Otavio Salvador (1): linux-firmware: Bump revision to 1baa348 Paul Eggleton (1): socat: fix LICENSE Peter Kjellerstedt (3): meson: Correct use of the _append operator bitbake: bitbake-diffsigs: Merge with bitbake-dumpsig bitbake: bitbake-diffsigs: Support recursive deps with signature files Richard Purdie (9): meson: Disable rpath stripping at install time scripts/runqemu: Fix logic error causing failures with MACHINE from the environment meta-selftest/error: Cleanup large trailing whitespace bitbake: server/process: Make lockfile handling clearer bitbake: server/process: Show the last 60 log lines, not the last 10 bitbake: server/process: Show last 60 lines of the log if the server didn't start poky.conf: Update the distros we test against on the autobuilder poky: Update version to 2.6.1 build-appliance-image: Update to thud head revision Robert Yang (5): classes: Remove tab indentations in python code recipes: Remove tab indentations in python code bugzilla.bbclass: Remove it since obsoleted sstate.bbclass: Only remove sstate file when task is existed bitbake: server/process: print a message when no logfile Ross Burton (12): cpio: fix crash when appending to archives image_types: use cpio-native to build cpio images libtasn1: no need to inherit binconfig binconfig: only try to mangles files piglit: add missing waffle-bin dependency ruby: upgrade 2.5.1 -> 2.5.3 insane: Clarify GNU_HASH warning patchreview: Various fixes/improvements python3: don't cripple target distutils python3: drop redundant patch bitbake: layerindex: don't use shell=True when cloning bitbake: fetch: don't use shell=True when listing ar files Zheng Ruoqin (2): nss: Fix SHA_HTONL bug for arm 32be. createrepo-c: Fix setup of logging (log domains) Change-Id: I025dd27f44e57af38abff110ebc331e371bc461b Signed-off-by: Brad Bishop --- poky/bitbake/bin/bitbake-diffsigs | 143 +++++++++++++++++++------------- poky/bitbake/bin/bitbake-dumpsig | 95 +-------------------- poky/bitbake/lib/bb/fetch2/__init__.py | 2 +- poky/bitbake/lib/bb/server/process.py | 24 ++++-- poky/bitbake/lib/bblayers/layerindex.py | 2 +- 5 files changed, 105 insertions(+), 161 deletions(-) mode change 100755 => 120000 poky/bitbake/bin/bitbake-dumpsig (limited to 'poky/bitbake') diff --git a/poky/bitbake/bin/bitbake-diffsigs b/poky/bitbake/bin/bitbake-diffsigs index 4e6bbddcd8..fa430bb3b3 100755 --- a/poky/bitbake/bin/bitbake-diffsigs +++ b/poky/bitbake/bin/bitbake-diffsigs @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# bitbake-diffsigs -# BitBake task signature data comparison utility +# bitbake-diffsigs / bitbake-dumpsig +# BitBake task signature data dump and comparison utility # # Copyright (C) 2012-2013, 2017 Intel Corporation # @@ -21,7 +21,6 @@ import os import sys import warnings -import fnmatch import argparse import logging import pickle @@ -32,7 +31,10 @@ import bb.tinfoil import bb.siggen import bb.msg -logger = bb.msg.logger_create('bitbake-diffsigs') +myname = os.path.basename(sys.argv[0]) +logger = bb.msg.logger_create(myname) + +is_dump = myname == 'bitbake-dumpsig' def find_siginfo(tinfoil, pn, taskname, sigs=None): result = None @@ -59,8 +61,8 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None): sys.exit(2) return result -def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False): - """ Find the most recent signature files for the specified PN/task and compare them """ +def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None): + """ Find the most recent signature files for the specified PN/task """ if not taskname.startswith('do_'): taskname = 'do_%s' % taskname @@ -79,73 +81,81 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False latestfiles = [sigfiles[sig1], sigfiles[sig2]] else: filedates = find_siginfo(bbhandler, pn, taskname) - latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:] + latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:] if not latestfiles: logger.error('No sigdata files found matching %s %s' % (pn, taskname)) sys.exit(1) - elif len(latestfiles) < 2: - logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname)) - sys.exit(1) - # Define recursion callback - def recursecb(key, hash1, hash2): - hashes = [hash1, hash2] - hashfiles = find_siginfo(bbhandler, key, None, hashes) - - recout = [] - if len(hashfiles) == 0: - recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) - elif not hash1 in hashfiles: - recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1)) - elif not hash2 in hashfiles: - recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2)) - else: - out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) - for change in out2: - for line in change.splitlines(): - recout.append(' ' + line) + return latestfiles - return recout - # Recurse into signature comparison - logger.debug("Signature file (previous): %s" % latestfiles[-2]) - logger.debug("Signature file (latest): %s" % latestfiles[-1]) - output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color) - if output: - print('\n'.join(output)) - sys.exit(0) +# Define recursion callback +def recursecb(key, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = find_siginfo(tinfoil, key, None, hashes) + + recout = [] + if len(hashfiles) == 0: + recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) + elif not hash1 in hashfiles: + recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1)) + elif not hash2 in hashfiles: + recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) + for change in out2: + for line in change.splitlines(): + recout.append(' ' + line) + return recout parser = argparse.ArgumentParser( - description="Compares siginfo/sigdata files written out by BitBake") + description=("Dumps" if is_dump else "Compares") + " siginfo/sigdata files written out by BitBake") -parser.add_argument('-d', '--debug', +parser.add_argument('-D', '--debug', help='Enable debug output', action='store_true') -parser.add_argument('--color', - help='Colorize output (where %(metavar)s is %(choices)s)', - choices=['auto', 'always', 'never'], default='auto', metavar='color') +if is_dump: + parser.add_argument("-t", "--task", + help="find the signature data file for the last run of the specified task", + action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname')) -parser.add_argument("-t", "--task", - help="find the signature data files for last two runs of the specified task and compare them", - action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname')) + parser.add_argument("sigdatafile1", + help="Signature file to dump. Not used when using -t/--task.", + action="store", nargs='?', metavar="sigdatafile") +else: + parser.add_argument('-c', '--color', + help='Colorize the output (where %(metavar)s is %(choices)s)', + choices=['auto', 'always', 'never'], default='auto', metavar='color') + + parser.add_argument('-d', '--dump', + help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)', + action='store_true') -parser.add_argument("-s", "--signature", - help="With -t/--task, specify the signatures to look for instead of taking the last two", - action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig')) + parser.add_argument("-t", "--task", + help="find the signature data files for the last two runs of the specified task and compare them", + action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname')) -parser.add_argument("sigdatafile1", - help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.", - action="store", nargs='?') + parser.add_argument("-s", "--signature", + help="With -t/--task, specify the signatures to look for instead of taking the last two", + action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig')) -parser.add_argument("sigdatafile2", - help="Second signature file to compare", - action="store", nargs='?') + parser.add_argument("sigdatafile1", + help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.", + action="store", nargs='?') + parser.add_argument("sigdatafile2", + help="Second signature file to compare", + action="store", nargs='?') options = parser.parse_args() +if is_dump: + options.color = 'never' + options.dump = True + options.sigdatafile2 = None + options.sigargs = None if options.debug: logger.setLevel(logging.DEBUG) @@ -155,17 +165,32 @@ color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.is if options.taskargs: with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare(config_only=True) - if options.sigargs: - find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color) + if not options.dump and options.sigargs: + files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1]) + else: + files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1]) + + if options.dump: + logger.debug("Signature file: %s" % files[-1]) + output = bb.siggen.dump_sigfile(files[-1]) else: - find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color) + if len(files) < 2: + logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1])) + sys.exit(1) + + # Recurse into signature comparison + logger.debug("Signature file (previous): %s" % files[-2]) + logger.debug("Signature file (latest): %s" % files[-1]) + output = bb.siggen.compare_sigfiles(files[-2], files[-1], recursecb, color=color) else: if options.sigargs: logger.error('-s/--signature can only be used together with -t/--task') sys.exit(1) try: - if options.sigdatafile1 and options.sigdatafile2: - output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color) + if not options.dump and options.sigdatafile1 and options.sigdatafile2: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color) elif options.sigdatafile1: output = bb.siggen.dump_sigfile(options.sigdatafile1) else: @@ -179,5 +204,5 @@ else: logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files') sys.exit(1) - if output: - print('\n'.join(output)) +if output: + print('\n'.join(output)) diff --git a/poky/bitbake/bin/bitbake-dumpsig b/poky/bitbake/bin/bitbake-dumpsig deleted file mode 100755 index 95ebd93546..0000000000 --- a/poky/bitbake/bin/bitbake-dumpsig +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 - -# bitbake-dumpsig -# BitBake task signature dump utility -# -# Copyright (C) 2013 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import sys -import warnings -import optparse -import logging -import pickle - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) - -import bb.tinfoil -import bb.siggen -import bb.msg - -logger = bb.msg.logger_create('bitbake-dumpsig') - -def find_siginfo_task(bbhandler, pn, taskname): - """ Find the most recent signature file for the specified PN/task """ - - if not hasattr(bb.siggen, 'find_siginfo'): - logger.error('Metadata does not support finding signature data files') - sys.exit(1) - - if not taskname.startswith('do_'): - taskname = 'do_%s' % taskname - - filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data) - latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:] - if not latestfiles: - logger.error('No sigdata files found matching %s %s' % (pn, taskname)) - sys.exit(1) - - return latestfiles[0] - -parser = optparse.OptionParser( - description = "Dumps siginfo/sigdata files written out by BitBake", - usage = """ - %prog -t recipename taskname - %prog sigdatafile""") - -parser.add_option("-D", "--debug", - help = "enable debug", - action = "store_true", dest="debug", default = False) - -parser.add_option("-t", "--task", - help = "find the signature data file for the specified task", - action="store", dest="taskargs", nargs=2, metavar='recipename taskname') - -options, args = parser.parse_args(sys.argv) - -if options.debug: - logger.setLevel(logging.DEBUG) - -if options.taskargs: - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(config_only = True) - file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1]) - logger.debug("Signature file: %s" % file) -elif len(args) == 1: - parser.print_help() - sys.exit(0) -else: - file = args[1] - -try: - output = bb.siggen.dump_sigfile(file) -except IOError as e: - logger.error(str(e)) - sys.exit(1) -except (pickle.UnpicklingError, EOFError): - logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file') - sys.exit(1) - -if output: - print('\n'.join(output)) diff --git a/poky/bitbake/bin/bitbake-dumpsig b/poky/bitbake/bin/bitbake-dumpsig new file mode 120000 index 0000000000..b1e8489b45 --- /dev/null +++ b/poky/bitbake/bin/bitbake-dumpsig @@ -0,0 +1 @@ +bitbake-diffsigs \ No newline at end of file diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 3b8389e0a2..709372e16f 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1469,7 +1469,7 @@ class FetchMethod(object): else: cmd = 'rpm2cpio.sh %s | cpio -id' % (file) elif file.endswith('.deb') or file.endswith('.ipk'): - output = subprocess.check_output('ar -t %s' % file, preexec_fn=subprocess_setup, shell=True) + output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup) datafile = None if output: for line in output.decode().splitlines(): diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 42d79079a9..070da4fe7a 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -239,6 +239,12 @@ class ProcessServer(multiprocessing.Process): while not lock: with bb.utils.timeout(3): lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) + if lock: + # We hold the lock so we can remove the file (hide stale pid data) + bb.utils.remove(lockfile) + bb.utils.unlockfile(lock) + return + if not lock: # Some systems may not have lsof available procs = None @@ -259,10 +265,6 @@ class ProcessServer(multiprocessing.Process): if procs: msg += ":\n%s" % str(procs) print(msg) - return - # We hold the lock so we can remove the file (hide stale pid data) - bb.utils.remove(lockfile) - bb.utils.unlockfile(lock) def idle_commands(self, delay, fds=None): nextsleep = delay @@ -412,23 +414,33 @@ class BitBakeServer(object): logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] + lastlines = [] with open(logfile, "r") as f: for line in f: if started: lines.append(line) else: + lastlines.append(line) res = logstart_re.match(line.rstrip()) if res: ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) + if len(lastlines) > 60: + lastlines = lastlines[-60:] if lines: - if len(lines) > 10: - bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:]))) + if len(lines) > 60: + bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:]))) else: bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines))) + elif lastlines: + bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines))) + else: + bb.error("%s doesn't exist" % logfile) + raise SystemExit(1) + ready.close() def _startServer(self): diff --git a/poky/bitbake/lib/bblayers/layerindex.py b/poky/bitbake/lib/bblayers/layerindex.py index 9f02a9da2e..b2ff2268ea 100644 --- a/poky/bitbake/lib/bblayers/layerindex.py +++ b/poky/bitbake/lib/bblayers/layerindex.py @@ -28,7 +28,7 @@ class LayerIndexPlugin(ActionPlugin): layerdir = os.path.join(repodir, subdir) if not os.path.exists(repodir): if fetch_layer: - result = subprocess.call('git clone %s %s' % (url, repodir), shell = True) + result = subprocess.call(['git', 'clone', url, repodir]) if result: logger.error("Failed to download %s" % url) return None, None, None -- cgit v1.2.3