From 5199d831602da71945df7cef62eb3c01183cf20e Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Fri, 24 Sep 2021 16:47:35 -0500 Subject: subtree updates meta-security: 1f18c623e9..de6712a806: Armin Kuster (8): cryfs: drop recipe trousers: set precise BSD license ibmtpm2tss: set precise BSD license ibmswtpm2: set precise BSD license opendnssec: set precise BSD license checksec: set precise BSD license isic: set precise BSD license tpm-quote-tools: Update SRC_URI Christer Fletcher (1): dm-verity-img.bbclass: Expose --data-block-size for configuration Kai Kang (1): sssd: 2.5.1 -> 2.5.2 meta-raspberrypi: a6fa6b3aec..9eb4879cf4: Andrew Penner (1): rpi-cmdline: Support ethernet over USB Khem Raj (2): linux-raspberrypi: Update to 5.10.63 raspberrypi-firmware: Update to latest meta-openembedded: e4a3c66505..cff8331f96: Armin Kuster (21): python3-cycler: set precise BSD license python3-dill: set precise BSD license python3-ipython-genutils: set precise BSD license python3-traitlets: set precise BSD license python3-parallax: set precise BSD license python3-ipython:set precise BSD license python3-mpmath: set precise BSD license python3-sympy: set precise BSD license python3-sqlparse: set precise BSD license python3-webencodings: set precise BSD license python3-pyperclip:set precise BSD license python3-geojson: set precise BSD license python3-aenum: set precise BSD license python3-gnupg: set precise BSD license python3-kiwisolver: set precise BSD license python3-jdcal: set precise BSD license python3-send2trash: set precise BSD license python3-flask-bootstrap: Update LICENSES autossh: set precise BSD licenses jemalloc: set precise BSD license gpsd-machine-conf: set precise BSD license Bruce Ashfield (1): vboxguestdrivers: fix build against 5.14+ Ed Tanous (1): Boost-url Move to latest version Khem Raj (57): gdm: Add polkit to required distro features python3-lxml: Inherit pkgconfig python3-icu: Inherit pkgconfig python3-h5py: Inherit pkgconfig python3-pyparted: Inherit pkgconfig python3-systemd: Inherit pkgconfig rp-pppoe: Add configure cached variable via recipe site: Remove local site files postfix: Inherit pkgconfig emacs: Inherit pkgconfig libgnt: Inherit pkgconfig libgnt: Inherit pkgconfig portaudio-v19: Inherit pkgconfig sshfs-fuse: Inherit pkgconfig appstream-glib: Inherit pkgconfig volume-key: Inherit pkgconfig kronosnet: Inherit pkgconfig rrdtool: Inherit pkgconfig libbytesize: Inherit pkgconfig dlt-daemon: Inherit pkgconfig libmypaint: Inherit pkgconfig libubox: Inherit pkgconfig xfsprogs: Inherit pkgconfig pavucontrol: Inherit pkgconfig blueman: Inherit pkgconfig mimic: Inherit pkgconfig libchamplain: Inherit pkgconfig gst-shark: Inherit pkgconfig zchunk: Inherit pkgconfig libvdpau: Inherit pkgconfig tigervnc: Inherit pkgconfig mpc: Inherit pkgconfig avro-c: Inherit pkgconfig udevil: Inherit pkgconfig remmina: Inherit pkgconfig transmission: Inherit pkgconfig libuvc: Inherit pkgconfig crda: Inherit pkgconfig wxwidgets: Inherit pkgconfig mdbus2: Inherit pkgconfig firewalld: Inherit pkgconfig renderdoc: Inherit pkgconfig fetchmail: Inherit pkgconfig ncmpc: Inherit pkgconfig yad: Inherit pkgconfig mscgen: Inherit pkgconfig libldb: Inherit pkgconfig pahole: Inherit missing pkgconfig gerbera: Inherit pkgconfig xfce4-datetime-setter: Inherit pkgconfig libblockdev: Inherit pkgconfig ntopng: Inherit pkgconfig mosquitto: Inherit pkgconfig samba: Inherit pkgconfig fio: Upgrade to 3.28 rdma-core: Inherit pkgconfig postfix: Add missing dependency on m4 Marek Vasut (1): dstat: Add missing python-six runtime dependency Matteo Croce (1): pahole: call python via env in the shebang Pascal Bach (1): poco: update to 1.11.0 Peter Kjellerstedt (1): libiio: Make libiio-python3 depend on python3-core Pierre-Jean Texier (1): cppzmq: upgrade 4.8.0 -> 4.8.1 Sakib Sajal (3): bats: source files from correct directory gd: upgrade 2.3.2 -> 2.3.3 lmdb: replace tag with commit id in SRCREV Trevor Woerner (2): vk-gl-cts: allow the user to specify the target vk-gl-cts: fix soname linking Yi Zhao (2): samba: upgrade 4.14.5 -> 4.14.7 net-snmp: remove perllocal.pod when enable packageconfig[perl] jan (1): netdata: Fixed the recipe. wangmy (3): byacc: upgrade 20200910 -> 20210808 nghttp2: upgrade 1.44.0 -> 1.45.1 apache2: upgrade 2.4.48 -> 2.4.49 zangrc (5): python3-beautifulsoup4: upgrade 4.9.3 -> 4.10.0 python3-bitarray: upgrade 2.3.3 -> 2.3.4 python3-decorator: upgrade 5.0.9 -> 5.1.0 python3-grpcio-tools: upgrade 1.39.0 -> 1.40.0 python3-grpcio: upgrade 1.39.0 -> 1.40.0 zhengruoqin (5): python3-openpyxl: upgrade 3.0.7 -> 3.0.8 python3-pandas: upgrade 1.3.2 -> 1.3.3 python3-pulsectl: upgrade 21.5.18 -> 21.9.1 protobuf: upgrade 3.17.3 -> 3.18.0 span-lite: upgrade 0.10.0 -> 0.10.1 poky: 359e1cb62f..06dcace68b: Alexander Kanavin (13): lttng: update 2.12 -> 2.13.0 core-image-ptest-all: bump RAM requirement to 4G bitbake: bitbake: drop old rules for python warnings bitbake: bitbake: correct the collections vs collections.abc deprecation bitbake: bitbake: fix regexp deprecation warnings bitbake: bitbake: do not import imp in layerindexlib bitbake: bitbake: adjust parser error check for python 3.10 compatibility bitbake: bitbake: correct deprecation warning in process.py bitbake: bitbake: enable python warnings at the first opportunity meta: correct collections vs collections.abc deprecation wic: keep rootfs_size as integer cpan-base.bbclass: use raw string for regexp testimage: symlink the task log and qemu console log to tmp/log/oeqa Armin Kuster (2): apr: Security fix for CVE-2021-35940 tar: ignore node-tar CVEs Bruce Ashfield (11): linux-yocto/5.13: update to v5.13.13 linux-yocto/5.13: update to v5.13.15 linux-yocto/5.10: update to v5.10.61 linux-yocto/5.10: update to v5.10.63 yocto-bsp/5.10: update to v5.10.63 yocto-bsp/5.13: update to v5.13.15 libc-headers: bump to v5.14 linux-yocto: introduce 5.14 reference kernel systemtap: update to 4.5-latest conf/machine: bump qemu preferred versions to 5.14 poky: set default kernel to 5.14 Changqing Li (1): lttng-ust: fix do_compile error when PACKAGECONFIG examples is enabled Chanho Park (1): binutils: inherit pkgconfig to address libdebuginfod depdency Claudius Heine (1): rng-tools: add systemd-udev-settle wants to service Daniel Ammann (1): bitbake: fetch2/wget: Enable ftps Daniel Wagenknecht (2): mirrors.bbclass: provide additional rule for git repo fallbacks mirrors.bbclass: remove redundant server-specific mirrors Denys Dmytriyenko (1): readline: correct pkg-config dependency for termcap Hsia-Jun(Randy) Li (1): cross-canadian: make android pass target sys check Jon Mason (6): Update mailing list address README: update mailing list address dev-manual: update mailing list address core-image-sato: Fix runqemu error for qemuarmv5 machine/qemuarm*: use virtio graphics testimage: remove aarch64 xorg exclusion Joshua Watt (17): Add SPDX licenses classes/package: Add extended packaged data classes/create-spdx: Add class classes/create-spdx: Change creator classes/create-spdx: Add SHA1 to index file classes/create-spdx: Add index to DEPLOYDIR classes/create-spdx: Add runtime dependency mapping classes/create-spdx: Add NOASSERTION for unknown debug sources classes/create-spdx: Fix another creator classes/create-spdx: Fix up license reporting classes/create-spdx: Speed up hash calculations classes/create-spdx: Fix file:// in downloadLocation classes/create-spdx: Add special exception for Public Domain license classes/create-spdx: Collect all task dependencies classes/create-spdx: Skip package processing for native recipes classes/create-spdx: Comment out placeholder license warning bitbake: cooker: Allow upstream for local hash equivalence server Kai Kang (2): perl: fix CVE-2021-36770 rust-common.bbclass: make sure ccache exist Kevin Hao (1): meta-yocto-bsp: Update the default kernel to v5.14 Khem Raj (3): vim: Add packageconfig for sound notification support site: Drop caching libIDL_cv_long_long_format site: Drop ORBit2 relared cached variables Konrad Weihmann (1): expat: pull from github releases Kristian Klausen (3): systemd: Add homed PACKAGECONFIG wic: Add extra-space argument systemd: Add tpm2 PACKAGECONFIG Mark Hatle (3): reproducible_build: Remove BUILD_REPRODUCIBLE_BINARIES checking externalsrc: Work with reproducible_build tcf-agent: Move to the latest master version Markus Volk (1): util-linux: disable raw Martin Jansa (3): default-distrovars.inc: Set BBINCLUDELOGS to empty to disable printing failed task output multiple times bitbake: bitbake.conf: fix vars_from_file() call qemu-native: add direct dependency on ninja-native and meson-native Michael Halstead (1): releases: update to include 3.3.3 Michael Opdenacker (9): dev-manual: explicit that devpyshell is a task bitbake: bitbake-user-manual: replace "file name" by "filename" manuals: replace Freenode by Libera Chat as IRC host manuals: delete unmaintained history sections ref-manual: document UPSTREAM_CHECK_COMMITS and UPSTREAM_VERSION_UNKNOWN ref-manual: remove checkpkg task ref-manual: improve "devtool check-upgrade-status" details ref-manual: improve documentation for RECIPE_NO_UPDATE_REASON ref-manual: update "devtool check-upgrade-status" output Mingli Yu (6): coreutils: add pkgconfig for selinux findutils: add pkgconfig for selinux tar: add pkgconfig for selinux multilib.bbclass: add RDEPENDS related check back insane.bbclass: add FILERDEPENDS related check back python3: fix multilib qa issue Peter Bergin (1): systemd: add packageconfig for wheel-group Peter Kjellerstedt (2): common-licenses, licenses.conf: Remove duplicate licenses create-spdx.bbclass: Search all license directories for licenses Quentin Schulz (3): bitbake: doc: bitbake-user-manual-execution: remove mention to long-gone BBHASHDEPS variable conf/mips: mips16e: prepend override to MACHINEOVERRIDES bitbake: doc: bitbake-user-manual-fetching: S should be set to WORKDIR/git for git fetcher Randy MacLeod (1): tcmode-default: add rust to the default toolchains Ranjitsinh Rathod (1): rpm: Handle proper return value to avoid major issues Richard Purdie (67): oeqa/runtime/parselogs: Make DVD ata error apply to all qemux86 machines tcl: Exclude CVE-2021-35331 from checks xdg-utils: Add fix for CVE-2020-27748 build-appliance-image: Update to master head revision utils: Drop unused variable staging_install from oe_libinstall utils: Drop obsolete oe_machinstall function flex: Add CVE-2019-6293 to exclusions for checks go: Exclude CVE-2021-29923 from report list bitbake: runqueue: Avoid deadlock avoidance task graph corruption bitbake: runqueue: Fix issues with multiconfig deferred task deadlock messages oeqa/oescripts: Fix after tar recipe changes pseudo: Update with fcntl and glibc 2.34 fixes bitbake: persist_data: Drop deprecated/unused function bitbake: parse_py: Drop deprecated function reference bitbake: build: Match markup to real function name bitbake: build: Handle SystemExit in python tasks correctly bitbake: process: Don't include logs in error message if piping them bitbake: build: Avoid duplicating logs in verbose mode bitbake: data_smart: Make ExpansionErrors more readable bitbake: build: Catch and error upon circular task references bitbake: data_smart: Improve error display for handled exceptions bitbake: fetch2: Add recursion guard bitbake: cookerdata: Improve missing core layer error message bitbake: cookerdata: Show error for no BBLAYERS in bblayers.conf bitbake: runqueue: Clean up task stats handling Revert "default-distrovars.inc: Set BBINCLUDELOGS to empty to disable printing failed task output multiple times" bitbake.conf: Ensure XZ_THREADS doesn't change sstate checksums sstate: Avoid problems with recipes using SRCPV when fetching sstate local.conf.sample: Update sstate mirror entry with new hash equivalence setting useradd: Ensure preinst data is expanded correctly in pkgdata package: Fix pkgdata determinism issues sstate: Ensure SDE is accounted for in package task timestamps bash: Ensure deterministic build sstatesig: Allow exclusion of the root directory for do_package bitbake: bitbake-worker: Improve error handling bitbake: runqueue/knotty: Improve UI handling of setscene task counting bitbake: fetch2/git: Avoid races over mirror tarball creation README: Update email address for Bruce bitbake: cookerdata: Show a readable error for invalid multiconfig name bitbake: fetch2/git: Use os.rename instead of mv bitbake: tests/fetch2: Fix quoting warning bitbake: data_smart: Don't add None to ExpansionError varlist bitbake: fetch2/svn: Allow peg-revision functionality to be disabled vim: Backport fix for CVE-2021-3770 libgcrypt: Upgrade 1.9.3 -> 1.9.4 sqlite3: Exclude CVE-2021-36690 from cve checks recipes: Add missing pkgconfig inherit lttng-tools: Add missing DEPENDS on bison-native cross: Drop unused do_install pybootchart: Avoid divide by zero bitbake: tests/fetch2: Use our own git server for dtc test repo scripts/oe-publish-sdk: Disable git gc to avoid build errors image/qemu: Add explict depends for qemu-helper addto_recipe_sysroot task siteinfo/autotools: Ensure task checksums reflect site files package_ipk/deb/rpm: Drop recursive do_build task dependencies reproducible_build/package_XXX: Ensure SDE task is in dependency chain populate_sdk_base/images: Drop use of 'meta' class and hence do_build dependencies buildtools-tarball/uninative-tarball/meta-ide-support: Drop useless meta class meta: Drop useless class staging: Mark deploy an sstate task sstate: Ensure deploy tasks don't pull in toolchains sstate: Avoid deploy_source_date_epoch sstate when unneeded ssate: Cleanup directtasks handling bitbake: build: Ensure python stdout/stderr is logged correctly bitbake: build: Make exception printing clearer bitbake: build: Fix log flushing race oeqa/selftest: Add tests for bitbake shell/python task output Robert P. J. Day (16): dev-manual: pass False to d.getVar() for devpyshell example ref-manual: add missing "${PN}-src" to default PACKAGES list dev-manual: small number of minor aesthetic tweaks dev-manual: various pedantic nitpickery dev-manual: drop "three" since there are four requirements ref-manual: update SYSROOT_DIRS_* variable entries README: update manual list and names, online docs URL image_types_wic.bbclass: alphabetize list of WICVARS systemd: '${systemd_unitdir}/system' => '${systemd_system_unitdir}' ref-manual: render options in monospace to show quotes properly ref-manual: remove mention of obsolete devtool "--any-recipe" option ref-manual: correct typo in "classes" section, "${BPN}/{PV}" ref-manual: add potential of parallelism to defn of "Task" ref-manual: couple minor tweaks to Chapter 1 dev-manual: emphasize that new layers live outside of poky dev-manual: update output of "wic list images" Robert Yang (1): assimp: Remove it Ross Burton (40): lz4: remove redundant BSD license python3-numpy: remove redundant BSD license quota: remove BSD license nfs-utils: set precise BSD license dtc: set precise BSD license acpica: set precise BSD license libevent: set precise BSD license openssh: remove redundant BSD license python3-packaging: fix license statement iputils: set precise BSD license libx11-compose-data: set precise BSD license webkitgtk: set precise BSD license libwpe: set precise BSD license wpebackend-fdo: set precise BSD license common-licenses: add missing SPDX licences dev-manual/common-tasks: sync libxpm fragment with the recipe lsof: correct LICENSE selftest/python-async-test: set precise BSD license lsof: add upstream check xinetd: correct LICENSE oeqa/recipeutils: update for license change to python-async-test libxfont: set precise BSD license valgrind: set precise BSD license shadow-sysroot: sync license with shadow ovmf: set precise BSD license ppp: set precise BSD license ffmpeg: update LICENSE hdparm: set correct license recipetool/create_buildsys_python: treat BSD as BSD-3-Clause oeqa/selftest/recipetool: update for license changes create-spdx: transform license list into a dict for faster lookups create-spdx: remove redundant test create-spdx: embed unknown license texts create-spdx: don't duplicate license texts in each package create-spdx: handle CLOSED license ffmpeg: fix LICENSE avahi: remove obsolete intltool-native dependency shared-mime-info: use a more concise description libsoup-2.4: remove obsolete intltool dependency oeqa/target/ssh: don't assume target_dumper is set Sakib Sajal (1): go: upgrade 1.16.5 -> 1.16.7 Saul Wold (2): classes/create-spdx: extend DocumentRef to include name create-spdx: remove trailing comma Scott Weaver (3): bitbake: bitbake: fetch2: fix premirror URI when downloadfilename defined bitbake: bitbake: tests/fetch: add downloadfilename tests bitbake: bitbake: tests/fetch: add and fix npm tests Steve Sakoman (1): connman: add CVE_PRODUCT Tom Rini (1): common-tasks: Add an example of using bbappends to add a file Trevor Woerner (1): hello-mod/hello.c: convert to module_init/module_exit Valentin Danaila (1): bitbake: fetch2/s3: allow to switch profile from environment variable Vyacheslav Yurkov (1): ref-manual: add overlayfs class Signed-off-by: Andrew Geissler Change-Id: I194b13991cbaac7ae9e20cc2b552b508ab879905 --- poky/bitbake/bin/bitbake | 2 + poky/bitbake/bin/bitbake-diffsigs | 1 + poky/bitbake/bin/bitbake-getvar | 2 + poky/bitbake/bin/bitbake-hashclient | 2 + poky/bitbake/bin/bitbake-hashserv | 2 + poky/bitbake/bin/bitbake-layers | 2 + poky/bitbake/bin/bitbake-prserv | 2 + poky/bitbake/bin/bitbake-selftest | 2 + poky/bitbake/bin/bitbake-server | 1 + poky/bitbake/bin/bitbake-worker | 9 ++- poky/bitbake/bin/git-make-shallow | 2 + poky/bitbake/bin/toaster-eventreplay | 2 + .../bitbake-user-manual-execution.rst | 7 +- .../bitbake-user-manual-fetching.rst | 7 ++ .../bitbake-user-manual-hello.rst | 2 +- .../bitbake-user-manual-metadata.rst | 6 +- .../bitbake-user-manual-ref-variables.rst | 4 +- poky/bitbake/lib/bb/build.py | 82 ++++++++++++---------- poky/bitbake/lib/bb/cache.py | 3 +- poky/bitbake/lib/bb/cooker.py | 7 +- poky/bitbake/lib/bb/cookerdata.py | 7 ++ poky/bitbake/lib/bb/data_smart.py | 18 +++-- poky/bitbake/lib/bb/fetch2/__init__.py | 10 ++- poky/bitbake/lib/bb/fetch2/git.py | 21 +++++- poky/bitbake/lib/bb/fetch2/svn.py | 10 ++- poky/bitbake/lib/bb/fetch2/wget.py | 2 +- poky/bitbake/lib/bb/main.py | 7 -- poky/bitbake/lib/bb/parse/parse_py/BBHandler.py | 3 - poky/bitbake/lib/bb/persist_data.py | 55 +-------------- poky/bitbake/lib/bb/process.py | 3 + poky/bitbake/lib/bb/runqueue.py | 52 +++++++++----- poky/bitbake/lib/bb/server/process.py | 2 +- poky/bitbake/lib/bb/tests/fetch.py | 61 +++++++++++++--- poky/bitbake/lib/bb/ui/knotty.py | 2 +- poky/bitbake/lib/bb/ui/uihelper.py | 4 +- poky/bitbake/lib/bblayers/query.py | 6 +- poky/bitbake/lib/layerindexlib/__init__.py | 1 - 37 files changed, 251 insertions(+), 160 deletions(-) (limited to 'poky/bitbake') diff --git a/poky/bitbake/bin/bitbake b/poky/bitbake/bin/bitbake index dc1873af6..d3ee8e946 100755 --- a/poky/bitbake/bin/bitbake +++ b/poky/bitbake/bin/bitbake @@ -12,6 +12,8 @@ import os import sys +import warnings +warnings.simplefilter("default") sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) diff --git a/poky/bitbake/bin/bitbake-diffsigs b/poky/bitbake/bin/bitbake-diffsigs index 19420a2df..6646dccdf 100755 --- a/poky/bitbake/bin/bitbake-diffsigs +++ b/poky/bitbake/bin/bitbake-diffsigs @@ -11,6 +11,7 @@ import os import sys import warnings +warnings.simplefilter("default") import argparse import logging import pickle diff --git a/poky/bitbake/bin/bitbake-getvar b/poky/bitbake/bin/bitbake-getvar index 942321925..5435a8d79 100755 --- a/poky/bitbake/bin/bitbake-getvar +++ b/poky/bitbake/bin/bitbake-getvar @@ -9,6 +9,8 @@ import argparse import io import os import sys +import warnings +warnings.simplefilter("default") bindir = os.path.dirname(__file__) topdir = os.path.dirname(bindir) diff --git a/poky/bitbake/bin/bitbake-hashclient b/poky/bitbake/bin/bitbake-hashclient index a89290217..494f17592 100755 --- a/poky/bitbake/bin/bitbake-hashclient +++ b/poky/bitbake/bin/bitbake-hashclient @@ -13,6 +13,8 @@ import pprint import sys import threading import time +import warnings +warnings.simplefilter("default") try: import tqdm diff --git a/poky/bitbake/bin/bitbake-hashserv b/poky/bitbake/bin/bitbake-hashserv index 153f65a37..00af76b2d 100755 --- a/poky/bitbake/bin/bitbake-hashserv +++ b/poky/bitbake/bin/bitbake-hashserv @@ -10,6 +10,8 @@ import sys import logging import argparse import sqlite3 +import warnings +warnings.simplefilter("default") sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) diff --git a/poky/bitbake/bin/bitbake-layers b/poky/bitbake/bin/bitbake-layers index ff085d674..449434d46 100755 --- a/poky/bitbake/bin/bitbake-layers +++ b/poky/bitbake/bin/bitbake-layers @@ -14,6 +14,8 @@ import logging import os import sys import argparse +import warnings +warnings.simplefilter("default") bindir = os.path.dirname(__file__) topdir = os.path.dirname(bindir) diff --git a/poky/bitbake/bin/bitbake-prserv b/poky/bitbake/bin/bitbake-prserv index bef5ef689..323df66dd 100755 --- a/poky/bitbake/bin/bitbake-prserv +++ b/poky/bitbake/bin/bitbake-prserv @@ -6,6 +6,8 @@ import os import sys,logging import optparse +import warnings +warnings.simplefilter("default") sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) diff --git a/poky/bitbake/bin/bitbake-selftest b/poky/bitbake/bin/bitbake-selftest index aec470692..f25f23b1a 100755 --- a/poky/bitbake/bin/bitbake-selftest +++ b/poky/bitbake/bin/bitbake-selftest @@ -7,6 +7,8 @@ import os import sys, logging +import warnings +warnings.simplefilter("default") sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) import unittest diff --git a/poky/bitbake/bin/bitbake-server b/poky/bitbake/bin/bitbake-server index 8e5394736..f53f88b6b 100755 --- a/poky/bitbake/bin/bitbake-server +++ b/poky/bitbake/bin/bitbake-server @@ -8,6 +8,7 @@ import os import sys import warnings +warnings.simplefilter("default") import logging sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) diff --git a/poky/bitbake/bin/bitbake-worker b/poky/bitbake/bin/bitbake-worker index 7d982f90b..6ead2da6d 100755 --- a/poky/bitbake/bin/bitbake-worker +++ b/poky/bitbake/bin/bitbake-worker @@ -6,6 +6,7 @@ import os import sys import warnings +warnings.simplefilter("default") sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) from bb import fetch2 import logging @@ -513,9 +514,11 @@ except BaseException as e: import traceback sys.stderr.write(traceback.format_exc()) sys.stderr.write(str(e)) - -worker_thread_exit = True -worker_thread.join() +finally: + worker_thread_exit = True + worker_thread.join() workerlog_write("exiting") +if not normalexit: + sys.exit(1) sys.exit(0) diff --git a/poky/bitbake/bin/git-make-shallow b/poky/bitbake/bin/git-make-shallow index 57069f7ed..1d00fbf18 100755 --- a/poky/bitbake/bin/git-make-shallow +++ b/poky/bitbake/bin/git-make-shallow @@ -16,6 +16,8 @@ import itertools import os import subprocess import sys +import warnings +warnings.simplefilter("default") version = 1.0 diff --git a/poky/bitbake/bin/toaster-eventreplay b/poky/bitbake/bin/toaster-eventreplay index 8fa4ab711..404b61f51 100755 --- a/poky/bitbake/bin/toaster-eventreplay +++ b/poky/bitbake/bin/toaster-eventreplay @@ -19,6 +19,8 @@ import sys import json import pickle import codecs +import warnings +warnings.simplefilter("default") from collections import namedtuple diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst index a6ef90db1..a4b1efbe8 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst @@ -208,8 +208,8 @@ metadata. For example, in ``bitbake.conf`` the recipe name and version are used to set the variables :term:`PN` and :term:`PV`:: - PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" - PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" In this example, a recipe called "something_1.2.3.bb" would set :term:`PN` to "something" and :term:`PV` to "1.2.3". @@ -578,9 +578,6 @@ the build. This information includes: - ``BB_BASEHASH_``\ *filename:taskname*: The base hashes for each dependent task. -- ``BBHASHDEPS_``\ *filename:taskname*: The task dependencies for - each task. - - :term:`BB_TASKHASH`: The hash of the currently running task. It is worth noting that BitBake's "-S" option lets you debug BitBake's diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 40b245b6d..a944d0f78 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -439,6 +439,13 @@ Here are some example URLs:: SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1" SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http" +.. note:: + + When using ``git`` as the fetcher of the main source code of your software, + ``S`` should be set accordingly:: + + S = "${WORKDIR}/git" + .. note:: Specifying passwords directly in ``git://`` urls is not supported. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst index c5a4ce60b..83a415d99 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst @@ -216,7 +216,7 @@ Following is the complete "Hello World" example. use some editor to create the ``bitbake.conf`` so that it contains the following:: - PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" TMPDIR = "${TOPDIR}/tmp" CACHE = "${TMPDIR}/cache" diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index b0494d080..119720d52 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -413,8 +413,8 @@ variables from BitBake's internal data dictionary, ``d``. The following lines select the values of a package name and its version number, respectively:: - PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" - PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" .. note:: @@ -1941,7 +1941,7 @@ Wildcard Support in Variables ============================= Support for wildcard use in variables varies depending on the context in -which it is used. For example, some variables and file names allow +which it is used. For example, some variables and filenames allow limited use of wildcards through the "``%``" and "``*``" characters. Other variables or names support Python's `glob `_ syntax, diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 2392ec425..8862e16b8 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -339,7 +339,7 @@ overview of their function and contents. :term:`BB_LOGFMT` Specifies the name of the log files saved into ``${``\ :term:`T`\ ``}``. By default, the :term:`BB_LOGFMT` - variable is undefined and the log file names get created using the + variable is undefined and the log filenames get created using the following form:: log.{task}.{pid} @@ -393,7 +393,7 @@ overview of their function and contents. :term:`BB_RUNFMT` Specifies the name of the executable script files (i.e. run files) saved into ``${``\ :term:`T`\ ``}``. By default, the - :term:`BB_RUNFMT` variable is undefined and the run file names get + :term:`BB_RUNFMT` variable is undefined and the run filenames get created using the following form:: run.{task}.{pid} diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index 6ce8f1e6d..7e4ab9f64 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -295,9 +295,13 @@ def exec_func_python(func, d, runfile, cwd=None): lineno = int(d.getVarFlag(func, "lineno", False)) bb.methodpool.insert_method(func, text, fn, lineno - 1) - comp = utils.better_compile(code, func, "exec_python_func() autogenerated") - utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated") + comp = utils.better_compile(code, func, "exec_func_python() autogenerated") + utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") finally: + # We want any stdout/stderr to be printed before any other log messages to make debugging + # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. + sys.stdout.flush() + sys.stderr.flush() bb.debug(2, "Python function %s finished" % func) if cwd and olddir: @@ -682,47 +686,51 @@ def _exec_task(fn, task, d, quieterr): try: try: event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) - except (bb.BBHandledException, SystemExit): - return 1 - try: for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) for func in (postfuncs or '').split(): exec_func(func, localdata) - except bb.BBHandledException: - event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata) - return 1 - except Exception as exc: - if quieterr: - event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) - else: - errprinted = errchk.triggered - logger.error(str(exc)) - event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) - return 1 - finally: - sys.stdout.flush() - sys.stderr.flush() - - bblogger.removeHandler(handler) - - # Restore the backup fds - os.dup2(osi[0], osi[1]) - os.dup2(oso[0], oso[1]) - os.dup2(ose[0], ose[1]) - - # Close the backup fds - os.close(osi[0]) - os.close(oso[0]) - os.close(ose[0]) + finally: + # Need to flush and close the logs before sending events where the + # UI may try to look at the logs. + sys.stdout.flush() + sys.stderr.flush() + + bblogger.removeHandler(handler) + + # Restore the backup fds + os.dup2(osi[0], osi[1]) + os.dup2(oso[0], oso[1]) + os.dup2(ose[0], ose[1]) + + # Close the backup fds + os.close(osi[0]) + os.close(oso[0]) + os.close(ose[0]) + + logfile.close() + if os.path.exists(logfn) and os.path.getsize(logfn) == 0: + logger.debug2("Zero size logfn %s, removing", logfn) + bb.utils.remove(logfn) + bb.utils.remove(loglink) + except bb.BBHandledException: + event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata) + return 1 + except (Exception, SystemExit) as exc: + if quieterr: + event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) + else: + errprinted = errchk.triggered + # If the output is already on stdout, we've printed the information in the + # logs once already so don't duplicate + if verboseStdoutLogging: + errprinted = True + logger.error(repr(exc)) + event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) + return 1 - logfile.close() - if os.path.exists(logfn) and os.path.getsize(logfn) == 0: - logger.debug2("Zero size logfn %s, removing", logfn) - bb.utils.remove(logfn) - bb.utils.remove(loglink) event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): @@ -1030,6 +1038,8 @@ def tasksbetween(task_start, task_end, d): def follow_chain(task, endtask, chain=None): if not chain: chain = [] + if task in chain: + bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) chain.append(task) for othertask in tasks: if othertask == task: diff --git a/poky/bitbake/lib/bb/cache.py b/poky/bitbake/lib/bb/cache.py index 73bc6e966..4e08c100a 100644 --- a/poky/bitbake/lib/bb/cache.py +++ b/poky/bitbake/lib/bb/cache.py @@ -19,7 +19,8 @@ import os import logging import pickle -from collections import defaultdict, Mapping +from collections import defaultdict +from collections.abc import Mapping import bb.utils from bb import PrefixLoggerAdapter import re diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index db991702e..1f55d9ad7 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -389,7 +389,12 @@ class BBCooker: if not self.hashserv: dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") - self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False) + self.hashserv = hashserv.create_server( + self.hashservaddr, + dbfile, + sync=False, + upstream=self.data.getVar("BB_HASHSERVE_UPSTREAM") or None, + ) self.hashserv.serve_as_process() self.data.setVar("BB_HASHSERVE", self.hashservaddr) self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) diff --git a/poky/bitbake/lib/bb/cookerdata.py b/poky/bitbake/lib/bb/cookerdata.py index 1c1e008c6..ba657c03b 100644 --- a/poky/bitbake/lib/bb/cookerdata.py +++ b/poky/bitbake/lib/bb/cookerdata.py @@ -291,6 +291,8 @@ class CookerDataBuilder(object): multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() for config in multiconfig: + if config[0].isdigit(): + bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config) mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) bb.event.fire(bb.event.ConfigParsed(), mcdata) self.mcdata[config] = mcdata @@ -342,6 +344,9 @@ class CookerDataBuilder(object): layers = (data.getVar('BBLAYERS') or "").split() broken_layers = [] + if not layers: + bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition") + data = bb.data.createCopy(data) approved = bb.utils.approved_variables() @@ -396,6 +401,8 @@ class CookerDataBuilder(object): if c in collections_tmp: bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c) compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split()) + if compat and not layerseries: + bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c) if compat and not (compat & layerseries): bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)" % (c, " ".join(layerseries), " ".join(compat))) diff --git a/poky/bitbake/lib/bb/data_smart.py b/poky/bitbake/lib/bb/data_smart.py index 65528c6ae..8d235da12 100644 --- a/poky/bitbake/lib/bb/data_smart.py +++ b/poky/bitbake/lib/bb/data_smart.py @@ -17,7 +17,7 @@ BitBake build tools. # Based on functions from the base bb module, Copyright 2003 Holger Schurig import copy, re, sys, traceback -from collections import MutableMapping +from collections.abc import MutableMapping import logging import hashlib import bb, bb.codeparser @@ -151,6 +151,7 @@ class ExpansionError(Exception): self.expression = expression self.variablename = varname self.exception = exception + self.varlist = [varname or expression or ""] if varname: if expression: self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) @@ -160,8 +161,14 @@ class ExpansionError(Exception): self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) Exception.__init__(self, self.msg) self.args = (varname, expression, exception) + + def addVar(self, varname): + if varname: + self.varlist.append(varname) + def __str__(self): - return self.msg + chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) + return self.msg + chain class IncludeHistory(object): def __init__(self, parent = None, filename = '[TOP LEVEL]'): @@ -403,14 +410,17 @@ class DataSmart(MutableMapping): s = __expand_python_regexp__.sub(varparse.python_sub, s) except SyntaxError as e: # Likely unmatched brackets, just don't expand the expression - if e.msg != "EOL while scanning string literal": + if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): raise if s == olds: break - except ExpansionError: + except ExpansionError as e: + e.addVar(varname) raise except bb.parse.SkipRecipe: raise + except bb.BBHandledException: + raise except Exception as exc: tb = sys.exc_info()[2] raise ExpansionError(varname, s, exc).with_traceback(tb) from exc diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 914fa5c02..ee29d89b1 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -466,7 +466,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): # Kill parameters, they make no sense for mirror tarballs uri_decoded[5] = {} elif ud.localpath and ud.method.supports_checksum(ud): - basename = os.path.basename(ud.localpath) + basename = os.path.basename(uri_decoded[loc]) if basename and not result_decoded[loc].endswith(basename): result_decoded[loc] = os.path.join(result_decoded[loc], basename) else: @@ -754,6 +754,11 @@ def get_srcrev(d, method_name='sortable_revision'): that fetcher provides a method with the given name and the same signature as sortable_revision. """ + recursion = d.getVar("__BBINSRCREV") + if recursion: + raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") + d.setVar("__BBINSRCREV", True) + scms = [] fetcher = Fetch(d.getVar('SRC_URI').split(), d) urldata = fetcher.ud @@ -768,6 +773,7 @@ def get_srcrev(d, method_name='sortable_revision'): autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) if len(rev) > 10: rev = rev[:10] + d.delVar("__BBINSRCREV") if autoinc: return "AUTOINC+" + rev return rev @@ -802,6 +808,7 @@ def get_srcrev(d, method_name='sortable_revision'): if seenautoinc: format = "AUTOINC+" + format + d.delVar("__BBINSRCREV") return format def localpath(url, d): @@ -827,6 +834,7 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH', 'DBUS_SESSION_BUS_ADDRESS', 'P4CONFIG', 'SSL_CERT_FILE', + 'AWS_PROFILE', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_DEFAULT_REGION'] diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 488f4c741..e8ddf2c76 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -68,6 +68,7 @@ import subprocess import tempfile import bb import bb.progress +from contextlib import contextmanager from bb.fetch2 import FetchMethod from bb.fetch2 import runfetchcmd from bb.fetch2 import logger @@ -418,6 +419,20 @@ class Git(FetchMethod): bb.utils.remove(tmpdir, recurse=True) def build_mirror_data(self, ud, d): + + # Create as a temp file and move atomically into position to avoid races + @contextmanager + def create_atomic(filename): + fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) + try: + yield tfile + umask = os.umask(0o666) + os.umask(umask) + os.chmod(tfile, (0o666 & ~umask)) + os.rename(tfile, filename) + finally: + os.close(fd) + if ud.shallow and ud.write_shallow_tarballs: if not os.path.exists(ud.fullshallow): if os.path.islink(ud.fullshallow): @@ -428,7 +443,8 @@ class Git(FetchMethod): self.clone_shallow_local(ud, shallowclone, d) logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) + with create_atomic(ud.fullshallow) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) runfetchcmd("touch %s.done" % ud.fullshallow, d) finally: bb.utils.remove(tempdir, recurse=True) @@ -437,7 +453,8 @@ class Git(FetchMethod): os.unlink(ud.fullmirror) logger.info("Creating tarball of git repository") - runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) + with create_atomic(ud.fullmirror) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir) runfetchcmd("touch %s.done" % ud.fullmirror, d) def clone_shallow_local(self, ud, dest, d): diff --git a/poky/bitbake/lib/bb/fetch2/svn.py b/poky/bitbake/lib/bb/fetch2/svn.py index 80102b44f..d40e4d290 100644 --- a/poky/bitbake/lib/bb/fetch2/svn.py +++ b/poky/bitbake/lib/bb/fetch2/svn.py @@ -57,7 +57,12 @@ class Svn(FetchMethod): if 'rev' in ud.parm: ud.revision = ud.parm['rev'] - ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision)) + # Whether to use the @REV peg-revision syntax in the svn command or not + ud.pegrevision = True + if 'nopegrevision' in ud.parm: + ud.pegrevision = False + + ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision])) def _buildsvncommand(self, ud, d, command): """ @@ -98,7 +103,8 @@ class Svn(FetchMethod): if ud.revision: options.append("-r %s" % ud.revision) - suffix = "@%s" % (ud.revision) + if ud.pegrevision: + suffix = "@%s" % (ud.revision) if command == "fetch": transportuser = ud.parm.get("transportuser", "") diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index 9a49e64a0..349891e85 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -69,7 +69,7 @@ class Wget(FetchMethod): """ Check to see if a given url can be fetched with wget. """ - return ud.type in ['http', 'https', 'ftp'] + return ud.type in ['http', 'https', 'ftp', 'ftps'] def recommends_checksum(self, urldata): return True diff --git a/poky/bitbake/lib/bb/main.py b/poky/bitbake/lib/bb/main.py index 06bad495a..639fc1885 100755 --- a/poky/bitbake/lib/bb/main.py +++ b/poky/bitbake/lib/bb/main.py @@ -112,13 +112,6 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): warnlog.warning(s) warnings.showwarning = _showwarning -warnings.filterwarnings("ignore") -warnings.filterwarnings("default", module="($|(oe|bb)\.)") -warnings.filterwarnings("ignore", category=PendingDeprecationWarning) -warnings.filterwarnings("ignore", category=ImportWarning) -warnings.filterwarnings("ignore", category=DeprecationWarning, module="$") -warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers") - def create_bitbake_parser(): parser = optparse.OptionParser( diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index 152ef6ab7..ee9bd760c 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -19,9 +19,6 @@ from . import ConfHandler from .. import resolve_file, ast, logger, ParseError from .ConfHandler import include, init -# For compatibility -bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) - __func_start_regexp__ = re.compile(r"(((?Ppython(?=(\s|\()))|(?Pfakeroot(?=\s)))\s*)*(?P[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) diff --git a/poky/bitbake/lib/bb/persist_data.py b/poky/bitbake/lib/bb/persist_data.py index c6a209fb3..49c9a0d51 100644 --- a/poky/bitbake/lib/bb/persist_data.py +++ b/poky/bitbake/lib/bb/persist_data.py @@ -12,14 +12,14 @@ currently, providing a key/value store accessed by 'domain'. # import collections +import collections.abc import contextlib import functools import logging import os.path import sqlite3 import sys -import warnings -from collections import Mapping +from collections.abc import Mapping sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): @@ -29,7 +29,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): logger = logging.getLogger("BitBake.PersistData") @functools.total_ordering -class SQLTable(collections.MutableMapping): +class SQLTable(collections.abc.MutableMapping): class _Decorators(object): @staticmethod def retry(*, reconnect=True): @@ -238,55 +238,6 @@ class SQLTable(collections.MutableMapping): def has_key(self, key): return key in self - -class PersistData(object): - """Deprecated representation of the bitbake persistent data store""" - def __init__(self, d): - warnings.warn("Use of PersistData is deprecated. Please use " - "persist(domain, d) instead.", - category=DeprecationWarning, - stacklevel=2) - - self.data = persist(d) - logger.debug("Using '%s' as the persistent data cache", - self.data.filename) - - def addDomain(self, domain): - """ - Add a domain (pending deprecation) - """ - return self.data[domain] - - def delDomain(self, domain): - """ - Removes a domain and all the data it contains - """ - del self.data[domain] - - def getKeyValues(self, domain): - """ - Return a list of key + value pairs for a domain - """ - return list(self.data[domain].items()) - - def getValue(self, domain, key): - """ - Return the value of a key for a domain - """ - return self.data[domain][key] - - def setValue(self, domain, key, value): - """ - Sets the value of a key for a domain - """ - self.data[domain][key] = value - - def delValue(self, domain, key): - """ - Deletes a key/value pair - """ - del self.data[domain][key] - def persist(domain, d): """Convenience factory for SQLTable objects based upon metadata""" import bb.utils diff --git a/poky/bitbake/lib/bb/process.py b/poky/bitbake/lib/bb/process.py index 7c3995cce..d5a1775fc 100644 --- a/poky/bitbake/lib/bb/process.py +++ b/poky/bitbake/lib/bb/process.py @@ -181,5 +181,8 @@ def run(cmd, input=None, log=None, extrafiles=None, **options): stderr = stderr.decode("utf-8") if pipe.returncode != 0: + if log: + # Don't duplicate the output in the exception if logging it + raise ExecutionError(cmd, pipe.returncode, None, None) raise ExecutionError(cmd, pipe.returncode, stdout, stderr) return stdout, stderr diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 25e012125..10511a09d 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -85,15 +85,19 @@ class RunQueueStats: """ Holds statistics on the tasks handled by the associated runQueue """ - def __init__(self, total): + def __init__(self, total, setscene_total): self.completed = 0 self.skipped = 0 self.failed = 0 self.active = 0 + self.setscene_active = 0 + self.setscene_covered = 0 + self.setscene_notcovered = 0 + self.setscene_total = setscene_total self.total = total def copy(self): - obj = self.__class__(self.total) + obj = self.__class__(self.total, self.setscene_total) obj.__dict__.update(self.__dict__) return obj @@ -112,6 +116,13 @@ class RunQueueStats: def taskActive(self): self.active = self.active + 1 + def updateCovered(self, covered, notcovered): + self.setscene_covered = covered + self.setscene_notcovered = notcovered + + def updateActiveSetscene(self, active): + self.setscene_active = active + # These values indicate the next step due to be run in the # runQueue state machine runQueuePrepare = 2 @@ -1735,8 +1746,7 @@ class RunQueueExecute: self.holdoff_need_update = True self.sqdone = False - self.stats = RunQueueStats(len(self.rqdata.runtaskentries)) - self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids)) + self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids)) for mc in rq.worker: rq.worker[mc].pipe.setrunqueueexec(self) @@ -1787,6 +1797,7 @@ class RunQueueExecute: else: self.sq_task_complete(task) self.sq_live.remove(task) + self.stats.updateActiveSetscene(len(self.sq_live)) else: if status != 0: self.task_fail(task, status, fakerootlog=fakerootlog) @@ -1820,7 +1831,7 @@ class RunQueueExecute: def finish(self): self.rq.state = runQueueCleanUp - active = self.stats.active + self.sq_stats.active + active = self.stats.active + len(self.sq_live) if active > 0: bb.event.fire(runQueueExitWait(active), self.cfgData) self.rq.read_workers() @@ -1853,7 +1864,7 @@ class RunQueueExecute: return valid def can_start_task(self): - active = self.stats.active + self.sq_stats.active + active = self.stats.active + len(self.sq_live) can_start = active < self.number_tasks return can_start @@ -1904,6 +1915,12 @@ class RunQueueExecute: self.setbuildable(revdep) logger.debug("Marking task %s as buildable", revdep) + for t in self.sq_deferred.copy(): + if self.sq_deferred[t] == task: + logger.debug2("Deferred task %s now buildable" % t) + del self.sq_deferred[t] + update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) + def task_complete(self, task): self.stats.taskCompleted() bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) @@ -1950,7 +1967,7 @@ class RunQueueExecute: err = False if not self.sqdone: logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered))) - completeevent = sceneQueueComplete(self.sq_stats, self.rq) + completeevent = sceneQueueComplete(self.stats, self.rq) bb.event.fire(completeevent, self.cfgData) if self.sq_deferred: logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred)) @@ -2061,7 +2078,7 @@ class RunQueueExecute: self.sq_task_failoutright(task) return True - startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq) + startevent = sceneQueueTaskStarted(task, self.stats, self.rq) bb.event.fire(startevent, self.cfgData) taskdepdata = self.sq_build_taskdepdata(task) @@ -2082,7 +2099,7 @@ class RunQueueExecute: self.build_stamps2.append(self.build_stamps[task]) self.sq_running.add(task) self.sq_live.add(task) - self.sq_stats.taskActive() + self.stats.updateActiveSetscene(len(self.sq_live)) if self.can_start_task(): return True @@ -2172,7 +2189,7 @@ class RunQueueExecute: if self.can_start_task(): return True - if self.stats.active > 0 or self.sq_stats.active > 0: + if self.stats.active > 0 or len(self.sq_live) > 0: self.rq.read_workers() return self.rq.active_fds() @@ -2180,7 +2197,8 @@ class RunQueueExecute: if self.sq_deferred: tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0]) logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid) - self.sq_task_failoutright(tid) + if tid not in self.runq_complete: + self.sq_task_failoutright(tid) return True if len(self.failed_tids) != 0: @@ -2457,6 +2475,7 @@ class RunQueueExecute: self.sq_task_failoutright(tid) if changed: + self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) self.holdoff_need_update = True def scenequeue_updatecounters(self, task, fail=False): @@ -2490,6 +2509,7 @@ class RunQueueExecute: new.add(dep) next = new + self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) self.holdoff_need_update = True def sq_task_completeoutright(self, task): @@ -2513,13 +2533,11 @@ class RunQueueExecute: self.rq.state = runQueueCleanUp def sq_task_complete(self, task): - self.sq_stats.taskCompleted() - bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData) + bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) self.sq_task_completeoutright(task) def sq_task_fail(self, task, result): - self.sq_stats.taskFailed() - bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData) + bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData) self.scenequeue_notcovered.add(task) self.scenequeue_updatecounters(task, True) self.sq_check_taskfail(task) @@ -2527,8 +2545,6 @@ class RunQueueExecute: def sq_task_failoutright(self, task): self.sq_running.add(task) self.sq_buildable.add(task) - self.sq_stats.taskSkipped() - self.sq_stats.taskCompleted() self.scenequeue_notcovered.add(task) self.scenequeue_updatecounters(task, True) @@ -2536,8 +2552,6 @@ class RunQueueExecute: self.sq_running.add(task) self.sq_buildable.add(task) self.sq_task_completeoutright(task) - self.sq_stats.taskSkipped() - self.sq_stats.taskCompleted() def sq_build_taskdepdata(self, task): def getsetscenedeps(tid): diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index b593830cc..8fdcc66dc 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -661,7 +661,7 @@ class BBUIEventQueue: self.reader = ConnectionReader(readfd) self.t = threading.Thread() - self.t.setDaemon(True) + self.t.daemon = True self.t.run = self.startCallbackHandler self.t.start() diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 9291ce4a0..af292a216 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -866,6 +866,27 @@ class FetcherNetworkTest(FetcherTest): fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + @skipIfNoNetwork() + def test_fetch_specify_downloadfilename(self): + fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) + + @skipIfNoNetwork() + def test_fetch_premirror_specify_downloadfilename_regex_uri(self): + self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake/") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) + + @skipIfNoNetwork() + # BZ13039 + def test_fetch_premirror_specify_downloadfilename_specific_uri(self): + self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake http://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) + @skipIfNoNetwork() def gitfetcher(self, url1, url2): def checkrevision(self, fetcher): @@ -1232,7 +1253,7 @@ class FetchLatestVersionTest(FetcherTest): ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") : "1.0", # version pattern "pkg_name-vX.Y.Z" - ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") + ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") : "1.4.0", # combination version pattern ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") @@ -1244,13 +1265,13 @@ class FetchLatestVersionTest(FetcherTest): : "20120614", # packages with a valid UPSTREAM_CHECK_GITTAGREGEX # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing - ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P(\d+\.(\d\.?)*))") + ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P(\d+\.(\d\.?)*))") : "0.4.3", - ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P(([0-9][\.|_]?)+[0-9]))") + ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P(([0-9][\.|_]?)+[0-9]))") : "11.0.0", - ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P((\d+[\.\-_]*)+))") + ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P((\d+[\.\-_]*)+))") : "1.3.59", - ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") + ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") : "3.82+dbg0.9", } @@ -1290,11 +1311,11 @@ class FetchLatestVersionTest(FetcherTest): # # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 # https://github.com/apple/cups/releases - ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?Pcups\-)(?P((\d+[\.\-_]*)+))\-source\.tar\.gz") + ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?Pcups\-)(?P((\d+[\.\-_]*)+))\-source\.tar\.gz") : "2.0.0", # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz # http://ftp.debian.org/debian/pool/main/d/db5.3/ - ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?Pdb5\.3_)(?P\d+(\.\d+)+).+\.orig\.tar\.xz") + ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?Pdb5\.3_)(?P\d+(\.\d+)+).+\.orig\.tar\.xz") : "5.3.10", } @@ -2257,9 +2278,10 @@ class NPMTest(FetcherTest): fetcher.download() self.assertTrue(os.path.exists(ud.localpath)) # Setup the mirror + pkgname = os.path.basename(ud.proxy.urls[0].split(';')[0]) mirrordir = os.path.join(self.tempdir, 'mirror') bb.utils.mkdirhier(mirrordir) - os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + os.replace(ud.localpath, os.path.join(mirrordir, pkgname)) self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') # Fetch again @@ -2267,6 +2289,27 @@ class NPMTest(FetcherTest): fetcher.download() self.assertTrue(os.path.exists(ud.localpath)) + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_premirrors_with_specified_filename(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) + os.replace(ud.localpath, mirrorfilename) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s\n' % mirrorfilename) + self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') + # Fetch again + self.assertFalse(os.path.exists(ud.localpath)) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + @skipIfNoNpm() @skipIfNoNetwork() def test_npm_mirrors(self): @@ -2329,7 +2372,7 @@ class NPMTest(FetcherTest): @skipIfNoNpm() @skipIfNoNetwork() def test_npm_registry_alternate(self): - url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() fetcher.unpack(self.unpackdir) diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index 65ff2727d..8df745d13 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -753,7 +753,7 @@ def main(server, eventHandler, params, tf = TerminalFilter): continue if isinstance(event, bb.runqueue.sceneQueueTaskStarted): - logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring)) + logger.info("Running setscene task %d of %d (%s)" % (event.stats.setscene_covered + event.stats.setscene_active + event.stats.setscene_notcovered + 1, event.stats.setscene_total, event.taskstring)) continue if isinstance(event, bb.runqueue.runQueueTaskStarted): diff --git a/poky/bitbake/lib/bb/ui/uihelper.py b/poky/bitbake/lib/bb/ui/uihelper.py index 48d808ae2..52fdae3fe 100644 --- a/poky/bitbake/lib/bb/ui/uihelper.py +++ b/poky/bitbake/lib/bb/ui/uihelper.py @@ -49,8 +49,8 @@ class BBUIHelper: tid = event._fn + ":" + event._task removetid(event.pid, tid) self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)}) - elif isinstance(event, bb.runqueue.runQueueTaskStarted): - self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1 + elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted): + self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + event.stats.setscene_active + 1 self.tasknumber_total = event.stats.total self.needUpdate = True elif isinstance(event, bb.build.TaskProgress): diff --git a/poky/bitbake/lib/bblayers/query.py b/poky/bitbake/lib/bblayers/query.py index 6e94c8307..525d4f0d4 100644 --- a/poky/bitbake/lib/bblayers/query.py +++ b/poky/bitbake/lib/bblayers/query.py @@ -441,10 +441,10 @@ NOTE: .bbappend files can impact the dependencies. line = fnfile.readline() # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass - conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$") - inc_re = re.compile(".*\.inc$") + conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$") + inc_re = re.compile(r".*\.inc$") # The "inherit xxx" in .bbclass - bbclass_re = re.compile(".*\.bbclass$") + bbclass_re = re.compile(r".*\.bbclass$") for layerdir in self.bblayers: layername = self.get_layer_name(layerdir) for dirpath, dirnames, filenames in os.walk(layerdir): diff --git a/poky/bitbake/lib/layerindexlib/__init__.py b/poky/bitbake/lib/layerindexlib/__init__.py index 9ca127b9d..3159bf2f6 100644 --- a/poky/bitbake/lib/layerindexlib/__init__.py +++ b/poky/bitbake/lib/layerindexlib/__init__.py @@ -6,7 +6,6 @@ import datetime import logging -import imp import os from collections import OrderedDict -- cgit v1.2.3