summaryrefslogtreecommitdiff
path: root/poky/bitbake
diff options
context:
space:
mode:
authorPatrick Williams <patrick@stwcx.xyz>2024-04-13 00:51:14 +0300
committerPatrick Williams <patrick@stwcx.xyz>2024-04-13 01:10:08 +0300
commit44b3caf2214045a1492e7aa14e808ac6312930e1 (patch)
treebde7c89dc5680df7768224f05287625cfbcc79dc /poky/bitbake
parent03514f1996efa799e50da744818ba331c2e893b6 (diff)
downloadopenbmc-44b3caf2214045a1492e7aa14e808ac6312930e1.tar.xz
subtree updates
poky: 110ee701b3..5950c63d54: Alexander Kanavin (19): ovmf: update 202308 -> 202402 attr: update 2.5.1 -> 2.5.2 dpkg: update 1.22.0 -> 1.22.5 gptfdisk: update 1.0.9 -> 1.0.10 icu: update 74-1 -> 74-2 go-helloworld: update to latest revision libpam: update 1.5.3 -> 1.6.0 libtraceevent: update 1.7.3 -> 1.8.2 mdadm: update 4.2 -> 4.3 npth: update 1.6 -> 1.7 python3-lxml: update 5.0.0 -> 5.1.0 rpm: update 4.19.1 -> 4.19.1.1 ruby: update 3.2.2 -> 3.3.0 tcl: update 8.6.13 -> 8.6.14 texinfo: update 7.0.3 -> 7.1 waffle: update 1.7.2 -> 1.8.0 shadow: update 4.14.2 -> 4.15.0 meta/lib/oe/sstatesig.py: do not error out if sstate files fail on os.stat() scripts/oe-setup-build: write a build environment initialization one-liner into the build directory Bruce Ashfield (1): perf: make bpf asm include arch conditional Chen Qi (2): ovmf: set CVE_STATUS for CVE-2014-8271 ovmf: set CVE_STATUS for a few CVEs Denys Dmytriyenko (3): mtd-utils: upgrade 2.1.6 -> 2.2.0 lzip: upgrade 1.24 -> 1.24.1 wayland-protocols: upgrade 1.33 -> 1.34 Harish Sadineni (1): rust: set CVE_STATUS for CVE-2024-24576 Joao Marcos Costa (6): classes: document new go-vendor class migration updates for 5.0 release-notes updates for 5.0 ref-manual/variables: add new variables for v5.0 release-notes-5.0: add updates release-notes-5.0: mention cmake-qemu.bbclass Joe Slater (2): init-ifupdown: modify interfaces for busybox packagegroup-core-boot: recommend ifupdown Jon Mason (4): acpica: use github for SRC_URI acpica: update to 20240322 release yocto-bsp/linux-yocto-dev: add genericarm64 yocto-bsp/genericarm64: add virtio-gpu Joshua Watt (1): bitbake: siggen: Capture SSL environment for hashserver Julien Stephan (1): devtool: standard: throws appropriate error if source is in detached HEAD Jörg Sommer (1): kernel-dev: join mkdir commands with -p K Sanjay Nayak (1): xorg-xserver-config: Disable screen blanking for qemu images Khem Raj (5): llvm: Upgrade to 18.1.3 bugfix release sanity: Use diff instead of meld by default libseccomp: Fix build when python packageconfig is enabled linux-yocto: Enable nft modules for ptest images mdadm: Fix build with new musl Lee Chee Yang (2): release-notes-4.0.17: reorder CVEs migration-guides: add release notes for 4.3.4 Max Krummenacher (1): perf: add asm include required for v6.9+ Michael Haener (1): iproute2: add bridge package Michael Opdenacker (19): manuals: fix duplicate "stylecheck" target manuals: add initial sphinx-lint support manuals: fix trailing spaces manuals: fix incorrect double backticks migration-guides: add missing opening tag colon release-notes-5.0: documentation highlights manuals: remove tab characters dev-manual/debugging: mention new ``taskexp_ncurses`` option migration-guides: release-notes-5.0: update docs highlights bitbake: prserv: simplify the PRServerClient() interface bitbake: prserv: use double quotes by default bitbake: bitbake-prserv: replace deprecated optparse by argparse bitbake: prserv: use self.logger instead of logger directly bitbake: asyncrpc: include parse_address from hashserv bitbake: prserv: capitalization and spacing improvements bitbake: prserv: add extra requests bitbake: prserv: remove redundant exception handler bitbake: prserv: correct error message bitbake: prserv: remove unnecessary code Mikko Rapeli (1): linux-yocto-dev: remove duplicate DEPENDS Ninette Adhikari (3): oe-build-perf-report: Add apache echarts to make report interactive oe-build-perf-report: Display more than 300 commits and date instead of commit number oe-build-perf-report: Improve report styling and add descriptions Oleh Matiusha (1): nativesdk-gzip: fix reproducibility issues Peter Hoyes (1): u-boot-tools: Package mkeficapsule Peter Marko (2): bitbake.conf: remove comment about oldincludedir systemd: make predictable name mac policy opt-out Quentin Schulz (2): docs: conf.py: properly escape backslashes for latex_elements manuals: refer to new yocto-patches mailing list wherever appropriate Richard Purdie (10): xwayland: Upgrade 23.2.4 -> 23.2.5 curl: Upgrade 8.6.0 -> 8.7.1 nghttp2: Upgrade 1.60.1 -> 1.61.0 pseudo: Update to pull in fchmodat fix bitbake: doc/user-manual: Add BB_LOADFACTOR_MAX bitbake: BBHandler: Handle unclosed functions correctly testimage: Enable runtime 'login' screenshot tests Revert "testimage: Enable runtime 'login' screenshot tests" python3-websockets: Import from meta-python buildtools-tarball: Add python3-websockets Rob Woolley (1): bitbake: wget: Make wget --passive-ftp option conditional on ftp/ftps Ross Burton (2): eudev: update Upstream-Status on netifnames.patch classes/pypi: don't expose PYPI_ARCHIVE_NAME Simone Weiß (1): bitbake: doc: Add section for variable context Wang Mingyu (87): debianutils: upgrade 5.16 -> 5.17 diffoscope: upgrade 259 -> 260 encodings: upgrade 1.0.7 -> 1.1.0 gcr: upgrade 4.2.0 -> 4.2.1 ghostscript: upgrade 10.02.1 -> 10.03.0 libassuan: upgrade 2.5.6 -> 2.5.7 libfontenc: upgrade 1.1.7 -> 1.1.8 libpng: upgrade 1.6.42 -> 1.6.43 libsdl2: upgrade 2.30.0 -> 2.30.1 libxcb: upgrade 1.16 -> 1.16.1 libxcursor: upgrade 1.2.1 -> 1.2.2 libxdmcp: upgrade 1.1.4 -> 1.1.5 mkfontscale: upgrade 1.2.2 -> 1.2.3 pango: upgrade 1.52.0 -> 1.52.1 psmisc: upgrade 23.6 -> 23.7 python3-cython: upgrade 3.0.8 -> 3.0.9 python3-hypothesis: upgrade 6.98.15 -> 6.99.4 python3-importlib-metadata: upgrade 7.0.1 -> 7.0.2 python3-libarchive-c: upgrade 5.0 -> 5.1 python3-pygobject: update 3.46.0 -> 3.48.1 python3-pyopenssl: upgrade 24.0.0 -> 24.1.0 python3-pyparsing: upgrade 3.1.1 -> 3.1.2 python3-pytest-subtests: upgrade 0.11.0 -> 0.12.1 python3-pytest: upgrade 8.0.2 -> 8.1.1 python3-trove-classifiers: upgrade 2024.2.23 -> 2024.3.3 repo: upgrade 2.42 -> 2.44 shaderc: update 2023.8 -> 2024.0 stress-ng: upgrade 0.17.05 -> 0.17.06 xauth: upgrade 1.1.2 -> 1.1.3 xev: update 1.2.5 -> 1.2.6 gnupg: upgrade 2.4.4 -> 2.4.5 adwaita-icon-theme: upgrade 45.0 -> 46.0 at-spi2-core: upgrade 2.50.1 -> 2.52.0 bind: upgrade 9.18.24 -> 9.18.25 createrepo-c: upgrade 1.0.4 -> 1.1.0 enchant2: upgrade 2.6.7 -> 2.6.8 harfbuzz: upgrade 8.3.0 -> 8.3.1 libbsd: upgrade 0.12.1 -> 0.12.2 libcomps: upgrade 0.1.20 -> 0.1.21 libpciaccess: upgrade 0.18 -> 0.18.1 libwpe: upgrade 1.14.2 -> 1.16.0 libxkbcommon: upgrade 1.6.0 -> 1.7.0 libxml2: upgrade 2.12.5 -> 2.12.6 lighttpd: upgrade 1.4.74 -> 1.4.75 openssh: upgrade 9.6p1 -> 9.7p1 python3-hatchling: upgrade 1.21.1 -> 1.22.4 python3-importlib-metadata: upgrade 7.0.2 -> 7.1.0 python3-license-expression: upgrade 30.2.0 -> 30.3.0 python3-markdown: upgrade 3.5.2 -> 3.6 python3-packaging: upgrade 23.2 -> 24.0 python3-pyelftools: upgrade 0.30 -> 0.31 python3-referencing: upgrade 0.33.0 -> 0.34.0 python3-scons: upgrade 4.6.0 -> 4.7.0 python3-setuptools: upgrade 69.1.1 -> 69.2.0 python3-wheel: upgrade 0.42.0 -> 0.43.0 python3-zipp: upgrade 3.17.0 -> 3.18.1 vala: upgrade 0.56.15 -> 0.56.16 wget: upgrade 1.21.4 -> 1.24.5 mesa: upgrade 24.0.2 -> 24.0.3 vulkan: upgrade 1.3.275.0 -> 1.3.280.0 babeltrace2: upgrade 2.0.5 -> 2.0.6 bash-completion: upgrade 2.12.0 -> 2.13.0 btrfs-tools: upgrade 6.7.1 -> 6.8 coreutils: upgrade 9.4 -> 9.5 dnf: upgrade 4.19.0 -> 4.19.2 ell: upgrade 0.63 -> 0.64 enchant2: upgrade 2.6.8 -> 2.6.9 libdnf: upgrade 0.73.0 -> 0.73.1 libical: upgrade 3.0.17 -> 3.0.18 liburi-perl: upgrade 5.27 -> 5.28 libx11: upgrade 1.8.7 -> 1.8.9 libxmlb: upgrade 0.3.15 -> 0.3.17 libxmu: upgrade 1.1.4 -> 1.2.0 lttng-tools: upgrade 2.13.11 -> 2.13.13 man-db: upgrade 2.12.0 -> 2.12.1 mpg123: upgrade 1.32.5 -> 1.32.6 mtdev: upgrade 1.1.6 -> 1.1.7 pkgconf: upgrade 2.1.1 -> 2.2.0 python3-beartype: upgrade 0.17.2 -> 0.18.2 python3-build: upgrade 1.1.1 -> 1.2.1 python3-git: upgrade 3.1.42 -> 3.1.43 python3-pyasn1: upgrade 0.5.1 -> 0.6.0 python3-typing-extensions: upgrade 4.10.0 -> 4.11.0 rsync: upgrade 3.2.7 -> 3.3.0 ttyrun: upgrade 2.31.0 -> 2.32.0 u-boot: upgrade 2024.01 -> 2024.04 xorgproto: upgrade 2023.2 -> 2024.1 Yoann Congal (2): ref-manual: variables: document CVE_DB_INCR_UPDATE_AGE_THRES variable release-notes-5.0: document some cve, strace and qa changes meta-raspberrypi: d072cc8a48..1879cb831f: Max Stepanov (1): rpi-eeprom: Update to support raspberrypi5 machine meta-arm: d9e18ce792..17df9c4ebc: Anusmita Dutta Mazumder (2): arm-bsp/u-boot:corstone1000: add unique guid for fvp and mps3 arm-bsp/tf-m:corstone1000: add unique guid for fvp and mps3 Debbie Martin (1): arm-systemready: Change get_json_result_dir helper Harsimran Singh Tungal (1): corstone1000:arm-bsp/tftf: upgrade tftf version to v2.10 Jon Mason (7): arm-bsp: remove unused recipes arm-bsp: Remove tc1 CI: update to kas 4.3.2 arm/optee-ftpm: update to the latest SHA arm/trusted-firmware-a: update to 2.10.3 release arm/opencsd: update to 1.5.2 arm-bsp/corstone1000: reformat u-boot patches meta-security: 283a773f24..d1522af21d: Armin Kuster (1): README.md: update to new patches mailing list meta-openembedded: a6bcdca5b4..4958bfe013: Alex Kiernan (1): mdns: Upgrade 2200.80.16 -> 2200.100.94.0.2 Beniamin Sandu (5): mbedtls: upgrade 3.5.2 -> 3.6.0 mbedtls: upgrade 2.28.7 -> 2.28.8 unbound: upgrade 1.19.1 -> 1.19.3 libtorrent: remove CVE mention libtorrent-rasterbar: add initial recipe for 2.0.10 Changqing Li (1): nodejs: don't always disable io_uring Dan McGregor (2): dash: correct licence libfido2: new recipe Fathi Boudra (1): composefs: add a new recipe Guðni Már Gilbert (1): python3-ecdsa: upgrade 0.18.0 -> 0.19.0 Khem Raj (17): python3-pydantic-core: Enable benchmark tests python3-pydbus: Fix typo in ptest package name python3-netaddr: Ignore failing tests on musl python3-pydantic: Ignore failing testcases python3-pydantic-core: Skip failing ptests python3-whoosh: Fix an intermittent ptest python3-pyzmq: Fix ptests Revert "libqmi: upgrade 1.34.0 -> 1.35.2" Revert "libmbim: upgrade 1.30.0 -> 1.31.2" nftables: Fix ptest runs python3-flexparser,python3-flexcache: Add recipes python3-pint: Switch to using github SRC_URI libxml++: Delete recipe for 2.42.1 jemalloc: Update to tip of dev branch libteam: Add missing dependencies revealed by ptests oprofile: Fix failing ptests ptest-packagelists-meta-oe: jemalloc and oprofile are passing now Markus Volk (1): dav1d: update 1.4.0 -> 1.4.1 Maxim Perevozchikov (1): nginx: Disable login for www user Peter Kjellerstedt (1): libnice: Update to 0.1.22 Peter Marko (1): syslog-ng: fix build without ipv6 in distro features Randy MacLeod (5): ncftp: Upgrade to 3.2.7 pimd: switch SRC_URI to https tnftp: switch the SRC_URI to https postfix: switch SRC_URI to http libmad: switch links/SRC_URI to https sites Rui Costa (1): avro: add recipe for c++ Tom Geelen (4): python3-casttube: upgrade 0.2.0 -> 0.2.1 python3-sqlalchemy: upgrade 2.0.25 --> 2.0.27 python3-charset-normalizer: add native build option to recipe python3-chromecast: upgrade 13.1.0 -> 14.0.0 Xiangyu Chen (1): libgpiod: fix QA error in ptest RDEPENDS Yi Zhao (2): rocksdb: fix build error for DEBUG_BUILD rocksdb: fix build error for multilib Yongchang Qiao (1): packagegroup-meta-filesystems: Fix utils typo alperak (5): python3-bleak: enable ptest and add missing runtime dependency python3-pillow: Upgrade 10.1.0 -> 10.3.0 and fix ptest python3-flexcache: enable ptest python3-flexparser: enable ptest and add missing runtime dependencies python3-flexcache: add missing runtime dependencies Change-Id: I06aa4dd845848eec6e165878d482977f48422765 Signed-off-by: Patrick Williams <patrick@stwcx.xyz>
Diffstat (limited to 'poky/bitbake')
-rwxr-xr-xpoky/bitbake/bin/bitbake-prserv86
-rw-r--r--poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst91
-rw-r--r--poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst9
-rw-r--r--poky/bitbake/doc/index.rst1
-rw-r--r--poky/bitbake/lib/bb/asyncrpc/client.py23
-rw-r--r--poky/bitbake/lib/bb/fetch2/wget.py5
-rw-r--r--poky/bitbake/lib/bb/parse/parse_py/BBHandler.py8
-rw-r--r--poky/bitbake/lib/bb/siggen.py90
-rw-r--r--poky/bitbake/lib/hashserv/__init__.py27
-rw-r--r--poky/bitbake/lib/prserv/__init__.py6
-rw-r--r--poky/bitbake/lib/prserv/client.py41
-rw-r--r--poky/bitbake/lib/prserv/db.py140
-rw-r--r--poky/bitbake/lib/prserv/serv.py128
13 files changed, 461 insertions, 194 deletions
diff --git a/poky/bitbake/bin/bitbake-prserv b/poky/bitbake/bin/bitbake-prserv
index 5be42f3ce5..ad0a069401 100755
--- a/poky/bitbake/bin/bitbake-prserv
+++ b/poky/bitbake/bin/bitbake-prserv
@@ -7,49 +7,77 @@
import os
import sys,logging
-import optparse
+import argparse
import warnings
warnings.simplefilter("default")
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib"))
import prserv
import prserv.serv
-__version__="1.0.0"
+VERSION = "1.1.0"
-PRHOST_DEFAULT='0.0.0.0'
+PRHOST_DEFAULT="0.0.0.0"
PRPORT_DEFAULT=8585
def main():
- parser = optparse.OptionParser(
- version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
- usage = "%prog < --start | --stop > [options]")
+ parser = argparse.ArgumentParser(
+ description="BitBake PR Server. Version=%s" % VERSION,
+ formatter_class=argparse.RawTextHelpFormatter)
- parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store",
- dest="dbfile", type="string", default="prserv.sqlite3")
- parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store",
- dest="logfile", type="string", default="prserv.log")
- parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
- action = "store", type="string", dest="loglevel", default = "INFO")
- parser.add_option("--start", help="start daemon",
- action="store_true", dest="start")
- parser.add_option("--stop", help="stop daemon",
- action="store_true", dest="stop")
- parser.add_option("--host", help="ip address to bind", action="store",
- dest="host", type="string", default=PRHOST_DEFAULT)
- parser.add_option("--port", help="port number(default: 8585)", action="store",
- dest="port", type="int", default=PRPORT_DEFAULT)
- parser.add_option("-r", "--read-only", help="open database in read-only mode",
- action="store_true")
+ parser.add_argument(
+ "-f",
+ "--file",
+ default="prserv.sqlite3",
+ help="database filename (default: prserv.sqlite3)",
+ )
+ parser.add_argument(
+ "-l",
+ "--log",
+ default="prserv.log",
+ help="log filename(default: prserv.log)",
+ )
+ parser.add_argument(
+ "--loglevel",
+ default="INFO",
+ help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
+ )
+ parser.add_argument(
+ "--start",
+ action="store_true",
+ help="start daemon",
+ )
+ parser.add_argument(
+ "--stop",
+ action="store_true",
+ help="stop daemon",
+ )
+ parser.add_argument(
+ "--host",
+ help="ip address to bind",
+ default=PRHOST_DEFAULT,
+ )
+ parser.add_argument(
+ "--port",
+ type=int,
+ default=PRPORT_DEFAULT,
+ help="port number (default: 8585)",
+ )
+ parser.add_argument(
+ "-r",
+ "--read-only",
+ action="store_true",
+ help="open database in read-only mode",
+ )
- options, args = parser.parse_args(sys.argv)
- prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
+ args = parser.parse_args()
+ prserv.init_logger(os.path.abspath(args.log), args.loglevel)
- if options.start:
- ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile), options.read_only)
- elif options.stop:
- ret=prserv.serv.stop_daemon(options.host, options.port)
+ if args.start:
+ ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only)
+ elif args.stop:
+ ret=prserv.serv.stop_daemon(args.host, args.port)
else:
ret=parser.print_help()
return ret
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst
new file mode 100644
index 0000000000..e9c454ba11
--- /dev/null
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst
@@ -0,0 +1,91 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+================
+Variable Context
+================
+
+|
+
+Variables might only have an impact or can be used in certain contexts. Some
+should only be used in global files like ``.conf``, while others are intended only
+for local files like ``.bb``. This chapter aims to describe some important variable
+contexts.
+
+.. _ref-varcontext-configuration:
+
+BitBake's own configuration
+===========================
+
+Variables starting with ``BB_`` usually configure the behaviour of BitBake itself.
+For example, one could configure:
+
+- System resources, like disk space to be used (:term:`BB_DISKMON_DIRS`),
+ or the number of tasks to be run in parallel by BitBake (:term:`BB_NUMBER_THREADS`).
+
+- How the fetchers shall behave, e.g., :term:`BB_FETCH_PREMIRRORONLY` is used
+ by BitBake to determine if BitBake's fetcher shall search only
+ :term:`PREMIRRORS` for files.
+
+Those variables are usually configured globally.
+
+BitBake configuration
+=====================
+
+There are variables:
+
+- Like :term:`B` or :term:`T`, that are used to specify directories used by
+ BitBake during the build of a particular recipe. Those variables are
+ specified in ``bitbake.conf``. Some, like :term:`B`, are quite often
+ overwritten in recipes.
+
+- Starting with ``FAKEROOT``, to configure how the ``fakeroot`` command is
+ handled. Those are usually set by ``bitbake.conf`` and might get adapted in a
+ ``bbclass``.
+
+- Detailing where BitBake will store and fetch information from, for
+ data reuse between build runs like :term:`CACHE`, :term:`DL_DIR` or
+ :term:`PERSISTENT_DIR`. Those are usually global.
+
+
+Layers and files
+================
+
+Variables starting with ``LAYER`` configure how BitBake handles layers.
+Additionally, variables starting with ``BB`` configure how layers and files are
+handled. For example:
+
+- :term:`LAYERDEPENDS` is used to configure on which layers a given layer
+ depends.
+
+- The configured layers are contained in :term:`BBLAYERS` and files in
+ :term:`BBFILES`.
+
+Those variables are often used in the files ``layer.conf`` and ``bblayers.conf``.
+
+Recipes and packages
+====================
+
+Variables handling recipes and packages can be split into:
+
+- :term:`PN`, :term:`PV` or :term:`PF` for example, contain information about
+ the name or revision of a recipe or package. Usually, the default set in
+ ``bitbake.conf`` is used, but those are from time to time overwritten in
+ recipes.
+
+- :term:`SUMMARY`, :term:`DESCRIPTION`, :term:`LICENSE` or :term:`HOMEPAGE`
+ contain the expected information and should be set specifically for every
+ recipe.
+
+- In recipes, variables are also used to control build and runtime
+ dependencies between recipes/packages with other recipes/packages. The
+ most common should be: :term:`PROVIDES`, :term:`RPROVIDES`, :term:`DEPENDS`,
+ and :term:`RDEPENDS`.
+
+- There are further variables starting with ``SRC`` that specify the sources in
+ a recipe like :term:`SRC_URI` or :term:`SRCDATE`. Those are also usually set
+ in recipes.
+
+- Which version or provider of a recipe should be given preference when
+ multiple recipes would provide the same item, is controlled by variables
+ starting with ``PREFERRED_``. Those are normally set in the configuration
+ files of a ``MACHINE`` or ``DISTRO``.
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
index d3936935cd..899e584f91 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
@@ -432,6 +432,15 @@ overview of their function and contents.
``ConfigParsed`` event can set the variable to trigger the re-parse.
You must be careful to avoid recursive loops with this functionality.
+ :term:`BB_LOADFACTOR_MAX`
+ Setting this to a value will cause BitBake to check the system load
+ average before executing new tasks. If the load average is above the
+ the number of CPUs multipled by this factor, no new task will be started
+ unless there is no task executing. A value of "1.5" has been found to
+ work reasonably. This is helpful for systems which don't have pressure
+ regulation enabled, which is more granular. Pressure values take
+ precedence over loadfactor.
+
:term:`BB_LOGCONFIG`
Specifies the name of a config file that contains the user logging
configuration. See
diff --git a/poky/bitbake/doc/index.rst b/poky/bitbake/doc/index.rst
index 3ff8b1580f..ee1660ac15 100644
--- a/poky/bitbake/doc/index.rst
+++ b/poky/bitbake/doc/index.rst
@@ -13,6 +13,7 @@ BitBake User Manual
bitbake-user-manual/bitbake-user-manual-intro
bitbake-user-manual/bitbake-user-manual-execution
bitbake-user-manual/bitbake-user-manual-metadata
+ bitbake-user-manual/bitbake-user-manual-ref-variables-context
bitbake-user-manual/bitbake-user-manual-fetching
bitbake-user-manual/bitbake-user-manual-ref-variables
bitbake-user-manual/bitbake-user-manual-hello
diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py
index 29a5ab76aa..a350b4fb12 100644
--- a/poky/bitbake/lib/bb/asyncrpc/client.py
+++ b/poky/bitbake/lib/bb/asyncrpc/client.py
@@ -10,11 +10,34 @@ import json
import os
import socket
import sys
+import re
import contextlib
from threading import Thread
from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK
from .exceptions import ConnectionClosedError, InvokeError
+UNIX_PREFIX = "unix://"
+WS_PREFIX = "ws://"
+WSS_PREFIX = "wss://"
+
+ADDR_TYPE_UNIX = 0
+ADDR_TYPE_TCP = 1
+ADDR_TYPE_WS = 2
+
+def parse_address(addr):
+ if addr.startswith(UNIX_PREFIX):
+ return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
+ elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
+ return (ADDR_TYPE_WS, (addr,))
+ else:
+ m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
+ if m is not None:
+ host = m.group("host")
+ port = m.group("port")
+ else:
+ host, port = addr.split(":")
+
+ return (ADDR_TYPE_TCP, (host, int(port)))
class AsyncClient(object):
def __init__(
diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py
index dc025800e6..fbfa6938ac 100644
--- a/poky/bitbake/lib/bb/fetch2/wget.py
+++ b/poky/bitbake/lib/bb/fetch2/wget.py
@@ -87,7 +87,10 @@ class Wget(FetchMethod):
if not ud.localfile:
ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
- self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp"
+ self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+
+ if ud.type == 'ftp' or ud.type == 'ftps':
+ self.basecmd += " --passive-ftp"
if not self.check_certs(d):
self.basecmd += " --no-check-certificate"
diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
index cd1c998f8f..c13e4b9755 100644
--- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -34,6 +34,7 @@ __infunc__ = []
__inpython__ = False
__body__ = []
__classname__ = ""
+__residue__ = []
cached_statements = {}
@@ -80,7 +81,7 @@ def inherit(files, fn, lineno, d, deferred=False):
__inherit_cache = d.getVar('__inherit_cache', False) or []
def get_statements(filename, absolute_filename, base_name):
- global cached_statements
+ global cached_statements, __residue__, __body__
try:
return cached_statements[absolute_filename]
@@ -100,6 +101,11 @@ def get_statements(filename, absolute_filename, base_name):
# add a blank line to close out any python definition
feeder(lineno, "", filename, base_name, statements, eof=True)
+ if __residue__:
+ raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno)
+ if __body__:
+ raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno)
+
if filename.endswith(".bbclass") or filename.endswith(".inc"):
cached_statements[absolute_filename] = statements
return statements
diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py
index 2a0ecf57e1..0421885306 100644
--- a/poky/bitbake/lib/bb/siggen.py
+++ b/poky/bitbake/lib/bb/siggen.py
@@ -15,6 +15,7 @@ import difflib
import simplediff
import json
import types
+from contextlib import contextmanager
import bb.compress.zstd
from bb.checksum import FileChecksumCache
from bb import runqueue
@@ -28,6 +29,14 @@ hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
# The minimum version of the find_siginfo function we need
find_siginfo_minversion = 2
+HASHSERV_ENVVARS = [
+ "SSL_CERT_DIR",
+ "SSL_CERT_FILE",
+ "NO_PROXY",
+ "HTTPS_PROXY",
+ "HTTP_PROXY"
+]
+
def check_siggen_version(siggen):
if not hasattr(siggen, "find_siginfo_version"):
bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
@@ -537,14 +546,23 @@ class SignatureGeneratorUniHashMixIn(object):
self.unihash_exists_cache = set()
self.username = None
self.password = None
+ self.env = {}
+
+ origenv = data.getVar("BB_ORIGENV")
+ for e in HASHSERV_ENVVARS:
+ value = data.getVar(e)
+ if not value and origenv:
+ value = origenv.getVar(e)
+ if value:
+ self.env[e] = value
super().__init__(data)
def get_taskdata(self):
- return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password) + super().get_taskdata()
+ return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata()
def set_taskdata(self, data):
- self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password = data[:6]
- super().set_taskdata(data[6:])
+ self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7]
+ super().set_taskdata(data[7:])
def get_hashserv_creds(self):
if self.username and self.password:
@@ -555,15 +573,30 @@ class SignatureGeneratorUniHashMixIn(object):
return {}
+ @contextmanager
+ def _client_env(self):
+ orig_env = os.environ.copy()
+ try:
+ for k, v in self.env.items():
+ os.environ[k] = v
+
+ yield
+ finally:
+ os.environ = orig_env
+
+ @contextmanager
def client(self):
- if getattr(self, '_client', None) is None:
- self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
- return self._client
+ with self._client_env():
+ if getattr(self, '_client', None) is None:
+ self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+ yield self._client
+ @contextmanager
def client_pool(self):
- if getattr(self, '_client_pool', None) is None:
- self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
- return self._client_pool
+ with self._client_env():
+ if getattr(self, '_client_pool', None) is None:
+ self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
+ yield self._client_pool
def reset(self, data):
self.__close_clients()
@@ -574,12 +607,13 @@ class SignatureGeneratorUniHashMixIn(object):
return super().exit()
def __close_clients(self):
- if getattr(self, '_client', None) is not None:
- self._client.close()
- self._client = None
- if getattr(self, '_client_pool', None) is not None:
- self._client_pool.close()
- self._client_pool = None
+ with self._client_env():
+ if getattr(self, '_client', None) is not None:
+ self._client.close()
+ self._client = None
+ if getattr(self, '_client_pool', None) is not None:
+ self._client_pool.close()
+ self._client_pool = None
def get_stampfile_hash(self, tid):
if tid in self.taskhash:
@@ -650,11 +684,13 @@ class SignatureGeneratorUniHashMixIn(object):
if self.max_parallel <= 1 or len(uncached_query) <= 1:
# No parallelism required. Make the query serially with the single client
- uncached_result = {
- key: self.client().unihash_exists(value) for key, value in uncached_query.items()
- }
+ with self.client() as client:
+ uncached_result = {
+ key: client.unihash_exists(value) for key, value in uncached_query.items()
+ }
else:
- uncached_result = self.client_pool().unihashes_exist(uncached_query)
+ with self.client_pool() as client_pool:
+ uncached_result = client_pool.unihashes_exist(uncached_query)
for key, exists in uncached_result.items():
if exists:
@@ -687,10 +723,12 @@ class SignatureGeneratorUniHashMixIn(object):
if self.max_parallel <= 1 or len(queries) <= 1:
# No parallelism required. Make the query serially with the single client
- for tid, args in queries.items():
- query_result[tid] = self.client().get_unihash(*args)
+ with self.client() as client:
+ for tid, args in queries.items():
+ query_result[tid] = client.get_unihash(*args)
else:
- query_result = self.client_pool().get_unihashes(queries)
+ with self.client_pool() as client_pool:
+ query_result = client_pool.get_unihashes(queries)
for tid, unihash in query_result.items():
# In the absence of being able to discover a unique hash from the
@@ -785,7 +823,9 @@ class SignatureGeneratorUniHashMixIn(object):
if tid in self.extramethod:
method = method + self.extramethod[tid]
- data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
+ with self.client() as client:
+ data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
+
new_unihash = data['unihash']
if new_unihash != unihash:
@@ -816,7 +856,9 @@ class SignatureGeneratorUniHashMixIn(object):
if tid in self.extramethod:
method = method + self.extramethod[tid]
- data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+ with self.client() as client:
+ data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+
hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
if data is None:
diff --git a/poky/bitbake/lib/hashserv/__init__.py b/poky/bitbake/lib/hashserv/__init__.py
index 552a33278f..74367eb6b4 100644
--- a/poky/bitbake/lib/hashserv/__init__.py
+++ b/poky/bitbake/lib/hashserv/__init__.py
@@ -5,39 +5,14 @@
import asyncio
from contextlib import closing
-import re
import itertools
import json
from collections import namedtuple
from urllib.parse import urlparse
-
-UNIX_PREFIX = "unix://"
-WS_PREFIX = "ws://"
-WSS_PREFIX = "wss://"
-
-ADDR_TYPE_UNIX = 0
-ADDR_TYPE_TCP = 1
-ADDR_TYPE_WS = 2
+from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
User = namedtuple("User", ("username", "permissions"))
-
-def parse_address(addr):
- if addr.startswith(UNIX_PREFIX):
- return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
- elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
- return (ADDR_TYPE_WS, (addr,))
- else:
- m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
- if m is not None:
- host = m.group("host")
- port = m.group("port")
- else:
- host, port = addr.split(":")
-
- return (ADDR_TYPE_TCP, (host, int(port)))
-
-
def create_server(
addr,
dbname,
diff --git a/poky/bitbake/lib/prserv/__init__.py b/poky/bitbake/lib/prserv/__init__.py
index 38ced818ad..0e0aa34d0e 100644
--- a/poky/bitbake/lib/prserv/__init__.py
+++ b/poky/bitbake/lib/prserv/__init__.py
@@ -7,13 +7,13 @@
__version__ = "1.0.0"
import os, time
-import sys,logging
+import sys, logging
def init_logger(logfile, loglevel):
numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int):
- raise ValueError('Invalid log level: %s' % loglevel)
- FORMAT = '%(asctime)-15s %(message)s'
+ raise ValueError("Invalid log level: %s" % loglevel)
+ FORMAT = "%(asctime)-15s %(message)s"
logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
class NotFoundError(Exception):
diff --git a/poky/bitbake/lib/prserv/client.py b/poky/bitbake/lib/prserv/client.py
index 6b81356fac..8471ee3046 100644
--- a/poky/bitbake/lib/prserv/client.py
+++ b/poky/bitbake/lib/prserv/client.py
@@ -11,40 +11,61 @@ logger = logging.getLogger("BitBake.PRserv")
class PRAsyncClient(bb.asyncrpc.AsyncClient):
def __init__(self):
- super().__init__('PRSERVICE', '1.0', logger)
+ super().__init__("PRSERVICE", "1.0", logger)
async def getPR(self, version, pkgarch, checksum):
response = await self.invoke(
- {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}}
+ {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}}
)
if response:
- return response['value']
+ return response["value"]
+
+ async def test_pr(self, version, pkgarch, checksum):
+ response = await self.invoke(
+ {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}}
+ )
+ if response:
+ return response["value"]
+
+ async def test_package(self, version, pkgarch):
+ response = await self.invoke(
+ {"test-package": {"version": version, "pkgarch": pkgarch}}
+ )
+ if response:
+ return response["value"]
+
+ async def max_package_pr(self, version, pkgarch):
+ response = await self.invoke(
+ {"max-package-pr": {"version": version, "pkgarch": pkgarch}}
+ )
+ if response:
+ return response["value"]
async def importone(self, version, pkgarch, checksum, value):
response = await self.invoke(
- {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}}
+ {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}}
)
if response:
- return response['value']
+ return response["value"]
async def export(self, version, pkgarch, checksum, colinfo):
response = await self.invoke(
- {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}}
+ {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}}
)
if response:
- return (response['metainfo'], response['datainfo'])
+ return (response["metainfo"], response["datainfo"])
async def is_readonly(self):
response = await self.invoke(
- {'is-readonly': {}}
+ {"is-readonly": {}}
)
if response:
- return response['readonly']
+ return response["readonly"]
class PRClient(bb.asyncrpc.Client):
def __init__(self):
super().__init__()
- self._add_methods('getPR', 'importone', 'export', 'is_readonly')
+ self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly")
def _get_async_client(self):
return PRAsyncClient()
diff --git a/poky/bitbake/lib/prserv/db.py b/poky/bitbake/lib/prserv/db.py
index b4bda7078c..eb41508198 100644
--- a/poky/bitbake/lib/prserv/db.py
+++ b/poky/bitbake/lib/prserv/db.py
@@ -38,9 +38,9 @@ class PRTable(object):
self.read_only = read_only
self.dirty = False
if nohist:
- self.table = "%s_nohist" % table
+ self.table = "%s_nohist" % table
else:
- self.table = "%s_hist" % table
+ self.table = "%s_hist" % table
if self.read_only:
table_exists = self._execute(
@@ -64,7 +64,7 @@ class PRTable(object):
try:
return self.conn.execute(*query)
except sqlite3.OperationalError as exc:
- if 'is locked' in str(exc) and end > time.time():
+ if "is locked" in str(exc) and end > time.time():
continue
raise exc
@@ -78,7 +78,53 @@ class PRTable(object):
self.sync()
self.dirty = False
- def _getValueHist(self, version, pkgarch, checksum):
+ def test_package(self, version, pkgarch):
+ """Returns whether the specified package version is found in the database for the specified architecture"""
+
+ # Just returns the value if found or None otherwise
+ data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
+ (version, pkgarch))
+ row=data.fetchone()
+ if row is not None:
+ return True
+ else:
+ return False
+
+ def test_value(self, version, pkgarch, value):
+ """Returns whether the specified value is found in the database for the specified package and architecture"""
+
+ # Just returns the value if found or None otherwise
+ data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
+ (version, pkgarch, value))
+ row=data.fetchone()
+ if row is not None:
+ return True
+ else:
+ return False
+
+ def find_value(self, version, pkgarch, checksum):
+ """Returns the value for the specified checksum if found or None otherwise."""
+
+ data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
+ (version, pkgarch, checksum))
+ row=data.fetchone()
+ if row is not None:
+ return row[0]
+ else:
+ return None
+
+ def find_max_value(self, version, pkgarch):
+ """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
+
+ data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table),
+ (version, pkgarch))
+ row = data.fetchone()
+ if row is not None:
+ return row[0]
+ else:
+ return None
+
+ def _get_value_hist(self, version, pkgarch, checksum):
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row=data.fetchone()
@@ -87,7 +133,7 @@ class PRTable(object):
else:
#no value found, try to insert
if self.read_only:
- data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
+ data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
(version, pkgarch))
row = data.fetchone()
if row is not None:
@@ -96,9 +142,9 @@ class PRTable(object):
return 0
try:
- self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
- % (self.table,self.table),
- (version,pkgarch, checksum,version, pkgarch))
+ self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
+ % (self.table, self.table),
+ (version, pkgarch, checksum, version, pkgarch))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
@@ -112,10 +158,10 @@ class PRTable(object):
else:
raise prserv.NotFoundError
- def _getValueNohist(self, version, pkgarch, checksum):
+ def _get_value_no_hist(self, version, pkgarch, checksum):
data=self._execute("SELECT value FROM %s \
WHERE version=? AND pkgarch=? AND checksum=? AND \
- value >= (select max(value) from %s where version=? AND pkgarch=?);"
+ value >= (select max(value) from %s where version=? AND pkgarch=?);"
% (self.table, self.table),
(version, pkgarch, checksum, version, pkgarch))
row=data.fetchone()
@@ -124,17 +170,13 @@ class PRTable(object):
else:
#no value found, try to insert
if self.read_only:
- data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
+ data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
(version, pkgarch))
- row = data.fetchone()
- if row is not None:
- return row[0]
- else:
- return 0
+ return data.fetchone()[0]
try:
- self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
- % (self.table,self.table),
+ self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
+ % (self.table, self.table),
(version, pkgarch, checksum, version, pkgarch))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
@@ -150,17 +192,17 @@ class PRTable(object):
else:
raise prserv.NotFoundError
- def getValue(self, version, pkgarch, checksum):
+ def get_value(self, version, pkgarch, checksum):
if self.nohist:
- return self._getValueNohist(version, pkgarch, checksum)
+ return self._get_value_no_hist(version, pkgarch, checksum)
else:
- return self._getValueHist(version, pkgarch, checksum)
+ return self._get_value_hist(version, pkgarch, checksum)
- def _importHist(self, version, pkgarch, checksum, value):
+ def _import_hist(self, version, pkgarch, checksum, value):
if self.read_only:
return None
- val = None
+ val = None
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row = data.fetchone()
@@ -183,27 +225,27 @@ class PRTable(object):
val = row[0]
return val
- def _importNohist(self, version, pkgarch, checksum, value):
+ def _import_no_hist(self, version, pkgarch, checksum, value):
if self.read_only:
return None
try:
#try to insert
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
- (version, pkgarch, checksum,value))
+ (version, pkgarch, checksum, value))
except sqlite3.IntegrityError as exc:
#already have the record, try to update
try:
- self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
+ self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
% (self.table),
- (value,version,pkgarch,checksum,value))
+ (value, version, pkgarch, checksum, value))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
self.dirty = True
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
- (version,pkgarch,checksum,value))
+ (version, pkgarch, checksum, value))
row=data.fetchone()
if row is not None:
return row[0]
@@ -212,33 +254,33 @@ class PRTable(object):
def importone(self, version, pkgarch, checksum, value):
if self.nohist:
- return self._importNohist(version, pkgarch, checksum, value)
+ return self._import_no_hist(version, pkgarch, checksum, value)
else:
- return self._importHist(version, pkgarch, checksum, value)
+ return self._import_hist(version, pkgarch, checksum, value)
def export(self, version, pkgarch, checksum, colinfo):
metainfo = {}
- #column info
+ #column info
if colinfo:
- metainfo['tbl_name'] = self.table
- metainfo['core_ver'] = prserv.__version__
- metainfo['col_info'] = []
+ metainfo["tbl_name"] = self.table
+ metainfo["core_ver"] = prserv.__version__
+ metainfo["col_info"] = []
data = self._execute("PRAGMA table_info(%s);" % self.table)
for row in data:
col = {}
- col['name'] = row['name']
- col['type'] = row['type']
- col['notnull'] = row['notnull']
- col['dflt_value'] = row['dflt_value']
- col['pk'] = row['pk']
- metainfo['col_info'].append(col)
+ col["name"] = row["name"]
+ col["type"] = row["type"]
+ col["notnull"] = row["notnull"]
+ col["dflt_value"] = row["dflt_value"]
+ col["pk"] = row["pk"]
+ metainfo["col_info"].append(col)
#data info
datainfo = []
if self.nohist:
sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
- (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
+ (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
else:
sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
@@ -261,12 +303,12 @@ class PRTable(object):
else:
data = self._execute(sqlstmt)
for row in data:
- if row['version']:
+ if row["version"]:
col = {}
- col['version'] = row['version']
- col['pkgarch'] = row['pkgarch']
- col['checksum'] = row['checksum']
- col['value'] = row['value']
+ col["version"] = row["version"]
+ col["pkgarch"] = row["pkgarch"]
+ col["checksum"] = row["checksum"]
+ col["value"] = row["value"]
datainfo.append(col)
return (metainfo, datainfo)
@@ -275,7 +317,7 @@ class PRTable(object):
for line in self.conn.iterdump():
writeCount = writeCount + len(line) + 1
fd.write(line)
- fd.write('\n')
+ fd.write("\n")
return writeCount
class PRData(object):
@@ -302,7 +344,7 @@ class PRData(object):
def disconnect(self):
self.connection.close()
- def __getitem__(self,tblname):
+ def __getitem__(self, tblname):
if not isinstance(tblname, str):
raise TypeError("tblname argument must be a string, not '%s'" %
type(tblname))
@@ -316,4 +358,4 @@ class PRData(object):
if tblname in self._tables:
del self._tables[tblname]
logger.info("drop table %s" % (tblname))
- self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
+ self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
diff --git a/poky/bitbake/lib/prserv/serv.py b/poky/bitbake/lib/prserv/serv.py
index 5fc8863f70..dc4be5b620 100644
--- a/poky/bitbake/lib/prserv/serv.py
+++ b/poky/bitbake/lib/prserv/serv.py
@@ -20,16 +20,19 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
singleton = None
class PRServerClient(bb.asyncrpc.AsyncServerConnection):
- def __init__(self, socket, table, read_only):
- super().__init__(socket, 'PRSERVICE', logger)
+ def __init__(self, socket, server):
+ super().__init__(socket, "PRSERVICE", server.logger)
+ self.server = server
+
self.handlers.update({
- 'get-pr': self.handle_get_pr,
- 'import-one': self.handle_import_one,
- 'export': self.handle_export,
- 'is-readonly': self.handle_is_readonly,
+ "get-pr": self.handle_get_pr,
+ "test-pr": self.handle_test_pr,
+ "test-package": self.handle_test_package,
+ "max-package-pr": self.handle_max_package_pr,
+ "import-one": self.handle_import_one,
+ "export": self.handle_export,
+ "is-readonly": self.handle_is_readonly,
})
- self.table = table
- self.read_only = read_only
def validate_proto_version(self):
return (self.proto_version == (1, 0))
@@ -38,57 +41,80 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
try:
return await super().dispatch_message(msg)
except:
- self.table.sync()
+ self.server.table.sync()
raise
else:
- self.table.sync_if_dirty()
+ self.server.table.sync_if_dirty()
+
+ async def handle_test_pr(self, request):
+ '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value'''
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+ checksum = request["checksum"]
+
+ value = self.server.table.find_value(version, pkgarch, checksum)
+ return {"value": value}
+
+ async def handle_test_package(self, request):
+ '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False'''
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+
+ value = self.server.table.test_package(version, pkgarch)
+ return {"value": value}
+
+ async def handle_max_package_pr(self, request):
+ '''Finds the greatest PR value for (version, pkgarch) in the db. Returns None if no entry was found'''
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+
+ value = self.server.table.find_max_value(version, pkgarch)
+ return {"value": value}
async def handle_get_pr(self, request):
- version = request['version']
- pkgarch = request['pkgarch']
- checksum = request['checksum']
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+ checksum = request["checksum"]
response = None
try:
- value = self.table.getValue(version, pkgarch, checksum)
- response = {'value': value}
+ value = self.server.table.get_value(version, pkgarch, checksum)
+ response = {"value": value}
except prserv.NotFoundError:
- logger.error("can not find value for (%s, %s)",version, checksum)
- except sqlite3.Error as exc:
- logger.error(str(exc))
+ self.logger.error("failure storing value in database for (%s, %s)",version, checksum)
return response
async def handle_import_one(self, request):
response = None
- if not self.read_only:
- version = request['version']
- pkgarch = request['pkgarch']
- checksum = request['checksum']
- value = request['value']
+ if not self.server.read_only:
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+ checksum = request["checksum"]
+ value = request["value"]
- value = self.table.importone(version, pkgarch, checksum, value)
+ value = self.server.table.importone(version, pkgarch, checksum, value)
if value is not None:
- response = {'value': value}
+ response = {"value": value}
return response
async def handle_export(self, request):
- version = request['version']
- pkgarch = request['pkgarch']
- checksum = request['checksum']
- colinfo = request['colinfo']
+ version = request["version"]
+ pkgarch = request["pkgarch"]
+ checksum = request["checksum"]
+ colinfo = request["colinfo"]
try:
- (metainfo, datainfo) = self.table.export(version, pkgarch, checksum, colinfo)
+ (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo)
except sqlite3.Error as exc:
- logger.error(str(exc))
+ self.logger.error(str(exc))
metainfo = datainfo = None
- return {'metainfo': metainfo, 'datainfo': datainfo}
+ return {"metainfo": metainfo, "datainfo": datainfo}
async def handle_is_readonly(self, request):
- return {'readonly': self.read_only}
+ return {"readonly": self.server.read_only}
class PRServer(bb.asyncrpc.AsyncServer):
def __init__(self, dbfile, read_only=False):
@@ -98,14 +124,14 @@ class PRServer(bb.asyncrpc.AsyncServer):
self.read_only = read_only
def accept_client(self, socket):
- return PRServerClient(socket, self.table, self.read_only)
+ return PRServerClient(socket, self)
def start(self):
tasks = super().start()
self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only)
self.table = self.db["PRMAIN"]
- logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
+ self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
(self.dbfile, self.address, str(os.getpid())))
return tasks
@@ -135,7 +161,7 @@ class PRServSingleton(object):
if not self.prserv.address:
raise PRServiceConfigError
if not self.port:
- self.port = int(self.prserv.address.rsplit(':', 1)[1])
+ self.port = int(self.prserv.address.rsplit(":", 1)[1])
def run_as_daemon(func, pidfile, logfile):
"""
@@ -171,18 +197,18 @@ def run_as_daemon(func, pidfile, logfile):
# stdout/stderr or it could be 'real' unix fd forking where we need
# to physically close the fds to prevent the program launching us from
# potentially hanging on a pipe. Handle both cases.
- si = open('/dev/null', 'r')
+ si = open("/dev/null", "r")
try:
- os.dup2(si.fileno(),sys.stdin.fileno())
+ os.dup2(si.fileno(), sys.stdin.fileno())
except (AttributeError, io.UnsupportedOperation):
sys.stdin = si
- so = open(logfile, 'a+')
+ so = open(logfile, "a+")
try:
- os.dup2(so.fileno(),sys.stdout.fileno())
+ os.dup2(so.fileno(), sys.stdout.fileno())
except (AttributeError, io.UnsupportedOperation):
sys.stdout = so
try:
- os.dup2(so.fileno(),sys.stderr.fileno())
+ os.dup2(so.fileno(), sys.stderr.fileno())
except (AttributeError, io.UnsupportedOperation):
sys.stderr = so
@@ -200,7 +226,7 @@ def run_as_daemon(func, pidfile, logfile):
# write pidfile
pid = str(os.getpid())
- with open(pidfile, 'w') as pf:
+ with open(pidfile, "w") as pf:
pf.write("%s\n" % pid)
func()
@@ -245,15 +271,15 @@ def stop_daemon(host, port):
# so at least advise the user which ports the corresponding server is listening
ports = []
portstr = ""
- for pf in glob.glob(PIDPREFIX % (ip,'*')):
+ for pf in glob.glob(PIDPREFIX % (ip, "*")):
bn = os.path.basename(pf)
root, _ = os.path.splitext(bn)
- ports.append(root.split('_')[-1])
+ ports.append(root.split("_")[-1])
if len(ports):
- portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports))
+ portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports))
sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n"
- % (pidfile,portstr))
+ % (pidfile, portstr))
return 1
try:
@@ -284,7 +310,7 @@ def is_running(pid):
return True
def is_local_special(host, port):
- if (host == 'localhost' or host == '127.0.0.1') and not port:
+ if (host == "localhost" or host == "127.0.0.1") and not port:
return True
else:
return False
@@ -295,7 +321,7 @@ class PRServiceConfigError(Exception):
def auto_start(d):
global singleton
- host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
+ host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":")))
if not host_params:
# Shutdown any existing PR Server
auto_shutdown()
@@ -304,7 +330,7 @@ def auto_start(d):
if len(host_params) != 2:
# Shutdown any existing PR Server
auto_shutdown()
- logger.critical('\n'.join(['PRSERV_HOST: incorrect format',
+ logger.critical("\n".join(["PRSERV_HOST: incorrect format",
'Usage: PRSERV_HOST = "<hostname>:<port>"']))
raise PRServiceConfigError
@@ -357,8 +383,8 @@ def connect(host, port):
global singleton
- if host.strip().lower() == 'localhost' and not port:
- host = 'localhost'
+ if host.strip().lower() == "localhost" and not port:
+ host = "localhost"
port = singleton.port
conn = client.PRClient()