summaryrefslogtreecommitdiff
path: root/poky/bitbake/lib/hashserv
diff options
context:
space:
mode:
authorAndrew Geissler <geissonator@yahoo.com>2021-05-08 00:11:35 +0300
committerBrad Bishop <bradleyb@fuzziesquirrel.com>2021-05-27 15:46:22 +0300
commitc926e17c956a1babdf42d31f644bf0eedfa7f5f6 (patch)
tree1f86e19ce74be674e46d31a88a438050f83d3762 /poky/bitbake/lib/hashserv
parent5e7fd51182f375f58130989e8d0e206e3e14dee1 (diff)
downloadopenbmc-c926e17c956a1babdf42d31f644bf0eedfa7f5f6.tar.xz
poky: subtree update:1203d1f24d..2dcd1f2a21
Alejandro Enedino Hernandez Samaniego (2): python3: Improve logging, syntax and update deprecated modules to create_manifest python3: Upgrade 3.9.2 -> 3.9.4 Alexander Kanavin (22): scripts/oe-debuginfod: correct several issues libmicrohttpd: add a recipe from meta-oe maintainers.inc: add libmicrohttpd entry xwayland: add a standalone recipe weston: use standalone xwayland instead of outdated xserver-xorg version elfutils: correct debuginfod builds on x32 elfutils: adjust ptests for correct debuginfod testing default-distrovars.inc: add debuginfod to default DISTRO_FEATURES oeqa: tear down oeqa decorators if one of them raises an exception in setup meta/lib/oeqa/core/tests/cases/timeout.py: add a testcase for the previous fix core-image-weston: add sdk/ptest images oeqa/core/tests/test_data.py: use weston image instead of sato oeqa/selftest: transition to weston images core-image-multilib-example: base on weston, and not sato dev-manual/common-tasks.rst: correct the documentation for debuginfod diffoscope: add native libraries to LD_LIBRARY_PATH Revert "oeqa: Set LD_LIBRARY_PATH when executing native commands" boost: correct upstream version check vte: use tarballs again gdk-pixbuf: update 2.40.0 -> 2.42.6 glib-2.0: update 2.68.0 -> 2.68.1 gnu-config: update to latest revision Anatol Belski (1): cross-canadian: Whitelist "mingw32" as TARGET_OS Anders Wallin (3): lttng-tools: Fix missing legacy test files lttng-tools: Fix path for test_python_looging scripts/contrib/image-manifest: add new script Andreas Müller (1): xwayland: remove protocol.txt - it clashes with xserver-xorg Anthony Bagwell (1): systemd: upgrade 247.4 -> 247.6 Anuj Mittal (2): Revert "qemu: fix CVE-2021-3392" qemu: fix CVE-2021-3392 Armin Kuster (6): binutils: rename BRANCH var libseccomp: move recipe from meta-security to core gnutls: Enable seccomp if FEATURE is set systemd: Enable seccomp if FEATURE is set qemu: Enable seccomp if FEATURE is set default-distrovars.inc: Add seccomp to DISTRO_FEATURES_DEFAULT Bastian Krause (1): ccache: add packageconfig docs option Bruce Ashfield (20): kern-tools: add dropped options to audit output linux-yocto/5.4: update to v5.4.109 linux-yocto/5.10: update to v5.10.27 linux-yocto/5.10: BSP configuration fixes linux-yocto/5.10: update to v5.10.29 linux-yocto/5.4: update to v5.4.111 linux-yocto/5.10: update to v5.10.30 linux-yocto-rt/5.10: update to -rt34 linux-yocto/5.4: update to v5.4.112 linux-yocto/5.4: fix arm defconfig warnings linux-yocto/5.10: fix arm defconfig warnings linux-yocto/5.10: aufs fixes linux-yocto/5.10: qemuriscv32.cfg: RV32 only supports 1G physical memory linux-yocto/5.10: update to v5.10.32 perf: fix python-audit RDEPENDS linux-yocto/5.4: update to v5.4.114 linux-yocto/5.10: update to v5.10.34 linux-yocto/5.4: update to v5.4.116 linux-yocto/5.10: qemuppc32: reduce serial shutdown issues yocto-check-layer: Only note a layer without a conf/layer.conf (versus error) Changqing Li (2): libpam: make volatile files created successfully gcr: fix one parallel build failure Chen Qi (3): busybox: fix CVE-2021-28831 weston: fix build failure due to race condition rsync: fix CVE-2020-14387 Christophe Chapuis (1): rootfs.py: find .ko.gz and .ko.xz kernel modules as well Daniel Ammann (1): archiver: Fix typos Devendra Tewari (2): bitbake: lib/bb: Add bb.utils.rename() helper function and use for renaming classes/lib/scripts: Use bb.utils.rename() instead of os.rename() Diego Sueiro (3): oeqa/selftest/bblayers: Add test case for bitbake-layers layerindex-show-depends bitbake: layerindex: Fix bitbake-layers layerindex-show-depends command bitbake: layerindex: Add --fetchdir parameter to layerindex-fetch Douglas Royds (2): Revert "externalsrc: Detect code changes in submodules" externalsrc: Detect code changes in submodules Gavin Li (1): kmod: do not symlink config.guess/config.sub during autoreconf Harald Brinkmann (1): bitbake: fetch/svn: Fix parsing revision of SVN repos with redirects He Zhe (1): linux-yocto-dev: add features/scsi/scsi-debug.scc features/gpio/mockup.scc to KERNEL_FEATURES Henning Schild (3): bitbake: fetch/git: add support for disabling shared clones on unpack bitbake: tests/fetch: deduplicate local git testing code bitbake: tests/fetch: add tests for local and remote "noshared" git fetching Jon Mason (1): oeqa/runtime: space needed Jonas Höppner (1): ltp: fix empty ltp-dev package Jose Quaresma (4): gstreamer1.0: update patch upstream status ptest-runner: libgcc must be installed for pthread_cancel to work gstreamer1.0: rename patches gstreamer1.0: update ptest patch Joshua Watt (2): bitbake: knotty: Re-enable command line logging levels classes/image: Use xargs to set file timestamps Kai Kang (2): cmake.bbclass: remove ${B} before cmake_do_configure kernel-yocto.bbclass: chdir to ${WORKDIR} for do_kernel_checkout Kevin Hao (3): modutils-initscripts: Bail out when no module is installed sysvinit-inittab/start_getty: Check /sys for the tty device existence Revert "inittab: Add getty launch on hvc0 for qemuppc64" Khairul Rohaizzat Jamaluddin (1): qemu: Fix CVE-2020-35517 Khem Raj (54): gcc: Upgrade to 10.3.0 bug-fix release glibc: Rename glibc src package gcc-runtime: Make DEBUG_PREFIX_MAP relative to S valgrind: Delete trailing whitespaces valgrind: Add glibc-src to ptest rdeps valgrind: Add libstdc++ debug symbols for ptest vte: Upgrade to 0.64.0 release systemd: Fix build on mips/musl epiphany: Add missing dependency on gnutls cups: Turn gnutls into a packageconfig knob wpa-supplicant: Enable openssl curl: Use openssl backend libpsl: Add config knobs for runtime/builtin conversion choices glib-networking: Prefer openssl backend instead of gnutls gstreamer1.0-plugins-bad: Add packageconfigs for hls crypto backends ca-certificates: Fix openssl runtime cert dependencies weston: Drop loading xwayland.so module elfutils: Make 64bit time_t fix generic binutils: Fix linking failures when using dwarf-5 go: Use dl.google.com for SRC_URI musl: Update to latest master llvm: Upgrade to LLVM 12 release python3-docutils: Upgrade to 0.17.1 python3-markupsafe: Enable ptests python3-jinja2: Enable ptests python3-pyyaml: Add recipe apt: Fix build on musl when seccomp is enabled default-distrovars.inc: Remove seccomp for riscv32 gcc-target: Create a LTO plugin symlink in bfd-plugins directory bitbake.conf: Use gcc-nm as default NM gcc-cross: Install linker LTO plugin for binutils tools gcc-cross-canadian: Install LTO linker plugin to BFD searchable location gnutls: Point to staging area for finding seccomp libs and includes libjpeg-turbo: Use --reproducible option for nasm libid3tag: Filter -ffile-prefix-map too openssl: Filter out -ffile-prefix-map as well ltp: Filter out -ffile-prefix-map gcc-runtime: Fix __FILE__ related reproducablity issues reproducible_build.bbclass: Enable -Wdate-time pkgconfig: Fix nativesdk builds for mingw sdk hosts m4: Do not use SIGSTKSZ bluez: Fix shadowing of pause function from libc valgrind: Disable leak_cpp_interior test findutils: Do not use SIGSTKSZ bash: Include files needed for run-heredoc ptest libpam: Provide needed env for tst-pam_start_confdir ptest cml1.bbclass: Return sorted list of cfg files busybox: Enable long options for enabled applets webkitgtk: Fix reproducibility in minibrowser webkitgtk: Update patch status libgcc-initial: Do not build fp128 to decimal ppc functions gcc: Upgrade to GCC 11 busybox: Fix reproducibility strace: Upgrade to 5.12 Konrad Weihmann (2): cpan-base: set default UPSTREAM_CHECK_REGEX cve-update-db-native: skip on empty cpe23Uri Marek Vasut (1): linux-firmware: Package RSI 911x WiFi firmware Martin Jansa (2): xwayland: add opengl to REQUIRED_DISTRO_FEATURES ofono: prevent using bundled ell headers and fix build with ell-0.39 Michael Halstead (1): releases: update to include 3.3 Michael Opdenacker (7): dev-manual: fix code insertion manuals: simplify code insertion manuals: code insertion simplification over two lines bitbake: doc: bitbake-user-manual: simplify colon usage bitbake: doc: bitbake-user-manual: code insertion simplification over two lines dev-manual: update references to Docker installation instructions sanity.bbclass: mention CONNECTIVITY_CHECK_URIS in network failure message Mikko Rapeli (4): bitbake: bitbake: tests/fetch: fix test execution without .gitconfig bitbake: bitbake: tests/fetch: remove write protected files too lz4: use CFLAGS from bitbake unzip: use optimization from bitbake Mingli Yu (6): libxshmfence: Build fixes for riscv32 packagegroup-core-tools-profile: Remove valgrind for riscv32 packagegroup-core-tools-testapps.bb: Remove kexec for riscv32 libtool: make sure autoheader run before automake groff: not ship /usr/bin/grap2graph rpm: Upgrade to 4.16.1.3 Minjae Kim (1): qemu: fix CVE-2021-3392 Nicolas Dechesne (1): bitbake: doc: bitbake-user-manual: fix typo left over from Sphinx migration Niels Avonds (1): bitbake: fetch/gitsm: Fix crash when using git LFS and submodules Oleksandr Kravchuk (2): python3-setuptools: update to 56.0.0 autoconf-archive: update to 2021.02.19 Otavio Salvador (2): gstreamer1.0-plugins-base: Add 'viv-fb' OpenGL Window System option gstreamer1.0-plugins-base: Use bb.utils.filter to reduce code Paul Barker (10): bitbake: hashserv: Use generic ConnectionError bitbake: asyncrpc: Common implementation of RPC using json & asyncio bitbake: hashserv: Refactor to use asyncrpc bitbake: prserv: Drop obsolete python version check bitbake: prserv: Drop unused dump_db method bitbake: prserv: Add connect function prservice: Use new connect API bitbake: prserv: Use multiprocessing to auto start prserver bitbake: prserv: Extract daemonization from PRServer class bitbake: prserv: Handle requests in main thread Paulo Cesar Zaneti (1): perl: fix startperl configuration option for perl-native Peter Budny (1): lib/oe/terminal: Fix tmux new-session on older tmux versions (<1.9) Petr Vorel (1): ltp: Replace musl patches with do_patch[postfuncs] Przemyslaw Gorszkowski (2): bitbake: progress: LineFilterProgressHandler - Handle parsing line which ends with CR only bitbake: fetch/s3: Add progress handler for S3 cp command Randy MacLeod (2): sqlite3: upgrade 3.35.0 -> 3.35.3 oe-time-dd-test.sh: increase timeout to 15 sec Reto Schneider (2): license_image.bbclass: Detect broken symlinks license_image.bbclass: Fix symlink to generic license files Richard Purdie (32): oeqa/selftest: Hardcode test assumptions about heartbeat event timings pseudo: Upgrade to add trailing slashes ignore path fix oeqa/selftest: Ensure packages classes are set correctly for maintainers test layer.conf: Update to add post 3.3 release honister series sanity: Add error check for '%' in build path bitbake: runqueue: Fix deferred task issues bitbake: tinfoil/data_smart: Allow variable history emit() to function remotely sanity: Further improve directory sanity tests bitbake: bitbake-server: Remove now unneeded code bitbake: doc/user-manual-fetching: Remove basepath unpack parameter docs poky.conf: Post release version bump runqemu: Ensure we cleanup snapshot files after image run patchelf: Backport fix from upstream for note section overlap error pyyaml: Add missing HOMEPAGE yocto-check-layer: Avoid bug when iterating and autoadding dependencies libseccomp: Add MAINTAINERS entry and HOMEPAGE libseccomp: Fix reproducibility issue apt: Disable libseccomp libxcrypt: Update to 4.4.19 release and fix symbol version issues patchelf: Fix note section alignment issues bitbake: runqueue: Fix multiconfig deferred task sstate validity caching issue bitbake: runqueue: Handle deferred task rehashing in multiconfig builds patchelf: Fix alignment patch pybootchart/draw: Avoid divide by zero error yocto-uninative: Update to 3.1 which includes a patchelf fix Revert "perl: fix startperl configuration option for perl-native" bitbake: bin/bitbake-getvar: Add a new command to query a variable value (with history) bitbake: bitbake: Switch to post release version number 1.51.0 sanity.conf: Require bitbake 1.51.0 oeqa/qemurunner: Improve logging thread exit handling for qemu shutdown test oeqa/qemurunner: Handle path length issues for qmp socket lib/package_manager: Use shutil.copy instead of bb.utils.copyfile for intercepts Robert Joslyn (3): btrfs-tools: Update to 5.11.1 btrfs-tools: Add PACKAGECONFIG options btrfs-tools: Try to follow style guide Robert P. J. Day (3): sdk-manual: "beablebone" -> "beaglebone" sdk-manual: fix broken formatting of sample command bitbake.conf: sort MIRROR list, add missing SAMBA_MIRROR Ross Burton (4): glslang: strip whitespace in pkgconfig file insane: clean up some more warning messages bitbake: bitbake-server: ensure server timeout is a float oe-buildenv-internal: add BitBake's library to PYTHONPATH Sakib Sajal (12): oe-time-dd-test.sh: make executable oe-time-dd-test.sh: provide more information from "top" qemu: fix CVE-2021-20181 qemu: fix CVE-2020-29443 qemu: fix CVE-2021-20221 qemu: fix CVE-2021-3409 qemu: fix CVE-2021-3416 qemu: fix CVE-2021-20257 oe-time-dd-test.sh: collect cooker log when timeout is exceeded buildstats.bbclass: collect data in the same file. qemu: fix CVE-2020-27821 qemu: fix CVE-2021-20263 Samuli Piippo (1): assimp: BBCLASSEXTEND to native and nativesdk Saul Wold (4): pango: re-enable ptest qemu-system-native: install qmp python module qemurunner: Add support for qmp commands qemurunner: change warning to info Stefan Ghinea (3): wpa-supplicant: fix CVE-2021-30004 libssh2: fix build failure with option no-ecdsa xserver-xorg: fix CVE-2021-3472 Stefano Babic (1): libubootenv: upgrade 0.3.1 -> 0.3.2 Teoh Jay Shen (6): oeqa/manual/bsp-hw.json : remove boot_from_runlevel_3 and boot_from_runlevel_5 manual test oeqa/manual/bsp-hw.json : remove ethernet_static_ip_set_in_connman and ethernet_get_IP_in_connman_via_DHCP manual test oeqa/manual/bsp-hw.json : remove standby and Test_if_LAN_device_works_well_after_resume_from_suspend_state manual test oeqa/manual/bsp-hw.json : remove click_terminal_icon_on_X_desktop manual test oeqa/manual/bsp-hw.json :remove Check_if_RTC_(Real_Time_Clock)_can_work_correctly manual test oeqa/manual/bsp-hw.json : remove Test_if_usb_hid_device_works_well_after_resume_from_suspend_state manual test Trevor Gamblin (2): nettle: upgrade 3.7.1 -> 3.7.2 ref-manual/variables.rst: Add incompatibility warning for SERIAL_CONSOLES_CHECK Ulrich Ölmann (1): arch-armv6m.inc: fix access rights Vinay Kumar (1): binutils: Fix CVE-2021-20197 Vineela Tummalapalli (1): Adding dunfell 3.1.7 to the switcher and release list. Wang Mingyu (6): at-spi2-core: upgrade 2.38.0 -> 2.40.0 babeltrace2: upgrade 2.0.3 -> 2.0.4 boost-build-native: upgrade 4.3.0 -> 4.4.1 libassuan: upgrade 2.5.4 -> 2.5.5 webkitgtk: upgrade 2.30.5 -> 2.30.6 vte: upgrade 0.62.2 -> 0.62.3 Wes Lindauer (1): oeqa/runtime/cases: Only disable/enable for current boot Yanfei Xu (1): parselogs: ignore floppy error on qemu-system-x86 at boot stage Yi Fan Yu (7): valgrind: update 3.16.1 -> 3.17.0 valgrind: Disable ptest swapcontext.vgtest valgrind: Fix ptest swapcontext.vgtest Revert "glib-2.0: add workaround to fix codegen.py.test failing" re2c: Upgrade 2.0.3 -> 2.1.1 valgrind: Enable drd/tests/bar_bad* ptest libevent: Increase ptest timing tolerance 50 ms -> 100 ms Zqiang (1): rt-tests: Update rt-tests hongxu (1): deb: apply postinstall on sdk wangmy (34): ell: upgrade 0.38 -> 0.39 dbus-glib: upgrade 0.110 -> 0.112 ccache: upgrade 4.2 -> 4.2.1 gcr: upgrade 3.38.1 -> 3.40.0 ghostscript: upgrade 9.53.3 -> 9.54.0 libsolv: upgrade 0.7.17 -> 0.7.18 glib-2.0: upgrade 2.66.7 -> 2.68.0 file: upgrade 5.39 -> 5.40 curl: upgrade 7.75.0 -> 7.76.0 acpica: upgrade 20210105 -> 20210331 help2man: upgrade 1.48.2 -> 1.48.3 libportal: upgrade 0.3 -> 0.4 libksba: upgrade 1.5.0 -> 1.5.1 go: upgrade 1.16.2 -> 1.16.3 libcap: upgrade 2.48 -> 2.49 libcomps: upgrade 0.1.15 -> 0.1.16 icu: upgrade 68.2 -> 69.1 mpg123: upgrade 1.26.4 -> 1.26.5 man-pages: upgrade 5.10 -> 5.11 go: update SRC_URI to use https protocol mesa: upgrade 21.0.1 -> 21.0.2 openssh: upgrade 8.5p1 -> 8.6p1 mtools: upgrade 4.0.26 -> 4.0.27 python3-cython: upgrade 0.29.22 -> 0.29.23 tiff: upgrade 4.2.0 -> 4.3.0 boost: upgrade 1.75.0 -> 1.76.0 wpebackend-fdo: upgrade 1.8.2 -> 1.8.3 mesa: upgrade 21.0.2 -> 21.0.3 gdb: upgrade 10.1 -> 10.2 glib-networking: upgrade 2.66.0 -> 2.68.1 glslang: upgrade 11.2.0 -> 11.4.0 hdparm: upgrade 9.60 -> 9.61 libhandy: upgrade 1.2.1 -> 1.2.2 libjitterentropy: upgrade 3.0.1 -> 3.0.2 zangrc (1): maintainers.inc: Modify email address zhengruoqin (19): epiphany: upgrade 3.38.2 -> 3.38.3 wpebackend-fdo: upgrade 1.8.0 -> 1.8.2 netbase: upgrade 6.2 -> 6.3 python3-dbusmock: upgrade 0.22.0 -> 0.23.0 python3-gitdb: upgrade 4.0.5 -> 4.0.7 libva: upgrade 2.10.0 -> 2.11.0 ruby: upgrade 3.0.0 -> 3.0.1 libva-utils: upgrade 2.10.0 -> 2.11.1 libdazzle: upgrade 3.38.0 -> 3.40.0 librepo: upgrade 1.13.0 -> 1.14.0 libdrm: upgrade 2.4.104 -> 2.4.105 python3-pygobject: upgrade 3.38.0 -> 3.40.1 libedit: upgrade 20210216-3.1 -> 20210419-3.1 libhandy: upgrade 1.2.0 -> 1.2.1 libical: upgrade 3.0.9 -> 3.0.10 libsolv: upgrade 0.7.18 -> 0.7.19 libmicrohttpd: upgrade 0.9.72 -> 0.9.73 python3-numpy: upgrade 1.20.1 -> 1.20.2 wireless-regdb: upgrade 2020.11.20 -> 2021.04.21 Signed-off-by: Andrew Geissler <geissonator@yahoo.com> Change-Id: Ibdaea694cae40b0749d472bf08b53002a45b31d7
Diffstat (limited to 'poky/bitbake/lib/hashserv')
-rw-r--r--poky/bitbake/lib/hashserv/client.py147
-rw-r--r--poky/bitbake/lib/hashserv/server.py210
-rw-r--r--poky/bitbake/lib/hashserv/tests.py3
3 files changed, 45 insertions, 315 deletions
diff --git a/poky/bitbake/lib/hashserv/client.py b/poky/bitbake/lib/hashserv/client.py
index e05c1eb56..531170967 100644
--- a/poky/bitbake/lib/hashserv/client.py
+++ b/poky/bitbake/lib/hashserv/client.py
@@ -8,110 +8,26 @@ import json
import logging
import socket
import os
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client
+import bb.asyncrpc
+from . import create_async_client
logger = logging.getLogger("hashserv.client")
-class HashConnectionError(Exception):
- pass
-
-
-class AsyncClient(object):
+class AsyncClient(bb.asyncrpc.AsyncClient):
MODE_NORMAL = 0
MODE_GET_STREAM = 1
def __init__(self):
- self.reader = None
- self.writer = None
+ super().__init__('OEHASHEQUIV', '1.1', logger)
self.mode = self.MODE_NORMAL
- self.max_chunk = DEFAULT_MAX_CHUNK
-
- async def connect_tcp(self, address, port):
- async def connect_sock():
- return await asyncio.open_connection(address, port)
-
- self._connect_sock = connect_sock
-
- async def connect_unix(self, path):
- async def connect_sock():
- return await asyncio.open_unix_connection(path)
-
- self._connect_sock = connect_sock
-
- async def connect(self):
- if self.reader is None or self.writer is None:
- (self.reader, self.writer) = await self._connect_sock()
-
- self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8"))
- await self.writer.drain()
-
- cur_mode = self.mode
- self.mode = self.MODE_NORMAL
- await self._set_mode(cur_mode)
-
- async def close(self):
- self.reader = None
-
- if self.writer is not None:
- self.writer.close()
- self.writer = None
-
- async def _send_wrapper(self, proc):
- count = 0
- while True:
- try:
- await self.connect()
- return await proc()
- except (
- OSError,
- HashConnectionError,
- json.JSONDecodeError,
- UnicodeDecodeError,
- ) as e:
- logger.warning("Error talking to server: %s" % e)
- if count >= 3:
- if not isinstance(e, HashConnectionError):
- raise HashConnectionError(str(e))
- raise e
- await self.close()
- count += 1
-
- async def send_message(self, msg):
- async def get_line():
- line = await self.reader.readline()
- if not line:
- raise HashConnectionError("Connection closed")
-
- line = line.decode("utf-8")
-
- if not line.endswith("\n"):
- raise HashConnectionError("Bad message %r" % message)
-
- return line
-
- async def proc():
- for c in chunkify(json.dumps(msg), self.max_chunk):
- self.writer.write(c.encode("utf-8"))
- await self.writer.drain()
-
- l = await get_line()
-
- m = json.loads(l)
- if m and "chunk-stream" in m:
- lines = []
- while True:
- l = (await get_line()).rstrip("\n")
- if not l:
- break
- lines.append(l)
-
- m = json.loads("".join(lines))
-
- return m
- return await self._send_wrapper(proc)
+ async def setup_connection(self):
+ await super().setup_connection()
+ cur_mode = self.mode
+ self.mode = self.MODE_NORMAL
+ await self._set_mode(cur_mode)
async def send_stream(self, msg):
async def proc():
@@ -119,7 +35,7 @@ class AsyncClient(object):
await self.writer.drain()
l = await self.reader.readline()
if not l:
- raise HashConnectionError("Connection closed")
+ raise ConnectionError("Connection closed")
return l.decode("utf-8").rstrip()
return await self._send_wrapper(proc)
@@ -128,11 +44,11 @@ class AsyncClient(object):
if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
r = await self.send_stream("END")
if r != "ok":
- raise HashConnectionError("Bad response from server %r" % r)
+ raise ConnectionError("Bad response from server %r" % r)
elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
r = await self.send_message({"get-stream": None})
if r != "ok":
- raise HashConnectionError("Bad response from server %r" % r)
+ raise ConnectionError("Bad response from server %r" % r)
elif new_mode != self.mode:
raise Exception(
"Undefined mode transition %r -> %r" % (self.mode, new_mode)
@@ -189,12 +105,10 @@ class AsyncClient(object):
return (await self.send_message({"backfill-wait": None}))["tasks"]
-class Client(object):
+class Client(bb.asyncrpc.Client):
def __init__(self):
- self.client = AsyncClient()
- self.loop = asyncio.new_event_loop()
-
- for call in (
+ super().__init__()
+ self._add_methods(
"connect_tcp",
"close",
"get_unihash",
@@ -204,30 +118,7 @@ class Client(object):
"get_stats",
"reset_stats",
"backfill_wait",
- ):
- downcall = getattr(self.client, call)
- setattr(self, call, self._get_downcall_wrapper(downcall))
-
- def _get_downcall_wrapper(self, downcall):
- def wrapper(*args, **kwargs):
- return self.loop.run_until_complete(downcall(*args, **kwargs))
-
- return wrapper
-
- def connect_unix(self, path):
- # AF_UNIX has path length issues so chdir here to workaround
- cwd = os.getcwd()
- try:
- os.chdir(os.path.dirname(path))
- self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
- self.loop.run_until_complete(self.client.connect())
- finally:
- os.chdir(cwd)
-
- @property
- def max_chunk(self):
- return self.client.max_chunk
-
- @max_chunk.setter
- def max_chunk(self, value):
- self.client.max_chunk = value
+ )
+
+ def _get_async_client(self):
+ return AsyncClient()
diff --git a/poky/bitbake/lib/hashserv/server.py b/poky/bitbake/lib/hashserv/server.py
index a0dc0c170..c941c0e9d 100644
--- a/poky/bitbake/lib/hashserv/server.py
+++ b/poky/bitbake/lib/hashserv/server.py
@@ -14,7 +14,9 @@ import signal
import socket
import sys
import time
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client, TABLE_COLUMNS
+from . import create_async_client, TABLE_COLUMNS
+import bb.asyncrpc
+
logger = logging.getLogger('hashserv.server')
@@ -109,12 +111,6 @@ class Stats(object):
return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')}
-class ClientError(Exception):
- pass
-
-class ServerError(Exception):
- pass
-
def insert_task(cursor, data, ignore=False):
keys = sorted(data.keys())
query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % (
@@ -149,7 +145,7 @@ async def copy_outhash_from_upstream(client, db, method, outhash, taskhash):
return d
-class ServerClient(object):
+class ServerClient(bb.asyncrpc.AsyncServerConnection):
FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
ALL_QUERY = 'SELECT * FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
OUTHASH_QUERY = '''
@@ -168,21 +164,19 @@ class ServerClient(object):
'''
def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only):
- self.reader = reader
- self.writer = writer
+ super().__init__(reader, writer, 'OEHASHEQUIV', logger)
self.db = db
self.request_stats = request_stats
- self.max_chunk = DEFAULT_MAX_CHUNK
+ self.max_chunk = bb.asyncrpc.DEFAULT_MAX_CHUNK
self.backfill_queue = backfill_queue
self.upstream = upstream
- self.handlers = {
+ self.handlers.update({
'get': self.handle_get,
'get-outhash': self.handle_get_outhash,
'get-stream': self.handle_get_stream,
'get-stats': self.handle_get_stats,
- 'chunk-stream': self.handle_chunk,
- }
+ })
if not read_only:
self.handlers.update({
@@ -192,56 +186,19 @@ class ServerClient(object):
'backfill-wait': self.handle_backfill_wait,
})
+ def validate_proto_version(self):
+ return (self.proto_version > (1, 0) and self.proto_version <= (1, 1))
+
async def process_requests(self):
if self.upstream is not None:
self.upstream_client = await create_async_client(self.upstream)
else:
self.upstream_client = None
- try:
-
-
- self.addr = self.writer.get_extra_info('peername')
- logger.debug('Client %r connected' % (self.addr,))
-
- # Read protocol and version
- protocol = await self.reader.readline()
- if protocol is None:
- return
-
- (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split()
- if proto_name != 'OEHASHEQUIV':
- return
-
- proto_version = tuple(int(v) for v in proto_version.split('.'))
- if proto_version < (1, 0) or proto_version > (1, 1):
- return
-
- # Read headers. Currently, no headers are implemented, so look for
- # an empty line to signal the end of the headers
- while True:
- line = await self.reader.readline()
- if line is None:
- return
+ await super().process_requests()
- line = line.decode('utf-8').rstrip()
- if not line:
- break
-
- # Handle messages
- while True:
- d = await self.read_message()
- if d is None:
- break
- await self.dispatch_message(d)
- await self.writer.drain()
- except ClientError as e:
- logger.error(str(e))
- finally:
- if self.upstream_client is not None:
- await self.upstream_client.close()
-
- self.writer.close()
+ if self.upstream_client is not None:
+ await self.upstream_client.close()
async def dispatch_message(self, msg):
for k in self.handlers.keys():
@@ -255,47 +212,7 @@ class ServerClient(object):
await self.handlers[k](msg[k])
return
- raise ClientError("Unrecognized command %r" % msg)
-
- def write_message(self, msg):
- for c in chunkify(json.dumps(msg), self.max_chunk):
- self.writer.write(c.encode('utf-8'))
-
- async def read_message(self):
- l = await self.reader.readline()
- if not l:
- return None
-
- try:
- message = l.decode('utf-8')
-
- if not message.endswith('\n'):
- return None
-
- return json.loads(message)
- except (json.JSONDecodeError, UnicodeDecodeError) as e:
- logger.error('Bad message from client: %r' % message)
- raise e
-
- async def handle_chunk(self, request):
- lines = []
- try:
- while True:
- l = await self.reader.readline()
- l = l.rstrip(b"\n").decode("utf-8")
- if not l:
- break
- lines.append(l)
-
- msg = json.loads(''.join(lines))
- except (json.JSONDecodeError, UnicodeDecodeError) as e:
- logger.error('Bad message from client: %r' % message)
- raise e
-
- if 'chunk-stream' in msg:
- raise ClientError("Nested chunks are not allowed")
-
- await self.dispatch_message(msg)
+ raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg)
async def handle_get(self, request):
method = request['method']
@@ -499,74 +416,20 @@ class ServerClient(object):
cursor.close()
-class Server(object):
+class Server(bb.asyncrpc.AsyncServer):
def __init__(self, db, loop=None, upstream=None, read_only=False):
if upstream and read_only:
- raise ServerError("Read-only hashserv cannot pull from an upstream server")
+ raise bb.asyncrpc.ServerError("Read-only hashserv cannot pull from an upstream server")
+
+ super().__init__(logger, loop)
self.request_stats = Stats()
self.db = db
-
- if loop is None:
- self.loop = asyncio.new_event_loop()
- self.close_loop = True
- else:
- self.loop = loop
- self.close_loop = False
-
self.upstream = upstream
self.read_only = read_only
- self._cleanup_socket = None
-
- def start_tcp_server(self, host, port):
- self.server = self.loop.run_until_complete(
- asyncio.start_server(self.handle_client, host, port, loop=self.loop)
- )
-
- for s in self.server.sockets:
- logger.info('Listening on %r' % (s.getsockname(),))
- # Newer python does this automatically. Do it manually here for
- # maximum compatibility
- s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
- s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-
- name = self.server.sockets[0].getsockname()
- if self.server.sockets[0].family == socket.AF_INET6:
- self.address = "[%s]:%d" % (name[0], name[1])
- else:
- self.address = "%s:%d" % (name[0], name[1])
-
- def start_unix_server(self, path):
- def cleanup():
- os.unlink(path)
-
- cwd = os.getcwd()
- try:
- # Work around path length limits in AF_UNIX
- os.chdir(os.path.dirname(path))
- self.server = self.loop.run_until_complete(
- asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
- )
- finally:
- os.chdir(cwd)
-
- logger.info('Listening on %r' % path)
-
- self._cleanup_socket = cleanup
- self.address = "unix://%s" % os.path.abspath(path)
-
- async def handle_client(self, reader, writer):
- # writer.transport.set_write_buffer_limits(0)
- try:
- client = ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
- await client.process_requests()
- except Exception as e:
- import traceback
- logger.error('Error from client: %s' % str(e), exc_info=True)
- traceback.print_exc()
- writer.close()
- logger.info('Client disconnected')
+ def accept_client(self, reader, writer):
+ return ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
@contextmanager
def _backfill_worker(self):
@@ -597,31 +460,8 @@ class Server(object):
else:
yield
- def serve_forever(self):
- def signal_handler():
- self.loop.stop()
-
- asyncio.set_event_loop(self.loop)
- try:
- self.backfill_queue = asyncio.Queue()
-
- self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
-
- with self._backfill_worker():
- try:
- self.loop.run_forever()
- except KeyboardInterrupt:
- pass
-
- self.server.close()
-
- self.loop.run_until_complete(self.server.wait_closed())
- logger.info('Server shutting down')
- finally:
- if self.close_loop:
- if sys.version_info >= (3, 6):
- self.loop.run_until_complete(self.loop.shutdown_asyncgens())
- self.loop.close()
+ def run_loop_forever(self):
+ self.backfill_queue = asyncio.Queue()
- if self._cleanup_socket is not None:
- self._cleanup_socket()
+ with self._backfill_worker():
+ super().run_loop_forever()
diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py
index 1a696481e..e2b762dbf 100644
--- a/poky/bitbake/lib/hashserv/tests.py
+++ b/poky/bitbake/lib/hashserv/tests.py
@@ -6,7 +6,6 @@
#
from . import create_server, create_client
-from .client import HashConnectionError
import hashlib
import logging
import multiprocessing
@@ -277,7 +276,7 @@ class HashEquivalenceCommonTests(object):
outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
- with self.assertRaises(HashConnectionError):
+ with self.assertRaises(ConnectionError):
ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
# Ensure that the database was not modified