summaryrefslogtreecommitdiff
path: root/poky/bitbake/lib/hashserv
diff options
context:
space:
mode:
authorBrad Bishop <bradleyb@fuzziesquirrel.com>2019-08-20 16:16:51 +0300
committerBrad Bishop <bradleyb@fuzziesquirrel.com>2019-08-20 16:17:54 +0300
commit08902b01500fb82ac050ec2dce9b6c4358075a17 (patch)
tree76dad89580e2a758feb672731745c5f4c0f6ef30 /poky/bitbake/lib/hashserv
parent754b8faf0be432fcdcacb340fe95117cac890e40 (diff)
downloadopenbmc-08902b01500fb82ac050ec2dce9b6c4358075a17.tar.xz
poky: subtree update:835f7eac06..20946c63c2
Aaron Chan (1): python3-dbus: Add native and nativesdk variants Adrian Bunk (8): gnome: Remove the gnome class bind: Remove RECIPE_NO_UPDATE_REASON and follow the ESV releases webkitgtk: Reenable on mips mtd-utils: Upgrade to 2.1.1 Change ftp:// URIs to http(s):// webkitgtk: Stop disabling gold on aarch64 and mips grub/libmpc/gdb: Use GNU_MIRROR in more recipes screen: Backport fix for an implicit function declaration Alexander Kanavin (28): btrfs-tools: update 5.1.1 -> 5.2.1 libmodulemd: update to 2.6.0 libwebp: upgrade 1.0.2 -> 1.0.3 createrepo-c: upgrade 0.14.2 -> 0.14.3 webkitgtk: upgrade 2.24.2 -> 2.24.3 bzip2: fix upstream version check stress-ng: add a recipe that replaces the original stress meson: update 0.50.1 -> 0.51.1 meson.bbclass: do not pass native compiler/linker flags via command line meson: add a backported patch to address vala cross-compilation errors libedit: fix upstream verison check maintainers.inc: assign acpica to Ross stress-ng: add a patch to remove unneeded bash dependency elfutils: use PRIVATE_LIBS for the ptest package apt: add a missing perl runtime dependency attr: add a missing perl runtime dependency ofono: correct the python3 runtime dependency bluez5: correct the python3 runtime dependency local.conf.sample: do not add sdl to nativesdk qemu config maintainers.inc: give python recipes to Oleksandr Kravchuk python-numpy: remove the python 2.x version of the recipe python-scons: remove the python 2.x version of the recipe python-nose: remove the python 2.x version of the recipe lib/oeqa/utils/qemurunner.py: add runqemuparams after kvm/nographic/snapshot/slirp mesa: enable glx-tls option in native and nativesdk builds insane.bbclass: in file-rdeps do not look into RDEPENDS recursively sudo: correct SRC_URI ovmf: fix upstream version check Andreas Obergschwandtner (1): bzip2: set the autoconf package version to the recipe version Anuj Mittal (11): mpg123: upgrade 1.25.10 -> 1.25.11 libsdl: remove pulseaudio: don't include consolekit when systemd is enabled libsdl2: upgrade 2.0.9 -> 2.0.10 grub: upgrade 2.02 -> 2.04 patch: fix CVE-2019-13636 python: fix CVE-2018-20852 python: CVE-2019-9947 is same as CVE-2019-9740 libtasn1: upgrade 4.13 -> 4.14 pango: upgrade 1.42.4 -> 1.44.3 harfbuzz: upgrade 2.4.0 -> 2.5.3 Bartosz Golaszewski (1): qemu: add a patch fixing the native build on newer kernels Bedel, Alban (3): rng-tools: start rngd early in the boot process again kernel-uboot: remove useless special casing of arm64 Image boost: Fix build and enable context and coroutines on aarch64 Bruce Ashfield (2): linux-yocto/4.19: update to v4.19.61 linux-yocto-dev: bump to 5.3-rcX Changqing Li (6): runqemu: add lockfile for port used when slirp enabled runqemu: fix get portlock fail for multi users qemuboot-x86: move QB_SYSTEM_NAME to corresponding conf genericx86-64.conf/genericx86.conf: add QB_SYSTEM_NAME grub/grub-efi: fix conflict for aach64 go-runtime: remove conflict files from -dev packages Chen Qi (1): sudo: use nonarch_libdir instead of libdir for tmpfiles.d Chin Huat Ang (1): cve-update-db-native: fix https proxy issues Chris Laplante via bitbake-devel (1): bitbake: fetch2/wget: avoid 'maximum recursion depth' RuntimeErrors when handling 403 codes Daniel Ammann (2): image_types: Remove remnants of hdddirect bitbake: toaster: Sync list of fs_types with oe-core Denys Dmytriyenko (2): wayland-protocols: upgrade 1.17 -> 1.18 weston: upgrade 6.0.0 -> 6.0.1 Diego Rondini (1): image_types.bbclass: make gzipped images rsyncable Dmitry Eremin-Solenikov (1): kernel.bbclass: fix installation of modules signing certificates Frederic Ouellet (1): systemd: Add partial support of drop-in configuration files to systemd-systemctl-native Hongxu Jia (1): grub: add grub-native Jason Wessel (6): sqlite3: Fix zlib determinism problem pseudo: Fix openat() with a symlink pointing to a directory image_types_wic.bbclass: Copy the .wks and .env files to deploy image dir wic: Add partition type for msdos partition tables wic: Make disk partition size consistently computed dpkg: Provide update-alternative for start-stop-daemon Johann Fridriksson (1): ruby: Adding zlib-native to native dependencies Joshua Lock via Openembedded-core (3): sstate: fix log message classes/sstate: don't use unsigned sstate when verification enabled classes/sstate: regenerate sstate when signing enabled Joshua Watt (1): bitbake: hashserv: SQL Optimizations Kai Kang (3): subversion: add packageconfig boost epiphany: set imcompatible with tune mips e2fsprogs: 1.44.5 -> 1.45.3 Khem Raj (23): strace: Upgrade to 5.2 linux-libc-header: Fix ptrace.h and prctl.h conflict on aarch64 libnss-nis: Fix build with glibc 2.30 lttng-ust: Check for gettid libc API ltp: Fix build with glibc 2.30 lttng-tools: Fix build with glibc 2.30 xserver-xorg: Backport patch to remove using sys/io.h Apache-2.0-with-LLVM-exception: Add new license file libedit: Move from meta-oe groff: Fix math.h inclusion from system headers issue webkitgtk: Fix compile failures with clang glibc: Update to glibc 2.30 virglrender: Fix endianness check on musl syslinux: Override hardcoded toolnames in Makefile systemd-boot: Add option to specify cross objcopy and use it mesa,llvm,meson: Update llvm to 8.0.1 plus define and use LLVM version globally musl: Update to master tip oeqa/buildgalculator.py: Add dependency on gtk+3 oeqa/parselogs: grep for exact errors list keywords gcc-runtime: Move content from gcclibdir into libdir gdb: Do not set musl specific CFLAGS linuxloader: Add entries for riscv64 musl: Delete GLIBC_LDSO before creating symlink with lnr Luca Boccassi (1): python3-pygobject: remove python3-setuptools from RDEPENDS Mads Andreasen (1): bitbake: fetch2/npm: Use npm pack to download node modules instead of wget Mark Hatle (2): glibc-package.inc: Add linux-libc-headers-dev to glibc-dev bitbake: layerindexlib: Fix parsing of recursive layer dependencies Martin Jansa (3): icecc.bbclass: catch subprocess.CalledProcessError powertop: import a fix from buildroot meson: backport fix for builds with -Werror=return-type Ming Liu (5): libx11-compose-data: add recipe libxkbcommon: RDEPENDS on libx11 compose data weston: change to use meson build system license_image.bbclass: drop invalid comments opensbi: handle deploy task under sstate Naveen Saini (2): gdk-pixbuf: enable x11 PACKAGECONFIG option image_types_wic: add syslinux-native dependency conditional Oleksandr Kravchuk (17): python3-pip: update to 19.2.1 python3-git: update to 2.1.12 ethtool: update to 5.2 python3-git: update to 2.1.13 xorgproto: update to 2019.1 xserver-xorg: update to 1.20.5 ell: update to 0.21 libinput: update to 1.14.0 wpa-supplicant: update to 2.9 aspell: update to 0.60.7 linux-firmware: add PE back xf86-input-libinput: update to 0.29.0 git: update to 2.22.1 xrandr: update to 1.5.1 python3-git: update to 3.0.0 librepo: update to 1.10.5 libevent: update to 2.1.11 Pascal Bach (2): cmake: 3.14.5 -> 3.15.1 cmake: 3.15.1 -> 3.15.2 Paul Eggleton (2): scripts/create-pull-request: improve handling of non-SSH remote URLs scripts/create-pull-request: fix putting subject containing / into cover letter Piotr Tworek (2): pulseaudio: Backport upstream fix new alsa compatibility. libdrm: Move amdgpu.ids file into libdrm-amdgpu package. Randy MacLeod (1): ptest-runner: update from 2.3.1 to 2.3.2 Rasmus Villemoes (1): iproute2: drop pointless configure-cross.patch Ricardo Neri (5): ovmf: Update to version edk2-stable201905 ovmf: Set PV ovmf: Use HOSTTOOLS' python3 ovmf: Generate test Platform key and first Key Exchange Key runqemu: Add support to handle EnrollDefaultKeys PK/KEK1 certificate Ricardo Ribalda Delgado (2): packagegroup-core-base-utils: Make it machine specific inetutils: Fix abort on invalid files Richard Purdie (50): package: Improve determinism sstate: Reduce race windows bitbake: siggen: Import unihash code from OE-Core bitbake: cache: Add SimpleCache class bitbake: runqueue: Improve scenequeue processing logic bitbake: siggen: Add new unitaskhashes data variable which is cached bitbake: siggen: Convert to use self.unitaskhashes bitbake: runqueue: Enable dynamic task adjustment to hash equivalency bitbake: runqueue: Improve determinism bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE bitbake: hashserv: Turn off sqlite synchronous mode bitbake: prserv: Use a memory journal bitbake: hashserv: Use separate threads for answering requests and handling them bitbake: hashserv: Switch from threads to multiprocessing bitbake: runqueue: Clean up BB_HASHCHECK_FUNCTION API bitbake: siggen: Clean up task reference formats bitbake: build/utils: Drop bb.build.FuncFailed bitbake: tests/runqueue: Add hashserv+runqueue test bitbake: bitbake: Bump version to 1.43.1 for API changes sanity.conf: Require bitbake 1.43.1 classes/lib: Remove bb.build.FuncFailed sstatesig: Move unihash siggen code to bitbake sstatesig: Add debug for incorrect hash server settings sstatesig: Adpat to recent bitbake hash equiv runqueue changes sstatesig: Update to handle BB_HASHSERVE sstate/sstatesig: Update to new form of BB_HASHCHECK_FUNCTION sstatesig: Updates to match bitbake siggen changes gstreamer: Add fix for glibc 2.30 sstatesig: Fix leftover splitting issue from siggen change python3-pygobject: Add missing pkgutil RDEPENDS bitbake: runqueue: Fix corruption issue bitbake: runqueue: Improve setscene task handling logic bitbake: tests/runqueue: Add further hash equivalence tests bitbake: cooker: Improve hash server startup code to avoid exit tracebacks bitbake: runqueue: Wait for covered tasks to complete before trying setscene bitbake: runqueue: Fix next_buildable_task performance problem bitbake: runqueue: Improve scenequeue debugging bitbake: runqueue: Recompute holdoff tasks from scratch bitbake: runqueue: Fix event timing race bitbake: runqueue: Drop debug statement causing performance issues bitbake: runqueue: Add further debug information bitbake: runqueue: Add missing setscene task corner case bitbake: runqueue: Ensure we clear the stamp cache poky: Retire opensuse 42.3 from SANITY_TESTED_DISTROS gcc-cross-canadian: Drop obsolete shlibs exclusion bitbake: tests/runqueue: Fix tests bitbake: runqueue: Fix data corruption problem bitbake: runqueue: Ensure data is handled correctly bitbake: hashserv: Ensure we don't accumulate sockets in TIME_WAIT state bitbake: runqueue: Ensure target_tids is filtered Robert Yang (3): bitbake: cooker: Cleanup the queue before call process.join() bitbake: knotty: Fix for the Second Keyboard Interrupt bitbake: bitbake: server/process: Handle BBHandledException to avoid unexpected exceptions Ross Burton (23): libidn2: remove build paths from libidn2.pc gnutls: don't use HOSTTOOLS_DIR/bash as a shell on target libical: upgrade to 3.0.5 perl: fix whitespace perl: add PACKAGECONFIG for db fortran-helloworld: neaten recipe python3: remove empty python3-distutils-staticdev python3: support recommends in manifest python3: split out the Windows distutils installer stubs insane: check if the recipe incorrectly uses DEPENDS_${PN} libxx86misc: remove this now redundant library xserver-xorg: clean up xorgproto dependencies xserver-xorg: add PACKAGECONFIG for DGA xdpyinfo: don't depend on DGA libxx86dga: remove obsolete client libary xserver-xorg: remove embedded build path in the source libx11: update to 1.6.8 sanity: update for new bb.build.exec_func() behaviour libx11-diet: remove qemu: fix patch Upstream-Status xserver-xorg: refresh build path removal patch waffle: upgrade 1.5.2 -> 1.6.0 libx11: replace libtool patch with upstreamed patch Tim Blechmann (1): deb: allow custom dpkg command Trevor Gamblin (2): gzip: update ptest package dependencies patch: fix CVE-2019-13638 Wenlin Kang (1): db: add switch for building database verification Will Page (1): uboot: fixes to uboot-extlinux-config attribute values William Bourque (1): meta/lib/oeqa: Remove ext4 for bootimg-biosplusefi Yi Zhao (1): libx11-compose-data: upgrade 1.6.7 -> 1.6.8 Yuan Chao (4): glib-2.0:upgrade 2.60.5 -> 2.60.6 nettle:upgrade 3.4.1 -> 3.5.1 python3-pbr:upgrade 5.4.1 -> 5.4.2 gpgme:upgrade 1.13.0 -> 1.13.1 Zang Ruochen (8): msmtp: upgrade 1.8.4 -> 1.8.5 curl: upgrade 7.65.2 -> 7.65.3 iso-codes: upgrade 4.2 -> 4.3 python-scons:upgrade 3.0.5 -> 3.1.0 libgudev:upgrade 232 -> 233 libglu:upgrade 9.0.0 -> 9.0.1 man-db:upgrade 2.8.5 -> 2.8.6.1 libnewt:upgrade 0.52.20 -> 0.52.21 Zheng Ruoqin (1): python3-mako: 1.0.14 -> 1.1.0 Zoltan Kuscsik (1): kmscube: update to latest revision Change-Id: I2cd1a0d59da46725b1aba5a79b63eb6121b3c79e Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/bitbake/lib/hashserv')
-rw-r--r--poky/bitbake/lib/hashserv/__init__.py118
-rw-r--r--poky/bitbake/lib/hashserv/tests.py15
2 files changed, 111 insertions, 22 deletions
diff --git a/poky/bitbake/lib/hashserv/__init__.py b/poky/bitbake/lib/hashserv/__init__.py
index fdc9ced9f..eb03c3221 100644
--- a/poky/bitbake/lib/hashserv/__init__.py
+++ b/poky/bitbake/lib/hashserv/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Garmin Ltd.
+# Copyright (C) 2018-2019 Garmin Ltd.
#
# SPDX-License-Identifier: GPL-2.0-only
#
@@ -10,6 +10,12 @@ import sqlite3
import json
import traceback
import logging
+import socketserver
+import queue
+import threading
+import signal
+import socket
+import struct
from datetime import datetime
logger = logging.getLogger('hashserv')
@@ -18,8 +24,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
def log_message(self, f, *args):
logger.debug(f, *args)
+ def opendb(self):
+ self.db = sqlite3.connect(self.dbname)
+ self.db.row_factory = sqlite3.Row
+ self.db.execute("PRAGMA synchronous = OFF;")
+ self.db.execute("PRAGMA journal_mode = MEMORY;")
+
def do_GET(self):
try:
+ if not self.db:
+ self.opendb()
+
p = urllib.parse.urlparse(self.path)
if p.path != self.prefix + '/v1/equivalent':
@@ -32,7 +47,7 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
d = None
with contextlib.closing(self.db.cursor()) as cursor:
- cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1',
+ cursor.execute('SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1',
{'method': method, 'taskhash': taskhash})
row = cursor.fetchone()
@@ -52,6 +67,9 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
def do_POST(self):
try:
+ if not self.db:
+ self.opendb()
+
p = urllib.parse.urlparse(self.path)
if p.path != self.prefix + '/v1/equivalent':
@@ -63,15 +81,29 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
with contextlib.closing(self.db.cursor()) as cursor:
cursor.execute('''
- SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND outhash=:outhash
+ -- Find tasks with a matching outhash (that is, tasks that
+ -- are equivalent)
+ SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND outhash=:outhash
+
+ -- If there is an exact match on the taskhash, return it.
+ -- Otherwise return the oldest matching outhash of any
+ -- taskhash
ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
created ASC
+
+ -- Only return one row
LIMIT 1
''', {k: data[k] for k in ('method', 'outhash', 'taskhash')})
row = cursor.fetchone()
+ # If no matching outhash was found, or one *was* found but it
+ # wasn't an exact match on the taskhash, a new entry for this
+ # taskhash should be added
if row is None or row['taskhash'] != data['taskhash']:
+ # If a row matching the outhash was found, the unihash for
+ # the new taskhash should be the same as that one.
+ # Otherwise the caller provided unihash is used.
unihash = data['unihash']
if row is not None:
unihash = row['unihash']
@@ -88,18 +120,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
if k in data:
insert_data[k] = data[k]
- cursor.execute('''INSERT INTO tasks_v1 (%s) VALUES (%s)''' % (
+ cursor.execute('''INSERT INTO tasks_v2 (%s) VALUES (%s)''' % (
', '.join(sorted(insert_data.keys())),
', '.join(':' + k for k in sorted(insert_data.keys()))),
insert_data)
logger.info('Adding taskhash %s with unihash %s', data['taskhash'], unihash)
- cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE id=:id', {'id': cursor.lastrowid})
- row = cursor.fetchone()
self.db.commit()
-
- d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
+ d = {'taskhash': data['taskhash'], 'method': data['method'], 'unihash': unihash}
+ else:
+ d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
@@ -110,17 +141,67 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
self.send_error(400, explain=traceback.format_exc())
return
-def create_server(addr, db, prefix=''):
+class ThreadedHTTPServer(HTTPServer):
+ quit = False
+
+ def serve_forever(self):
+ self.requestqueue = queue.Queue()
+ self.handlerthread = threading.Thread(target=self.process_request_thread)
+ self.handlerthread.daemon = False
+
+ self.handlerthread.start()
+
+ signal.signal(signal.SIGTERM, self.sigterm_exception)
+ super().serve_forever()
+ os._exit(0)
+
+ def sigterm_exception(self, signum, stackframe):
+ self.server_close()
+ os._exit(0)
+
+ def server_bind(self):
+ HTTPServer.server_bind(self)
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
+
+ def process_request_thread(self):
+ while not self.quit:
+ try:
+ (request, client_address) = self.requestqueue.get(True)
+ except queue.Empty:
+ continue
+ if request is None:
+ continue
+ try:
+ self.finish_request(request, client_address)
+ except Exception:
+ self.handle_error(request, client_address)
+ finally:
+ self.shutdown_request(request)
+ os._exit(0)
+
+ def process_request(self, request, client_address):
+ self.requestqueue.put((request, client_address))
+
+ def server_close(self):
+ super().server_close()
+ self.quit = True
+ self.requestqueue.put((None, None))
+ self.handlerthread.join()
+
+def create_server(addr, dbname, prefix=''):
class Handler(HashEquivalenceServer):
pass
- Handler.prefix = prefix
- Handler.db = db
+ db = sqlite3.connect(dbname)
db.row_factory = sqlite3.Row
+ Handler.prefix = prefix
+ Handler.db = None
+ Handler.dbname = dbname
+
with contextlib.closing(db.cursor()) as cursor:
cursor.execute('''
- CREATE TABLE IF NOT EXISTS tasks_v1 (
+ CREATE TABLE IF NOT EXISTS tasks_v2 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
method TEXT NOT NULL,
outhash TEXT NOT NULL,
@@ -134,9 +215,16 @@ def create_server(addr, db, prefix=''):
PV TEXT,
PR TEXT,
task TEXT,
- outhash_siginfo TEXT
+ outhash_siginfo TEXT,
+
+ UNIQUE(method, outhash, taskhash)
)
''')
+ cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup ON tasks_v2 (method, taskhash)')
+ cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup ON tasks_v2 (method, outhash)')
+
+ ret = ThreadedHTTPServer(addr, Handler)
+
+ logger.info('Starting server on %s\n', ret.server_port)
- logger.info('Starting server on %s', addr)
- return HTTPServer(addr, Handler)
+ return ret
diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py
index 8300a2559..6845b5388 100644
--- a/poky/bitbake/lib/hashserv/tests.py
+++ b/poky/bitbake/lib/hashserv/tests.py
@@ -6,30 +6,31 @@
#
import unittest
-import threading
+import multiprocessing
import sqlite3
import hashlib
import urllib.request
import json
+import tempfile
from . import create_server
class TestHashEquivalenceServer(unittest.TestCase):
def setUp(self):
- # Start an in memory hash equivalence server in the background bound to
+ # Start a hash equivalence server in the background bound to
# an ephemeral port
- db = sqlite3.connect(':memory:', check_same_thread=False)
- self.server = create_server(('localhost', 0), db)
+ self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-")
+ self.server = create_server(('localhost', 0), self.dbfile.name)
self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
- self.server_thread = threading.Thread(target=self.server.serve_forever)
+ self.server_thread = multiprocessing.Process(target=self.server.serve_forever)
+ self.server_thread.daemon = True
self.server_thread.start()
def tearDown(self):
# Shutdown server
s = getattr(self, 'server', None)
if s is not None:
- self.server.shutdown()
+ self.server_thread.terminate()
self.server_thread.join()
- self.server.server_close()
def send_get(self, path):
url = '%s/%s' % (self.server_addr, path)