From 08902b01500fb82ac050ec2dce9b6c4358075a17 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Tue, 20 Aug 2019 09:16:51 -0400 Subject: poky: subtree update:835f7eac06..20946c63c2 Aaron Chan (1): python3-dbus: Add native and nativesdk variants Adrian Bunk (8): gnome: Remove the gnome class bind: Remove RECIPE_NO_UPDATE_REASON and follow the ESV releases webkitgtk: Reenable on mips mtd-utils: Upgrade to 2.1.1 Change ftp:// URIs to http(s):// webkitgtk: Stop disabling gold on aarch64 and mips grub/libmpc/gdb: Use GNU_MIRROR in more recipes screen: Backport fix for an implicit function declaration Alexander Kanavin (28): btrfs-tools: update 5.1.1 -> 5.2.1 libmodulemd: update to 2.6.0 libwebp: upgrade 1.0.2 -> 1.0.3 createrepo-c: upgrade 0.14.2 -> 0.14.3 webkitgtk: upgrade 2.24.2 -> 2.24.3 bzip2: fix upstream version check stress-ng: add a recipe that replaces the original stress meson: update 0.50.1 -> 0.51.1 meson.bbclass: do not pass native compiler/linker flags via command line meson: add a backported patch to address vala cross-compilation errors libedit: fix upstream verison check maintainers.inc: assign acpica to Ross stress-ng: add a patch to remove unneeded bash dependency elfutils: use PRIVATE_LIBS for the ptest package apt: add a missing perl runtime dependency attr: add a missing perl runtime dependency ofono: correct the python3 runtime dependency bluez5: correct the python3 runtime dependency local.conf.sample: do not add sdl to nativesdk qemu config maintainers.inc: give python recipes to Oleksandr Kravchuk python-numpy: remove the python 2.x version of the recipe python-scons: remove the python 2.x version of the recipe python-nose: remove the python 2.x version of the recipe lib/oeqa/utils/qemurunner.py: add runqemuparams after kvm/nographic/snapshot/slirp mesa: enable glx-tls option in native and nativesdk builds insane.bbclass: in file-rdeps do not look into RDEPENDS recursively sudo: correct SRC_URI ovmf: fix upstream version check Andreas Obergschwandtner (1): bzip2: set the autoconf package version to the recipe version Anuj Mittal (11): mpg123: upgrade 1.25.10 -> 1.25.11 libsdl: remove pulseaudio: don't include consolekit when systemd is enabled libsdl2: upgrade 2.0.9 -> 2.0.10 grub: upgrade 2.02 -> 2.04 patch: fix CVE-2019-13636 python: fix CVE-2018-20852 python: CVE-2019-9947 is same as CVE-2019-9740 libtasn1: upgrade 4.13 -> 4.14 pango: upgrade 1.42.4 -> 1.44.3 harfbuzz: upgrade 2.4.0 -> 2.5.3 Bartosz Golaszewski (1): qemu: add a patch fixing the native build on newer kernels Bedel, Alban (3): rng-tools: start rngd early in the boot process again kernel-uboot: remove useless special casing of arm64 Image boost: Fix build and enable context and coroutines on aarch64 Bruce Ashfield (2): linux-yocto/4.19: update to v4.19.61 linux-yocto-dev: bump to 5.3-rcX Changqing Li (6): runqemu: add lockfile for port used when slirp enabled runqemu: fix get portlock fail for multi users qemuboot-x86: move QB_SYSTEM_NAME to corresponding conf genericx86-64.conf/genericx86.conf: add QB_SYSTEM_NAME grub/grub-efi: fix conflict for aach64 go-runtime: remove conflict files from -dev packages Chen Qi (1): sudo: use nonarch_libdir instead of libdir for tmpfiles.d Chin Huat Ang (1): cve-update-db-native: fix https proxy issues Chris Laplante via bitbake-devel (1): bitbake: fetch2/wget: avoid 'maximum recursion depth' RuntimeErrors when handling 403 codes Daniel Ammann (2): image_types: Remove remnants of hdddirect bitbake: toaster: Sync list of fs_types with oe-core Denys Dmytriyenko (2): wayland-protocols: upgrade 1.17 -> 1.18 weston: upgrade 6.0.0 -> 6.0.1 Diego Rondini (1): image_types.bbclass: make gzipped images rsyncable Dmitry Eremin-Solenikov (1): kernel.bbclass: fix installation of modules signing certificates Frederic Ouellet (1): systemd: Add partial support of drop-in configuration files to systemd-systemctl-native Hongxu Jia (1): grub: add grub-native Jason Wessel (6): sqlite3: Fix zlib determinism problem pseudo: Fix openat() with a symlink pointing to a directory image_types_wic.bbclass: Copy the .wks and .env files to deploy image dir wic: Add partition type for msdos partition tables wic: Make disk partition size consistently computed dpkg: Provide update-alternative for start-stop-daemon Johann Fridriksson (1): ruby: Adding zlib-native to native dependencies Joshua Lock via Openembedded-core (3): sstate: fix log message classes/sstate: don't use unsigned sstate when verification enabled classes/sstate: regenerate sstate when signing enabled Joshua Watt (1): bitbake: hashserv: SQL Optimizations Kai Kang (3): subversion: add packageconfig boost epiphany: set imcompatible with tune mips e2fsprogs: 1.44.5 -> 1.45.3 Khem Raj (23): strace: Upgrade to 5.2 linux-libc-header: Fix ptrace.h and prctl.h conflict on aarch64 libnss-nis: Fix build with glibc 2.30 lttng-ust: Check for gettid libc API ltp: Fix build with glibc 2.30 lttng-tools: Fix build with glibc 2.30 xserver-xorg: Backport patch to remove using sys/io.h Apache-2.0-with-LLVM-exception: Add new license file libedit: Move from meta-oe groff: Fix math.h inclusion from system headers issue webkitgtk: Fix compile failures with clang glibc: Update to glibc 2.30 virglrender: Fix endianness check on musl syslinux: Override hardcoded toolnames in Makefile systemd-boot: Add option to specify cross objcopy and use it mesa,llvm,meson: Update llvm to 8.0.1 plus define and use LLVM version globally musl: Update to master tip oeqa/buildgalculator.py: Add dependency on gtk+3 oeqa/parselogs: grep for exact errors list keywords gcc-runtime: Move content from gcclibdir into libdir gdb: Do not set musl specific CFLAGS linuxloader: Add entries for riscv64 musl: Delete GLIBC_LDSO before creating symlink with lnr Luca Boccassi (1): python3-pygobject: remove python3-setuptools from RDEPENDS Mads Andreasen (1): bitbake: fetch2/npm: Use npm pack to download node modules instead of wget Mark Hatle (2): glibc-package.inc: Add linux-libc-headers-dev to glibc-dev bitbake: layerindexlib: Fix parsing of recursive layer dependencies Martin Jansa (3): icecc.bbclass: catch subprocess.CalledProcessError powertop: import a fix from buildroot meson: backport fix for builds with -Werror=return-type Ming Liu (5): libx11-compose-data: add recipe libxkbcommon: RDEPENDS on libx11 compose data weston: change to use meson build system license_image.bbclass: drop invalid comments opensbi: handle deploy task under sstate Naveen Saini (2): gdk-pixbuf: enable x11 PACKAGECONFIG option image_types_wic: add syslinux-native dependency conditional Oleksandr Kravchuk (17): python3-pip: update to 19.2.1 python3-git: update to 2.1.12 ethtool: update to 5.2 python3-git: update to 2.1.13 xorgproto: update to 2019.1 xserver-xorg: update to 1.20.5 ell: update to 0.21 libinput: update to 1.14.0 wpa-supplicant: update to 2.9 aspell: update to 0.60.7 linux-firmware: add PE back xf86-input-libinput: update to 0.29.0 git: update to 2.22.1 xrandr: update to 1.5.1 python3-git: update to 3.0.0 librepo: update to 1.10.5 libevent: update to 2.1.11 Pascal Bach (2): cmake: 3.14.5 -> 3.15.1 cmake: 3.15.1 -> 3.15.2 Paul Eggleton (2): scripts/create-pull-request: improve handling of non-SSH remote URLs scripts/create-pull-request: fix putting subject containing / into cover letter Piotr Tworek (2): pulseaudio: Backport upstream fix new alsa compatibility. libdrm: Move amdgpu.ids file into libdrm-amdgpu package. Randy MacLeod (1): ptest-runner: update from 2.3.1 to 2.3.2 Rasmus Villemoes (1): iproute2: drop pointless configure-cross.patch Ricardo Neri (5): ovmf: Update to version edk2-stable201905 ovmf: Set PV ovmf: Use HOSTTOOLS' python3 ovmf: Generate test Platform key and first Key Exchange Key runqemu: Add support to handle EnrollDefaultKeys PK/KEK1 certificate Ricardo Ribalda Delgado (2): packagegroup-core-base-utils: Make it machine specific inetutils: Fix abort on invalid files Richard Purdie (50): package: Improve determinism sstate: Reduce race windows bitbake: siggen: Import unihash code from OE-Core bitbake: cache: Add SimpleCache class bitbake: runqueue: Improve scenequeue processing logic bitbake: siggen: Add new unitaskhashes data variable which is cached bitbake: siggen: Convert to use self.unitaskhashes bitbake: runqueue: Enable dynamic task adjustment to hash equivalency bitbake: runqueue: Improve determinism bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE bitbake: hashserv: Turn off sqlite synchronous mode bitbake: prserv: Use a memory journal bitbake: hashserv: Use separate threads for answering requests and handling them bitbake: hashserv: Switch from threads to multiprocessing bitbake: runqueue: Clean up BB_HASHCHECK_FUNCTION API bitbake: siggen: Clean up task reference formats bitbake: build/utils: Drop bb.build.FuncFailed bitbake: tests/runqueue: Add hashserv+runqueue test bitbake: bitbake: Bump version to 1.43.1 for API changes sanity.conf: Require bitbake 1.43.1 classes/lib: Remove bb.build.FuncFailed sstatesig: Move unihash siggen code to bitbake sstatesig: Add debug for incorrect hash server settings sstatesig: Adpat to recent bitbake hash equiv runqueue changes sstatesig: Update to handle BB_HASHSERVE sstate/sstatesig: Update to new form of BB_HASHCHECK_FUNCTION sstatesig: Updates to match bitbake siggen changes gstreamer: Add fix for glibc 2.30 sstatesig: Fix leftover splitting issue from siggen change python3-pygobject: Add missing pkgutil RDEPENDS bitbake: runqueue: Fix corruption issue bitbake: runqueue: Improve setscene task handling logic bitbake: tests/runqueue: Add further hash equivalence tests bitbake: cooker: Improve hash server startup code to avoid exit tracebacks bitbake: runqueue: Wait for covered tasks to complete before trying setscene bitbake: runqueue: Fix next_buildable_task performance problem bitbake: runqueue: Improve scenequeue debugging bitbake: runqueue: Recompute holdoff tasks from scratch bitbake: runqueue: Fix event timing race bitbake: runqueue: Drop debug statement causing performance issues bitbake: runqueue: Add further debug information bitbake: runqueue: Add missing setscene task corner case bitbake: runqueue: Ensure we clear the stamp cache poky: Retire opensuse 42.3 from SANITY_TESTED_DISTROS gcc-cross-canadian: Drop obsolete shlibs exclusion bitbake: tests/runqueue: Fix tests bitbake: runqueue: Fix data corruption problem bitbake: runqueue: Ensure data is handled correctly bitbake: hashserv: Ensure we don't accumulate sockets in TIME_WAIT state bitbake: runqueue: Ensure target_tids is filtered Robert Yang (3): bitbake: cooker: Cleanup the queue before call process.join() bitbake: knotty: Fix for the Second Keyboard Interrupt bitbake: bitbake: server/process: Handle BBHandledException to avoid unexpected exceptions Ross Burton (23): libidn2: remove build paths from libidn2.pc gnutls: don't use HOSTTOOLS_DIR/bash as a shell on target libical: upgrade to 3.0.5 perl: fix whitespace perl: add PACKAGECONFIG for db fortran-helloworld: neaten recipe python3: remove empty python3-distutils-staticdev python3: support recommends in manifest python3: split out the Windows distutils installer stubs insane: check if the recipe incorrectly uses DEPENDS_${PN} libxx86misc: remove this now redundant library xserver-xorg: clean up xorgproto dependencies xserver-xorg: add PACKAGECONFIG for DGA xdpyinfo: don't depend on DGA libxx86dga: remove obsolete client libary xserver-xorg: remove embedded build path in the source libx11: update to 1.6.8 sanity: update for new bb.build.exec_func() behaviour libx11-diet: remove qemu: fix patch Upstream-Status xserver-xorg: refresh build path removal patch waffle: upgrade 1.5.2 -> 1.6.0 libx11: replace libtool patch with upstreamed patch Tim Blechmann (1): deb: allow custom dpkg command Trevor Gamblin (2): gzip: update ptest package dependencies patch: fix CVE-2019-13638 Wenlin Kang (1): db: add switch for building database verification Will Page (1): uboot: fixes to uboot-extlinux-config attribute values William Bourque (1): meta/lib/oeqa: Remove ext4 for bootimg-biosplusefi Yi Zhao (1): libx11-compose-data: upgrade 1.6.7 -> 1.6.8 Yuan Chao (4): glib-2.0:upgrade 2.60.5 -> 2.60.6 nettle:upgrade 3.4.1 -> 3.5.1 python3-pbr:upgrade 5.4.1 -> 5.4.2 gpgme:upgrade 1.13.0 -> 1.13.1 Zang Ruochen (8): msmtp: upgrade 1.8.4 -> 1.8.5 curl: upgrade 7.65.2 -> 7.65.3 iso-codes: upgrade 4.2 -> 4.3 python-scons:upgrade 3.0.5 -> 3.1.0 libgudev:upgrade 232 -> 233 libglu:upgrade 9.0.0 -> 9.0.1 man-db:upgrade 2.8.5 -> 2.8.6.1 libnewt:upgrade 0.52.20 -> 0.52.21 Zheng Ruoqin (1): python3-mako: 1.0.14 -> 1.1.0 Zoltan Kuscsik (1): kmscube: update to latest revision Change-Id: I2cd1a0d59da46725b1aba5a79b63eb6121b3c79e Signed-off-by: Brad Bishop --- poky/bitbake/lib/bb/siggen.py | 322 +++++++++++++++++++++++++++++++++--------- 1 file changed, 255 insertions(+), 67 deletions(-) (limited to 'poky/bitbake/lib/bb/siggen.py') diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 6a729f3b1e..b503559305 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -12,6 +12,7 @@ import bb.data import difflib import simplediff from bb.checksum import FileChecksumCache +from bb import runqueue logger = logging.getLogger('BitBake.SigGen') @@ -41,17 +42,17 @@ class SignatureGenerator(object): self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} + self.unitaskhashes = {} def finalise(self, fn, d, varient): return - def get_unihash(self, task): - return self.taskhash[task] + def get_unihash(self, tid): + return self.taskhash[tid] - def get_taskhash(self, fn, task, deps, dataCache): - k = fn + "." + task - self.taskhash[k] = hashlib.sha256(k.encode("utf-8")).hexdigest() - return self.taskhash[k] + def get_taskhash(self, tid, deps, dataCache): + self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest() + return self.taskhash[tid] def writeout_file_checksum_cache(self): """Write/update the file checksum cache onto disk""" @@ -73,14 +74,23 @@ class SignatureGenerator(object): return def get_taskdata(self): - return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) + return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes) def set_taskdata(self, data): - self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data + self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes = data def reset(self, data): self.__init__(data) + def get_taskhashes(self): + return self.taskhash, self.unitaskhashes + + def set_taskhashes(self, hashes): + self.taskhash, self.unitaskhashes = hashes + + def save_unitaskhashes(self): + return + class SignatureGeneratorBasic(SignatureGenerator): """ @@ -96,7 +106,6 @@ class SignatureGeneratorBasic(SignatureGenerator): self.taints = {} self.gendeps = {} self.lookupcache = {} - self.pkgnameextract = re.compile(r"(?P.*)\..*") self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) @@ -107,6 +116,9 @@ class SignatureGeneratorBasic(SignatureGenerator): else: self.checksum_cache = None + self.unihash_cache = bb.cache.SimpleCache("1") + self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {}) + def init_rundepcheck(self, data): self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None if self.taskwhitelist: @@ -122,16 +134,16 @@ class SignatureGeneratorBasic(SignatureGenerator): taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) for task in tasklist: - k = fn + "." + task - if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]: - bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k])) + tid = fn + ":" + task + if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]: + bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid])) bb.error("The following commands may help:") cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task) # Make sure sigdata is dumped before run printdiff bb.error("%s -Snone" % cmd) bb.error("Then:") bb.error("%s -Sprintdiff\n" % cmd) - self.basehash[k] = basehash[k] + self.basehash[tid] = basehash[tid] self.taskdeps[fn] = taskdeps self.gendeps[fn] = gendeps @@ -158,7 +170,7 @@ class SignatureGeneratorBasic(SignatureGenerator): # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: - d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) + d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task]) def rundep_check(self, fn, recipename, task, dep, depname, dataCache): # Return True if we should keep the dependency, False to drop it @@ -178,33 +190,26 @@ class SignatureGeneratorBasic(SignatureGenerator): pass return taint - def get_taskhash(self, fn, task, deps, dataCache): + def get_taskhash(self, tid, deps, dataCache): - mc = '' - if fn.startswith('mc:'): - mc = fn.split(':')[1] - k = fn + "." + task + (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) - data = dataCache.basetaskhash[k] - self.basehash[k] = data - self.runtaskdeps[k] = [] - self.file_checksum_values[k] = [] + data = dataCache.basetaskhash[tid] + self.basehash[tid] = data + self.runtaskdeps[tid] = [] + self.file_checksum_values[tid] = [] recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): - pkgname = self.pkgnameextract.search(dep).group('fn') - if mc: - depmc = pkgname.split(':')[1] - if mc != depmc: - continue - if dep.startswith("mc:") and not mc: + (depmc, _, deptaskname, depfn) = bb.runqueue.split_tid_mcfn(dep) + if mc != depmc: continue - depname = dataCache.pkg_fn[pkgname] + depname = dataCache.pkg_fn[depfn] if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): continue if dep not in self.taskhash: bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) data = data + self.get_unihash(dep) - self.runtaskdeps[k].append(dep) + self.runtaskdeps[tid].append(dep) if task in dataCache.file_checksums[fn]: if self.checksum_cache: @@ -212,7 +217,7 @@ class SignatureGeneratorBasic(SignatureGenerator): else: checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) for (f,cs) in checksums: - self.file_checksum_values[k].append((f,cs)) + self.file_checksum_values[tid].append((f,cs)) if cs: data = data + cs @@ -222,16 +227,16 @@ class SignatureGeneratorBasic(SignatureGenerator): import uuid taint = str(uuid.uuid4()) data = data + taint - self.taints[k] = "nostamp:" + taint + self.taints[tid] = "nostamp:" + taint taint = self.read_taint(fn, task, dataCache.stamp[fn]) if taint: data = data + taint - self.taints[k] = taint - logger.warning("%s is tainted from a forced run" % k) + self.taints[tid] = taint + logger.warning("%s is tainted from a forced run" % tid) h = hashlib.sha256(data.encode("utf-8")).hexdigest() - self.taskhash[k] = h + self.taskhash[tid] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) return h @@ -244,17 +249,20 @@ class SignatureGeneratorBasic(SignatureGenerator): bb.fetch2.fetcher_parse_save() bb.fetch2.fetcher_parse_done() + def save_unitaskhashes(self): + self.unihash_cache.save(self.unitaskhashes) + def dump_sigtask(self, fn, task, stampbase, runtime): - k = fn + "." + task + tid = fn + ":" + task referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase referencestamp = runtime[11:] - elif runtime and k in self.taskhash: - sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] + elif runtime and tid in self.taskhash: + sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[tid] else: - sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] + sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid] bb.utils.mkdirhier(os.path.dirname(sigfile)) @@ -263,7 +271,7 @@ class SignatureGeneratorBasic(SignatureGenerator): data['basewhitelist'] = self.basewhitelist data['taskwhitelist'] = self.taskwhitelist data['taskdeps'] = self.taskdeps[fn][task] - data['basehash'] = self.basehash[k] + data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] @@ -273,30 +281,30 @@ class SignatureGeneratorBasic(SignatureGenerator): data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] - if runtime and k in self.taskhash: - data['runtaskdeps'] = self.runtaskdeps[k] - data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] + if runtime and tid in self.taskhash: + data['runtaskdeps'] = self.runtaskdeps[tid] + data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.get_unihash(dep) - data['taskhash'] = self.taskhash[k] + data['taskhash'] = self.taskhash[tid] taint = self.read_taint(fn, task, referencestamp) if taint: data['taint'] = taint - if runtime and k in self.taints: - if 'nostamp:' in self.taints[k]: - data['taint'] = self.taints[k] + if runtime and tid in self.taints: + if 'nostamp:' in self.taints[tid]: + data['taint'] = self.taints[tid] computed_basehash = calc_basehash(data) - if computed_basehash != self.basehash[k]: - bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) - if runtime and k in self.taskhash: + if computed_basehash != self.basehash[tid]: + bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[tid], tid)) + if runtime and tid in self.taskhash: computed_taskhash = calc_taskhash(data) - if computed_taskhash != self.taskhash[k]: - bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) - sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) + if computed_taskhash != self.taskhash[tid]: + bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid)) + sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash) fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: @@ -316,34 +324,33 @@ class SignatureGeneratorBasic(SignatureGenerator): if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task - (mc, _, _) = bb.runqueue.split_tid(tid) - k = fn + "." + task - if k not in self.taskhash: + mc = bb.runqueue.mc_from_tid(tid) + if tid not in self.taskhash: continue - if dataCaches[mc].basetaskhash[k] != self.basehash[k]: - bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) - bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) + if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]: + bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid) + bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid])) self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) class SignatureGeneratorBasicHash(SignatureGeneratorBasic): name = "basichash" - def get_stampfile_hash(self, task): - if task in self.taskhash: - return self.taskhash[task] + def get_stampfile_hash(self, tid): + if tid in self.taskhash: + return self.taskhash[tid] # If task is not in basehash, then error - return self.basehash[task] + return self.basehash[tid] def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): if taskname != "do_setscene" and taskname.endswith("_setscene"): - k = fn + "." + taskname[:-9] + tid = fn + ":" + taskname[:-9] else: - k = fn + "." + taskname + tid = fn + ":" + taskname if clean: h = "*" else: - h = self.get_stampfile_hash(k) + h = self.get_stampfile_hash(tid) return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') @@ -354,6 +361,187 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic): bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) bb.build.write_taint(task, d, fn) +class SignatureGeneratorUniHashMixIn(object): + def get_taskdata(self): + return (self.server, self.method) + super().get_taskdata() + + def set_taskdata(self, data): + self.server, self.method = data[:2] + super().set_taskdata(data[2:]) + + def __get_task_unihash_key(self, tid): + # TODO: The key only *needs* to be the taskhash, the tid is just + # convenient + return '%s:%s' % (tid, self.taskhash[tid]) + + def get_stampfile_hash(self, tid): + if tid in self.taskhash: + # If a unique hash is reported, use it as the stampfile hash. This + # ensures that if a task won't be re-run if the taskhash changes, + # but it would result in the same output hash + unihash = self.unitaskhashes.get(self.__get_task_unihash_key(tid), None) + if unihash is not None: + return unihash + + return super().get_stampfile_hash(tid) + + def set_unihash(self, tid, unihash): + self.unitaskhashes[self.__get_task_unihash_key(tid)] = unihash + + def get_unihash(self, tid): + import urllib + import json + + taskhash = self.taskhash[tid] + + key = self.__get_task_unihash_key(tid) + + # TODO: This cache can grow unbounded. It probably only needs to keep + # for each task + unihash = self.unitaskhashes.get(key, None) + if unihash is not None: + return unihash + + # In the absence of being able to discover a unique hash from the + # server, make it be equivalent to the taskhash. The unique "hash" only + # really needs to be a unique string (not even necessarily a hash), but + # making it match the taskhash has a few advantages: + # + # 1) All of the sstate code that assumes hashes can be the same + # 2) It provides maximal compatibility with builders that don't use + # an equivalency server + # 3) The value is easy for multiple independent builders to derive the + # same unique hash from the same input. This means that if the + # independent builders find the same taskhash, but it isn't reported + # to the server, there is a better chance that they will agree on + # the unique hash. + unihash = taskhash + + try: + url = '%s/v1/equivalent?%s' % (self.server, + urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[tid]})) + + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + data = response.read().decode('utf-8') + + json_data = json.loads(data) + + if json_data: + unihash = json_data['unihash'] + # A unique hash equal to the taskhash is not very interesting, + # so it is reported it at debug level 2. If they differ, that + # is much more interesting, so it is reported at debug level 1 + bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server)) + else: + bb.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server)) + except urllib.error.URLError as e: + bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) + except (KeyError, json.JSONDecodeError) as e: + bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) + + self.unitaskhashes[key] = unihash + return unihash + + def report_unihash(self, path, task, d): + import urllib + import json + import tempfile + import base64 + import importlib + + taskhash = d.getVar('BB_TASKHASH') + unihash = d.getVar('BB_UNIHASH') + report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1' + tempdir = d.getVar('T') + fn = d.getVar('BB_FILENAME') + key = fn + ':do_' + task + ':' + taskhash + + # Sanity checks + cache_unihash = self.unitaskhashes.get(key, None) + if cache_unihash is None: + bb.fatal('%s not in unihash cache. Please report this error' % key) + + if cache_unihash != unihash: + bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash)) + + sigfile = None + sigfile_name = "depsig.do_%s.%d" % (task, os.getpid()) + sigfile_link = "depsig.do_%s" % task + + try: + sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') + + locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} + + if "." in self.method: + (module, method) = self.method.rsplit('.', 1) + locs['method'] = getattr(importlib.import_module(module), method) + outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) + else: + outhash = bb.utils.better_eval(self.method + '(path, sigfile, task, d)', locs) + + try: + url = '%s/v1/equivalent' % self.server + task_data = { + 'taskhash': taskhash, + 'method': self.method, + 'outhash': outhash, + 'unihash': unihash, + 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER') + } + + if report_taskdata: + sigfile.seek(0) + + task_data['PN'] = d.getVar('PN') + task_data['PV'] = d.getVar('PV') + task_data['PR'] = d.getVar('PR') + task_data['task'] = task + task_data['outhash_siginfo'] = sigfile.read().decode('utf-8') + + headers = {'content-type': 'application/json'} + + request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers) + response = urllib.request.urlopen(request) + data = response.read().decode('utf-8') + + json_data = json.loads(data) + new_unihash = json_data['unihash'] + + if new_unihash != unihash: + bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) + bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d) + else: + bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) + except urllib.error.URLError as e: + bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) + except (KeyError, json.JSONDecodeError) as e: + bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) + finally: + if sigfile: + sigfile.close() + + sigfile_link_path = os.path.join(tempdir, sigfile_link) + bb.utils.remove(sigfile_link_path) + + try: + os.symlink(sigfile_name, sigfile_link_path) + except OSError: + pass + + +# +# Dummy class used for bitbake-selftest +# +class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureGeneratorBasicHash): + name = "TestEquivHash" + def init_rundepcheck(self, data): + super().init_rundepcheck(data) + self.server = "http://" + data.getVar('BB_HASHSERVE') + self.method = "sstate_output_hash" + + def dump_this_task(outfile, d): import bb.parse fn = d.getVar("BB_FILENAME") -- cgit v1.2.3