diff options
Diffstat (limited to 'poky/meta/lib/oe/sstatesig.py')
| -rw-r--r-- | poky/meta/lib/oe/sstatesig.py | 261 |
1 files changed, 50 insertions, 211 deletions
diff --git a/poky/meta/lib/oe/sstatesig.py b/poky/meta/lib/oe/sstatesig.py index 13af16e47..c566ce5a0 100644 --- a/poky/meta/lib/oe/sstatesig.py +++ b/poky/meta/lib/oe/sstatesig.py @@ -59,7 +59,7 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): # is machine specific. # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes) # and we reccomend a kernel-module, we exclude the dependency. - depfn = dep.rsplit(".", 1)[0] + depfn = dep.rsplit(":", 1)[0] if dataCache and isKernel(depfn) and not isKernel(fn): for pkg in dataCache.runrecs[fn]: if " ".join(dataCache.runrecs[fn][pkg]).find("kernel-module-") != -1: @@ -90,8 +90,7 @@ class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) -class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): - name = "OEBasicHash" +class SignatureGeneratorOEBasicHashMixIn(object): def init_rundepcheck(self, data): self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() @@ -129,12 +128,11 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) def get_taskdata(self): - data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata() - return (data, self.lockedpnmap, self.lockedhashfn) + return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata() def set_taskdata(self, data): - coredata, self.lockedpnmap, self.lockedhashfn = data - super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata) + self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3] + super().set_taskdata(data[3:]) def dump_sigs(self, dataCache, options): sigfile = os.getcwd() + "/locked-sigs.inc" @@ -142,8 +140,10 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): self.dump_lockedsigs(sigfile) return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) - def get_taskhash(self, fn, task, deps, dataCache): - h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache) + def get_taskhash(self, tid, deps, dataCache): + h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(tid, deps, dataCache) + + (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) recipename = dataCache.pkg_fn[fn] self.lockedpnmap[fn] = recipename @@ -153,37 +153,27 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): if recipename in self.unlockedrecipes: unlocked = True else: - def get_mc(tid): - tid = tid.rsplit('.', 1)[0] - if tid.startswith('mc:'): - elems = tid.split(':') - return elems[1] def recipename_from_dep(dep): - # The dep entry will look something like - # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, - # ... - - fn = dep.rsplit('.', 1)[0] + fn = bb.runqueue.fn_from_tid(dep) return dataCache.pkg_fn[fn] - mc = get_mc(fn) # If any unlocked recipe is in the direct dependencies then the # current recipe should be unlocked as well. - depnames = [ recipename_from_dep(x) for x in deps if mc == get_mc(x)] + depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)] if any(x in y for y in depnames for x in self.unlockedrecipes): self.unlockedrecipes[recipename] = '' unlocked = True if not unlocked and recipename in self.lockedsigs: if task in self.lockedsigs[recipename]: - k = fn + "." + task h_locked = self.lockedsigs[recipename][task][0] var = self.lockedsigs[recipename][task][1] - self.lockedhashes[k] = h_locked - self.taskhash[k] = h_locked + self.lockedhashes[tid] = h_locked + unihash = super().get_unihash(tid) + self.taskhash[tid] = h_locked #bb.warn("Using %s %s %s" % (recipename, task, h)) - if h != h_locked: + if h != h_locked and h_locked != unihash: self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' % (recipename, task, h, h_locked, var)) @@ -191,37 +181,41 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): #bb.warn("%s %s %s" % (recipename, task, h)) return h + def get_unihash(self, tid): + if tid in self.lockedhashes: + return self.lockedhashes[tid] + return super().get_unihash(tid) + def dump_sigtask(self, fn, task, stampbase, runtime): - k = fn + "." + task - if k in self.lockedhashes: + tid = fn + ":" + task + if tid in self.lockedhashes: return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) def dump_lockedsigs(self, sigfile, taskfilter=None): types = {} - for k in self.runtaskdeps: + for tid in self.runtaskdeps: if taskfilter: - if not k in taskfilter: + if not tid in taskfilter: continue - fn = k.rsplit(".",1)[0] + fn = bb.runqueue.fn_from_tid(tid) t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] t = 't-' + t.replace('_', '-') if t not in types: types[t] = [] - types[t].append(k) + types[t].append(tid) with open(sigfile, "w") as f: l = sorted(types) for t in l: f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) types[t].sort() - sortedk = sorted(types[t], key=lambda k: self.lockedpnmap[k.rsplit(".",1)[0]]) - for k in sortedk: - fn = k.rsplit(".",1)[0] - task = k.rsplit(".",1)[1] - if k not in self.taskhash: + sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) + for tid in sortedtid: + (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) + if tid not in self.taskhash: continue - f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n") + f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n") f.write(' "\n') f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(l))) @@ -229,25 +223,26 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): with open(sigfile, "w") as f: tasks = [] for taskitem in self.taskhash: - (fn, task) = taskitem.rsplit(".", 1) + (fn, task) = taskitem.rsplit(":", 1) pn = self.lockedpnmap[fn] tasks.append((pn, task, fn, self.taskhash[taskitem])) for (pn, task, fn, taskhash) in sorted(tasks): - f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash)) + f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash)) - def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): + def checkhashes(self, sq_data, missed, found, d): warn_msgs = [] error_msgs = [] sstate_missing_msgs = [] - for task in range(len(sq_fn)): - if task not in ret: + for tid in sq_data['hash']: + if tid not in found: for pn in self.lockedsigs: - if sq_hash[task] in iter(self.lockedsigs[pn].values()): - if sq_task[task] == 'do_shared_workdir': + taskname = bb.runqueue.taskname_from_tid(tid) + if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()): + if taskname == 'do_shared_workdir': continue sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" - % (pn, sq_task[task], sq_hash[task])) + % (pn, taskname, sq_data['hash'][tid])) checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") if checklevel == 'warn': @@ -266,176 +261,20 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): if error_msgs: bb.fatal("\n".join(error_msgs)) -class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): +class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash): + name = "OEBasicHash" + +class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash): name = "OEEquivHash" def init_rundepcheck(self, data): super().init_rundepcheck(data) - self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') + self.server = data.getVar('BB_HASHSERVE') + if not self.server: + bb.fatal("OEEquivHash requires BB_HASHSERVE to be set") self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') - self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) - - def get_taskdata(self): - return (self.server, self.method) + super().get_taskdata() - - def set_taskdata(self, data): - self.server, self.method = data[:2] - super().set_taskdata(data[2:]) - - def __get_task_unihash_key(self, task): - # TODO: The key only *needs* to be the taskhash, the task is just - # convenient - return '%s:%s' % (task, self.taskhash[task]) - - def get_stampfile_hash(self, task): - if task in self.taskhash: - # If a unique hash is reported, use it as the stampfile hash. This - # ensures that if a task won't be re-run if the taskhash changes, - # but it would result in the same output hash - unihash = self.unihashes.get(self.__get_task_unihash_key(task)) - if unihash is not None: - return unihash - - return super().get_stampfile_hash(task) - - def get_unihash(self, task): - import urllib - import json - - taskhash = self.taskhash[task] - - key = self.__get_task_unihash_key(task) - - # TODO: This cache can grow unbounded. It probably only needs to keep - # for each task - unihash = self.unihashes.get(key) - if unihash is not None: - return unihash - - # In the absence of being able to discover a unique hash from the - # server, make it be equivalent to the taskhash. The unique "hash" only - # really needs to be a unique string (not even necessarily a hash), but - # making it match the taskhash has a few advantages: - # - # 1) All of the sstate code that assumes hashes can be the same - # 2) It provides maximal compatibility with builders that don't use - # an equivalency server - # 3) The value is easy for multiple independent builders to derive the - # same unique hash from the same input. This means that if the - # independent builders find the same taskhash, but it isn't reported - # to the server, there is a better chance that they will agree on - # the unique hash. - unihash = taskhash - - try: - url = '%s/v1/equivalent?%s' % (self.server, - urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]})) - - request = urllib.request.Request(url) - response = urllib.request.urlopen(request) - data = response.read().decode('utf-8') - - json_data = json.loads(data) - - if json_data: - unihash = json_data['unihash'] - # A unique hash equal to the taskhash is not very interesting, - # so it is reported it at debug level 2. If they differ, that - # is much more interesting, so it is reported at debug level 1 - bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server)) - else: - bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server)) - except urllib.error.URLError as e: - bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) - except (KeyError, json.JSONDecodeError) as e: - bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) - - self.unihashes[key] = unihash - return unihash - - def report_unihash(self, path, task, d): - import urllib - import json - import tempfile - import base64 - import importlib - - taskhash = d.getVar('BB_TASKHASH') - unihash = d.getVar('BB_UNIHASH') - report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1' - tempdir = d.getVar('T') - fn = d.getVar('BB_FILENAME') - key = fn + '.do_' + task + ':' + taskhash - - # Sanity checks - cache_unihash = self.unihashes.get(key) - if cache_unihash is None: - bb.fatal('%s not in unihash cache. Please report this error' % key) - - if cache_unihash != unihash: - bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash)) - - sigfile = None - sigfile_name = "depsig.do_%s.%d" % (task, os.getpid()) - sigfile_link = "depsig.do_%s" % task - - try: - sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') - - locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} - - (module, method) = self.method.rsplit('.', 1) - locs['method'] = getattr(importlib.import_module(module), method) - - outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) - - try: - url = '%s/v1/equivalent' % self.server - task_data = { - 'taskhash': taskhash, - 'method': self.method, - 'outhash': outhash, - 'unihash': unihash, - 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER') - } - - if report_taskdata: - sigfile.seek(0) - - task_data['PN'] = d.getVar('PN') - task_data['PV'] = d.getVar('PV') - task_data['PR'] = d.getVar('PR') - task_data['task'] = task - task_data['outhash_siginfo'] = sigfile.read().decode('utf-8') - - headers = {'content-type': 'application/json'} - - request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers) - response = urllib.request.urlopen(request) - data = response.read().decode('utf-8') - - json_data = json.loads(data) - new_unihash = json_data['unihash'] - - if new_unihash != unihash: - bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) - else: - bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) - except urllib.error.URLError as e: - bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) - except (KeyError, json.JSONDecodeError) as e: - bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) - finally: - if sigfile: - sigfile.close() - - sigfile_link_path = os.path.join(tempdir, sigfile_link) - bb.utils.remove(sigfile_link_path) - - try: - os.symlink(sigfile_name, sigfile_link_path) - except OSError: - pass + if not self.method: + bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set") # Insert these classes into siggen's namespace so it can see and select them bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic @@ -452,7 +291,7 @@ def find_siginfo(pn, taskname, taskhashlist, d): if not taskname: # We have to derive pn and taskname key = pn - splitit = key.split('.bb.') + splitit = key.split('.bb:') taskname = splitit[1] pn = os.path.basename(splitit[0]).split('_')[0] if key.startswith('virtual:native:'): |