diff options
author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2019-04-05 22:28:33 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2019-04-05 22:31:28 +0300 |
commit | 193236933b0f4ab91b1625b64e2187e2db4e0e8f (patch) | |
tree | e12769d7c76d8b0517d6de3d3c72189753d253ed /poky/bitbake/lib/bb/runqueue.py | |
parent | bd93df9478f2f56ffcbc8cb88f1709c735dcd85b (diff) | |
download | openbmc-193236933b0f4ab91b1625b64e2187e2db4e0e8f.tar.xz |
reset upstream subtrees to HEAD
Reset the following subtrees on HEAD:
poky: 8217b477a1(master)
meta-xilinx: 64aa3d35ae(master)
meta-openembedded: 0435c9e193(master)
meta-raspberrypi: 490a4441ac(master)
meta-security: cb6d1c85ee(master)
Squashed patches:
meta-phosphor: drop systemd 239 patches
meta-phosphor: mrw-api: use correct install path
Change-Id: I268e2646d9174ad305630c6bbd3fbc1a6105f43d
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/bitbake/lib/bb/runqueue.py')
-rw-r--r-- | poky/bitbake/lib/bb/runqueue.py | 92 |
1 files changed, 60 insertions, 32 deletions
diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 383c183235..329cda33a4 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -37,11 +37,12 @@ from bb import monitordisk import subprocess import pickle from multiprocessing import Process +import shlex bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") -__find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' ) +__find_sha256__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{64}(?![a-z0-9])' ) def fn_from_tid(tid): return tid.rsplit(":", 1)[0] @@ -351,6 +352,7 @@ class RunTaskEntry(object): self.depends = set() self.revdeps = set() self.hash = None + self.unihash = None self.task = None self.weight = 1 @@ -390,6 +392,9 @@ class RunQueueData: def get_task_hash(self, tid): return self.runtaskentries[tid].hash + def get_task_unihash(self, tid): + return self.runtaskentries[tid].unihash + def get_user_idstring(self, tid, task_name_suffix = ""): return tid + task_name_suffix @@ -1161,18 +1166,21 @@ class RunQueueData: if len(self.runtaskentries[tid].depends - dealtwith) == 0: dealtwith.add(tid) todeal.remove(tid) - procdep = [] - for dep in self.runtaskentries[tid].depends: - procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep)) - (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) - self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc]) - task = self.runtaskentries[tid].task + self.prepare_task_hash(tid) bb.parse.siggen.writeout_file_checksum_cache() #self.dump_data() return len(self.runtaskentries) + def prepare_task_hash(self, tid): + procdep = [] + for dep in self.runtaskentries[tid].depends: + procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep)) + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc]) + self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(taskfn + "." + taskname) + def dump_data(self): """ Dump some debug information on the internal data structures @@ -1224,28 +1232,23 @@ class RunQueue: if fakeroot: magic = magic + "beef" mcdata = self.cooker.databuilder.mcdata[mc] - fakerootcmd = mcdata.getVar("FAKEROOTCMD") + fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD")) fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split() env = os.environ.copy() for key, value in (var.split('=') for var in fakerootenv): env[key] = value - worker = subprocess.Popen([fakerootcmd, "bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) + worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) else: worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE) bb.utils.nonblockingfd(worker.stdout) workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec) - runqhash = {} - for tid in self.rqdata.runtaskentries: - runqhash[tid] = self.rqdata.runtaskentries[tid].hash - workerdata = { "taskdeps" : self.rqdata.dataCaches[mc].task_deps, "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv, "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs, "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv, "sigdata" : bb.parse.siggen.get_taskdata(), - "runq_hash" : runqhash, "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel, "logdefaultverbose" : bb.msg.loggerDefaultVerbose, "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, @@ -1387,6 +1390,26 @@ class RunQueue: cache[tid] = iscurrent return iscurrent + def validate_hash(self, *, sq_fn, sq_task, sq_hash, sq_hashfn, siginfo, sq_unihash, d): + locs = {"sq_fn" : sq_fn, "sq_task" : sq_task, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, + "sq_unihash" : sq_unihash, "siginfo" : siginfo, "d" : d} + + hashvalidate_args = ("(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=siginfo, sq_unihash=sq_unihash)", + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=siginfo)", + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)") + + for args in hashvalidate_args[:-1]: + try: + call = self.hashvalidate + args + return bb.utils.better_eval(call, locs) + except TypeError: + continue + + # Call the last entry without a try...catch to propagate any thrown + # TypeError + call = self.hashvalidate + hashvalidate_args[-1] + return bb.utils.better_eval(call, locs) + def _execute_runqueue(self): """ Run the tasks in a queue prepared by rqdata.prepare() @@ -1558,6 +1581,7 @@ class RunQueue: valid = [] sq_hash = [] sq_hashfn = [] + sq_unihash = [] sq_fn = [] sq_taskname = [] sq_task = [] @@ -1576,16 +1600,13 @@ class RunQueue: sq_fn.append(fn) sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn]) sq_hash.append(self.rqdata.runtaskentries[tid].hash) + sq_unihash.append(self.rqdata.runtaskentries[tid].unihash) sq_taskname.append(taskname) sq_task.append(tid) - locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data } - try: - call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)" - valid = bb.utils.better_eval(call, locs) - # Handle version with no siginfo parameter - except TypeError: - call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" - valid = bb.utils.better_eval(call, locs) + + valid = self.validate_hash(sq_fn=sq_fn, sq_task=sq_taskname, sq_hash=sq_hash, sq_hashfn=sq_hashfn, + siginfo=True, sq_unihash=sq_unihash, d=self.cooker.data) + for v in valid: valid_new.add(sq_task[v]) @@ -1667,7 +1688,7 @@ class RunQueue: matches = {k : v for k, v in iter(matches.items()) if h not in k} if matches: latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1] - prevh = __find_md5__.search(latestmatch).group(0) + prevh = __find_sha256__.search(latestmatch).group(0) output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb) bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output)) @@ -2042,6 +2063,8 @@ class RunQueueExecuteTasks(RunQueueExecute): taskdepdata = self.build_taskdepdata(task) taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] + taskhash = self.rqdata.get_task_hash(task) + unihash = self.rqdata.get_task_unihash(task) if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): if not mc in self.rq.fakeworker: try: @@ -2051,10 +2074,10 @@ class RunQueueExecuteTasks(RunQueueExecute): self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) @@ -2109,8 +2132,9 @@ class RunQueueExecuteTasks(RunQueueExecute): deps = self.rqdata.runtaskentries[revdep].depends provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] taskhash = self.rqdata.runtaskentries[revdep].hash - taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash] + unihash = self.rqdata.runtaskentries[revdep].unihash deps = self.filtermcdeps(task, deps) + taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] for revdep2 in deps: if revdep2 not in taskdepdata: additional.append(revdep2) @@ -2313,6 +2337,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): if self.rq.hashvalidate: sq_hash = [] sq_hashfn = [] + sq_unihash = [] sq_fn = [] sq_taskname = [] sq_task = [] @@ -2344,14 +2369,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute): sq_fn.append(fn) sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn]) sq_hash.append(self.rqdata.runtaskentries[tid].hash) + sq_unihash.append(self.rqdata.runtaskentries[tid].unihash) sq_taskname.append(taskname) sq_task.append(tid) self.cooker.data.setVar("BB_SETSCENE_STAMPCURRENT_COUNT", len(stamppresent)) - call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" - locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data } - valid = bb.utils.better_eval(call, locs) + valid = self.rq.validate_hash(sq_fn=sq_fn, sq_task=sq_taskname, sq_hash=sq_hash, sq_hashfn=sq_hashfn, + siginfo=False, sq_unihash=sq_unihash, d=self.cooker.data) self.cooker.data.delVar("BB_SETSCENE_STAMPCURRENT_COUNT") @@ -2482,13 +2507,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute): taskdepdata = self.build_taskdepdata(task) taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] + taskhash = self.rqdata.get_task_hash(task) + unihash = self.rqdata.get_task_unihash(task) if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) @@ -2552,7 +2579,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute): deps = getsetscenedeps(revdep) provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] taskhash = self.rqdata.runtaskentries[revdep].hash - taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash] + unihash = self.rqdata.runtaskentries[revdep].unihash + taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] for revdep2 in deps: if revdep2 not in taskdepdata: additional.append(revdep2) |