diff options
Diffstat (limited to 'poky/bitbake/lib/bb/runqueue.py')
-rw-r--r-- | poky/bitbake/lib/bb/runqueue.py | 160 |
1 files changed, 69 insertions, 91 deletions
diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 10511a09dc..8ae3fe85f1 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -385,7 +385,6 @@ class RunQueueData: self.rq = rq self.warn_multi_bb = False - self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or "" self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split() self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData, targets) self.setscenewhitelist_checked = False @@ -547,7 +546,7 @@ class RunQueueData: next_points.append(revdep) task_done[revdep] = True endpoints = next_points - if len(next_points) == 0: + if not next_points: break # Circular dependency sanity check @@ -589,7 +588,7 @@ class RunQueueData: found = False for mc in self.taskData: - if len(taskData[mc].taskentries) > 0: + if taskData[mc].taskentries: found = True break if not found: @@ -773,7 +772,7 @@ class RunQueueData: # Find the dependency chain endpoints endpoints = set() for tid in self.runtaskentries: - if len(deps[tid]) == 0: + if not deps[tid]: endpoints.add(tid) # Iterate the chains collating dependencies while endpoints: @@ -784,11 +783,11 @@ class RunQueueData: cumulativedeps[dep].update(cumulativedeps[tid]) if tid in deps[dep]: deps[dep].remove(tid) - if len(deps[dep]) == 0: + if not deps[dep]: next.add(dep) endpoints = next #for tid in deps: - # if len(deps[tid]) != 0: + # if deps[tid]: # bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid])) # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to @@ -926,39 +925,37 @@ class RunQueueData: # # Once all active tasks are marked, prune the ones we don't need. - delcount = {} - for tid in list(self.runtaskentries.keys()): - if tid not in runq_build: - delcount[tid] = self.runtaskentries[tid] - del self.runtaskentries[tid] - # Handle --runall if self.cooker.configuration.runall: # re-run the mark_active and then drop unused tasks from new list + reduced_tasklist = set(self.runtaskentries.keys()) + for tid in list(self.runtaskentries.keys()): + if tid not in runq_build: + reduced_tasklist.remove(tid) runq_build = {} for task in self.cooker.configuration.runall: if not task.startswith("do_"): task = "do_{0}".format(task) runall_tids = set() - for tid in list(self.runtaskentries): + for tid in reduced_tasklist: wanttid = "{0}:{1}".format(fn_from_tid(tid), task) - if wanttid in delcount: - self.runtaskentries[wanttid] = delcount[wanttid] if wanttid in self.runtaskentries: runall_tids.add(wanttid) for tid in list(runall_tids): - mark_active(tid,1) + mark_active(tid, 1) if self.cooker.configuration.force: invalidate_task(tid, False) - for tid in list(self.runtaskentries.keys()): - if tid not in runq_build: - delcount[tid] = self.runtaskentries[tid] - del self.runtaskentries[tid] + delcount = set() + for tid in list(self.runtaskentries.keys()): + if tid not in runq_build: + delcount.add(tid) + del self.runtaskentries[tid] - if len(self.runtaskentries) == 0: + if self.cooker.configuration.runall: + if not self.runtaskentries: bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets))) self.init_progress_reporter.next_stage() @@ -971,19 +968,19 @@ class RunQueueData: for task in self.cooker.configuration.runonly: if not task.startswith("do_"): task = "do_{0}".format(task) - runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task } + runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task] - for tid in list(runonly_tids): - mark_active(tid,1) + for tid in runonly_tids: + mark_active(tid, 1) if self.cooker.configuration.force: invalidate_task(tid, False) for tid in list(self.runtaskentries.keys()): if tid not in runq_build: - delcount[tid] = self.runtaskentries[tid] + delcount.add(tid) del self.runtaskentries[tid] - if len(self.runtaskentries) == 0: + if not self.runtaskentries: bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets))) # @@ -991,7 +988,7 @@ class RunQueueData: # # Check to make sure we still have tasks to run - if len(self.runtaskentries) == 0: + if not self.runtaskentries: if not taskData[''].abort: bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") else: @@ -1015,7 +1012,7 @@ class RunQueueData: endpoints = [] for tid in self.runtaskentries: revdeps = self.runtaskentries[tid].revdeps - if len(revdeps) == 0: + if not revdeps: endpoints.append(tid) for dep in revdeps: if dep in self.runtaskentries[tid].depends: @@ -1061,12 +1058,12 @@ class RunQueueData: seen_pn.append(pn) else: bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn)) - msg = "Multiple .bb files are due to be built which each provide %s:\n %s" % (prov, "\n ".join(prov_list[prov])) + msgs = ["Multiple .bb files are due to be built which each provide %s:\n %s" % (prov, "\n ".join(prov_list[prov]))] # # Construct a list of things which uniquely depend on each provider # since this may help the user figure out which dependency is triggering this warning # - msg += "\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from." + msgs.append("\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from.") deplist = {} commondeps = None for provfn in prov_list[prov]: @@ -1086,12 +1083,12 @@ class RunQueueData: commondeps &= deps deplist[provfn] = deps for provfn in deplist: - msg += "\n%s has unique dependees:\n %s" % (provfn, "\n ".join(deplist[provfn] - commondeps)) + msgs.append("\n%s has unique dependees:\n %s" % (provfn, "\n ".join(deplist[provfn] - commondeps))) # # Construct a list of provides and runtime providers for each recipe # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC) # - msg += "\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful." + msgs.append("\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful.") provide_results = {} rprovide_results = {} commonprovs = None @@ -1118,29 +1115,18 @@ class RunQueueData: else: commonrprovs &= rprovides rprovide_results[provfn] = rprovides - #msg += "\nCommon provides:\n %s" % ("\n ".join(commonprovs)) - #msg += "\nCommon rprovides:\n %s" % ("\n ".join(commonrprovs)) + #msgs.append("\nCommon provides:\n %s" % ("\n ".join(commonprovs))) + #msgs.append("\nCommon rprovides:\n %s" % ("\n ".join(commonrprovs))) for provfn in prov_list[prov]: - msg += "\n%s has unique provides:\n %s" % (provfn, "\n ".join(provide_results[provfn] - commonprovs)) - msg += "\n%s has unique rprovides:\n %s" % (provfn, "\n ".join(rprovide_results[provfn] - commonrprovs)) + msgs.append("\n%s has unique provides:\n %s" % (provfn, "\n ".join(provide_results[provfn] - commonprovs))) + msgs.append("\n%s has unique rprovides:\n %s" % (provfn, "\n ".join(rprovide_results[provfn] - commonrprovs))) if self.warn_multi_bb: - logger.verbnote(msg) + logger.verbnote("".join(msgs)) else: - logger.error(msg) + logger.error("".join(msgs)) self.init_progress_reporter.next_stage() - - # Create a whitelist usable by the stamp checks - self.stampfnwhitelist = {} - for mc in self.taskData: - self.stampfnwhitelist[mc] = [] - for entry in self.stampwhitelist.split(): - if entry not in self.taskData[mc].build_targets: - continue - fn = self.taskData.build_targets[entry][0] - self.stampfnwhitelist[mc].append(fn) - self.init_progress_reporter.next_stage() # Iterate over the task list looking for tasks with a 'setscene' function @@ -1188,9 +1174,9 @@ class RunQueueData: # Iterate over the task list and call into the siggen code dealtwith = set() todeal = set(self.runtaskentries) - while len(todeal) > 0: + while todeal: for tid in todeal.copy(): - if len(self.runtaskentries[tid].depends - dealtwith) == 0: + if not (self.runtaskentries[tid].depends - dealtwith): dealtwith.add(tid) todeal.remove(tid) self.prepare_task_hash(tid) @@ -1229,7 +1215,6 @@ class RunQueue: self.cfgData = cfgData self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets) - self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile" self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None @@ -1358,14 +1343,6 @@ class RunQueue: if taskname is None: taskname = tn - if self.stamppolicy == "perfile": - fulldeptree = False - else: - fulldeptree = True - stampwhitelist = [] - if self.stamppolicy == "whitelist": - stampwhitelist = self.rqdata.stampfnwhitelist[mc] - stampfile = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn) # If the stamp is missing, it's not current @@ -1397,7 +1374,7 @@ class RunQueue: continue if t3 and t3 > t2: continue - if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist): + if fn == fn2: if not t2: logger.debug2('Stampfile %s does not exist', stampfile2) iscurrent = False @@ -1500,10 +1477,10 @@ class RunQueue: self.rqexe = RunQueueExecute(self) # If we don't have any setscene functions, skip execution - if len(self.rqdata.runq_setscene_tids) == 0: + if not self.rqdata.runq_setscene_tids: logger.info('No setscene tasks') for tid in self.rqdata.runtaskentries: - if len(self.rqdata.runtaskentries[tid].depends) == 0: + if not self.rqdata.runtaskentries[tid].depends: self.rqexe.setbuildable(tid) self.rqexe.tasks_notcovered.add(tid) self.rqexe.sqdone = True @@ -1780,7 +1757,7 @@ class RunQueueExecute: bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % (self.scheduler, ", ".join(obj.name for obj in schedulers))) - #if len(self.rqdata.runq_setscene_tids) > 0: + #if self.rqdata.runq_setscene_tids: self.sqdata = SQData() build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self) @@ -1821,7 +1798,7 @@ class RunQueueExecute: # worker must have died? pass - if len(self.failed_tids) != 0: + if self.failed_tids: self.rq.state = runQueueFailed return @@ -1837,7 +1814,7 @@ class RunQueueExecute: self.rq.read_workers() return self.rq.active_fds() - if len(self.failed_tids) != 0: + if self.failed_tids: self.rq.state = runQueueFailed return True @@ -1935,7 +1912,7 @@ class RunQueueExecute: self.stats.taskFailed() self.failed_tids.append(task) - fakeroot_log = "" + fakeroot_log = [] if fakerootlog and os.path.exists(fakerootlog): with open(fakerootlog) as fakeroot_log_file: fakeroot_failed = False @@ -1945,12 +1922,12 @@ class RunQueueExecute: fakeroot_failed = True if 'doing new pid setup and server start' in line: break - fakeroot_log = line + fakeroot_log + fakeroot_log.append(line) if not fakeroot_failed: - fakeroot_log = None + fakeroot_log = [] - bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=fakeroot_log), self.cfgData) + bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=("".join(fakeroot_log) or None)), self.cfgData) if self.rqdata.taskData[''].abort: self.rq.state = runQueueCleanUp @@ -2001,7 +1978,7 @@ class RunQueueExecute: if x not in self.tasks_scenequeue_done: logger.error("Task %s was never processed by the setscene code" % x) err = True - if len(self.rqdata.runtaskentries[x].depends) == 0 and x not in self.runq_buildable: + if not self.rqdata.runtaskentries[x].depends and x not in self.runq_buildable: logger.error("Task %s was never marked as buildable by the setscene code" % x) err = True return err @@ -2025,7 +2002,7 @@ class RunQueueExecute: # Find the next setscene to run for nexttask in self.sorted_setscene_tids: if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): - if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): + if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): if nexttask not in self.rqdata.target_tids: logger.debug2("Skipping setscene for task %s" % nexttask) self.sq_task_skip(nexttask) @@ -2189,7 +2166,7 @@ class RunQueueExecute: if self.can_start_task(): return True - if self.stats.active > 0 or len(self.sq_live) > 0: + if self.stats.active > 0 or self.sq_live: self.rq.read_workers() return self.rq.active_fds() @@ -2201,7 +2178,7 @@ class RunQueueExecute: self.sq_task_failoutright(tid) return True - if len(self.failed_tids) != 0: + if self.failed_tids: self.rq.state = runQueueFailed return True @@ -2280,7 +2257,7 @@ class RunQueueExecute: covered.intersection_update(self.tasks_scenequeue_done) for tid in notcovered | covered: - if len(self.rqdata.runtaskentries[tid].depends) == 0: + if not self.rqdata.runtaskentries[tid].depends: self.setbuildable(tid) elif self.rqdata.runtaskentries[tid].depends.issubset(self.runq_complete): self.setbuildable(tid) @@ -2339,7 +2316,7 @@ class RunQueueExecute: # Now iterate those tasks in dependency order to regenerate their taskhash/unihash next = set() for p in total: - if len(self.rqdata.runtaskentries[p].depends) == 0: + if not self.rqdata.runtaskentries[p].depends: next.add(p) elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): next.add(p) @@ -2349,7 +2326,7 @@ class RunQueueExecute: current = next.copy() next = set() for tid in current: - if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): + if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): continue orighash = self.rqdata.runtaskentries[tid].hash dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid)) @@ -2436,7 +2413,7 @@ class RunQueueExecute: if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered): if tid not in self.sq_buildable: self.sq_buildable.add(tid) - if len(self.sqdata.sq_revdeps[tid]) == 0: + if not self.sqdata.sq_revdeps[tid]: self.sq_buildable.add(tid) if tid in self.sqdata.outrightfail: @@ -2608,12 +2585,13 @@ class RunQueueExecute: pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): if tid in self.rqdata.runq_setscene_tids: - msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname) + msg = ['Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)] else: - msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname) + msg = ['Task %s.%s attempted to execute unexpectedly' % (pn, taskname)] for t in self.scenequeue_notcovered: - msg = msg + "\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash) - logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered)) + msg.append("\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)) + msg.append('\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered)) + logger.error("".join(msg)) return True return False @@ -2652,7 +2630,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): for tid in rqdata.runtaskentries: sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps) sq_revdeps_squash[tid] = set() - if (len(sq_revdeps[tid]) == 0) and tid not in rqdata.runq_setscene_tids: + if not sq_revdeps[tid] and tid not in rqdata.runq_setscene_tids: #bb.warn("Added endpoint %s" % (tid)) endpoints[tid] = set() @@ -2693,9 +2671,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): sq_revdeps[dep].remove(point) if tasks: sq_revdeps_squash[dep] |= tasks - if len(sq_revdeps[dep]) == 0 and dep not in rqdata.runq_setscene_tids: + if not sq_revdeps[dep] and dep not in rqdata.runq_setscene_tids: newendpoints[dep] = task - if len(newendpoints) != 0: + if newendpoints: process_endpoints(newendpoints) process_endpoints(endpoints) @@ -2707,7 +2685,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): # Take the build endpoints (no revdeps) and find the sstate tasks they depend upon new = True for tid in rqdata.runtaskentries: - if len(rqdata.runtaskentries[tid].revdeps) == 0: + if not rqdata.runtaskentries[tid].revdeps: sqdata.unskippable.add(tid) sqdata.unskippable |= sqrq.cantskip while new: @@ -2716,7 +2694,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): for tid in sorted(orig, reverse=True): if tid in rqdata.runq_setscene_tids: continue - if len(rqdata.runtaskentries[tid].depends) == 0: + if not rqdata.runtaskentries[tid].depends: # These are tasks which have no setscene tasks in their chain, need to mark as directly buildable sqrq.setbuildable(tid) sqdata.unskippable |= rqdata.runtaskentries[tid].depends @@ -2731,7 +2709,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): for taskcounter, tid in enumerate(rqdata.runtaskentries): if tid in rqdata.runq_setscene_tids: pass - elif len(sq_revdeps_squash[tid]) != 0: + elif sq_revdeps_squash[tid]: bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") else: del sq_revdeps_squash[tid] @@ -2796,7 +2774,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): sqdata.multiconfigs = set() for tid in sqdata.sq_revdeps: sqdata.multiconfigs.add(mc_from_tid(tid)) - if len(sqdata.sq_revdeps[tid]) == 0: + if not sqdata.sq_revdeps[tid]: sqrq.sq_buildable.add(tid) rqdata.init_progress_reporter.finish() @@ -3050,7 +3028,7 @@ class runQueuePipe(): raise end = len(self.queue) found = True - while found and len(self.queue): + while found and self.queue: found = False index = self.queue.find(b"</event>") while index != -1 and self.queue.startswith(b"<event>"): @@ -3088,7 +3066,7 @@ class runQueuePipe(): def close(self): while self.read(): continue - if len(self.queue) > 0: + if self.queue: print("Warning, worker left partial message: %s" % self.queue) self.input.close() |