diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/bb/siggen.py')
| -rw-r--r-- | import-layers/yocto-poky/bitbake/lib/bb/siggen.py | 189 |
1 files changed, 146 insertions, 43 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py index 542bbb9d1..f71190ad4 100644 --- a/import-layers/yocto-poky/bitbake/lib/bb/siggen.py +++ b/import-layers/yocto-poky/bitbake/lib/bb/siggen.py @@ -5,6 +5,8 @@ import re import tempfile import pickle import bb.data +import difflib +import simplediff from bb.checksum import FileChecksumCache logger = logging.getLogger('BitBake.SigGen') @@ -13,7 +15,7 @@ def init(d): siggens = [obj for obj in globals().values() if type(obj) is type and issubclass(obj, SignatureGenerator)] - desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop" + desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop" for sg in siggens: if desired == sg.name: return sg(d) @@ -82,10 +84,10 @@ class SignatureGeneratorBasic(SignatureGenerator): self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P<fn>.*)\..*") - self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split()) + self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) self.taskwhitelist = None self.init_rundepcheck(data) - checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True) + checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") if checksum_cache_file: self.checksum_cache = FileChecksumCache() self.checksum_cache.init_cache(data, checksum_cache_file) @@ -93,7 +95,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.checksum_cache = None def init_rundepcheck(self, data): - self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None + self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None if self.taskwhitelist: self.twl = re.compile(self.taskwhitelist) else: @@ -101,6 +103,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def _build_data(self, fn, d): + ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) taskdeps = {} @@ -135,7 +138,7 @@ class SignatureGeneratorBasic(SignatureGenerator): data = data + str(var) datahash = hashlib.md5(data.encode("utf-8")).hexdigest() k = fn + "." + task - if k in self.basehash and self.basehash[k] != datahash: + if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash: bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash)) self.basehash[k] = datahash taskdeps[task] = alldeps @@ -154,13 +157,15 @@ class SignatureGeneratorBasic(SignatureGenerator): try: taskdeps = self._build_data(fn, d) + except bb.parse.SkipRecipe: + raise except: bb.warn("Error during finalise of %s" % fn) raise #Slow but can be useful for debugging mismatched basehashes #for task in self.taskdeps[fn]: - # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) + # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) for task in taskdeps: d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) @@ -306,8 +311,8 @@ class SignatureGeneratorBasic(SignatureGenerator): pass raise err - def dump_sigs(self, dataCaches, options): - for fn in self.taskdeps: + def dump_sigfn(self, fn, dataCaches, options): + if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task (mc, _, _) = bb.runqueue.split_tid(tid) @@ -345,16 +350,67 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic): def dump_this_task(outfile, d): import bb.parse - fn = d.getVar("BB_FILENAME", True) - task = "do_" + d.getVar("BB_CURRENTTASK", True) + fn = d.getVar("BB_FILENAME") + task = "do_" + d.getVar("BB_CURRENTTASK") referencestamp = bb.build.stamp_internal(task, d, None, True) bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) +def init_colors(enable_color): + """Initialise colour dict for passing to compare_sigfiles()""" + # First set up the colours + colors = {'color_title': '\033[1;37;40m', + 'color_default': '\033[0;37;40m', + 'color_add': '\033[1;32;40m', + 'color_remove': '\033[1;31;40m', + } + # Leave all keys present but clear the values + if not enable_color: + for k in colors.keys(): + colors[k] = '' + return colors + +def worddiff_str(oldstr, newstr, colors=None): + if not colors: + colors = init_colors(False) + diff = simplediff.diff(oldstr.split(' '), newstr.split(' ')) + ret = [] + for change, value in diff: + value = ' '.join(value) + if change == '=': + ret.append(value) + elif change == '+': + item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors) + ret.append(item) + elif change == '-': + item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors) + ret.append(item) + whitespace_note = '' + if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()): + whitespace_note = ' (whitespace changed)' + return '"%s"%s' % (' '.join(ret), whitespace_note) + +def list_inline_diff(oldlist, newlist, colors=None): + if not colors: + colors = init_colors(False) + diff = simplediff.diff(oldlist, newlist) + ret = [] + for change, value in diff: + value = ' '.join(value) + if change == '=': + ret.append("'%s'" % value) + elif change == '+': + item = '{color_add}+{value}{color_default}'.format(value=value, **colors) + ret.append(item) + elif change == '-': + item = '{color_remove}-{value}{color_default}'.format(value=value, **colors) + ret.append(item) + return '[%s]' % (', '.join(ret)) + def clean_basepath(a): mc = None if a.startswith("multiconfig:"): _, mc, a = a.split(":", 2) - b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2] + b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] if a.startswith("virtual:"): b = b + ":" + a.rsplit(":", 1)[0] if mc: @@ -373,9 +429,26 @@ def clean_basepaths_list(a): b.append(clean_basepath(x)) return b -def compare_sigfiles(a, b, recursecb = None): +def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): output = [] + colors = init_colors(color) + def color_format(formatstr, **values): + """ + Return colour formatted string. + NOTE: call with the format string, not an already formatted string + containing values (otherwise you could have trouble with { and } + characters) + """ + if not formatstr.endswith('{color_default}'): + formatstr += '{color_default}' + # In newer python 3 versions you can pass both of these directly, + # but we only require 3.4 at the moment + formatparams = {} + formatparams.update(colors) + formatparams.update(values) + return formatstr.format(**formatparams) + with open(a, 'rb') as f: p1 = pickle.Unpickler(f) a_data = p1.load() @@ -429,39 +502,59 @@ def compare_sigfiles(a, b, recursecb = None): return changed, added, removed if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: - output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist'])) + output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) if a_data['basewhitelist'] and b_data['basewhitelist']: output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: - output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist'])) + output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) if a_data['taskwhitelist'] and b_data['taskwhitelist']: output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) if a_data['taskdeps'] != b_data['taskdeps']: - output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) + output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) - if a_data['basehash'] != b_data['basehash']: - output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash'])) + if a_data['basehash'] != b_data['basehash'] and not collapsed: + output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) if changed: for dep in changed: - output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) + output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) if a_data['gendeps'][dep] and b_data['gendeps'][dep]: output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) if added: for dep in added: - output.append("Dependency on variable %s was added" % (dep)) + output.append(color_format("{color_title}Dependency on variable %s was added") % (dep)) if removed: for dep in removed: - output.append("Dependency on Variable %s was removed" % (dep)) + output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep)) changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) if changed: for dep in changed: - output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep])) + oldval = a_data['varvals'][dep] + newval = b_data['varvals'][dep] + if newval and oldval and ('\n' in oldval or '\n' in newval): + diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='') + # Cut off the first two lines, since we aren't interested in + # the old/new filename (they are blank anyway in this case) + difflines = list(diff)[2:] + if color: + # Add colour to diff output + for i, line in enumerate(difflines): + if line.startswith('+'): + line = color_format('{color_add}{line}', line=line) + difflines[i] = line + elif line.startswith('-'): + line = color_format('{color_remove}{line}', line=line) + difflines[i] = line + output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines))) + elif newval and oldval and (' ' in oldval or ' ' in newval): + output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors))) + else: + output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval)) if not 'file_checksum_values' in a_data: a_data['file_checksum_values'] = {} @@ -471,32 +564,38 @@ def compare_sigfiles(a, b, recursecb = None): changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) if changed: for f, old, new in changed: - output.append("Checksum for file %s changed from %s to %s" % (f, old, new)) + output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new)) if added: for f in added: - output.append("Dependency on checksum of file %s was added" % (f)) + output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f)) if removed: for f in removed: - output.append("Dependency on checksum of file %s was removed" % (f)) + output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f)) if not 'runtaskdeps' in a_data: a_data['runtaskdeps'] = {} if not 'runtaskdeps' in b_data: b_data['runtaskdeps'] = {} - if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): - changed = ["Number of task dependencies changed"] - else: - changed = [] - for idx, task in enumerate(a_data['runtaskdeps']): - a = a_data['runtaskdeps'][idx] - b = b_data['runtaskdeps'][idx] - if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]: - changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b])) + if not collapsed: + if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): + changed = ["Number of task dependencies changed"] + else: + changed = [] + for idx, task in enumerate(a_data['runtaskdeps']): + a = a_data['runtaskdeps'][idx] + b = b_data['runtaskdeps'][idx] + if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed: + changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b])) - if changed: - output.append("runtaskdeps changed from %s to %s" % (clean_basepaths_list(a_data['runtaskdeps']), clean_basepaths_list(b_data['runtaskdeps']))) - output.append("\n".join(changed)) + if changed: + clean_a = clean_basepaths_list(a_data['runtaskdeps']) + clean_b = clean_basepaths_list(b_data['runtaskdeps']) + if clean_a != clean_b: + output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors)) + else: + output.append(color_format("{color_title}runtaskdeps changed:")) + output.append("\n".join(changed)) if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: @@ -512,7 +611,7 @@ def compare_sigfiles(a, b, recursecb = None): #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) bdep_found = True if not bdep_found: - output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep])) + output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep])) if removed: for dep in removed: adep_found = False @@ -522,21 +621,25 @@ def compare_sigfiles(a, b, recursecb = None): #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) adep_found = True if not adep_found: - output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep])) + output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep])) if changed: for dep in changed: - output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep])) + if not collapsed: + output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep])) if callable(recursecb): - # If a dependent hash changed, might as well print the line above and then defer to the changes in - # that hash since in all likelyhood, they're the same changes this task also saw. recout = recursecb(dep, a[dep], b[dep]) if recout: - output = [output[-1]] + recout + if collapsed: + output.extend(recout) + else: + # If a dependent hash changed, might as well print the line above and then defer to the changes in + # that hash since in all likelyhood, they're the same changes this task also saw. + output = [output[-1]] + recout a_taint = a_data.get('taint', None) b_taint = b_data.get('taint', None) if a_taint != b_taint: - output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)) + output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint)) return output |