diff options
Diffstat (limited to 'poky/meta/lib/oeqa/selftest/cases/reproducible.py')
| -rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/reproducible.py | 130 |
1 files changed, 84 insertions, 46 deletions
diff --git a/poky/meta/lib/oeqa/selftest/cases/reproducible.py b/poky/meta/lib/oeqa/selftest/cases/reproducible.py index a9110565a..5d3959be7 100644 --- a/poky/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/poky/meta/lib/oeqa/selftest/cases/reproducible.py @@ -1,7 +1,7 @@ # # SPDX-License-Identifier: MIT # -# Copyright 2019 by Garmin Ltd. or its subsidiaries +# Copyright 2019-2020 by Garmin Ltd. or its subsidiaries from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars @@ -15,6 +15,7 @@ import tempfile import shutil import stat import os +import datetime MISSING = 'MISSING' DIFFERENT = 'DIFFERENT' @@ -78,8 +79,18 @@ def compare_file(reference, test, diffutils_sysroot): class ReproducibleTests(OESelftestTestCase): package_classes = ['deb', 'ipk'] - images = ['core-image-minimal'] + images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline'] save_results = False + if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: + save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] + + # This variable controls if one of the test builds is allowed to pull from + # an sstate cache/mirror. The other build is always done clean as a point of + # comparison. + # If you know that your sstate archives are reproducible, enabling this + # will test that and also make the test run faster. If your sstate is not + # reproducible, disable this in your derived test class + build_from_sstate = True def setUpLocal(self): super().setUpLocal() @@ -88,12 +99,12 @@ class ReproducibleTests(OESelftestTestCase): for v in needed_vars: setattr(self, v.lower(), bb_vars[v]) - self.extrasresults = {} - self.extrasresults.setdefault('reproducible.rawlogs', {})['log'] = '' - self.extrasresults.setdefault('reproducible', {}).setdefault('files', {}) + self.extraresults = {} + self.extraresults.setdefault('reproducible.rawlogs', {})['log'] = '' + self.extraresults.setdefault('reproducible', {}).setdefault('files', {}) def append_to_log(self, msg): - self.extrasresults['reproducible.rawlogs']['log'] += msg + self.extraresults['reproducible.rawlogs']['log'] += msg def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): result = PackageCompareResults() @@ -120,60 +131,69 @@ class ReproducibleTests(OESelftestTestCase): return result def write_package_list(self, package_class, name, packages): - self.extrasresults['reproducible']['files'].setdefault(package_class, {})[name] = [ + self.extraresults['reproducible']['files'].setdefault(package_class, {})[name] = [ {'reference': p.reference, 'test': p.test} for p in packages] def copy_file(self, source, dest): bb.utils.mkdirhier(os.path.dirname(dest)) shutil.copyfile(source, dest) - def test_reproducible_builds(self): + def do_test_build(self, name, use_sstate): capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes] - if self.save_results: - save_dir = tempfile.mkdtemp(prefix='oe-reproducible-') - os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) - self.logger.info('Non-reproducible packages will be copied to %s', save_dir) + tmpdir = os.path.join(self.topdir, name, 'tmp') + if os.path.exists(tmpdir): + bb.utils.remove(tmpdir, recurse=True) + + config = textwrap.dedent('''\ + INHERIT += "reproducible_build" + PACKAGE_CLASSES = "{package_classes}" + INHIBIT_PACKAGE_STRIP = "1" + TMPDIR = "{tmpdir}" + ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), + tmpdir=tmpdir) + + if not use_sstate: + # This config fragment will disable using shared and the sstate + # mirror, forcing a complete build from scratch + config += textwrap.dedent('''\ + SSTATE_DIR = "${TMPDIR}/sstate" + SSTATE_MIRROR = "" + ''') + + self.write_config(config) + d = get_bb_vars(capture_vars) + bitbake(' '.join(self.images)) + return d + + def test_reproducible_builds(self): + def strip_topdir(s): + if s.startswith(self.topdir): + return s[len(self.topdir):] + return s # Build native utilities self.write_config('') - bitbake("diffutils-native -c addto_recipe_sysroot") + bitbake("diffoscope-native diffutils-native jquery-native -c addto_recipe_sysroot") diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") + diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") + jquery_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "jquery-native") - # Reproducible builds should not pull from sstate or mirrors, but - # sharing DL_DIR is fine - common_config = textwrap.dedent('''\ - INHERIT += "reproducible_build" - PACKAGE_CLASSES = "%s" - SSTATE_DIR = "${TMPDIR}/sstate" - ''') % (' '.join('package_%s' % c for c in self.package_classes)) - - # Perform a build. - reproducibleA_tmp = os.path.join(self.topdir, 'reproducibleA', 'tmp') - if os.path.exists(reproducibleA_tmp): - bb.utils.remove(reproducibleA_tmp, recurse=True) - - self.write_config((textwrap.dedent('''\ - TMPDIR = "%s" - ''') % reproducibleA_tmp) + common_config) - vars_A = get_bb_vars(capture_vars) - bitbake(' '.join(self.images)) - - # Perform another build. - reproducibleB_tmp = os.path.join(self.topdir, 'reproducibleB', 'tmp') - if os.path.exists(reproducibleB_tmp): - bb.utils.remove(reproducibleB_tmp, recurse=True) + if self.save_results: + os.makedirs(self.save_results, exist_ok=True) + datestr = datetime.datetime.now().strftime('%Y%m%d') + save_dir = tempfile.mkdtemp(prefix='oe-reproducible-%s-' % datestr, dir=self.save_results) + os.chmod(save_dir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + self.logger.info('Non-reproducible packages will be copied to %s', save_dir) - self.write_config((textwrap.dedent('''\ - SSTATE_MIRROR = "" - TMPDIR = "%s" - ''') % reproducibleB_tmp) + common_config) - vars_B = get_bb_vars(capture_vars) - bitbake(' '.join(self.images)) + vars_A = self.do_test_build('reproducibleA', self.build_from_sstate) + vars_B = self.do_test_build('reproducibleB', False) # NOTE: The temp directories from the reproducible build are purposely # kept after the build so it can be diffed for debugging. + fails = [] + for c in self.package_classes: with self.subTest(package_class=c): package_class = 'package_' + c @@ -193,10 +213,28 @@ class ReproducibleTests(OESelftestTestCase): if self.save_results: for d in result.different: - self.copy_file(d.reference, '/'.join([save_dir, d.reference])) - self.copy_file(d.test, '/'.join([save_dir, d.test])) + self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) + self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) if result.missing or result.different: - self.fail("The following %s packages are missing or different: %s" % - (c, ' '.join(r.test for r in (result.missing + result.different)))) + fails.append("The following %s packages are missing or different: %s" % + (c, '\n'.join(r.test for r in (result.missing + result.different)))) + + # Clean up empty directories + if self.save_results: + if not os.listdir(save_dir): + os.rmdir(save_dir) + else: + self.logger.info('Running diffoscope') + package_dir = os.path.join(save_dir, 'packages') + package_html_dir = os.path.join(package_dir, 'diff-html') + + # Copy jquery to improve the diffoscope output usability + self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) + + runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--html-dir', package_html_dir, 'reproducibleA', 'reproducibleB'], + native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) + + if fails: + self.fail('\n'.join(fails)) |