diff options
Diffstat (limited to 'poky/meta/lib')
-rw-r--r-- | poky/meta/lib/oe/recipeutils.py | 2 | ||||
-rw-r--r-- | poky/meta/lib/oe/reproducible.py | 104 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/selftest/cases/sstatetests.py | 5 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/utils/qemurunner.py | 17 | ||||
-rw-r--r-- | poky/meta/lib/oeqa/utils/qemutinyrunner.py | 2 |
5 files changed, 124 insertions, 6 deletions
diff --git a/poky/meta/lib/oe/recipeutils.py b/poky/meta/lib/oe/recipeutils.py index 29473a988..36427eec9 100644 --- a/poky/meta/lib/oe/recipeutils.py +++ b/poky/meta/lib/oe/recipeutils.py @@ -24,7 +24,7 @@ from collections import OrderedDict, defaultdict from bb.utils import vercmp_string # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND'] # Variables that sometimes are a bit long but shouldn't be wrapped nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] diff --git a/poky/meta/lib/oe/reproducible.py b/poky/meta/lib/oe/reproducible.py new file mode 100644 index 000000000..f4f58dd95 --- /dev/null +++ b/poky/meta/lib/oe/reproducible.py @@ -0,0 +1,104 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +import os +import subprocess +import bb + +def get_source_date_epoch_from_known_files(d, sourcedir): + source_date_epoch = None + newest_file = None + known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) + for file in known_files: + filepath = os.path.join(sourcedir, file) + if os.path.isfile(filepath): + mtime = int(os.lstat(filepath).st_mtime) + # There may be more than one "known_file" present, if so, use the youngest one + if not source_date_epoch or mtime > source_date_epoch: + source_date_epoch = mtime + newest_file = filepath + if newest_file: + bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) + return source_date_epoch + +def find_git_folder(d, sourcedir): + # First guess: WORKDIR/git + # This is the default git fetcher unpack path + workdir = d.getVar('WORKDIR') + gitpath = os.path.join(workdir, "git/.git") + if os.path.isdir(gitpath): + return gitpath + + # Second guess: ${S} + gitpath = os.path.join(sourcedir, ".git") + if os.path.isdir(gitpath): + return gitpath + + # Perhaps there was a subpath or destsuffix specified. + # Go looking in the WORKDIR + exclude = set(["build", "image", "license-destdir", "patches", "pseudo", + "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) + for root, dirs, files in os.walk(workdir, topdown=True): + dirs[:] = [d for d in dirs if d not in exclude] + if '.git' in dirs: + return root + + bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) + return None + +def get_source_date_epoch_from_git(d, sourcedir): + if not "git://" in d.getVar('SRC_URI'): + return None + + gitpath = find_git_folder(d, sourcedir) + if not gitpath: + return None + + # Check that the repository has a valid HEAD; it may not if subdir is used + # in SRC_URI + p = subprocess.run(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=gitpath) + if p.returncode != 0: + bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8'))) + return None + + bb.debug(1, "git repository: %s" % gitpath) + p = subprocess.run(['git','log','-1','--pretty=%ct'], check=True, stdout=subprocess.PIPE, cwd=gitpath) + return int(p.stdout.decode('utf-8')) + +def get_source_date_epoch_from_youngest_file(d, sourcedir): + if sourcedir == d.getVar('WORKDIR'): + # These sources are almost certainly not from a tarball + return None + + # Do it the hard way: check all files and find the youngest one... + source_date_epoch = None + newest_file = None + for root, dirs, files in os.walk(sourcedir, topdown=True): + files = [f for f in files if not f[0] == '.'] + + for fname in files: + filename = os.path.join(root, fname) + try: + mtime = int(os.lstat(filename).st_mtime) + except ValueError: + mtime = 0 + if not source_date_epoch or mtime > source_date_epoch: + source_date_epoch = mtime + newest_file = filename + + if newest_file: + bb.debug(1, "Newest file found: %s" % newest_file) + return source_date_epoch + +def fixed_source_date_epoch(): + bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") + return 0 + +def get_source_date_epoch(d, sourcedir): + return ( + get_source_date_epoch_from_git(d, sourcedir) or + get_source_date_epoch_from_known_files(d, sourcedir) or + get_source_date_epoch_from_youngest_file(d, sourcedir) or + fixed_source_date_epoch() # Last resort + ) + diff --git a/poky/meta/lib/oeqa/selftest/cases/sstatetests.py b/poky/meta/lib/oeqa/selftest/cases/sstatetests.py index 9adb51196..c46e8ba48 100644 --- a/poky/meta/lib/oeqa/selftest/cases/sstatetests.py +++ b/poky/meta/lib/oeqa/selftest/cases/sstatetests.py @@ -19,10 +19,13 @@ class SStateTests(SStateBase): # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} # when PV does not contain SRCPV - tempdir = tempfile.mkdtemp(prefix='oeqa') + tempdir = tempfile.mkdtemp(prefix='sstate_autorev') + tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') self.track_for_cleanup(tempdir) + self.track_for_cleanup(tempdldir) create_temp_layer(tempdir, 'selftestrecipetool') self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) + self.append_config("DL_DIR = \"%s\"" % tempdldir) runCmd('bitbake-layers add-layer %s' % tempdir) # Use dbus-wait as a local git repo we can add a commit between two builds in diff --git a/poky/meta/lib/oeqa/utils/qemurunner.py b/poky/meta/lib/oeqa/utils/qemurunner.py index 992fff937..519aa9aa1 100644 --- a/poky/meta/lib/oeqa/utils/qemurunner.py +++ b/poky/meta/lib/oeqa/utils/qemurunner.py @@ -32,7 +32,7 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, - use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False): + use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None): # Popen object for runqemu self.runqemu = None @@ -63,7 +63,9 @@ class QemuRunner: self.boot_patterns = boot_patterns self.runqemutime = 120 - self.qemu_pidfile = 'pidfile_'+str(os.getpid()) + if not workdir: + workdir = os.getcwd() + self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) self.monitorpipe = None @@ -252,8 +254,17 @@ class QemuRunner: if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) + + qemu_pid = None + if os.path.isfile(self.qemu_pidfile): + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + + self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s" + % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid)))) + # Dump all processes to help us to figure out what is going on... - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() diff --git a/poky/meta/lib/oeqa/utils/qemutinyrunner.py b/poky/meta/lib/oeqa/utils/qemutinyrunner.py index 364005bd2..5c92941c0 100644 --- a/poky/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/poky/meta/lib/oeqa/utils/qemutinyrunner.py @@ -138,7 +138,7 @@ class QemuTinyRunner(QemuRunner): # # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command'], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} |