From eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 Mon Sep 17 00:00:00 2001 From: Dave Cobbley Date: Tue, 14 Aug 2018 10:05:37 -0700 Subject: [Subtree] Removing import-layers directory As part of the move to subtrees, need to bring all the import layers content to the top level. Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f Signed-off-by: Dave Cobbley Signed-off-by: Brad Bishop --- poky/meta/classes/staging.bbclass | 591 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 591 insertions(+) create mode 100644 poky/meta/classes/staging.bbclass (limited to 'poky/meta/classes/staging.bbclass') diff --git a/poky/meta/classes/staging.bbclass b/poky/meta/classes/staging.bbclass new file mode 100644 index 000000000..3fcbc9f15 --- /dev/null +++ b/poky/meta/classes/staging.bbclass @@ -0,0 +1,591 @@ +# These directories will be staged in the sysroot +SYSROOT_DIRS = " \ + ${includedir} \ + ${libdir} \ + ${base_libdir} \ + ${nonarch_base_libdir} \ + ${datadir} \ +" + +# These directories are also staged in the sysroot when they contain files that +# are usable on the build system +SYSROOT_DIRS_NATIVE = " \ + ${bindir} \ + ${sbindir} \ + ${base_bindir} \ + ${base_sbindir} \ + ${libexecdir} \ + ${sysconfdir} \ + ${localstatedir} \ +" +SYSROOT_DIRS_append_class-native = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS_append_class-cross = " ${SYSROOT_DIRS_NATIVE}" +SYSROOT_DIRS_append_class-crosssdk = " ${SYSROOT_DIRS_NATIVE}" + +# These directories will not be staged in the sysroot +SYSROOT_DIRS_BLACKLIST = " \ + ${mandir} \ + ${docdir} \ + ${infodir} \ + ${datadir}/locale \ + ${datadir}/applications \ + ${datadir}/fonts \ + ${datadir}/pixmaps \ + ${libdir}/${PN}/ptest \ +" + +sysroot_stage_dir() { + src="$1" + dest="$2" + # if the src doesn't exist don't do anything + if [ ! -d "$src" ]; then + return + fi + + mkdir -p "$dest" + ( + cd $src + find . -print0 | cpio --null -pdlu $dest + ) +} + +sysroot_stage_dirs() { + from="$1" + to="$2" + + for dir in ${SYSROOT_DIRS}; do + sysroot_stage_dir "$from$dir" "$to$dir" + done + + # Remove directories we do not care about + for dir in ${SYSROOT_DIRS_BLACKLIST}; do + rm -rf "$to$dir" + done +} + +sysroot_stage_all() { + sysroot_stage_dirs ${D} ${SYSROOT_DESTDIR} +} + +python sysroot_strip () { + inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP') + if inhibit_sysroot and oe.types.boolean(inhibit_sysroot): + return 0 + + dstdir = d.getVar('SYSROOT_DESTDIR') + pn = d.getVar('PN') + libdir = os.path.abspath(dstdir + os.sep + d.getVar("libdir")) + base_libdir = os.path.abspath(dstdir + os.sep + d.getVar("base_libdir")) + qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split() + strip_cmd = d.getVar("STRIP") + + oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, + qa_already_stripped=qa_already_stripped) +} + +do_populate_sysroot[dirs] = "${SYSROOT_DESTDIR}" +do_populate_sysroot[umask] = "022" + +addtask populate_sysroot after do_install + +SYSROOT_PREPROCESS_FUNCS ?= "" +SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir" + +python do_populate_sysroot () { + bb.build.exec_func("sysroot_stage_all", d) + bb.build.exec_func("sysroot_strip", d) + for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split(): + bb.build.exec_func(f, d) + pn = d.getVar("PN") + multiprov = d.getVar("MULTI_PROVIDER_WHITELIST").split() + provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/") + bb.utils.mkdirhier(provdir) + for p in d.getVar("PROVIDES").split(): + if p in multiprov: + continue + p = p.replace("/", "_") + with open(provdir + p, "w") as f: + f.write(pn) +} + +do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}" +do_populate_sysroot[vardepsexclude] += "MULTI_PROVIDER_WHITELIST" + +POPULATESYSROOTDEPS = "" +POPULATESYSROOTDEPS_class-target = "virtual/${MLPREFIX}${TARGET_PREFIX}binutils:do_populate_sysroot" +POPULATESYSROOTDEPS_class-nativesdk = "virtual/${TARGET_PREFIX}binutils-crosssdk:do_populate_sysroot" +do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}" + +SSTATETASKS += "do_populate_sysroot" +do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}" +do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}" +do_populate_sysroot[sstate-outputdirs] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}" +do_populate_sysroot[sstate-fixmedir] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}" + +python do_populate_sysroot_setscene () { + sstate_setscene(d) +} +addtask do_populate_sysroot_setscene + +def staging_copyfile(c, target, dest, postinsts, seendirs): + import errno + + destdir = os.path.dirname(dest) + if destdir not in seendirs: + bb.utils.mkdirhier(destdir) + seendirs.add(destdir) + if "/usr/bin/postinst-" in c: + postinsts.append(dest) + if os.path.islink(c): + linkto = os.readlink(c) + if os.path.lexists(dest): + if not os.path.islink(dest): + raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest) + if os.readlink(dest) == linkto: + return dest + raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest) + os.symlink(linkto, dest) + #bb.warn(c) + else: + try: + os.link(c, dest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(c, dest) + else: + raise + return dest + +def staging_copydir(c, target, dest, seendirs): + if dest not in seendirs: + bb.utils.mkdirhier(dest) + seendirs.add(dest) + +def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d): + import subprocess + + if not fixme: + return + cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative) + for fixmevar in ['COMPONENTS_DIR', 'HOSTTOOLS_DIR', 'PKGDATA_DIR', 'PSEUDO_LOCALSTATEDIR', 'LOGFIFO']: + fixme_path = d.getVar(fixmevar) + cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) + bb.debug(2, cmd) + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + + +def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d): + import glob + import subprocess + import errno + + fixme = [] + postinsts = [] + seendirs = set() + stagingdir = d.getVar("STAGING_DIR") + if native: + pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*'] + targetdir = nativesysroot + else: + pkgarchs = ['${MACHINE_ARCH}'] + pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split())) + pkgarchs.append('allarch') + targetdir = targetsysroot + + bb.utils.mkdirhier(targetdir) + for pkgarch in pkgarchs: + for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)): + if manifest.endswith("-initial.populate_sysroot"): + # skip glibc-initial and libgcc-initial due to file overlap + continue + tmanifest = targetdir + "/" + os.path.basename(manifest) + if os.path.exists(tmanifest): + continue + try: + os.link(manifest, tmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(manifest, tmanifest) + else: + raise + with open(manifest, "r") as f: + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fixme.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = targetdir + "/" + "/".join(dest.split("/")[3:]) + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + try: + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + except FileExistsError: + continue + + staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d) + for p in postinsts: + subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT) + +# +# Manifests here are complicated. The main sysroot area has the unpacked sstate +# which us unrelocated and tracked by the main sstate manifests. Each recipe +# specific sysroot has manifests for each dependency that is installed there. +# The task hash is used to tell whether the data needs to be reinstalled. We +# use a symlink to point to the currently installed hash. There is also a +# "complete" stamp file which is used to mark if installation completed. If +# something fails (e.g. a postinst), this won't get written and we would +# remove and reinstall the dependency. This also means partially installed +# dependencies should get cleaned up correctly. +# + +python extend_recipe_sysroot() { + import copy + import subprocess + import errno + import collections + import glob + + taskdepdata = d.getVar("BB_TASKDEPDATA", False) + mytaskname = d.getVar("BB_RUNTASK") + if mytaskname.endswith("_setscene"): + mytaskname = mytaskname.replace("_setscene", "") + workdir = d.getVar("WORKDIR") + #bb.warn(str(taskdepdata)) + pn = d.getVar("PN") + + stagingdir = d.getVar("STAGING_DIR") + sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests" + recipesysroot = d.getVar("RECIPE_SYSROOT") + recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE") + current_variant = d.getVar("BBEXTENDVARIANT") + + # Detect bitbake -b usage + nodeps = d.getVar("BB_LIMITEDDEPS") or False + if nodeps: + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d) + staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d) + bb.utils.unlockfile(lock) + return + + start = None + configuredeps = [] + for dep in taskdepdata: + data = taskdepdata[dep] + if data[1] == mytaskname and data[0] == pn: + start = dep + break + if start is None: + bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") + + # We need to figure out which sysroot files we need to expose to this task. + # This needs to match what would get restored from sstate, which is controlled + # ultimately by calls from bitbake to setscene_depvalid(). + # That function expects a setscene dependency tree. We build a dependency tree + # condensed to inter-sstate task dependencies, similar to that used by setscene + # tasks. We can then call into setscene_depvalid() and decide + # which dependencies we can "see" and should expose in the recipe specific sysroot. + setscenedeps = copy.deepcopy(taskdepdata) + + start = set([start]) + + sstatetasks = d.getVar("SSTATETASKS").split() + + def print_dep_tree(deptree): + data = "" + for dep in deptree: + deps = " " + "\n ".join(deptree[dep][3]) + "\n" + data = "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5]) + return data + + #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata)) + + #bb.note(" start2 is %s" % str(start)) + + # If start is an sstate task (like do_package) we need to add in its direct dependencies + # else the code below won't recurse into them. + for dep in set(start): + for dep2 in setscenedeps[dep][3]: + start.add(dep2) + start.remove(dep) + + #bb.note(" start3 is %s" % str(start)) + + # Create collapsed do_populate_sysroot -> do_populate_sysroot tree + for dep in taskdepdata: + data = setscenedeps[dep] + if data[1] not in sstatetasks: + for dep2 in setscenedeps: + data2 = setscenedeps[dep2] + if dep in data2[3]: + data2[3].update(setscenedeps[dep][3]) + data2[3].remove(dep) + if dep in start: + start.update(setscenedeps[dep][3]) + start.remove(dep) + del setscenedeps[dep] + + # Remove circular references + for dep in setscenedeps: + if dep in setscenedeps[dep][3]: + setscenedeps[dep][3].remove(dep) + + #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps)) + #bb.note(" start is %s" % str(start)) + + # Direct dependencies should be present and can be depended upon + for dep in set(start): + if setscenedeps[dep][1] == "do_populate_sysroot": + if dep not in configuredeps: + configuredeps.append(dep) + bb.note("Direct dependencies are %s" % str(configuredeps)) + #bb.note(" or %s" % str(start)) + + msgbuf = [] + # Call into setscene_depvalid for each sub-dependency and only copy sysroot files + # for ones that would be restored from sstate. + done = list(start) + next = list(start) + while next: + new = [] + for dep in next: + data = setscenedeps[dep] + for datadep in data[3]: + if datadep in done: + continue + taskdeps = {} + taskdeps[dep] = setscenedeps[dep][:2] + taskdeps[datadep] = setscenedeps[datadep][:2] + retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf) + if retval: + msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep) + continue + done.append(datadep) + new.append(datadep) + if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot": + configuredeps.append(datadep) + msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0]) + else: + msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0]) + next = new + + # This logging is too verbose for day to day use sadly + #bb.debug(2, "\n".join(msgbuf)) + + depdir = recipesysrootnative + "/installeddeps" + bb.utils.mkdirhier(depdir) + bb.utils.mkdirhier(sharedmanifests) + + lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock") + + fixme = {} + fixme[''] = [] + fixme['native'] = [] + seendirs = set() + postinsts = [] + multilibs = {} + manifests = {} + # All files that we're going to be installing, to find conflicts. + fileset = {} + + for f in os.listdir(depdir): + if not f.endswith(".complete"): + continue + f = depdir + "/" + f + if os.path.islink(f) and not os.path.exists(f): + bb.note("%s no longer exists, removing from sysroot" % f) + lnk = os.readlink(f.replace(".complete", "")) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(f) + os.unlink(f.replace(".complete", "")) + + installed = [] + for dep in configuredeps: + c = setscenedeps[dep][0] + if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"): + bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c) + continue + installed.append(c) + + # We want to remove anything which this task previously installed but is no longer a dependency + taskindex = depdir + "/" + "index." + mytaskname + if os.path.exists(taskindex): + potential = [] + with open(taskindex, "r") as f: + for l in f: + l = l.strip() + if l not in installed: + fl = depdir + "/" + l + if not os.path.exists(fl): + # Was likely already uninstalled + continue + potential.append(l) + # We need to ensure not other task needs this dependency. We hold the sysroot + # lock so we ca search the indexes to check + if potential: + for i in glob.glob(depdir + "/index.*"): + if i.endswith("." + mytaskname): + continue + with open(i, "r") as f: + for l in f: + l = l.strip() + if l in potential: + potential.remove(l) + for l in potential: + fl = depdir + "/" + l + bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l)) + lnk = os.readlink(fl) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(fl) + os.unlink(fl + ".complete") + + msg_exists = [] + msg_adding = [] + for dep in configuredeps: + c = setscenedeps[dep][0] + if c not in installed: + continue + taskhash = setscenedeps[dep][5] + taskmanifest = depdir + "/" + c + "." + taskhash + + if os.path.exists(depdir + "/" + c): + lnk = os.readlink(depdir + "/" + c) + if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"): + msg_exists.append(c) + continue + else: + bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash)) + sstate_clean_manifest(depdir + "/" + lnk, d, workdir) + os.unlink(depdir + "/" + c) + if os.path.lexists(depdir + "/" + c + ".complete"): + os.unlink(depdir + "/" + c + ".complete") + elif os.path.lexists(depdir + "/" + c): + os.unlink(depdir + "/" + c) + + msg_adding.append(c) + + os.symlink(c + "." + taskhash, depdir + "/" + c) + + manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "populate_sysroot", d, multilibs) + destsysroot = d2.getVar("RECIPE_SYSROOT") + + native = False + if c.endswith("-native") or "-cross-" in c or "-crosssdk" in c: + native = True + + if manifest: + newmanifest = collections.OrderedDict() + if native: + fm = fixme['native'] + targetdir = recipesysrootnative + else: + fm = fixme[''] + targetdir = destsysroot + with open(manifest, "r") as f: + manifests[dep] = manifest + for l in f: + l = l.strip() + if l.endswith("/fixmepath"): + fm.append(l) + continue + if l.endswith("/fixmepath.cmd"): + continue + dest = l.replace(stagingdir, "") + dest = "/" + "/".join(dest.split("/")[3:]) + newmanifest[l] = targetdir + dest + + # Check if files have already been installed by another + # recipe and abort if they have, explaining what recipes are + # conflicting. + hashname = targetdir + dest + if not hashname.endswith("/"): + if hashname in fileset: + bb.fatal("The file %s is installed by both %s and %s, aborting" % (dest, c, fileset[hashname])) + else: + fileset[hashname] = c + + # Having multiple identical manifests in each sysroot eats diskspace so + # create a shared pool of them and hardlink if we can. + # We create the manifest in advance so that if something fails during installation, + # or the build is interrupted, subsequent exeuction can cleanup. + sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest) + if not os.path.exists(sharedm): + smlock = bb.utils.lockfile(sharedm + ".lock") + # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other + # but python can lose file handles so we need to do this under a lock. + if not os.path.exists(sharedm): + with open(sharedm, 'w') as m: + for l in newmanifest: + dest = newmanifest[l] + m.write(dest.replace(workdir + "/", "") + "\n") + bb.utils.unlockfile(smlock) + try: + os.link(sharedm, taskmanifest) + except OSError as err: + if err.errno == errno.EXDEV: + bb.utils.copyfile(sharedm, taskmanifest) + else: + raise + # Finally actually install the files + for l in newmanifest: + dest = newmanifest[l] + if l.endswith("/"): + staging_copydir(l, targetdir, dest, seendirs) + continue + staging_copyfile(l, targetdir, dest, postinsts, seendirs) + + bb.note("Installed into sysroot: %s" % str(msg_adding)) + bb.note("Skipping as already exists in sysroot: %s" % str(msg_exists)) + + for f in fixme: + if f == '': + staging_processfixme(fixme[f], recipesysroot, recipesysroot, recipesysrootnative, d) + elif f == 'native': + staging_processfixme(fixme[f], recipesysrootnative, recipesysroot, recipesysrootnative, d) + else: + staging_processfixme(fixme[f], multilibs[f].getVar("RECIPE_SYSROOT"), recipesysroot, recipesysrootnative, d) + + for p in postinsts: + subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT) + + for dep in manifests: + c = setscenedeps[dep][0] + os.symlink(manifests[dep], depdir + "/" + c + ".complete") + + with open(taskindex, "w") as f: + for l in sorted(installed): + f.write(l + "\n") + + bb.utils.unlockfile(lock) +} +extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA" + +python do_prepare_recipe_sysroot () { + bb.build.exec_func("extend_recipe_sysroot", d) +} +addtask do_prepare_recipe_sysroot before do_configure after do_fetch + +# Clean out the recipe specific sysroots before do_fetch +# (use a prefunc so we can order before extend_recipe_sysroot if it gets added) +python clean_recipe_sysroot() { + # We remove these stamps since we're removing any content they'd have added with + # cleandirs. This removes the sigdata too, likely not a big deal, + oe.path.remove(d.getVar("STAMP") + "*addto_recipe_sysroot*") + return +} +clean_recipe_sysroot[cleandirs] += "${RECIPE_SYSROOT} ${RECIPE_SYSROOT_NATIVE}" +do_fetch[prefuncs] += "clean_recipe_sysroot" + +python staging_taskhandler() { + bbtasks = e.tasklist + for task in bbtasks: + deps = d.getVarFlag(task, "depends") + if deps and "populate_sysroot" in deps: + d.appendVarFlag(task, "prefuncs", " extend_recipe_sysroot") +} +staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess" +addhandler staging_taskhandler -- cgit v1.2.3