diff options
Diffstat (limited to 'import-layers/yocto-poky/meta/classes/toaster.bbclass')
-rw-r--r-- | import-layers/yocto-poky/meta/classes/toaster.bbclass | 397 |
1 files changed, 397 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/meta/classes/toaster.bbclass b/import-layers/yocto-poky/meta/classes/toaster.bbclass new file mode 100644 index 000000000..1a70f14a9 --- /dev/null +++ b/import-layers/yocto-poky/meta/classes/toaster.bbclass @@ -0,0 +1,397 @@ +# +# Toaster helper class +# +# Copyright (C) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) +# +# This bbclass is designed to extract data used by OE-Core during the build process, +# for recording in the Toaster system. +# The data access is synchronous, preserving the build data integrity across +# different builds. +# +# The data is transferred through the event system, using the MetadataEvent objects. +# +# The model is to enable the datadump functions as postfuncs, and have the dump +# executed after the real taskfunc has been executed. This prevents task signature changing +# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled. +# +# To enable, use INHERIT in local.conf: +# +# INHERIT += "toaster" +# +# +# +# + +# Find and dump layer info when we got the layers parsed + + + +python toaster_layerinfo_dumpdata() { + import subprocess + + def _get_git_branch(layer_path): + branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] + branch = branch.replace('refs/heads/', '').rstrip() + return branch + + def _get_git_revision(layer_path): + revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip() + return revision + + def _get_url_map_name(layer_name): + """ Some layers have a different name on openembedded.org site, + this method returns the correct name to use in the URL + """ + + url_name = layer_name + url_mapping = {'meta': 'openembedded-core'} + + for key in url_mapping.keys(): + if key == layer_name: + url_name = url_mapping[key] + + return url_name + + def _get_layer_version_information(layer_path): + + layer_version_info = {} + layer_version_info['branch'] = _get_git_branch(layer_path) + layer_version_info['commit'] = _get_git_revision(layer_path) + layer_version_info['priority'] = 0 + + return layer_version_info + + + def _get_layer_dict(layer_path): + + layer_info = {} + layer_name = layer_path.split('/')[-1] + layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' + layer_url_name = _get_url_map_name(layer_name) + + layer_info['name'] = layer_url_name + layer_info['local_path'] = layer_path + layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) + layer_info['version'] = _get_layer_version_information(layer_path) + + return layer_info + + + bblayers = e.data.getVar("BBLAYERS", True) + + llayerinfo = {} + + for layer in { l for l in bblayers.strip().split(" ") if len(l) }: + llayerinfo[layer] = _get_layer_dict(layer) + + + bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data) +} + +# Dump package file info data + +def _toaster_load_pkgdatafile(dirpath, filepath): + import json + import re + pkgdata = {} + with open(os.path.join(dirpath, filepath), "r") as fin: + for line in fin: + try: + kn, kv = line.strip().split(": ", 1) + m = re.match(r"^PKG_([^A-Z:]*)", kn) + if m: + pkgdata['OPKGN'] = m.group(1) + kn = "_".join([x for x in kn.split("_") if x.isupper()]) + pkgdata[kn] = kv.strip() + if kn == 'FILES_INFO': + pkgdata[kn] = json.loads(kv) + + except ValueError: + pass # ignore lines without valid key: value pairs + return pkgdata + +python toaster_package_dumpdata() { + """ + Dumps the data about the packages created by a recipe + """ + + # No need to try and dumpdata if the recipe isn't generating packages + if not d.getVar('PACKAGES', True): + return + + pkgdatadir = d.getVar('PKGDESTWORK', True) + lpkgdata = {} + datadir = os.path.join(pkgdatadir, 'runtime') + + # scan and send data for each generated package + for datafile in os.listdir(datadir): + if not datafile.endswith('.packaged'): + lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) + # Fire an event containing the pkg data + bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) +} + +# 2. Dump output image files information + +python toaster_image_dumpdata() { + """ + Image filename for output images is not standardized. + image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually + have hardcoded ways to create image file names in them. + So we look for files starting with the set name. + + We also look for other files in the images/ directory which don't + match IMAGE_NAME, such as the kernel bzImage, modules tarball etc. + """ + + dir_to_walk = d.getVar('DEPLOY_DIR_IMAGE', True); + image_name = d.getVar('IMAGE_NAME', True); + image_info_data = {} + artifact_info_data = {} + + # collect all images and artifacts in the images directory + for dirpath, dirnames, filenames in os.walk(dir_to_walk): + for filename in filenames: + full_path = os.path.join(dirpath, filename) + try: + if filename.startswith(image_name): + # image + image_info_data[full_path] = os.stat(full_path).st_size + else: + # other non-image artifact + if not os.path.islink(full_path): + artifact_info_data[full_path] = os.stat(full_path).st_size + except OSError as e: + bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) + + bb.event.fire(bb.event.MetadataEvent("ImageFileSize", image_info_data), d) + bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) +} + +python toaster_artifact_dumpdata() { + """ + Dump data about artifacts in the SDK_DEPLOY directory + """ + + dir_to_walk = d.getVar("SDK_DEPLOY", True) + artifact_info_data = {} + + # collect all artifacts in the sdk directory + for dirpath, dirnames, filenames in os.walk(dir_to_walk): + for filename in filenames: + full_path = os.path.join(dirpath, filename) + try: + if not os.path.islink(full_path): + artifact_info_data[full_path] = os.stat(full_path).st_size + except OSError as e: + bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) + + bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) +} + +# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data + +python toaster_collect_task_stats() { + import bb.build + import bb.event + import bb.data + import bb.utils + import os + + toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist") + + if not e.data.getVar('BUILDSTATS_BASE', True): + return # if we don't have buildstats, we cannot collect stats + + def stat_to_float(value): + return float(value.strip('% \n\r')) + + def _append_read_list(v): + lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) + + with open(toaster_statlist_file, "a") as fout: + taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}") + fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) + + bb.utils.unlockfile(lock) + + def _read_stats(filename): + # seconds + cpu_time_user = 0 + cpu_time_system = 0 + + # bytes + disk_io_read = 0 + disk_io_write = 0 + + started = 0 + ended = 0 + + taskname = '' + + statinfo = {} + + with open(filename, 'r') as task_bs: + for line in task_bs.readlines(): + k,v = line.strip().split(": ", 1) + statinfo[k] = v + + if "Started" in statinfo: + started = stat_to_float(statinfo["Started"]) + + if "Ended" in statinfo: + ended = stat_to_float(statinfo["Ended"]) + + if "Child rusage ru_utime" in statinfo: + cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"]) + + if "Child rusage ru_stime" in statinfo: + cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"]) + + if "IO write_bytes" in statinfo: + write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r')) + disk_io_write = disk_io_write + write_bytes + + if "IO read_bytes" in statinfo: + read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r')) + disk_io_read = disk_io_read + read_bytes + + return { + 'stat_file': filename, + 'cpu_time_user': cpu_time_user, + 'cpu_time_system': cpu_time_system, + 'disk_io_read': disk_io_read, + 'disk_io_write': disk_io_write, + 'started': started, + 'ended': ended + } + + if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): + _append_read_list(e) + pass + + if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file): + events = [] + with open(toaster_statlist_file, "r") as fin: + for line in fin: + (taskfile, taskname, filename, recipename) = line.strip().split("::") + stats = _read_stats(filename) + events.append((taskfile, taskname, stats, recipename)) + bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) + os.unlink(toaster_statlist_file) +} + +# dump relevant build history data as an event when the build is completed + +python toaster_buildhistory_dump() { + import re + BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") + BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) + pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) + + + # scan the build targets for this build + images = {} + allpkgs = {} + files = {} + for target in e._pkgs: + target = target.split(':')[0] # strip ':<task>' suffix from the target + installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) + if os.path.exists(installed_img_path): + images[target] = {} + files[target] = {} + files[target]['dirs'] = [] + files[target]['syms'] = [] + files[target]['files'] = [] + with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: + for line in fin: + line = line.rstrip(";") + psize, px = line.split("\t") + punit, pname = px.split(" ") + # this size is "installed-size" as it measures how much space it takes on disk + images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} + + with open("%s/depends.dot" % installed_img_path, "r") as fin: + p = re.compile(r' -> ') + dot = re.compile(r'.*style=dotted') + for line in fin: + line = line.rstrip(';') + linesplit = p.split(line) + if len(linesplit) == 2: + pname = linesplit[0].rstrip('"').strip('"') + dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') + deptype = "depends" + if dot.match(line): + deptype = "recommends" + if not pname in images[target]: + images[target][pname] = {'size': 0, 'depends' : []} + if not dependsname in images[target]: + images[target][dependsname] = {'size': 0, 'depends' : []} + images[target][pname]['depends'].append((dependsname, deptype)) + + with open("%s/files-in-image.txt" % installed_img_path, "r") as fin: + for line in fin: + lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] + if lc[0].startswith("l"): + files[target]['syms'].append(lc) + elif lc[0].startswith("d"): + files[target]['dirs'].append(lc) + else: + files[target]['files'].append(lc) + + for pname in images[target]: + if not pname in allpkgs: + try: + pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname) + except IOError as err: + if err.errno == 2: + # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime + continue + else: + raise + allpkgs[pname] = pkgdata + + + data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files } + + bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data) + +} + +# dump information related to license manifest path + +python toaster_licensemanifest_dump() { + deploy_dir = d.getVar('DEPLOY_DIR', True); + image_name = d.getVar('IMAGE_NAME', True); + + data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name } + + bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d) +} + +# set event handlers +addhandler toaster_layerinfo_dumpdata +toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted" + +addhandler toaster_collect_task_stats +toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed" + +addhandler toaster_buildhistory_dump +toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" + +do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata " +do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata " + +do_package[postfuncs] += "toaster_package_dumpdata " +do_package[vardepsexclude] += "toaster_package_dumpdata " + +do_image_complete[postfuncs] += "toaster_image_dumpdata " +do_image_complete[vardepsexclude] += "toaster_image_dumpdata " + +do_rootfs[postfuncs] += "toaster_licensemanifest_dump " +do_rootfs[vardepsexclude] += "toaster_licensemanifest_dump " + +do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata " +do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata " |