diff options
Diffstat (limited to 'yocto-poky/meta/classes/toaster.bbclass')
| -rw-r--r-- | yocto-poky/meta/classes/toaster.bbclass | 397 |
1 files changed, 0 insertions, 397 deletions
diff --git a/yocto-poky/meta/classes/toaster.bbclass b/yocto-poky/meta/classes/toaster.bbclass deleted file mode 100644 index 1a70f14a9..000000000 --- a/yocto-poky/meta/classes/toaster.bbclass +++ /dev/null @@ -1,397 +0,0 @@ -# -# Toaster helper class -# -# Copyright (C) 2013 Intel Corporation -# -# Released under the MIT license (see COPYING.MIT) -# -# This bbclass is designed to extract data used by OE-Core during the build process, -# for recording in the Toaster system. -# The data access is synchronous, preserving the build data integrity across -# different builds. -# -# The data is transferred through the event system, using the MetadataEvent objects. -# -# The model is to enable the datadump functions as postfuncs, and have the dump -# executed after the real taskfunc has been executed. This prevents task signature changing -# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled. -# -# To enable, use INHERIT in local.conf: -# -# INHERIT += "toaster" -# -# -# -# - -# Find and dump layer info when we got the layers parsed - - - -python toaster_layerinfo_dumpdata() { - import subprocess - - def _get_git_branch(layer_path): - branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] - branch = branch.replace('refs/heads/', '').rstrip() - return branch - - def _get_git_revision(layer_path): - revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip() - return revision - - def _get_url_map_name(layer_name): - """ Some layers have a different name on openembedded.org site, - this method returns the correct name to use in the URL - """ - - url_name = layer_name - url_mapping = {'meta': 'openembedded-core'} - - for key in url_mapping.keys(): - if key == layer_name: - url_name = url_mapping[key] - - return url_name - - def _get_layer_version_information(layer_path): - - layer_version_info = {} - layer_version_info['branch'] = _get_git_branch(layer_path) - layer_version_info['commit'] = _get_git_revision(layer_path) - layer_version_info['priority'] = 0 - - return layer_version_info - - - def _get_layer_dict(layer_path): - - layer_info = {} - layer_name = layer_path.split('/')[-1] - layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' - layer_url_name = _get_url_map_name(layer_name) - - layer_info['name'] = layer_url_name - layer_info['local_path'] = layer_path - layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) - layer_info['version'] = _get_layer_version_information(layer_path) - - return layer_info - - - bblayers = e.data.getVar("BBLAYERS", True) - - llayerinfo = {} - - for layer in { l for l in bblayers.strip().split(" ") if len(l) }: - llayerinfo[layer] = _get_layer_dict(layer) - - - bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data) -} - -# Dump package file info data - -def _toaster_load_pkgdatafile(dirpath, filepath): - import json - import re - pkgdata = {} - with open(os.path.join(dirpath, filepath), "r") as fin: - for line in fin: - try: - kn, kv = line.strip().split(": ", 1) - m = re.match(r"^PKG_([^A-Z:]*)", kn) - if m: - pkgdata['OPKGN'] = m.group(1) - kn = "_".join([x for x in kn.split("_") if x.isupper()]) - pkgdata[kn] = kv.strip() - if kn == 'FILES_INFO': - pkgdata[kn] = json.loads(kv) - - except ValueError: - pass # ignore lines without valid key: value pairs - return pkgdata - -python toaster_package_dumpdata() { - """ - Dumps the data about the packages created by a recipe - """ - - # No need to try and dumpdata if the recipe isn't generating packages - if not d.getVar('PACKAGES', True): - return - - pkgdatadir = d.getVar('PKGDESTWORK', True) - lpkgdata = {} - datadir = os.path.join(pkgdatadir, 'runtime') - - # scan and send data for each generated package - for datafile in os.listdir(datadir): - if not datafile.endswith('.packaged'): - lpkgdata = _toaster_load_pkgdatafile(datadir, datafile) - # Fire an event containing the pkg data - bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) -} - -# 2. Dump output image files information - -python toaster_image_dumpdata() { - """ - Image filename for output images is not standardized. - image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually - have hardcoded ways to create image file names in them. - So we look for files starting with the set name. - - We also look for other files in the images/ directory which don't - match IMAGE_NAME, such as the kernel bzImage, modules tarball etc. - """ - - dir_to_walk = d.getVar('DEPLOY_DIR_IMAGE', True); - image_name = d.getVar('IMAGE_NAME', True); - image_info_data = {} - artifact_info_data = {} - - # collect all images and artifacts in the images directory - for dirpath, dirnames, filenames in os.walk(dir_to_walk): - for filename in filenames: - full_path = os.path.join(dirpath, filename) - try: - if filename.startswith(image_name): - # image - image_info_data[full_path] = os.stat(full_path).st_size - else: - # other non-image artifact - if not os.path.islink(full_path): - artifact_info_data[full_path] = os.stat(full_path).st_size - except OSError as e: - bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) - - bb.event.fire(bb.event.MetadataEvent("ImageFileSize", image_info_data), d) - bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) -} - -python toaster_artifact_dumpdata() { - """ - Dump data about artifacts in the SDK_DEPLOY directory - """ - - dir_to_walk = d.getVar("SDK_DEPLOY", True) - artifact_info_data = {} - - # collect all artifacts in the sdk directory - for dirpath, dirnames, filenames in os.walk(dir_to_walk): - for filename in filenames: - full_path = os.path.join(dirpath, filename) - try: - if not os.path.islink(full_path): - artifact_info_data[full_path] = os.stat(full_path).st_size - except OSError as e: - bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d) - - bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d) -} - -# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data - -python toaster_collect_task_stats() { - import bb.build - import bb.event - import bb.data - import bb.utils - import os - - toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist") - - if not e.data.getVar('BUILDSTATS_BASE', True): - return # if we don't have buildstats, we cannot collect stats - - def stat_to_float(value): - return float(value.strip('% \n\r')) - - def _append_read_list(v): - lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) - - with open(toaster_statlist_file, "a") as fout: - taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}") - fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) - - bb.utils.unlockfile(lock) - - def _read_stats(filename): - # seconds - cpu_time_user = 0 - cpu_time_system = 0 - - # bytes - disk_io_read = 0 - disk_io_write = 0 - - started = 0 - ended = 0 - - taskname = '' - - statinfo = {} - - with open(filename, 'r') as task_bs: - for line in task_bs.readlines(): - k,v = line.strip().split(": ", 1) - statinfo[k] = v - - if "Started" in statinfo: - started = stat_to_float(statinfo["Started"]) - - if "Ended" in statinfo: - ended = stat_to_float(statinfo["Ended"]) - - if "Child rusage ru_utime" in statinfo: - cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"]) - - if "Child rusage ru_stime" in statinfo: - cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"]) - - if "IO write_bytes" in statinfo: - write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r')) - disk_io_write = disk_io_write + write_bytes - - if "IO read_bytes" in statinfo: - read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r')) - disk_io_read = disk_io_read + read_bytes - - return { - 'stat_file': filename, - 'cpu_time_user': cpu_time_user, - 'cpu_time_system': cpu_time_system, - 'disk_io_read': disk_io_read, - 'disk_io_write': disk_io_write, - 'started': started, - 'ended': ended - } - - if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): - _append_read_list(e) - pass - - if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file): - events = [] - with open(toaster_statlist_file, "r") as fin: - for line in fin: - (taskfile, taskname, filename, recipename) = line.strip().split("::") - stats = _read_stats(filename) - events.append((taskfile, taskname, stats, recipename)) - bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) - os.unlink(toaster_statlist_file) -} - -# dump relevant build history data as an event when the build is completed - -python toaster_buildhistory_dump() { - import re - BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") - BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) - pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) - - - # scan the build targets for this build - images = {} - allpkgs = {} - files = {} - for target in e._pkgs: - target = target.split(':')[0] # strip ':<task>' suffix from the target - installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) - if os.path.exists(installed_img_path): - images[target] = {} - files[target] = {} - files[target]['dirs'] = [] - files[target]['syms'] = [] - files[target]['files'] = [] - with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: - for line in fin: - line = line.rstrip(";") - psize, px = line.split("\t") - punit, pname = px.split(" ") - # this size is "installed-size" as it measures how much space it takes on disk - images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} - - with open("%s/depends.dot" % installed_img_path, "r") as fin: - p = re.compile(r' -> ') - dot = re.compile(r'.*style=dotted') - for line in fin: - line = line.rstrip(';') - linesplit = p.split(line) - if len(linesplit) == 2: - pname = linesplit[0].rstrip('"').strip('"') - dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') - deptype = "depends" - if dot.match(line): - deptype = "recommends" - if not pname in images[target]: - images[target][pname] = {'size': 0, 'depends' : []} - if not dependsname in images[target]: - images[target][dependsname] = {'size': 0, 'depends' : []} - images[target][pname]['depends'].append((dependsname, deptype)) - - with open("%s/files-in-image.txt" % installed_img_path, "r") as fin: - for line in fin: - lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] - if lc[0].startswith("l"): - files[target]['syms'].append(lc) - elif lc[0].startswith("d"): - files[target]['dirs'].append(lc) - else: - files[target]['files'].append(lc) - - for pname in images[target]: - if not pname in allpkgs: - try: - pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname) - except IOError as err: - if err.errno == 2: - # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime - continue - else: - raise - allpkgs[pname] = pkgdata - - - data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files } - - bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data) - -} - -# dump information related to license manifest path - -python toaster_licensemanifest_dump() { - deploy_dir = d.getVar('DEPLOY_DIR', True); - image_name = d.getVar('IMAGE_NAME', True); - - data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name } - - bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d) -} - -# set event handlers -addhandler toaster_layerinfo_dumpdata -toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted" - -addhandler toaster_collect_task_stats -toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed" - -addhandler toaster_buildhistory_dump -toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" - -do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata " -do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata " - -do_package[postfuncs] += "toaster_package_dumpdata " -do_package[vardepsexclude] += "toaster_package_dumpdata " - -do_image_complete[postfuncs] += "toaster_image_dumpdata " -do_image_complete[vardepsexclude] += "toaster_image_dumpdata " - -do_rootfs[postfuncs] += "toaster_licensemanifest_dump " -do_rootfs[vardepsexclude] += "toaster_licensemanifest_dump " - -do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata " -do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata " |