diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py')
| -rw-r--r-- | yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py | 1521 |
1 files changed, 0 insertions, 1521 deletions
diff --git a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py deleted file mode 100644 index 93979054d..000000000 --- a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py +++ /dev/null @@ -1,1521 +0,0 @@ -# -# BitBake ToasterUI Implementation -# -# Copyright (C) 2013 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import sys -import bb -import re -import os - -os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings" - - -import django -from django.utils import timezone - - -def _configure_toaster(): - """ Add toaster to sys path for importing modules - """ - sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) -_configure_toaster() - -django.setup() - -from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText -from orm.models import Target_Image_File, BuildArtifact -from orm.models import Variable, VariableHistory -from orm.models import Package, Package_File, Target_Installed_Package, Target_File -from orm.models import Task_Dependency, Package_Dependency -from orm.models import Recipe_Dependency, Provides -from orm.models import Project, CustomImagePackage, CustomImageRecipe - -from bldcontrol.models import BuildEnvironment, BuildRequest - -from bb.msg import BBLogFormatter as formatter -from django.db import models -from pprint import pformat -import logging -from datetime import datetime, timedelta - -from django.db import transaction, connection - -# pylint: disable=invalid-name -# the logger name is standard throughout BitBake -logger = logging.getLogger("ToasterLogger") - - -class NotExisting(Exception): - pass - -class ORMWrapper(object): - """ This class creates the dictionaries needed to store information in the database - following the format defined by the Django models. It is also used to save this - information in the database. - """ - - def __init__(self): - self.layer_version_objects = [] - self.layer_version_built = [] - self.task_objects = {} - self.recipe_objects = {} - - @staticmethod - def _build_key(**kwargs): - key = "0" - for k in sorted(kwargs.keys()): - if isinstance(kwargs[k], models.Model): - key += "-%d" % kwargs[k].id - else: - key += "-%s" % str(kwargs[k]) - return key - - - def _cached_get_or_create(self, clazz, **kwargs): - """ This is a memory-cached get_or_create. We assume that the objects will not be created in the - database through any other means. - """ - - assert issubclass(clazz, models.Model), "_cached_get_or_create needs to get the class as first argument" - - key = ORMWrapper._build_key(**kwargs) - dictname = "objects_%s" % clazz.__name__ - if not dictname in vars(self).keys(): - vars(self)[dictname] = {} - - created = False - if not key in vars(self)[dictname].keys(): - vars(self)[dictname][key], created = \ - clazz.objects.get_or_create(**kwargs) - - return (vars(self)[dictname][key], created) - - - def _cached_get(self, clazz, **kwargs): - """ This is a memory-cached get. We assume that the objects will not change in the database between gets. - """ - assert issubclass(clazz, models.Model), "_cached_get needs to get the class as first argument" - - key = ORMWrapper._build_key(**kwargs) - dictname = "objects_%s" % clazz.__name__ - - if not dictname in vars(self).keys(): - vars(self)[dictname] = {} - - if not key in vars(self)[dictname].keys(): - vars(self)[dictname][key] = clazz.objects.get(**kwargs) - - return vars(self)[dictname][key] - - def _timestamp_to_datetime(self, secs): - """ - Convert timestamp in seconds to Python datetime - """ - return datetime(1970, 1, 1) + timedelta(seconds=secs) - - # pylint: disable=no-self-use - # we disable detection of no self use in functions because the methods actually work on the object - # even if they don't touch self anywhere - - # pylint: disable=bad-continuation - # we do not follow the python conventions for continuation indentation due to long lines here - - def create_build_object(self, build_info, brbe, project_id): - assert 'machine' in build_info - assert 'distro' in build_info - assert 'distro_version' in build_info - assert 'started_on' in build_info - assert 'cooker_log_path' in build_info - assert 'build_name' in build_info - assert 'bitbake_version' in build_info - - prj = None - buildrequest = None - if brbe is not None: # this build was triggered by a request from a user - logger.debug(1, "buildinfohelper: brbe is %s" % brbe) - br, _ = brbe.split(":") - buildrequest = BuildRequest.objects.get(pk = br) - prj = buildrequest.project - - elif project_id is not None: # this build was triggered by an external system for a specific project - logger.debug(1, "buildinfohelper: project is %s" % prj) - prj = Project.objects.get(pk = project_id) - - else: # this build was triggered by a legacy system, or command line interactive mode - prj = Project.objects.get_or_create_default_project() - logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) - - - if buildrequest is not None: - build = buildrequest.build - logger.info("Updating existing build, with %s", build_info) - build.project = prj - build.machine=build_info['machine'] - build.distro=build_info['distro'] - build.distro_version=build_info['distro_version'] - build.cooker_log_path=build_info['cooker_log_path'] - build.build_name=build_info['build_name'] - build.bitbake_version=build_info['bitbake_version'] - build.save() - - else: - build = Build.objects.create( - project = prj, - machine=build_info['machine'], - distro=build_info['distro'], - distro_version=build_info['distro_version'], - started_on=build_info['started_on'], - completed_on=build_info['started_on'], - cooker_log_path=build_info['cooker_log_path'], - build_name=build_info['build_name'], - bitbake_version=build_info['bitbake_version']) - - logger.debug(1, "buildinfohelper: build is created %s" % build) - - if buildrequest is not None: - buildrequest.build = build - buildrequest.save() - - return build - - @staticmethod - def get_or_create_targets(target_info): - result = [] - for target in target_info['targets']: - task = '' - if ':' in target: - target, task = target.split(':', 1) - if task.startswith('do_'): - task = task[3:] - if task == 'build': - task = '' - obj, created = Target.objects.get_or_create(build=target_info['build'], - target=target) - if created: - obj.is_image = False - if task: - obj.task = task - obj.save() - result.append(obj) - return result - - def update_build_object(self, build, errors, warnings, taskfailures): - assert isinstance(build,Build) - assert isinstance(errors, int) - assert isinstance(warnings, int) - - if build.outcome == Build.CANCELLED: - return - try: - if build.buildrequest.state == BuildRequest.REQ_CANCELLING: - return - except AttributeError: - # We may not have a buildrequest if this is a command line build - pass - - outcome = Build.SUCCEEDED - if errors or taskfailures: - outcome = Build.FAILED - - build.completed_on = timezone.now() - build.outcome = outcome - build.save() - - def update_target_set_license_manifest(self, target, license_manifest_path): - target.license_manifest_path = license_manifest_path - target.save() - - def update_task_object(self, build, task_name, recipe_name, task_stats): - """ - Find the task for build which matches the recipe and task name - to be stored - """ - task_to_update = Task.objects.get( - build = build, - task_name = task_name, - recipe__name = recipe_name - ) - - if 'started' in task_stats and 'ended' in task_stats: - task_to_update.started = self._timestamp_to_datetime(task_stats['started']) - task_to_update.ended = self._timestamp_to_datetime(task_stats['ended']) - task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started']) - task_to_update.cpu_time_user = task_stats.get('cpu_time_user') - task_to_update.cpu_time_system = task_stats.get('cpu_time_system') - if 'disk_io_read' in task_stats and 'disk_io_write' in task_stats: - task_to_update.disk_io_read = task_stats['disk_io_read'] - task_to_update.disk_io_write = task_stats['disk_io_write'] - task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write'] - - task_to_update.save() - - def get_update_task_object(self, task_information, must_exist = False): - assert 'build' in task_information - assert 'recipe' in task_information - assert 'task_name' in task_information - - # we use must_exist info for database look-up optimization - task_object, created = self._cached_get_or_create(Task, - build=task_information['build'], - recipe=task_information['recipe'], - task_name=task_information['task_name'] - ) - if created and must_exist: - task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) - raise NotExisting("Task object created when expected to exist", task_information) - - object_changed = False - for v in vars(task_object): - if v in task_information.keys(): - if vars(task_object)[v] != task_information[v]: - vars(task_object)[v] = task_information[v] - object_changed = True - - # update setscene-related information if the task has a setscene - if task_object.outcome == Task.OUTCOME_COVERED and 1 == task_object.get_related_setscene().count(): - task_object.outcome = Task.OUTCOME_CACHED - object_changed = True - - outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build, - recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome - if outcome_task_setscene == Task.OUTCOME_SUCCESS: - task_object.sstate_result = Task.SSTATE_RESTORED - object_changed = True - elif outcome_task_setscene == Task.OUTCOME_FAILED: - task_object.sstate_result = Task.SSTATE_FAILED - object_changed = True - - if object_changed: - task_object.save() - return task_object - - - def get_update_recipe_object(self, recipe_information, must_exist = False): - assert 'layer_version' in recipe_information - assert 'file_path' in recipe_information - assert 'pathflags' in recipe_information - - assert not recipe_information['file_path'].startswith("/") # we should have layer-relative paths at all times - - - def update_recipe_obj(recipe_object): - object_changed = False - for v in vars(recipe_object): - if v in recipe_information.keys(): - object_changed = True - vars(recipe_object)[v] = recipe_information[v] - - if object_changed: - recipe_object.save() - - recipe, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'], - file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags']) - - update_recipe_obj(recipe) - - built_recipe = None - # Create a copy of the recipe for historical puposes and update it - for built_layer in self.layer_version_built: - if built_layer.layer == recipe_information['layer_version'].layer: - built_recipe, c = self._cached_get_or_create(Recipe, - layer_version=built_layer, - file_path=recipe_information['file_path'], - pathflags = recipe_information['pathflags']) - update_recipe_obj(built_recipe) - break - - - # If we're in analysis mode or if this is a custom recipe - # then we are wholly responsible for the data - # and therefore we return the 'real' recipe rather than the build - # history copy of the recipe. - if recipe_information['layer_version'].build is not None and \ - recipe_information['layer_version'].build.project == \ - Project.objects.get_or_create_default_project(): - return recipe - - if built_recipe is None: - return recipe - - return built_recipe - - def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): - if isinstance(layer_obj, Layer_Version): - # Special case the toaster-custom-images layer which is created - # on the fly so don't update the values which may cause the layer - # to be duplicated on a future get_or_create - if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME: - return layer_obj - # We already found our layer version for this build so just - # update it with the new build information - logger.debug("We found our layer from toaster") - layer_obj.local_path = layer_version_information['local_path'] - layer_obj.save() - self.layer_version_objects.append(layer_obj) - - # create a new copy of this layer version as a snapshot for - # historical purposes - layer_copy, c = Layer_Version.objects.get_or_create( - build=build_obj, - layer=layer_obj.layer, - up_branch=layer_obj.up_branch, - branch=layer_version_information['branch'], - commit=layer_version_information['commit'], - local_path=layer_version_information['local_path'], - ) - - logger.info("created new historical layer version %d", - layer_copy.pk) - - self.layer_version_built.append(layer_copy) - - return layer_obj - - assert isinstance(build_obj, Build) - assert isinstance(layer_obj, Layer) - assert 'branch' in layer_version_information - assert 'commit' in layer_version_information - assert 'priority' in layer_version_information - assert 'local_path' in layer_version_information - - # If we're doing a command line build then associate this new layer with the - # project to avoid it 'contaminating' toaster data - project = None - if build_obj.project == Project.objects.get_or_create_default_project(): - project = build_obj.project - - layer_version_object, _ = Layer_Version.objects.get_or_create( - build = build_obj, - layer = layer_obj, - branch = layer_version_information['branch'], - commit = layer_version_information['commit'], - priority = layer_version_information['priority'], - local_path = layer_version_information['local_path'], - project=project) - - self.layer_version_objects.append(layer_version_object) - - return layer_version_object - - def get_update_layer_object(self, layer_information, brbe): - assert 'name' in layer_information - assert 'layer_index_url' in layer_information - - if brbe is None: - layer_object, _ = Layer.objects.get_or_create( - name=layer_information['name'], - layer_index_url=layer_information['layer_index_url']) - return layer_object - else: - # we are under managed mode; we must match the layer used in the Project Layer - br_id, be_id = brbe.split(":") - - # find layer by checkout path; - from bldcontrol import bbcontroller - bc = bbcontroller.getBuildEnvironmentController(pk = be_id) - - # we might have a race condition here, as the project layers may change between the build trigger and the actual build execution - # but we can only match on the layer name, so the worst thing can happen is a mis-identification of the layer, not a total failure - - # note that this is different - buildrequest = BuildRequest.objects.get(pk = br_id) - for brl in buildrequest.brlayer_set.all(): - localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath) - # we get a relative path, unless running in HEAD mode where the path is absolute - if not localdirname.startswith("/"): - localdirname = os.path.join(bc.be.sourcedir, localdirname) - #logger.debug(1, "Localdirname %s lcal_path %s" % (localdirname, layer_information['local_path'])) - if localdirname.startswith(layer_information['local_path']): - # If the build request came from toaster this field - # should contain the information from the layer_version - # That created this build request. - if brl.layer_version: - return brl.layer_version - - # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build() - #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname)) - - for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name): - if pl.layercommit.layer.vcs_url == brl.giturl : - layer = pl.layercommit.layer - layer.save() - return layer - - raise NotExisting("Unidentified layer %s" % pformat(layer_information)) - - - def save_target_file_information(self, build_obj, target_obj, filedata): - assert isinstance(build_obj, Build) - assert isinstance(target_obj, Target) - dirs = filedata['dirs'] - files = filedata['files'] - syms = filedata['syms'] - - # always create the root directory as a special case; - # note that this is never displayed, so the owner, group, - # size, permission are irrelevant - tf_obj = Target_File.objects.create(target = target_obj, - path = '/', - size = 0, - owner = '', - group = '', - permission = '', - inodetype = Target_File.ITYPE_DIRECTORY) - tf_obj.save() - - # insert directories, ordered by name depth - for d in sorted(dirs, key=lambda x:len(x[-1].split("/"))): - (user, group, size) = d[1:4] - permission = d[0][1:] - path = d[4].lstrip(".") - - # we already created the root directory, so ignore any - # entry for it - if len(path) == 0: - continue - - parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) - if len(parent_path) == 0: - parent_path = "/" - parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) - tf_obj = Target_File.objects.create( - target = target_obj, - path = unicode(path, 'utf-8'), - size = size, - inodetype = Target_File.ITYPE_DIRECTORY, - permission = permission, - owner = user, - group = group, - directory = parent_obj) - - - # we insert files - for d in files: - (user, group, size) = d[1:4] - permission = d[0][1:] - path = d[4].lstrip(".") - parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) - inodetype = Target_File.ITYPE_REGULAR - if d[0].startswith('b'): - inodetype = Target_File.ITYPE_BLOCK - if d[0].startswith('c'): - inodetype = Target_File.ITYPE_CHARACTER - if d[0].startswith('p'): - inodetype = Target_File.ITYPE_FIFO - - tf_obj = Target_File.objects.create( - target = target_obj, - path = unicode(path, 'utf-8'), - size = size, - inodetype = inodetype, - permission = permission, - owner = user, - group = group) - parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) - tf_obj.directory = parent_obj - tf_obj.save() - - # we insert symlinks - for d in syms: - (user, group, size) = d[1:4] - permission = d[0][1:] - path = d[4].lstrip(".") - filetarget_path = d[6] - - parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1]) - if not filetarget_path.startswith("/"): - # we have a relative path, get a normalized absolute one - filetarget_path = parent_path + "/" + filetarget_path - fcp = filetarget_path.split("/") - fcpl = [] - for i in fcp: - if i == "..": - fcpl.pop() - else: - fcpl.append(i) - filetarget_path = "/".join(fcpl) - - try: - filetarget_obj = Target_File.objects.get( - target = target_obj, - path = unicode(filetarget_path, 'utf-8')) - except Target_File.DoesNotExist: - # we might have an invalid link; no way to detect this. just set it to None - filetarget_obj = None - - parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY) - - tf_obj = Target_File.objects.create( - target = target_obj, - path = unicode(path, 'utf-8'), - size = size, - inodetype = Target_File.ITYPE_SYMLINK, - permission = permission, - owner = user, - group = group, - directory = parent_obj, - sym_target = filetarget_obj) - - - def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes, built_package=False): - assert isinstance(build_obj, Build) - assert isinstance(target_obj, Target) - - errormsg = "" - for p in packagedict: - # Search name swtiches round the installed name vs package name - # by default installed name == package name - searchname = p - if p not in pkgpnmap: - logger.warning("Image packages list contains %p, but is" - " missing from all packages list where the" - " metadata comes from. Skipping...", p) - continue - - if 'OPKGN' in pkgpnmap[p].keys(): - searchname = pkgpnmap[p]['OPKGN'] - - built_recipe = recipes[pkgpnmap[p]['PN']] - - if built_package: - packagedict[p]['object'], created = Package.objects.get_or_create( build = build_obj, name = searchname ) - recipe = built_recipe - else: - packagedict[p]['object'], created = \ - CustomImagePackage.objects.get_or_create(name=searchname) - # Clear the Package_Dependency objects as we're going to update - # the CustomImagePackage with the latest dependency information - packagedict[p]['object'].package_dependencies_target.all().delete() - packagedict[p]['object'].package_dependencies_source.all().delete() - try: - recipe = self._cached_get( - Recipe, - name=built_recipe.name, - layer_version__build=None, - layer_version__up_branch= - built_recipe.layer_version.up_branch, - file_path=built_recipe.file_path, - version=built_recipe.version - ) - except (Recipe.DoesNotExist, - Recipe.MultipleObjectsReturned) as e: - logger.info("We did not find one recipe for the" - "configuration data package %s %s" % (p, e)) - continue - - if created or packagedict[p]['object'].size == -1: # save the data anyway we can, not just if it was not created here; bug [YOCTO #6887] - # fill in everything we can from the runtime-reverse package data - try: - packagedict[p]['object'].recipe = recipe - packagedict[p]['object'].version = pkgpnmap[p]['PV'] - packagedict[p]['object'].installed_name = p - packagedict[p]['object'].revision = pkgpnmap[p]['PR'] - packagedict[p]['object'].license = pkgpnmap[p]['LICENSE'] - packagedict[p]['object'].section = pkgpnmap[p]['SECTION'] - packagedict[p]['object'].summary = pkgpnmap[p]['SUMMARY'] - packagedict[p]['object'].description = pkgpnmap[p]['DESCRIPTION'] - packagedict[p]['object'].size = int(pkgpnmap[p]['PKGSIZE']) - - # no files recorded for this package, so save files info - packagefile_objects = [] - for targetpath in pkgpnmap[p]['FILES_INFO']: - targetfilesize = pkgpnmap[p]['FILES_INFO'][targetpath] - packagefile_objects.append(Package_File( package = packagedict[p]['object'], - path = targetpath, - size = targetfilesize)) - if len(packagefile_objects): - Package_File.objects.bulk_create(packagefile_objects) - except KeyError as e: - errormsg += " stpi: Key error, package %s key %s \n" % ( p, e ) - - # save disk installed size - packagedict[p]['object'].installed_size = packagedict[p]['size'] - packagedict[p]['object'].save() - - if built_package: - Target_Installed_Package.objects.create(target = target_obj, package = packagedict[p]['object']) - - packagedeps_objs = [] - for p in packagedict: - for (px,deptype) in packagedict[p]['depends']: - if deptype == 'depends': - tdeptype = Package_Dependency.TYPE_TRDEPENDS - elif deptype == 'recommends': - tdeptype = Package_Dependency.TYPE_TRECOMMENDS - - try: - packagedeps_objs.append(Package_Dependency( - package = packagedict[p]['object'], - depends_on = packagedict[px]['object'], - dep_type = tdeptype, - target = target_obj)) - except KeyError as e: - logger.warn("Could not add dependency to the package %s " - "because %s is an unknown package", p, px) - - if len(packagedeps_objs) > 0: - Package_Dependency.objects.bulk_create(packagedeps_objs) - else: - logger.info("No package dependencies created") - - if len(errormsg) > 0: - logger.warn("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) - - def save_target_image_file_information(self, target_obj, file_name, file_size): - Target_Image_File.objects.create( target = target_obj, - file_name = file_name, - file_size = file_size) - - def save_artifact_information(self, build_obj, file_name, file_size): - # we skip the image files from other builds - if Target_Image_File.objects.filter(file_name = file_name).count() > 0: - return - - # do not update artifacts found in other builds - if BuildArtifact.objects.filter(file_name = file_name).count() > 0: - return - - BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size) - - def create_logmessage(self, log_information): - assert 'build' in log_information - assert 'level' in log_information - assert 'message' in log_information - - log_object = LogMessage.objects.create( - build = log_information['build'], - level = log_information['level'], - message = log_information['message']) - - for v in vars(log_object): - if v in log_information.keys(): - vars(log_object)[v] = log_information[v] - - return log_object.save() - - - def save_build_package_information(self, build_obj, package_info, recipes, - built_package): - # assert isinstance(build_obj, Build) - - # create and save the object - pname = package_info['PKG'] - built_recipe = recipes[package_info['PN']] - if 'OPKGN' in package_info.keys(): - pname = package_info['OPKGN'] - - if built_package: - bp_object, _ = Package.objects.get_or_create( build = build_obj, - name = pname ) - recipe = built_recipe - else: - bp_object, created = \ - CustomImagePackage.objects.get_or_create(name=pname) - try: - recipe = self._cached_get(Recipe, - name=built_recipe.name, - layer_version__build=None, - file_path=built_recipe.file_path, - version=built_recipe.version) - - except (Recipe.DoesNotExist, Recipe.MultipleObjectsReturned): - logger.debug("We did not find one recipe for the configuration" - "data package %s" % pname) - return - - bp_object.installed_name = package_info['PKG'] - bp_object.recipe = recipe - bp_object.version = package_info['PKGV'] - bp_object.revision = package_info['PKGR'] - bp_object.summary = package_info['SUMMARY'] - bp_object.description = package_info['DESCRIPTION'] - bp_object.size = int(package_info['PKGSIZE']) - bp_object.section = package_info['SECTION'] - bp_object.license = package_info['LICENSE'] - bp_object.save() - - # save any attached file information - packagefile_objects = [] - for path in package_info['FILES_INFO']: - packagefile_objects.append(Package_File( package = bp_object, - path = path, - size = package_info['FILES_INFO'][path] )) - if len(packagefile_objects): - Package_File.objects.bulk_create(packagefile_objects) - - def _po_byname(p): - if built_package: - pkg, created = Package.objects.get_or_create(build=build_obj, - name=p) - else: - pkg, created = CustomImagePackage.objects.get_or_create(name=p) - - if created: - pkg.size = -1 - pkg.save() - return pkg - - packagedeps_objs = [] - # save soft dependency information - if 'RDEPENDS' in package_info and package_info['RDEPENDS']: - for p in bb.utils.explode_deps(package_info['RDEPENDS']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RDEPENDS)) - if 'RPROVIDES' in package_info and package_info['RPROVIDES']: - for p in bb.utils.explode_deps(package_info['RPROVIDES']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RPROVIDES)) - if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']: - for p in bb.utils.explode_deps(package_info['RRECOMMENDS']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RRECOMMENDS)) - if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']: - for p in bb.utils.explode_deps(package_info['RSUGGESTS']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RSUGGESTS)) - if 'RREPLACES' in package_info and package_info['RREPLACES']: - for p in bb.utils.explode_deps(package_info['RREPLACES']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RREPLACES)) - if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']: - for p in bb.utils.explode_deps(package_info['RCONFLICTS']): - packagedeps_objs.append(Package_Dependency( package = bp_object, - depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS)) - - if len(packagedeps_objs) > 0: - Package_Dependency.objects.bulk_create(packagedeps_objs) - - return bp_object - - def save_build_variables(self, build_obj, vardump): - assert isinstance(build_obj, Build) - - for k in vardump: - desc = vardump[k]['doc'] - if desc is None: - var_words = [word for word in k.split('_')] - root_var = "_".join([word for word in var_words if word.isupper()]) - if root_var and root_var != k and root_var in vardump: - desc = vardump[root_var]['doc'] - if desc is None: - desc = '' - if len(desc): - HelpText.objects.get_or_create(build=build_obj, - area=HelpText.VARIABLE, - key=k, text=desc) - if not bool(vardump[k]['func']): - value = vardump[k]['v'] - if value is None: - value = '' - variable_obj = Variable.objects.create( build = build_obj, - variable_name = k, - variable_value = value, - description = desc) - - varhist_objects = [] - for vh in vardump[k]['history']: - if not 'documentation.conf' in vh['file']: - varhist_objects.append(VariableHistory( variable = variable_obj, - file_name = vh['file'], - line_number = vh['line'], - operation = vh['op'])) - if len(varhist_objects): - VariableHistory.objects.bulk_create(varhist_objects) - - -class MockEvent(object): - """ This object is used to create event, for which normal event-processing methods can - be used, out of data that is not coming via an actual event - """ - def __init__(self): - self.msg = None - self.levelno = None - self.taskname = None - self.taskhash = None - self.pathname = None - self.lineno = None - - -class BuildInfoHelper(object): - """ This class gathers the build information from the server and sends it - towards the ORM wrapper for storing in the database - It is instantiated once per build - Keeps in memory all data that needs matching before writing it to the database - """ - - # pylint: disable=protected-access - # the code will look into the protected variables of the event; no easy way around this - # pylint: disable=bad-continuation - # we do not follow the python conventions for continuation indentation due to long lines here - - def __init__(self, server, has_build_history = False, brbe = None): - self.internal_state = {} - self.internal_state['taskdata'] = {} - self.internal_state['targets'] = [] - self.task_order = 0 - self.autocommit_step = 1 - self.server = server - # we use manual transactions if the database doesn't autocommit on us - if not connection.features.autocommits_when_autocommit_is_off: - transaction.set_autocommit(False) - self.orm_wrapper = ORMWrapper() - self.has_build_history = has_build_history - self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0] - - # this is set for Toaster-triggered builds by localhostbecontroller - # via toasterui - self.brbe = brbe - - self.project = None - - logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) - - - ################### - ## methods to convert event/external info into objects that the ORM layer uses - - - def _get_build_information(self, build_log_path): - build_info = {} - build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] - build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] - build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] - build_info['started_on'] = timezone.now() - build_info['completed_on'] = timezone.now() - build_info['cooker_log_path'] = build_log_path - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] - build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] - build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] - return build_info - - def _get_task_information(self, event, recipe): - assert 'taskname' in vars(event) - - task_information = {} - task_information['build'] = self.internal_state['build'] - task_information['outcome'] = Task.OUTCOME_NA - task_information['recipe'] = recipe - task_information['task_name'] = event.taskname - try: - # some tasks don't come with a hash. and that's ok - task_information['sstate_checksum'] = event.taskhash - except AttributeError: - pass - return task_information - - def _get_layer_version_for_path(self, path): - assert path.startswith("/") - assert 'build' in self.internal_state - - def _slkey_interactive(layer_version): - assert isinstance(layer_version, Layer_Version) - return len(layer_version.local_path) - - # Heuristics: we always match recipe to the deepest layer path in the discovered layers - for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive): - # we can match to the recipe file path - if path.startswith(lvo.local_path): - return lvo - - #if we get here, we didn't read layers correctly; dump whatever information we have on the error log - logger.warn("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects) - - #mockup the new layer - unknown_layer, _ = Layer.objects.get_or_create(name="Unidentified layer", layer_index_url="") - unknown_layer_version_obj, _ = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) - - # append it so we don't run into this error again and again - self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) - - return unknown_layer_version_obj - - def _get_recipe_information_from_taskfile(self, taskfile): - localfilepath = taskfile.split(":")[-1] - filepath_flags = ":".join(sorted(taskfile.split(":")[:-1])) - layer_version_obj = self._get_layer_version_for_path(localfilepath) - - - - recipe_info = {} - recipe_info['layer_version'] = layer_version_obj - recipe_info['file_path'] = localfilepath - recipe_info['pathflags'] = filepath_flags - - if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): - recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") - else: - raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) - - return recipe_info - - def _get_path_information(self, task_object): - assert isinstance(task_object, Task) - build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" - build_stats_path = [] - - for t in self.internal_state['targets']: - buildname = self.internal_state['build'].build_name - pe, pv = task_object.recipe.version.split(":",1) - if len(pe) > 0: - package = task_object.recipe.name + "-" + pe + "_" + pv - else: - package = task_object.recipe.name + "-" + pv - - build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, - buildname=buildname, - package=package)) - - return build_stats_path - - - ################################ - ## external available methods to store information - @staticmethod - def _get_data_from_event(event): - evdata = None - if '_localdata' in vars(event): - evdata = event._localdata - elif 'data' in vars(event): - evdata = event.data - else: - raise Exception("Event with neither _localdata or data properties") - return evdata - - def store_layer_info(self, event): - layerinfos = BuildInfoHelper._get_data_from_event(event) - self.internal_state['lvs'] = {} - for layer in layerinfos: - try: - self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] - self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] - except NotExisting as nee: - logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) - - - def store_started_build(self, event, build_log_path): - assert '_pkgs' in vars(event) - build_information = self._get_build_information(build_log_path) - - # Update brbe and project as they can be changed for every build - self.project = build_information['project'] - - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) - - self.internal_state['build'] = build_obj - - # save layer version information for this build - if not 'lvs' in self.internal_state: - logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.") - else: - for layer_obj in self.internal_state['lvs']: - self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) - - del self.internal_state['lvs'] - - # create target information - target_information = {} - target_information['targets'] = event._pkgs - target_information['build'] = build_obj - - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) - - # Save build configuration - data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] - - # convert the paths from absolute to relative to either the build directory or layer checkouts - path_prefixes = [] - - if self.brbe is not None: - _, be_id = self.brbe.split(":") - be = BuildEnvironment.objects.get(pk = be_id) - path_prefixes.append(be.builddir) - - for layer in sorted(self.orm_wrapper.layer_version_objects, key = lambda x:len(x.local_path), reverse=True): - path_prefixes.append(layer.local_path) - - # we strip the prefixes - for k in data: - if not bool(data[k]['func']): - for vh in data[k]['history']: - if not 'documentation.conf' in vh['file']: - abs_file_name = vh['file'] - for pp in path_prefixes: - if abs_file_name.startswith(pp + "/"): - vh['file']=abs_file_name[len(pp + "/"):] - break - - # save the variables - self.orm_wrapper.save_build_variables(build_obj, data) - - return self.brbe - - - def update_target_image_file(self, event): - evdata = BuildInfoHelper._get_data_from_event(event) - - for t in self.internal_state['targets']: - if t.is_image == True: - output_files = list(evdata.viewkeys()) - for output in output_files: - if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): - self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) - - def update_artifact_image_file(self, event): - evdata = BuildInfoHelper._get_data_from_event(event) - for artifact_path in evdata.keys(): - self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) - - def update_build_information(self, event, errors, warnings, taskfailures): - if 'build' in self.internal_state: - self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) - - - def store_license_manifest_path(self, event): - deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir'] - image_name = BuildInfoHelper._get_data_from_event(event)['image_name'] - path = deploy_dir + "/licenses/" + image_name + "/license.manifest" - for target in self.internal_state['targets']: - if target.target in image_name: - self.orm_wrapper.update_target_set_license_manifest(target, path) - - - def store_started_task(self, event): - assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) - assert 'taskfile' in vars(event) - localfilepath = event.taskfile.split(":")[-1] - assert localfilepath.startswith("/") - - identifier = event.taskfile + ":" + event.taskname - - recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) - recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) - - task_information = self._get_task_information(event, recipe) - task_information['outcome'] = Task.OUTCOME_NA - - if isinstance(event, bb.runqueue.runQueueTaskSkipped): - assert 'reason' in vars(event) - task_information['task_executed'] = False - if event.reason == "covered": - task_information['outcome'] = Task.OUTCOME_COVERED - if event.reason == "existing": - task_information['outcome'] = Task.OUTCOME_PREBUILT - else: - task_information['task_executed'] = True - if 'noexec' in vars(event) and event.noexec == True: - task_information['task_executed'] = False - task_information['outcome'] = Task.OUTCOME_EMPTY - task_information['script_type'] = Task.CODING_NA - - # do not assign order numbers to scene tasks - if not isinstance(event, bb.runqueue.sceneQueueTaskStarted): - self.task_order += 1 - task_information['order'] = self.task_order - - self.orm_wrapper.get_update_task_object(task_information) - - self.internal_state['taskdata'][identifier] = { - 'outcome': task_information['outcome'], - } - - - def store_tasks_stats(self, event): - task_data = BuildInfoHelper._get_data_from_event(event) - - for (task_file, task_name, task_stats, recipe_name) in task_data: - build = self.internal_state['build'] - self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats) - - def update_and_store_task(self, event): - assert 'taskfile' in vars(event) - localfilepath = event.taskfile.split(":")[-1] - assert localfilepath.startswith("/") - - identifier = event.taskfile + ":" + event.taskname - if not identifier in self.internal_state['taskdata']: - if isinstance(event, bb.build.TaskBase): - # we do a bit of guessing - candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)] - if len(candidates) == 1: - identifier = candidates[0] - - assert identifier in self.internal_state['taskdata'] - identifierlist = identifier.split(":") - realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1]) - recipe_information = self._get_recipe_information_from_taskfile(realtaskfile) - recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) - task_information = self._get_task_information(event,recipe) - - task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome'] - - if 'logfile' in vars(event): - task_information['logfile'] = event.logfile - - if '_message' in vars(event): - task_information['message'] = event._message - - if 'taskflags' in vars(event): - # with TaskStarted, we get even more information - if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1': - task_information['script_type'] = Task.CODING_PYTHON - else: - task_information['script_type'] = Task.CODING_SHELL - - if task_information['outcome'] == Task.OUTCOME_NA: - if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)): - task_information['outcome'] = Task.OUTCOME_SUCCESS - del self.internal_state['taskdata'][identifier] - - if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runqueue.sceneQueueTaskFailed)): - task_information['outcome'] = Task.OUTCOME_FAILED - del self.internal_state['taskdata'][identifier] - - if not connection.features.autocommits_when_autocommit_is_off: - # we force a sync point here, to get the progress bar to show - if self.autocommit_step % 3 == 0: - transaction.set_autocommit(True) - transaction.set_autocommit(False) - self.autocommit_step += 1 - - self.orm_wrapper.get_update_task_object(task_information, True) # must exist - - - def store_missed_state_tasks(self, event): - for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: - - # identifier = fn + taskname + "_setscene" - recipe_information = self._get_recipe_information_from_taskfile(fn) - recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) - mevent = MockEvent() - mevent.taskname = taskname - mevent.taskhash = taskhash - task_information = self._get_task_information(mevent,recipe) - - task_information['start_time'] = timezone.now() - task_information['outcome'] = Task.OUTCOME_NA - task_information['sstate_checksum'] = taskhash - task_information['sstate_result'] = Task.SSTATE_MISS - task_information['path_to_sstate_obj'] = sstatefile - - self.orm_wrapper.get_update_task_object(task_information) - - for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: - - # identifier = fn + taskname + "_setscene" - recipe_information = self._get_recipe_information_from_taskfile(fn) - recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) - mevent = MockEvent() - mevent.taskname = taskname - mevent.taskhash = taskhash - task_information = self._get_task_information(mevent,recipe) - - task_information['path_to_sstate_obj'] = sstatefile - - self.orm_wrapper.get_update_task_object(task_information) - - - def store_target_package_data(self, event): - # for all image targets - for target in self.internal_state['targets']: - if target.is_image: - pkgdata = BuildInfoHelper._get_data_from_event(event)['pkgdata'] - imgdata = BuildInfoHelper._get_data_from_event(event)['imgdata'].get(target.target, {}) - filedata = BuildInfoHelper._get_data_from_event(event)['filedata'].get(target.target, {}) - - try: - self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'], built_package=True) - self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata.copy(), pkgdata, self.internal_state['recipes'], built_package=False) - except KeyError as e: - logger.warn("KeyError in save_target_package_information" - "%s ", e) - - try: - self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata) - except KeyError as e: - logger.warn("KeyError in save_target_file_information" - "%s ", e) - - - - - def store_dependency_information(self, event): - assert '_depgraph' in vars(event) - assert 'layer-priorities' in event._depgraph - assert 'pn' in event._depgraph - assert 'tdepends' in event._depgraph - - errormsg = "" - - # save layer version priorities - if 'layer-priorities' in event._depgraph.keys(): - for lv in event._depgraph['layer-priorities']: - (_, path, _, priority) = lv - layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ - assert layer_version_obj is not None - layer_version_obj.priority = priority - layer_version_obj.save() - - # save recipe information - self.internal_state['recipes'] = {} - for pn in event._depgraph['pn']: - - file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1] - pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1])) - layer_version_obj = self._get_layer_version_for_path(file_name) - - assert layer_version_obj is not None - - recipe_info = {} - recipe_info['name'] = pn - recipe_info['layer_version'] = layer_version_obj - - if 'version' in event._depgraph['pn'][pn]: - recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") - - if 'summary' in event._depgraph['pn'][pn]: - recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] - - if 'license' in event._depgraph['pn'][pn]: - recipe_info['license'] = event._depgraph['pn'][pn]['license'] - - if 'description' in event._depgraph['pn'][pn]: - recipe_info['description'] = event._depgraph['pn'][pn]['description'] - - if 'section' in event._depgraph['pn'][pn]: - recipe_info['section'] = event._depgraph['pn'][pn]['section'] - - if 'homepage' in event._depgraph['pn'][pn]: - recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] - - if 'bugtracker' in event._depgraph['pn'][pn]: - recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] - - recipe_info['file_path'] = file_name - recipe_info['pathflags'] = pathflags - - if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path): - recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/") - else: - raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path)) - - recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) - recipe.is_image = False - if 'inherits' in event._depgraph['pn'][pn].keys(): - for cls in event._depgraph['pn'][pn]['inherits']: - if cls.endswith('/image.bbclass'): - recipe.is_image = True - recipe_info['is_image'] = True - # Save the is_image state to the relevant recipe objects - self.orm_wrapper.get_update_recipe_object(recipe_info) - break - if recipe.is_image: - for t in self.internal_state['targets']: - if pn == t.target: - t.is_image = True - t.save() - self.internal_state['recipes'][pn] = recipe - - # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED - - assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() - - # save recipe dependency - # buildtime - recipedeps_objects = [] - for recipe in event._depgraph['depends']: - target = self.internal_state['recipes'][recipe] - for dep in event._depgraph['depends'][recipe]: - if dep in assume_provided: - continue - via = None - if 'providermap' in event._depgraph and dep in event._depgraph['providermap']: - deprecipe = event._depgraph['providermap'][dep][0] - dependency = self.internal_state['recipes'][deprecipe] - via = Provides.objects.get_or_create(name=dep, - recipe=dependency)[0] - elif dep in self.internal_state['recipes']: - dependency = self.internal_state['recipes'][dep] - else: - errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep) - continue - recipe_dep = Recipe_Dependency(recipe=target, - depends_on=dependency, - via=via, - dep_type=Recipe_Dependency.TYPE_DEPENDS) - recipedeps_objects.append(recipe_dep) - - Recipe_Dependency.objects.bulk_create(recipedeps_objects) - - # save all task information - def _save_a_task(taskdesc): - spec = re.split(r'\.', taskdesc) - pn = ".".join(spec[0:-1]) - taskname = spec[-1] - e = event - e.taskname = pn - recipe = self.internal_state['recipes'][pn] - task_info = self._get_task_information(e, recipe) - task_info['task_name'] = taskname - task_obj = self.orm_wrapper.get_update_task_object(task_info) - return task_obj - - # create tasks - tasks = {} - for taskdesc in event._depgraph['tdepends']: - tasks[taskdesc] = _save_a_task(taskdesc) - - # create dependencies between tasks - taskdeps_objects = [] - for taskdesc in event._depgraph['tdepends']: - target = tasks[taskdesc] - for taskdep in event._depgraph['tdepends'][taskdesc]: - if taskdep not in tasks: - # Fetch tasks info is not collected previously - dep = _save_a_task(taskdep) - else: - dep = tasks[taskdep] - taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) - Task_Dependency.objects.bulk_create(taskdeps_objects) - - if len(errormsg) > 0: - logger.warn("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) - - - def store_build_package_information(self, event): - package_info = BuildInfoHelper._get_data_from_event(event) - self.orm_wrapper.save_build_package_information( - self.internal_state['build'], - package_info, - self.internal_state['recipes'], - built_package=True) - - self.orm_wrapper.save_build_package_information( - self.internal_state['build'], - package_info, - self.internal_state['recipes'], - built_package=False) - - def _store_build_done(self, errorcode): - logger.info("Build exited with errorcode %d", errorcode) - br_id, be_id = self.brbe.split(":") - be = BuildEnvironment.objects.get(pk = be_id) - be.lock = BuildEnvironment.LOCK_LOCK - be.save() - br = BuildRequest.objects.get(pk = br_id) - - # if we're 'done' because we got cancelled update the build outcome - if br.state == BuildRequest.REQ_CANCELLING: - logger.info("Build cancelled") - br.build.outcome = Build.CANCELLED - br.build.save() - self.internal_state['build'] = br.build - errorcode = 0 - - if errorcode == 0: - # request archival of the project artifacts - br.state = BuildRequest.REQ_COMPLETED - else: - br.state = BuildRequest.REQ_FAILED - br.save() - - - def store_log_error(self, text): - mockevent = MockEvent() - mockevent.levelno = formatter.ERROR - mockevent.msg = text - mockevent.pathname = '-- None' - mockevent.lineno = LogMessage.ERROR - self.store_log_event(mockevent) - - def store_log_exception(self, text, backtrace = ""): - mockevent = MockEvent() - mockevent.levelno = -1 - mockevent.msg = text - mockevent.pathname = backtrace - mockevent.lineno = -1 - self.store_log_event(mockevent) - - - def store_log_event(self, event): - if event.levelno < formatter.WARNING: - return - - if 'args' in vars(event): - event.msg = event.msg % event.args - - if not 'build' in self.internal_state: - if self.brbe is None: - if not 'backlog' in self.internal_state: - self.internal_state['backlog'] = [] - self.internal_state['backlog'].append(event) - return - else: # we're under Toaster control, the build is already created - br, _ = self.brbe.split(":") - buildrequest = BuildRequest.objects.get(pk = br) - self.internal_state['build'] = buildrequest.build - - if 'build' in self.internal_state and 'backlog' in self.internal_state: - # if we have a backlog of events, do our best to save them here - if len(self.internal_state['backlog']): - tempevent = self.internal_state['backlog'].pop() - logger.debug(1, "buildinfohelper: Saving stored event %s " % tempevent) - self.store_log_event(tempevent) - else: - logger.info("buildinfohelper: All events saved") - del self.internal_state['backlog'] - - log_information = {} - log_information['build'] = self.internal_state['build'] - if event.levelno == formatter.CRITICAL: - log_information['level'] = LogMessage.CRITICAL - elif event.levelno == formatter.ERROR: - log_information['level'] = LogMessage.ERROR - elif event.levelno == formatter.WARNING: - log_information['level'] = LogMessage.WARNING - elif event.levelno == -2: # toaster self-logging - log_information['level'] = -2 - else: - log_information['level'] = LogMessage.INFO - - log_information['message'] = event.msg - log_information['pathname'] = event.pathname - log_information['lineno'] = event.lineno - logger.info("Logging error 2: %s", log_information) - - self.orm_wrapper.create_logmessage(log_information) - - def close(self, errorcode): - if self.brbe is not None: - self._store_build_done(errorcode) - - if 'backlog' in self.internal_state: - if 'build' in self.internal_state: - # we save missed events in the database for the current build - tempevent = self.internal_state['backlog'].pop() - self.store_log_event(tempevent) - else: - # we have no build, and we still have events; something amazingly wrong happend - for event in self.internal_state['backlog']: - logger.error("UNSAVED log: %s", event.msg) - - if not connection.features.autocommits_when_autocommit_is_off: - transaction.set_autocommit(True) - - # unset the brbe; this is to prevent subsequent command-line builds - # being incorrectly attached to the previous Toaster-triggered build; - # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 - self.brbe = None |