diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/toaster/toastergui/views.py')
| -rwxr-xr-x | yocto-poky/bitbake/lib/toaster/toastergui/views.py | 1200 |
1 files changed, 515 insertions, 685 deletions
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/yocto-poky/bitbake/lib/toaster/toastergui/views.py index 0e255f1b8..bd5bf6334 100755 --- a/yocto-poky/bitbake/lib/toaster/toastergui/views.py +++ b/yocto-poky/bitbake/lib/toaster/toastergui/views.py @@ -27,15 +27,15 @@ import operator,re from django.db.models import F, Q, Sum, Count, Max from django.db import IntegrityError, Error -from django.shortcuts import render, redirect +from django.shortcuts import render, redirect, get_object_or_404 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency -from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact +from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage from orm.models import BitbakeVersion, CustomImageRecipe from bldcontrol import bbcontroller from django.views.decorators.cache import cache_control from django.core.urlresolvers import reverse, resolve -from django.core.exceptions import MultipleObjectsReturned +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone @@ -43,7 +43,9 @@ from django.utils.html import escape from datetime import timedelta, datetime from django.utils import formats from toastergui.templatetags.projecttags import json as jsonfilter +from decimal import Decimal import json +import os from os.path import dirname from functools import wraps import itertools @@ -73,13 +75,9 @@ class MimeTypeFinder(object): def landing(request): # in build mode, we redirect to the command-line builds page # if there are any builds for the default (cli builds) project - default_project = Project.objects.get_default_project() + default_project = Project.objects.get_or_create_default_project() default_project_builds = Build.objects.filter(project = default_project) - if (not toastermain.settings.BUILD_MODE) and default_project_builds.count() > 0: - args = (default_project.id,) - return redirect(reverse('projectbuilds', args = args), permanent = False) - # we only redirect to projects page if there is a user-generated project num_builds = Build.objects.all().count() user_projects = Project.objects.filter(is_default = False) @@ -95,52 +93,6 @@ def landing(request): return render(request, 'landing.html', context) -# returns a list for most recent builds; -def _get_latest_builds(prj=None): - queryset = Build.objects.all() - - if prj is not None: - queryset = queryset.filter(project = prj) - - if not toastermain.settings.BUILD_MODE: - queryset = queryset.exclude(project__is_default=False) - - return list(itertools.chain( - queryset.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), - queryset.filter(outcome__lt=Build.IN_PROGRESS).order_by("-started_on")[:3] )) - - -# a JSON-able dict of recent builds; for use in the Project page, xhr_ updates, and other places, as needed -def _project_recent_build_list(prj): - data = [] - # take the most recent 3 completed builds, plus any builds in progress - for x in _get_latest_builds(prj): - d = { - "id": x.pk, - "targets" : map(lambda y: {"target": y.target, "task": y.task }, x.target_set.all()), # TODO: create the task entry in the Target table - "status": x.get_current_status(), - "errors": map(lambda y: {"type": y.lineno, "msg": y.message, "tb": y.pathname}, (x.logmessage_set.filter(level__gte=LogMessage.WARNING)|x.logmessage_set.filter(level=LogMessage.EXCEPTION))), - "updated": x.completed_on.strftime('%s')+"000", - "command_time": (x.completed_on - x.started_on).total_seconds(), - "br_page_url": reverse('builddashboard', args=(x.pk,) ), - "build" : map( lambda y: {"id": y.pk, - "status": y.get_outcome_display(), - "completed_on" : y.completed_on.strftime('%s')+"000", - "build_time" : (y.completed_on - y.started_on).total_seconds(), - "build_page_url" : reverse('builddashboard', args=(y.pk,)), - 'build_time_page_url': reverse('buildtime', args=(y.pk,)), - "errors": y.errors.count(), - "warnings": y.warnings.count(), - "completeper": y.completeper() if y.outcome == Build.IN_PROGRESS else "0", - "eta": y.eta().strftime('%s')+"000" if y.outcome == Build.IN_PROGRESS else "0", - }, [x]), - } - data.append(d) - - return data - - - def objtojson(obj): from django.db.models.query import QuerySet from django.db.models import Model @@ -151,6 +103,8 @@ def objtojson(obj): return obj.total_seconds() elif isinstance(obj, QuerySet) or isinstance(obj, set): return list(obj) + elif isinstance(obj, Decimal): + return str(obj) elif type(obj).__name__ == "RelatedManager": return [x.pk for x in obj.all()] elif hasattr( obj, '__dict__') and isinstance(obj, Model): @@ -509,7 +463,7 @@ def builddashboard( request, build_id ): for t in tgts: elem = { } elem[ 'target' ] = t - if ( t.is_image ): + if t.is_image: hasImages = True npkg = 0 pkgsz = 0 @@ -527,9 +481,13 @@ def builddashboard( request, build_id ): if ( ndx < 0 ): ndx = 0; f = i.file_name[ ndx + 1: ] - imageFiles.append({ 'id': i.id, 'path': f, 'size' : i.file_size }) - if ( t.is_image and - (( len( imageFiles ) <= 0 ) or ( len( t.license_manifest_path ) <= 0 ))): + imageFiles.append({ + 'id': i.id, + 'path': f, + 'size': i.file_size, + 'suffix': i.suffix + }) + if t.is_image and (len(imageFiles) <= 0 or len(t.license_manifest_path) <= 0): targetHasNoImages = True elem[ 'imageFiles' ] = imageFiles elem[ 'targetHasNoImages' ] = targetHasNoImages @@ -549,6 +507,7 @@ def builddashboard( request, build_id ): context = { 'build' : build, + 'project' : build.project, 'hasImages' : hasImages, 'ntargets' : ntargets, 'targets' : targets, @@ -705,7 +664,9 @@ def recipe_packages(request, build_id, recipe_id): def target_common( request, build_id, target_id, variant ): template = "target.html" - (pagesize, orderby) = _get_parameters_values(request, 25, 'name:+') + default_orderby = 'name:+' + + (pagesize, orderby) = _get_parameters_values(request, 25, default_orderby) mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby': orderby } retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: @@ -725,8 +686,6 @@ def target_common( request, build_id, target_id, variant ): queryset = queryset.select_related("recipe", "recipe__layer_version", "recipe__layer_version__layer") packages = _build_page_range( Paginator(queryset, pagesize), request.GET.get( 'page', 1 )) - - build = Build.objects.get( pk = build_id ) # bring in package dependencies @@ -839,11 +798,12 @@ eans multiple licenses exist that cover different parts of the source', context = { 'objectname': variant, 'build' : build, + 'project' : build.project, 'target' : Target.objects.filter( pk = target_id )[ 0 ], 'objects' : packages, 'packages_sum' : packages_sum[ 'installed_size__sum' ], 'object_search_display': "packages included", - 'default_orderby' : orderby, + 'default_orderby' : default_orderby, 'tablecols' : [ tc_package, tc_packageVersion, @@ -979,7 +939,10 @@ def dirinfo(request, build_id, target_id, file_path=None): if head != sep: dir_list.insert(0, head) - context = { 'build': Build.objects.get(pk=build_id), + build = Build.objects.get(pk=build_id) + + context = { 'build': build, + 'project': build.project, 'target': Target.objects.get(pk=target_id), 'packages_sum': packages_sum['installed_size__sum'], 'objects': objects, @@ -1038,29 +1001,29 @@ def tasks_common(request, build_id, variant, task_anchor): anchor=task_anchor # default ordering depends on variant - if 'buildtime' == variant: - title_variant='Time' - object_search_display="time data" - filter_search_display="tasks" - (pagesize, orderby) = _get_parameters_values(request, 25, 'elapsed_time:-') - elif 'diskio' == variant: - title_variant='Disk I/O' - object_search_display="disk I/O data" - filter_search_display="tasks" - (pagesize, orderby) = _get_parameters_values(request, 25, 'disk_io:-') - elif 'cpuusage' == variant: - title_variant='CPU usage' - object_search_display="CPU usage data" - filter_search_display="tasks" - (pagesize, orderby) = _get_parameters_values(request, 25, 'cpu_usage:-') - else : - title_variant='Tasks' - object_search_display="tasks" - filter_search_display="tasks" - (pagesize, orderby) = _get_parameters_values(request, 25, 'order:+') + default_orderby = None + filter_search_display = 'tasks' + + if 'buildtime' == variant: + default_orderby = 'elapsed_time:-' + title_variant = 'Time' + object_search_display = 'time data' + elif 'diskio' == variant: + default_orderby = 'disk_io:-' + title_variant = 'Disk I/O' + object_search_display = 'disk I/O data' + elif 'cputime' == variant: + default_orderby = 'cpu_time_system:-' + title_variant='CPU time' + object_search_display = 'CPU time data' + else: + default_orderby = 'order:+' + title_variant = 'Tasks' + object_search_display = 'tasks' + (pagesize, orderby) = _get_parameters_values(request, 25, default_orderby) - mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby': orderby } + mandatory_parameters = {'count': pagesize, 'page' : 1, 'orderby': orderby} template = 'tasks.html' retval = _verify_parameters( request.GET, mandatory_parameters ) @@ -1204,23 +1167,38 @@ def tasks_common(request, build_id, variant, task_anchor): del tc_time['clclass'] tc_cache['hidden']='1' - tc_cpu={ - 'name':'CPU usage', - 'qhelp':'The percentage of task CPU utilization', - 'orderfield': _get_toggle_order(request, "cpu_usage", True), - 'ordericon':_get_toggle_order_icon(request, "cpu_usage"), - 'orderkey' : 'cpu_usage', - 'clclass': 'cpu_used', 'hidden' : 1, + tc_cpu_time_system={ + 'name':'System CPU time (secs)', + 'qhelp':'Total amount of time spent executing in kernel mode, in ' + + 'seconds. Note that this time can be greater than the task ' + + 'time due to parallel execution.', + 'orderfield': _get_toggle_order(request, "cpu_time_system", True), + 'ordericon':_get_toggle_order_icon(request, "cpu_time_system"), + 'orderkey' : 'cpu_time_system', + 'clclass': 'cpu_time_system', 'hidden' : 1, } - if 'cpuusage' == variant: - tc_cpu['hidden']='0' - del tc_cpu['clclass'] + tc_cpu_time_user={ + 'name':'User CPU time (secs)', + 'qhelp':'Total amount of time spent executing in user mode, in seconds. ' + + 'Note that this time can be greater than the task time due to ' + + 'parallel execution.', + 'orderfield': _get_toggle_order(request, "cpu_time_user", True), + 'ordericon':_get_toggle_order_icon(request, "cpu_time_user"), + 'orderkey' : 'cpu_time_user', + 'clclass': 'cpu_time_user', 'hidden' : 1, + } + + if 'cputime' == variant: + tc_cpu_time_system['hidden']='0' + tc_cpu_time_user['hidden']='0' + del tc_cpu_time_system['clclass'] + del tc_cpu_time_user['clclass'] tc_cache['hidden']='1' tc_diskio={ - 'name':'Disk I/O (ms)', - 'qhelp':'Number of miliseconds the task spent doing disk input and output', + 'name':'Disk I/O (bytes)', + 'qhelp':'Number of bytes written to and read from the disk during the task', 'orderfield': _get_toggle_order(request, "disk_io", True), 'ordericon':_get_toggle_order_icon(request, "disk_io"), 'orderkey' : 'disk_io', @@ -1238,8 +1216,9 @@ def tasks_common(request, build_id, variant, task_anchor): 'filter_search_display': filter_search_display, 'mainheading': title_variant, 'build': build, + 'project': build.project, 'objects': task_objects, - 'default_orderby' : orderby, + 'default_orderby' : default_orderby, 'search_term': search_term, 'total_count': queryset_with_search.count(), 'tablecols':[ @@ -1251,7 +1230,8 @@ def tasks_common(request, build_id, variant, task_anchor): tc_outcome, tc_cache, tc_time, - tc_cpu, + tc_cpu_time_system, + tc_cpu_time_user, tc_diskio, ]} @@ -1272,9 +1252,8 @@ def buildtime(request, build_id): def diskio(request, build_id): return tasks_common(request, build_id, 'diskio', '') -def cpuusage(request, build_id): - return tasks_common(request, build_id, 'cpuusage', '') - +def cputime(request, build_id): + return tasks_common(request, build_id, 'cputime', '') def recipes(request, build_id): template = 'recipes.html' @@ -1284,7 +1263,10 @@ def recipes(request, build_id): if retval: return _redirect_parameters( 'recipes', request.GET, mandatory_parameters, build_id = build_id) (filter_string, search_term, ordering_string) = _search_tuple(request, Recipe) - queryset = Recipe.objects.filter(layer_version__id__in=Layer_Version.objects.filter(build=build_id)).select_related("layer_version", "layer_version__layer") + + build = Build.objects.get(pk=build_id) + + queryset = build.get_recipes() queryset = _get_queryset(Recipe, queryset, filter_string, search_term, ordering_string, 'name') recipes = _build_page_range(Paginator(queryset, pagesize),request.GET.get('page', 1)) @@ -1303,11 +1285,10 @@ def recipes(request, build_id): revlist.append(recipe_dep) revs[recipe.id] = revlist - build = Build.objects.get(pk=build_id) - context = { 'objectname': 'recipes', 'build': build, + 'project': build.project, 'objects': recipes, 'default_orderby' : 'name:+', 'recipe_deps' : deps, @@ -1392,10 +1373,12 @@ def configuration(request, build_id): 'MACHINE', 'DISTRO', 'DISTRO_VERSION', 'TUNE_FEATURES', 'TARGET_FPU') context = dict(Variable.objects.filter(build=build_id, variable_name__in=var_names)\ .values_list('variable_name', 'variable_value')) + build = Build.objects.get(pk=build_id) context.update({'objectname': 'configuration', 'object_search_display':'variables', 'filter_search_display':'variables', - 'build': Build.objects.get(pk=build_id), + 'build': build, + 'project': build.project, 'targets': Target.objects.filter(build=build_id)}) return render(request, template, context) @@ -1432,12 +1415,15 @@ def configvars(request, build_id): file_filter += '/bitbake.conf' build_dir=re.sub("/tmp/log/.*","",Build.objects.get(pk=build_id).cooker_log_path) + build = Build.objects.get(pk=build_id) + context = { 'objectname': 'configvars', 'object_search_display':'BitBake variables', 'filter_search_display':'variables', 'file_filter': file_filter, - 'build': Build.objects.get(pk=build_id), + 'build': build, + 'project': build.project, 'objects' : variables, 'total_count':queryset_with_search.count(), 'default_orderby' : 'variable_name:+', @@ -1506,6 +1492,7 @@ def bpackage(request, build_id): context = { 'objectname': 'packages built', 'build': build, + 'project': build.project, 'objects' : packages, 'default_orderby' : 'name:+', 'tablecols':[ @@ -1580,7 +1567,12 @@ def bpackage(request, build_id): def bfile(request, build_id, package_id): template = 'bfile.html' files = Package_File.objects.filter(package = package_id) - context = {'build': Build.objects.get(pk=build_id), 'objects' : files} + build = Build.objects.get(pk=build_id) + context = { + 'build': build, + 'project': build.project, + 'objects' : files + } return render(request, template, context) @@ -1893,11 +1885,6 @@ def managedcontextprocessor(request): "projects": projects, "non_cli_projects": projects.exclude(is_default=True), "DEBUG" : toastermain.settings.DEBUG, - - # True if Toaster is in build mode, False otherwise - "BUILD_MODE": toastermain.settings.BUILD_MODE, - - "CUSTOM_IMAGE" : toastermain.settings.CUSTOM_IMAGE, "TOASTER_BRANCH": toastermain.settings.TOASTER_BRANCH, "TOASTER_REVISION" : toastermain.settings.TOASTER_REVISION, } @@ -1908,6 +1895,7 @@ def managedcontextprocessor(request): import toastermain.settings from orm.models import Project, ProjectLayer, ProjectTarget, ProjectVariable +from bldcontrol.models import BuildEnvironment # we have a set of functions if we're in managed mode, or # a default "page not available" simple functions for interactive mode @@ -1926,263 +1914,6 @@ if True: ''' The exception raised on invalid POST requests ''' pass - # shows the "all builds" page for managed mode; it displays build requests (at least started!) instead of actual builds - # WARNING _build_list_helper() may raise a RedirectException, which - # will set the GET parameters and redirect back to the - # all-builds or projectbuilds page as appropriate; - # TODO don't use exceptions to control program flow - @_template_renderer("builds.html") - def builds(request): - # define here what parameters the view needs in the GET portion in order to - # be able to display something. 'count' and 'page' are mandatory for all views - # that use paginators. - - queryset = Build.objects.all() - - # if in analysis mode, exclude builds for all projects except - # command line builds - if not toastermain.settings.BUILD_MODE: - queryset = queryset.exclude(project__is_default=False) - - redirect_page = resolve(request.path_info).url_name - - context, pagesize, orderby = _build_list_helper(request, - queryset, - redirect_page) - # all builds page as a Project column - context['tablecols'].append({ - 'name': 'Project', - 'clclass': 'project_column' - }) - - _set_parameters_values(pagesize, orderby, request) - return context - - - # helper function, to be used on "all builds" and "project builds" pages - def _build_list_helper(request, queryset_all, redirect_page, pid=None): - default_orderby = 'completed_on:-' - (pagesize, orderby) = _get_parameters_values(request, 10, default_orderby) - mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } - retval = _verify_parameters( request.GET, mandatory_parameters ) - if retval: - params = {} - if pid: - params = {'pid': pid} - raise RedirectException(redirect_page, - request.GET, - mandatory_parameters, - **params) - - # boilerplate code that takes a request for an object type and returns a queryset - # for that object type. copypasta for all needed table searches - (filter_string, search_term, ordering_string) = _search_tuple(request, Build) - - # post-process any date range filters - filter_string, daterange_selected = _modify_date_range_filter(filter_string) - - # don't show "in progress" builds in "all builds" or "project builds" - queryset_all = queryset_all.exclude(outcome = Build.IN_PROGRESS) - - # append project info - queryset_all = queryset_all.select_related("project") - - # annotate with number of ERROR and EXCEPTION log messages - queryset_all = queryset_all.annotate( - errors_no = Count( - 'logmessage', - only=Q(logmessage__level=LogMessage.ERROR) | - Q(logmessage__level=LogMessage.EXCEPTION) - ) - ) - - # annotate with number of warnings - q_warnings = Q(logmessage__level=LogMessage.WARNING) - queryset_all = queryset_all.annotate( - warnings_no = Count('logmessage', only=q_warnings) - ) - - # add timespent field - timespent = 'completed_on - started_on' - queryset_all = queryset_all.extra(select={'timespent': timespent}) - - queryset_with_search = _get_queryset(Build, queryset_all, - None, search_term, - ordering_string, '-completed_on') - - queryset = _get_queryset(Build, queryset_all, - filter_string, search_term, - ordering_string, '-completed_on') - - # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display - build_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) - - # build view-specific information; this is rendered specifically in the builds page, at the top of the page (i.e. Recent builds) - build_mru = _get_latest_builds()[:3] - - # calculate the exact begining of local today and yesterday, append context - context_date,today_begin,yesterday_begin = _add_daterange_context(queryset_all, request, {'started_on','completed_on'}) - - # set up list of fstypes for each build - fstypes_map = {}; - for build in build_info: - targets = Target.objects.filter( build_id = build.id ) - comma = ""; - extensions = ""; - for t in targets: - if ( not t.is_image ): - continue - tif = Target_Image_File.objects.filter( target_id = t.id ) - for i in tif: - s=re.sub('.*tar.bz2', 'tar.bz2', i.file_name) - if s == i.file_name: - s=re.sub('.*\.', '', i.file_name) - if None == re.search(s,extensions): - extensions += comma + s - comma = ", " - fstypes_map[build.id]=extensions - - # send the data to the template - context = { - # specific info for - 'mru' : build_mru, - # TODO: common objects for all table views, adapt as needed - 'objects' : build_info, - 'objectname' : "builds", - 'default_orderby' : default_orderby, - 'fstypes' : fstypes_map, - 'search_term' : search_term, - 'total_count' : queryset_with_search.count(), - 'daterange_selected' : daterange_selected, - # Specifies the display of columns for the table, appearance in "Edit columns" box, toggling default show/hide, and specifying filters for columns - 'tablecols' : [ - {'name': 'Outcome', # column with a single filter - 'qhelp' : "The outcome tells you if a build successfully completed or failed", # the help button content - 'dclass' : "span2", # indication about column width; comes from the design - 'orderfield': _get_toggle_order(request, "outcome"), # adds ordering by the field value; default ascending unless clicked from ascending into descending - 'ordericon':_get_toggle_order_icon(request, "outcome"), - # filter field will set a filter on that column with the specs in the filter description - # the class field in the filter has no relation with clclass; the control different aspects of the UI - # still, it is recommended for the values to be identical for easy tracking in the generated HTML - 'filter' : {'class' : 'outcome', - 'label': 'Show:', - 'options' : [ - ('Successful builds', 'outcome:' + str(Build.SUCCEEDED), queryset_with_search.filter(outcome=str(Build.SUCCEEDED)).count()), # this is the field search expression - ('Failed builds', 'outcome:'+ str(Build.FAILED), queryset_with_search.filter(outcome=str(Build.FAILED)).count()), - ] - } - }, - {'name': 'Recipe', # default column, disabled box, with just the name in the list - 'qhelp': "What you built (i.e. one or more recipes or image recipes)", - 'orderfield': _get_toggle_order(request, "target__target"), - 'ordericon':_get_toggle_order_icon(request, "target__target"), - }, - {'name': 'Machine', - 'qhelp': "The machine is the hardware for which you are building a recipe or image recipe", - 'orderfield': _get_toggle_order(request, "machine"), - 'ordericon':_get_toggle_order_icon(request, "machine"), - 'dclass': 'span3' - }, # a slightly wider column - {'name': 'Started on', 'clclass': 'started_on', 'hidden' : 1, # this is an unchecked box, which hides the column - 'qhelp': "The date and time you started the build", - 'orderfield': _get_toggle_order(request, "started_on", True), - 'ordericon':_get_toggle_order_icon(request, "started_on"), - 'orderkey' : "started_on", - 'filter' : {'class' : 'started_on', - 'label': 'Show:', - 'options' : [ - ("Today's builds" , 'started_on__gte:'+today_begin.strftime("%Y-%m-%d"), queryset_all.filter(started_on__gte=today_begin).count()), - ("Yesterday's builds", - 'started_on__gte!started_on__lt:' - +yesterday_begin.strftime("%Y-%m-%d")+'!' - +today_begin.strftime("%Y-%m-%d"), - queryset_all.filter( - started_on__gte=yesterday_begin, - started_on__lt=today_begin - ).count()), - ("Build date range", 'daterange', 1, '', 'started_on'), - ] - } - }, - {'name': 'Completed on', - 'qhelp': "The date and time the build finished", - 'orderfield': _get_toggle_order(request, "completed_on", True), - 'ordericon':_get_toggle_order_icon(request, "completed_on"), - 'orderkey' : 'completed_on', - 'filter' : {'class' : 'completed_on', - 'label': 'Show:', - 'options' : [ - ("Today's builds" , 'completed_on__gte:'+today_begin.strftime("%Y-%m-%d"), queryset_all.filter(completed_on__gte=today_begin).count()), - ("Yesterday's builds", - 'completed_on__gte!completed_on__lt:' - +yesterday_begin.strftime("%Y-%m-%d")+'!' - +today_begin.strftime("%Y-%m-%d"), - queryset_all.filter( - completed_on__gte=yesterday_begin, - completed_on__lt=today_begin - ).count()), - ("Build date range", 'daterange', 1, '', 'completed_on'), - ] - } - }, - {'name': 'Failed tasks', 'clclass': 'failed_tasks', # specifing a clclass will enable the checkbox - 'qhelp': "How many tasks failed during the build", - 'filter' : {'class' : 'failed_tasks', - 'label': 'Show:', - 'options' : [ - ('Builds with failed tasks', 'task_build__outcome:4', queryset_with_search.filter(task_build__outcome=4).count()), - ('Builds without failed tasks', 'task_build__outcome:NOT4', queryset_with_search.filter(~Q(task_build__outcome=4)).count()), - ] - } - }, - {'name': 'Errors', 'clclass': 'errors_no', - 'qhelp': "How many errors were encountered during the build (if any)", - # Comment out sorting and filter until YOCTO #8131 is fixed - #'orderfield': _get_toggle_order(request, "errors_no", True), - #'ordericon':_get_toggle_order_icon(request, "errors_no"), - #'orderkey' : 'errors_no', - #'filter' : {'class' : 'errors_no', - # 'label': 'Show:', - # 'options' : [ - # ('Builds with errors', 'errors_no__gte:1', queryset_with_search.filter(errors_no__gte=1).count()), - # ('Builds without errors', 'errors_no:0', queryset_with_search.filter(errors_no=0).count()), - # ] - # } - }, - {'name': 'Warnings', 'clclass': 'warnings_no', - 'qhelp': "How many warnings were encountered during the build (if any)", - # Comment out sorting and filter until YOCTO #8131 is fixed - #'orderfield': _get_toggle_order(request, "warnings_no", True), - #'ordericon':_get_toggle_order_icon(request, "warnings_no"), - #'orderkey' : 'warnings_no', - #'filter' : {'class' : 'warnings_no', - # 'label': 'Show:', - # 'options' : [ - # ('Builds with warnings','warnings_no__gte:1', queryset_with_search.filter(warnings_no__gte=1).count()), - # ('Builds without warnings','warnings_no:0', queryset_with_search.filter(warnings_no=0).count()), - # ] - # } - }, - {'name': 'Time', 'clclass': 'time', 'hidden' : 1, - 'qhelp': "How long it took the build to finish", - # Comment out sorting until YOCTO #8131 is fixed - #'orderfield': _get_toggle_order(request, "timespent", True), - #'ordericon':_get_toggle_order_icon(request, "timespent"), - #'orderkey' : 'timespent', - }, - {'name': 'Image files', 'clclass': 'output', - 'qhelp': "The root file system types produced by the build. You can find them in your <code>/build/tmp/deploy/images/</code> directory", - # TODO: compute image fstypes from Target_Image_File - } - ] - } - - # merge daterange values - context.update(context_date) - return context, pagesize, orderby - - - # new project def newproject(request): template = "newproject.html" @@ -2280,16 +2011,17 @@ if True: prj.bitbake_version = prj.release.bitbake_version prj.save() # we need to change the layers - for i in prj.projectlayer_set.all(): + for project in prj.projectlayer_set.all(): # find and add a similarly-named layer on the new branch try: - lv = prj.compatible_layerversions(layer_name = i.layercommit.layer.name)[0] - ProjectLayer.objects.get_or_create(project = prj, layercommit = lv) + layer_versions = prj.get_all_compatible_layer_versions() + layer_versions = layer_versions.filter(layer__name = project.layercommit.layer.name) + ProjectLayer.objects.get_or_create(project = prj, layercommit = layer_versions.first()) except IndexError: pass finally: # get rid of the old entry - i.delete() + project.delete() if 'machineName' in request.POST: machinevar = prj.projectvariable_set.get(name="MACHINE") @@ -2316,7 +2048,7 @@ if True: "completedbuilds": Build.objects.exclude(outcome = Build.IN_PROGRESS).filter(project_id = pid), "prj" : {"name": prj.name, }, "buildrequests" : prj.build_set.filter(outcome=Build.IN_PROGRESS), - "builds" : _project_recent_build_list(prj), + "builds" : Build.get_recent(prj), "layers" : map(lambda x: { "id": x.layercommit.pk, "orderid": x.pk, @@ -2371,13 +2103,23 @@ if True: name = "_js_unit_test_prj_" - # If there is an existing project by this name delete it. We don't want - # Lots of duplicates cluttering up the projects. + # If there is an existing project by this name delete it. + # We don't want Lots of duplicates cluttering up the projects. Project.objects.filter(name=name).delete() - new_project = Project.objects.create_project(name=name, release=release) + new_project = Project.objects.create_project(name=name, + release=release) + # Add a layer + layer = new_project.get_all_compatible_layer_versions().first() + + ProjectLayer.objects.get_or_create(layercommit=layer, + project=new_project) - context = { 'project' : new_project } + # make sure we have a machine set for this project + ProjectVariable.objects.get_or_create(project=new_project, + name="MACHINE", + value="qemux86") + context = {'project': new_project} return render(request, "js-unit-tests.html", context) from django.views.decorators.csrf import csrf_exempt @@ -2399,12 +2141,17 @@ if True: retval = [] - for i in prj.projectlayer_set.all(): - lv = prj.compatible_layerversions(release = Release.objects.get(pk=new_release_id)).filter(layer__name = i.layercommit.layer.name) + for project in prj.projectlayer_set.all(): + release = Release.objects.get(pk = new_release_id) + + layer_versions = prj.get_all_compatible_layer_versions() + layer_versions = layer_versions.filter(release = release) + layer_versions = layer_versions.filter(layer__name = project.layercommit.layer.name) + # there is no layer_version with the new release id, # and the same name - if lv.count() < 1: - retval.append(i) + if layer_versions.count() < 1: + retval.append(project) return response({"error":"ok", "rows" : map( _lv_to_dict(prj), @@ -2461,6 +2208,10 @@ if True: except ProjectVariable.DoesNotExist: pass try: + return_data['dl_dir'] = ProjectVariable.objects.get(project = prj, name = "DL_DIR").value, + except ProjectVariable.DoesNotExist: + pass + try: return_data['fstypes'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_FSTYPES").value, except ProjectVariable.DoesNotExist: pass @@ -2473,7 +2224,7 @@ if True: except ProjectVariable.DoesNotExist: pass try: - return_data['sdk_machine'] = ProjectVariable.objects.get(project = prj, name = "SDKMACHINE").value, + return_data['sstate_dir'] = ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value, except ProjectVariable.DoesNotExist: pass @@ -2493,8 +2244,11 @@ if True: layers_added = []; # Rudimentary check for any possible html tags - if "<" in request.POST: - return HttpResponse(jsonfilter({"error": "Invalid character <"}), content_type = "application/json") + for val in request.POST.values(): + if "<" in val: + return HttpResponse(jsonfilter( + {"error": "Invalid character <"}), + content_type="application/json") prj = Project.objects.get(pk=request.POST['project_id']) @@ -2669,33 +2423,106 @@ if True: # create custom recipe try: - recipe = CustomImageRecipe.objects.create( - name=request.POST["name"], - base_recipe=params["base"], - project=params["project"]) + + # Only allowed chars in name are a-z, 0-9 and - + if re.search(r'[^a-z|0-9|-]', request.POST["name"]): + return {"error": "invalid-name"} + + custom_images = CustomImageRecipe.objects.all() + + # Are there any recipes with this name already in our project? + existing_image_recipes_in_project = custom_images.filter( + name=request.POST["name"], project=params["project"]) + + if existing_image_recipes_in_project.count() > 0: + return {"error": "image-already-exists"} + + # Are there any recipes with this name which aren't custom + # image recipes? + custom_image_ids = custom_images.values_list('id', flat=True) + existing_non_image_recipes = Recipe.objects.filter( + Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids) + ) + + if existing_non_image_recipes.count() > 0: + return {"error": "recipe-already-exists"} + + # create layer 'Custom layer' and verion if needed + layer = Layer.objects.get_or_create( + name=CustomImageRecipe.LAYER_NAME, + summary="Layer for custom recipes", + vcs_url="file:///toaster_created_layer")[0] + + # Check if we have a layer version already + # We don't use get_or_create here because the dirpath will change + # and is a required field + lver = Layer_Version.objects.filter(Q(project=params['project']) & + Q(layer=layer) & + Q(build=None)).last() + if lver == None: + lver, created = Layer_Version.objects.get_or_create( + project=params['project'], + layer=layer, + dirpath="toaster_created_layer") + + # Add a dependency on our layer to the base recipe's layer + LayerVersionDependency.objects.get_or_create( + layer_version=lver, + depends_on=params["base"].layer_version) + + # Add it to our current project if needed + ProjectLayer.objects.get_or_create(project=params['project'], + layercommit=lver, + optional=False) + + # Create the actual recipe + recipe, created = CustomImageRecipe.objects.get_or_create( + name=request.POST["name"], + base_recipe=params["base"], + project=params["project"], + layer_version=lver, + is_image=True) + + # If we created the object then setup these fields. They may get + # overwritten later on and cause the get_or_create to create a + # duplicate if they've changed. + if created: + recipe.file_path = request.POST["name"] + recipe.license = "MIT" + recipe.version = "0.1" + recipe.save() + except Error as err: return {"error": "Can't create custom recipe: %s" % err} # Find the package list from the last build of this recipe/target - build = Build.objects.filter(target__target=params['base'].name, - project=params['project']).last() - - if build: + target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & + Q(build__project=params['project']) & + (Q(target=params['base'].name) | + Q(target=recipe.name))).last() + if target: # Copy in every package # We don't want these packages to be linked to anything because # that underlying data may change e.g. delete a build - for package in build.package_set.all(): - # Create the duplicate - package.pk = None - package.save() - # Disassociate the package from the build - package.build = None - package.save() - recipe.packages.add(package) - else: - logger.warn("No packages found for this base recipe") + for tpackage in target.target_installed_package_set.all(): + try: + built_package = tpackage.package + # The package had no recipe information so is a ghost + # package skip it + if built_package.recipe == None: + continue; + + config_package = CustomImagePackage.objects.get( + name=built_package.name) + + recipe.includes_set.add(config_package) + except Exception as e: + logger.warning("Error adding package %s %s" % + (tpackage.package.name, e)) + pass return {"error": "ok", + "packages" : recipe.get_all_packages().count(), "url": reverse('customrecipe', args=(params['project'].pk, recipe.id))} @@ -2720,34 +2547,105 @@ if True: or {"error": <error message>} """ - objects = CustomImageRecipe.objects.filter(id=recipe_id) - if not objects: + try: + custom_recipe = CustomImageRecipe.objects.get(id=recipe_id) + except CustomImageRecipe.DoesNotExist: return {"error": "Custom recipe with id=%s " "not found" % recipe_id} + if request.method == 'GET': - values = CustomImageRecipe.objects.filter(id=recipe_id).values() - if values: - return {"error": "ok", "info": values[0]} - else: - return {"error": "Custom recipe with id=%s " - "not found" % recipe_id} - return {"error": "ok", "info": objects.values()[0]} + info = {"id" : custom_recipe.id, + "name" : custom_recipe.name, + "base_recipe_id": custom_recipe.base_recipe.id, + "project_id": custom_recipe.project.id, + } + + return {"error": "ok", "info": info} + elif request.method == 'DELETE': - objects.delete() + custom_recipe.delete() return {"error": "ok"} else: return {"error": "Method %s is not supported" % request.method} + def customrecipe_download(request, pid, recipe_id): + recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) + + file_data = recipe.generate_recipe_file_contents() + + response = HttpResponse(file_data, content_type='text/plain') + response['Content-Disposition'] = \ + 'attachment; filename="%s_%s.bb"' % (recipe.name, + recipe.version) + + return response + + def _traverse_dependents(next_package_id, rev_deps, all_current_packages, tree_level=0): + """ + Recurse through reverse dependency tree for next_package_id. + Limit the reverse dependency search to packages not already scanned, + that is, not already in rev_deps. + Limit the scan to a depth (tree_level) not exceeding the count of + all packages in the custom image, and if that depth is exceeded + return False, pop out of the recursion, and write a warning + to the log, but this is unlikely, suggesting a dependency loop + not caught by bitbake. + On return, the input/output arg rev_deps is appended with queryset + dictionary elements, annotated for use in the customimage template. + The list has unsorted, but unique elements. + """ + max_dependency_tree_depth = all_current_packages.count() + if tree_level >= max_dependency_tree_depth: + logger.warning( + "The number of reverse dependencies " + "for this package exceeds " + max_dependency_tree_depth + + " and the remaining reverse dependencies will not be removed") + return True + + package = CustomImagePackage.objects.get(id=next_package_id) + dependents = \ + package.package_dependencies_target.annotate( + name=F('package__name'), + pk=F('package__pk'), + size=F('package__size'), + ).values("name", "pk", "size").exclude( + ~Q(pk__in=all_current_packages) + ) + + for pkg in dependents: + if pkg in rev_deps: + # already seen, skip dependent search + continue + + rev_deps.append(pkg) + if (_traverse_dependents( + pkg["pk"], rev_deps, all_current_packages, tree_level+1)): + return True + + return False + + def _get_all_dependents(package_id, all_current_packages): + """ + Returns sorted list of recursive reverse dependencies for package_id, + as a list of dictionary items, by recursing through dependency + relationships. + """ + rev_deps = [] + _traverse_dependents(package_id, rev_deps, all_current_packages) + rev_deps = sorted(rev_deps, key=lambda x: x["name"]) + return rev_deps + @xhr_response def xhr_customrecipe_packages(request, recipe_id, package_id): """ ReST API to add/remove packages to/from custom recipe. - Entry point: /xhr_customrecipe/<recipe_id>/packages/ + Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id> Methods: PUT - Add package to the recipe DELETE - Delete package from the recipe + GET - Get package information Returns: {"error": "ok"} @@ -2760,26 +2658,149 @@ if True: return {"error": "Custom recipe with id=%s " "not found" % recipe_id} - if request.method == 'GET' and not package_id: - return {"error": "ok", - "packages": list(recipe.packages.values_list('id'))} + if package_id: + try: + package = CustomImagePackage.objects.get(id=package_id) + except Package.DoesNotExist: + return {"error": "Package with id=%s " + "not found" % package_id} - try: - package = Package.objects.get(id=package_id) - except Package.DoesNotExist: - return {"error": "Package with id=%s " - "not found" % package_id} + if request.method == 'GET': + # If no package_id then list the current packages + if not package_id: + total_size = 0 + packages = recipe.get_all_packages().values("id", + "name", + "version", + "size") + for package in packages: + package['size_formatted'] = \ + filtered_filesizeformat(package['size']) + total_size += package['size'] + + return {"error": "ok", + "packages" : list(packages), + "total" : len(packages), + "total_size" : total_size, + "total_size_formatted" : + filtered_filesizeformat(total_size) + } + else: + all_current_packages = recipe.get_all_packages() + + # Dependencies for package which aren't satisfied by the + # current packages in the custom image recipe + deps = package.package_dependencies_source.annotate( + name=F('depends_on__name'), + pk=F('depends_on__pk'), + size=F('depends_on__size'), + ).values("name", "pk", "size").filter( + # There are two depends types we don't know why + (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) | + Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) & + ~Q(pk__in=all_current_packages) + ) + + # Reverse dependencies which are needed by packages that are + # in the image. Recursive search providing all dependents, + # not just immediate dependents. + reverse_deps = _get_all_dependents(package_id, all_current_packages) + total_size_deps = 0 + total_size_reverse_deps = 0 + + for dep in deps: + dep['size_formatted'] = \ + filtered_filesizeformat(dep['size']) + total_size_deps += dep['size'] + + for dep in reverse_deps: + dep['size_formatted'] = \ + filtered_filesizeformat(dep['size']) + total_size_reverse_deps += dep['size'] + + + return {"error": "ok", + "id": package.pk, + "name": package.name, + "version": package.version, + "unsatisfied_dependencies": list(deps), + "unsatisfied_dependencies_size": total_size_deps, + "unsatisfied_dependencies_size_formatted": + filtered_filesizeformat(total_size_deps), + "reverse_dependencies": list(reverse_deps), + "reverse_dependencies_size": total_size_reverse_deps, + "reverse_dependencies_size_formatted": + filtered_filesizeformat(total_size_reverse_deps)} + + included_packages = recipe.includes_set.values_list('pk', flat=True) if request.method == 'PUT': - recipe.packages.add(package) + # If we're adding back a package which used to be included in this + # image all we need to do is remove it from the excludes + if package.pk in included_packages: + try: + recipe.excludes_set.remove(package) + return {"error": "ok"} + except Package.DoesNotExist: + return {"error": + "Package %s not found in excludes but was in " + "included list" % package.name} + + else: + recipe.appends_set.add(package) + # Make sure that package is not in the excludes set + try: + recipe.excludes_set.remove(package) + except: + pass + # Add the dependencies we think will be added to the recipe + # as a result of appending this package. + # TODO this should recurse down the entire deps tree + for dep in package.package_dependencies_source.all_depends(): + try: + cust_package = CustomImagePackage.objects.get( + name=dep.depends_on.name) + + recipe.includes_set.add(cust_package) + try: + # When adding the pre-requisite package, make + # sure it's not in the excluded list from a + # prior removal. + recipe.excludes_set.remove(cust_package) + except Package.DoesNotExist: + # Don't care if the package had never been excluded + pass + except: + logger.warning("Could not add package's suggested" + "dependencies to the list") + return {"error": "ok"} + elif request.method == 'DELETE': - if package in recipe.packages.all(): - recipe.packages.remove(package) + try: + # If we're deleting a package which is included we need to + # Add it to the excludes list. + if package.pk in included_packages: + recipe.excludes_set.add(package) + else: + recipe.appends_set.remove(package) + all_current_packages = recipe.get_all_packages() + reverse_deps_dictlist = _get_all_dependents(package.pk, all_current_packages) + ids = [entry['pk'] for entry in reverse_deps_dictlist] + reverse_deps = CustomImagePackage.objects.filter(id__in=ids) + for r in reverse_deps: + try: + if r.id in included_packages: + recipe.excludes_set.add(r) + else: + recipe.appends_set.remove(r) + except: + pass + return {"error": "ok"} - else: - return {"error": "Package '%s' is not in the recipe '%s'" % \ - (package.name, recipe.name)} + except CustomImageRecipe.DoesNotExist: + return {"error": "Tried to remove package that wasn't present"} + else: return {"error": "Method %s is not supported" % request.method} @@ -2816,30 +2837,16 @@ if True: } vars_blacklist = { - 'DL_DR','PARALLEL_MAKE','BB_NUMBER_THREADS','SSTATE_DIR', + 'PARALLEL_MAKE','BB_NUMBER_THREADS', 'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT', - 'DL_DIR','PARALLEL_MAKE','SSTATE_DIR','SSTATE_DIR','SSTATE_MIRRORS','TMPDIR', + 'PARALLEL_MAKE','SSTATE_MIRRORS','TMPDIR', 'all_proxy','ftp_proxy','http_proxy ','https_proxy' } - vars_fstypes = { - 'btrfs','cpio','cpio.gz','cpio.lz4','cpio.lzma','cpio.xz','cramfs', - 'elf','ext2','ext2.bz2','ext2.gz','ext2.lzma', 'ext4', 'ext4.gz', 'ext3','ext3.gz','hddimg', - 'iso','jffs2','jffs2.sum','squashfs','squashfs-lzo','squashfs-xz','tar.bz2', - 'tar.lz4','tar.xz','tartar.gz','ubi','ubifs','vmdk' - } + vars_fstypes = Target_Image_File.SUFFIXES return(vars_managed,sorted(vars_fstypes),vars_blacklist) - def customrecipe(request, pid, recipe_id): - project = Project.objects.get(pk=pid) - context = {'project' : project, - 'projectlayers': [], - 'recipe' : CustomImageRecipe.objects.get(pk=recipe_id) - } - - return render(request, "customrecipe.html", context) - @_template_renderer("projectconf.html") def projectconf(request, pid): @@ -2870,6 +2877,19 @@ if True: except ProjectVariable.DoesNotExist: pass try: + if ProjectVariable.objects.get(project = prj, name = "DL_DIR").value == "${TOPDIR}/../downloads": + be = BuildEnvironment.objects.get(pk = str(1)) + dl_dir = os.path.join(dirname(be.builddir), "downloads") + context['dl_dir'] = dl_dir + pv, created = ProjectVariable.objects.get_or_create(project = prj, name = "DL_DIR") + pv.value = dl_dir + pv.save() + else: + context['dl_dir'] = ProjectVariable.objects.get(project = prj, name = "DL_DIR").value + context['dl_dir_defined'] = "1" + except ProjectVariable.DoesNotExist,BuildEnvironment.DoesNotExist: + pass + try: context['fstypes'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_FSTYPES").value context['fstypes_defined'] = "1" except ProjectVariable.DoesNotExist: @@ -2885,268 +2905,78 @@ if True: except ProjectVariable.DoesNotExist: pass try: - context['sdk_machine'] = ProjectVariable.objects.get(project = prj, name = "SDKMACHINE").value - context['sdk_machine_defined'] = "1" - except ProjectVariable.DoesNotExist: + if ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value == "${TOPDIR}/../sstate-cache": + be = BuildEnvironment.objects.get(pk = str(1)) + sstate_dir = os.path.join(dirname(be.builddir), "sstate-cache") + context['sstate_dir'] = sstate_dir + pv, created = ProjectVariable.objects.get_or_create(project = prj, name = "SSTATE_DIR") + pv.value = sstate_dir + pv.save() + else: + context['sstate_dir'] = ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value + context['sstate_dir_defined'] = "1" + except ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist: pass return context - # WARNING _build_list_helper() may raise a RedirectException, which - # will set the GET parameters and redirect back to the - # all-builds or projectbuilds page as appropriate; - # TODO don't use exceptions to control program flow - @_template_renderer('projectbuilds.html') - def projectbuilds(request, pid): - prj = Project.objects.get(id = pid) - - if request.method == "POST": - # process any build request - - if 'buildCancel' in request.POST: - for i in request.POST['buildCancel'].strip().split(" "): - try: - br = BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_QUEUED) - br.state = BuildRequest.REQ_DELETED - br.save() - except BuildRequest.DoesNotExist: - pass - - if 'buildDelete' in request.POST: - for i in request.POST['buildDelete'].strip().split(" "): - try: - BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() - except BuildRequest.DoesNotExist: - pass - - if 'targets' in request.POST: - ProjectTarget.objects.filter(project = prj).delete() - s = str(request.POST['targets']) - for t in s.translate(None, ";%|\"").split(" "): - if ":" in t: - target, task = t.split(":") - else: - target = t - task = "" - ProjectTarget.objects.create(project = prj, - target = target, - task = task) - prj.schedule_build() - - queryset = Build.objects.filter(project_id = pid) - - redirect_page = resolve(request.path_info).url_name - - context, pagesize, orderby = _build_list_helper(request, - queryset, - redirect_page, - pid) - - context['project'] = prj - _set_parameters_values(pagesize, orderby, request) - - # add the most recent builds for this project - context['mru'] = _get_latest_builds(prj) - - return context + def _file_names_for_artifact(build, artifact_type, artifact_id): + """ + Return a tuple (file path, file name for the download response) for an + artifact of type artifact_type with ID artifact_id for build; if + artifact type is not supported, returns (None, None) + """ + file_name = None + response_file_name = None + if artifact_type == "cookerlog": + file_name = build.cooker_log_path + response_file_name = "cooker.log" - def _file_name_for_artifact(b, artifact_type, artifact_id): - file_name = None - # Target_Image_File file_name - if artifact_type == "imagefile": - file_name = Target_Image_File.objects.get(target__build = b, pk = artifact_id).file_name + elif artifact_type == "imagefile": + file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name elif artifact_type == "buildartifact": - file_name = BuildArtifact.objects.get(build = b, pk = artifact_id).file_name + file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name - elif artifact_type == "licensemanifest": - file_name = Target.objects.get(build = b, pk = artifact_id).license_manifest_path + elif artifact_type == "licensemanifest": + file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path elif artifact_type == "tasklogfile": - file_name = Task.objects.get(build = b, pk = artifact_id).logfile + file_name = Task.objects.get(build = build, pk = artifact_id).logfile elif artifact_type == "logmessagefile": - file_name = LogMessage.objects.get(build = b, pk = artifact_id).pathname - else: - raise Exception("FIXME: artifact type %s not implemented" % (artifact_type)) + file_name = LogMessage.objects.get(build = build, pk = artifact_id).pathname - return file_name + if file_name and not response_file_name: + response_file_name = os.path.basename(file_name) + return (file_name, response_file_name) def build_artifact(request, build_id, artifact_type, artifact_id): - if artifact_type in ["cookerlog"]: - try: - build = Build.objects.get(pk = build_id) - file_name = build.cooker_log_path + """ + View which returns a build artifact file as a response + """ + file_name = None + response_file_name = None + + try: + build = Build.objects.get(pk = build_id) + file_name, response_file_name = _file_names_for_artifact( + build, artifact_type, artifact_id + ) + + if file_name and response_file_name: fsock = open(file_name, "r") content_type = MimeTypeFinder.get_mimetype(file_name) response = HttpResponse(fsock, content_type = content_type) - disposition = 'attachment; filename=cooker.log' - response['Content-Disposition'] = disposition + disposition = "attachment; filename=" + response_file_name + response["Content-Disposition"] = disposition return response - except IOError: - context = { - 'build' : Build.objects.get(pk = build_id), - } - return render(request, "unavailable_artifact.html", context) - - else: - # retrieve the artifact directly from the build environment - return _get_be_artifact(request, build_id, artifact_type, artifact_id) - - - def _get_be_artifact(request, build_id, artifact_type, artifact_id): - try: - b = Build.objects.get(pk=build_id) - if b.buildrequest is None or b.buildrequest.environment is None: - raise Exception("Artifact not available for download (missing build request or build environment)") - - file_name = _file_name_for_artifact(b, artifact_type, artifact_id) - fsock = None - content_type='application/force-download' - - if file_name is None: - raise Exception("Could not handle artifact %s id %s" % (artifact_type, artifact_id)) else: - content_type = MimeTypeFinder.get_mimetype(file_name) - fsock = b.buildrequest.environment.get_artifact(file_name) - file_name = os.path.basename(file_name) # we assume that the build environment system has the same path conventions as host - - response = HttpResponse(fsock, content_type = content_type) - - # returns a file from the environment - response['Content-Disposition'] = 'attachment; filename=' + file_name - return response - except IOError: - context = { - 'build' : Build.objects.get(pk = build_id), - } - return render(request, "unavailable_artifact.html", context) - - - - - @_template_renderer("projects.html") - def projects(request): - (pagesize, orderby) = _get_parameters_values(request, 10, 'updated:-') - mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } - retval = _verify_parameters( request.GET, mandatory_parameters ) - if retval: - raise RedirectException( 'all-projects', request.GET, mandatory_parameters ) - - queryset_all = Project.objects.all() - - # annotate each project with its number of builds - queryset_all = queryset_all.annotate(num_builds=Count('build')) - - # exclude the command line builds project if it has no builds - q_default_with_builds = Q(is_default=True) & Q(num_builds__gt=0) - queryset_all = queryset_all.filter(Q(is_default=False) | - q_default_with_builds) - - # if in BUILD_MODE, exclude everything but the command line builds project - if not toastermain.settings.BUILD_MODE: - queryset_all = queryset_all.exclude(is_default=False) - - # boilerplate code that takes a request for an object type and returns a queryset - # for that object type. copypasta for all needed table searches - (filter_string, search_term, ordering_string) = _search_tuple(request, Project) - queryset_with_search = _get_queryset(Project, queryset_all, None, search_term, ordering_string, '-updated') - queryset = _get_queryset(Project, queryset_all, filter_string, search_term, ordering_string, '-updated') - - # retrieve the objects that will be displayed in the table; projects a paginator and gets a page range to display - project_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) - - # add fields needed in JSON dumps for API call support - for p in project_info.object_list: - p.id = p.pk - p.projectPageUrl = reverse('project', args=(p.id,)) - p.layersTypeAheadUrl = reverse('xhr_layerstypeahead', args=(p.id,)) - p.recipesTypeAheadUrl = reverse('xhr_recipestypeahead', args=(p.id,)) - p.projectBuildsUrl = reverse('projectbuilds', args=(p.id,)) - - # build view-specific information; this is rendered specifically in the builds page, at the top of the page (i.e. Recent builds) - build_mru = _get_latest_builds() - - # translate the project's build target strings - fstypes_map = {}; - for project in project_info: - try: - targets = Target.objects.filter( build_id = project.get_last_build_id() ) - comma = ""; - extensions = ""; - for t in targets: - if ( not t.is_image ): - continue - tif = Target_Image_File.objects.filter( target_id = t.id ) - for i in tif: - s=re.sub('.*tar.bz2', 'tar.bz2', i.file_name) - if s == i.file_name: - s=re.sub('.*\.', '', i.file_name) - if None == re.search(s,extensions): - extensions += comma + s - comma = ", " - fstypes_map[project.id]=extensions - except (Target.DoesNotExist,IndexError): - fstypes_map[project.id]=project.get_last_imgfiles - - context = { - 'mru' : build_mru, - - 'objects' : project_info, - 'objectname' : "projects", - 'default_orderby' : 'id:-', - 'search_term' : search_term, - 'total_count' : queryset_with_search.count(), - 'fstypes' : fstypes_map, - 'build_FAILED' : Build.FAILED, - 'build_SUCCEEDED' : Build.SUCCEEDED, - 'tablecols': [ - {'name': 'Project', - 'orderfield': _get_toggle_order(request, "name"), - 'ordericon':_get_toggle_order_icon(request, "name"), - 'orderkey' : 'name', - }, - {'name': 'Last activity on', - 'clclass': 'updated', - 'qhelp': "Shows the starting date and time of the last project build. If the project has no builds, it shows the date the project was created", - 'orderfield': _get_toggle_order(request, "updated", True), - 'ordericon':_get_toggle_order_icon(request, "updated"), - 'orderkey' : 'updated', - }, - {'name': 'Release', - 'qhelp' : "The version of the build system used by the project", - 'orderfield': _get_toggle_order(request, "release__name"), - 'ordericon':_get_toggle_order_icon(request, "release__name"), - 'orderkey' : 'release__name', - }, - {'name': 'Machine', - 'qhelp': "The hardware currently selected for the project", - }, - {'name': 'Number of builds', - 'qhelp': "How many builds have been run for the project", - }, - {'name': 'Last build outcome', 'clclass': 'loutcome', - 'qhelp': "Tells you if the last project build completed successfully or failed", - }, - {'name': 'Recipe', 'clclass': 'ltarget', - 'qhelp': "The last recipe that was built in this project", - }, - {'name': 'Errors', 'clclass': 'lerrors', - 'qhelp': "How many errors were encountered during the last project build (if any)", - }, - {'name': 'Warnings', 'clclass': 'lwarnings', - 'qhelp': "How many warnigns were encountered during the last project build (if any)", - }, - {'name': 'Image files', 'clclass': 'limagefiles', 'hidden': 1, - 'qhelp': "The root file system types produced by the last project build", - }, - ] - } - - _set_parameters_values(pagesize, orderby, request) - return context + return render(request, "unavailable_artifact.html") + except ObjectDoesNotExist, IOError: + return render(request, "unavailable_artifact.html") |