diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/toaster/toastergui')
4 files changed, 183 insertions, 77 deletions
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js index e0df46397..895e61b2a 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js +++ b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js @@ -57,8 +57,8 @@ function basePageInit(ctx) { if ($(".total-builds").length !== 0){ libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl, function(prjInfo){ - if (prjInfo.builds) - $(".total-builds").text(prjInfo.builds.length); + if (prjInfo.completedbuilds) + $(".total-builds").text(prjInfo.completedbuilds.length); }); } diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html index df809de40..27cfcd7dc 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html +++ b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html @@ -16,8 +16,8 @@ <script> // initialize the date range controls $(document).ready(function () { - date_init('created','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); - date_init('updated','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); + date_init('started_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); + date_init('completed_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); }); </script> diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py index 1a8b4787d..4d1549b0a 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py +++ b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py @@ -24,10 +24,11 @@ from django.test import TestCase from django.core.urlresolvers import reverse from django.utils import timezone -from orm.models import Project, Release, BitbakeVersion, Build -from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer +from orm.models import Project, Release, BitbakeVersion, ProjectTarget +from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build from orm.models import Layer_Version, Recipe, Machine, ProjectLayer import json +from bs4 import BeautifulSoup PROJECT_NAME = "test project" @@ -41,7 +42,6 @@ class ViewTests(TestCase): bitbake_version=bbv) self.project = Project.objects.create_project(name=PROJECT_NAME, release=release) - layersrc = LayerSource.objects.create(sourcetype=LayerSource.TYPE_IMPORTED) self.priority = ReleaseLayerSourcePriority.objects.create(release=release, layer_source=layersrc) @@ -292,3 +292,88 @@ class ProjectsPageTests(TestCase): 'should be a project row in the page') self.assertTrue(self.PROJECT_NAME in response.content, 'default project "cli builds" should be in page') + +class ProjectBuildsDisplayTest(TestCase): + """ Test data at /project/X/builds is displayed correctly """ + + def setUp(self): + bbv = BitbakeVersion.objects.create(name="bbv1", giturl="/tmp/", + branch="master", dirpath="") + release = Release.objects.create(name="release1", + bitbake_version=bbv) + self.project1 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + self.project2 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + + # parameters for builds to associate with the projects + now = timezone.now() + + self.project1_build_success = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project1_build_in_progress = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + self.project2_build_success = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project2_build_in_progress = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + def _get_rows_for_project(self, project_id): + url = reverse("projectbuilds", args=(project_id,)) + response = self.client.get(url, follow=True) + soup = BeautifulSoup(response.content) + return soup.select('tr[class="data"]') + + def test_show_builds_for_project(self): + """ Builds for a project should be displayed """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2) + + def test_show_builds_for_project_only(self): + """ Builds for other projects should be excluded """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build1c = Build.objects.create(**self.project1_build_success) + + # shouldn't see these two + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 3) + + def test_show_builds_exclude_in_progress(self): + """ "in progress" builds should not be shown """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + + # shouldn't see this one + build1c = Build.objects.create(**self.project1_build_in_progress) + + # shouldn't see these two either, as they belong to a different project + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2)
\ No newline at end of file diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/yocto-poky/bitbake/lib/toaster/toastergui/views.py index 4e8f69e80..8689a1251 100755 --- a/yocto-poky/bitbake/lib/toaster/toastergui/views.py +++ b/yocto-poky/bitbake/lib/toaster/toastergui/views.py @@ -40,17 +40,26 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone from django.utils.html import escape -from datetime import timedelta, datetime, date +from datetime import timedelta, datetime from django.utils import formats from toastergui.templatetags.projecttags import json as jsonfilter import json from os.path import dirname import itertools +import magic import logging logger = logging.getLogger("toaster") +class MimeTypeFinder(object): + _magic = magic.Magic(flags = magic.MAGIC_MIME_TYPE) + + # returns the mimetype for a file path + @classmethod + def get_mimetype(self, path): + return self._magic.id_filename(path) + # all new sessions should come through the landing page; # determine in which mode we are running in, and redirect appropriately def landing(request): @@ -68,8 +77,6 @@ def landing(request): return render(request, 'landing.html', context) - - # returns a list for most recent builds; def _get_latest_builds(prj=None): queryset = Build.objects.all() @@ -435,8 +442,7 @@ def _modify_date_range_filter(filter_string): def _add_daterange_context(queryset_all, request, daterange_list): # calculate the exact begining of local today and yesterday today_begin = timezone.localtime(timezone.now()) - today_begin = date(today_begin.year,today_begin.month,today_begin.day) - yesterday_begin = today_begin-timedelta(days=1) + yesterday_begin = today_begin - timedelta(days=1) # add daterange persistent context_date = {} context_date['last_date_from'] = request.GET.get('last_date_from',timezone.localtime(timezone.now()).strftime("%d/%m/%Y")) @@ -1890,45 +1896,87 @@ if True: pass # shows the "all builds" page for managed mode; it displays build requests (at least started!) instead of actual builds + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer("builds.html") def builds(request): # define here what parameters the view needs in the GET portion in order to # be able to display something. 'count' and 'page' are mandatory for all views # that use paginators. - queryset = Build.objects.exclude(outcome = Build.IN_PROGRESS) + queryset = Build.objects.all() - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - # all builds page as a Project column - context['tablecols'].append({'name': 'Project', 'clcalss': 'project_column', }) - except RedirectException as re: - # rewrite the RedirectException - re.view = resolve(request.path_info).url_name - raise re + redirect_page = resolve(request.path_info).url_name + + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page) + # all builds page as a Project column + context['tablecols'].append({ + 'name': 'Project', + 'clclass': 'project_column' + }) _set_parameters_values(pagesize, orderby, request) return context # helper function, to be used on "all builds" and "project builds" pages - def _build_list_helper(request, queryset_all): - + def _build_list_helper(request, queryset_all, redirect_page, pid=None): default_orderby = 'completed_on:-' (pagesize, orderby) = _get_parameters_values(request, 10, default_orderby) mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: - raise RedirectException( None, request.GET, mandatory_parameters) + params = {} + if pid: + params = {'pid': pid} + raise RedirectException(redirect_page, + request.GET, + mandatory_parameters, + **params) # boilerplate code that takes a request for an object type and returns a queryset # for that object type. copypasta for all needed table searches (filter_string, search_term, ordering_string) = _search_tuple(request, Build) + # post-process any date range filters - filter_string,daterange_selected = _modify_date_range_filter(filter_string) - queryset_all = queryset_all.select_related("project").annotate(errors_no = Count('logmessage', only=Q(logmessage__level=LogMessage.ERROR)|Q(logmessage__level=LogMessage.EXCEPTION))).annotate(warnings_no = Count('logmessage', only=Q(logmessage__level=LogMessage.WARNING))).extra(select={'timespent':'completed_on - started_on'}) - queryset_with_search = _get_queryset(Build, queryset_all, None, search_term, ordering_string, '-completed_on') - queryset = _get_queryset(Build, queryset_all, filter_string, search_term, ordering_string, '-completed_on') + filter_string, daterange_selected = _modify_date_range_filter(filter_string) + + # don't show "in progress" builds in "all builds" or "project builds" + queryset_all = queryset_all.exclude(outcome = Build.IN_PROGRESS) + + # append project info + queryset_all = queryset_all.select_related("project") + + # annotate with number of ERROR and EXCEPTION log messages + queryset_all = queryset_all.annotate( + errors_no = Count( + 'logmessage', + only=Q(logmessage__level=LogMessage.ERROR) | + Q(logmessage__level=LogMessage.EXCEPTION) + ) + ) + + # annotate with number of warnings + q_warnings = Q(logmessage__level=LogMessage.WARNING) + queryset_all = queryset_all.annotate( + warnings_no = Count('logmessage', only=q_warnings) + ) + + # add timespent field + timespent = 'completed_on - started_on' + queryset_all = queryset_all.extra(select={'timespent': timespent}) + + queryset_with_search = _get_queryset(Build, queryset_all, + None, search_term, + ordering_string, '-completed_on') + + queryset = _get_queryset(Build, queryset_all, + filter_string, search_term, + ordering_string, '-completed_on') # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display build_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) @@ -2226,7 +2274,7 @@ if True: context = { "project" : prj, "lvs_nos" : Layer_Version.objects.all().count(), - "completedbuilds": Build.objects.filter(project_id = pid).filter(outcome__lte = Build.IN_PROGRESS), + "completedbuilds": Build.objects.exclude(outcome = Build.IN_PROGRESS).filter(project_id = pid), "prj" : {"name": prj.name, }, "buildrequests" : prj.build_set.filter(outcome=Build.IN_PROGRESS), "builds" : _project_recent_build_list(prj), @@ -2632,6 +2680,10 @@ if True: return context + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer('projectbuilds.html') def projectbuilds(request, pid): prj = Project.objects.get(id = pid) @@ -2651,7 +2703,7 @@ if True: if 'buildDelete' in request.POST: for i in request.POST['buildDelete'].strip().split(" "): try: - br = BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() + BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() except BuildRequest.DoesNotExist: pass @@ -2664,20 +2716,19 @@ if True: else: target = t task = "" - ProjectTarget.objects.create(project = prj, target = target, task = task) - - br = prj.schedule_build() + ProjectTarget.objects.create(project = prj, + target = target, + task = task) + prj.schedule_build() + queryset = Build.objects.filter(project_id = pid) - queryset = Build.objects.filter(outcome__lte = Build.IN_PROGRESS) + redirect_page = resolve(request.path_info).url_name - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - except RedirectException as re: - # rewrite the RedirectException with our current url information - re.view = resolve(request.path_info).url_name - re.okwargs = {"pid" : pid} - raise re + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page, + pid) context['project'] = prj _set_parameters_values(pagesize, orderby, request) @@ -2710,47 +2761,17 @@ if True: def build_artifact(request, build_id, artifact_type, artifact_id): if artifact_type in ["cookerlog"]: - # these artifacts are saved after building, so they are on the server itself - def _mimetype_for_artifact(path): - try: - import magic - - # fair warning: this is a mess; there are multiple competing and incompatible - # magic modules floating around, so we try some of the most common combinations - - try: # we try ubuntu's python-magic 5.4 - m = magic.open(magic.MAGIC_MIME_TYPE) - m.load() - return m.file(path) - except AttributeError: - pass - - try: # we try python-magic 0.4.6 - m = magic.Magic(magic.MAGIC_MIME) - return m.from_file(path) - except AttributeError: - pass - - try: # we try pip filemagic 1.6 - m = magic.Magic(flags=magic.MAGIC_MIME_TYPE) - return m.id_filename(path) - except AttributeError: - pass - - return "binary/octet-stream" - except ImportError: - return "binary/octet-stream" try: - # match code with runbuilds.Command.archive() - build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id)) - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - + build = Build.objects.get(pk = build_id) + file_name = build.cooker_log_path fsock = open(file_name, "r") - content_type=_mimetype_for_artifact(file_name) + content_type = MimeTypeFinder.get_mimetype(file_name) response = HttpResponse(fsock, content_type = content_type) - response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name) + disposition = 'attachment; filename=cooker.log' + response['Content-Disposition'] = disposition + return response except IOError: context = { |