diff options
author | Patrick Williams <patrick@stwcx.xyz> | 2015-09-22 16:09:05 +0300 |
---|---|---|
committer | Patrick Williams <patrick@stwcx.xyz> | 2015-09-22 16:09:05 +0300 |
commit | 3445365503e1e4d5601acf7c05609cc9673ec414 (patch) | |
tree | 7eb70c5bab200b0515a1b4d16873a75855df4c89 /yocto-poky/bitbake/lib | |
parent | d10502479a70bd72ca4e09569b6ee738875e4823 (diff) | |
parent | d7e963193b4e6541206a320316a158a65f1fee89 (diff) | |
download | openbmc-3445365503e1e4d5601acf7c05609cc9673ec414.tar.xz |
Merge commit 'd7e963193b4e6541206a320316a158a65f1fee89' into HEAD
Diffstat (limited to 'yocto-poky/bitbake/lib')
15 files changed, 277 insertions, 152 deletions
diff --git a/yocto-poky/bitbake/lib/bb/cache.py b/yocto-poky/bitbake/lib/bb/cache.py index ef4d660e8..ab09b08b5 100644 --- a/yocto-poky/bitbake/lib/bb/cache.py +++ b/yocto-poky/bitbake/lib/bb/cache.py @@ -528,7 +528,20 @@ class Cache(object): if hasattr(info_array[0], 'file_checksums'): for _, fl in info_array[0].file_checksums.items(): - for f in fl.split(): + fl = fl.strip() + while fl: + # A .split() would be simpler but means spaces or colons in filenames would break + a = fl.find(":True") + b = fl.find(":False") + if ((a < 0) and b) or ((b > 0) and (b < a)): + f = fl[:b+6] + fl = fl[b+7:] + elif ((b < 0) and a) or ((a > 0) and (a < b)): + f = fl[:a+5] + fl = fl[a+6:] + else: + break + fl = fl.strip() if "*" in f: continue f, exist = f.split(":") diff --git a/yocto-poky/bitbake/lib/bb/data_smart.py b/yocto-poky/bitbake/lib/bb/data_smart.py index 79b4ed932..70558c15a 100644 --- a/yocto-poky/bitbake/lib/bb/data_smart.py +++ b/yocto-poky/bitbake/lib/bb/data_smart.py @@ -413,9 +413,11 @@ class DataSmart(MutableMapping): self.overrides = None def need_overrides(self): - if self.overrides is None: - if self.inoverride: - return + if self.overrides is not None: + return + if self.inoverride: + return + for count in range(5): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] @@ -424,6 +426,13 @@ class DataSmart(MutableMapping): self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {} + newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + if newoverrides == self.overrides: + break + self.overrides = newoverrides + self.overridesset = set(self.overrides) + else: + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") def initVar(self, var): self.expand_cache = {} @@ -484,10 +493,8 @@ class DataSmart(MutableMapping): if '_' in var: self._setvar_update_overrides(base, **loginfo) - if base in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) return if not var in self.dict: @@ -520,8 +527,21 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) if var in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) + + def _setvar_update_overridevars(self, var, value): + vardata = self.expandWithRefs(value, var) + new = vardata.references + new.update(vardata.contains.keys()) + while not new.issubset(self.overridevars): + nextnew = set() + self.overridevars.update(new) + for i in new: + vardata = self.expandWithRefs(self.getVar(i, True), i) + nextnew.update(vardata.references) + nextnew.update(vardata.contains.keys()) + new = nextnew + self.internal_finalize(True) def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster @@ -628,6 +648,8 @@ class DataSmart(MutableMapping): if flag == "_defaultval" and '_' in var: self._setvar_update_overrides(var, **loginfo) + if flag == "_defaultval" and var in self.overridevars: + self._setvar_update_overridevars(var, value) if flag == "unexport" or flag == "export": if not "__exportlist" in self.dict: diff --git a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py index 3d53b63b3..288a1c8fd 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py @@ -464,7 +464,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d): for k in replacements: uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) - result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc]) + result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) if loc == 2: # Handle path manipulations basename = None @@ -862,7 +862,7 @@ def build_mirroruris(origud, mirrors, ld): replacements["BASENAME"] = origud.path.split("/")[-1] replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') - def adduri(ud, uris, uds): + def adduri(ud, uris, uds, mirrors): for line in mirrors: try: (find, replace) = line @@ -876,6 +876,12 @@ def build_mirroruris(origud, mirrors, ld): logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) continue + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) + try: newud = FetchData(newuri, ld) newud.setup_localpath(ld) @@ -885,16 +891,16 @@ def build_mirroruris(origud, mirrors, ld): try: # setup_localpath of file:// urls may fail, we should still see # if mirrors of the url exist - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) except UnboundLocalError: pass continue uris.append(newuri) uds.append(newud) - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) - adduri(origud, uris, uds) + adduri(origud, uris, uds, mirrors) return uris, uds diff --git a/yocto-poky/bitbake/lib/bb/fetch2/git.py b/yocto-poky/bitbake/lib/bb/fetch2/git.py index 40658ff94..9bd87ad25 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/git.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/git.py @@ -66,6 +66,7 @@ Supported SRC_URI options are: # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import errno import os import re import bb @@ -181,8 +182,6 @@ class Git(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): bb.utils.mkdirhier(ud.clonedir) @@ -220,7 +219,11 @@ class Git(FetchMethod): runfetchcmd(fetch_cmd, d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise os.chdir(ud.clonedir) for name in ud.names: if not self._contains_ref(ud, d, name): @@ -228,7 +231,7 @@ class Git(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/yocto-poky/bitbake/lib/bb/fetch2/hg.py index d978630ba..bbb4ed95d 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/hg.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/hg.py @@ -163,8 +163,6 @@ class Hg(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") # If the checkout doesn't exist and the mirror tarball does, extract it @@ -189,7 +187,11 @@ class Hg(FetchMethod): logger.debug(1, "Running %s", pullcmd) bb.fetch2.check_network_access(d, pullcmd, ud.url) runfetchcmd(pullcmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise # No source found, clone it. if not os.path.exists(ud.moddir): @@ -238,7 +240,7 @@ class Hg(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/yocto-poky/bitbake/lib/bb/tests/fetch.py b/yocto-poky/bitbake/lib/bb/tests/fetch.py index 1e61f3a11..94173c14a 100644 --- a/yocto-poky/bitbake/lib/bb/tests/fetch.py +++ b/yocto-poky/bitbake/lib/bb/tests/fetch.py @@ -405,6 +405,16 @@ class MirrorUriTest(FetcherTest): 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) + recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ + "https://.*/[^/]* https://BBBB/B/B/B/ \n" + + def test_recursive(self): + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', + 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', + 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) class FetcherLocalTest(FetcherTest): def setUp(self): diff --git a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py index 2d1ed5111..6e313fee8 100644 --- a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py +++ b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py @@ -704,7 +704,7 @@ class BuildInfoHelper(object): ## methods to convert event/external info into objects that the ORM layer uses - def _get_build_information(self): + def _get_build_information(self, consolelogfile): build_info = {} # Generate an identifier for each new build @@ -713,7 +713,7 @@ class BuildInfoHelper(object): build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() - build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0] + build_info['cooker_log_path'] = consolelogfile build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] @@ -847,9 +847,9 @@ class BuildInfoHelper(object): logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) - def store_started_build(self, event): + def store_started_build(self, event, consolelogfile): assert '_pkgs' in vars(event) - build_information = self._get_build_information() + build_information = self._get_build_information(consolelogfile) build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) diff --git a/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/yocto-poky/bitbake/lib/bb/ui/toasterui.py index 9c7e87dd1..e0c278bb3 100644 --- a/yocto-poky/bitbake/lib/bb/ui/toasterui.py +++ b/yocto-poky/bitbake/lib/bb/ui/toasterui.py @@ -126,7 +126,7 @@ def main(server, eventHandler, params ): # the code will look into the protected variables of the event; no easy way around this if isinstance(event, bb.event.BuildStarted): - buildinfohelper.store_started_build(event) + buildinfohelper.store_started_build(event, consolelogfile) if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): buildinfohelper.update_and_store_task(event) diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 3ccc7c67c..b2c573c9e 100644 --- a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -64,27 +64,6 @@ class Command(NoArgsCommand): return "" return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4)) - - def _verify_artifact_storage_dir(self): - # verify that we have a settings for downloading artifacts - while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: - guessedpath = os.getcwd() + "/toaster_build_artifacts/" - print("\nToaster needs to know in which directory it can download build log files and other artifacts.\nToaster suggests \"%s\"." % guessedpath) - artifacts_storage_dir = raw_input("Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) - if len(artifacts_storage_dir) == 0: - artifacts_storage_dir = guessedpath - if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): - try: - os.makedirs(artifacts_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) - return 0 - - def _verify_build_environment(self): # refuse to start if we have no build environments while BuildEnvironment.objects.count() == 0: @@ -239,7 +218,6 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): retval = 0 - retval += self._verify_artifact_storage_dir() retval += self._verify_build_environment() retval += self._verify_default_settings() retval += self._verify_builds_in_progress() diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index c3e9b74c0..718e1441d 100644 --- a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -77,31 +77,11 @@ class Command(NoArgsCommand): bec.be.save() def archive(self): - ''' archives data from the builds ''' - artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): - # save cooker log if br.build == None: br.state = BuildRequest.REQ_FAILED - br.save() - continue - build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) - try: - os.makedirs(build_artifact_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - try: - with open(file_name, "w") as f: - f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) - except IOError: - os.unlink(file_name) - - br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_COMPLETED br.save() def cleanup(self): diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js index e0df46397..895e61b2a 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js +++ b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js @@ -57,8 +57,8 @@ function basePageInit(ctx) { if ($(".total-builds").length !== 0){ libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl, function(prjInfo){ - if (prjInfo.builds) - $(".total-builds").text(prjInfo.builds.length); + if (prjInfo.completedbuilds) + $(".total-builds").text(prjInfo.completedbuilds.length); }); } diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html index df809de40..27cfcd7dc 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html +++ b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html @@ -16,8 +16,8 @@ <script> // initialize the date range controls $(document).ready(function () { - date_init('created','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); - date_init('updated','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); + date_init('started_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); + date_init('completed_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); }); </script> diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py index 1a8b4787d..4d1549b0a 100644 --- a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py +++ b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py @@ -24,10 +24,11 @@ from django.test import TestCase from django.core.urlresolvers import reverse from django.utils import timezone -from orm.models import Project, Release, BitbakeVersion, Build -from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer +from orm.models import Project, Release, BitbakeVersion, ProjectTarget +from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build from orm.models import Layer_Version, Recipe, Machine, ProjectLayer import json +from bs4 import BeautifulSoup PROJECT_NAME = "test project" @@ -41,7 +42,6 @@ class ViewTests(TestCase): bitbake_version=bbv) self.project = Project.objects.create_project(name=PROJECT_NAME, release=release) - layersrc = LayerSource.objects.create(sourcetype=LayerSource.TYPE_IMPORTED) self.priority = ReleaseLayerSourcePriority.objects.create(release=release, layer_source=layersrc) @@ -292,3 +292,88 @@ class ProjectsPageTests(TestCase): 'should be a project row in the page') self.assertTrue(self.PROJECT_NAME in response.content, 'default project "cli builds" should be in page') + +class ProjectBuildsDisplayTest(TestCase): + """ Test data at /project/X/builds is displayed correctly """ + + def setUp(self): + bbv = BitbakeVersion.objects.create(name="bbv1", giturl="/tmp/", + branch="master", dirpath="") + release = Release.objects.create(name="release1", + bitbake_version=bbv) + self.project1 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + self.project2 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + + # parameters for builds to associate with the projects + now = timezone.now() + + self.project1_build_success = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project1_build_in_progress = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + self.project2_build_success = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project2_build_in_progress = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + def _get_rows_for_project(self, project_id): + url = reverse("projectbuilds", args=(project_id,)) + response = self.client.get(url, follow=True) + soup = BeautifulSoup(response.content) + return soup.select('tr[class="data"]') + + def test_show_builds_for_project(self): + """ Builds for a project should be displayed """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2) + + def test_show_builds_for_project_only(self): + """ Builds for other projects should be excluded """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build1c = Build.objects.create(**self.project1_build_success) + + # shouldn't see these two + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 3) + + def test_show_builds_exclude_in_progress(self): + """ "in progress" builds should not be shown """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + + # shouldn't see this one + build1c = Build.objects.create(**self.project1_build_in_progress) + + # shouldn't see these two either, as they belong to a different project + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2)
\ No newline at end of file diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/yocto-poky/bitbake/lib/toaster/toastergui/views.py index 4e8f69e80..8689a1251 100755 --- a/yocto-poky/bitbake/lib/toaster/toastergui/views.py +++ b/yocto-poky/bitbake/lib/toaster/toastergui/views.py @@ -40,17 +40,26 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone from django.utils.html import escape -from datetime import timedelta, datetime, date +from datetime import timedelta, datetime from django.utils import formats from toastergui.templatetags.projecttags import json as jsonfilter import json from os.path import dirname import itertools +import magic import logging logger = logging.getLogger("toaster") +class MimeTypeFinder(object): + _magic = magic.Magic(flags = magic.MAGIC_MIME_TYPE) + + # returns the mimetype for a file path + @classmethod + def get_mimetype(self, path): + return self._magic.id_filename(path) + # all new sessions should come through the landing page; # determine in which mode we are running in, and redirect appropriately def landing(request): @@ -68,8 +77,6 @@ def landing(request): return render(request, 'landing.html', context) - - # returns a list for most recent builds; def _get_latest_builds(prj=None): queryset = Build.objects.all() @@ -435,8 +442,7 @@ def _modify_date_range_filter(filter_string): def _add_daterange_context(queryset_all, request, daterange_list): # calculate the exact begining of local today and yesterday today_begin = timezone.localtime(timezone.now()) - today_begin = date(today_begin.year,today_begin.month,today_begin.day) - yesterday_begin = today_begin-timedelta(days=1) + yesterday_begin = today_begin - timedelta(days=1) # add daterange persistent context_date = {} context_date['last_date_from'] = request.GET.get('last_date_from',timezone.localtime(timezone.now()).strftime("%d/%m/%Y")) @@ -1890,45 +1896,87 @@ if True: pass # shows the "all builds" page for managed mode; it displays build requests (at least started!) instead of actual builds + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer("builds.html") def builds(request): # define here what parameters the view needs in the GET portion in order to # be able to display something. 'count' and 'page' are mandatory for all views # that use paginators. - queryset = Build.objects.exclude(outcome = Build.IN_PROGRESS) + queryset = Build.objects.all() - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - # all builds page as a Project column - context['tablecols'].append({'name': 'Project', 'clcalss': 'project_column', }) - except RedirectException as re: - # rewrite the RedirectException - re.view = resolve(request.path_info).url_name - raise re + redirect_page = resolve(request.path_info).url_name + + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page) + # all builds page as a Project column + context['tablecols'].append({ + 'name': 'Project', + 'clclass': 'project_column' + }) _set_parameters_values(pagesize, orderby, request) return context # helper function, to be used on "all builds" and "project builds" pages - def _build_list_helper(request, queryset_all): - + def _build_list_helper(request, queryset_all, redirect_page, pid=None): default_orderby = 'completed_on:-' (pagesize, orderby) = _get_parameters_values(request, 10, default_orderby) mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: - raise RedirectException( None, request.GET, mandatory_parameters) + params = {} + if pid: + params = {'pid': pid} + raise RedirectException(redirect_page, + request.GET, + mandatory_parameters, + **params) # boilerplate code that takes a request for an object type and returns a queryset # for that object type. copypasta for all needed table searches (filter_string, search_term, ordering_string) = _search_tuple(request, Build) + # post-process any date range filters - filter_string,daterange_selected = _modify_date_range_filter(filter_string) - queryset_all = queryset_all.select_related("project").annotate(errors_no = Count('logmessage', only=Q(logmessage__level=LogMessage.ERROR)|Q(logmessage__level=LogMessage.EXCEPTION))).annotate(warnings_no = Count('logmessage', only=Q(logmessage__level=LogMessage.WARNING))).extra(select={'timespent':'completed_on - started_on'}) - queryset_with_search = _get_queryset(Build, queryset_all, None, search_term, ordering_string, '-completed_on') - queryset = _get_queryset(Build, queryset_all, filter_string, search_term, ordering_string, '-completed_on') + filter_string, daterange_selected = _modify_date_range_filter(filter_string) + + # don't show "in progress" builds in "all builds" or "project builds" + queryset_all = queryset_all.exclude(outcome = Build.IN_PROGRESS) + + # append project info + queryset_all = queryset_all.select_related("project") + + # annotate with number of ERROR and EXCEPTION log messages + queryset_all = queryset_all.annotate( + errors_no = Count( + 'logmessage', + only=Q(logmessage__level=LogMessage.ERROR) | + Q(logmessage__level=LogMessage.EXCEPTION) + ) + ) + + # annotate with number of warnings + q_warnings = Q(logmessage__level=LogMessage.WARNING) + queryset_all = queryset_all.annotate( + warnings_no = Count('logmessage', only=q_warnings) + ) + + # add timespent field + timespent = 'completed_on - started_on' + queryset_all = queryset_all.extra(select={'timespent': timespent}) + + queryset_with_search = _get_queryset(Build, queryset_all, + None, search_term, + ordering_string, '-completed_on') + + queryset = _get_queryset(Build, queryset_all, + filter_string, search_term, + ordering_string, '-completed_on') # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display build_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) @@ -2226,7 +2274,7 @@ if True: context = { "project" : prj, "lvs_nos" : Layer_Version.objects.all().count(), - "completedbuilds": Build.objects.filter(project_id = pid).filter(outcome__lte = Build.IN_PROGRESS), + "completedbuilds": Build.objects.exclude(outcome = Build.IN_PROGRESS).filter(project_id = pid), "prj" : {"name": prj.name, }, "buildrequests" : prj.build_set.filter(outcome=Build.IN_PROGRESS), "builds" : _project_recent_build_list(prj), @@ -2632,6 +2680,10 @@ if True: return context + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer('projectbuilds.html') def projectbuilds(request, pid): prj = Project.objects.get(id = pid) @@ -2651,7 +2703,7 @@ if True: if 'buildDelete' in request.POST: for i in request.POST['buildDelete'].strip().split(" "): try: - br = BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() + BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() except BuildRequest.DoesNotExist: pass @@ -2664,20 +2716,19 @@ if True: else: target = t task = "" - ProjectTarget.objects.create(project = prj, target = target, task = task) - - br = prj.schedule_build() + ProjectTarget.objects.create(project = prj, + target = target, + task = task) + prj.schedule_build() + queryset = Build.objects.filter(project_id = pid) - queryset = Build.objects.filter(outcome__lte = Build.IN_PROGRESS) + redirect_page = resolve(request.path_info).url_name - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - except RedirectException as re: - # rewrite the RedirectException with our current url information - re.view = resolve(request.path_info).url_name - re.okwargs = {"pid" : pid} - raise re + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page, + pid) context['project'] = prj _set_parameters_values(pagesize, orderby, request) @@ -2710,47 +2761,17 @@ if True: def build_artifact(request, build_id, artifact_type, artifact_id): if artifact_type in ["cookerlog"]: - # these artifacts are saved after building, so they are on the server itself - def _mimetype_for_artifact(path): - try: - import magic - - # fair warning: this is a mess; there are multiple competing and incompatible - # magic modules floating around, so we try some of the most common combinations - - try: # we try ubuntu's python-magic 5.4 - m = magic.open(magic.MAGIC_MIME_TYPE) - m.load() - return m.file(path) - except AttributeError: - pass - - try: # we try python-magic 0.4.6 - m = magic.Magic(magic.MAGIC_MIME) - return m.from_file(path) - except AttributeError: - pass - - try: # we try pip filemagic 1.6 - m = magic.Magic(flags=magic.MAGIC_MIME_TYPE) - return m.id_filename(path) - except AttributeError: - pass - - return "binary/octet-stream" - except ImportError: - return "binary/octet-stream" try: - # match code with runbuilds.Command.archive() - build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id)) - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - + build = Build.objects.get(pk = build_id) + file_name = build.cooker_log_path fsock = open(file_name, "r") - content_type=_mimetype_for_artifact(file_name) + content_type = MimeTypeFinder.get_mimetype(file_name) response = HttpResponse(fsock, content_type = content_type) - response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name) + disposition = 'attachment; filename=cooker.log' + response['Content-Disposition'] = disposition + return response except IOError: context = { diff --git a/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py b/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py index 343d3114c..ff93e549d 100644 --- a/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py +++ b/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist from orm.models import Build from django.db import OperationalError import os @@ -6,12 +7,16 @@ import os class Command(BaseCommand): - args = "buildId" + args = '<buildID1 buildID2 .....>' help = "Deletes selected build(s)" - def handle(self, buildId, *args, **options): - for bid in buildId.split(","): - b = Build.objects.get(pk = bid) + def handle(self, *args, **options): + for bid in args: + try: + b = Build.objects.get(pk = bid) + except ObjectDoesNotExist: + print 'build %s does not exist, skipping...' %(bid) + continue # theoretically, just b.delete() would suffice # however SQLite runs into problems when you try to # delete too many rows at once, so we delete some direct |