Merge commit 'd7e963193b4e6541206a320316a158a65f1fee89' into HEAD

16 files changed, 279 insertions, 152 deletions

diff --git a/yocto-poky/bitbake/lib/bb/cache.py b/yocto-poky/bitbake/lib/bb/cache.py
index ef4d660e8..ab09b08b5 100644
--- a/yocto-poky/bitbake/lib/bb/cache.py
+++ b/yocto-poky/bitbake/lib/bb/cache.py
@@ -528,7 +528,20 @@ class Cache(object):
 
         if hasattr(info_array[0], 'file_checksums'):
             for _, fl in info_array[0].file_checksums.items():
-                for f in fl.split():
+                fl = fl.strip()
+                while fl:
+                    # A .split() would be simpler but means spaces or colons in filenames would break
+                    a = fl.find(":True")
+                    b = fl.find(":False")
+                    if ((a < 0) and b) or ((b > 0) and (b < a)):
+                       f = fl[:b+6]
+                       fl = fl[b+7:]
+                    elif ((b < 0) and a) or ((a > 0) and (a < b)):
+                       f = fl[:a+5]
+                       fl = fl[a+6:]
+                    else:
+                       break
+                    fl = fl.strip()
                     if "*" in f:
                         continue
                     f, exist = f.split(":")
diff --git a/yocto-poky/bitbake/lib/bb/data_smart.py b/yocto-poky/bitbake/lib/bb/data_smart.py
index 79b4ed932..70558c15a 100644
--- a/yocto-poky/bitbake/lib/bb/data_smart.py
+++ b/yocto-poky/bitbake/lib/bb/data_smart.py
@@ -413,9 +413,11 @@ class DataSmart(MutableMapping):
         self.overrides = None
 
     def need_overrides(self):
-        if self.overrides is None:
-            if self.inoverride:
-                return
+        if self.overrides is not None:
+            return
+        if self.inoverride:
+            return
+        for count in range(5):
             self.inoverride = True
             # Can end up here recursively so setup dummy values
             self.overrides = []
@@ -424,6 +426,13 @@ class DataSmart(MutableMapping):
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
+            newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            if newoverrides == self.overrides:
+                break
+            self.overrides = newoverrides
+            self.overridesset = set(self.overrides)
+        else:
+            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -484,10 +493,8 @@ class DataSmart(MutableMapping):
             if '_' in var:
                 self._setvar_update_overrides(base, **loginfo)
 
-
             if base in self.overridevars:
-                self.overridevars.update(self.expandWithRefs(value, var).references)
-                self.internal_finalize(True)
+                self._setvar_update_overridevars(var, value)
             return
 
         if not var in self.dict:
@@ -520,8 +527,21 @@ class DataSmart(MutableMapping):
         self.varhistory.record(**loginfo)
 
         if var in self.overridevars:
-            self.overridevars.update(self.expandWithRefs(value, var).references)
-            self.internal_finalize(True)
+            self._setvar_update_overridevars(var, value)
+
+    def _setvar_update_overridevars(self, var, value):
+        vardata = self.expandWithRefs(value, var)
+        new = vardata.references
+        new.update(vardata.contains.keys())
+        while not new.issubset(self.overridevars):
+            nextnew = set()
+            self.overridevars.update(new)
+            for i in new:
+                vardata = self.expandWithRefs(self.getVar(i, True), i)
+                nextnew.update(vardata.references)
+                nextnew.update(vardata.contains.keys())
+            new = nextnew
+        self.internal_finalize(True)
 
     def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
@@ -628,6 +648,8 @@ class DataSmart(MutableMapping):
 
         if flag == "_defaultval" and '_' in var:
             self._setvar_update_overrides(var, **loginfo)
+        if flag == "_defaultval" and var in self.overridevars:
+            self._setvar_update_overridevars(var, value)
 
         if flag == "unexport" or flag == "export":
             if not "__exportlist" in self.dict:
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
index 3d53b63b3..288a1c8fd 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/__init__.py
@@ -464,7 +464,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
                 for k in replacements:
                     uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
                 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
-                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc])
+                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
             if loc == 2:
                 # Handle path manipulations
                 basename = None
@@ -862,7 +862,7 @@ def build_mirroruris(origud, mirrors, ld):
     replacements["BASENAME"] = origud.path.split("/")[-1]
     replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
 
-    def adduri(ud, uris, uds):
+    def adduri(ud, uris, uds, mirrors):
         for line in mirrors:
             try:
                 (find, replace) = line
@@ -876,6 +876,12 @@ def build_mirroruris(origud, mirrors, ld):
                 logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" %  (newuri))
                 continue
 
+            # Create a local copy of the mirrors minus the current line
+            # this will prevent us from recursively processing the same line
+            # as well as indirect recursion A -> B -> C -> A
+            localmirrors = list(mirrors)
+            localmirrors.remove(line)
+
             try:
                 newud = FetchData(newuri, ld)
                 newud.setup_localpath(ld)
@@ -885,16 +891,16 @@ def build_mirroruris(origud, mirrors, ld):
                 try:
                     # setup_localpath of file:// urls may fail, we should still see 
                     # if mirrors of the url exist
-                    adduri(newud, uris, uds)
+                    adduri(newud, uris, uds, localmirrors)
                 except UnboundLocalError:
                     pass
                 continue   
             uris.append(newuri)
             uds.append(newud)
 
-            adduri(newud, uris, uds)
+            adduri(newud, uris, uds, localmirrors)
 
-    adduri(origud, uris, uds)
+    adduri(origud, uris, uds, mirrors)
 
     return uris, uds
 
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/git.py b/yocto-poky/bitbake/lib/bb/fetch2/git.py
index 40658ff94..9bd87ad25 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/git.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/git.py
@@ -66,6 +66,7 @@ Supported SRC_URI options are:
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import errno
 import os
 import re
 import bb
@@ -181,8 +182,6 @@ class Git(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        ud.repochanged = not os.path.exists(ud.fullmirror)
-
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
             bb.utils.mkdirhier(ud.clonedir)
@@ -220,7 +219,11 @@ class Git(FetchMethod):
             runfetchcmd(fetch_cmd, d)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
-            ud.repochanged = True
+            try:
+                os.unlink(ud.fullmirror)
+            except OSError as exc:
+                if exc.errno != errno.ENOENT:
+                    raise
         os.chdir(ud.clonedir)
         for name in ud.names:
             if not self._contains_ref(ud, d, name):
@@ -228,7 +231,7 @@ class Git(FetchMethod):
 
     def build_mirror_data(self, ud, d):
         # Generate a mirror tarball if needed
-        if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
             # it's possible that this symlink points to read-only filesystem with PREMIRROR
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/hg.py b/yocto-poky/bitbake/lib/bb/fetch2/hg.py
index d978630ba..bbb4ed95d 100644
--- a/yocto-poky/bitbake/lib/bb/fetch2/hg.py
+++ b/yocto-poky/bitbake/lib/bb/fetch2/hg.py
@@ -163,8 +163,6 @@ class Hg(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        ud.repochanged = not os.path.exists(ud.fullmirror)
-
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
@@ -189,7 +187,11 @@ class Hg(FetchMethod):
                 logger.debug(1, "Running %s", pullcmd)
                 bb.fetch2.check_network_access(d, pullcmd, ud.url)
                 runfetchcmd(pullcmd, d)
-                ud.repochanged = True
+                try:
+                    os.unlink(ud.fullmirror)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
 
         # No source found, clone it.
         if not os.path.exists(ud.moddir):
@@ -238,7 +240,7 @@ class Hg(FetchMethod):
 
     def build_mirror_data(self, ud, d):
         # Generate a mirror tarball if needed
-        if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+        if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror):
             # it's possible that this symlink points to read-only filesystem with PREMIRROR
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
diff --git a/yocto-poky/bitbake/lib/bb/tests/fetch.py b/yocto-poky/bitbake/lib/bb/tests/fetch.py
index 1e61f3a11..94173c14a 100644
--- a/yocto-poky/bitbake/lib/bb/tests/fetch.py
+++ b/yocto-poky/bitbake/lib/bb/tests/fetch.py
@@ -405,6 +405,16 @@ class MirrorUriTest(FetcherTest):
                                 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
                                 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
 
+    recmirrorvar = "https://.*/[^/]*    http://AAAA/A/A/A/ \n" \
+                   "https://.*/[^/]*    https://BBBB/B/B/B/ \n"
+
+    def test_recursive(self):
+        fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
+        mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
+        uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
+        self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
+                                'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
+                                'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
 
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
diff --git a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
index 2d1ed5111..6e313fee8 100644
--- a/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/yocto-poky/bitbake/lib/bb/ui/buildinfohelper.py
@@ -704,7 +704,7 @@ class BuildInfoHelper(object):
     ## methods to convert event/external info into objects that the ORM layer uses
 
 
-    def _get_build_information(self):
+    def _get_build_information(self, consolelogfile):
         build_info = {}
         # Generate an identifier for each new build
 
@@ -713,7 +713,7 @@ class BuildInfoHelper(object):
         build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
         build_info['started_on'] = timezone.now()
         build_info['completed_on'] = timezone.now()
-        build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
+        build_info['cooker_log_path'] = consolelogfile
         build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
         build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
 
@@ -847,9 +847,9 @@ class BuildInfoHelper(object):
                 logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
 
 
-    def store_started_build(self, event):
+    def store_started_build(self, event, consolelogfile):
         assert '_pkgs' in vars(event)
-        build_information = self._get_build_information()
+        build_information = self._get_build_information(consolelogfile)
 
         build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
 
diff --git a/yocto-poky/bitbake/lib/bb/ui/toasterui.py b/yocto-poky/bitbake/lib/bb/ui/toasterui.py
index 9c7e87dd1..e0c278bb3 100644
--- a/yocto-poky/bitbake/lib/bb/ui/toasterui.py
+++ b/yocto-poky/bitbake/lib/bb/ui/toasterui.py
@@ -126,7 +126,7 @@ def main(server, eventHandler, params ):
             # the code will look into the protected variables of the event; no easy way around this
 
             if isinstance(event, bb.event.BuildStarted):
-                buildinfohelper.store_started_build(event)
+                buildinfohelper.store_started_build(event, consolelogfile)
 
             if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
                 buildinfohelper.update_and_store_task(event)
diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
index 3ccc7c67c..b2c573c9e 100644
--- a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
+++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
@@ -64,27 +64,6 @@ class Command(NoArgsCommand):
             return ""
         return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4))
 
-
-    def _verify_artifact_storage_dir(self):
-        # verify that we have a settings for downloading artifacts
-        while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0:
-            guessedpath = os.getcwd() + "/toaster_build_artifacts/"
-            print("\nToaster needs to know in which directory it can download build log files and other artifacts.\nToaster suggests \"%s\"." % guessedpath)
-            artifacts_storage_dir = raw_input("Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath)
-            if len(artifacts_storage_dir) == 0:
-                artifacts_storage_dir = guessedpath
-            if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"):
-                try:
-                    os.makedirs(artifacts_storage_dir)
-                except OSError as ose:
-                    if "File exists" in str(ose):
-                        pass
-                    else:
-                        raise ose
-                ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir)
-        return 0
-
-
     def _verify_build_environment(self):
         # refuse to start if we have no build environments
         while BuildEnvironment.objects.count() == 0:
@@ -239,7 +218,6 @@ class Command(NoArgsCommand):
 
     def handle_noargs(self, **options):
         retval = 0
-        retval += self._verify_artifact_storage_dir()
         retval += self._verify_build_environment()
         retval += self._verify_default_settings()
         retval += self._verify_builds_in_progress()
diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index c3e9b74c0..718e1441d 100644
--- a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -77,31 +77,11 @@ class Command(NoArgsCommand):
             bec.be.save()
 
     def archive(self):
-        ''' archives data from the builds '''
-        artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value
         for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE):
-            # save cooker log
             if br.build == None:
                 br.state = BuildRequest.REQ_FAILED
-                br.save()
-                continue
-            build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk)
-            try:
-                os.makedirs(build_artifact_storage_dir)
-            except OSError as ose:
-                if "File exists" in str(ose):
-                    pass
-                else:
-                    raise ose
-
-            file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt")
-            try:
-                with open(file_name, "w") as f:
-                    f.write(br.environment.get_artifact(br.build.cooker_log_path).read())
-            except IOError:
-                os.unlink(file_name)
-
-            br.state = BuildRequest.REQ_COMPLETED
+            else:
+                br.state = BuildRequest.REQ_COMPLETED
             br.save()
 
     def cleanup(self):
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js
index e0df46397..895e61b2a 100644
--- a/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js
+++ b/yocto-poky/bitbake/lib/toaster/toastergui/static/js/base.js
@@ -57,8 +57,8 @@ function basePageInit(ctx) {
 
   if ($(".total-builds").length !== 0){
     libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl, function(prjInfo){
-      if (prjInfo.builds)
-        $(".total-builds").text(prjInfo.builds.length);
+      if (prjInfo.completedbuilds)
+        $(".total-builds").text(prjInfo.completedbuilds.length);
     });
   }
 
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
index df809de40..27cfcd7dc 100644
--- a/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
+++ b/yocto-poky/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
@@ -16,8 +16,8 @@
 <script>
     // initialize the date range controls
     $(document).ready(function () {
-        date_init('created','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}');
-        date_init('updated','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}');
+        date_init('started_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}');
+        date_init('completed_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}');
     });
 </script>
 
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py
index 1a8b4787d..4d1549b0a 100644
--- a/yocto-poky/bitbake/lib/toaster/toastergui/tests.py
+++ b/yocto-poky/bitbake/lib/toaster/toastergui/tests.py
@@ -24,10 +24,11 @@
 from django.test import TestCase
 from django.core.urlresolvers import reverse
 from django.utils import timezone
-from orm.models import Project, Release, BitbakeVersion, Build
-from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer
+from orm.models import Project, Release, BitbakeVersion, ProjectTarget
+from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build
 from orm.models import Layer_Version, Recipe, Machine, ProjectLayer
 import json
+from bs4 import BeautifulSoup
 
 PROJECT_NAME = "test project"
 
@@ -41,7 +42,6 @@ class ViewTests(TestCase):
                                          bitbake_version=bbv)
         self.project = Project.objects.create_project(name=PROJECT_NAME,
                                                       release=release)
-
         layersrc = LayerSource.objects.create(sourcetype=LayerSource.TYPE_IMPORTED)
         self.priority = ReleaseLayerSourcePriority.objects.create(release=release,
                                                                   layer_source=layersrc)
@@ -292,3 +292,88 @@ class ProjectsPageTests(TestCase):
                         'should be a project row in the page')
         self.assertTrue(self.PROJECT_NAME in response.content,
                         'default project "cli builds" should be in page')
+
+class ProjectBuildsDisplayTest(TestCase):
+    """ Test data at /project/X/builds is displayed correctly """
+
+    def setUp(self):
+        bbv = BitbakeVersion.objects.create(name="bbv1", giturl="/tmp/",
+                                            branch="master", dirpath="")
+        release = Release.objects.create(name="release1",
+                                         bitbake_version=bbv)
+        self.project1 = Project.objects.create_project(name=PROJECT_NAME,
+                                                       release=release)
+        self.project2 = Project.objects.create_project(name=PROJECT_NAME,
+                                                       release=release)
+
+        # parameters for builds to associate with the projects
+        now = timezone.now()
+
+        self.project1_build_success = {
+            "project": self.project1,
+            "started_on": now,
+            "completed_on": now,
+            "outcome": Build.SUCCEEDED
+        }
+
+        self.project1_build_in_progress = {
+            "project": self.project1,
+            "started_on": now,
+            "completed_on": now,
+            "outcome": Build.IN_PROGRESS
+        }
+
+        self.project2_build_success = {
+            "project": self.project2,
+            "started_on": now,
+            "completed_on": now,
+            "outcome": Build.SUCCEEDED
+        }
+
+        self.project2_build_in_progress = {
+            "project": self.project2,
+            "started_on": now,
+            "completed_on": now,
+            "outcome": Build.IN_PROGRESS
+        }
+
+    def _get_rows_for_project(self, project_id):
+        url = reverse("projectbuilds", args=(project_id,))
+        response = self.client.get(url, follow=True)
+        soup = BeautifulSoup(response.content)
+        return soup.select('tr[class="data"]')
+
+    def test_show_builds_for_project(self):
+        """ Builds for a project should be displayed """
+        build1a = Build.objects.create(**self.project1_build_success)
+        build1b = Build.objects.create(**self.project1_build_success)
+        build_rows = self._get_rows_for_project(self.project1.id)
+        self.assertEqual(len(build_rows), 2)
+
+    def test_show_builds_for_project_only(self):
+        """ Builds for other projects should be excluded """
+        build1a = Build.objects.create(**self.project1_build_success)
+        build1b = Build.objects.create(**self.project1_build_success)
+        build1c = Build.objects.create(**self.project1_build_success)
+
+        # shouldn't see these two
+        build2a = Build.objects.create(**self.project2_build_success)
+        build2b = Build.objects.create(**self.project2_build_in_progress)
+
+        build_rows = self._get_rows_for_project(self.project1.id)
+        self.assertEqual(len(build_rows), 3)
+
+    def test_show_builds_exclude_in_progress(self):
+        """ "in progress" builds should not be shown """
+        build1a = Build.objects.create(**self.project1_build_success)
+        build1b = Build.objects.create(**self.project1_build_success)
+
+        # shouldn't see this one
+        build1c = Build.objects.create(**self.project1_build_in_progress)
+
+        # shouldn't see these two either, as they belong to a different project
+        build2a = Build.objects.create(**self.project2_build_success)
+        build2b = Build.objects.create(**self.project2_build_in_progress)
+
+        build_rows = self._get_rows_for_project(self.project1.id)
+        self.assertEqual(len(build_rows), 2)
\ No newline at end of file
diff --git a/yocto-poky/bitbake/lib/toaster/toastergui/views.py b/yocto-poky/bitbake/lib/toaster/toastergui/views.py
index 4e8f69e80..8689a1251 100755
--- a/yocto-poky/bitbake/lib/toaster/toastergui/views.py
+++ b/yocto-poky/bitbake/lib/toaster/toastergui/views.py
@@ -40,17 +40,26 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.http import HttpResponseBadRequest, HttpResponseNotFound
 from django.utils import timezone
 from django.utils.html import escape
-from datetime import timedelta, datetime, date
+from datetime import timedelta, datetime
 from django.utils import formats
 from toastergui.templatetags.projecttags import json as jsonfilter
 import json
 from os.path import dirname
 import itertools
 
+import magic
 import logging
 
 logger = logging.getLogger("toaster")
 
+class MimeTypeFinder(object):
+    _magic = magic.Magic(flags = magic.MAGIC_MIME_TYPE)
+
+    # returns the mimetype for a file path
+    @classmethod
+    def get_mimetype(self, path):
+        return self._magic.id_filename(path)
+
 # all new sessions should come through the landing page;
 # determine in which mode we are running in, and redirect appropriately
 def landing(request):
@@ -68,8 +77,6 @@ def landing(request):
 
     return render(request, 'landing.html', context)
 
-
-
 # returns a list for most recent builds;
 def _get_latest_builds(prj=None):
     queryset = Build.objects.all()
@@ -435,8 +442,7 @@ def _modify_date_range_filter(filter_string):
 def _add_daterange_context(queryset_all, request, daterange_list):
     # calculate the exact begining of local today and yesterday
     today_begin = timezone.localtime(timezone.now())
-    today_begin = date(today_begin.year,today_begin.month,today_begin.day)
-    yesterday_begin = today_begin-timedelta(days=1)
+    yesterday_begin = today_begin - timedelta(days=1)
     # add daterange persistent
     context_date = {}
     context_date['last_date_from'] = request.GET.get('last_date_from',timezone.localtime(timezone.now()).strftime("%d/%m/%Y"))
@@ -1890,45 +1896,87 @@ if True:
         pass
 
     # shows the "all builds" page for managed mode; it displays build requests (at least started!) instead of actual builds
+    # WARNING _build_list_helper() may raise a RedirectException, which
+    # will set the GET parameters and redirect back to the
+    # all-builds or projectbuilds page as appropriate;
+    # TODO don't use exceptions to control program flow
     @_template_renderer("builds.html")
     def builds(request):
         # define here what parameters the view needs in the GET portion in order to
         # be able to display something.  'count' and 'page' are mandatory for all views
         # that use paginators.
 
-        queryset = Build.objects.exclude(outcome = Build.IN_PROGRESS)
+        queryset = Build.objects.all()
 
-        try:
-            context, pagesize, orderby = _build_list_helper(request, queryset)
-            # all builds page as a Project column
-            context['tablecols'].append({'name': 'Project', 'clcalss': 'project_column', })
-        except RedirectException as re:
-            # rewrite the RedirectException
-            re.view = resolve(request.path_info).url_name
-            raise re
+        redirect_page = resolve(request.path_info).url_name
+
+        context, pagesize, orderby = _build_list_helper(request,
+                                                        queryset,
+                                                        redirect_page)
+        # all builds page as a Project column
+        context['tablecols'].append({
+            'name': 'Project',
+            'clclass': 'project_column'
+        })
 
         _set_parameters_values(pagesize, orderby, request)
         return context
 
 
     # helper function, to be used on "all builds" and "project builds" pages
-    def _build_list_helper(request, queryset_all):
-
+    def _build_list_helper(request, queryset_all, redirect_page, pid=None):
         default_orderby = 'completed_on:-'
         (pagesize, orderby) = _get_parameters_values(request, 10, default_orderby)
         mandatory_parameters = { 'count': pagesize,  'page' : 1, 'orderby' : orderby }
         retval = _verify_parameters( request.GET, mandatory_parameters )
         if retval:
-            raise RedirectException( None, request.GET, mandatory_parameters)
+            params = {}
+            if pid:
+                params = {'pid': pid}
+            raise RedirectException(redirect_page,
+                                    request.GET,
+                                    mandatory_parameters,
+                                    **params)
 
         # boilerplate code that takes a request for an object type and returns a queryset
         # for that object type. copypasta for all needed table searches
         (filter_string, search_term, ordering_string) = _search_tuple(request, Build)
+
         # post-process any date range filters
-        filter_string,daterange_selected = _modify_date_range_filter(filter_string)
-        queryset_all = queryset_all.select_related("project").annotate(errors_no = Count('logmessage', only=Q(logmessage__level=LogMessage.ERROR)|Q(logmessage__level=LogMessage.EXCEPTION))).annotate(warnings_no = Count('logmessage', only=Q(logmessage__level=LogMessage.WARNING))).extra(select={'timespent':'completed_on - started_on'})
-        queryset_with_search = _get_queryset(Build, queryset_all, None, search_term, ordering_string, '-completed_on')
-        queryset = _get_queryset(Build, queryset_all, filter_string, search_term, ordering_string, '-completed_on')
+        filter_string, daterange_selected = _modify_date_range_filter(filter_string)
+
+        # don't show "in progress" builds in "all builds" or "project builds"
+        queryset_all = queryset_all.exclude(outcome = Build.IN_PROGRESS)
+
+        # append project info
+        queryset_all = queryset_all.select_related("project")
+
+        # annotate with number of ERROR and EXCEPTION log messages
+        queryset_all = queryset_all.annotate(
+            errors_no = Count(
+                'logmessage',
+                only=Q(logmessage__level=LogMessage.ERROR) |
+                     Q(logmessage__level=LogMessage.EXCEPTION)
+            )
+        )
+
+        # annotate with number of warnings
+        q_warnings = Q(logmessage__level=LogMessage.WARNING)
+        queryset_all = queryset_all.annotate(
+            warnings_no = Count('logmessage', only=q_warnings)
+        )
+
+        # add timespent field
+        timespent = 'completed_on - started_on'
+        queryset_all = queryset_all.extra(select={'timespent': timespent})
+
+        queryset_with_search = _get_queryset(Build, queryset_all,
+                                             None, search_term,
+                                             ordering_string, '-completed_on')
+
+        queryset = _get_queryset(Build, queryset_all,
+                                 filter_string, search_term,
+                                 ordering_string, '-completed_on')
 
         # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display
         build_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1))
@@ -2226,7 +2274,7 @@ if True:
         context = {
             "project" : prj,
             "lvs_nos" : Layer_Version.objects.all().count(),
-            "completedbuilds": Build.objects.filter(project_id = pid).filter(outcome__lte = Build.IN_PROGRESS),
+            "completedbuilds": Build.objects.exclude(outcome = Build.IN_PROGRESS).filter(project_id = pid),
             "prj" : {"name": prj.name, },
             "buildrequests" : prj.build_set.filter(outcome=Build.IN_PROGRESS),
             "builds" : _project_recent_build_list(prj),
@@ -2632,6 +2680,10 @@ if True:
 
         return context
 
+    # WARNING _build_list_helper() may raise a RedirectException, which
+    # will set the GET parameters and redirect back to the
+    # all-builds or projectbuilds page as appropriate;
+    # TODO don't use exceptions to control program flow
     @_template_renderer('projectbuilds.html')
     def projectbuilds(request, pid):
         prj = Project.objects.get(id = pid)
@@ -2651,7 +2703,7 @@ if True:
             if 'buildDelete' in request.POST:
                 for i in request.POST['buildDelete'].strip().split(" "):
                     try:
-                        br = BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete()
+                        BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete()
                     except BuildRequest.DoesNotExist:
                         pass
 
@@ -2664,20 +2716,19 @@ if True:
                     else:
                         target = t
                         task = ""
-                    ProjectTarget.objects.create(project = prj, target = target, task = task)
-
-                br = prj.schedule_build()
+                    ProjectTarget.objects.create(project = prj,
+                                                 target = target,
+                                                 task = task)
+                prj.schedule_build()
 
+        queryset = Build.objects.filter(project_id = pid)
 
-        queryset = Build.objects.filter(outcome__lte = Build.IN_PROGRESS)
+        redirect_page = resolve(request.path_info).url_name
 
-        try:
-            context, pagesize, orderby = _build_list_helper(request, queryset)
-        except RedirectException as re:
-            # rewrite the RedirectException with our current url information
-            re.view = resolve(request.path_info).url_name
-            re.okwargs = {"pid" : pid}
-            raise re
+        context, pagesize, orderby = _build_list_helper(request,
+                                                        queryset,
+                                                        redirect_page,
+                                                        pid)
 
         context['project'] = prj
         _set_parameters_values(pagesize, orderby, request)
@@ -2710,47 +2761,17 @@ if True:
 
     def build_artifact(request, build_id, artifact_type, artifact_id):
         if artifact_type in ["cookerlog"]:
-            # these artifacts are saved after building, so they are on the server itself
-            def _mimetype_for_artifact(path):
-                try:
-                    import magic
-
-                    # fair warning: this is a mess; there are multiple competing and incompatible
-                    # magic modules floating around, so we try some of the most common combinations
-
-                    try:    # we try ubuntu's python-magic 5.4
-                        m = magic.open(magic.MAGIC_MIME_TYPE)
-                        m.load()
-                        return m.file(path)
-                    except AttributeError:
-                        pass
-
-                    try:    # we try python-magic 0.4.6
-                        m = magic.Magic(magic.MAGIC_MIME)
-                        return m.from_file(path)
-                    except AttributeError:
-                        pass
-
-                    try:    # we try pip filemagic 1.6
-                        m = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
-                        return m.id_filename(path)
-                    except AttributeError:
-                        pass
-
-                    return "binary/octet-stream"
-                except ImportError:
-                    return "binary/octet-stream"
             try:
-                # match code with runbuilds.Command.archive()
-                build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id))
-                file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt")
-
+                build = Build.objects.get(pk = build_id)
+                file_name = build.cooker_log_path
                 fsock = open(file_name, "r")
-                content_type=_mimetype_for_artifact(file_name)
+                content_type = MimeTypeFinder.get_mimetype(file_name)
 
                 response = HttpResponse(fsock, content_type = content_type)
 
-                response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name)
+                disposition = 'attachment; filename=cooker.log'
+                response['Content-Disposition'] = disposition
+
                 return response
             except IOError:
                 context = {
diff --git a/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py b/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py
index 343d3114c..ff93e549d 100644
--- a/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py
+++ b/yocto-poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py
@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand, CommandError
+from django.core.exceptions import ObjectDoesNotExist
 from orm.models import Build
 from django.db import OperationalError
 import os
@@ -6,12 +7,16 @@ import os
 
 
 class Command(BaseCommand):
-    args    = "buildId"
+    args    = '<buildID1 buildID2 .....>'
     help    = "Deletes selected build(s)"
 
-    def handle(self, buildId, *args, **options):
-        for bid in buildId.split(","):
-            b = Build.objects.get(pk = bid)
+    def handle(self, *args, **options):
+        for bid in args:
+            try:
+                b = Build.objects.get(pk = bid)
+            except ObjectDoesNotExist:
+                print 'build %s does not exist, skipping...' %(bid)
+                continue
             # theoretically, just b.delete() would suffice
             # however SQLite runs into problems when you try to
             # delete too many rows at once, so we delete some direct
diff --git a/yocto-poky/bitbake/toaster-requirements.txt b/yocto-poky/bitbake/toaster-requirements.txt
index 19b529372..c4a222155 100644
--- a/yocto-poky/bitbake/toaster-requirements.txt
+++ b/yocto-poky/bitbake/toaster-requirements.txt
@@ -2,3 +2,5 @@ Django==1.6
 South==0.8.4
 argparse==1.2.1
 wsgiref==0.1.2
+filemagic==1.6
+beautifulsoup4>=4.4.0