diff options
41 files changed, 761 insertions, 173 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index ef4d660e8..ab09b08b5 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -528,7 +528,20 @@ class Cache(object): if hasattr(info_array[0], 'file_checksums'): for _, fl in info_array[0].file_checksums.items(): - for f in fl.split(): + fl = fl.strip() + while fl: + # A .split() would be simpler but means spaces or colons in filenames would break + a = fl.find(":True") + b = fl.find(":False") + if ((a < 0) and b) or ((b > 0) and (b < a)): + f = fl[:b+6] + fl = fl[b+7:] + elif ((b < 0) and a) or ((a > 0) and (a < b)): + f = fl[:a+5] + fl = fl[a+6:] + else: + break + fl = fl.strip() if "*" in f: continue f, exist = f.split(":") diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 79b4ed932..70558c15a 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -413,9 +413,11 @@ class DataSmart(MutableMapping): self.overrides = None def need_overrides(self): - if self.overrides is None: - if self.inoverride: - return + if self.overrides is not None: + return + if self.inoverride: + return + for count in range(5): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] @@ -424,6 +426,13 @@ class DataSmart(MutableMapping): self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {} + newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] + if newoverrides == self.overrides: + break + self.overrides = newoverrides + self.overridesset = set(self.overrides) + else: + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") def initVar(self, var): self.expand_cache = {} @@ -484,10 +493,8 @@ class DataSmart(MutableMapping): if '_' in var: self._setvar_update_overrides(base, **loginfo) - if base in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) return if not var in self.dict: @@ -520,8 +527,21 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) if var in self.overridevars: - self.overridevars.update(self.expandWithRefs(value, var).references) - self.internal_finalize(True) + self._setvar_update_overridevars(var, value) + + def _setvar_update_overridevars(self, var, value): + vardata = self.expandWithRefs(value, var) + new = vardata.references + new.update(vardata.contains.keys()) + while not new.issubset(self.overridevars): + nextnew = set() + self.overridevars.update(new) + for i in new: + vardata = self.expandWithRefs(self.getVar(i, True), i) + nextnew.update(vardata.references) + nextnew.update(vardata.contains.keys()) + new = nextnew + self.internal_finalize(True) def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster @@ -628,6 +648,8 @@ class DataSmart(MutableMapping): if flag == "_defaultval" and '_' in var: self._setvar_update_overrides(var, **loginfo) + if flag == "_defaultval" and var in self.overridevars: + self._setvar_update_overridevars(var, value) if flag == "unexport" or flag == "export": if not "__exportlist" in self.dict: diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 3d53b63b3..288a1c8fd 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -464,7 +464,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d): for k in replacements: uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) - result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc]) + result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) if loc == 2: # Handle path manipulations basename = None @@ -862,7 +862,7 @@ def build_mirroruris(origud, mirrors, ld): replacements["BASENAME"] = origud.path.split("/")[-1] replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') - def adduri(ud, uris, uds): + def adduri(ud, uris, uds, mirrors): for line in mirrors: try: (find, replace) = line @@ -876,6 +876,12 @@ def build_mirroruris(origud, mirrors, ld): logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) continue + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) + try: newud = FetchData(newuri, ld) newud.setup_localpath(ld) @@ -885,16 +891,16 @@ def build_mirroruris(origud, mirrors, ld): try: # setup_localpath of file:// urls may fail, we should still see # if mirrors of the url exist - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) except UnboundLocalError: pass continue uris.append(newuri) uds.append(newud) - adduri(newud, uris, uds) + adduri(newud, uris, uds, localmirrors) - adduri(origud, uris, uds) + adduri(origud, uris, uds, mirrors) return uris, uds diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 40658ff94..9bd87ad25 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -66,6 +66,7 @@ Supported SRC_URI options are: # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import errno import os import re import bb @@ -181,8 +182,6 @@ class Git(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): bb.utils.mkdirhier(ud.clonedir) @@ -220,7 +219,11 @@ class Git(FetchMethod): runfetchcmd(fetch_cmd, d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise os.chdir(ud.clonedir) for name in ud.names: if not self._contains_ref(ud, d, name): @@ -228,7 +231,7 @@ class Git(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index d978630ba..bbb4ed95d 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -163,8 +163,6 @@ class Hg(FetchMethod): def download(self, ud, d): """Fetch url""" - ud.repochanged = not os.path.exists(ud.fullmirror) - logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") # If the checkout doesn't exist and the mirror tarball does, extract it @@ -189,7 +187,11 @@ class Hg(FetchMethod): logger.debug(1, "Running %s", pullcmd) bb.fetch2.check_network_access(d, pullcmd, ud.url) runfetchcmd(pullcmd, d) - ud.repochanged = True + try: + os.unlink(ud.fullmirror) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise # No source found, clone it. if not os.path.exists(ud.moddir): @@ -238,7 +240,7 @@ class Hg(FetchMethod): def build_mirror_data(self, ud, d): # Generate a mirror tarball if needed - if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)): + if ud.write_tarballs == "1" and not os.path.exists(ud.fullmirror): # it's possible that this symlink points to read-only filesystem with PREMIRROR if os.path.islink(ud.fullmirror): os.unlink(ud.fullmirror) diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 1e61f3a11..94173c14a 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py @@ -405,6 +405,16 @@ class MirrorUriTest(FetcherTest): 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) + recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \ + "https://.*/[^/]* https://BBBB/B/B/B/ \n" + + def test_recursive(self): + fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) + mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) + uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) + self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', + 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', + 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) class FetcherLocalTest(FetcherTest): def setUp(self): diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 2d1ed5111..6e313fee8 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -704,7 +704,7 @@ class BuildInfoHelper(object): ## methods to convert event/external info into objects that the ORM layer uses - def _get_build_information(self): + def _get_build_information(self, consolelogfile): build_info = {} # Generate an identifier for each new build @@ -713,7 +713,7 @@ class BuildInfoHelper(object): build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() - build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0] + build_info['cooker_log_path'] = consolelogfile build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] @@ -847,9 +847,9 @@ class BuildInfoHelper(object): logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) - def store_started_build(self, event): + def store_started_build(self, event, consolelogfile): assert '_pkgs' in vars(event) - build_information = self._get_build_information() + build_information = self._get_build_information(consolelogfile) build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 9c7e87dd1..e0c278bb3 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -126,7 +126,7 @@ def main(server, eventHandler, params ): # the code will look into the protected variables of the event; no easy way around this if isinstance(event, bb.event.BuildStarted): - buildinfohelper.store_started_build(event) + buildinfohelper.store_started_build(event, consolelogfile) if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): buildinfohelper.update_and_store_task(event) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 3ccc7c67c..b2c573c9e 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -64,27 +64,6 @@ class Command(NoArgsCommand): return "" return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4)) - - def _verify_artifact_storage_dir(self): - # verify that we have a settings for downloading artifacts - while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: - guessedpath = os.getcwd() + "/toaster_build_artifacts/" - print("\nToaster needs to know in which directory it can download build log files and other artifacts.\nToaster suggests \"%s\"." % guessedpath) - artifacts_storage_dir = raw_input("Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) - if len(artifacts_storage_dir) == 0: - artifacts_storage_dir = guessedpath - if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): - try: - os.makedirs(artifacts_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) - return 0 - - def _verify_build_environment(self): # refuse to start if we have no build environments while BuildEnvironment.objects.count() == 0: @@ -239,7 +218,6 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): retval = 0 - retval += self._verify_artifact_storage_dir() retval += self._verify_build_environment() retval += self._verify_default_settings() retval += self._verify_builds_in_progress() diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index c3e9b74c0..718e1441d 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -77,31 +77,11 @@ class Command(NoArgsCommand): bec.be.save() def archive(self): - ''' archives data from the builds ''' - artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): - # save cooker log if br.build == None: br.state = BuildRequest.REQ_FAILED - br.save() - continue - build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) - try: - os.makedirs(build_artifact_storage_dir) - except OSError as ose: - if "File exists" in str(ose): - pass - else: - raise ose - - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - try: - with open(file_name, "w") as f: - f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) - except IOError: - os.unlink(file_name) - - br.state = BuildRequest.REQ_COMPLETED + else: + br.state = BuildRequest.REQ_COMPLETED br.save() def cleanup(self): diff --git a/bitbake/lib/toaster/toastergui/static/js/base.js b/bitbake/lib/toaster/toastergui/static/js/base.js index e0df46397..895e61b2a 100644 --- a/bitbake/lib/toaster/toastergui/static/js/base.js +++ b/bitbake/lib/toaster/toastergui/static/js/base.js @@ -57,8 +57,8 @@ function basePageInit(ctx) { if ($(".total-builds").length !== 0){ libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl, function(prjInfo){ - if (prjInfo.builds) - $(".total-builds").text(prjInfo.builds.length); + if (prjInfo.completedbuilds) + $(".total-builds").text(prjInfo.completedbuilds.length); }); } diff --git a/bitbake/lib/toaster/toastergui/templates/projectbuilds.html b/bitbake/lib/toaster/toastergui/templates/projectbuilds.html index df809de40..27cfcd7dc 100644 --- a/bitbake/lib/toaster/toastergui/templates/projectbuilds.html +++ b/bitbake/lib/toaster/toastergui/templates/projectbuilds.html @@ -16,8 +16,8 @@ <script> // initialize the date range controls $(document).ready(function () { - date_init('created','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); - date_init('updated','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); + date_init('started_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_started_on}}','{{dateMax_started_on}}','{{daterange_selected}}'); + date_init('completed_on','{{last_date_from}}','{{last_date_to}}','{{dateMin_completed_on}}','{{dateMax_completed_on}}','{{daterange_selected}}'); }); </script> diff --git a/bitbake/lib/toaster/toastergui/tests.py b/bitbake/lib/toaster/toastergui/tests.py index 1a8b4787d..4d1549b0a 100644 --- a/bitbake/lib/toaster/toastergui/tests.py +++ b/bitbake/lib/toaster/toastergui/tests.py @@ -24,10 +24,11 @@ from django.test import TestCase from django.core.urlresolvers import reverse from django.utils import timezone -from orm.models import Project, Release, BitbakeVersion, Build -from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer +from orm.models import Project, Release, BitbakeVersion, ProjectTarget +from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build from orm.models import Layer_Version, Recipe, Machine, ProjectLayer import json +from bs4 import BeautifulSoup PROJECT_NAME = "test project" @@ -41,7 +42,6 @@ class ViewTests(TestCase): bitbake_version=bbv) self.project = Project.objects.create_project(name=PROJECT_NAME, release=release) - layersrc = LayerSource.objects.create(sourcetype=LayerSource.TYPE_IMPORTED) self.priority = ReleaseLayerSourcePriority.objects.create(release=release, layer_source=layersrc) @@ -292,3 +292,88 @@ class ProjectsPageTests(TestCase): 'should be a project row in the page') self.assertTrue(self.PROJECT_NAME in response.content, 'default project "cli builds" should be in page') + +class ProjectBuildsDisplayTest(TestCase): + """ Test data at /project/X/builds is displayed correctly """ + + def setUp(self): + bbv = BitbakeVersion.objects.create(name="bbv1", giturl="/tmp/", + branch="master", dirpath="") + release = Release.objects.create(name="release1", + bitbake_version=bbv) + self.project1 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + self.project2 = Project.objects.create_project(name=PROJECT_NAME, + release=release) + + # parameters for builds to associate with the projects + now = timezone.now() + + self.project1_build_success = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project1_build_in_progress = { + "project": self.project1, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + self.project2_build_success = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.SUCCEEDED + } + + self.project2_build_in_progress = { + "project": self.project2, + "started_on": now, + "completed_on": now, + "outcome": Build.IN_PROGRESS + } + + def _get_rows_for_project(self, project_id): + url = reverse("projectbuilds", args=(project_id,)) + response = self.client.get(url, follow=True) + soup = BeautifulSoup(response.content) + return soup.select('tr[class="data"]') + + def test_show_builds_for_project(self): + """ Builds for a project should be displayed """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2) + + def test_show_builds_for_project_only(self): + """ Builds for other projects should be excluded """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + build1c = Build.objects.create(**self.project1_build_success) + + # shouldn't see these two + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 3) + + def test_show_builds_exclude_in_progress(self): + """ "in progress" builds should not be shown """ + build1a = Build.objects.create(**self.project1_build_success) + build1b = Build.objects.create(**self.project1_build_success) + + # shouldn't see this one + build1c = Build.objects.create(**self.project1_build_in_progress) + + # shouldn't see these two either, as they belong to a different project + build2a = Build.objects.create(**self.project2_build_success) + build2b = Build.objects.create(**self.project2_build_in_progress) + + build_rows = self._get_rows_for_project(self.project1.id) + self.assertEqual(len(build_rows), 2)
\ No newline at end of file diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 4e8f69e80..8689a1251 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py @@ -40,17 +40,26 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone from django.utils.html import escape -from datetime import timedelta, datetime, date +from datetime import timedelta, datetime from django.utils import formats from toastergui.templatetags.projecttags import json as jsonfilter import json from os.path import dirname import itertools +import magic import logging logger = logging.getLogger("toaster") +class MimeTypeFinder(object): + _magic = magic.Magic(flags = magic.MAGIC_MIME_TYPE) + + # returns the mimetype for a file path + @classmethod + def get_mimetype(self, path): + return self._magic.id_filename(path) + # all new sessions should come through the landing page; # determine in which mode we are running in, and redirect appropriately def landing(request): @@ -68,8 +77,6 @@ def landing(request): return render(request, 'landing.html', context) - - # returns a list for most recent builds; def _get_latest_builds(prj=None): queryset = Build.objects.all() @@ -435,8 +442,7 @@ def _modify_date_range_filter(filter_string): def _add_daterange_context(queryset_all, request, daterange_list): # calculate the exact begining of local today and yesterday today_begin = timezone.localtime(timezone.now()) - today_begin = date(today_begin.year,today_begin.month,today_begin.day) - yesterday_begin = today_begin-timedelta(days=1) + yesterday_begin = today_begin - timedelta(days=1) # add daterange persistent context_date = {} context_date['last_date_from'] = request.GET.get('last_date_from',timezone.localtime(timezone.now()).strftime("%d/%m/%Y")) @@ -1890,45 +1896,87 @@ if True: pass # shows the "all builds" page for managed mode; it displays build requests (at least started!) instead of actual builds + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer("builds.html") def builds(request): # define here what parameters the view needs in the GET portion in order to # be able to display something. 'count' and 'page' are mandatory for all views # that use paginators. - queryset = Build.objects.exclude(outcome = Build.IN_PROGRESS) + queryset = Build.objects.all() - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - # all builds page as a Project column - context['tablecols'].append({'name': 'Project', 'clcalss': 'project_column', }) - except RedirectException as re: - # rewrite the RedirectException - re.view = resolve(request.path_info).url_name - raise re + redirect_page = resolve(request.path_info).url_name + + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page) + # all builds page as a Project column + context['tablecols'].append({ + 'name': 'Project', + 'clclass': 'project_column' + }) _set_parameters_values(pagesize, orderby, request) return context # helper function, to be used on "all builds" and "project builds" pages - def _build_list_helper(request, queryset_all): - + def _build_list_helper(request, queryset_all, redirect_page, pid=None): default_orderby = 'completed_on:-' (pagesize, orderby) = _get_parameters_values(request, 10, default_orderby) mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: - raise RedirectException( None, request.GET, mandatory_parameters) + params = {} + if pid: + params = {'pid': pid} + raise RedirectException(redirect_page, + request.GET, + mandatory_parameters, + **params) # boilerplate code that takes a request for an object type and returns a queryset # for that object type. copypasta for all needed table searches (filter_string, search_term, ordering_string) = _search_tuple(request, Build) + # post-process any date range filters - filter_string,daterange_selected = _modify_date_range_filter(filter_string) - queryset_all = queryset_all.select_related("project").annotate(errors_no = Count('logmessage', only=Q(logmessage__level=LogMessage.ERROR)|Q(logmessage__level=LogMessage.EXCEPTION))).annotate(warnings_no = Count('logmessage', only=Q(logmessage__level=LogMessage.WARNING))).extra(select={'timespent':'completed_on - started_on'}) - queryset_with_search = _get_queryset(Build, queryset_all, None, search_term, ordering_string, '-completed_on') - queryset = _get_queryset(Build, queryset_all, filter_string, search_term, ordering_string, '-completed_on') + filter_string, daterange_selected = _modify_date_range_filter(filter_string) + + # don't show "in progress" builds in "all builds" or "project builds" + queryset_all = queryset_all.exclude(outcome = Build.IN_PROGRESS) + + # append project info + queryset_all = queryset_all.select_related("project") + + # annotate with number of ERROR and EXCEPTION log messages + queryset_all = queryset_all.annotate( + errors_no = Count( + 'logmessage', + only=Q(logmessage__level=LogMessage.ERROR) | + Q(logmessage__level=LogMessage.EXCEPTION) + ) + ) + + # annotate with number of warnings + q_warnings = Q(logmessage__level=LogMessage.WARNING) + queryset_all = queryset_all.annotate( + warnings_no = Count('logmessage', only=q_warnings) + ) + + # add timespent field + timespent = 'completed_on - started_on' + queryset_all = queryset_all.extra(select={'timespent': timespent}) + + queryset_with_search = _get_queryset(Build, queryset_all, + None, search_term, + ordering_string, '-completed_on') + + queryset = _get_queryset(Build, queryset_all, + filter_string, search_term, + ordering_string, '-completed_on') # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display build_info = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) @@ -2226,7 +2274,7 @@ if True: context = { "project" : prj, "lvs_nos" : Layer_Version.objects.all().count(), - "completedbuilds": Build.objects.filter(project_id = pid).filter(outcome__lte = Build.IN_PROGRESS), + "completedbuilds": Build.objects.exclude(outcome = Build.IN_PROGRESS).filter(project_id = pid), "prj" : {"name": prj.name, }, "buildrequests" : prj.build_set.filter(outcome=Build.IN_PROGRESS), "builds" : _project_recent_build_list(prj), @@ -2632,6 +2680,10 @@ if True: return context + # WARNING _build_list_helper() may raise a RedirectException, which + # will set the GET parameters and redirect back to the + # all-builds or projectbuilds page as appropriate; + # TODO don't use exceptions to control program flow @_template_renderer('projectbuilds.html') def projectbuilds(request, pid): prj = Project.objects.get(id = pid) @@ -2651,7 +2703,7 @@ if True: if 'buildDelete' in request.POST: for i in request.POST['buildDelete'].strip().split(" "): try: - br = BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() + BuildRequest.objects.select_for_update().get(project = prj, pk = i, state__lte = BuildRequest.REQ_DELETED).delete() except BuildRequest.DoesNotExist: pass @@ -2664,20 +2716,19 @@ if True: else: target = t task = "" - ProjectTarget.objects.create(project = prj, target = target, task = task) - - br = prj.schedule_build() + ProjectTarget.objects.create(project = prj, + target = target, + task = task) + prj.schedule_build() + queryset = Build.objects.filter(project_id = pid) - queryset = Build.objects.filter(outcome__lte = Build.IN_PROGRESS) + redirect_page = resolve(request.path_info).url_name - try: - context, pagesize, orderby = _build_list_helper(request, queryset) - except RedirectException as re: - # rewrite the RedirectException with our current url information - re.view = resolve(request.path_info).url_name - re.okwargs = {"pid" : pid} - raise re + context, pagesize, orderby = _build_list_helper(request, + queryset, + redirect_page, + pid) context['project'] = prj _set_parameters_values(pagesize, orderby, request) @@ -2710,47 +2761,17 @@ if True: def build_artifact(request, build_id, artifact_type, artifact_id): if artifact_type in ["cookerlog"]: - # these artifacts are saved after building, so they are on the server itself - def _mimetype_for_artifact(path): - try: - import magic - - # fair warning: this is a mess; there are multiple competing and incompatible - # magic modules floating around, so we try some of the most common combinations - - try: # we try ubuntu's python-magic 5.4 - m = magic.open(magic.MAGIC_MIME_TYPE) - m.load() - return m.file(path) - except AttributeError: - pass - - try: # we try python-magic 0.4.6 - m = magic.Magic(magic.MAGIC_MIME) - return m.from_file(path) - except AttributeError: - pass - - try: # we try pip filemagic 1.6 - m = magic.Magic(flags=magic.MAGIC_MIME_TYPE) - return m.id_filename(path) - except AttributeError: - pass - - return "binary/octet-stream" - except ImportError: - return "binary/octet-stream" try: - # match code with runbuilds.Command.archive() - build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id)) - file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") - + build = Build.objects.get(pk = build_id) + file_name = build.cooker_log_path fsock = open(file_name, "r") - content_type=_mimetype_for_artifact(file_name) + content_type = MimeTypeFinder.get_mimetype(file_name) response = HttpResponse(fsock, content_type = content_type) - response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name) + disposition = 'attachment; filename=cooker.log' + response['Content-Disposition'] = disposition + return response except IOError: context = { diff --git a/bitbake/lib/toaster/toastermain/management/commands/builddelete.py b/bitbake/lib/toaster/toastermain/management/commands/builddelete.py index 343d3114c..ff93e549d 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/builddelete.py +++ b/bitbake/lib/toaster/toastermain/management/commands/builddelete.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist from orm.models import Build from django.db import OperationalError import os @@ -6,12 +7,16 @@ import os class Command(BaseCommand): - args = "buildId" + args = '<buildID1 buildID2 .....>' help = "Deletes selected build(s)" - def handle(self, buildId, *args, **options): - for bid in buildId.split(","): - b = Build.objects.get(pk = bid) + def handle(self, *args, **options): + for bid in args: + try: + b = Build.objects.get(pk = bid) + except ObjectDoesNotExist: + print 'build %s does not exist, skipping...' %(bid) + continue # theoretically, just b.delete() would suffice # however SQLite runs into problems when you try to # delete too many rows at once, so we delete some direct diff --git a/bitbake/toaster-requirements.txt b/bitbake/toaster-requirements.txt index 19b529372..c4a222155 100644 --- a/bitbake/toaster-requirements.txt +++ b/bitbake/toaster-requirements.txt @@ -2,3 +2,5 @@ Django==1.6 South==0.8.4 argparse==1.2.1 wsgiref==0.1.2 +filemagic==1.6 +beautifulsoup4>=4.4.0 diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass index 9ccd7d2e2..819045a3b 100644 --- a/meta/classes/autotools.bbclass +++ b/meta/classes/autotools.bbclass @@ -105,7 +105,7 @@ autotools_preconfigure() { if [ "${S}" != "${B}" ]; then echo "Previously configured separate build directory detected, cleaning ${B}" rm -rf ${B} - mkdir ${B} + mkdir -p ${B} else # At least remove the .la files since automake won't automatically # regenerate them even if CFLAGS/LDFLAGS are different diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass index fc7d64d7e..86a98bb11 100644 --- a/meta/classes/image.bbclass +++ b/meta/classes/image.bbclass @@ -106,14 +106,30 @@ python () { d.setVarFlag(var, 'func', '1') } +def fstype_variables(d): + import oe.image + + image = oe.image.Image(d) + alltypes, fstype_groups, cimages = image._get_image_types() + fstype_vars = set() + for fstype_group in fstype_groups: + for fstype in fstype_group: + fstype_vars.add('IMAGE_CMD_' + fstype) + if fstype in cimages: + for ctype in cimages[fstype]: + fstype_vars.add('COMPRESS_CMD_' + ctype) + + return sorted(fstype_vars) + def rootfs_variables(d): from oe.rootfs import variable_depends - variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPEDEP_','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', + variables = ['IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE', 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','RM_OLD_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS','SDK_OS', 'SDK_OUTPUT','SDKPATHNATIVE','SDKTARGETSYSROOT','SDK_DIR','SDK_VENDOR','SDKIMAGE_INSTALL_COMPLEMENTARY','SDK_PACKAGE_ARCHS','SDK_OUTPUT','SDKTARGETSYSROOT','MULTILIBRE_ALLOW_REP', 'MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS', 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', 'COMPRESSIONTYPES', 'IMAGE_GEN_DEBUGFS'] + variables.extend(fstype_variables(d)) variables.extend(command_variables(d)) variables.extend(variable_depends(d)) return " ".join(variables) diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py index 236195597..f9e9bfd58 100644 --- a/meta/lib/oe/image.py +++ b/meta/lib/oe/image.py @@ -172,6 +172,8 @@ class Image(ImageDepGraph): if base_size != int(base_size): base_size = int(base_size + 1) + else: + base_size = int(base_size) base_size += rootfs_alignment - 1 base_size -= base_size % rootfs_alignment diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 0fe68d4d5..a6f89b6a8 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -151,6 +151,12 @@ class oeRuntimeTest(oeTest): elif (type(self.target).__name__ == "QemuTarget"): self.assertTrue(self.target.check(), msg = "Qemu not running?") + self.setUpLocal() + + # a setup method before tests but after the class instantiation + def setUpLocal(self): + pass + def tearDown(self): # If a test fails or there is an exception if not exc_info() == (None, None, None): diff --git a/meta/lib/oeqa/selftest/sstatetests.py b/meta/lib/oeqa/selftest/sstatetests.py index 6906b2123..c4efc47fe 100644 --- a/meta/lib/oeqa/selftest/sstatetests.py +++ b/meta/lib/oeqa/selftest/sstatetests.py @@ -3,6 +3,7 @@ import unittest import os import re import shutil +import glob import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -276,6 +277,8 @@ NATIVELSBSTRING = \"DistroB\" """ The sstate checksums off allarch packages should be independent of whichever MACHINE is set. Check this using bitbake -S. + Also, rather than duplicate the test, check nativesdk stamps are the same between + the two MACHINE values. """ topdir = get_bb_var('TOPDIR') @@ -286,18 +289,20 @@ TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" MACHINE = \"qemux86\" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") - bitbake("world -S none") + bitbake("world meta-toolchain -S none") self.write_config(""" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" MACHINE = \"qemuarm\" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") - bitbake("world -S none") + bitbake("world meta-toolchain -S none") def get_files(d): f = [] for root, dirs, files in os.walk(d): for name in files: + if "meta-environment" in root or "cross-canadian" in root: + continue if "do_build" not in name: f.append(os.path.join(root, name)) return f @@ -306,3 +311,12 @@ MACHINE = \"qemuarm\" files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None self.assertItemsEqual(files1, files2) + + nativesdkdir = os.path.basename(glob.glob(topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) + + files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) + files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertItemsEqual(files1, files2) + diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index 162a88fb7..b6adcb184 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -220,3 +220,28 @@ def getAllTags(obj): ret = __gettags(obj) ret.update(__gettags(tc_method)) return ret + +def timeout_handler(seconds): + def decorator(fn): + if hasattr(signal, 'alarm'): + @wraps(fn) + def wrapped_f(self, *args, **kw): + current_frame = sys._getframe() + def raiseTimeOut(signal, frame): + if frame is not current_frame: + try: + self.target.restart() + raise TimeOut('%s seconds' % seconds) + except: + raise TimeOut('%s seconds' % seconds) + prev_handler = signal.signal(signal.SIGALRM, raiseTimeOut) + try: + signal.alarm(seconds) + return fn(self, *args, **kw) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, prev_handler) + return wrapped_f + else: + return fn + return decorator diff --git a/meta/recipes-core/glibc/glibc-ld.inc b/meta/recipes-core/glibc/glibc-ld.inc index 962d66688..c5f4db229 100644 --- a/meta/recipes-core/glibc/glibc-ld.inc +++ b/meta/recipes-core/glibc/glibc-ld.inc @@ -54,3 +54,4 @@ def glibc_dl_info(d): EGLIBC_KNOWN_INTERPRETER_NAMES = "${@glibc_dl_info(d)['ldconfig']}" RTLDLIST = "${@glibc_dl_info(d)['lddrewrite']}" +glibc_dl_info[vardepsexclude] = "OVERRIDES" diff --git a/meta/recipes-core/glibc/glibc/nscd-no-bash.patch b/meta/recipes-core/glibc/glibc/nscd-no-bash.patch new file mode 100644 index 000000000..c306ce6af --- /dev/null +++ b/meta/recipes-core/glibc/glibc/nscd-no-bash.patch @@ -0,0 +1,61 @@ +Don't use bashisms (except for echo -n, which busybox supports) to avoid needing bash to start nscd. + +Upstream-Status: Pending +Signed-off-by: Ross Burton <ross.burton@intel.com> + +diff --git a/nscd/nscd.init b/nscd/nscd.init +index a882da7..b02986e 100644 +--- a/nscd/nscd.init ++++ b/nscd/nscd.init +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/bin/sh + # + # nscd: Starts the Name Switch Cache Daemon + # +@@ -49,7 +49,7 @@ prog=nscd + start () { + [ -d /var/run/nscd ] || mkdir /var/run/nscd + [ -d /var/db/nscd ] || mkdir /var/db/nscd +- echo -n $"Starting $prog: " ++ echo -n "Starting $prog: " + daemon /usr/sbin/nscd + RETVAL=$? + echo +@@ -58,7 +58,7 @@ start () { + } + + stop () { +- echo -n $"Stopping $prog: " ++ echo -n "Stopping $prog: " + /usr/sbin/nscd -K + RETVAL=$? + if [ $RETVAL -eq 0 ]; then +@@ -67,9 +67,9 @@ stop () { + # a non-privileged user + rm -f /var/run/nscd/nscd.pid + rm -f /var/run/nscd/socket +- success $"$prog shutdown" ++ success "$prog shutdown" + else +- failure $"$prog shutdown" ++ failure "$prog shutdown" + fi + echo + return $RETVAL +@@ -103,13 +103,13 @@ case "$1" in + RETVAL=$? + ;; + force-reload | reload) +- echo -n $"Reloading $prog: " ++ echo -n "Reloading $prog: " + killproc /usr/sbin/nscd -HUP + RETVAL=$? + echo + ;; + *) +- echo $"Usage: $0 {start|stop|status|restart|reload|condrestart}" ++ echo "Usage: $0 {start|stop|status|restart|reload|condrestart}" + RETVAL=1 + ;; + esac diff --git a/meta/recipes-core/glibc/glibc_2.22.bb b/meta/recipes-core/glibc/glibc_2.22.bb index f0e1fad45..09f0428ea 100644 --- a/meta/recipes-core/glibc/glibc_2.22.bb +++ b/meta/recipes-core/glibc/glibc_2.22.bb @@ -38,6 +38,7 @@ SRC_URI = "${GLIBC_GIT_URI};branch=${BRANCH};name=glibc \ file://0025-eglibc-Install-PIC-archives.patch \ file://0026-eglibc-dl_debug_mask-is-controlled-by-__OPTION_EGLIB.patch \ file://0027-eglibc-use-option-groups-Conditionally-exclude-c-tes.patch \ + file://nscd-no-bash.patch \ " SRC_URI += "\ diff --git a/meta/recipes-core/initrdscripts/files/init-install-efi.sh b/meta/recipes-core/initrdscripts/files/init-install-efi.sh index a3ed74b98..fc4908ef9 100644 --- a/meta/recipes-core/initrdscripts/files/init-install-efi.sh +++ b/meta/recipes-core/initrdscripts/files/init-install-efi.sh @@ -114,7 +114,11 @@ rm -f /etc/udev/scripts/mount* umount ${device}* 2> /dev/null || /bin/true mkdir -p /tmp -cat /proc/mounts > /etc/mtab + +# Create /etc/mtab if not present +if [ ! -e /etc/mtab ]; then + cat /proc/mounts > /etc/mtab +fi disk_size=$(parted ${device} unit mb print | grep Disk | cut -d" " -f 3 | sed -e "s/MB//") diff --git a/meta/recipes-core/systemd/systemd_225.bb b/meta/recipes-core/systemd/systemd_225.bb index 6ac99cd63..f7d4c7df4 100644 --- a/meta/recipes-core/systemd/systemd_225.bb +++ b/meta/recipes-core/systemd/systemd_225.bb @@ -46,7 +46,7 @@ SRC_URI = "git://github.com/systemd/systemd.git;protocol=git \ file://init \ file://run-ptest \ " -SRC_URI_append_qemuall = "file://qemuall_io_latency-core-device.c-Change-the-default-device-timeout-to-2.patch" +SRC_URI_append_qemuall = " file://qemuall_io_latency-core-device.c-Change-the-default-device-timeout-to-2.patch" S = "${WORKDIR}/git" diff --git a/meta/recipes-devtools/mkelfimage/mkelfimage_git.bb b/meta/recipes-devtools/mkelfimage/mkelfimage_git.bb index 2845b8cc3..e1c33a631 100644 --- a/meta/recipes-devtools/mkelfimage/mkelfimage_git.bb +++ b/meta/recipes-devtools/mkelfimage/mkelfimage_git.bb @@ -30,6 +30,7 @@ inherit autotools-brokensep do_install_append() { rmdir ${D}${datadir}/mkelfImage/elf32-i386 rmdir ${D}${datadir}/mkelfImage + chown root:root ${D}/${sbindir}/mkelfImage } BBCLASSEXTEND = "native" diff --git a/meta/recipes-devtools/prelink/prelink_git.bb b/meta/recipes-devtools/prelink/prelink_git.bb index 6ff6917e9..79a5f5011 100644 --- a/meta/recipes-devtools/prelink/prelink_git.bb +++ b/meta/recipes-devtools/prelink/prelink_git.bb @@ -8,7 +8,7 @@ and executables, so that far fewer relocations need to be resolved at \ runtime and thus programs come up faster." LICENSE = "GPLv2" LIC_FILES_CHKSUM = "file://COPYING;md5=c93c0550bd3173f4504b2cbd8991e50b" -SRCREV = "40327fb99654e96db6ef15e2f2d5ef140ac3e998" +SRCREV = "cdee5a4dd226cc5e9f30f370067a9031f398ef3c" PV = "1.0+git${SRCPV}" # diff --git a/meta/recipes-devtools/python/python.inc b/meta/recipes-devtools/python/python.inc index e18ab8e99..4d428f3d0 100644 --- a/meta/recipes-devtools/python/python.inc +++ b/meta/recipes-devtools/python/python.inc @@ -16,7 +16,6 @@ PYTHON_MAJMIN = "2.7" inherit autotools -PYTHONLSBOPTS = "--with-wctype-functions" PYTHONLSBOPTS_linuxstdbase = "ac_cv_sizeof_off_t=8" EXTRA_OECONF = "\ diff --git a/meta/recipes-devtools/python/python_2.7.9.bb b/meta/recipes-devtools/python/python_2.7.9.bb index ae4557716..f7e2f272f 100644 --- a/meta/recipes-devtools/python/python_2.7.9.bb +++ b/meta/recipes-devtools/python/python_2.7.9.bb @@ -161,7 +161,8 @@ FILES_${PN}-dbg += "${libdir}/python${PYTHON_MAJMIN}/lib-dynload/.debug" # catch all the rest (unsorted) PACKAGES += "${PN}-misc" FILES_${PN}-misc = "${libdir}/python${PYTHON_MAJMIN}" -RDEPENDS_${PN}-ptest = "${PN}-modules ${PN}-misc" +RDEPENDS_${PN}-modules += "${PN}-misc" +RDEPENDS_${PN}-ptest = "${PN}-modules" #inherit ptest after "require python-${PYTHON_MAJMIN}-manifest.inc" so PACKAGES doesn't get overwritten inherit ptest diff --git a/meta/recipes-devtools/squashfs-tools/squashfs-tools/0001-mksquashfs.c-get-inline-functions-work-with-C99.patch b/meta/recipes-devtools/squashfs-tools/squashfs-tools/0001-mksquashfs.c-get-inline-functions-work-with-C99.patch new file mode 100644 index 000000000..a5bab0544 --- /dev/null +++ b/meta/recipes-devtools/squashfs-tools/squashfs-tools/0001-mksquashfs.c-get-inline-functions-work-with-C99.patch @@ -0,0 +1,154 @@ +From ac6268e843c43286eebff2a1052182c2393cdb2e Mon Sep 17 00:00:00 2001 +From: Roy Li <rongqing.li@windriver.com> +Date: Mon, 14 Sep 2015 12:31:42 +0800 +Subject: [PATCH] mksquashfs.c: get inline functions work with both gnu11 and gnu89 + +Upstream-Status: Pending + +After gcc upgraded to gcc5, and if the codes is compiled without optimization(-O0), +and the below error will happen: + +| mksquashfs.o: In function `create_inode': +| git/squashfs-tools/mksquashfs.c:897: undefined reference to `get_inode_no' +| git/squashfs-tools/mksquashfs.c:960: undefined reference to `get_parent_no' +| git/squashfs-tools/mksquashfs.c:983: undefined reference to `get_parent_no' +| mksquashfs.o: In function `reader_read_process': +| git/squashfs-tools/mksquashfs.c:2132: undefined reference to `is_fragment' +| mksquashfs.o: In function `reader_read_file': +| git/squashfs-tools/mksquashfs.c:2228: undefined reference to `is_fragment' +| mksquashfs.o: In function `dir_scan': +| git/squashfs-tools/mksquashfs.c:3101: undefined reference to `create_dir_entry' + +gcc5 defaults to -std=gnu11 instead of -std=gnu89, and it requires that exactly one C +source file has the callable copy of the inline function. Consider the following +program: + + inline int + foo (void) + { + return 42; + } + + int + main (void) + { + return foo (); + } + +The program above will not link with the C99 inline semantics, because no out-of-line +function foo is generated. To fix this, either mark the function foo as static, or +add the following declaration: + static inline int foo (void); + +more information refer to: https://gcc.gnu.org/gcc-5/porting_to.html; + +but the use of "extern inline" will lead to the compilation issue if gcc is not +gcc5, as the commit in oe-core d0af30c92fde [alsa-lib: Change function type to +"static __inline__"] + "extern __inline__ function()" is the inlined version that + can be used in this compilation unit, but there will be another + definition of this function somewhere, so compiler will not emit + any code for the function body. This causes problem in -O0, + where functions are never inlined, the function call is preserved, + but linker can't find the symbol, thus the error happens. + +so replace "inline" with "static inline" to make it work with both gnu11 and gnu89 + +Signed-off-by: Roy Li <rongqing.li@windriver.com> +--- + squashfs-tools/mksquashfs.c | 20 ++++++++++---------- + 1 file changed, 10 insertions(+), 10 deletions(-) + +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index d221c35..6bba1d2 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -828,13 +828,13 @@ char *subpathname(struct dir_ent *dir_ent) + } + + +-inline unsigned int get_inode_no(struct inode_info *inode) ++static inline unsigned int get_inode_no(struct inode_info *inode) + { + return inode->inode_number; + } + + +-inline unsigned int get_parent_no(struct dir_info *dir) ++static inline unsigned int get_parent_no(struct dir_info *dir) + { + return dir->depth ? get_inode_no(dir->dir_ent->inode) : inode_no; + } +@@ -2027,7 +2027,7 @@ struct file_info *duplicate(long long file_size, long long bytes, + } + + +-inline int is_fragment(struct inode_info *inode) ++static inline int is_fragment(struct inode_info *inode) + { + off_t file_size = inode->buf.st_size; + +@@ -2996,13 +2996,13 @@ struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id) + } + + +-inline struct inode_info *lookup_inode(struct stat *buf) ++static inline struct inode_info *lookup_inode(struct stat *buf) + { + return lookup_inode2(buf, 0, 0); + } + + +-inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) ++static inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) + { + if (inode->inode_number == 0) { + inode->inode_number = use_this ? : inode_no ++; +@@ -3013,7 +3013,7 @@ inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) + } + + +-inline struct dir_ent *create_dir_entry(char *name, char *source_name, ++static inline struct dir_ent *create_dir_entry(char *name, char *source_name, + char *nonstandard_pathname, struct dir_info *dir) + { + struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent)); +@@ -3031,7 +3031,7 @@ inline struct dir_ent *create_dir_entry(char *name, char *source_name, + } + + +-inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, ++static inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, + struct inode_info *inode_info) + { + struct dir_info *dir = dir_ent->our_dir; +@@ -3047,7 +3047,7 @@ inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, + } + + +-inline void add_dir_entry2(char *name, char *source_name, ++static inline void add_dir_entry2(char *name, char *source_name, + char *nonstandard_pathname, struct dir_info *sub_dir, + struct inode_info *inode_info, struct dir_info *dir) + { +@@ -3059,7 +3059,7 @@ inline void add_dir_entry2(char *name, char *source_name, + } + + +-inline void free_dir_entry(struct dir_ent *dir_ent) ++static inline void free_dir_entry(struct dir_ent *dir_ent) + { + if(dir_ent->name) + free(dir_ent->name); +@@ -3080,7 +3080,7 @@ inline void free_dir_entry(struct dir_ent *dir_ent) + } + + +-inline void add_excluded(struct dir_info *dir) ++static inline void add_excluded(struct dir_info *dir) + { + dir->excluded ++; + } +-- +1.9.1 + diff --git a/meta/recipes-devtools/squashfs-tools/squashfs-tools_git.bb b/meta/recipes-devtools/squashfs-tools/squashfs-tools_git.bb index 497b28207..7aebd00e0 100644 --- a/meta/recipes-devtools/squashfs-tools/squashfs-tools_git.bb +++ b/meta/recipes-devtools/squashfs-tools/squashfs-tools_git.bb @@ -12,6 +12,7 @@ PV = "4.3+gitr${SRCPV}" SRCREV = "9c1db6d13a51a2e009f0027ef336ce03624eac0d" SRC_URI = "git://github.com/plougher/squashfs-tools.git;protocol=https \ http://downloads.sourceforge.net/sevenzip/lzma465.tar.bz2;name=lzma \ + file://0001-mksquashfs.c-get-inline-functions-work-with-C99.patch;striplevel=2 \ " SRC_URI[lzma.md5sum] = "29d5ffd03a5a3e51aef6a74e9eafb759" SRC_URI[lzma.sha256sum] = "c935fd04dd8e0e8c688a3078f3675d699679a90be81c12686837e0880aa0fa1e" diff --git a/meta/recipes-gnome/gnome/adwaita-icon-theme_3.16.2.1.bb b/meta/recipes-gnome/gnome/adwaita-icon-theme_3.16.2.1.bb index 526e699e9..0d7fa0cd6 100644 --- a/meta/recipes-gnome/gnome/adwaita-icon-theme_3.16.2.1.bb +++ b/meta/recipes-gnome/gnome/adwaita-icon-theme_3.16.2.1.bb @@ -27,6 +27,10 @@ do_install_append() { PACKAGES = "${PN}-cursors ${PN}-symbolic ${PN}-hires ${PN}" +RREPLACES_${PN} = "gnome-icon-theme" +RCONFLICTS_${PN} = "gnome-icon-theme" +RPROVIDES_${PN} = "gnome-icon-theme" + FILES_${PN}-cursors = "${prefix}/share/icons/Adwaita/cursors/" FILES_${PN}-symbolic = "${prefix}/share/icons/Adwaita/*/*/*.symbolic.png" FILES_${PN}-hires = "${prefix}/share/icons/Adwaita/256x256/" diff --git a/meta/recipes-kernel/cryptodev/cryptodev-tests_1.7.bb b/meta/recipes-kernel/cryptodev/cryptodev-tests_1.7.bb index efc41ae42..be59a4af2 100644 --- a/meta/recipes-kernel/cryptodev/cryptodev-tests_1.7.bb +++ b/meta/recipes-kernel/cryptodev/cryptodev-tests_1.7.bb @@ -9,7 +9,7 @@ file://0001-Add-the-compile-and-install-rules-for-cryptodev-test.patch \ file://0002-Fix-tests-Makefile-usage-of-LDLIBS-vs.-LDFLAGS.patch \ " -EXTRA_OEMAKE='KERNEL_DIR="${STAGING_KERNEL_DIR}" PREFIX="${D}"' +EXTRA_OEMAKE='KERNEL_DIR="${STAGING_EXECPREFIXDIR}" PREFIX="${D}"' do_compile() { oe_runmake testprogs diff --git a/meta/recipes-kernel/kmod/kmod_git.bb b/meta/recipes-kernel/kmod/kmod_git.bb index ba4d85eda..e0bb95c8f 100644 --- a/meta/recipes-kernel/kmod/kmod_git.bb +++ b/meta/recipes-kernel/kmod/kmod_git.bb @@ -21,9 +21,9 @@ do_install_append () { install -dm755 ${D}${base_bindir} install -dm755 ${D}${base_sbindir} # add symlinks to kmod - ln -s ..${base_bindir}/kmod ${D}${base_bindir}/lsmod + lnr ${D}${base_bindir}/kmod ${D}${base_bindir}/lsmod for tool in insmod rmmod depmod modinfo modprobe; do - ln -s ..${base_bindir}/kmod ${D}${base_sbindir}/${tool} + lnr ${D}${base_bindir}/kmod ${D}${base_sbindir}/${tool} done # configuration directories install -dm755 ${D}${base_libdir}/depmod.d diff --git a/meta/recipes-kernel/linux-firmware/linux-firmware_git.bb b/meta/recipes-kernel/linux-firmware/linux-firmware_git.bb index ef8117c9b..4939ca67c 100644 --- a/meta/recipes-kernel/linux-firmware/linux-firmware_git.bb +++ b/meta/recipes-kernel/linux-firmware/linux-firmware_git.bb @@ -168,7 +168,7 @@ PACKAGES =+ "${PN}-ralink-license ${PN}-ralink \ ${PN}-ti-connectivity-license ${PN}-wl12xx ${PN}-wl18xx \ ${PN}-vt6656-license ${PN}-vt6656 \ ${PN}-rtl-license ${PN}-rtl8192cu ${PN}-rtl8192ce ${PN}-rtl8192su \ - ${PN}-broadcom-license ${PN}-bcm4329 ${PN}-bcm4330 ${PN}-bcm4334 \ + ${PN}-broadcom-license ${PN}-bcm4329 ${PN}-bcm4330 ${PN}-bcm4334 ${PN}-bcm4354 \ ${PN}-atheros-license ${PN}-ar9170 ${PN}-ar3k ${PN}-ath6k ${PN}-ath9k \ \ ${PN}-iwlwifi-license ${PN}-iwlwifi-135-6 \ @@ -311,6 +311,7 @@ RDEPENDS_${PN}-vt6656 = "${PN}-vt6656-license" LICENSE_${PN}-bcm4329 = "Firmware-broadcom_bcm43xx" LICENSE_${PN}-bcm4330 = "Firmware-broadcom_bcm43xx" LICENSE_${PN}-bcm4334 = "Firmware-broadcom_bcm43xx" +LICENSE_${PN}-bcm4354 = "Firmware-broadcom_bcm43xx" FILES_${PN}-broadcom-license = " \ /lib/firmware/LICENCE.broadcom_bcm43xx \ @@ -324,11 +325,16 @@ FILES_${PN}-bcm4330 = " \ FILES_${PN}-bcm4334 = " \ /lib/firmware/brcm/brcmfmac4334-sdio.bin \ " +FILES_${PN}-bcm4354 = " \ + /lib/firmware/brcm/brcmfmac4354-sdio.bin \ +" ALTERNATIVE_LINK_NAME[brcmfmac-sdio.bin] = "/lib/firmware/brcm/brcmfmac-sdio.bin" ALTERNATIVE_linux-firmware-bcm4334 = "brcmfmac-sdio.bin" ALTERNATIVE_TARGET_linux-firmware-bcm4334[brcmfmac-sdio.bin] = "/lib/firmware/brcm/brcmfmac4334-sdio.bin" +ALTERNATIVE_linux_firmware-bcm4354 = "brcmfmac-sdio.bin" +ALTERNATIVE_TARGET_linux-firmware-bcm4354[brcmfmac-sdio.bin] = "/lib/firmware/brcm/brcmfmac4354-sdio.bin" ALTERNATIVE_linux-firmware-bcm4329 = "brcmfmac-sdio.bin" ALTERNATIVE_TARGET_linux-firmware-bcm4329[brcmfmac-sdio.bin] = "/lib/firmware/brcm/brcmfmac4329-sdio.bin" ALTERNATIVE_linux-firmware-bcm4330 = "brcmfmac-sdio.bin" @@ -337,6 +343,7 @@ ALTERNATIVE_TARGET_linux-firmware-bcm4330[brcmfmac-sdio.bin] = "/lib/firmware/br RDEPENDS_${PN}-bcm4329 += "${PN}-broadcom-license" RDEPENDS_${PN}-bcm4330 += "${PN}-broadcom-license" RDEPENDS_${PN}-bcm4334 += "${PN}-broadcom-license" +RDEPENDS_${PN}-bcm4354 += "${PN}-broadcom-license" # For iwlwifi LICENSE_${PN}-iwlwifi-135-6 = "Firmware-iwlwifi_firmware" diff --git a/meta/recipes-kernel/perf/perf.bb b/meta/recipes-kernel/perf/perf.bb index adb3a2cdc..22bd3c820 100644 --- a/meta/recipes-kernel/perf/perf.bb +++ b/meta/recipes-kernel/perf/perf.bb @@ -120,7 +120,7 @@ do_install() { do_configure_prepend () { # Fix for rebuilding rm -rf ${B}/ - mkdir ${B}/ + mkdir -p ${B}/ # If building a multlib based perf, the incorrect library path will be # detected by perf, since it triggers via: ifeq ($(ARCH),x86_64). In a 32 bit @@ -148,7 +148,7 @@ do_configure_prepend () { ${S}/tools/perf/Makefile.perf fi sed -i -e "s,--root='/\$(DESTDIR_SQ)',--prefix='\$(DESTDIR_SQ)/usr' --install-lib='\$(DESTDIR)\$(PYTHON_SITEPACKAGES_DIR)',g" \ - ${S}/tools/perf/Makefile + ${S}/tools/perf/Makefile* if [ -e "${S}/tools/build/Makefile.build" ]; then sed -i -e 's,\ .config-detected, $(OUTPUT)/config-detected,g' \ diff --git a/meta/recipes-support/nspr/nspr_4.10.8.bb b/meta/recipes-support/nspr/nspr_4.10.8.bb index 944994e17..bc6001888 100644 --- a/meta/recipes-support/nspr/nspr_4.10.8.bb +++ b/meta/recipes-support/nspr/nspr_4.10.8.bb @@ -5,7 +5,7 @@ LIC_FILES_CHKSUM = "file://configure.in;beginline=3;endline=6;md5=90c2fdee38e45d file://Makefile.in;beginline=4;endline=38;md5=beda1dbb98a515f557d3e58ef06bca99" SECTION = "libs/network" -SRC_URI = "ftp://ftp.mozilla.org/pub/mozilla.org/nspr/releases/v${PV}/src/nspr-${PV}.tar.gz \ +SRC_URI = "http://ftp.mozilla.org/pub/nspr/releases/v${PV}/src/nspr-${PV}.tar.gz \ file://remove-rpath-from-tests.patch \ file://fix-build-on-x86_64.patch \ file://remove-srcdir-from-configure-in.patch \ diff --git a/scripts/contrib/mkefidisk.sh b/scripts/contrib/mkefidisk.sh index 55f72b0f5..cd4de0533 100755 --- a/scripts/contrib/mkefidisk.sh +++ b/scripts/contrib/mkefidisk.sh @@ -152,8 +152,55 @@ unmount() { # Parse and validate arguments # if [ $# -lt 3 ] || [ $# -gt 4 ]; then - usage - exit 1 + if [ $# -eq 1 ]; then + AVAILABLE_DISK=`lsblk | grep "disk" | cut -f 1 -d " "` + X=0 + for disk in `echo $AVAILABLE_DISK`; do + mounted=`lsblk /dev/$disk | awk {'print $7'} | sed "s/MOUNTPOINT//"` + if [ -z "$mounted" ]; then + UNMOUNTED_AVAILABLES="$UNMOUNTED_AVAILABLES /dev/$disk" + info "$X - /dev/$disk" + X=`expr $X + 1` + fi + done + if [ $X -eq 0 ]; then + die "No unmounted device found." + fi + read -p "Choose unmounted device number: " DISK_NUMBER + X=0 + for line in `echo $UNMOUNTED_AVAILABLES`; do + if [ $DISK_NUMBER -eq $X ]; then + DISK_TO_BE_FLASHED=$line + break + else + X=`expr $X + 1` + fi + done + if [ -z "$DISK_TO_BE_FLASHED" ]; then + die "Option \"$DISK_NUMBER\" is invalid. Choose a valid option" + else + if [ -z `echo $DISK_TO_BE_FLASHED | grep "mmc"` ]; then + TARGET_TO_BE_BOOT="/dev/sda" + else + TARGET_TO_BE_BOOT="/dev/mmcblk0" + fi + fi + echo "" + echo "Choose a name of the device that will be boot from" + echo -n "Recommended name is: " + info "$TARGET_TO_BE_BOOT" + read -p "Is target device okay? [y/N]: " RESPONSE + if [ "$RESPONSE" != "y" ]; then + read -p "Choose target device name: " TARGET_TO_BE_BOOT + fi + echo "" + if [ -z "$TARGET_TO_BE_BOOT" ]; then + die "Error: choose a valid target name" + fi + else + usage + exit 1 + fi fi if [ "$1" = "-v" ]; then @@ -162,9 +209,15 @@ if [ "$1" = "-v" ]; then shift fi -DEVICE=$1 -HDDIMG=$2 -TARGET_DEVICE=$3 +if [ -z "$AVAILABLE_DISK" ]; then + DEVICE=$1 + HDDIMG=$2 + TARGET_DEVICE=$3 +else + DEVICE=$DISK_TO_BE_FLASHED + HDDIMG=$1 + TARGET_DEVICE=$TARGET_TO_BE_BOOT +fi LINK=$(readlink $DEVICE) if [ $? -eq 0 ]; then diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py new file mode 100644 index 000000000..77b74cb73 --- /dev/null +++ b/scripts/lib/recipetool/newappend.py @@ -0,0 +1,111 @@ +# Recipe creation tool - newappend plugin +# +# This sub-command creates a bbappend for the specified target and prints the +# path to the bbappend. +# +# Example: recipetool newappend meta-mylayer busybox +# +# Copyright (C) 2015 Christopher Larson <kergoth@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import argparse +import errno +import logging +import os +import re +import sys + + +logger = logging.getLogger('recipetool') +tinfoil = None + + +def plugin_init(pluginlist): + # Don't need to do anything here right now, but plugins must have this function defined + pass + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +def _provide_to_pn(cooker, provide): + """Get the name of the preferred recipe for the specified provide.""" + import bb.providers + filenames = cooker.recipecache.providers[provide] + eligible, foundUnique = bb.providers.filterProviders(filenames, provide, cooker.expanded_data, cooker.recipecache) + filename = eligible[0] + pn = cooker.recipecache.pkg_fn[filename] + return pn + + +def _get_recipe_file(cooker, pn): + import oe.recipeutils + recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) + if not recipefile: + skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) + if skipreasons: + logger.error('\n'.join(skipreasons)) + else: + logger.error("Unable to find any recipe file matching %s" % pn) + return recipefile + + +def layer(layerpath): + if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')): + raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath)) + return layerpath + + +def newappend(args): + import oe.recipeutils + + pn = _provide_to_pn(tinfoil.cooker, args.target) + recipe_path = _get_recipe_file(tinfoil.cooker, pn) + + rd = tinfoil.config_data.createCopy() + rd.setVar('FILE', recipe_path) + append_path, path_ok = oe.recipeutils.get_bbappend_path(rd, args.destlayer, args.wildcard_version) + if not append_path: + logger.error('Unable to determine layer directory containing %s', recipe_path) + return 1 + + if not path_ok: + logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)) + + layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()] + if not os.path.abspath(args.destlayer) in layerdirs: + logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') + + if not os.path.exists(append_path): + bb.utils.mkdirhier(os.path.dirname(append_path)) + + try: + open(append_path, 'a') + except (OSError, IOError) as exc: + logger.critical(str(exc)) + return 1 + + print(append_path) + + +def register_command(subparsers): + parser = subparsers.add_parser('newappend', + help='Create a bbappend for the specified target in the specified layer') + parser.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true') + parser.add_argument('destlayer', help='Base directory of the destination layer to write the bbappend to', type=layer) + parser.add_argument('target', help='Target recipe/provide to append') + parser.set_defaults(func=newappend, parserecipes=True) |