summaryrefslogtreecommitdiff
path: root/poky/bitbake/lib/bb
diff options
context:
space:
mode:
Diffstat (limited to 'poky/bitbake/lib/bb')
-rw-r--r--poky/bitbake/lib/bb/__init__.py2
-rw-r--r--poky/bitbake/lib/bb/build.py17
-rw-r--r--poky/bitbake/lib/bb/cache.py3
-rw-r--r--poky/bitbake/lib/bb/codeparser.py6
-rw-r--r--poky/bitbake/lib/bb/cooker.py34
-rw-r--r--poky/bitbake/lib/bb/event.py31
-rw-r--r--poky/bitbake/lib/bb/fetch2/__init__.py4
-rw-r--r--poky/bitbake/lib/bb/fetch2/az.py93
-rw-r--r--poky/bitbake/lib/bb/providers.py84
-rw-r--r--poky/bitbake/lib/bb/runqueue.py67
-rw-r--r--poky/bitbake/lib/bb/tests/color.py2
-rw-r--r--poky/bitbake/lib/bb/tests/fetch.py2
-rw-r--r--poky/bitbake/lib/bb/tinfoil.py2
13 files changed, 282 insertions, 65 deletions
diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py
index 84a9051c1..afce5ccb8 100644
--- a/poky/bitbake/lib/bb/__init__.py
+++ b/poky/bitbake/lib/bb/__init__.py
@@ -58,7 +58,7 @@ class BBLoggerMixin(object):
if not bb.event.worker_pid:
if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
return
- if loglevel > bb.msg.loggerDefaultLogLevel:
+ if loglevel < bb.msg.loggerDefaultLogLevel:
return
return self.log(loglevel, msg, *args, **kwargs)
diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py
index f4f897e41..b2715fc53 100644
--- a/poky/bitbake/lib/bb/build.py
+++ b/poky/bitbake/lib/bb/build.py
@@ -854,6 +854,23 @@ def make_stamp(task, d, file_name = None):
file_name = d.getVar('BB_FILENAME')
bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
+def find_stale_stamps(task, d, file_name=None):
+ current = stamp_internal(task, d, file_name)
+ current2 = stamp_internal(task + "_setscene", d, file_name)
+ cleanmask = stamp_cleanmask_internal(task, d, file_name)
+ found = []
+ for mask in cleanmask:
+ for name in glob.glob(mask):
+ if "sigdata" in name or "sigbasedata" in name:
+ continue
+ if name.endswith('.taint'):
+ continue
+ if name == current or name == current2:
+ continue
+ logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2))
+ found.append(name)
+ return found
+
def del_stamp(task, d, file_name = None):
"""
Removes a stamp for a given task
diff --git a/poky/bitbake/lib/bb/cache.py b/poky/bitbake/lib/bb/cache.py
index aea2b8bc1..27eb27179 100644
--- a/poky/bitbake/lib/bb/cache.py
+++ b/poky/bitbake/lib/bb/cache.py
@@ -126,6 +126,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
self.inherits = self.getvar('__inherit_cache', metadata, expand=False)
self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
+ self.fakerootlogs = self.getvar('FAKEROOTLOGS', metadata)
self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata)
self.extradepsfunc = self.getvar('calculate_extra_depends', metadata)
@@ -163,6 +164,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
cachedata.fakerootenv = {}
cachedata.fakerootnoenv = {}
cachedata.fakerootdirs = {}
+ cachedata.fakerootlogs = {}
cachedata.extradepsfunc = {}
def add_cacheData(self, cachedata, fn):
@@ -231,6 +233,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
cachedata.fakerootenv[fn] = self.fakerootenv
cachedata.fakerootnoenv[fn] = self.fakerootnoenv
cachedata.fakerootdirs[fn] = self.fakerootdirs
+ cachedata.fakerootlogs[fn] = self.fakerootlogs
cachedata.extradepsfunc[fn] = self.extradepsfunc
def virtualfn2realfn(virtualfn):
diff --git a/poky/bitbake/lib/bb/codeparser.py b/poky/bitbake/lib/bb/codeparser.py
index 25a7ac69d..0cec452c0 100644
--- a/poky/bitbake/lib/bb/codeparser.py
+++ b/poky/bitbake/lib/bb/codeparser.py
@@ -212,9 +212,9 @@ class PythonParser():
funcstr = codegen.to_source(func)
argstr = codegen.to_source(arg)
except TypeError:
- self.log.debug(2, 'Failed to convert function and argument to source form')
+ self.log.debug2('Failed to convert function and argument to source form')
else:
- self.log.debug(1, self.unhandled_message % (funcstr, argstr))
+ self.log.debug(self.unhandled_message % (funcstr, argstr))
def visit_Call(self, node):
name = self.called_node_name(node.func)
@@ -450,7 +450,7 @@ class ShellParser():
cmd = word[1]
if cmd.startswith("$"):
- self.log.debug(1, self.unhandled_template % cmd)
+ self.log.debug(self.unhandled_template % cmd)
elif cmd == "eval":
command = " ".join(word for _, word in words[1:])
self._parse_shell(command)
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py
index f4ab797ed..39e10e613 100644
--- a/poky/bitbake/lib/bb/cooker.py
+++ b/poky/bitbake/lib/bb/cooker.py
@@ -502,22 +502,30 @@ class BBCooker:
def showVersions(self):
- (latest_versions, preferred_versions) = self.findProviders()
+ (latest_versions, preferred_versions, required) = self.findProviders()
- logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
- logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
+ logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
+ logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
for p in sorted(self.recipecaches[''].pkg_pn):
- pref = preferred_versions[p]
+ preferred = preferred_versions[p]
latest = latest_versions[p]
+ requiredstr = ""
+ preferredstr = ""
+ if required[p]:
+ if preferred[0] is not None:
+ requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
+ else:
+ bb.fatal("REQUIRED_VERSION of package %s not available" % p)
+ else:
+ preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
- prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
- if pref == latest:
- prefstr = ""
+ if preferred == latest:
+ preferredstr = ""
- logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
+ logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
def showEnvironment(self, buildfile=None, pkgs_to_build=None):
"""
@@ -1063,10 +1071,16 @@ class BBCooker:
if pn in self.recipecaches[mc].providers:
filenames = self.recipecaches[mc].providers[pn]
eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
- filename = eligible[0]
+ if eligible is not None:
+ filename = eligible[0]
+ else:
+ filename = None
return None, None, None, filename
elif pn in self.recipecaches[mc].pkg_pn:
- return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+ (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+ if required and preferred_file is None:
+ return None, None, None, None
+ return (latest, latest_f, preferred_ver, preferred_file)
else:
return None, None, None, None
diff --git a/poky/bitbake/lib/bb/event.py b/poky/bitbake/lib/bb/event.py
index 23e1f3187..0454c7533 100644
--- a/poky/bitbake/lib/bb/event.py
+++ b/poky/bitbake/lib/bb/event.py
@@ -118,7 +118,7 @@ def fire_class_handlers(event, d):
if _eventfilter:
if not _eventfilter(name, handler, event, d):
continue
- if d and not name in (d.getVar("__BBHANDLERS_MC") or []):
+ if d is not None and not name in (d.getVar("__BBHANDLERS_MC") or set()):
continue
execute_handler(name, handler, event, d)
@@ -232,12 +232,16 @@ noop = lambda _: None
def register(name, handler, mask=None, filename=None, lineno=None, data=None):
"""Register an Event handler"""
- if data and data.getVar("BB_CURRENT_MC"):
+ if data is not None and data.getVar("BB_CURRENT_MC"):
mc = data.getVar("BB_CURRENT_MC")
name = '%s%s' % (mc.replace('-', '_'), name)
# already registered
if name in _handlers:
+ if data is not None:
+ bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+ bbhands_mc.add(name)
+ data.setVar("__BBHANDLERS_MC", bbhands_mc)
return AlreadyRegistered
if handler is not None:
@@ -274,16 +278,16 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None):
_event_handler_map[m] = {}
_event_handler_map[m][name] = True
- if data:
- bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
- bbhands_mc.append(name)
+ if data is not None:
+ bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+ bbhands_mc.add(name)
data.setVar("__BBHANDLERS_MC", bbhands_mc)
return Registered
def remove(name, handler, data=None):
"""Remove an Event handler"""
- if data:
+ if data is not None:
if data.getVar("BB_CURRENT_MC"):
mc = data.getVar("BB_CURRENT_MC")
name = '%s%s' % (mc.replace('-', '_'), name)
@@ -295,8 +299,8 @@ def remove(name, handler, data=None):
if name in _event_handler_map[event]:
_event_handler_map[event].pop(name)
- if data:
- bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
+ if data is not None:
+ bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
if name in bbhands_mc:
bbhands_mc.remove(name)
data.setVar("__BBHANDLERS_MC", bbhands_mc)
@@ -666,6 +670,17 @@ class ReachableStamps(Event):
Event.__init__(self)
self.stamps = stamps
+class StaleSetSceneTasks(Event):
+ """
+ An event listing setscene tasks which are 'stale' and will
+ be rerun. The metadata may use to clean up stale data.
+ tasks is a mapping of tasks and matching stale stamps.
+ """
+
+ def __init__(self, tasks):
+ Event.__init__(self)
+ self.tasks = tasks
+
class FilesMatchingFound(Event):
"""
Event when a list of files matching the supplied pattern has
diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py
index 19169d780..cf0201c49 100644
--- a/poky/bitbake/lib/bb/fetch2/__init__.py
+++ b/poky/bitbake/lib/bb/fetch2/__init__.py
@@ -1243,7 +1243,7 @@ class FetchData(object):
if checksum_name in self.parm:
checksum_expected = self.parm[checksum_name]
- elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+ elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]:
checksum_expected = None
else:
checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
@@ -1908,6 +1908,7 @@ from . import repo
from . import clearcase
from . import npm
from . import npmsw
+from . import az
methods.append(local.Local())
methods.append(wget.Wget())
@@ -1927,3 +1928,4 @@ methods.append(repo.Repo())
methods.append(clearcase.ClearCase())
methods.append(npm.Npm())
methods.append(npmsw.NpmShrinkWrap())
+methods.append(az.Az())
diff --git a/poky/bitbake/lib/bb/fetch2/az.py b/poky/bitbake/lib/bb/fetch2/az.py
new file mode 100644
index 000000000..3ccc594c2
--- /dev/null
+++ b/poky/bitbake/lib/bb/fetch2/az.py
@@ -0,0 +1,93 @@
+"""
+BitBake 'Fetch' Azure Storage implementation
+
+"""
+
+# Copyright (C) 2021 Alejandro Hernandez Samaniego
+#
+# Based on bb.fetch2.wget:
+# Copyright (C) 2003, 2004 Chris Larson
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import shlex
+import os
+import bb
+from bb.fetch2 import FetchError
+from bb.fetch2 import logger
+from bb.fetch2.wget import Wget
+
+
+class Az(Wget):
+
+ def supports(self, ud, d):
+ """
+ Check to see if a given url can be fetched from Azure Storage
+ """
+ return ud.type in ['az']
+
+
+ def checkstatus(self, fetch, ud, d, try_again=True):
+
+ # checkstatus discards parameters either way, we need to do this before adding the SAS
+ ud.url = ud.url.replace('az://','https://').split(';')[0]
+
+ az_sas = d.getVar('AZ_SAS')
+ if az_sas and az_sas not in ud.url:
+ ud.url += az_sas
+
+ return Wget.checkstatus(self, fetch, ud, d, try_again)
+
+ # Override download method, include retries
+ def download(self, ud, d, retries=3):
+ """Fetch urls"""
+
+ # If were reaching the account transaction limit we might be refused a connection,
+ # retrying allows us to avoid false negatives since the limit changes over time
+ fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+
+ # We need to provide a localpath to avoid wget using the SAS
+ # ud.localfile either has the downloadfilename or ud.path
+ localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+ bb.utils.mkdirhier(os.path.dirname(localpath))
+ fetchcmd += " -O %s" % shlex.quote(localpath)
+
+
+ if ud.user and ud.pswd:
+ fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+
+ # Check if a Shared Access Signature was given and use it
+ az_sas = d.getVar('AZ_SAS')
+
+ if az_sas:
+ azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
+ else:
+ azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+
+ if os.path.exists(ud.localpath):
+ # file exists, but we didnt complete it.. trying again.
+ fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+ else:
+ fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+
+ try:
+ self._runwget(ud, d, fetchcmd, False)
+ except FetchError as e:
+ # Azure fails on handshake sometimes when using wget after some stress, producing a
+ # FetchError from the fetcher, if the artifact exists retyring should succeed
+ if 'Unable to establish SSL connection' in str(e):
+ logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
+ self.download(ud, d, retries -1)
+
+ # Sanity check since wget can pretend it succeed when it didn't
+ # Also, this used to happen if sourceforge sent us to the mirror page
+ if not os.path.exists(ud.localpath):
+ raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
+
+ if os.path.getsize(ud.localpath) == 0:
+ os.remove(ud.localpath)
+ raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
+
+ return True
diff --git a/poky/bitbake/lib/bb/providers.py b/poky/bitbake/lib/bb/providers.py
index b5a6cd009..3ec11a40e 100644
--- a/poky/bitbake/lib/bb/providers.py
+++ b/poky/bitbake/lib/bb/providers.py
@@ -38,16 +38,17 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
localdata = data.createCopy(cfgData)
bb.data.expandKeys(localdata)
+ required = {}
preferred_versions = {}
latest_versions = {}
for pn in pkg_pn:
- (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+ (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
preferred_versions[pn] = (pref_ver, pref_file)
latest_versions[pn] = (last_ver, last_file)
+ required[pn] = req
- return (latest_versions, preferred_versions)
-
+ return (latest_versions, preferred_versions, required)
def allProviders(dataCache):
"""
@@ -59,7 +60,6 @@ def allProviders(dataCache):
all_providers[pn].append((ver, fn))
return all_providers
-
def sortPriorities(pn, dataCache, pkg_pn = None):
"""
Reorder pkg_pn by file priority and default preference
@@ -87,6 +87,21 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
return tmp_pn
+def versionVariableMatch(cfgData, keyword, pn):
+ """
+ Return the value of the <keyword>_VERSION variable if set.
+ """
+
+ # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
+ # hence we do this manually rather than use OVERRIDES
+ ver = cfgData.getVar("%s_VERSION_pn-%s" % (keyword, pn))
+ if not ver:
+ ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
+ if not ver:
+ ver = cfgData.getVar("%s_VERSION" % keyword)
+
+ return ver
+
def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
"""
Check if the version pe,pv,pr is the preferred one.
@@ -102,19 +117,28 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
"""
- Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+ Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
"""
preferred_file = None
preferred_ver = None
+ required = False
- # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
- # hence we do this manually rather than use OVERRIDES
- preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
- if not preferred_v:
- preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
- if not preferred_v:
- preferred_v = cfgData.getVar("PREFERRED_VERSION")
+ required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
+ preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
+
+ itemstr = ""
+ if item:
+ itemstr = " (for item %s)" % item
+
+ if required_v is not None:
+ if preferred_v is not None:
+ logger.warn("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
+ else:
+ logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
+ # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
+ preferred_v = required_v
+ required = True
if preferred_v:
m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
@@ -147,11 +171,9 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
pv_str = preferred_v
if not (preferred_e is None):
pv_str = '%s:%s' % (preferred_e, pv_str)
- itemstr = ""
- if item:
- itemstr = " (for item %s)" % item
if preferred_file is None:
- logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+ if not required:
+ logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
available_vers = []
for file_set in pkg_pn:
for f in file_set:
@@ -164,11 +186,15 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
if available_vers:
available_vers.sort()
logger.warn("versions of %s available: %s", pn, ' '.join(available_vers))
+ if required:
+ logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
else:
- logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
-
- return (preferred_ver, preferred_file)
+ if required:
+ logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+ else:
+ logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+ return (preferred_ver, preferred_file, required)
def findLatestProvider(pn, cfgData, dataCache, file_set):
"""
@@ -189,7 +215,6 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
return (latest, latest_f)
-
def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
"""
If there is a PREFERRED_VERSION, find the highest-priority bbfile
@@ -198,17 +223,16 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
"""
sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
- # Find the highest priority provider with a PREFERRED_VERSION set
- (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+ # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
+ (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
# Find the latest version of the highest priority provider
(latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
- if preferred_file is None:
+ if not required and preferred_file is None:
preferred_file = latest_f
preferred_ver = latest
- return (latest, latest_f, preferred_ver, preferred_file)
-
+ return (latest, latest_f, preferred_ver, preferred_file, required)
def _filterProviders(providers, item, cfgData, dataCache):
"""
@@ -234,10 +258,13 @@ def _filterProviders(providers, item, cfgData, dataCache):
logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
- # First add PREFERRED_VERSIONS
+ # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
for pn in sorted(pkg_pn):
sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
- preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+ preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+ if required and preferred_file is None:
+ return eligible
+ preferred_versions[pn] = (preferred_ver, preferred_file)
if preferred_versions[pn][1]:
eligible.append(preferred_versions[pn][1])
@@ -249,7 +276,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
eligible.append(preferred_versions[pn][1])
if not eligible:
- logger.error("no eligible providers for %s", item)
return eligible
# If pn == item, give it a slight default preference
@@ -266,7 +292,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
return eligible
-
def filterProviders(providers, item, cfgData, dataCache):
"""
Take a list of providers and filter/reorder according to the
@@ -388,7 +413,6 @@ def getRuntimeProviders(dataCache, rdepend):
return rproviders
-
def buildWorldTargetList(dataCache, task=None):
"""
Build package list for "bitbake world"
diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py
index 54ef245a6..80d7f6ca6 100644
--- a/poky/bitbake/lib/bb/runqueue.py
+++ b/poky/bitbake/lib/bb/runqueue.py
@@ -1242,6 +1242,7 @@ class RunQueue:
magic = "decafbad"
if self.cooker.configuration.profile:
magic = "decafbadbad"
+ fakerootlogs = None
if fakeroot:
magic = magic + "beef"
mcdata = self.cooker.databuilder.mcdata[mc]
@@ -1251,10 +1252,11 @@ class RunQueue:
for key, value in (var.split('=') for var in fakerootenv):
env[key] = value
worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+ fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
else:
worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
bb.utils.nonblockingfd(worker.stdout)
- workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+ workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
workerdata = {
"taskdeps" : self.rqdata.dataCaches[mc].task_deps,
@@ -1772,7 +1774,7 @@ class RunQueueExecute:
self.sqdata = SQData()
build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
- def runqueue_process_waitpid(self, task, status):
+ def runqueue_process_waitpid(self, task, status, fakerootlog=None):
# self.build_stamps[pid] may not exist when use shared work directory.
if task in self.build_stamps:
@@ -1787,7 +1789,7 @@ class RunQueueExecute:
self.sq_live.remove(task)
else:
if status != 0:
- self.task_fail(task, status)
+ self.task_fail(task, status, fakerootlog=fakerootlog)
else:
self.task_complete(task)
return True
@@ -1908,14 +1910,31 @@ class RunQueueExecute:
self.task_completeoutright(task)
self.runq_tasksrun.add(task)
- def task_fail(self, task, exitcode):
+ def task_fail(self, task, exitcode, fakerootlog=None):
"""
Called when a task has failed
Updates the state engine with the failure
"""
self.stats.taskFailed()
self.failed_tids.append(task)
- bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
+
+ fakeroot_log = ""
+ if fakerootlog and os.path.exists(fakerootlog):
+ with open(fakerootlog) as fakeroot_log_file:
+ fakeroot_failed = False
+ for line in reversed(fakeroot_log_file.readlines()):
+ for fakeroot_error in ['mismatch', 'error', 'fatal']:
+ if fakeroot_error in line.lower():
+ fakeroot_failed = True
+ if 'doing new pid setup and server start' in line:
+ break
+ fakeroot_log = line + fakeroot_log
+
+ if not fakeroot_failed:
+ fakeroot_log = None
+
+ bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=fakeroot_log), self.cfgData)
+
if self.rqdata.taskData[''].abort:
self.rq.state = runQueueCleanUp
@@ -1943,6 +1962,10 @@ class RunQueueExecute:
logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
err = True
+ for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered):
+ # No task should end up in both covered and uncovered, that is a bug.
+ logger.error("Setscene task %s in both covered and notcovered." % tid)
+
for tid in self.rqdata.runq_setscene_tids:
if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
err = True
@@ -2431,6 +2454,9 @@ class RunQueueExecute:
for dep in sorted(self.sqdata.sq_deps[task]):
if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
+ if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
+ # dependency could be already processed, e.g. noexec setscene task
+ continue
logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
self.sq_task_failoutright(dep)
continue
@@ -2755,6 +2781,20 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
+ # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+ # in any stamp files. Pass the list out to metadata as an event.
+ found = {}
+ for tid in rqdata.runq_setscene_tids:
+ (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+ stamps = bb.build.find_stale_stamps(taskname, rqdata.dataCaches[mc], taskfn)
+ if stamps:
+ if mc not in found:
+ found[mc] = {}
+ found[mc][tid] = stamps
+ for mc in found:
+ event = bb.event.StaleSetSceneTasks(found[mc])
+ bb.event.fire(event, cooker.databuilder.mcdata[mc])
+
def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
tocheck = set()
@@ -2876,12 +2916,16 @@ class runQueueTaskFailed(runQueueEvent):
"""
Event notifying a task failed
"""
- def __init__(self, task, stats, exitcode, rq):
+ def __init__(self, task, stats, exitcode, rq, fakeroot_log=None):
runQueueEvent.__init__(self, task, stats, rq)
self.exitcode = exitcode
+ self.fakeroot_log = fakeroot_log
def __str__(self):
- return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+ if self.fakeroot_log:
+ return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log)
+ else:
+ return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
class sceneQueueTaskFailed(sceneQueueEvent):
"""
@@ -2933,7 +2977,7 @@ class runQueuePipe():
"""
Abstraction for a pipe between a worker thread and the server
"""
- def __init__(self, pipein, pipeout, d, rq, rqexec):
+ def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None):
self.input = pipein
if pipeout:
pipeout.close()
@@ -2942,6 +2986,7 @@ class runQueuePipe():
self.d = d
self.rq = rq
self.rqexec = rqexec
+ self.fakerootlogs = fakerootlogs
def setrunqueueexec(self, rqexec):
self.rqexec = rqexec
@@ -2987,7 +3032,11 @@ class runQueuePipe():
task, status = pickle.loads(self.queue[10:index])
except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
- self.rqexec.runqueue_process_waitpid(task, status)
+ (_, _, _, taskfn) = split_tid_mcfn(task)
+ fakerootlog = None
+ if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs:
+ fakerootlog = self.fakerootlogs[taskfn]
+ self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog)
found = True
self.queue = self.queue[index+11:]
index = self.queue.find(b"</exitcode>")
diff --git a/poky/bitbake/lib/bb/tests/color.py b/poky/bitbake/lib/bb/tests/color.py
index bf03750c6..88dd27800 100644
--- a/poky/bitbake/lib/bb/tests/color.py
+++ b/poky/bitbake/lib/bb/tests/color.py
@@ -31,7 +31,7 @@ class ColorCodeTests(unittest.TestCase):
def setUp(self):
self.d = bb.data.init()
self._progress_watcher = ProgressWatcher()
- bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+ bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event, data=self.d)
def tearDown(self):
bb.event.remove("bb.build.TaskProgress", None)
diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py
index 7b2dac7b8..ddf6e9743 100644
--- a/poky/bitbake/lib/bb/tests/fetch.py
+++ b/poky/bitbake/lib/bb/tests/fetch.py
@@ -1345,7 +1345,7 @@ class FetchCheckStatusTest(FetcherTest):
"http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
"http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
"https://yoctoproject.org/",
- "https://yoctoproject.org/documentation",
+ "https://docs.yoctoproject.org",
"http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
"http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
"ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
diff --git a/poky/bitbake/lib/bb/tinfoil.py b/poky/bitbake/lib/bb/tinfoil.py
index 763c32981..796a98f05 100644
--- a/poky/bitbake/lib/bb/tinfoil.py
+++ b/poky/bitbake/lib/bb/tinfoil.py
@@ -440,7 +440,7 @@ class Tinfoil:
to initialise Tinfoil and use it with config_only=True first and
then conditionally call this function to parse recipes later.
"""
- config_params = TinfoilConfigParameters(config_only=False)
+ config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
self.run_actions(config_params)
self.recipes_parsed = True
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-22 00:28:04 +0300