diff options
Diffstat (limited to 'poky/bitbake')
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 9 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 60 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/asyncrpc/serv.py | 4 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/runqueue.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 6 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/utils.py | 7 | ||||
| -rw-r--r-- | poky/bitbake/lib/bblayers/layerindex.py | 25 | ||||
| -rw-r--r-- | poky/bitbake/lib/ply/yacc.py | 7 |
10 files changed, 98 insertions, 26 deletions
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index af9947199c..b7c3d8091f 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -330,7 +330,8 @@ Removal (Override Style Syntax) You can remove values from lists using the removal override style syntax. Specifying a value for removal causes all occurrences of that -value to be removed from the variable. +value to be removed from the variable. Unlike ":append" and ":prepend", +there is no need to add a leading or trailing space to the value. When you use this syntax, BitBake expects one or more strings. Surrounding spaces and spacing are preserved. Here is an example:: @@ -421,6 +422,12 @@ documentation to a BitBake variable as follows:: CACHE[doc] = "The directory holding the cache of the metadata." +.. note:: + + Variable flag names starting with an underscore (``_``) character + are allowed but are ignored by ``d.getVarFlags("VAR")`` + in Python code. Such flag names are used internally by BitBake. + Inline Python Variable Expansion -------------------------------- diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 725e6c2cd5..3522d2b77b 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -484,29 +484,55 @@ overview of their function and contents. for it to work. :term:`BB_PRESSURE_MAX_CPU` - The threshold for maximum CPU pressure before BitBake prevents the - scheduling of new tasks. Once the :term:`BB_PRESSURE_MAX_CPU` threshold - is exceeded, new tasks are not started until the pressure subsides to - below the threshold. If :term:`BB_PRESSURE_MAX_CPU` is not set, CPU - pressure is not monitored. A threshold can be set in ``conf/local.conf`` - as:: + Specifies a maximum CPU pressure threshold, above which BitBake's + scheduler will not start new tasks (providing there is at least + one active task). If no value is set, CPU pressure is not + monitored when starting tasks. + + The pressure data is calculated based upon what Linux kernels since + version 4.20 expose under ``/proc/pressure``. The threshold represents + the difference in "total" pressure from the previous second. The + minimum value is 1.0 (extremely slow builds) and the maximum is + 1000000 (a pressure value unlikely to ever be reached). + + This threshold can be set in ``conf/local.conf`` as:: BB_PRESSURE_MAX_CPU = "500" :term:`BB_PRESSURE_MAX_IO` - The threshold for maximum IO pressure experienced before BitBake - prevents the scheduling of new tasks. The IO pressure is regulated in the - same way as :term:`BB_PRESSURE_MAX_CPU`. At this point in time, - experiments show that IO pressure tends to be short-lived and regulating - just the CPU can help to reduce it. + Specifies a maximum I/O pressure threshold, above which BitBake's + scheduler will not start new tasks (providing there is at least + one active task). If no value is set, I/O pressure is not + monitored when starting tasks. + + The pressure data is calculated based upon what Linux kernels since + version 4.20 expose under ``/proc/pressure``. The threshold represents + the difference in "total" pressure from the previous second. The + minimum value is 1.0 (extremely slow builds) and the maximum is + 1000000 (a pressure value unlikely to ever be reached). + + At this point in time, experiments show that IO pressure tends to + be short-lived and regulating just the CPU with + :term:`BB_PRESSURE_MAX_CPU` can help to reduce it. :term:`BB_PRESSURE_MAX_MEMORY` - The threshold for maximum memory pressure experienced before BitBake - prevents the scheduling of new tasks. The memory pressure is regulated in - the same way as :term:`BB_PRESSURE_MAX_CPU`. Note that any memory - pressure indicates that a system is being pushed beyond its capacity. At - this point in time, experiments show that memory pressure tends to be - short-lived and regulating just the CPU can help to reduce it. + + Specifies a maximum memory pressure threshold, above which BitBake's + scheduler will not start new tasks (providing there is at least + one active task). If no value is set, memory pressure is not + monitored when starting tasks. + + The pressure data is calculated based upon what Linux kernels since + version 4.20 expose under ``/proc/pressure``. The threshold represents + the difference in "total" pressure from the previous second. The + minimum value is 1.0 (extremely slow builds) and the maximum is + 1000000 (a pressure value unlikely to ever be reached). + + Memory pressure is experienced when time is spent swapping, + refaulting pages from the page cache or performing direct reclaim. + This is why memory pressure is rarely seen, but setting this variable + might be useful as a last resort to prevent OOM errors if they are + occurring during builds. :term:`BB_RUNFMT` Specifies the name of the executable script files (i.e. run files) diff --git a/poky/bitbake/lib/bb/asyncrpc/serv.py b/poky/bitbake/lib/bb/asyncrpc/serv.py index 5cf45f908a..d2de4891b8 100644 --- a/poky/bitbake/lib/bb/asyncrpc/serv.py +++ b/poky/bitbake/lib/bb/asyncrpc/serv.py @@ -42,7 +42,7 @@ class AsyncServerConnection(object): # Read protocol and version client_protocol = await self.reader.readline() - if client_protocol is None: + if not client_protocol: return (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split() @@ -59,7 +59,7 @@ class AsyncServerConnection(object): # an empty line to signal the end of the headers while True: line = await self.reader.readline() - if line is None: + if not line: return line = line.decode('utf-8').rstrip() diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index 32a529f0f9..1da2f03197 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -250,7 +250,7 @@ class BBCooker: def process_inotify_updates(self): for n in [self.confignotifier, self.notifier]: if n and n.check_events(timeout=0): - # read notified events and enqeue them + # read notified events and enqueue them n.read_events() n.process_events() diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 17d4904927..578edc5914 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -243,7 +243,7 @@ class Git(FetchMethod): for name in ud.names: ud.unresolvedrev[name] = 'HEAD' - ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false -c core.pager=cat" + ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat" write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" ud.write_tarballs = write_tarballs != "0" or ud.rebaseable diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 48e25401ba..338d1fe36f 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -2256,7 +2256,7 @@ class RunQueueExecute: if self.sq_deferred: deferred_tid = list(self.sq_deferred.keys())[0] blocking_tid = self.sq_deferred.pop(deferred_tid) - logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid)) + logger.warning("Runqueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid)) return True if self.failed_tids: diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index b4ed691f33..0af06e46e5 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -1331,12 +1331,14 @@ class URLHandle(unittest.TestCase): "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), + r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main'}), } # we require a pathname to encodeurl but users can still pass such urls to # decodeurl and we need to handle them decodedata = datatable.copy() decodedata.update({ "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), + "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), }) def test_decodeurl(self): @@ -1869,7 +1871,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) - self.git('submodule add file://%s' % smdir, cwd=self.srcdir) + self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) self.git('submodule update', cwd=self.srcdir) self.git('commit -m submodule -a', cwd=self.srcdir) @@ -1899,7 +1901,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) - self.git('submodule add file://%s' % smdir, cwd=self.srcdir) + self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) self.git('submodule update', cwd=self.srcdir) self.git('commit -m submodule -a', cwd=self.srcdir) diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index e6e21e20fe..64a004d0d8 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -547,7 +547,12 @@ def md5_file(filename): Return the hex string representation of the MD5 checksum of filename. """ import hashlib - return _hasher(hashlib.new('MD5', usedforsecurity=False), filename) + try: + sig = hashlib.new('MD5', usedforsecurity=False) + except TypeError: + # Some configurations don't appear to support two arguments + sig = hashlib.new('MD5') + return _hasher(sig, filename) def sha256_file(filename): """ diff --git a/poky/bitbake/lib/bblayers/layerindex.py b/poky/bitbake/lib/bblayers/layerindex.py index 0ac8fd2ec7..ba91fac669 100644 --- a/poky/bitbake/lib/bblayers/layerindex.py +++ b/poky/bitbake/lib/bblayers/layerindex.py @@ -49,6 +49,31 @@ class LayerIndexPlugin(ActionPlugin): else: logger.plain("Repository %s needs to be fetched" % url) return subdir, layername, layerdir + elif os.path.exists(repodir) and branch: + """ + If the repo is already cloned, ensure it is on the correct branch, + switching branches if necessary and possible. + """ + base_cmd = ['git', '--git-dir=%s/.git' % repodir, '--work-tree=%s' % repodir] + cmd = base_cmd + ['branch'] + completed_proc = subprocess.run(cmd, text=True, capture_output=True) + if completed_proc.returncode: + logger.error("Unable to validate repo %s (%s)" % (repodir, stderr)) + return None, None, None + else: + if branch != completed_proc.stdout[2:-1]: + cmd = base_cmd + ['status', '--short'] + completed_proc = subprocess.run(cmd, text=True, capture_output=True) + if completed_proc.stdout.count('\n') != 0: + logger.warning("There are uncommitted changes in repo %s" % repodir) + cmd = base_cmd + ['checkout', branch] + completed_proc = subprocess.run(cmd, text=True, capture_output=True) + if completed_proc.returncode: + # Could be due to original shallow clone on a different branch for example + logger.error("Unable to automatically switch %s to desired branch '%s' (%s)" + % (repodir, branch, completed_proc.stderr)) + return None, None, None + return subdir, layername, layerdir elif os.path.exists(layerdir): return subdir, layername, layerdir else: diff --git a/poky/bitbake/lib/ply/yacc.py b/poky/bitbake/lib/ply/yacc.py index 767c4e4674..381b50cf0b 100644 --- a/poky/bitbake/lib/ply/yacc.py +++ b/poky/bitbake/lib/ply/yacc.py @@ -2798,7 +2798,14 @@ class ParserReflect(object): def signature(self): try: import hashlib + except ImportError: + raise RuntimeError("Unable to import hashlib") + try: sig = hashlib.new('MD5', usedforsecurity=False) + except TypeError: + # Some configurations don't appear to support two arguments + sig = hashlib.new('MD5') + try: if self.start: sig.update(self.start.encode('latin-1')) if self.prec: |