diff options
Diffstat (limited to 'poky/bitbake')
40 files changed, 199 insertions, 45 deletions
diff --git a/poky/bitbake/bin/bitbake-layers b/poky/bitbake/bin/bitbake-layers index 449434d468..d4b1d1aaf2 100755 --- a/poky/bitbake/bin/bitbake-layers +++ b/poky/bitbake/bin/bitbake-layers @@ -68,11 +68,11 @@ def main(): registered = False for plugin in plugins: + if hasattr(plugin, 'tinfoil_init'): + plugin.tinfoil_init(tinfoil) if hasattr(plugin, 'register_commands'): registered = True plugin.register_commands(subparsers) - if hasattr(plugin, 'tinfoil_init'): - plugin.tinfoil_init(tinfoil) if not registered: logger.error("No commands registered - missing plugins?") diff --git a/poky/bitbake/bin/bitbake-prserv b/poky/bitbake/bin/bitbake-prserv index 323df66dd0..5be42f3ce5 100755 --- a/poky/bitbake/bin/bitbake-prserv +++ b/poky/bitbake/bin/bitbake-prserv @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/bin/bitbake-worker b/poky/bitbake/bin/bitbake-worker index 9d850ec77c..7be39370b3 100755 --- a/poky/bitbake/bin/bitbake-worker +++ b/poky/bitbake/bin/bitbake-worker @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -455,6 +457,7 @@ class BitbakeWorker(object): for mc in self.databuilder.mcdata: self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"]) self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"]) + self.databuilder.mcdata[mc].setVar("__bbclasstype", "recipe") def handle_newtaskhashes(self, data): self.workerdata["newhashes"] = pickle.loads(data) diff --git a/poky/bitbake/bin/git-make-shallow b/poky/bitbake/bin/git-make-shallow index 1d00fbf183..d0532c5ab8 100755 --- a/poky/bitbake/bin/git-make-shallow +++ b/poky/bitbake/bin/git-make-shallow @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/COW.py b/poky/bitbake/lib/bb/COW.py index 23c22b65ef..76bc08a3ea 100644 --- a/poky/bitbake/lib/bb/COW.py +++ b/poky/bitbake/lib/bb/COW.py @@ -3,6 +3,8 @@ # # Copyright (C) 2006 Tim Ansell # +# SPDX-License-Identifier: GPL-2.0-only +# # Please Note: # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. # Assign a file to __warn__ to get warnings about slow operations. diff --git a/poky/bitbake/lib/bb/asyncrpc/__init__.py b/poky/bitbake/lib/bb/asyncrpc/__init__.py index c2f2b3c00b..9a85e9965b 100644 --- a/poky/bitbake/lib/bb/asyncrpc/__init__.py +++ b/poky/bitbake/lib/bb/asyncrpc/__init__.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py index 34960197d1..881434d2e9 100644 --- a/poky/bitbake/lib/bb/asyncrpc/client.py +++ b/poky/bitbake/lib/bb/asyncrpc/client.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/asyncrpc/serv.py b/poky/bitbake/lib/bb/asyncrpc/serv.py index 585bc121da..5cf45f908a 100644 --- a/poky/bitbake/lib/bb/asyncrpc/serv.py +++ b/poky/bitbake/lib/bb/asyncrpc/serv.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index 55f68b98c5..b8c1099ef5 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -20,6 +20,7 @@ import itertools import time import re import stat +import datetime import bb import bb.msg import bb.process @@ -618,7 +619,8 @@ def _exec_task(fn, task, d, quieterr): logorder = os.path.join(tempdir, 'log.task_order') try: with open(logorder, 'a') as logorderfile: - logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") + logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) except OSError: logger.exception("Opening log file '%s'", logorder) pass diff --git a/poky/bitbake/lib/bb/codeparser.py b/poky/bitbake/lib/bb/codeparser.py index 3b3c3b41ff..9d66d3ae41 100644 --- a/poky/bitbake/lib/bb/codeparser.py +++ b/poky/bitbake/lib/bb/codeparser.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/compress/_pipecompress.py b/poky/bitbake/lib/bb/compress/_pipecompress.py index 5de17a82e2..4a403d62cf 100644 --- a/poky/bitbake/lib/bb/compress/_pipecompress.py +++ b/poky/bitbake/lib/bb/compress/_pipecompress.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # # Helper library to implement streaming compression and decompression using an diff --git a/poky/bitbake/lib/bb/compress/lz4.py b/poky/bitbake/lib/bb/compress/lz4.py index 0f6bc51a5b..88b0989322 100644 --- a/poky/bitbake/lib/bb/compress/lz4.py +++ b/poky/bitbake/lib/bb/compress/lz4.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/compress/zstd.py b/poky/bitbake/lib/bb/compress/zstd.py index 50c42133fb..cdbbe9d60f 100644 --- a/poky/bitbake/lib/bb/compress/zstd.py +++ b/poky/bitbake/lib/bb/compress/zstd.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index 2adf4d297d..1b6ee3032c 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -402,6 +402,7 @@ class BBCooker: for mc in self.databuilder.mcdata.values(): mc.renameVar("__depends", "__base_depends") self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher) + mc.setVar("__bbclasstype", "recipe") self.baseconfig_valid = True self.parsecache_valid = False diff --git a/poky/bitbake/lib/bb/cookerdata.py b/poky/bitbake/lib/bb/cookerdata.py index d54ac932e5..9706948ab3 100644 --- a/poky/bitbake/lib/bb/cookerdata.py +++ b/poky/bitbake/lib/bb/cookerdata.py @@ -254,6 +254,7 @@ class CookerDataBuilder(object): filtered_keys = bb.utils.approved_variables() bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys) self.basedata.setVar("BB_ORIGENV", self.savedenv) + self.basedata.setVar("__bbclasstype", "global") if worker: self.basedata.setVar("BB_WORKERCONTEXT", "1") diff --git a/poky/bitbake/lib/bb/daemonize.py b/poky/bitbake/lib/bb/daemonize.py index 4957bfd4b8..7689404436 100644 --- a/poky/bitbake/lib/bb/daemonize.py +++ b/poky/bitbake/lib/bb/daemonize.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/data.py b/poky/bitbake/lib/bb/data.py index c09d9b04bb..53fe34825d 100644 --- a/poky/bitbake/lib/bb/data.py +++ b/poky/bitbake/lib/bb/data.py @@ -441,7 +441,7 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): def inherits_class(klass, d): val = d.getVar('__inherit_cache', False) or [] - needle = os.path.join('classes', '%s.bbclass' % klass) + needle = '/%s.bbclass' % klass for v in val: if v.endswith(needle): return True diff --git a/poky/bitbake/lib/bb/exceptions.py b/poky/bitbake/lib/bb/exceptions.py index ecbad59970..801db9c82f 100644 --- a/poky/bitbake/lib/bb/exceptions.py +++ b/poky/bitbake/lib/bb/exceptions.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index c1950e4819..25d5db0e5b 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -115,6 +115,9 @@ class GitSM(Git): # This has to be a file reference proto = "file" url = "gitsm://" + uris[module] + if "{}{}".format(ud.host, ud.path) in url: + raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ + "Consider using git fetcher instead.") url += ';protocol=%s' % proto url += ";name=%s" % module diff --git a/poky/bitbake/lib/bb/fetch2/npm.py b/poky/bitbake/lib/bb/fetch2/npm.py index 8f7c10ac9b..8a179a339a 100644 --- a/poky/bitbake/lib/bb/fetch2/npm.py +++ b/poky/bitbake/lib/bb/fetch2/npm.py @@ -156,7 +156,7 @@ class Npm(FetchMethod): raise ParameterError("Invalid 'version' parameter", ud.url) # Extract the 'registry' part of the url - ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) + ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) # Using the 'downloadfilename' parameter as local filename # or the npm package name. diff --git a/poky/bitbake/lib/bb/fetch2/osc.py b/poky/bitbake/lib/bb/fetch2/osc.py index 86f8ddf47b..495ac8a30a 100644 --- a/poky/bitbake/lib/bb/fetch2/osc.py +++ b/poky/bitbake/lib/bb/fetch2/osc.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # """ diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index 68415735fd..18e6868387 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -44,23 +44,36 @@ def inherit(files, fn, lineno, d): __inherit_cache = d.getVar('__inherit_cache', False) or [] files = d.expand(files).split() for file in files: - if not os.path.isabs(file) and not file.endswith(".bbclass"): - file = os.path.join('classes', '%s.bbclass' % file) - - if not os.path.isabs(file): - bbpath = d.getVar("BBPATH") - abs_fn, attempts = bb.utils.which(bbpath, file, history=True) - for af in attempts: - if af != abs_fn: - bb.parse.mark_dependency(d, af) - if abs_fn: - file = abs_fn + classtype = d.getVar("__bbclasstype", False) + origfile = file + for t in ["classes-" + classtype, "classes"]: + file = origfile + if not os.path.isabs(file) and not file.endswith(".bbclass"): + file = os.path.join(t, '%s.bbclass' % file) + + if not os.path.isabs(file): + bbpath = d.getVar("BBPATH") + abs_fn, attempts = bb.utils.which(bbpath, file, history=True) + for af in attempts: + if af != abs_fn: + bb.parse.mark_dependency(d, af) + if abs_fn: + file = abs_fn + + if os.path.exists(file): + break + + if not os.path.exists(file): + raise ParseError("Could not inherit file %s" % (file), fn, lineno) if not file in __inherit_cache: logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno)) __inherit_cache.append( file ) d.setVar('__inherit_cache', __inherit_cache) - include(fn, file, lineno, d, "inherit") + try: + bb.parse.handle(file, d, True) + except (IOError, OSError) as exc: + raise ParseError("Could not inherit file %s: %s" % (fn, exc.strerror), fn, lineno) __inherit_cache = d.getVar('__inherit_cache', False) or [] def get_statements(filename, absolute_filename, base_name): diff --git a/poky/bitbake/lib/bb/process.py b/poky/bitbake/lib/bb/process.py index be2c15a188..4c7b6d39df 100644 --- a/poky/bitbake/lib/bb/process.py +++ b/poky/bitbake/lib/bb/process.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 359b503297..48e25401ba 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -168,15 +168,19 @@ class RunQueueScheduler(object): openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported) is returned. """ - if self.rq.max_cpu_pressure or self.rq.max_io_pressure: + if self.rq.max_cpu_pressure or self.rq.max_io_pressure or self.rq.max_memory_pressure: try: - with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds: + with open("/proc/pressure/cpu") as cpu_pressure_fds, \ + open("/proc/pressure/io") as io_pressure_fds, \ + open("/proc/pressure/memory") as memory_pressure_fds: + self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] + self.prev_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1] self.prev_pressure_time = time.time() self.check_pressure = True except: - bb.warn("The /proc/pressure files can't be read. Continuing build without monitoring pressure") + bb.note("The /proc/pressure files can't be read. Continuing build without monitoring pressure") self.check_pressure = False else: self.check_pressure = False @@ -184,21 +188,26 @@ class RunQueueScheduler(object): def exceeds_max_pressure(self): """ Monitor the difference in total pressure at least once per second, if - BB_PRESSURE_MAX_{CPU|IO} are set, return True if above threshold. + BB_PRESSURE_MAX_{CPU|IO|MEMORY} are set, return True if above threshold. """ if self.check_pressure: - with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds: + with open("/proc/pressure/cpu") as cpu_pressure_fds, \ + open("/proc/pressure/io") as io_pressure_fds, \ + open("/proc/pressure/memory") as memory_pressure_fds: # extract "total" from /proc/pressure/{cpu|io} curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] + curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1] exceeds_cpu_pressure = self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) > self.rq.max_cpu_pressure exceeds_io_pressure = self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) > self.rq.max_io_pressure + exceeds_memory_pressure = self.rq.max_memory_pressure and (float(curr_memory_pressure) - float(self.prev_memory_pressure)) > self.rq.max_memory_pressure now = time.time() if now - self.prev_pressure_time > 1.0: self.prev_cpu_pressure = curr_cpu_pressure self.prev_io_pressure = curr_io_pressure + self.prev_memory_pressure = curr_memory_pressure self.prev_pressure_time = now - return (exceeds_cpu_pressure or exceeds_io_pressure) + return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure) return False def next_buildable_task(self): @@ -1748,6 +1757,7 @@ class RunQueueExecute: self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed" self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU") self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO") + self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY") self.sq_buildable = set() self.sq_running = set() @@ -1798,6 +1808,13 @@ class RunQueueExecute: if self.max_io_pressure > upper_limit: bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure)) + if self.max_memory_pressure: + self.max_memory_pressure = float(self.max_memory_pressure) + if self.max_memory_pressure < lower_limit: + bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit)) + if self.max_memory_pressure > upper_limit: + bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure)) + # List of setscene tasks which we've covered self.scenequeue_covered = set() # List of tasks which are covered (including setscene ones) @@ -2237,10 +2254,9 @@ class RunQueueExecute: # No more tasks can be run. If we have deferred setscene tasks we should run them. if self.sq_deferred: - tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0]) - logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid) - if tid not in self.runq_complete: - self.sq_task_failoutright(tid) + deferred_tid = list(self.sq_deferred.keys())[0] + blocking_tid = self.sq_deferred.pop(deferred_tid) + logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid)) return True if self.failed_tids: @@ -2506,11 +2522,14 @@ class RunQueueExecute: if update_tasks: self.sqdone = False - for tid in [t[0] for t in update_tasks]: - h = pending_hash_index(tid, self.rqdata) - if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]: - self.sq_deferred[tid] = self.sqdata.hashes[h] - bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h])) + for mc in sorted(self.sqdata.multiconfigs): + for tid in sorted([t[0] for t in update_tasks]): + if mc_from_tid(tid) != mc: + continue + h = pending_hash_index(tid, self.rqdata) + if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]: + self.sq_deferred[tid] = self.sqdata.hashes[h] + bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h])) update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False) for (tid, harddepfail, origvalid) in update_tasks: diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 3f3d6df54d..07bb529452 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -425,7 +427,7 @@ class SignatureGeneratorBasic(SignatureGenerator): bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid)) sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash) - fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") + fd, tmpfile = bb.utils.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f: json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder) diff --git a/poky/bitbake/lib/bb/tests/compression.py b/poky/bitbake/lib/bb/tests/compression.py index d3ddf67f1c..95af3f96d7 100644 --- a/poky/bitbake/lib/bb/tests/compression.py +++ b/poky/bitbake/lib/bb/tests/compression.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/tests/cooker.py b/poky/bitbake/lib/bb/tests/cooker.py index c82d4b7b81..9e524ae345 100644 --- a/poky/bitbake/lib/bb/tests/cooker.py +++ b/poky/bitbake/lib/bb/tests/cooker.py @@ -1,6 +1,8 @@ # # BitBake Tests for cooker.py # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 7fcf57e7ea..b4ed691f33 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -11,6 +11,7 @@ import hashlib import tempfile import collections import os +import signal import tarfile from bb.fetch2 import URI from bb.fetch2 import FetchMethod @@ -22,6 +23,24 @@ def skipIfNoNetwork(): return unittest.skip("network test") return lambda f: f +class TestTimeout(Exception): + pass + +class Timeout(): + + def __init__(self, seconds): + self.seconds = seconds + + def handle_timeout(self, signum, frame): + raise TestTimeout("Test failed: timeout reached") + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, exc_type, exc_val, exc_tb): + signal.alarm(0) + class URITest(unittest.TestCase): test_uris = { "http://www.google.com/index.html" : { @@ -1172,6 +1191,15 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') + @skipIfNoNetwork() + def test_git_submodule_reference_to_parent(self): + self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" + self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") + with Timeout(60): + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + class SVNTest(FetcherTest): def skipIfNoSvn(): import shutil diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 1a3b74934d..ee7f2534f1 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -164,6 +164,7 @@ python () { # become unset/disappear. # def test_parse_classextend_contamination(self): + self.d.setVar("__bbclasstype", "recipe") cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass") #clsname = os.path.basename(cls.name).replace(".bbclass", "") self.classextend = self.classextend.replace("###CLASS###", cls.name) diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index 19ed68ea62..92d44c5260 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -28,6 +28,8 @@ import signal import collections import copy import ctypes +import random +import tempfile from subprocess import getstatusoutput from contextlib import contextmanager from ctypes import cdll @@ -429,12 +431,14 @@ def better_eval(source, locals, extraglobals = None): return eval(source, ctx, locals) @contextmanager -def fileslocked(files): +def fileslocked(files, *args, **kwargs): """Context manager for locking and unlocking file locks.""" locks = [] if files: for lockfile in files: - locks.append(bb.utils.lockfile(lockfile)) + l = bb.utils.lockfile(lockfile, *args, **kwargs) + if l is not None: + locks.append(l) try: yield @@ -1754,3 +1758,22 @@ def is_local_uid(uid=''): if str(uid) == line_split[2]: return True return False + +def mkstemp(suffix=None, prefix=None, dir=None, text=False): + """ + Generates a unique filename, independent of time. + + mkstemp() in glibc (at least) generates unique file names based on the + current system time. When combined with highly parallel builds, and + operating over NFS (e.g. shared sstate/downloads) this can result in + conflicts and race conditions. + + This function adds additional entropy to the file name so that a collision + is independent of time and thus extremely unlikely. + """ + entropy = "".join(random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890", k=20)) + if prefix: + prefix = prefix + entropy + else: + prefix = tempfile.gettempprefix() + entropy + return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text) diff --git a/poky/bitbake/lib/bblayers/__init__.py b/poky/bitbake/lib/bblayers/__init__.py index 4e7c09da04..78efd29750 100644 --- a/poky/bitbake/lib/bblayers/__init__.py +++ b/poky/bitbake/lib/bblayers/__init__.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bblayers/action.py b/poky/bitbake/lib/bblayers/action.py index 6723e2c605..454c251410 100644 --- a/poky/bitbake/lib/bblayers/action.py +++ b/poky/bitbake/lib/bblayers/action.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bblayers/common.py b/poky/bitbake/lib/bblayers/common.py index 6c76ef3505..f7b9cee371 100644 --- a/poky/bitbake/lib/bblayers/common.py +++ b/poky/bitbake/lib/bblayers/common.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bblayers/layerindex.py b/poky/bitbake/lib/bblayers/layerindex.py index 7936516209..0ac8fd2ec7 100644 --- a/poky/bitbake/lib/bblayers/layerindex.py +++ b/poky/bitbake/lib/bblayers/layerindex.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/bblayers/query.py b/poky/bitbake/lib/bblayers/query.py index 525d4f0d47..afd39518e5 100644 --- a/poky/bitbake/lib/bblayers/query.py +++ b/poky/bitbake/lib/bblayers/query.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # @@ -55,11 +57,12 @@ are overlayed will also be listed, with a " (skipped)" suffix. # Check for overlayed .bbclass files classes = collections.defaultdict(list) for layerdir in self.bblayers: - classdir = os.path.join(layerdir, 'classes') - if os.path.exists(classdir): - for classfile in os.listdir(classdir): - if os.path.splitext(classfile)[1] == '.bbclass': - classes[classfile].append(classdir) + for c in ["classes-global", "classes-recipe", "classes"]: + classdir = os.path.join(layerdir, c) + if os.path.exists(classdir): + for classfile in os.listdir(classdir): + if os.path.splitext(classfile)[1] == '.bbclass': + classes[classfile].append(classdir) # Locating classes and other files is a bit more complicated than recipes - # layer priority is not a factor; instead BitBake uses the first matching @@ -122,9 +125,14 @@ skipped recipes will also be listed, with a " (skipped)" suffix. if inherits: bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) for classname in inherits: - classfile = 'classes/%s.bbclass' % classname - if not bb.utils.which(bbpath, classfile, history=False): - logger.error('No class named %s found in BBPATH', classfile) + found = False + for c in ["classes-global", "classes-recipe", "classes"]: + cfile = c + '/%s.bbclass' % classname + if bb.utils.which(bbpath, cfile, history=False): + found = True + break + if not found: + logger.error('No class named %s found in BBPATH', classname) sys.exit(1) pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn @@ -172,7 +180,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix. logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() - cls_re = re.compile('classes/') + cls_re = re.compile('classes.*/') preffiles = [] show_unique_pn = [] @@ -405,7 +413,7 @@ NOTE: .bbappend files can impact the dependencies. self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) # The inherit class - cls_re = re.compile('classes/') + cls_re = re.compile('classes.*/') if f in self.tinfoil.cooker_data.inherits: inherits = self.tinfoil.cooker_data.inherits[f] for cls in inherits: diff --git a/poky/bitbake/lib/prserv/__init__.py b/poky/bitbake/lib/prserv/__init__.py index 9961040b58..38ced818ad 100644 --- a/poky/bitbake/lib/prserv/__init__.py +++ b/poky/bitbake/lib/prserv/__init__.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/prserv/client.py b/poky/bitbake/lib/prserv/client.py index a3f19ddafc..69ab7a4ac9 100644 --- a/poky/bitbake/lib/prserv/client.py +++ b/poky/bitbake/lib/prserv/client.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/prserv/db.py b/poky/bitbake/lib/prserv/db.py index 2710d4a225..b4bda7078c 100644 --- a/poky/bitbake/lib/prserv/db.py +++ b/poky/bitbake/lib/prserv/db.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/prserv/serv.py b/poky/bitbake/lib/prserv/serv.py index 0a20b927c7..c686b2065c 100644 --- a/poky/bitbake/lib/prserv/serv.py +++ b/poky/bitbake/lib/prserv/serv.py @@ -1,4 +1,6 @@ # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # diff --git a/poky/bitbake/lib/toaster/manage.py b/poky/bitbake/lib/toaster/manage.py index ae32619d12..f8de49c264 100755 --- a/poky/bitbake/lib/toaster/manage.py +++ b/poky/bitbake/lib/toaster/manage.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 # +# Copyright BitBake Contributors +# # SPDX-License-Identifier: GPL-2.0-only # |