diff options
Diffstat (limited to 'poky/bitbake')
| -rwxr-xr-x | poky/bitbake/bin/bitbake-hashclient | 2 | ||||
| -rwxr-xr-x | poky/bitbake/bin/bitbake-worker | 18 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst | 7 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 9 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 4 | ||||
| -rwxr-xr-x | poky/bitbake/lib/bb/main.py | 4 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/msg.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/runqueue.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/siggen.py | 21 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 15 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/ui/knotty.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/utils.py | 5 |
12 files changed, 67 insertions, 24 deletions
diff --git a/poky/bitbake/bin/bitbake-hashclient b/poky/bitbake/bin/bitbake-hashclient index 47dd27cd3c..610787ed2b 100755 --- a/poky/bitbake/bin/bitbake-hashclient +++ b/poky/bitbake/bin/bitbake-hashclient @@ -346,6 +346,8 @@ def main(): login, _, password = auth except FileNotFoundError: pass + except netrc.NetrcParseError as e: + sys.stderr.write(f"Error parsing {e.filename}:{e.lineno}: {e.msg}\n") func = getattr(args, 'func', None) if func: diff --git a/poky/bitbake/bin/bitbake-worker b/poky/bitbake/bin/bitbake-worker index eba9c562c7..e8073f2ac3 100755 --- a/poky/bitbake/bin/bitbake-worker +++ b/poky/bitbake/bin/bitbake-worker @@ -183,7 +183,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run: fakeroot = True envvars = (runtask['fakerootenv'] or "").split() - for key, value in (var.split('=') for var in envvars): + for key, value in (var.split('=',1) for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value fakeenv[key] = value @@ -195,7 +195,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): (fn, taskname, ', '.join(fakedirs))) else: envvars = (runtask['fakerootnoenv'] or "").split() - for key, value in (var.split('=') for var in envvars): + for key, value in (var.split('=',1) for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value fakeenv[key] = value @@ -237,11 +237,13 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): # Let SIGHUP exit as SIGTERM signal.signal(signal.SIGHUP, sigterm_handler) - # No stdin - newsi = os.open(os.devnull, os.O_RDWR) - os.dup2(newsi, sys.stdin.fileno()) + # No stdin & stdout + # stdout is used as a status report channel and must not be used by child processes. + dumbio = os.open(os.devnull, os.O_RDWR) + os.dup2(dumbio, sys.stdin.fileno()) + os.dup2(dumbio, sys.stdout.fileno()) - if umask: + if umask is not None: os.umask(umask) try: @@ -305,6 +307,10 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): if not quieterrors: logger.critical(traceback.format_exc()) os._exit(1) + + sys.stdout.flush() + sys.stderr.flush() + try: if dry_run: return 0 diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst index 4fa3ca4b54..d58fbb32ea 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst @@ -586,10 +586,11 @@ or possibly those defined in the metadata/signature handler itself. The simplest parameter to pass is "none", which causes a set of signature information to be written out into ``STAMPS_DIR`` corresponding to the targets specified. The other currently available parameter is -"printdiff", which causes BitBake to try to establish the closest +"printdiff", which causes BitBake to try to establish the most recent signature match it can (e.g. in the sstate cache) and then run -``bitbake-diffsigs`` over the matches to determine the stamps and delta -where these two stamp trees diverge. +compare the matched signatures to determine the stamps and delta +where these two stamp trees diverge. This can be used to determine why +tasks need to be re-run in situations where that is not expected. .. note:: diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 3529531ca1..5bf2c4b8cf 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -290,12 +290,12 @@ class URI(object): def _param_str_split(self, string, elmdelim, kvdelim="="): ret = collections.OrderedDict() - for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: + for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]: ret[k] = v return ret def _param_str_join(self, dict_, elmdelim, kvdelim="="): - return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) + return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()]) @property def hostport(self): @@ -943,7 +943,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): elif e.stderr: output = "output:\n%s" % e.stderr else: - output = "no output" + if log: + output = "see logfile for output" + else: + output = "no output" error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) except bb.process.CmdError as e: error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index b9dc576d47..c7ff769fdf 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -277,7 +277,7 @@ class Git(FetchMethod): ud.unresolvedrev[name] = ud.revisions[name] ud.revisions[name] = self.latest_revision(ud, d, name) - gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) + gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) if gitsrcname.startswith('.'): gitsrcname = gitsrcname[1:] @@ -673,7 +673,7 @@ class Git(FetchMethod): elif not need_lfs: bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) else: - runfetchcmd("%s lfs install" % ud.basecmd, d, workdir=destdir) + runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) if not ud.nocheckout: if subpath: diff --git a/poky/bitbake/lib/bb/main.py b/poky/bitbake/lib/bb/main.py index 92d8dc0293..bca8ebfa09 100755 --- a/poky/bitbake/lib/bb/main.py +++ b/poky/bitbake/lib/bb/main.py @@ -217,7 +217,9 @@ def create_bitbake_parser(): "execution. The SIGNATURE_HANDLER parameter is passed to the " "handler. Two common values are none and printdiff but the handler " "may define more/less. none means only dump the signature, printdiff" - " means compare the dumped signature with the cached one.") + " means recursively compare the dumped signature with the most recent" + " one in a local build or sstate cache (can be used to find out why tasks re-run" + " when that is not expected)") exec_group.add_argument("--revisions-changed", action="store_true", help="Set the exit code depending on whether upstream floating " diff --git a/poky/bitbake/lib/bb/msg.py b/poky/bitbake/lib/bb/msg.py index 93575d89c4..3e18596faa 100644 --- a/poky/bitbake/lib/bb/msg.py +++ b/poky/bitbake/lib/bb/msg.py @@ -230,7 +230,7 @@ def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers console = logging.StreamHandler(output) console.addFilter(bb.msg.LogFilterShowOnce()) format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if color == 'always' or (color == 'auto' and output.isatty()): + if color == 'always' or (color == 'auto' and output.isatty() and os.environ.get('NO_COLOR', '') == ''): format.enable_color() console.setFormatter(format) if preserve_handlers: diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 6987de3e29..bc7e18175d 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -1359,7 +1359,7 @@ class RunQueue: fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD")) fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split() env = os.environ.copy() - for key, value in (var.split('=') for var in fakerootenv): + for key, value in (var.split('=',1) for var in fakerootenv): env[key] = value worker = subprocess.Popen(fakerootcmd + [sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 3ab8431a08..2a0ecf57e1 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -535,23 +535,34 @@ class SignatureGeneratorUniHashMixIn(object): # hashes to appear over time, but much less likely for them to # disappear self.unihash_exists_cache = set() + self.username = None + self.password = None super().__init__(data) def get_taskdata(self): - return (self.server, self.method, self.extramethod, self.max_parallel) + super().get_taskdata() + return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password) + super().get_taskdata() def set_taskdata(self, data): - self.server, self.method, self.extramethod, self.max_parallel = data[:4] - super().set_taskdata(data[4:]) + self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password = data[:6] + super().set_taskdata(data[6:]) + + def get_hashserv_creds(self): + if self.username and self.password: + return { + "username": self.username, + "password": self.password, + } + + return {} def client(self): if getattr(self, '_client', None) is None: - self._client = hashserv.create_client(self.server) + self._client = hashserv.create_client(self.server, **self.get_hashserv_creds()) return self._client def client_pool(self): if getattr(self, '_client_pool', None) is None: - self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel) + self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds()) return self._client_pool def reset(self, data): diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index e988e26c0a..85c1f79ff3 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -308,6 +308,21 @@ class URITest(unittest.TestCase): 'params': {"someparam" : "1"}, 'query': {}, 'relative': True + }, + "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { + 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', + 'scheme': 'https', + 'hostname': 'www.innodisk.com', + 'port': None, + 'hostport': 'www.innodisk.com', + 'path': '/Download_file', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {"downloadfilename" : "EGPL-T101.zip"}, + 'query': {"9BE0BF6657": None}, + 'relative': False } } diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index 5a97d040b0..f86999bb09 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -179,7 +179,7 @@ class TerminalFilter(object): new[3] = new[3] & ~termios.ECHO termios.tcsetattr(fd, termios.TCSADRAIN, new) curses.setupterm() - if curses.tigetnum("colors") > 2: + if curses.tigetnum("colors") > 2 and os.environ.get('NO_COLOR', '') == '': for h in handlers: try: h.formatter.enable_color() diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index ad13d04331..ebee65d3dd 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -1142,7 +1142,10 @@ def get_referenced_vars(start_expr, d): def cpu_count(): - return multiprocessing.cpu_count() + try: + return len(os.sched_getaffinity(0)) + except OSError: + return multiprocessing.cpu_count() def nonblockingfd(fd): fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) |