diff options
Diffstat (limited to 'poky/bitbake/lib/bb')
-rw-r--r-- | poky/bitbake/lib/bb/__init__.py | 2 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/asyncrpc/__init__.py | 31 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/asyncrpc/client.py | 150 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/asyncrpc/serv.py | 223 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/build.py | 5 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/command.py | 12 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 5 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 8 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/s3.py | 41 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/svn.py | 2 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/wget.py | 2 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/progress.py | 7 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/runqueue.py | 24 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 9 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/siggen.py | 8 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 90 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/tinfoil.py | 4 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/ui/knotty.py | 8 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/utils.py | 18 |
19 files changed, 595 insertions, 54 deletions
diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py index 9f61dae14..a144bd609 100644 --- a/poky/bitbake/lib/bb/__init__.py +++ b/poky/bitbake/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "1.50.0" +__version__ = "1.51.0" import sys if sys.version_info < (3, 5, 0): diff --git a/poky/bitbake/lib/bb/asyncrpc/__init__.py b/poky/bitbake/lib/bb/asyncrpc/__init__.py new file mode 100644 index 000000000..b2bec31ab --- /dev/null +++ b/poky/bitbake/lib/bb/asyncrpc/__init__.py @@ -0,0 +1,31 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import itertools +import json + +# The Python async server defaults to a 64K receive buffer, so we hardcode our +# maximum chunk size. It would be better if the client and server reported to +# each other what the maximum chunk sizes were, but that will slow down the +# connection setup with a round trip delay so I'd rather not do that unless it +# is necessary +DEFAULT_MAX_CHUNK = 32 * 1024 + + +def chunkify(msg, max_chunk): + if len(msg) < max_chunk - 1: + yield ''.join((msg, "\n")) + else: + yield ''.join((json.dumps({ + 'chunk-stream': None + }), "\n")) + + args = [iter(msg)] * (max_chunk - 1) + for m in map(''.join, itertools.zip_longest(*args, fillvalue='')): + yield ''.join(itertools.chain(m, "\n")) + yield "\n" + + +from .client import AsyncClient, Client +from .serv import AsyncServer, AsyncServerConnection diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py new file mode 100644 index 000000000..79919c5be --- /dev/null +++ b/poky/bitbake/lib/bb/asyncrpc/client.py @@ -0,0 +1,150 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import abc +import asyncio +import json +import os +import socket +from . import chunkify, DEFAULT_MAX_CHUNK + + +class AsyncClient(object): + def __init__(self, proto_name, proto_version, logger): + self.reader = None + self.writer = None + self.max_chunk = DEFAULT_MAX_CHUNK + self.proto_name = proto_name + self.proto_version = proto_version + self.logger = logger + + async def connect_tcp(self, address, port): + async def connect_sock(): + return await asyncio.open_connection(address, port) + + self._connect_sock = connect_sock + + async def connect_unix(self, path): + async def connect_sock(): + return await asyncio.open_unix_connection(path) + + self._connect_sock = connect_sock + + async def setup_connection(self): + s = '%s %s\n\n' % (self.proto_name, self.proto_version) + self.writer.write(s.encode("utf-8")) + await self.writer.drain() + + async def connect(self): + if self.reader is None or self.writer is None: + (self.reader, self.writer) = await self._connect_sock() + await self.setup_connection() + + async def close(self): + self.reader = None + + if self.writer is not None: + self.writer.close() + self.writer = None + + async def _send_wrapper(self, proc): + count = 0 + while True: + try: + await self.connect() + return await proc() + except ( + OSError, + ConnectionError, + json.JSONDecodeError, + UnicodeDecodeError, + ) as e: + self.logger.warning("Error talking to server: %s" % e) + if count >= 3: + if not isinstance(e, ConnectionError): + raise ConnectionError(str(e)) + raise e + await self.close() + count += 1 + + async def send_message(self, msg): + async def get_line(): + line = await self.reader.readline() + if not line: + raise ConnectionError("Connection closed") + + line = line.decode("utf-8") + + if not line.endswith("\n"): + raise ConnectionError("Bad message %r" % msg) + + return line + + async def proc(): + for c in chunkify(json.dumps(msg), self.max_chunk): + self.writer.write(c.encode("utf-8")) + await self.writer.drain() + + l = await get_line() + + m = json.loads(l) + if m and "chunk-stream" in m: + lines = [] + while True: + l = (await get_line()).rstrip("\n") + if not l: + break + lines.append(l) + + m = json.loads("".join(lines)) + + return m + + return await self._send_wrapper(proc) + + async def ping(self): + return await self.send_message( + {'ping': {}} + ) + + +class Client(object): + def __init__(self): + self.client = self._get_async_client() + self.loop = asyncio.new_event_loop() + + self._add_methods('connect_tcp', 'close', 'ping') + + @abc.abstractmethod + def _get_async_client(self): + pass + + def _get_downcall_wrapper(self, downcall): + def wrapper(*args, **kwargs): + return self.loop.run_until_complete(downcall(*args, **kwargs)) + + return wrapper + + def _add_methods(self, *methods): + for m in methods: + downcall = getattr(self.client, m) + setattr(self, m, self._get_downcall_wrapper(downcall)) + + def connect_unix(self, path): + # AF_UNIX has path length issues so chdir here to workaround + cwd = os.getcwd() + try: + os.chdir(os.path.dirname(path)) + self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path))) + self.loop.run_until_complete(self.client.connect()) + finally: + os.chdir(cwd) + + @property + def max_chunk(self): + return self.client.max_chunk + + @max_chunk.setter + def max_chunk(self, value): + self.client.max_chunk = value diff --git a/poky/bitbake/lib/bb/asyncrpc/serv.py b/poky/bitbake/lib/bb/asyncrpc/serv.py new file mode 100644 index 000000000..ef20cb71d --- /dev/null +++ b/poky/bitbake/lib/bb/asyncrpc/serv.py @@ -0,0 +1,223 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import abc +import asyncio +import json +import os +import signal +import socket +import sys +from . import chunkify, DEFAULT_MAX_CHUNK + + +class ClientError(Exception): + pass + + +class ServerError(Exception): + pass + + +class AsyncServerConnection(object): + def __init__(self, reader, writer, proto_name, logger): + self.reader = reader + self.writer = writer + self.proto_name = proto_name + self.max_chunk = DEFAULT_MAX_CHUNK + self.handlers = { + 'chunk-stream': self.handle_chunk, + 'ping': self.handle_ping, + } + self.logger = logger + + async def process_requests(self): + try: + self.addr = self.writer.get_extra_info('peername') + self.logger.debug('Client %r connected' % (self.addr,)) + + # Read protocol and version + client_protocol = await self.reader.readline() + if client_protocol is None: + return + + (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split() + if client_proto_name != self.proto_name: + self.logger.debug('Rejecting invalid protocol %s' % (self.proto_name)) + return + + self.proto_version = tuple(int(v) for v in client_proto_version.split('.')) + if not self.validate_proto_version(): + self.logger.debug('Rejecting invalid protocol version %s' % (client_proto_version)) + return + + # Read headers. Currently, no headers are implemented, so look for + # an empty line to signal the end of the headers + while True: + line = await self.reader.readline() + if line is None: + return + + line = line.decode('utf-8').rstrip() + if not line: + break + + # Handle messages + while True: + d = await self.read_message() + if d is None: + break + await self.dispatch_message(d) + await self.writer.drain() + except ClientError as e: + self.logger.error(str(e)) + finally: + self.writer.close() + + async def dispatch_message(self, msg): + for k in self.handlers.keys(): + if k in msg: + self.logger.debug('Handling %s' % k) + await self.handlers[k](msg[k]) + return + + raise ClientError("Unrecognized command %r" % msg) + + def write_message(self, msg): + for c in chunkify(json.dumps(msg), self.max_chunk): + self.writer.write(c.encode('utf-8')) + + async def read_message(self): + l = await self.reader.readline() + if not l: + return None + + try: + message = l.decode('utf-8') + + if not message.endswith('\n'): + return None + + return json.loads(message) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self.logger.error('Bad message from client: %r' % message) + raise e + + async def handle_chunk(self, request): + lines = [] + try: + while True: + l = await self.reader.readline() + l = l.rstrip(b"\n").decode("utf-8") + if not l: + break + lines.append(l) + + msg = json.loads(''.join(lines)) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + self.logger.error('Bad message from client: %r' % lines) + raise e + + if 'chunk-stream' in msg: + raise ClientError("Nested chunks are not allowed") + + await self.dispatch_message(msg) + + async def handle_ping(self, request): + response = {'alive': True} + self.write_message(response) + + +class AsyncServer(object): + def __init__(self, logger, loop=None): + if loop is None: + self.loop = asyncio.new_event_loop() + self.close_loop = True + else: + self.loop = loop + self.close_loop = False + + self._cleanup_socket = None + self.logger = logger + + def start_tcp_server(self, host, port): + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, host, port, loop=self.loop) + ) + + for s in self.server.sockets: + self.logger.debug('Listening on %r' % (s.getsockname(),)) + # Newer python does this automatically. Do it manually here for + # maximum compatibility + s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1) + + name = self.server.sockets[0].getsockname() + if self.server.sockets[0].family == socket.AF_INET6: + self.address = "[%s]:%d" % (name[0], name[1]) + else: + self.address = "%s:%d" % (name[0], name[1]) + + def start_unix_server(self, path): + def cleanup(): + os.unlink(path) + + cwd = os.getcwd() + try: + # Work around path length limits in AF_UNIX + os.chdir(os.path.dirname(path)) + self.server = self.loop.run_until_complete( + asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop) + ) + finally: + os.chdir(cwd) + + self.logger.debug('Listening on %r' % path) + + self._cleanup_socket = cleanup + self.address = "unix://%s" % os.path.abspath(path) + + @abc.abstractmethod + def accept_client(self, reader, writer): + pass + + async def handle_client(self, reader, writer): + # writer.transport.set_write_buffer_limits(0) + try: + client = self.accept_client(reader, writer) + await client.process_requests() + except Exception as e: + import traceback + self.logger.error('Error from client: %s' % str(e), exc_info=True) + traceback.print_exc() + writer.close() + self.logger.debug('Client disconnected') + + def run_loop_forever(self): + try: + self.loop.run_forever() + except KeyboardInterrupt: + pass + + def signal_handler(self): + self.loop.stop() + + def serve_forever(self): + asyncio.set_event_loop(self.loop) + try: + self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler) + + self.run_loop_forever() + self.server.close() + + self.loop.run_until_complete(self.server.wait_closed()) + self.logger.debug('Server shutting down') + finally: + if self.close_loop: + if sys.version_info >= (3, 6): + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + if self._cleanup_socket is not None: + self._cleanup_socket() diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index b2715fc53..6ce8f1e6d 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -927,6 +927,11 @@ def add_tasks(tasklist, d): task_deps[name] = {} if name in flags: deptask = d.expand(flags[name]) + if name in ['noexec', 'fakeroot', 'nostamp']: + if deptask != '1': + bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " + "will result in the flag not being set. See YP bug #13808.".format(name)) + task_deps[name][task] = deptask getTask('mcdepends') getTask('depends') diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index dd77cdd6e..f530cf844 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -20,6 +20,7 @@ Commands are queued in a CommandQueue from collections import OrderedDict, defaultdict +import io import bb.event import bb.cooker import bb.remotedata @@ -500,6 +501,17 @@ class CommandsSync: d = command.remotedatastores[dsindex].varhistory return getattr(d, method)(*args, **kwargs) + def dataStoreConnectorVarHistCmdEmit(self, command, params): + dsindex = params[0] + var = params[1] + oval = params[2] + val = params[3] + d = command.remotedatastores[params[4]] + + o = io.StringIO() + command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d) + return o.getvalue() + def dataStoreConnectorIncHistCmd(self, command, params): dsindex = params[0] method = params[1] diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index cf0201c49..c8e91262a 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -562,6 +562,9 @@ def verify_checksum(ud, d, precomputed={}): checksum_expected = getattr(ud, "%s_expected" % checksum_id) + if checksum_expected == '': + checksum_expected = None + return { "id": checksum_id, "name": checksum_name, @@ -612,7 +615,7 @@ def verify_checksum(ud, d, precomputed={}): for ci in checksum_infos: if ci["expected"] and ci["expected"] != ci["data"]: - messages.append("File: '%s' has %s checksum %s when %s was " \ + messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) bad_checksum = ci["data"] diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index e3ba80a3f..5e65c83c6 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -168,7 +168,11 @@ class Git(FetchMethod): if len(branches) != len(ud.names): raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) - ud.cloneflags = "-s -n" + ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" + + ud.cloneflags = "-n" + if not ud.noshared: + ud.cloneflags += " -s" if ud.bareclone: ud.cloneflags += " --mirror" @@ -394,7 +398,7 @@ class Git(FetchMethod): tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) try: # Do the checkout. This implicitly involves a Git LFS fetch. - self.unpack(ud, tmpdir, d) + Git.unpack(self, ud, tmpdir, d) # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into # the bare clonedir. diff --git a/poky/bitbake/lib/bb/fetch2/s3.py b/poky/bitbake/lib/bb/fetch2/s3.py index ffca73c8e..6b8ffd535 100644 --- a/poky/bitbake/lib/bb/fetch2/s3.py +++ b/poky/bitbake/lib/bb/fetch2/s3.py @@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use. import os import bb import urllib.request, urllib.parse, urllib.error +import re from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError from bb.fetch2 import runfetchcmd +def convertToBytes(value, unit): + value = float(value) + if (unit == "KiB"): + value = value*1024.0; + elif (unit == "MiB"): + value = value*1024.0*1024.0; + elif (unit == "GiB"): + value = value*1024.0*1024.0*1024.0; + return value + +class S3ProgressHandler(bb.progress.LineFilterProgressHandler): + """ + Extract progress information from s3 cp output, e.g.: + Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining + """ + def __init__(self, d): + super(S3ProgressHandler, self).__init__(d) + # Send an initial progress event so the bar gets shown + self._fire_progress(0) + + def writeline(self, line): + percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line) + if percs: + completed = (percs[-1][0]) + completedUnit = (percs[-1][1]) + total = (percs[-1][2]) + totalUnit = (percs[-1][3]) + completed = convertToBytes(completed, completedUnit) + total = convertToBytes(total, totalUnit) + progress = (completed/total)*100.0 + rate = percs[-1][4] + self.update(progress, rate) + return False + return True + + class S3(FetchMethod): """Class to fetch urls via 'aws s3'""" @@ -52,7 +89,9 @@ class S3(FetchMethod): cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) bb.fetch2.check_network_access(d, cmd, ud.url) - runfetchcmd(cmd, d) + + progresshandler = S3ProgressHandler(d) + runfetchcmd(cmd, d, False, log=progresshandler) # Additional sanity checks copied from the wget class (although there # are no known issues which mean these are required, treat the aws cli diff --git a/poky/bitbake/lib/bb/fetch2/svn.py b/poky/bitbake/lib/bb/fetch2/svn.py index 8856ef1c6..80102b44f 100644 --- a/poky/bitbake/lib/bb/fetch2/svn.py +++ b/poky/bitbake/lib/bb/fetch2/svn.py @@ -86,7 +86,7 @@ class Svn(FetchMethod): if command == "info": svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) elif command == "log1": - svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) + svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) else: suffix = "" diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index 6d82f3af0..784df70c9 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -472,7 +472,7 @@ class Wget(FetchMethod): version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") + dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))") s = dirver_regex.search(dirver) if s: version_dir[1] = s.group('ver') diff --git a/poky/bitbake/lib/bb/progress.py b/poky/bitbake/lib/bb/progress.py index d051ba019..52d704d64 100644 --- a/poky/bitbake/lib/bb/progress.py +++ b/poky/bitbake/lib/bb/progress.py @@ -94,12 +94,15 @@ class LineFilterProgressHandler(ProgressHandler): while True: breakpos = self._linebuffer.find('\n') + 1 if breakpos == 0: - break + # for the case when the line with progress ends with only '\r' + breakpos = self._linebuffer.find('\r') + 1 + if breakpos == 0: + break line = self._linebuffer[:breakpos] self._linebuffer = self._linebuffer[breakpos:] # Drop any line feeds and anything that precedes them lbreakpos = line.rfind('\r') + 1 - if lbreakpos: + if lbreakpos and lbreakpos != breakpos: line = line[lbreakpos:] if self.writeline(filter_color(line)): super().write(line) diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index cd56a5547..6c41fe6d4 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -2030,8 +2030,6 @@ class RunQueueExecute: logger.debug("%s didn't become valid, skipping setscene" % nexttask) self.sq_task_failoutright(nexttask) return True - else: - self.sqdata.outrightfail.remove(nexttask) if nexttask in self.sqdata.outrightfail: logger.debug2('No package found, so skipping setscene task %s', nexttask) self.sq_task_failoutright(nexttask) @@ -2296,10 +2294,16 @@ class RunQueueExecute: self.updated_taskhash_queue.remove((tid, unihash)) if unihash != self.rqdata.runtaskentries[tid].unihash: - hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash)) - self.rqdata.runtaskentries[tid].unihash = unihash - bb.parse.siggen.set_unihash(tid, unihash) - toprocess.add(tid) + # Make sure we rehash any other tasks with the same task hash that we're deferred against. + torehash = [tid] + for deftid in self.sq_deferred: + if self.sq_deferred[deftid] == tid: + torehash.append(deftid) + for hashtid in torehash: + hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash)) + self.rqdata.runtaskentries[hashtid].unihash = unihash + bb.parse.siggen.set_unihash(hashtid, unihash) + toprocess.add(hashtid) # Work out all tasks which depend upon these total = set() @@ -2827,6 +2831,8 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s sqdata.stamppresent.remove(tid) if tid in sqdata.valid: sqdata.valid.remove(tid) + if tid in sqdata.outrightfail: + sqdata.outrightfail.remove(tid) noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True) @@ -2845,6 +2851,7 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary) sqdata.hashes = {} + sqrq.sq_deferred = {} for mc in sorted(sqdata.multiconfigs): for tid in sorted(sqdata.sq_revdeps): if mc_from_tid(tid) != mc: @@ -2857,10 +2864,13 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s continue if tid in sqrq.scenequeue_notcovered: continue - sqdata.outrightfail.add(tid) + if tid in sqrq.scenequeue_covered: + continue h = pending_hash_index(tid, rqdata) if h not in sqdata.hashes: + if tid in tids: + sqdata.outrightfail.add(tid) sqdata.hashes[h] = tid else: sqrq.sq_deferred[tid] = sqdata.hashes[h] diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index b27b4aefe..155e8d131 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -367,7 +367,12 @@ class ProcessServer(): self.next_heartbeat = now + self.heartbeat_seconds if hasattr(self.cooker, "data"): heartbeat = bb.event.HeartbeatEvent(now) - bb.event.fire(heartbeat, self.cooker.data) + try: + bb.event.fire(heartbeat, self.cooker.data) + except Exception as exc: + if not isinstance(exc, bb.BBHandledException): + logger.exception('Running heartbeat function') + self.quit = True if nextsleep and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for # the heartbeat. @@ -509,7 +514,7 @@ class BitBakeServer(object): os.set_inheritable(self.bitbake_lock.fileno(), True) os.set_inheritable(self.readypipein, True) serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") - os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) + os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface): diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 0d88c6ec6..07692e673 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -402,7 +402,7 @@ class SignatureGeneratorBasic(SignatureGenerator): p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) - os.rename(tmpfile, sigfile) + bb.utils.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) @@ -542,7 +542,7 @@ class SignatureGeneratorUniHashMixIn(object): hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server)) else: hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server)) - except hashserv.client.HashConnectionError as e: + except ConnectionError as e: bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) self.set_unihash(tid, unihash) @@ -621,7 +621,7 @@ class SignatureGeneratorUniHashMixIn(object): d.setVar('BB_UNIHASH', new_unihash) else: hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) - except hashserv.client.HashConnectionError as e: + except ConnectionError as e: bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) finally: if sigfile: @@ -661,7 +661,7 @@ class SignatureGeneratorUniHashMixIn(object): # TODO: What to do here? hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash)) - except hashserv.client.HashConnectionError as e: + except ConnectionError as e: bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) return False diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index ddf6e9743..9291ce4a0 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -390,6 +390,7 @@ class FetcherTest(unittest.TestCase): if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": print("Not cleaning up %s. Please remove manually." % self.tempdir) else: + bb.process.run('chmod u+rw -R %s' % self.tempdir) bb.utils.prunedir(self.tempdir) class MirrorUriTest(FetcherTest): @@ -673,12 +674,14 @@ class FetcherLocalTest(FetcherTest): with self.assertRaises(bb.fetch2.UnpackError): self.fetchUnpack(['file://a;subdir=/bin/sh']) - def test_local_gitfetch_usehead(self): + def dummyGitTest(self, suffix): # Create dummy local Git repo src_dir = tempfile.mkdtemp(dir=self.tempdir, prefix='gitfetch_localusehead_') src_dir = os.path.abspath(src_dir) bb.process.run("git init", cwd=src_dir) + bb.process.run("git config user.email 'you@example.com'", cwd=src_dir) + bb.process.run("git config user.name 'Your Name'", cwd=src_dir) bb.process.run("git commit --allow-empty -m'Dummy commit'", cwd=src_dir) # Use other branch than master @@ -690,7 +693,7 @@ class FetcherLocalTest(FetcherTest): # Fetch and check revision self.d.setVar("SRCREV", "AUTOINC") - url = "git://" + src_dir + ";protocol=file;usehead=1" + url = "git://" + src_dir + ";protocol=file;" + suffix fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() fetcher.unpack(self.unpackdir) @@ -699,31 +702,23 @@ class FetcherLocalTest(FetcherTest): unpack_rev = stdout[0].strip() self.assertEqual(orig_rev, unpack_rev) + def test_local_gitfetch_usehead(self): + self.dummyGitTest("usehead=1") + def test_local_gitfetch_usehead_withname(self): - # Create dummy local Git repo - src_dir = tempfile.mkdtemp(dir=self.tempdir, - prefix='gitfetch_localusehead_') - src_dir = os.path.abspath(src_dir) - bb.process.run("git init", cwd=src_dir) - bb.process.run("git commit --allow-empty -m'Dummy commit'", - cwd=src_dir) - # Use other branch than master - bb.process.run("git checkout -b my-devel", cwd=src_dir) - bb.process.run("git commit --allow-empty -m'Dummy commit 2'", - cwd=src_dir) - stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir) - orig_rev = stdout[0].strip() + self.dummyGitTest("usehead=1;name=newName") - # Fetch and check revision - self.d.setVar("SRCREV", "AUTOINC") - url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName" - fetcher = bb.fetch.Fetch([url], self.d) - fetcher.download() - fetcher.unpack(self.unpackdir) - stdout = bb.process.run("git rev-parse HEAD", - cwd=os.path.join(self.unpackdir, 'git')) - unpack_rev = stdout[0].strip() - self.assertEqual(orig_rev, unpack_rev) + def test_local_gitfetch_shared(self): + self.dummyGitTest("usehead=1;name=sharedName") + alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') + self.assertTrue(os.path.exists(alt)) + + def test_local_gitfetch_noshared(self): + self.d.setVar('BB_GIT_NOSHARED', '1') + self.unpackdir += '_noshared' + self.dummyGitTest("usehead=1;name=noSharedName") + alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') + self.assertFalse(os.path.exists(alt)) class FetcherNoNetworkTest(FetcherTest): def setUp(self): @@ -1390,6 +1385,8 @@ class GitMakeShallowTest(FetcherTest): self.gitdir = os.path.join(self.tempdir, 'gitshallow') bb.utils.mkdirhier(self.gitdir) bb.process.run('git init', cwd=self.gitdir) + bb.process.run('git config user.email "you@example.com"', cwd=self.gitdir) + bb.process.run('git config user.name "Your Name"', cwd=self.gitdir) def assertRefs(self, expected_refs): actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() @@ -1513,6 +1510,8 @@ class GitShallowTest(FetcherTest): bb.utils.mkdirhier(self.srcdir) self.git('init', cwd=self.srcdir) + self.git('config user.email "you@example.com"', cwd=self.srcdir) + self.git('config user.name "Your Name"', cwd=self.srcdir) self.d.setVar('WORKDIR', self.tempdir) self.d.setVar('S', self.gitdir) self.d.delVar('PREMIRRORS') @@ -1594,6 +1593,7 @@ class GitShallowTest(FetcherTest): # fetch and unpack, from the shallow tarball bb.utils.remove(self.gitdir, recurse=True) + bb.process.run('chmod u+w -R "%s"' % ud.clonedir) bb.utils.remove(ud.clonedir, recurse=True) bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) @@ -1746,6 +1746,8 @@ class GitShallowTest(FetcherTest): smdir = os.path.join(self.tempdir, 'gitsubmodule') bb.utils.mkdirhier(smdir) self.git('init', cwd=smdir) + self.git('config user.email "you@example.com"', cwd=smdir) + self.git('config user.name "Your Name"', cwd=smdir) # Make this look like it was cloned from a remote... self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) @@ -1776,6 +1778,8 @@ class GitShallowTest(FetcherTest): smdir = os.path.join(self.tempdir, 'gitsubmodule') bb.utils.mkdirhier(smdir) self.git('init', cwd=smdir) + self.git('config user.email "you@example.com"', cwd=smdir) + self.git('config user.name "Your Name"', cwd=smdir) # Make this look like it was cloned from a remote... self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) @@ -1794,7 +1798,7 @@ class GitShallowTest(FetcherTest): # Set up the mirror mirrordir = os.path.join(self.tempdir, 'mirror') - os.rename(self.dldir, mirrordir) + bb.utils.rename(self.dldir, mirrordir) self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) # Fetch from the mirror @@ -1818,8 +1822,8 @@ class GitShallowTest(FetcherTest): self.git('annex init', cwd=self.srcdir) open(os.path.join(self.srcdir, 'c'), 'w').close() self.git('annex add c', cwd=self.srcdir) - self.git('commit -m annex-c -a', cwd=self.srcdir) - bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex')) + self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) + bb.process.run('chmod u+w -R %s' % self.srcdir) uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir fetcher, ud = self.fetch_shallow(uri) @@ -1912,7 +1916,7 @@ class GitShallowTest(FetcherTest): bb.utils.mkdirhier(mirrordir) self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir) - os.rename(os.path.join(self.dldir, mirrortarball), + bb.utils.rename(os.path.join(self.dldir, mirrortarball), os.path.join(mirrordir, mirrortarball)) # Fetch from the mirror @@ -2094,6 +2098,8 @@ class GitLfsTest(FetcherTest): bb.utils.mkdirhier(self.srcdir) self.git('init', cwd=self.srcdir) + self.git('config user.email "you@example.com"', cwd=self.srcdir) + self.git('config user.name "Your Name"', cwd=self.srcdir) with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: attrs.write('*.mp3 filter=lfs -text') self.git(['add', '.gitattributes'], cwd=self.srcdir) @@ -2634,3 +2640,29 @@ class NPMTest(FetcherTest): fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) fetcher.download() self.assertTrue(os.path.exists(ud.localpath)) + +class GitSharedTest(FetcherTest): + def setUp(self): + super(GitSharedTest, self).setUp() + self.recipe_url = "git://git.openembedded.org/bitbake" + self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') + + @skipIfNoNetwork() + def test_shared_unpack(self): + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + + fetcher.download() + fetcher.unpack(self.unpackdir) + alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') + self.assertTrue(os.path.exists(alt)) + + @skipIfNoNetwork() + def test_noshared_unpack(self): + self.d.setVar('BB_GIT_NOSHARED', '1') + self.unpackdir += '_noshared' + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + + fetcher.download() + fetcher.unpack(self.unpackdir) + alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') + self.assertFalse(os.path.exists(alt)) diff --git a/poky/bitbake/lib/bb/tinfoil.py b/poky/bitbake/lib/bb/tinfoil.py index 796a98f05..27a341541 100644 --- a/poky/bitbake/lib/bb/tinfoil.py +++ b/poky/bitbake/lib/bb/tinfoil.py @@ -52,6 +52,10 @@ class TinfoilDataStoreConnectorVarHistory: def remoteCommand(self, cmd, *args, **kwargs): return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs) + def emit(self, var, oval, val, o, d): + ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex) + o.write(ret) + def __getattr__(self, name): if not hasattr(bb.data_smart.VariableHistory, name): raise AttributeError("VariableHistory has no such method %s" % name) diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index 0efa614df..65ff2727d 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -21,6 +21,7 @@ import fcntl import struct import copy import atexit +from itertools import groupby from bb.ui import uihelper @@ -539,6 +540,13 @@ def main(server, eventHandler, params, tf = TerminalFilter): except OSError: pass + # Add the logging domains specified by the user on the command line + for (domainarg, iterator) in groupby(params.debug_domains): + dlevel = len(tuple(iterator)) + l = logconfig["loggers"].setdefault("BitBake.%s" % domainarg, {}) + l["level"] = logging.DEBUG - dlevel + 1 + l.setdefault("handlers", []).extend(["BitBake.verbconsole"]) + conf = bb.msg.setLoggingConfig(logconfig, logconfigfile) if sys.stdin.isatty() and sys.stdout.isatty(): diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index b282d09ab..6ba1d2a37 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -782,7 +782,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]: try: - os.rename(src, destpath) + bb.utils.rename(src, destpath) renamefailed = 0 except Exception as e: if e.errno != errno.EXDEV: @@ -796,7 +796,7 @@ def movefile(src, dest, newmtime = None, sstat = None): if stat.S_ISREG(sstat[stat.ST_MODE]): try: # For safety copy then move it over. shutil.copyfile(src, destpath + "#new") - os.rename(destpath + "#new", destpath) + bb.utils.rename(destpath + "#new", destpath) didcopy = 1 except Exception as e: print('movefile: copy', src, '->', dest, 'failed.', e) @@ -874,7 +874,7 @@ def copyfile(src, dest, newmtime = None, sstat = None): # For safety copy then move it over. shutil.copyfile(src, dest + "#new") - os.rename(dest + "#new", dest) + bb.utils.rename(dest + "#new", dest) except Exception as e: logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e)) return False @@ -1669,3 +1669,15 @@ def is_semver(version): return False return True + +# Wrapper around os.rename which can handle cross device problems +# e.g. from container filesystems +def rename(src, dst): + try: + os.rename(src, dst) + except OSError as err: + if err.errno == 18: + # Invalid cross-device link error + shutil.move(src, dst) + else: + raise err |