From eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 Mon Sep 17 00:00:00 2001 From: Dave Cobbley Date: Tue, 14 Aug 2018 10:05:37 -0700 Subject: [Subtree] Removing import-layers directory As part of the move to subtrees, need to bring all the import layers content to the top level. Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f Signed-off-by: Dave Cobbley Signed-off-by: Brad Bishop --- poky/bitbake/bin/bitbake | 56 ++++ poky/bitbake/bin/bitbake-diffsigs | 183 ++++++++++++ poky/bitbake/bin/bitbake-dumpsig | 94 +++++++ poky/bitbake/bin/bitbake-layers | 110 ++++++++ poky/bitbake/bin/bitbake-prserv | 55 ++++ poky/bitbake/bin/bitbake-selftest | 73 +++++ poky/bitbake/bin/bitbake-worker | 501 +++++++++++++++++++++++++++++++++ poky/bitbake/bin/bitdoc | 531 +++++++++++++++++++++++++++++++++++ poky/bitbake/bin/git-make-shallow | 165 +++++++++++ poky/bitbake/bin/toaster | 314 +++++++++++++++++++++ poky/bitbake/bin/toaster-eventreplay | 126 +++++++++ 11 files changed, 2208 insertions(+) create mode 100755 poky/bitbake/bin/bitbake create mode 100755 poky/bitbake/bin/bitbake-diffsigs create mode 100755 poky/bitbake/bin/bitbake-dumpsig create mode 100755 poky/bitbake/bin/bitbake-layers create mode 100755 poky/bitbake/bin/bitbake-prserv create mode 100755 poky/bitbake/bin/bitbake-selftest create mode 100755 poky/bitbake/bin/bitbake-worker create mode 100755 poky/bitbake/bin/bitdoc create mode 100755 poky/bitbake/bin/git-make-shallow create mode 100755 poky/bitbake/bin/toaster create mode 100755 poky/bitbake/bin/toaster-eventreplay (limited to 'poky/bitbake/bin') diff --git a/poky/bitbake/bin/bitbake b/poky/bitbake/bin/bitbake new file mode 100755 index 000000000..342cef916 --- /dev/null +++ b/poky/bitbake/bin/bitbake @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# Copyright (C) 2006 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), + 'lib')) +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +from bb import cookerdata +from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException + +if sys.getfilesystemencoding() != "utf-8": + sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") + +__version__ = "1.38.0" + +if __name__ == "__main__": + if __version__ != bb.__version__: + sys.exit("Bitbake core version and program version mismatch!") + try: + sys.exit(bitbake_main(BitBakeConfigParameters(sys.argv), + cookerdata.CookerConfiguration())) + except BBMainException as err: + sys.exit(err) + except bb.BBHandledException: + sys.exit(1) + except Exception: + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/poky/bitbake/bin/bitbake-diffsigs b/poky/bitbake/bin/bitbake-diffsigs new file mode 100755 index 000000000..4e6bbddcd --- /dev/null +++ b/poky/bitbake/bin/bitbake-diffsigs @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +# bitbake-diffsigs +# BitBake task signature data comparison utility +# +# Copyright (C) 2012-2013, 2017 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import warnings +import fnmatch +import argparse +import logging +import pickle + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +import bb.tinfoil +import bb.siggen +import bb.msg + +logger = bb.msg.logger_create('bitbake-diffsigs') + +def find_siginfo(tinfoil, pn, taskname, sigs=None): + result = None + tinfoil.set_event_mask(['bb.event.FindSigInfoResult', + 'logging.LogRecord', + 'bb.command.CommandCompleted', + 'bb.command.CommandFailed']) + ret = tinfoil.run_command('findSigInfo', pn, taskname, sigs) + if ret: + while True: + event = tinfoil.wait_event(1) + if event: + if isinstance(event, bb.command.CommandCompleted): + break + elif isinstance(event, bb.command.CommandFailed): + logger.error(str(event)) + sys.exit(2) + elif isinstance(event, bb.event.FindSigInfoResult): + result = event.result + elif isinstance(event, logging.LogRecord): + logger.handle(event) + else: + logger.error('No result returned from findSigInfo command') + sys.exit(2) + return result + +def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False): + """ Find the most recent signature files for the specified PN/task and compare them """ + + if not taskname.startswith('do_'): + taskname = 'do_%s' % taskname + + if sig1 and sig2: + sigfiles = find_siginfo(bbhandler, pn, taskname, [sig1, sig2]) + if len(sigfiles) == 0: + logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2)) + sys.exit(1) + elif not sig1 in sigfiles: + logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig1)) + sys.exit(1) + elif not sig2 in sigfiles: + logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2)) + sys.exit(1) + latestfiles = [sigfiles[sig1], sigfiles[sig2]] + else: + filedates = find_siginfo(bbhandler, pn, taskname) + latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:] + if not latestfiles: + logger.error('No sigdata files found matching %s %s' % (pn, taskname)) + sys.exit(1) + elif len(latestfiles) < 2: + logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname)) + sys.exit(1) + + # Define recursion callback + def recursecb(key, hash1, hash2): + hashes = [hash1, hash2] + hashfiles = find_siginfo(bbhandler, key, None, hashes) + + recout = [] + if len(hashfiles) == 0: + recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) + elif not hash1 in hashfiles: + recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1)) + elif not hash2 in hashfiles: + recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2)) + else: + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) + for change in out2: + for line in change.splitlines(): + recout.append(' ' + line) + + return recout + + # Recurse into signature comparison + logger.debug("Signature file (previous): %s" % latestfiles[-2]) + logger.debug("Signature file (latest): %s" % latestfiles[-1]) + output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color) + if output: + print('\n'.join(output)) + sys.exit(0) + + + +parser = argparse.ArgumentParser( + description="Compares siginfo/sigdata files written out by BitBake") + +parser.add_argument('-d', '--debug', + help='Enable debug output', + action='store_true') + +parser.add_argument('--color', + help='Colorize output (where %(metavar)s is %(choices)s)', + choices=['auto', 'always', 'never'], default='auto', metavar='color') + +parser.add_argument("-t", "--task", + help="find the signature data files for last two runs of the specified task and compare them", + action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname')) + +parser.add_argument("-s", "--signature", + help="With -t/--task, specify the signatures to look for instead of taking the last two", + action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig')) + +parser.add_argument("sigdatafile1", + help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.", + action="store", nargs='?') + +parser.add_argument("sigdatafile2", + help="Second signature file to compare", + action="store", nargs='?') + + +options = parser.parse_args() + +if options.debug: + logger.setLevel(logging.DEBUG) + +color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.isatty())) + +if options.taskargs: + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare(config_only=True) + if options.sigargs: + find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color) + else: + find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color) +else: + if options.sigargs: + logger.error('-s/--signature can only be used together with -t/--task') + sys.exit(1) + try: + if options.sigdatafile1 and options.sigdatafile2: + output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color) + elif options.sigdatafile1: + output = bb.siggen.dump_sigfile(options.sigdatafile1) + else: + logger.error('Must specify signature file(s) or -t/--task') + parser.print_help() + sys.exit(1) + except IOError as e: + logger.error(str(e)) + sys.exit(1) + except (pickle.UnpicklingError, EOFError): + logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files') + sys.exit(1) + + if output: + print('\n'.join(output)) diff --git a/poky/bitbake/bin/bitbake-dumpsig b/poky/bitbake/bin/bitbake-dumpsig new file mode 100755 index 000000000..95ebd9354 --- /dev/null +++ b/poky/bitbake/bin/bitbake-dumpsig @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +# bitbake-dumpsig +# BitBake task signature dump utility +# +# Copyright (C) 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import warnings +import optparse +import logging +import pickle + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +import bb.tinfoil +import bb.siggen +import bb.msg + +logger = bb.msg.logger_create('bitbake-dumpsig') + +def find_siginfo_task(bbhandler, pn, taskname): + """ Find the most recent signature file for the specified PN/task """ + + if not hasattr(bb.siggen, 'find_siginfo'): + logger.error('Metadata does not support finding signature data files') + sys.exit(1) + + if not taskname.startswith('do_'): + taskname = 'do_%s' % taskname + + filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data) + latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:] + if not latestfiles: + logger.error('No sigdata files found matching %s %s' % (pn, taskname)) + sys.exit(1) + + return latestfiles[0] + +parser = optparse.OptionParser( + description = "Dumps siginfo/sigdata files written out by BitBake", + usage = """ + %prog -t recipename taskname + %prog sigdatafile""") + +parser.add_option("-D", "--debug", + help = "enable debug", + action = "store_true", dest="debug", default = False) + +parser.add_option("-t", "--task", + help = "find the signature data file for the specified task", + action="store", dest="taskargs", nargs=2, metavar='recipename taskname') + +options, args = parser.parse_args(sys.argv) + +if options.debug: + logger.setLevel(logging.DEBUG) + +if options.taskargs: + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only = True) + file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1]) + logger.debug("Signature file: %s" % file) +elif len(args) == 1: + parser.print_help() + sys.exit(0) +else: + file = args[1] + +try: + output = bb.siggen.dump_sigfile(file) +except IOError as e: + logger.error(str(e)) + sys.exit(1) +except (pickle.UnpicklingError, EOFError): + logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file') + sys.exit(1) + +if output: + print('\n'.join(output)) diff --git a/poky/bitbake/bin/bitbake-layers b/poky/bitbake/bin/bitbake-layers new file mode 100755 index 000000000..d184011ea --- /dev/null +++ b/poky/bitbake/bin/bitbake-layers @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# This script has subcommands which operate against your bitbake layers, either +# displaying useful information, or acting against them. +# See the help output for details on available commands. + +# Copyright (C) 2011 Mentor Graphics Corporation +# Copyright (C) 2011-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import os +import sys +import argparse +import signal + +bindir = os.path.dirname(__file__) +topdir = os.path.dirname(bindir) +sys.path[0:0] = [os.path.join(topdir, 'lib')] + +import bb.tinfoil +import bb.msg + +logger = bb.msg.logger_create('bitbake-layers', sys.stdout) + +def main(): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + parser = argparse.ArgumentParser( + description="BitBake layers utility", + epilog="Use %(prog)s --help to get help on a specific command", + add_help=False) + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + parser.add_argument('-F', '--force', help='Force add without recipe parse verification', action='store_true') + parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR') + + global_args, unparsed_args = parser.parse_known_args() + + # Help is added here rather than via add_help=True, as we don't want it to + # be handled by parse_known_args() + parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, + help='show this help message and exit') + subparsers = parser.add_subparsers(title='subcommands', metavar='') + subparsers.required = True + + if global_args.debug: + logger.setLevel(logging.DEBUG) + elif global_args.quiet: + logger.setLevel(logging.ERROR) + + # Need to re-run logger_create with color argument + # (will be the same logger since it has the same name) + bb.msg.logger_create('bitbake-layers', output=sys.stdout, color=global_args.color) + + plugins = [] + tinfoil = bb.tinfoil.Tinfoil(tracking=True) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + try: + tinfoil.prepare(True) + for path in ([topdir] + + tinfoil.config_data.getVar('BBPATH').split(':')): + pluginpath = os.path.join(path, 'lib', 'bblayers') + bb.utils.load_plugins(logger, plugins, pluginpath) + + registered = False + for plugin in plugins: + if hasattr(plugin, 'register_commands'): + registered = True + plugin.register_commands(subparsers) + if hasattr(plugin, 'tinfoil_init'): + plugin.tinfoil_init(tinfoil) + + if not registered: + logger.error("No commands registered - missing plugins?") + sys.exit(1) + + args = parser.parse_args(unparsed_args, namespace=global_args) + + if getattr(args, 'parserecipes', False): + tinfoil.config_data.disableTracking() + tinfoil.parse_recipes() + tinfoil.config_data.enableTracking() + + return args.func(args) + finally: + tinfoil.shutdown() + + +if __name__ == "__main__": + try: + ret = main() + except bb.BBHandledException: + ret = 1 + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) diff --git a/poky/bitbake/bin/bitbake-prserv b/poky/bitbake/bin/bitbake-prserv new file mode 100755 index 000000000..f38d2dd88 --- /dev/null +++ b/poky/bitbake/bin/bitbake-prserv @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import os +import sys,logging +import optparse + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) + +import prserv +import prserv.serv + +__version__="1.0.0" + +PRHOST_DEFAULT='0.0.0.0' +PRPORT_DEFAULT=8585 + +def main(): + parser = optparse.OptionParser( + version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__), + usage = "%prog < --start | --stop > [options]") + + parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store", + dest="dbfile", type="string", default="prserv.sqlite3") + parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store", + dest="logfile", type="string", default="prserv.log") + parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", + action = "store", type="string", dest="loglevel", default = "INFO") + parser.add_option("--start", help="start daemon", + action="store_true", dest="start") + parser.add_option("--stop", help="stop daemon", + action="store_true", dest="stop") + parser.add_option("--host", help="ip address to bind", action="store", + dest="host", type="string", default=PRHOST_DEFAULT) + parser.add_option("--port", help="port number(default: 8585)", action="store", + dest="port", type="int", default=PRPORT_DEFAULT) + + options, args = parser.parse_args(sys.argv) + prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) + + if options.start: + ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile)) + elif options.stop: + ret=prserv.serv.stop_daemon(options.host, options.port) + else: + ret=parser.print_help() + return ret + +if __name__ == "__main__": + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/poky/bitbake/bin/bitbake-selftest b/poky/bitbake/bin/bitbake-selftest new file mode 100755 index 000000000..afe1603d0 --- /dev/null +++ b/poky/bitbake/bin/bitbake-selftest @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2012 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys, logging +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) + +import unittest +try: + import bb +except RuntimeError as exc: + sys.exit(str(exc)) + +tests = ["bb.tests.codeparser", + "bb.tests.cow", + "bb.tests.data", + "bb.tests.event", + "bb.tests.fetch", + "bb.tests.parse", + "bb.tests.utils"] + +for t in tests: + t = '.'.join(t.split('.')[:3]) + __import__(t) + + +# Set-up logging +class StdoutStreamHandler(logging.StreamHandler): + """Special handler so that unittest is able to capture stdout""" + def __init__(self): + # Override __init__() because we don't want to set self.stream here + logging.Handler.__init__(self) + + @property + def stream(self): + # We want to dynamically write wherever sys.stdout is pointing to + return sys.stdout + + +handler = StdoutStreamHandler() +bb.logger.addHandler(handler) +bb.logger.setLevel(logging.DEBUG) + + +ENV_HELP = """\ +Environment variables: + BB_SKIP_NETTESTS set to 'yes' in order to skip tests using network + connection + BB_TMPDIR_NOCLEAN set to 'yes' to preserve test tmp directories +""" + +class main(unittest.main): + def _print_help(self, *args, **kwargs): + super(main, self)._print_help(*args, **kwargs) + print(ENV_HELP) + + +if __name__ == '__main__': + main(defaultTest=tests, buffer=True) diff --git a/poky/bitbake/bin/bitbake-worker b/poky/bitbake/bin/bitbake-worker new file mode 100755 index 000000000..e925054b7 --- /dev/null +++ b/poky/bitbake/bin/bitbake-worker @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 + +import os +import sys +import warnings +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) +from bb import fetch2 +import logging +import bb +import select +import errno +import signal +import pickle +import traceback +import queue +from multiprocessing import Lock +from threading import Thread + +if sys.getfilesystemencoding() != "utf-8": + sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") + +# Users shouldn't be running this code directly +if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"): + print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.") + sys.exit(1) + +profiling = False +if sys.argv[1].startswith("decafbadbad"): + profiling = True + try: + import cProfile as profile + except: + import profile + +# Unbuffer stdout to avoid log truncation in the event +# of an unorderly exit as well as to provide timely +# updates to log files for use with tail +try: + if sys.stdout.name == '': + import fcntl + fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL) + fl |= os.O_SYNC + fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl) + #sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) +except: + pass + +logger = logging.getLogger("BitBake") + +worker_pipe = sys.stdout.fileno() +bb.utils.nonblockingfd(worker_pipe) +# Need to guard against multiprocessing being used in child processes +# and multiple processes trying to write to the parent at the same time +worker_pipe_lock = None + +handler = bb.event.LogHandler() +logger.addHandler(handler) + +if 0: + # Code to write out a log file of all events passing through the worker + logfilename = "/tmp/workerlogfile" + format_str = "%(levelname)s: %(message)s" + conlogformat = bb.msg.BBLogFormatter(format_str) + consolelog = logging.FileHandler(logfilename) + bb.msg.addDefaultlogFilter(consolelog) + consolelog.setFormatter(conlogformat) + logger.addHandler(consolelog) + +worker_queue = queue.Queue() + +def worker_fire(event, d): + data = b"" + pickle.dumps(event) + b"" + worker_fire_prepickled(data) + +def worker_fire_prepickled(event): + global worker_queue + + worker_queue.put(event) + +# +# We can end up with write contention with the cooker, it can be trying to send commands +# and we can be trying to send event data back. Therefore use a separate thread for writing +# back data to cooker. +# +worker_thread_exit = False + +def worker_flush(worker_queue): + worker_queue_int = b"" + global worker_pipe, worker_thread_exit + + while True: + try: + worker_queue_int = worker_queue_int + worker_queue.get(True, 1) + except queue.Empty: + pass + while (worker_queue_int or not worker_queue.empty()): + try: + (_, ready, _) = select.select([], [worker_pipe], [], 1) + if not worker_queue.empty(): + worker_queue_int = worker_queue_int + worker_queue.get() + written = os.write(worker_pipe, worker_queue_int) + worker_queue_int = worker_queue_int[written:] + except (IOError, OSError) as e: + if e.errno != errno.EAGAIN and e.errno != errno.EPIPE: + raise + if worker_thread_exit and worker_queue.empty() and not worker_queue_int: + return + +worker_thread = Thread(target=worker_flush, args=(worker_queue,)) +worker_thread.start() + +def worker_child_fire(event, d): + global worker_pipe + global worker_pipe_lock + + data = b"" + pickle.dumps(event) + b"" + try: + worker_pipe_lock.acquire() + worker_pipe.write(data) + worker_pipe_lock.release() + except IOError: + sigterm_handler(None, None) + raise + +bb.event.worker_fire = worker_fire + +lf = None +#lf = open("/tmp/workercommandlog", "w+") +def workerlog_write(msg): + if lf: + lf.write(msg) + lf.flush() + +def sigterm_handler(signum, frame): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.killpg(0, signal.SIGTERM) + sys.exit() + +def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False): + # We need to setup the environment BEFORE the fork, since + # a fork() or exec*() activates PSEUDO... + + envbackup = {} + fakeenv = {} + umask = None + + taskdep = workerdata["taskdeps"][fn] + if 'umask' in taskdep and taskname in taskdep['umask']: + # umask might come in as a number or text string.. + try: + umask = int(taskdep['umask'][taskname],8) + except TypeError: + umask = taskdep['umask'][taskname] + + dry_run = cfg.dry_run or dry_run_exec + + # We can't use the fakeroot environment in a dry run as it possibly hasn't been built + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run: + envvars = (workerdata["fakerootenv"][fn] or "").split() + for key, value in (var.split('=') for var in envvars): + envbackup[key] = os.environ.get(key) + os.environ[key] = value + fakeenv[key] = value + + fakedirs = (workerdata["fakerootdirs"][fn] or "").split() + for p in fakedirs: + bb.utils.mkdirhier(p) + logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' % + (fn, taskname, ', '.join(fakedirs))) + else: + envvars = (workerdata["fakerootnoenv"][fn] or "").split() + for key, value in (var.split('=') for var in envvars): + envbackup[key] = os.environ.get(key) + os.environ[key] = value + fakeenv[key] = value + + sys.stdout.flush() + sys.stderr.flush() + + try: + pipein, pipeout = os.pipe() + pipein = os.fdopen(pipein, 'rb', 4096) + pipeout = os.fdopen(pipeout, 'wb', 0) + pid = os.fork() + except OSError as e: + logger.critical("fork failed: %d (%s)" % (e.errno, e.strerror)) + sys.exit(1) + + if pid == 0: + def child(): + global worker_pipe + global worker_pipe_lock + pipein.close() + + signal.signal(signal.SIGTERM, sigterm_handler) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, sigterm_handler) + bb.utils.signal_on_parent_exit("SIGTERM") + + # Save out the PID so that the event can include it the + # events + bb.event.worker_pid = os.getpid() + bb.event.worker_fire = worker_child_fire + worker_pipe = pipeout + worker_pipe_lock = Lock() + + # Make the child the process group leader and ensure no + # child process will be controlled by the current terminal + # This ensures signals sent to the controlling terminal like Ctrl+C + # don't stop the child processes. + os.setsid() + # No stdin + newsi = os.open(os.devnull, os.O_RDWR) + os.dup2(newsi, sys.stdin.fileno()) + + if umask: + os.umask(umask) + + try: + bb_cache = bb.cache.NoCache(databuilder) + (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn) + the_data = databuilder.mcdata[mc] + the_data.setVar("BB_WORKERCONTEXT", "1") + the_data.setVar("BB_TASKDEPDATA", taskdepdata) + if cfg.limited_deps: + the_data.setVar("BB_LIMITEDDEPS", "1") + the_data.setVar("BUILDNAME", workerdata["buildname"]) + the_data.setVar("DATE", workerdata["date"]) + the_data.setVar("TIME", workerdata["time"]) + for varname, value in extraconfigdata.items(): + the_data.setVar(varname, value) + + bb.parse.siggen.set_taskdata(workerdata["sigdata"]) + ret = 0 + + the_data = bb_cache.loadDataFull(fn, appends) + the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task]) + + bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", ""))) + + # exported_vars() returns a generator which *cannot* be passed to os.environ.update() + # successfully. We also need to unset anything from the environment which shouldn't be there + exports = bb.data.exported_vars(the_data) + + bb.utils.empty_environment() + for e, v in exports: + os.environ[e] = v + + for e in fakeenv: + os.environ[e] = fakeenv[e] + the_data.setVar(e, fakeenv[e]) + the_data.setVarFlag(e, 'export', "1") + + task_exports = the_data.getVarFlag(taskname, 'exports') + if task_exports: + for e in task_exports.split(): + the_data.setVarFlag(e, 'export', '1') + v = the_data.getVar(e) + if v is not None: + os.environ[e] = v + + if quieterrors: + the_data.setVarFlag(taskname, "quieterrors", "1") + + except Exception: + if not quieterrors: + logger.critical(traceback.format_exc()) + os._exit(1) + try: + if dry_run: + return 0 + return bb.build.exec_task(fn, taskname, the_data, cfg.profile) + except: + os._exit(1) + if not profiling: + os._exit(child()) + else: + profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname) + prof = profile.Profile() + try: + ret = profile.Profile.runcall(prof, child) + finally: + prof.dump_stats(profname) + bb.utils.process_profilelog(profname) + os._exit(ret) + else: + for key, value in iter(envbackup.items()): + if value is None: + del os.environ[key] + else: + os.environ[key] = value + + return pid, pipein, pipeout + +class runQueueWorkerPipe(): + """ + Abstraction for a pipe between a worker thread and the worker server + """ + def __init__(self, pipein, pipeout): + self.input = pipein + if pipeout: + pipeout.close() + bb.utils.nonblockingfd(self.input) + self.queue = b"" + + def read(self): + start = len(self.queue) + try: + self.queue = self.queue + (self.input.read(102400) or b"") + except (OSError, IOError) as e: + if e.errno != errno.EAGAIN: + raise + + end = len(self.queue) + index = self.queue.find(b"") + while index != -1: + worker_fire_prepickled(self.queue[:index+8]) + self.queue = self.queue[index+8:] + index = self.queue.find(b"") + return (end > start) + + def close(self): + while self.read(): + continue + if len(self.queue) > 0: + print("Warning, worker child left partial message: %s" % self.queue) + self.input.close() + +normalexit = False + +class BitbakeWorker(object): + def __init__(self, din): + self.input = din + bb.utils.nonblockingfd(self.input) + self.queue = b"" + self.cookercfg = None + self.databuilder = None + self.data = None + self.extraconfigdata = None + self.build_pids = {} + self.build_pipes = {} + + signal.signal(signal.SIGTERM, self.sigterm_exception) + # Let SIGHUP exit as SIGTERM + signal.signal(signal.SIGHUP, self.sigterm_exception) + if "beef" in sys.argv[1]: + bb.utils.set_process_name("Worker (Fakeroot)") + else: + bb.utils.set_process_name("Worker") + + def sigterm_exception(self, signum, stackframe): + if signum == signal.SIGTERM: + bb.warn("Worker received SIGTERM, shutting down...") + elif signum == signal.SIGHUP: + bb.warn("Worker received SIGHUP, shutting down...") + self.handle_finishnow(None) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + + def serve(self): + while True: + (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) + if self.input in ready: + try: + r = self.input.read() + if len(r) == 0: + # EOF on pipe, server must have terminated + self.sigterm_exception(signal.SIGTERM, None) + self.queue = self.queue + r + except (OSError, IOError): + pass + if len(self.queue): + self.handle_item(b"cookerconfig", self.handle_cookercfg) + self.handle_item(b"extraconfigdata", self.handle_extraconfigdata) + self.handle_item(b"workerdata", self.handle_workerdata) + self.handle_item(b"runtask", self.handle_runtask) + self.handle_item(b"finishnow", self.handle_finishnow) + self.handle_item(b"ping", self.handle_ping) + self.handle_item(b"quit", self.handle_quit) + + for pipe in self.build_pipes: + if self.build_pipes[pipe].input in ready: + self.build_pipes[pipe].read() + if len(self.build_pids): + while self.process_waitpid(): + continue + + + def handle_item(self, item, func): + if self.queue.startswith(b"<" + item + b">"): + index = self.queue.find(b"") + while index != -1: + func(self.queue[(len(item) + 2):index]) + self.queue = self.queue[(index + len(item) + 3):] + index = self.queue.find(b"") + + def handle_cookercfg(self, data): + self.cookercfg = pickle.loads(data) + self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True) + self.databuilder.parseBaseConfiguration() + self.data = self.databuilder.data + + def handle_extraconfigdata(self, data): + self.extraconfigdata = pickle.loads(data) + + def handle_workerdata(self, data): + self.workerdata = pickle.loads(data) + bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"] + bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"] + bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"] + bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] + for mc in self.databuilder.mcdata: + self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"]) + + def handle_ping(self, _): + workerlog_write("Handling ping\n") + + logger.warning("Pong from bitbake-worker!") + + def handle_quit(self, data): + workerlog_write("Handling quit\n") + + global normalexit + normalexit = True + sys.exit(0) + + def handle_runtask(self, data): + fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data) + workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + + pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec) + + self.build_pids[pid] = task + self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) + + def process_waitpid(self): + """ + Return none is there are no processes awaiting result collection, otherwise + collect the process exit codes and close the information pipe. + """ + try: + pid, status = os.waitpid(-1, os.WNOHANG) + if pid == 0 or os.WIFSTOPPED(status): + return False + except OSError: + return False + + workerlog_write("Exit code of %s for pid %s\n" % (status, pid)) + + if os.WIFEXITED(status): + status = os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + # Per shell conventions for $?, when a process exits due to + # a signal, we return an exit code of 128 + SIGNUM + status = 128 + os.WTERMSIG(status) + + task = self.build_pids[pid] + del self.build_pids[pid] + + self.build_pipes[pid].close() + del self.build_pipes[pid] + + worker_fire_prepickled(b"" + pickle.dumps((task, status)) + b"") + + return True + + def handle_finishnow(self, _): + if self.build_pids: + logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids)) + for k, v in iter(self.build_pids.items()): + try: + os.kill(-k, signal.SIGTERM) + os.waitpid(-1, 0) + except: + pass + for pipe in self.build_pipes: + self.build_pipes[pipe].read() + +try: + worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb')) + if not profiling: + worker.serve() + else: + profname = "profile-worker.log" + prof = profile.Profile() + try: + profile.Profile.runcall(prof, worker.serve) + finally: + prof.dump_stats(profname) + bb.utils.process_profilelog(profname) +except BaseException as e: + if not normalexit: + import traceback + sys.stderr.write(traceback.format_exc()) + sys.stderr.write(str(e)) + +worker_thread_exit = True +worker_thread.join() + +workerlog_write("exitting") +sys.exit(0) diff --git a/poky/bitbake/bin/bitdoc b/poky/bitbake/bin/bitdoc new file mode 100755 index 000000000..274467882 --- /dev/null +++ b/poky/bitbake/bin/bitdoc @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2005 Holger Hans Peter Freyther +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import optparse, os, sys + +# bitbake +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__), 'lib')) +import bb +import bb.parse +from string import split, join + +__version__ = "0.0.2" + +class HTMLFormatter: + """ + Simple class to help to generate some sort of HTML files. It is + quite inferior solution compared to docbook, gtkdoc, doxygen but it + should work for now. + We've a global introduction site (index.html) and then one site for + the list of keys (alphabetical sorted) and one for the list of groups, + one site for each key with links to the relations and groups. + + index.html + all_keys.html + all_groups.html + groupNAME.html + keyNAME.html + """ + + def replace(self, text, *pairs): + """ + From pydoc... almost identical at least + """ + while pairs: + (a, b) = pairs[0] + text = join(split(text, a), b) + pairs = pairs[1:] + return text + def escape(self, text): + """ + Escape string to be conform HTML + """ + return self.replace(text, + ('&', '&'), + ('<', '<' ), + ('>', '>' ) ) + def createNavigator(self): + """ + Create the navgiator + """ + return """ + + + + +
HomeGroupsKeys
+""" + + def relatedKeys(self, item): + """ + Create HTML to link to foreign keys + """ + + if len(item.related()) == 0: + return "" + + txt = "

See also:
" + txts = [] + for it in item.related(): + txts.append("""%(it)s""" % vars() ) + + return txt + ",".join(txts) + + def groups(self, item): + """ + Create HTML to link to related groups + """ + + if len(item.groups()) == 0: + return "" + + + txt = "

See also:
" + txts = [] + for group in item.groups(): + txts.append( """%s """ % (group, group) ) + + return txt + ",".join(txts) + + + def createKeySite(self, item): + """ + Create a site for a key. It contains the header/navigator, a heading, + the description, links to related keys and to the groups. + """ + + return """ +Key %s + + +%s +

%s

+ +
+

Synopsis

+

+%s +

+
+ +
+

Related Keys

+

+%s +

+
+ +
+

Groups

+

+%s +

+
+ + + +""" % (item.name(), self.createNavigator(), item.name(), + self.escape(item.description()), self.relatedKeys(item), self.groups(item)) + + def createGroupsSite(self, doc): + """ + Create the Group Overview site + """ + + groups = "" + sorted_groups = sorted(doc.groups()) + for group in sorted_groups: + groups += """%s
""" % (group, group) + + return """ +Group overview + + +%s +

Available Groups

+%s + +""" % (self.createNavigator(), groups) + + def createIndex(self): + """ + Create the index file + """ + + return """ +Bitbake Documentation + + +%s +

Documentation Entrance

+All available groups
+All available keys
+ +""" % self.createNavigator() + + def createKeysSite(self, doc): + """ + Create Overview of all avilable keys + """ + keys = "" + sorted_keys = sorted(doc.doc_keys()) + for key in sorted_keys: + keys += """%s
""" % (key, key) + + return """ +Key overview + + +%s +

Available Keys

+%s + +""" % (self.createNavigator(), keys) + + def createGroupSite(self, gr, items, _description = None): + """ + Create a site for a group: + Group the name of the group, items contain the name of the keys + inside this group + """ + groups = "" + description = "" + + # create a section with the group descriptions + if _description: + description += "

" % gr + description += _description + + items.sort(lambda x, y:cmp(x.name(), y.name())) + for group in items: + groups += """%s
""" % (group.name(), group.name()) + + return """ +Group %s + + +%s +%s +
+

Keys in Group %s

+
+%s
+
+
+ +""" % (gr, self.createNavigator(), description, gr, groups) + + + + def createCSS(self): + """ + Create the CSS file + """ + return """.synopsis, .classsynopsis +{ + background: #eeeeee; + border: solid 1px #aaaaaa; + padding: 0.5em; +} +.programlisting +{ + background: #eeeeff; + border: solid 1px #aaaaff; + padding: 0.5em; +} +.variablelist +{ + padding: 4px; + margin-left: 3em; +} +.variablelist td:first-child +{ + vertical-align: top; +} +table.navigation +{ + background: #ffeeee; + border: solid 1px #ffaaaa; + margin-top: 0.5em; + margin-bottom: 0.5em; +} +.navigation a +{ + color: #770000; +} +.navigation a:visited +{ + color: #550000; +} +.navigation .title +{ + font-size: 200%; +} +div.refnamediv +{ + margin-top: 2em; +} +div.gallery-float +{ + float: left; + padding: 10px; +} +div.gallery-float img +{ + border-style: none; +} +div.gallery-spacer +{ + clear: both; +} +a +{ + text-decoration: none; +} +a:hover +{ + text-decoration: underline; + color: #FF0000; +} +""" + + + +class DocumentationItem: + """ + A class to hold information about a configuration + item. It contains the key name, description, a list of related names, + and the group this item is contained in. + """ + + def __init__(self): + self._groups = [] + self._related = [] + self._name = "" + self._desc = "" + + def groups(self): + return self._groups + + def name(self): + return self._name + + def description(self): + return self._desc + + def related(self): + return self._related + + def setName(self, name): + self._name = name + + def setDescription(self, desc): + self._desc = desc + + def addGroup(self, group): + self._groups.append(group) + + def addRelation(self, relation): + self._related.append(relation) + + def sort(self): + self._related.sort() + self._groups.sort() + + +class Documentation: + """ + Holds the documentation... with mappings from key to items... + """ + + def __init__(self): + self.__keys = {} + self.__groups = {} + + def insert_doc_item(self, item): + """ + Insert the Doc Item into the internal list + of representation + """ + item.sort() + self.__keys[item.name()] = item + + for group in item.groups(): + if not group in self.__groups: + self.__groups[group] = [] + self.__groups[group].append(item) + self.__groups[group].sort() + + + def doc_item(self, key): + """ + Return the DocumentationInstance describing the key + """ + try: + return self.__keys[key] + except KeyError: + return None + + def doc_keys(self): + """ + Return the documented KEYS (names) + """ + return self.__keys.keys() + + def groups(self): + """ + Return the names of available groups + """ + return self.__groups.keys() + + def group_content(self, group_name): + """ + Return a list of keys/names that are in a specefic + group or the empty list + """ + try: + return self.__groups[group_name] + except KeyError: + return [] + + +def parse_cmdline(args): + """ + Parse the CMD line and return the result as a n-tuple + """ + + parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__, __version__)) + usage = """%prog [options] + +Create a set of html pages (documentation) for a bitbake.conf.... +""" + + # Add the needed options + parser.add_option( "-c", "--config", help = "Use the specified configuration file as source", + action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") ) + + parser.add_option( "-o", "--output", help = "Output directory for html files", + action = "store", dest = "output", default = "html/" ) + + parser.add_option( "-D", "--debug", help = "Increase the debug level", + action = "count", dest = "debug", default = 0 ) + + parser.add_option( "-v", "--verbose", help = "output more chit-char to the terminal", + action = "store_true", dest = "verbose", default = False ) + + options, args = parser.parse_args( sys.argv ) + + bb.msg.init_msgconfig(options.verbose, options.debug) + + return options.config, options.output + +def main(): + """ + The main Method + """ + + (config_file, output_dir) = parse_cmdline( sys.argv ) + + # right to let us load the file now + try: + documentation = bb.parse.handle( config_file, bb.data.init() ) + except IOError: + bb.fatal( "Unable to open %s" % config_file ) + except bb.parse.ParseError: + bb.fatal( "Unable to parse %s" % config_file ) + + if isinstance(documentation, dict): + documentation = documentation[""] + + # Assuming we've the file loaded now, we will initialize the 'tree' + doc = Documentation() + + # defined states + state_begin = 0 + state_see = 1 + state_group = 2 + + for key in bb.data.keys(documentation): + data = documentation.getVarFlag(key, "doc", False) + if not data: + continue + + # The Documentation now starts + doc_ins = DocumentationItem() + doc_ins.setName(key) + + + tokens = data.split(' ') + state = state_begin + string= "" + for token in tokens: + token = token.strip(',') + + if not state == state_see and token == "@see": + state = state_see + continue + elif not state == state_group and token == "@group": + state = state_group + continue + + if state == state_begin: + string += " %s" % token + elif state == state_see: + doc_ins.addRelation(token) + elif state == state_group: + doc_ins.addGroup(token) + + # set the description + doc_ins.setDescription(string) + doc.insert_doc_item(doc_ins) + + # let us create the HTML now + bb.utils.mkdirhier(output_dir) + os.chdir(output_dir) + + # Let us create the sites now. We do it in the following order + # Start with the index.html. It will point to sites explaining all + # keys and groups + html_slave = HTMLFormatter() + + f = file('style.css', 'w') + print >> f, html_slave.createCSS() + + f = file('index.html', 'w') + print >> f, html_slave.createIndex() + + f = file('all_groups.html', 'w') + print >> f, html_slave.createGroupsSite(doc) + + f = file('all_keys.html', 'w') + print >> f, html_slave.createKeysSite(doc) + + # now for each group create the site + for group in doc.groups(): + f = file('group%s.html' % group, 'w') + print >> f, html_slave.createGroupSite(group, doc.group_content(group)) + + # now for the keys + for key in doc.doc_keys(): + f = file('key%s.html' % doc.doc_item(key).name(), 'w') + print >> f, html_slave.createKeySite(doc.doc_item(key)) + + +if __name__ == "__main__": + main() diff --git a/poky/bitbake/bin/git-make-shallow b/poky/bitbake/bin/git-make-shallow new file mode 100755 index 000000000..296d3a3db --- /dev/null +++ b/poky/bitbake/bin/git-make-shallow @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""git-make-shallow: make the current git repository shallow + +Remove the history of the specified revisions, then optionally filter the +available refs to those specified. +""" + +import argparse +import collections +import errno +import itertools +import os +import subprocess +import sys + +version = 1.0 + + +def main(): + if sys.version_info < (3, 4, 0): + sys.exit('Python 3.4 or greater is required') + + git_dir = check_output(['git', 'rev-parse', '--git-dir']).rstrip() + shallow_file = os.path.join(git_dir, 'shallow') + if os.path.exists(shallow_file): + try: + check_output(['git', 'fetch', '--unshallow']) + except subprocess.CalledProcessError: + try: + os.unlink(shallow_file) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + args = process_args() + revs = check_output(['git', 'rev-list'] + args.revisions).splitlines() + + make_shallow(shallow_file, args.revisions, args.refs) + + ref_revs = check_output(['git', 'rev-list'] + args.refs).splitlines() + remaining_history = set(revs) & set(ref_revs) + for rev in remaining_history: + if check_output(['git', 'rev-parse', '{}^@'.format(rev)]): + sys.exit('Error: %s was not made shallow' % rev) + + filter_refs(args.refs) + + if args.shrink: + shrink_repo(git_dir) + subprocess.check_call(['git', 'fsck', '--unreachable']) + + +def process_args(): + # TODO: add argument to automatically keep local-only refs, since they + # can't be easily restored with a git fetch. + parser = argparse.ArgumentParser(description='Remove the history of the specified revisions, then optionally filter the available refs to those specified.') + parser.add_argument('--ref', '-r', metavar='REF', action='append', dest='refs', help='remove all but the specified refs (cumulative)') + parser.add_argument('--shrink', '-s', action='store_true', help='shrink the git repository by repacking and pruning') + parser.add_argument('revisions', metavar='REVISION', nargs='+', help='a git revision/commit') + if len(sys.argv) < 2: + parser.print_help() + sys.exit(2) + + args = parser.parse_args() + + if args.refs: + args.refs = check_output(['git', 'rev-parse', '--symbolic-full-name'] + args.refs).splitlines() + else: + args.refs = get_all_refs(lambda r, t, tt: t == 'commit' or tt == 'commit') + + args.refs = list(filter(lambda r: not r.endswith('/HEAD'), args.refs)) + args.revisions = check_output(['git', 'rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines() + return args + + +def check_output(cmd, input=None): + return subprocess.check_output(cmd, universal_newlines=True, input=input) + + +def make_shallow(shallow_file, revisions, refs): + """Remove the history of the specified revisions.""" + for rev in follow_history_intersections(revisions, refs): + print("Processing %s" % rev) + with open(shallow_file, 'a') as f: + f.write(rev + '\n') + + +def get_all_refs(ref_filter=None): + """Return all the existing refs in this repository, optionally filtering the refs.""" + ref_output = check_output(['git', 'for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)']) + ref_split = [tuple(iter_extend(l.rsplit('\t'), 3)) for l in ref_output.splitlines()] + if ref_filter: + ref_split = (e for e in ref_split if ref_filter(*e)) + refs = [r[0] for r in ref_split] + return refs + + +def iter_extend(iterable, length, obj=None): + """Ensure that iterable is the specified length by extending with obj.""" + return itertools.islice(itertools.chain(iterable, itertools.repeat(obj)), length) + + +def filter_refs(refs): + """Remove all but the specified refs from the git repository.""" + all_refs = get_all_refs() + to_remove = set(all_refs) - set(refs) + if to_remove: + check_output(['xargs', '-0', '-n', '1', 'git', 'update-ref', '-d', '--no-deref'], + input=''.join(l + '\0' for l in to_remove)) + + +def follow_history_intersections(revisions, refs): + """Determine all the points where the history of the specified revisions intersects the specified refs.""" + queue = collections.deque(revisions) + seen = set() + + for rev in iter_except(queue.popleft, IndexError): + if rev in seen: + continue + + parents = check_output(['git', 'rev-parse', '%s^@' % rev]).splitlines() + + yield rev + seen.add(rev) + + if not parents: + continue + + check_refs = check_output(['git', 'merge-base', '--independent'] + sorted(refs)).splitlines() + for parent in parents: + for ref in check_refs: + print("Checking %s vs %s" % (parent, ref)) + try: + merge_base = check_output(['git', 'merge-base', parent, ref]).rstrip() + except subprocess.CalledProcessError: + continue + else: + queue.append(merge_base) + + +def iter_except(func, exception, start=None): + """Yield a function repeatedly until it raises an exception.""" + try: + if start is not None: + yield start() + while True: + yield func() + except exception: + pass + + +def shrink_repo(git_dir): + """Shrink the newly shallow repository, removing the unreachable objects.""" + subprocess.check_call(['git', 'reflog', 'expire', '--expire-unreachable=now', '--all']) + subprocess.check_call(['git', 'repack', '-ad']) + try: + os.unlink(os.path.join(git_dir, 'objects', 'info', 'alternates')) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + subprocess.check_call(['git', 'prune', '--expire', 'now']) + + +if __name__ == '__main__': + main() diff --git a/poky/bitbake/bin/toaster b/poky/bitbake/bin/toaster new file mode 100755 index 000000000..ed365ee82 --- /dev/null +++ b/poky/bitbake/bin/toaster @@ -0,0 +1,314 @@ +#!/bin/echo ERROR: This script needs to be sourced. Please run as . + +# toaster - shell script to start Toaster + +# Copyright (C) 2013-2015 Intel Corp. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +HELP=" +Usage: source toaster start|stop [webport=] [noweb] [nobuild] + Optional arguments: + [nobuild] Setup the environment for capturing builds with toaster but disable managed builds + [noweb] Setup the environment for capturing builds with toaster but don't start the web server + [webport] Set the development server (default: localhost:8000) +" + +custom_extention() +{ + custom_extension=$BBBASEDIR/lib/toaster/orm/fixtures/custom_toaster_append.sh + if [ -f $custom_extension ] ; then + $custom_extension $* + fi +} + +databaseCheck() +{ + retval=0 + # you can always add a superuser later via + # ../bitbake/lib/toaster/manage.py createsuperuser --username= + $MANAGE migrate --noinput || retval=1 + + if [ $retval -eq 1 ]; then + echo "Failed migrations, aborting system start" 1>&2 + return $retval + fi + # Make sure that checksettings can pick up any value for TEMPLATECONF + export TEMPLATECONF + $MANAGE checksettings --traceback || retval=1 + + if [ $retval -eq 1 ]; then + printf "\nError while checking settings; aborting\n" + return $retval + fi + + return $retval +} + +webserverKillAll() +{ + local pidfile + if [ -f ${BUILDDIR}/.toastermain.pid ] ; then + custom_extention web_stop_postpend + else + custom_extention noweb_stop_postpend + fi + for pidfile in ${BUILDDIR}/.toastermain.pid ${BUILDDIR}/.runbuilds.pid; do + if [ -f ${pidfile} ]; then + pid=`cat ${pidfile}` + while kill -0 $pid 2>/dev/null; do + kill -SIGTERM $pid 2>/dev/null + sleep 1 + done + rm ${pidfile} + fi + done +} + +webserverStartAll() +{ + # do not start if toastermain points to a valid process + if ! cat "${BUILDDIR}/.toastermain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then + retval=1 + rm "${BUILDDIR}/.toastermain.pid" + fi + + retval=0 + + # check the database + databaseCheck || return 1 + + echo "Starting webserver..." + + $MANAGE runserver --noreload "$ADDR_PORT" \ + >${BUILDDIR}/toaster_web.log 2>&1 \ + & echo $! >${BUILDDIR}/.toastermain.pid + + sleep 1 + + if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then + retval=1 + rm "${BUILDDIR}/.toastermain.pid" + else + echo "Toaster development webserver started at http://$ADDR_PORT" + echo -e "\nYou can now run 'bitbake ' on the command line and monitor your build in Toaster.\nYou can also use a Toaster project to configure and run a build.\n" + custom_extention web_start_postpend $ADDR_PORT + fi + + return $retval +} + +INSTOPSYSTEM=0 + +# define the stop command +stop_system() +{ + # prevent reentry + if [ $INSTOPSYSTEM -eq 1 ]; then return; fi + INSTOPSYSTEM=1 + webserverKillAll + # unset exported variables + unset TOASTER_DIR + unset BITBAKE_UI + unset BBBASEDIR + trap - SIGHUP + #trap - SIGCHLD + INSTOPSYSTEM=0 +} + +verify_prereq() { + # Verify Django version + reqfile=$(python3 -c "import os; print(os.path.realpath('$BBBASEDIR/toaster-requirements.txt'))") + exp='s/Django\([><=]\+\)\([^,]\+\),\([><=]\+\)\(.\+\)/' + # expand version parts to 2 digits to support 1.10.x > 1.8 + # (note:helper functions hard to insert in-line) + exp=$exp'import sys,django;' + exp=$exp'version=["%02d" % int(n) for n in django.get_version().split(".")];' + exp=$exp'vmin=["%02d" % int(n) for n in "\2".split(".")];' + exp=$exp'vmax=["%02d" % int(n) for n in "\4".split(".")];' + exp=$exp'sys.exit(not (version \1 vmin and version \3 vmax))' + exp=$exp'/p' + if ! sed -n "$exp" $reqfile | python3 - ; then + req=`grep ^Django $reqfile` + echo "This program needs $req" + echo "Please install with pip3 install -r $reqfile" + return 2 + fi + + return 0 +} + +# read command line parameters +if [ -n "$BASH_SOURCE" ] ; then + TOASTER=${BASH_SOURCE} +elif [ -n "$ZSH_NAME" ] ; then + TOASTER=${(%):-%x} +else + TOASTER=$0 +fi + +export BBBASEDIR=`dirname $TOASTER`/.. +MANAGE="python3 $BBBASEDIR/lib/toaster/manage.py" +OE_ROOT=`dirname $TOASTER`/../.. + +# this is the configuraton file we are using for toaster +# we are using the same logic that oe-setup-builddir uses +# (based on TEMPLATECONF and .templateconf) to determine +# which toasterconf.json to use. +# note: There are a number of relative path assumptions +# in the local layers that currently make using an arbitrary +# toasterconf.json difficult. + +. $OE_ROOT/.templateconf +if [ -n "$TEMPLATECONF" ]; then + if [ ! -d "$TEMPLATECONF" ]; then + # Allow TEMPLATECONF=meta-xyz/conf as a shortcut + if [ -d "$OE_ROOT/$TEMPLATECONF" ]; then + TEMPLATECONF="$OE_ROOT/$TEMPLATECONF" + fi + fi +fi + +unset OE_ROOT + + +WEBSERVER=1 +export TOASTER_BUILDSERVER=1 +ADDR_PORT="localhost:8000" +unset CMD +for param in $*; do + case $param in + noweb ) + WEBSERVER=0 + ;; + nobuild ) + TOASTER_BUILDSERVER=0 + ;; + start ) + CMD=$param + ;; + stop ) + CMD=$param + ;; + webport=*) + ADDR_PORT="${param#*=}" + # Split the addr:port string + ADDR=`echo $ADDR_PORT | cut -f 1 -d ':'` + PORT=`echo $ADDR_PORT | cut -f 2 -d ':'` + # If only a port has been speified then set address to localhost. + if [ $ADDR = $PORT ] ; then + ADDR_PORT="localhost:$PORT" + fi + ;; + --help) + echo "$HELP" + return 0 + ;; + *) + echo "$HELP" + return 1 + ;; + + esac +done + +if [ `basename \"$0\"` = `basename \"${TOASTER}\"` ]; then + echo "Error: This script needs to be sourced. Please run as . $TOASTER" + return 1 +fi + +verify_prereq || return 1 + +# We make sure we're running in the current shell and in a good environment +if [ -z "$BUILDDIR" ] || ! which bitbake >/dev/null 2>&1 ; then + echo "Error: Build environment is not setup or bitbake is not in path." 1>&2 + return 2 +fi + +# this defines the dir toaster will use for +# 1) clones of layers (in _toaster_clones ) +# 2) the build dir (in build) +# 3) the sqlite db if that is being used. +# 4) pid's we need to clean up on exit/shutdown +export TOASTER_DIR=`dirname $BUILDDIR` +export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR" + +# Determine the action. If specified by arguments, fine, if not, toggle it +if [ "$CMD" = "start" ] ; then + if [ -n "$BBSERVER" ]; then + echo " Toaster is already running. Exiting..." + return 1 +fi +elif [ "$CMD" = "" ]; then + echo "No command specified" + echo "$HELP" + return 1 +fi + +echo "The system will $CMD." + +# Execute the commands +custom_extention toaster_prepend $CMD $ADDR_PORT + +case $CMD in + start ) + # check if addr:port is not in use + if [ "$CMD" == 'start' ]; then + if [ $WEBSERVER -gt 0 ]; then + $MANAGE checksocket "$ADDR_PORT" || return 1 + fi + fi + + # Create configuration file + conf=${BUILDDIR}/conf/local.conf + line='INHERIT+="toaster buildhistory"' + grep -q "$line" $conf || echo $line >> $conf + + if [ $WEBSERVER -eq 0 ] ; then + # Do not update the database for "noweb" unless + # it does not yet exist + if [ ! -f "$TOASTER_DIR/toaster.sqlite" ] ; then + if ! databaseCheck; then + echo "Failed ${CMD}." + return 4 + fi + fi + custom_extention noweb_start_postpend $ADDR_PORT + fi + if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then + echo "Failed ${CMD}." + return 4 + fi + export BITBAKE_UI='toasterui' + if [ $TOASTER_BUILDSERVER -eq 1 ] ; then + $MANAGE runbuilds \ + >${BUILDDIR}/toaster_runbuilds.log 2>&1 \ + & echo $! >${BUILDDIR}/.runbuilds.pid + else + echo "Toaster build server not started." + fi + + # set fail safe stop system on terminal exit + trap stop_system SIGHUP + echo "Successful ${CMD}." + custom_extention toaster_postpend $CMD $ADDR_PORT + return 0 + ;; + stop ) + stop_system + echo "Successful ${CMD}." + ;; +esac +custom_extention toaster_postpend $CMD $ADDR_PORT + diff --git a/poky/bitbake/bin/toaster-eventreplay b/poky/bitbake/bin/toaster-eventreplay new file mode 100755 index 000000000..80967a093 --- /dev/null +++ b/poky/bitbake/bin/toaster-eventreplay @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2014 Alex Damian +# +# This file re-uses code spread throughout other Bitbake source files. +# As such, all other copyrights belong to their own right holders. +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +This command takes a filename as a single parameter. The filename is read +as a build eventlog, and the ToasterUI is used to process events in the file +and log data in the database +""" + +import os +import sys +import json +import pickle +import codecs + +from collections import namedtuple + +# mangle syspath to allow easy import of modules +from os.path import join, dirname, abspath +sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib')) + +import bb.cooker +from bb.ui import toasterui + +class EventPlayer: + """Emulate a connection to a bitbake server.""" + + def __init__(self, eventfile, variables): + self.eventfile = eventfile + self.variables = variables + self.eventmask = [] + + def waitEvent(self, _timeout): + """Read event from the file.""" + line = self.eventfile.readline().strip() + if not line: + return + try: + event_str = json.loads(line)['vars'].encode('utf-8') + event = pickle.loads(codecs.decode(event_str, 'base64')) + event_name = "%s.%s" % (event.__module__, event.__class__.__name__) + if event_name not in self.eventmask: + return + return event + except ValueError as err: + print("Failed loading ", line) + raise err + + def runCommand(self, command_line): + """Emulate running a command on the server.""" + name = command_line[0] + + if name == "getVariable": + var_name = command_line[1] + variable = self.variables.get(var_name) + if variable: + return variable['v'], None + return None, "Missing variable %s" % var_name + + elif name == "getAllKeysWithFlags": + dump = {} + flaglist = command_line[1] + for key, val in self.variables.items(): + try: + if not key.startswith("__"): + dump[key] = { + 'v': val['v'], + 'history' : val['history'], + } + for flag in flaglist: + dump[key][flag] = val[flag] + except Exception as err: + print(err) + return (dump, None) + + elif name == 'setEventMask': + self.eventmask = command_line[-1] + return True, None + + else: + raise Exception("Command %s not implemented" % command_line[0]) + + def getEventHandle(self): + """ + This method is called by toasterui. + The return value is passed to self.runCommand but not used there. + """ + pass + +def main(argv): + with open(argv[-1]) as eventfile: + # load variables from the first line + variables = json.loads(eventfile.readline().strip())['allvariables'] + + params = namedtuple('ConfigParams', ['observe_only'])(True) + player = EventPlayer(eventfile, variables) + + return toasterui.main(player, player, params) + +# run toaster ui on our mock bitbake class +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: %s " % os.path.basename(sys.argv[0])) + sys.exit(1) + + sys.exit(main(sys.argv)) -- cgit v1.2.3