diff options
Diffstat (limited to 'poky/bitbake')
101 files changed, 8965 insertions, 904 deletions
diff --git a/poky/bitbake/bin/bitbake b/poky/bitbake/bin/bitbake index 6c73710cf..61db6b70f 100755 --- a/poky/bitbake/bin/bitbake +++ b/poky/bitbake/bin/bitbake @@ -26,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException if sys.getfilesystemencoding() != "utf-8": sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") -__version__ = "1.46.0" +__version__ = "1.47.0" if __name__ == "__main__": if __version__ != bb.__version__: diff --git a/poky/bitbake/bin/bitbake-layers b/poky/bitbake/bin/bitbake-layers index 149f1b1ac..ff085d674 100755 --- a/poky/bitbake/bin/bitbake-layers +++ b/poky/bitbake/bin/bitbake-layers @@ -14,7 +14,6 @@ import logging import os import sys import argparse -import signal bindir = os.path.dirname(__file__) topdir = os.path.dirname(bindir) @@ -26,7 +25,6 @@ import bb.msg logger = bb.msg.logger_create('bitbake-layers', sys.stdout) def main(): - signal.signal(signal.SIGPIPE, signal.SIG_DFL) parser = argparse.ArgumentParser( description="BitBake layers utility", epilog="Use %(prog)s <subcommand> --help to get help on a specific command", diff --git a/poky/bitbake/bin/bitbake-selftest b/poky/bitbake/bin/bitbake-selftest index 041a2719f..6c0737416 100755 --- a/poky/bitbake/bin/bitbake-selftest +++ b/poky/bitbake/bin/bitbake-selftest @@ -18,6 +18,7 @@ except RuntimeError as exc: sys.exit(str(exc)) tests = ["bb.tests.codeparser", + "bb.tests.color", "bb.tests.cooker", "bb.tests.cow", "bb.tests.data", @@ -26,6 +27,7 @@ tests = ["bb.tests.codeparser", "bb.tests.parse", "bb.tests.persist_data", "bb.tests.runqueue", + "bb.tests.siggen", "bb.tests.utils", "hashserv.tests", "layerindexlib.tests.layerindexobj", diff --git a/poky/bitbake/bin/bitbake-server b/poky/bitbake/bin/bitbake-server new file mode 100755 index 000000000..ffbc7894e --- /dev/null +++ b/poky/bitbake/bin/bitbake-server @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2020 Richard Purdie +# + +import os +import sys +import warnings +import logging +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +if sys.getfilesystemencoding() != "utf-8": + sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") + +# Users shouldn't be running this code directly +if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"): + print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.") + sys.exit(1) + +import bb.server.process + +lockfd = int(sys.argv[2]) +readypipeinfd = int(sys.argv[3]) +logfile = sys.argv[4] +lockname = sys.argv[5] +sockname = sys.argv[6] +timeout = sys.argv[7] +xmlrpcinterface = (sys.argv[8], int(sys.argv[9])) +if xmlrpcinterface[0] == "None": + xmlrpcinterface = (None, xmlrpcinterface[1]) +if timeout == "None": + timeout = None + +# Replace standard fds with our own +with open('/dev/null', 'r') as si: + os.dup2(si.fileno(), sys.stdin.fileno()) + +so = open(logfile, 'a+') +os.dup2(so.fileno(), sys.stdout.fileno()) +os.dup2(so.fileno(), sys.stderr.fileno()) + +# Have stdout and stderr be the same so log output matches chronologically +# and there aren't two seperate buffers +sys.stderr = sys.stdout + +logger = logging.getLogger("BitBake") +# Ensure logging messages get sent to the UI as events +handler = bb.event.LogHandler() +logger.addHandler(handler) + +bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface) + diff --git a/poky/bitbake/bin/bitbake-worker b/poky/bitbake/bin/bitbake-worker index 97cc0fd60..9334f11fb 100755 --- a/poky/bitbake/bin/bitbake-worker +++ b/poky/bitbake/bin/bitbake-worker @@ -413,9 +413,9 @@ class BitbakeWorker(object): def handle_workerdata(self, data): self.workerdata = pickle.loads(data) + bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"] + bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"] bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"] - bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"] - bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"] bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] for mc in self.databuilder.mcdata: self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"]) diff --git a/poky/bitbake/contrib/bbparse-torture.py b/poky/bitbake/contrib/bbparse-torture.py new file mode 100755 index 000000000..c25d547bb --- /dev/null +++ b/poky/bitbake/contrib/bbparse-torture.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python3 +# +# Copyright (C) 2020 Joshua Watt <JPEWhacker@gmail.com> +# +# SPDX-License-Identifier: MIT + +import argparse +import os +import random +import shutil +import signal +import subprocess +import sys +import time + + +def try_unlink(path): + try: + os.unlink(path) + except: + pass + + +def main(): + def cleanup(): + shutil.rmtree("tmp/cache", ignore_errors=True) + try_unlink("bitbake-cookerdaemon.log") + try_unlink("bitbake.sock") + try_unlink("bitbake.lock") + + parser = argparse.ArgumentParser( + description="Bitbake parser torture test", + epilog=""" + A torture test for bitbake's parser. Repeatedly interrupts parsing until + bitbake decides to deadlock. + """, + ) + + args = parser.parse_args() + + if not "BUILDDIR" in os.environ: + print( + "'BUILDDIR' not found in the environment. Did you initialize the build environment?" + ) + return 1 + + os.chdir(os.environ["BUILDDIR"]) + + run_num = 0 + while True: + if run_num % 100 == 0: + print("Calibrating wait time...") + cleanup() + + start_time = time.monotonic() + r = subprocess.run(["bitbake", "-p"]) + max_wait_time = time.monotonic() - start_time + + if r.returncode != 0: + print("Calibration run exited with %d" % r.returncode) + return 1 + + print("Maximum wait time is %f seconds" % max_wait_time) + + run_num += 1 + wait_time = random.random() * max_wait_time + + print("Run #%d" % run_num) + print("Will sleep for %f seconds" % wait_time) + + cleanup() + with subprocess.Popen(["bitbake", "-p"]) as proc: + time.sleep(wait_time) + proc.send_signal(signal.SIGINT) + try: + proc.wait(45) + except subprocess.TimeoutExpired: + print("Run #%d: Waited too long. Possible deadlock!" % run_num) + proc.wait() + return 1 + + if proc.returncode == 0: + print("Exited successfully. Timeout too long?") + else: + print("Exited with %d" % proc.returncode) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/poky/bitbake/contrib/vim/LICENSE.txt b/poky/bitbake/contrib/vim/LICENSE.txt new file mode 100644 index 000000000..c7d915024 --- /dev/null +++ b/poky/bitbake/contrib/vim/LICENSE.txt @@ -0,0 +1,18 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/poky/bitbake/contrib/vim/ftdetect/bitbake.vim b/poky/bitbake/contrib/vim/ftdetect/bitbake.vim index 200f8ae49..09fc4dc74 100644 --- a/poky/bitbake/contrib/vim/ftdetect/bitbake.vim +++ b/poky/bitbake/contrib/vim/ftdetect/bitbake.vim @@ -6,12 +6,12 @@ " " This sets up the syntax highlighting for BitBake files, like .bb, .bbclass and .inc -if &compatible || version < 600 +if &compatible || version < 600 || exists("b:loaded_bitbake_plugin") finish endif " .bb, .bbappend and .bbclass -au BufNewFile,BufRead *.{bb,bbappend,bbclass} set filetype=bitbake +au BufNewFile,BufRead *.{bb,bbappend,bbclass} set filetype=bitbake " .inc au BufNewFile,BufRead *.inc set filetype=bitbake diff --git a/poky/bitbake/contrib/vim/ftplugin/bitbake.vim b/poky/bitbake/contrib/vim/ftplugin/bitbake.vim index db0d75319..9e8d3e13c 100644 --- a/poky/bitbake/contrib/vim/ftplugin/bitbake.vim +++ b/poky/bitbake/contrib/vim/ftplugin/bitbake.vim @@ -1,2 +1,13 @@ -set sts=4 sw=4 et -set cms=#%s +" Only do this when not done yet for this buffer +if exists("b:did_ftplugin") + finish +endif + +" Don't load another plugin for this buffer +let b:did_ftplugin = 1 + +let b:undo_ftplugin = "setl cms< sts< sw< et< sua<" + +setlocal commentstring=#\ %s +setlocal softtabstop=4 shiftwidth=4 expandtab +setlocal suffixesadd+=.bb,.bbclass diff --git a/poky/bitbake/contrib/vim/plugin/newbb.vim b/poky/bitbake/contrib/vim/plugin/newbb.vim index 874e33805..3a4202736 100755..100644 --- a/poky/bitbake/contrib/vim/plugin/newbb.vim +++ b/poky/bitbake/contrib/vim/plugin/newbb.vim @@ -10,7 +10,7 @@ " " Will try to use git to find the user name and email -if &compatible || v:version < 600 +if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin") finish endif @@ -25,7 +25,7 @@ endfun fun! <SID>GetUserEmail() let l:user_email = system("git config --get user.email") if v:shell_error - return "unknow@user.org" + return "unknown@user.org" else return substitute(l:user_email, "\n", "", "") endfun @@ -41,6 +41,10 @@ fun! BBHeader() endfun fun! NewBBTemplate() + if line2byte(line('$') + 1) != -1 + return + endif + let l:paste = &paste set nopaste @@ -48,7 +52,7 @@ fun! NewBBTemplate() call BBHeader() " New the bb template - put ='DESCRIPTION = \"\"' + put ='SUMMARY = \"\"' put ='HOMEPAGE = \"\"' put ='LICENSE = \"\"' put ='SECTION = \"\"' @@ -58,7 +62,7 @@ fun! NewBBTemplate() " Go to the first place to edit 0 - /^DESCRIPTION =/ + /^SUMMARY =/ exec "normal 2f\"" if paste == 1 @@ -76,7 +80,7 @@ if v:progname =~ "vimdiff" endif augroup NewBB - au BufNewFile *.bb + au BufNewFile,BufReadPost *.bb \ if g:bb_create_on_empty | \ call NewBBTemplate() | \ endif diff --git a/poky/bitbake/contrib/vim/plugin/newbbappend.vim b/poky/bitbake/contrib/vim/plugin/newbbappend.vim new file mode 100644 index 000000000..e04174cf6 --- /dev/null +++ b/poky/bitbake/contrib/vim/plugin/newbbappend.vim @@ -0,0 +1,46 @@ +" Vim plugin file +" Purpose: Create a template for new bbappend file +" Author: Joshua Watt <JPEWhacker@gmail.com> +" Copyright: Copyright (C) 2017 Joshua Watt <JPEWhacker@gmail.com> +" +" This file is licensed under the MIT license, see COPYING.MIT in +" this source distribution for the terms. +" + +if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin") + finish +endif + +fun! NewBBAppendTemplate() + if line2byte(line('$') + 1) != -1 + return + endif + + let l:paste = &paste + set nopaste + + " New bbappend template + 0 put ='FILESEXTRAPATHS_prepend := \"${THISDIR}/${PN}:\"' + 2 + + if paste == 1 + set paste + endif +endfun + +if !exists("g:bb_create_on_empty") + let g:bb_create_on_empty = 1 +endif + +" disable in case of vimdiff +if v:progname =~ "vimdiff" + let g:bb_create_on_empty = 0 +endif + +augroup NewBBAppend + au BufNewFile,BufReadPost *.bbappend + \ if g:bb_create_on_empty | + \ call NewBBAppendTemplate() | + \ endif +augroup END + diff --git a/poky/bitbake/contrib/vim/syntax/bitbake.vim b/poky/bitbake/contrib/vim/syntax/bitbake.vim index fb55f9102..f964621ae 100644 --- a/poky/bitbake/contrib/vim/syntax/bitbake.vim +++ b/poky/bitbake/contrib/vim/syntax/bitbake.vim @@ -12,7 +12,7 @@ " " It's an entirely new type, just has specific syntax in shell and python code -if &compatible || v:version < 600 +if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin") finish endif if exists("b:current_syntax") @@ -58,8 +58,8 @@ syn match bbVarValue ".*$" contained contains=bbString,bbVarDeref,bbV syn region bbVarPyValue start=+${@+ skip=+\\$+ end=+}+ contained contains=@python " Vars metadata flags -syn match bbVarFlagDef "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag -syn region bbVarFlagFlag matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq +syn match bbVarFlagDef "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.+]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag +syn region bbVarFlagFlag matchgroup=bbArrayBrackets start="\[" end="\]\s*\(:=\|=\|.=\|=.|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq " Includes and requires syn keyword bbInclude inherit include require contained @@ -67,15 +67,15 @@ syn match bbIncludeRest ".*$" contained contains=bbString,bbVarDeref syn match bbIncludeLine "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest " Add taks and similar -syn keyword bbStatement addtask addhandler after before EXPORT_FUNCTIONS contained +syn keyword bbStatement addtask deltask addhandler after before EXPORT_FUNCTIONS contained syn match bbStatementRest ".*$" skipwhite contained contains=bbStatement -syn match bbStatementLine "^\(addtask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest +syn match bbStatementLine "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest " OE Important Functions syn keyword bbOEFunctions do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained " Generic Functions -syn match bbFunction "\h[0-9A-Za-z_-]*" display contained contains=bbOEFunctions +syn match bbFunction "\h[0-9A-Za-z_\-\.]*" display contained contains=bbOEFunctions " BitBake shell metadata syn include @shell syntax/sh.vim @@ -83,7 +83,7 @@ if exists("b:current_syntax") unlet b:current_syntax endif syn keyword bbShFakeRootFlag fakeroot contained -syn match bbShFuncDef "^\(fakeroot\s*\)\?\([0-9A-Za-z_${}-]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite +syn match bbShFuncDef "^\(fakeroot\s*\)\?\([\.0-9A-Za-z_${}\-\.]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite syn region bbShFuncRegion matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@shell " Python value inside shell functions @@ -91,7 +91,7 @@ syn region shDeref start=+${@+ skip=+\\$+ excludenl end=+}+ contained co " BitBake python metadata syn keyword bbPyFlag python contained -syn match bbPyFuncDef "^\(python\s\+\)\([0-9A-Za-z_${}-]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite +syn match bbPyFuncDef "^\(fakeroot\s*\)\?\(python\)\(\s\+[0-9A-Za-z_${}\-\.]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite syn region bbPyFuncRegion matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@python " BitBake 'def'd python functions diff --git a/poky/bitbake/doc/.gitignore b/poky/bitbake/doc/.gitignore new file mode 100644 index 000000000..69fa449dd --- /dev/null +++ b/poky/bitbake/doc/.gitignore @@ -0,0 +1 @@ +_build/ diff --git a/poky/bitbake/doc/Makefile.sphinx b/poky/bitbake/doc/Makefile.sphinx new file mode 100644 index 000000000..c663c2954 --- /dev/null +++ b/poky/bitbake/doc/Makefile.sphinx @@ -0,0 +1,31 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +DESTDIR = final + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile.sphinx clean publish + +publish: Makefile.sphinx html singlehtml + rm -rf $(BUILDDIR)/$(DESTDIR)/ + mkdir -p $(BUILDDIR)/$(DESTDIR)/ + cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/ + cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html + sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html + +clean: + @rm -rf $(BUILDDIR) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile.sphinx + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/poky/bitbake/doc/_templates/breadcrumbs.html b/poky/bitbake/doc/_templates/breadcrumbs.html new file mode 100644 index 000000000..eb6244b74 --- /dev/null +++ b/poky/bitbake/doc/_templates/breadcrumbs.html @@ -0,0 +1,14 @@ +{% extends "!breadcrumbs.html" %} + +{% block breadcrumbs %} + <li> + <span class="doctype_switcher_placeholder">{{ doctype or 'single' }}</span> + <span class="version_switcher_placeholder">{{ release }}</span> + </li> + <li> »</li> + {% for doc in parents %} + <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> »</li> + {% endfor %} + <li>{{ title }}</li> +{% endblock %} + diff --git a/poky/bitbake/doc/_templates/layout.html b/poky/bitbake/doc/_templates/layout.html new file mode 100644 index 000000000..308d5c7a2 --- /dev/null +++ b/poky/bitbake/doc/_templates/layout.html @@ -0,0 +1,7 @@ +{% extends "!layout.html" %} + +{% block extrabody %} +<div id="outdated-warning" style="text-align: center; background-color: #FFBABA; color: #6A0E0E;"> +</div> +{% endblock %} + diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst new file mode 100644 index 000000000..019afd22f --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst @@ -0,0 +1,733 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +========= +Execution +========= + +| + +The primary purpose for running BitBake is to produce some kind of +output such as a single installable package, a kernel, a software +development kit, or even a full, board-specific bootable Linux image, +complete with bootloader, kernel, and root filesystem. Of course, you +can execute the ``bitbake`` command with options that cause it to +execute single tasks, compile single recipe files, capture or clear +data, or simply return information about the execution environment. + +This chapter describes BitBake's execution process from start to finish +when you use it to create an image. The execution process is launched +using the following command form: :: + + $ bitbake target + +For information on +the BitBake command and its options, see ":ref:`The BitBake Command +<bitbake-user-manual-command>`" section. + +.. note:: + + Prior to executing BitBake, you should take advantage of available + parallel thread execution on your build host by setting the + :term:`BB_NUMBER_THREADS` variable in + your project's ``local.conf`` configuration file. + + A common method to determine this value for your build host is to run + the following: :: + + $ grep processor /proc/cpuinfo + + This command returns + the number of processors, which takes into account hyper-threading. + Thus, a quad-core build host with hyper-threading most likely shows + eight processors, which is the value you would then assign to + ``BB_NUMBER_THREADS``. + + A possibly simpler solution is that some Linux distributions (e.g. + Debian and Ubuntu) provide the ``ncpus`` command. + +Parsing the Base Configuration Metadata +======================================= + +The first thing BitBake does is parse base configuration metadata. Base +configuration metadata consists of your project's ``bblayers.conf`` file +to determine what layers BitBake needs to recognize, all necessary +``layer.conf`` files (one from each layer), and ``bitbake.conf``. The +data itself is of various types: + +- **Recipes:** Details about particular pieces of software. + +- **Class Data:** An abstraction of common build information (e.g. how to + build a Linux kernel). + +- **Configuration Data:** Machine-specific settings, policy decisions, + and so forth. Configuration data acts as the glue to bind everything + together. + +The ``layer.conf`` files are used to construct key variables such as +:term:`BBPATH` and :term:`BBFILES`. +``BBPATH`` is used to search for configuration and class files under the +``conf`` and ``classes`` directories, respectively. ``BBFILES`` is used +to locate both recipe and recipe append files (``.bb`` and +``.bbappend``). If there is no ``bblayers.conf`` file, it is assumed the +user has set the ``BBPATH`` and ``BBFILES`` directly in the environment. + +Next, the ``bitbake.conf`` file is located using the ``BBPATH`` variable +that was just constructed. The ``bitbake.conf`` file may also include +other configuration files using the ``include`` or ``require`` +directives. + +Prior to parsing configuration files, BitBake looks at certain +variables, including: + +- :term:`BB_ENV_WHITELIST` +- :term:`BB_ENV_EXTRAWHITE` +- :term:`BB_PRESERVE_ENV` +- :term:`BB_ORIGENV` +- :term:`BITBAKE_UI` + +The first four variables in this list relate to how BitBake treats shell +environment variables during task execution. By default, BitBake cleans +the environment variables and provides tight control over the shell +execution environment. However, through the use of these first four +variables, you can apply your control regarding the environment +variables allowed to be used by BitBake in the shell during execution of +tasks. See the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:Passing Information Into the Build Task Environment`" +section and the information about these variables in the variable +glossary for more information on how they work and on how to use them. + +The base configuration metadata is global and therefore affects all +recipes and tasks that are executed. + +BitBake first searches the current working directory for an optional +``conf/bblayers.conf`` configuration file. This file is expected to +contain a :term:`BBLAYERS` variable that is a +space-delimited list of 'layer' directories. Recall that if BitBake +cannot find a ``bblayers.conf`` file, then it is assumed the user has +set the ``BBPATH`` and ``BBFILES`` variables directly in the +environment. + +For each directory (layer) in this list, a ``conf/layer.conf`` file is +located and parsed with the :term:`LAYERDIR` variable +being set to the directory where the layer was found. The idea is these +files automatically set up :term:`BBPATH` and other +variables correctly for a given build directory. + +BitBake then expects to find the ``conf/bitbake.conf`` file somewhere in +the user-specified ``BBPATH``. That configuration file generally has +include directives to pull in any other metadata such as files specific +to the architecture, the machine, the local environment, and so forth. + +Only variable definitions and include directives are allowed in BitBake +``.conf`` files. Some variables directly influence BitBake's behavior. +These variables might have been set from the environment depending on +the environment variables previously mentioned or set in the +configuration files. The ":ref:`bitbake-user-manual/bitbake-user-manual-ref-variables:Variables Glossary`" +chapter presents a full list of +variables. + +After parsing configuration files, BitBake uses its rudimentary +inheritance mechanism, which is through class files, to inherit some +standard classes. BitBake parses a class when the inherit directive +responsible for getting that class is encountered. + +The ``base.bbclass`` file is always included. Other classes that are +specified in the configuration using the +:term:`INHERIT` variable are also included. BitBake +searches for class files in a ``classes`` subdirectory under the paths +in ``BBPATH`` in the same way as configuration files. + +A good way to get an idea of the configuration files and the class files +used in your execution environment is to run the following BitBake +command: :: + + $ bitbake -e > mybb.log + +Examining the top of the ``mybb.log`` +shows you the many configuration files and class files used in your +execution environment. + +.. note:: + + You need to be aware of how BitBake parses curly braces. If a recipe + uses a closing curly brace within the function and the character has + no leading spaces, BitBake produces a parsing error. If you use a + pair of curly braces in a shell function, the closing curly brace + must not be located at the start of the line without leading spaces. + + Here is an example that causes BitBake to produce a parsing error: :: + + fakeroot create_shar() { + cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + usage() + { + echo "test" + ###### The following "}" at the start of the line causes a parsing error ###### + } + EOF + } + + Writing the recipe this way avoids the error: + fakeroot create_shar() { + cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh + usage() + { + echo "test" + ###### The following "}" with a leading space at the start of the line avoids the error ###### + } + EOF + } + +Locating and Parsing Recipes +============================ + +During the configuration phase, BitBake will have set +:term:`BBFILES`. BitBake now uses it to construct a +list of recipes to parse, along with any append files (``.bbappend``) to +apply. ``BBFILES`` is a space-separated list of available files and +supports wildcards. An example would be: :: + + BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend" + +BitBake parses each +recipe and append file located with ``BBFILES`` and stores the values of +various variables into the datastore. + +.. note:: + + Append files are applied in the order they are encountered in BBFILES. + +For each file, a fresh copy of the base configuration is made, then the +recipe is parsed line by line. Any inherit statements cause BitBake to +find and then parse class files (``.bbclass``) using +:term:`BBPATH` as the search path. Finally, BitBake +parses in order any append files found in ``BBFILES``. + +One common convention is to use the recipe filename to define pieces of +metadata. For example, in ``bitbake.conf`` the recipe name and version +are used to set the variables :term:`PN` and +:term:`PV`: :: + + PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + +In this example, a recipe called "something_1.2.3.bb" would set +``PN`` to "something" and ``PV`` to "1.2.3". + +By the time parsing is complete for a recipe, BitBake has a list of +tasks that the recipe defines and a set of data consisting of keys and +values as well as dependency information about the tasks. + +BitBake does not need all of this information. It only needs a small +subset of the information to make decisions about the recipe. +Consequently, BitBake caches the values in which it is interested and +does not store the rest of the information. Experience has shown it is +faster to re-parse the metadata than to try and write it out to the disk +and then reload it. + +Where possible, subsequent BitBake commands reuse this cache of recipe +information. The validity of this cache is determined by first computing +a checksum of the base configuration data (see +:term:`BB_HASHCONFIG_WHITELIST`) and +then checking if the checksum matches. If that checksum matches what is +in the cache and the recipe and class files have not changed, BitBake is +able to use the cache. BitBake then reloads the cached information about +the recipe instead of reparsing it from scratch. + +Recipe file collections exist to allow the user to have multiple +repositories of ``.bb`` files that contain the same exact package. For +example, one could easily use them to make one's own local copy of an +upstream repository, but with custom modifications that one does not +want upstream. Here is an example: :: + + BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb" + BBFILE_COLLECTIONS = "upstream local" + BBFILE_PATTERN_upstream = "^/stuff/openembedded/" + BBFILE_PATTERN_local = "^/stuff/openembedded.modified/" + BBFILE_PRIORITY_upstream = "5" BBFILE_PRIORITY_local = "10" + +.. note:: + + The layers mechanism is now the preferred method of collecting code. + While the collections code remains, its main use is to set layer + priorities and to deal with overlap (conflicts) between layers. + +.. _bb-bitbake-providers: + +Providers +========= + +Assuming BitBake has been instructed to execute a target and that all +the recipe files have been parsed, BitBake starts to figure out how to +build the target. BitBake looks through the ``PROVIDES`` list for each +of the recipes. A ``PROVIDES`` list is the list of names by which the +recipe can be known. Each recipe's ``PROVIDES`` list is created +implicitly through the recipe's :term:`PN` variable and +explicitly through the recipe's :term:`PROVIDES` +variable, which is optional. + +When a recipe uses ``PROVIDES``, that recipe's functionality can be +found under an alternative name or names other than the implicit ``PN`` +name. As an example, suppose a recipe named ``keyboard_1.0.bb`` +contained the following: :: + + PROVIDES += "fullkeyboard" + +The ``PROVIDES`` +list for this recipe becomes "keyboard", which is implicit, and +"fullkeyboard", which is explicit. Consequently, the functionality found +in ``keyboard_1.0.bb`` can be found under two different names. + +.. _bb-bitbake-preferences: + +Preferences +=========== + +The ``PROVIDES`` list is only part of the solution for figuring out a +target's recipes. Because targets might have multiple providers, BitBake +needs to prioritize providers by determining provider preferences. + +A common example in which a target has multiple providers is +"virtual/kernel", which is on the ``PROVIDES`` list for each kernel +recipe. Each machine often selects the best kernel provider by using a +line similar to the following in the machine configuration file: :: + + PREFERRED_PROVIDER_virtual/kernel = "linux-yocto" + +The default :term:`PREFERRED_PROVIDER` is the provider +with the same name as the target. BitBake iterates through each target +it needs to build and resolves them and their dependencies using this +process. + +Understanding how providers are chosen is made complicated by the fact +that multiple versions might exist for a given provider. BitBake +defaults to the highest version of a provider. Version comparisons are +made using the same method as Debian. You can use the +:term:`PREFERRED_VERSION` variable to +specify a particular version. You can influence the order by using the +:term:`DEFAULT_PREFERENCE` variable. + +By default, files have a preference of "0". Setting +``DEFAULT_PREFERENCE`` to "-1" makes the recipe unlikely to be used +unless it is explicitly referenced. Setting ``DEFAULT_PREFERENCE`` to +"1" makes it likely the recipe is used. ``PREFERRED_VERSION`` overrides +any ``DEFAULT_PREFERENCE`` setting. ``DEFAULT_PREFERENCE`` is often used +to mark newer and more experimental recipe versions until they have +undergone sufficient testing to be considered stable. + +When there are multiple “versions” of a given recipe, BitBake defaults +to selecting the most recent version, unless otherwise specified. If the +recipe in question has a +:term:`DEFAULT_PREFERENCE` set lower than +the other recipes (default is 0), then it will not be selected. This +allows the person or persons maintaining the repository of recipe files +to specify their preference for the default selected version. +Additionally, the user can specify their preferred version. + +If the first recipe is named ``a_1.1.bb``, then the +:term:`PN` variable will be set to “a”, and the +:term:`PV` variable will be set to 1.1. + +Thus, if a recipe named ``a_1.2.bb`` exists, BitBake will choose 1.2 by +default. However, if you define the following variable in a ``.conf`` +file that BitBake parses, you can change that preference: :: + + PREFERRED_VERSION_a = "1.1" + +.. note:: + + It is common for a recipe to provide two versions -- a stable, + numbered (and preferred) version, and a version that is automatically + checked out from a source code repository that is considered more + "bleeding edge" but can be selected only explicitly. + + For example, in the OpenEmbedded codebase, there is a standard, + versioned recipe file for BusyBox, ``busybox_1.22.1.bb``, but there + is also a Git-based version, ``busybox_git.bb``, which explicitly + contains the line :: + + DEFAULT_PREFERENCE = "-1" + + to ensure that the + numbered, stable version is always preferred unless the developer + selects otherwise. + +.. _bb-bitbake-dependencies: + +Dependencies +============ + +Each target BitBake builds consists of multiple tasks such as ``fetch``, +``unpack``, ``patch``, ``configure``, and ``compile``. For best +performance on multi-core systems, BitBake considers each task as an +independent entity with its own set of dependencies. + +Dependencies are defined through several variables. You can find +information about variables BitBake uses in the +:doc:`bitbake-user-manual-ref-variables` near the end of this manual. At a +basic level, it is sufficient to know that BitBake uses the +:term:`DEPENDS` and +:term:`RDEPENDS` variables when calculating +dependencies. + +For more information on how BitBake handles dependencies, see the +:ref:`bitbake-user-manual/bitbake-user-manual-metadata:Dependencies` +section. + +.. _ref-bitbake-tasklist: + +The Task List +============= + +Based on the generated list of providers and the dependency information, +BitBake can now calculate exactly what tasks it needs to run and in what +order it needs to run them. The +:ref:`bitbake-user-manual/bitbake-user-manual-execution:executing tasks` +section has more information on how BitBake chooses which task to +execute next. + +The build now starts with BitBake forking off threads up to the limit +set in the :term:`BB_NUMBER_THREADS` +variable. BitBake continues to fork threads as long as there are tasks +ready to run, those tasks have all their dependencies met, and the +thread threshold has not been exceeded. + +It is worth noting that you can greatly speed up the build time by +properly setting the ``BB_NUMBER_THREADS`` variable. + +As each task completes, a timestamp is written to the directory +specified by the :term:`STAMP` variable. On subsequent +runs, BitBake looks in the build directory within ``tmp/stamps`` and +does not rerun tasks that are already completed unless a timestamp is +found to be invalid. Currently, invalid timestamps are only considered +on a per recipe file basis. So, for example, if the configure stamp has +a timestamp greater than the compile timestamp for a given target, then +the compile task would rerun. Running the compile task again, however, +has no effect on other providers that depend on that target. + +The exact format of the stamps is partly configurable. In modern +versions of BitBake, a hash is appended to the stamp so that if the +configuration changes, the stamp becomes invalid and the task is +automatically rerun. This hash, or signature used, is governed by the +signature policy that is configured (see the +:ref:`bitbake-user-manual/bitbake-user-manual-execution:checksums (signatures)` +section for information). It is also +possible to append extra metadata to the stamp using the +``[stamp-extra-info]`` task flag. For example, OpenEmbedded uses this +flag to make some tasks machine-specific. + +.. note:: + + Some tasks are marked as "nostamp" tasks. No timestamp file is + created when these tasks are run. Consequently, "nostamp" tasks are + always rerun. + +For more information on tasks, see the +:ref:`bitbake-user-manual/bitbake-user-manual-metadata:tasks` section. + +Executing Tasks +=============== + +Tasks can be either a shell task or a Python task. For shell tasks, +BitBake writes a shell script to +``${``\ :term:`T`\ ``}/run.do_taskname.pid`` and then +executes the script. The generated shell script contains all the +exported variables, and the shell functions with all variables expanded. +Output from the shell script goes to the file +``${T}/log.do_taskname.pid``. Looking at the expanded shell functions in +the run file and the output in the log files is a useful debugging +technique. + +For Python tasks, BitBake executes the task internally and logs +information to the controlling terminal. Future versions of BitBake will +write the functions to files similar to the way shell tasks are handled. +Logging will be handled in a way similar to shell tasks as well. + +The order in which BitBake runs the tasks is controlled by its task +scheduler. It is possible to configure the scheduler and define custom +implementations for specific use cases. For more information, see these +variables that control the behavior: + +- :term:`BB_SCHEDULER` + +- :term:`BB_SCHEDULERS` + +It is possible to have functions run before and after a task's main +function. This is done using the ``[prefuncs]`` and ``[postfuncs]`` +flags of the task that lists the functions to run. + +.. _checksums: + +Checksums (Signatures) +====================== + +A checksum is a unique signature of a task's inputs. The signature of a +task can be used to determine if a task needs to be run. Because it is a +change in a task's inputs that triggers running the task, BitBake needs +to detect all the inputs to a given task. For shell tasks, this turns +out to be fairly easy because BitBake generates a "run" shell script for +each task and it is possible to create a checksum that gives you a good +idea of when the task's data changes. + +To complicate the problem, some things should not be included in the +checksum. First, there is the actual specific build path of a given task +- the working directory. It does not matter if the working directory +changes because it should not affect the output for target packages. The +simplistic approach for excluding the working directory is to set it to +some fixed value and create the checksum for the "run" script. BitBake +goes one step better and uses the +:term:`BB_HASHBASE_WHITELIST` variable +to define a list of variables that should never be included when +generating the signatures. + +Another problem results from the "run" scripts containing functions that +might or might not get called. The incremental build solution contains +code that figures out dependencies between shell functions. This code is +used to prune the "run" scripts down to the minimum set, thereby +alleviating this problem and making the "run" scripts much more readable +as a bonus. + +So far we have solutions for shell scripts. What about Python tasks? The +same approach applies even though these tasks are more difficult. The +process needs to figure out what variables a Python function accesses +and what functions it calls. Again, the incremental build solution +contains code that first figures out the variable and function +dependencies, and then creates a checksum for the data used as the input +to the task. + +Like the working directory case, situations exist where dependencies +should be ignored. For these cases, you can instruct the build process +to ignore a dependency by using a line like the following: :: + + PACKAGE_ARCHS[vardepsexclude] = "MACHINE" + +This example ensures that the +``PACKAGE_ARCHS`` variable does not depend on the value of ``MACHINE``, +even if it does reference it. + +Equally, there are cases where we need to add dependencies BitBake is +not able to find. You can accomplish this by using a line like the +following: :: + + PACKAGE_ARCHS[vardeps] = "MACHINE" + +This example explicitly +adds the ``MACHINE`` variable as a dependency for ``PACKAGE_ARCHS``. + +Consider a case with in-line Python, for example, where BitBake is not +able to figure out dependencies. When running in debug mode (i.e. using +``-DDD``), BitBake produces output when it discovers something for which +it cannot figure out dependencies. + +Thus far, this section has limited discussion to the direct inputs into +a task. Information based on direct inputs is referred to as the +"basehash" in the code. However, there is still the question of a task's +indirect inputs - the things that were already built and present in the +build directory. The checksum (or signature) for a particular task needs +to add the hashes of all the tasks on which the particular task depends. +Choosing which dependencies to add is a policy decision. However, the +effect is to generate a master checksum that combines the basehash and +the hashes of the task's dependencies. + +At the code level, there are a variety of ways both the basehash and the +dependent task hashes can be influenced. Within the BitBake +configuration file, we can give BitBake some extra information to help +it construct the basehash. The following statement effectively results +in a list of global variable dependency excludes - variables never +included in any checksum. This example uses variables from OpenEmbedded +to help illustrate the concept: :: + + BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ + SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ + USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ + PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ + CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX" + +The previous example excludes the work directory, which is part of +``TMPDIR``. + +The rules for deciding which hashes of dependent tasks to include +through dependency chains are more complex and are generally +accomplished with a Python function. The code in +``meta/lib/oe/sstatesig.py`` shows two examples of this and also +illustrates how you can insert your own policy into the system if so +desired. This file defines the two basic signature generators +OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there +is a dummy "noop" signature handler enabled in BitBake. This means that +behavior is unchanged from previous versions. ``OE-Core`` uses the +"OEBasicHash" signature handler by default through this setting in the +``bitbake.conf`` file: :: + + BB_SIGNATURE_HANDLER ?= "OEBasicHash" + +The "OEBasicHash" ``BB_SIGNATURE_HANDLER`` is the same as the "OEBasic" +version but adds the task hash to the stamp files. This results in any +metadata change that changes the task hash, automatically causing the +task to be run again. This removes the need to bump +:term:`PR` values, and changes to metadata automatically +ripple across the build. + +It is also worth noting that the end result of these signature +generators is to make some dependency and hash information available to +the build. This information includes: + +- ``BB_BASEHASH_task-``\ *taskname*: The base hashes for each task in the + recipe. + +- ``BB_BASEHASH_``\ *filename:taskname*: The base hashes for each + dependent task. + +- ``BBHASHDEPS_``\ *filename:taskname*: The task dependencies for + each task. + +- ``BB_TASKHASH``: The hash of the currently running task. + +It is worth noting that BitBake's "-S" option lets you debug BitBake's +processing of signatures. The options passed to -S allow different +debugging modes to be used, either using BitBake's own debug functions +or possibly those defined in the metadata/signature handler itself. The +simplest parameter to pass is "none", which causes a set of signature +information to be written out into ``STAMPS_DIR`` corresponding to the +targets specified. The other currently available parameter is +"printdiff", which causes BitBake to try to establish the closest +signature match it can (e.g. in the sstate cache) and then run +``bitbake-diffsigs`` over the matches to determine the stamps and delta +where these two stamp trees diverge. + +.. note:: + + It is likely that future versions of BitBake will provide other + signature handlers triggered through additional "-S" parameters. + +You can find more information on checksum metadata in the +:ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene` +section. + +Setscene +======== + +The setscene process enables BitBake to handle "pre-built" artifacts. +The ability to handle and reuse these artifacts allows BitBake the +luxury of not having to build something from scratch every time. +Instead, BitBake can use, when possible, existing build artifacts. + +BitBake needs to have reliable data indicating whether or not an +artifact is compatible. Signatures, described in the previous section, +provide an ideal way of representing whether an artifact is compatible. +If a signature is the same, an object can be reused. + +If an object can be reused, the problem then becomes how to replace a +given task or set of tasks with the pre-built artifact. BitBake solves +the problem with the "setscene" process. + +When BitBake is asked to build a given target, before building anything, +it first asks whether cached information is available for any of the +targets it's building, or any of the intermediate targets. If cached +information is available, BitBake uses this information instead of +running the main tasks. + +BitBake first calls the function defined by the +:term:`BB_HASHCHECK_FUNCTION` variable +with a list of tasks and corresponding hashes it wants to build. This +function is designed to be fast and returns a list of the tasks for +which it believes in can obtain artifacts. + +Next, for each of the tasks that were returned as possibilities, BitBake +executes a setscene version of the task that the possible artifact +covers. Setscene versions of a task have the string "_setscene" appended +to the task name. So, for example, the task with the name ``xxx`` has a +setscene task named ``xxx_setscene``. The setscene version of the task +executes and provides the necessary artifacts returning either success +or failure. + +As previously mentioned, an artifact can cover more than one task. For +example, it is pointless to obtain a compiler if you already have the +compiled binary. To handle this, BitBake calls the +:term:`BB_SETSCENE_DEPVALID` function for +each successful setscene task to know whether or not it needs to obtain +the dependencies of that task. + +Finally, after all the setscene tasks have executed, BitBake calls the +function listed in +:term:`BB_SETSCENE_VERIFY_FUNCTION2` +with the list of tasks BitBake thinks has been "covered". The metadata +can then ensure that this list is correct and can inform BitBake that it +wants specific tasks to be run regardless of the setscene result. + +You can find more information on setscene metadata in the +:ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene` +section. + +Logging +======= + +In addition to the standard command line option to control how verbose +builds are when execute, bitbake also supports user defined +configuration of the `Python +logging <https://docs.python.org/3/library/logging.html>`__ facilities +through the :term:`BB_LOGCONFIG` variable. This +variable defines a json or yaml `logging +configuration <https://docs.python.org/3/library/logging.config.html>`__ +that will be intelligently merged into the default configuration. The +logging configuration is merged using the following rules: + +- The user defined configuration will completely replace the default + configuration if top level key ``bitbake_merge`` is set to the value + ``False``. In this case, all other rules are ignored. + +- The user configuration must have a top level ``version`` which must + match the value of the default configuration. + +- Any keys defined in the ``handlers``, ``formatters``, or ``filters``, + will be merged into the same section in the default configuration, + with the user specified keys taking replacing a default one if there + is a conflict. In practice, this means that if both the default + configuration and user configuration specify a handler named + ``myhandler``, the user defined one will replace the default. To + prevent the user from inadvertently replacing a default handler, + formatter, or filter, all of the default ones are named with a prefix + of "``BitBake.``" + +- If a logger is defined by the user with the key ``bitbake_merge`` set + to ``False``, that logger will be completely replaced by user + configuration. In this case, no other rules will apply to that + logger. + +- All user defined ``filter`` and ``handlers`` properties for a given + logger will be merged with corresponding properties from the default + logger. For example, if the user configuration adds a filter called + ``myFilter`` to the ``BitBake.SigGen``, and the default configuration + adds a filter called ``BitBake.defaultFilter``, both filters will be + applied to the logger + +As an example, consider the following user logging configuration file +which logs all Hash Equivalence related messages of VERBOSE or higher to +a file called ``hashequiv.log`` :: + + { + "version": 1, + "handlers": { + "autobuilderlog": { + "class": "logging.FileHandler", + "formatter": "logfileFormatter", + "level": "DEBUG", + "filename": "hashequiv.log", + "mode": "w" + } + }, + "formatters": { + "logfileFormatter": { + "format": "%(name)s: %(levelname)s: %(message)s" + } + }, + "loggers": { + "BitBake.SigGen.HashEquiv": { + "level": "VERBOSE", + "handlers": ["autobuilderlog"] + }, + "BitBake.RunQueue.HashEquiv": { + "level": "VERBOSE", + "handlers": ["autobuilderlog"] + } + } + } diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml index e4251dff5..04c5a26b9 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml @@ -765,7 +765,7 @@ the concept: <literallayout class='monospaced'> BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ - SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \ + SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX" diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst new file mode 100644 index 000000000..f62ddffe8 --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -0,0 +1,652 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +===================== +File Download Support +===================== + +| + +BitBake's fetch module is a standalone piece of library code that deals +with the intricacies of downloading source code and files from remote +systems. Fetching source code is one of the cornerstones of building +software. As such, this module forms an important part of BitBake. + +The current fetch module is called "fetch2" and refers to the fact that +it is the second major version of the API. The original version is +obsolete and has been removed from the codebase. Thus, in all cases, +"fetch" refers to "fetch2" in this manual. + +The Download (Fetch) +==================== + +BitBake takes several steps when fetching source code or files. The +fetcher codebase deals with two distinct processes in order: obtaining +the files from somewhere (cached or otherwise) and then unpacking those +files into a specific location and perhaps in a specific way. Getting +and unpacking the files is often optionally followed by patching. +Patching, however, is not covered by this module. + +The code to execute the first part of this process, a fetch, looks +something like the following: :: + + src_uri = (d.getVar('SRC_URI') or "").split() + fetcher = bb.fetch2.Fetch(src_uri, d) + fetcher.download() + +This code sets up an instance of the fetch class. The instance uses a +space-separated list of URLs from the :term:`SRC_URI` +variable and then calls the ``download`` method to download the files. + +The instantiation of the fetch class is usually followed by: :: + + rootdir = l.getVar('WORKDIR') + fetcher.unpack(rootdir) + +This code unpacks the downloaded files to the specified by ``WORKDIR``. + +.. note:: + + For convenience, the naming in these examples matches the variables + used by OpenEmbedded. If you want to see the above code in action, + examine the OpenEmbedded class file ``base.bbclass`` + . + +The ``SRC_URI`` and ``WORKDIR`` variables are not hardcoded into the +fetcher, since those fetcher methods can be (and are) called with +different variable names. In OpenEmbedded for example, the shared state +(sstate) code uses the fetch module to fetch the sstate files. + +When the ``download()`` method is called, BitBake tries to resolve the +URLs by looking for source files in a specific search order: + +- *Pre-mirror Sites:* BitBake first uses pre-mirrors to try and find + source files. These locations are defined using the + :term:`PREMIRRORS` variable. + +- *Source URI:* If pre-mirrors fail, BitBake uses the original URL (e.g + from ``SRC_URI``). + +- *Mirror Sites:* If fetch failures occur, BitBake next uses mirror + locations as defined by the :term:`MIRRORS` variable. + +For each URL passed to the fetcher, the fetcher calls the submodule that +handles that particular URL type. This behavior can be the source of +some confusion when you are providing URLs for the ``SRC_URI`` variable. +Consider the following two URLs: :: + + http://git.yoctoproject.org/git/poky;protocol=git + git://git.yoctoproject.org/git/poky;protocol=http + +In the former case, the URL is passed to the ``wget`` fetcher, which does not +understand "git". Therefore, the latter case is the correct form since the Git +fetcher does know how to use HTTP as a transport. + +Here are some examples that show commonly used mirror definitions: :: + + PREMIRRORS ?= "\ + bzr://.*/.\* http://somemirror.org/sources/ \\n \ + cvs://.*/.\* http://somemirror.org/sources/ \\n \ + git://.*/.\* http://somemirror.org/sources/ \\n \ + hg://.*/.\* http://somemirror.org/sources/ \\n \ + osc://.*/.\* http://somemirror.org/sources/ \\n \ + p4://.*/.\* http://somemirror.org/sources/ \\n \ + svn://.*/.\* http://somemirror.org/sources/ \\n" + + MIRRORS =+ "\ + ftp://.*/.\* http://somemirror.org/sources/ \\n \ + http://.*/.\* http://somemirror.org/sources/ \\n \ + https://.*/.\* http://somemirror.org/sources/ \\n" + +It is useful to note that BitBake +supports cross-URLs. It is possible to mirror a Git repository on an +HTTP server as a tarball. This is what the ``git://`` mapping in the +previous example does. + +Since network accesses are slow, BitBake maintains a cache of files +downloaded from the network. Any source files that are not local (i.e. +downloaded from the Internet) are placed into the download directory, +which is specified by the :term:`DL_DIR` variable. + +File integrity is of key importance for reproducing builds. For +non-local archive downloads, the fetcher code can verify SHA-256 and MD5 +checksums to ensure the archives have been downloaded correctly. You can +specify these checksums by using the ``SRC_URI`` variable with the +appropriate varflags as follows: :: + + SRC_URI[md5sum] = "value" + SRC_URI[sha256sum] = "value" + +You can also specify the checksums as +parameters on the ``SRC_URI`` as shown below: :: + + SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d" + +If multiple URIs exist, you can specify the checksums either directly as +in the previous example, or you can name the URLs. The following syntax +shows how you name the URIs: :: + + SRC_URI = "http://example.com/foobar.tar.bz2;name=foo" + SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d + +After a file has been downloaded and +has had its checksum checked, a ".done" stamp is placed in ``DL_DIR``. +BitBake uses this stamp during subsequent builds to avoid downloading or +comparing a checksum for the file again. + +.. note:: + + It is assumed that local storage is safe from data corruption. If + this were not the case, there would be bigger issues to worry about. + +If :term:`BB_STRICT_CHECKSUM` is set, any +download without a checksum triggers an error message. The +:term:`BB_NO_NETWORK` variable can be used to +make any attempted network access a fatal error, which is useful for +checking that mirrors are complete as well as other things. + +.. _bb-the-unpack: + +The Unpack +========== + +The unpack process usually immediately follows the download. For all +URLs except Git URLs, BitBake uses the common ``unpack`` method. + +A number of parameters exist that you can specify within the URL to +govern the behavior of the unpack stage: + +- *unpack:* Controls whether the URL components are unpacked. If set to + "1", which is the default, the components are unpacked. If set to + "0", the unpack stage leaves the file alone. This parameter is useful + when you want an archive to be copied in and not be unpacked. + +- *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether + to use DOS line ending conversion on text files. + +- *basepath:* Instructs the unpack stage to strip the specified + directories from the source path when unpacking. + +- *subdir:* Unpacks the specific URL to the specified subdirectory + within the root directory. + +The unpack call automatically decompresses and extracts files with ".Z", +".z", ".gz", ".xz", ".zip", ".jar", ".ipk", ".rpm". ".srpm", ".deb" and +".bz2" extensions as well as various combinations of tarball extensions. + +As mentioned, the Git fetcher has its own unpack method that is +optimized to work with Git trees. Basically, this method works by +cloning the tree into the final directory. The process is completed +using references so that there is only one central copy of the Git +metadata needed. + +.. _bb-fetchers: + +Fetchers +======== + +As mentioned earlier, the URL prefix determines which fetcher submodule +BitBake uses. Each submodule can support different URL parameters, which +are described in the following sections. + +.. _local-file-fetcher: + +Local file fetcher (``file://``) +-------------------------------- + +This submodule handles URLs that begin with ``file://``. The filename +you specify within the URL can be either an absolute or relative path to +a file. If the filename is relative, the contents of the +:term:`FILESPATH` variable is used in the same way +``PATH`` is used to find executables. If the file cannot be found, it is +assumed that it is available in :term:`DL_DIR` by the +time the ``download()`` method is called. + +If you specify a directory, the entire directory is unpacked. + +Here are a couple of example URLs, the first relative and the second +absolute: :: + + SRC_URI = "file://relativefile.patch" + SRC_URI = "file:///Users/ich/very_important_software" + +.. _http-ftp-fetcher: + +HTTP/FTP wget fetcher (``http://``, ``ftp://``, ``https://``) +------------------------------------------------------------- + +This fetcher obtains files from web and FTP servers. Internally, the +fetcher uses the wget utility. + +The executable and parameters used are specified by the +``FETCHCMD_wget`` variable, which defaults to sensible values. The +fetcher supports a parameter "downloadfilename" that allows the name of +the downloaded file to be specified. Specifying the name of the +downloaded file is useful for avoiding collisions in +:term:`DL_DIR` when dealing with multiple files that +have the same name. + +Some example URLs are as follows: :: + + SRC_URI = "http://oe.handhelds.org/not_there.aac" + SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac" + SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan" + +.. note:: + + Because URL parameters are delimited by semi-colons, this can + introduce ambiguity when parsing URLs that also contain semi-colons, + for example: + :: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47" + + + Such URLs should should be modified by replacing semi-colons with '&' + characters: + :: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47" + + + In most cases this should work. Treating semi-colons and '&' in + queries identically is recommended by the World Wide Web Consortium + (W3C). Note that due to the nature of the URL, you may have to + specify the name of the downloaded file as well: + :: + + SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2" + + +.. _cvs-fetcher: + +CVS fetcher (``(cvs://``) +------------------------- + +This submodule handles checking out files from the CVS version control +system. You can configure it using a number of different variables: + +- :term:`FETCHCMD_cvs <FETCHCMD>`: The name of the executable to use when running + the ``cvs`` command. This name is usually "cvs". + +- :term:`SRCDATE`: The date to use when fetching the CVS source code. A + special value of "now" causes the checkout to be updated on every + build. + +- :term:`CVSDIR`: Specifies where a temporary + checkout is saved. The location is often ``DL_DIR/cvs``. + +- CVS_PROXY_HOST: The name to use as a "proxy=" parameter to the + ``cvs`` command. + +- CVS_PROXY_PORT: The port number to use as a "proxyport=" + parameter to the ``cvs`` command. + +As well as the standard username and password URL syntax, you can also +configure the fetcher with various URL parameters: + +The supported parameters are as follows: + +- *"method":* The protocol over which to communicate with the CVS + server. By default, this protocol is "pserver". If "method" is set to + "ext", BitBake examines the "rsh" parameter and sets ``CVS_RSH``. You + can use "dir" for local directories. + +- *"module":* Specifies the module to check out. You must supply this + parameter. + +- *"tag":* Describes which CVS TAG should be used for the checkout. By + default, the TAG is empty. + +- *"date":* Specifies a date. If no "date" is specified, the + :term:`SRCDATE` of the configuration is used to + checkout a specific date. The special value of "now" causes the + checkout to be updated on every build. + +- *"localdir":* Used to rename the module. Effectively, you are + renaming the output directory to which the module is unpacked. You + are forcing the module into a special directory relative to + :term:`CVSDIR`. + +- *"rsh":* Used in conjunction with the "method" parameter. + +- *"scmdata":* Causes the CVS metadata to be maintained in the tarball + the fetcher creates when set to "keep". The tarball is expanded into + the work directory. By default, the CVS metadata is removed. + +- *"fullpath":* Controls whether the resulting checkout is at the + module level, which is the default, or is at deeper paths. + +- *"norecurse":* Causes the fetcher to only checkout the specified + directory with no recurse into any subdirectories. + +- *"port":* The port to which the CVS server connects. + +Some example URLs are as follows: :: + + SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext" + SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat" + +.. _svn-fetcher: + +Subversion (SVN) Fetcher (``svn://``) +------------------------------------- + +This fetcher submodule fetches code from the Subversion source control +system. The executable used is specified by ``FETCHCMD_svn``, which +defaults to "svn". The fetcher's temporary working directory is set by +:term:`SVNDIR`, which is usually ``DL_DIR/svn``. + +The supported parameters are as follows: + +- *"module":* The name of the svn module to checkout. You must provide + this parameter. You can think of this parameter as the top-level + directory of the repository data you want. + +- *"path_spec":* A specific directory in which to checkout the + specified svn module. + +- *"protocol":* The protocol to use, which defaults to "svn". If + "protocol" is set to "svn+ssh", the "ssh" parameter is also used. + +- *"rev":* The revision of the source code to checkout. + +- *"scmdata":* Causes the “.svn” directories to be available during + compile-time when set to "keep". By default, these directories are + removed. + +- *"ssh":* An optional parameter used when "protocol" is set to + "svn+ssh". You can use this parameter to specify the ssh program used + by svn. + +- *"transportuser":* When required, sets the username for the + transport. By default, this parameter is empty. The transport + username is different than the username used in the main URL, which + is passed to the subversion command. + +Following are three examples using svn: :: + + SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667" + SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh" + SRC_URI = "svn://myrepos/proj1;module=trunk;protocol=http;path_spec=${MY_DIR}/proj1" + +.. _git-fetcher: + +Git Fetcher (``git://``) +------------------------ + +This fetcher submodule fetches code from the Git source control system. +The fetcher works by creating a bare clone of the remote into +:term:`GITDIR`, which is usually ``DL_DIR/git2``. This +bare clone is then cloned into the work directory during the unpack +stage when a specific tree is checked out. This is done using alternates +and by reference to minimize the amount of duplicate data on the disk +and make the unpack process fast. The executable used can be set with +``FETCHCMD_git``. + +This fetcher supports the following parameters: + +- *"protocol":* The protocol used to fetch the files. The default is + "git" when a hostname is set. If a hostname is not set, the Git + protocol is "file". You can also use "http", "https", "ssh" and + "rsync". + +- *"nocheckout":* Tells the fetcher to not checkout source code when + unpacking when set to "1". Set this option for the URL where there is + a custom routine to checkout code. The default is "0". + +- *"rebaseable":* Indicates that the upstream Git repository can be + rebased. You should set this parameter to "1" if revisions can become + detached from branches. In this case, the source mirror tarball is + done per revision, which has a loss of efficiency. Rebasing the + upstream Git repository could cause the current revision to disappear + from the upstream repository. This option reminds the fetcher to + preserve the local cache carefully for future use. The default value + for this parameter is "0". + +- *"nobranch":* Tells the fetcher to not check the SHA validation for + the branch when set to "1". The default is "0". Set this option for + the recipe that refers to the commit that is valid for a tag instead + of the branch. + +- *"bareclone":* Tells the fetcher to clone a bare clone into the + destination directory without checking out a working tree. Only the + raw Git metadata is provided. This parameter implies the "nocheckout" + parameter as well. + +- *"branch":* The branch(es) of the Git tree to clone. If unset, this + is assumed to be "master". The number of branch parameters much match + the number of name parameters. + +- *"rev":* The revision to use for the checkout. The default is + "master". + +- *"tag":* Specifies a tag to use for the checkout. To correctly + resolve tags, BitBake must access the network. For that reason, tags + are often not used. As far as Git is concerned, the "tag" parameter + behaves effectively the same as the "rev" parameter. + +- *"subpath":* Limits the checkout to a specific subpath of the tree. + By default, the whole tree is checked out. + +- *"destsuffix":* The name of the path in which to place the checkout. + By default, the path is ``git/``. + +- *"usehead":* Enables local ``git://`` URLs to use the current branch + HEAD as the revision for use with ``AUTOREV``. The "usehead" + parameter implies no branch and only works when the transfer protocol + is ``file://``. + +Here are some example URLs: :: + + SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1" + SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http" + +.. _gitsm-fetcher: + +Git Submodule Fetcher (``gitsm://``) +------------------------------------ + +This fetcher submodule inherits from the :ref:`Git +fetcher<bitbake-user-manual/bitbake-user-manual-fetching:git fetcher +(\`\`git://\`\`)>` and extends that fetcher's behavior by fetching a +repository's submodules. :term:`SRC_URI` is passed to the Git fetcher as +described in the :ref:`bitbake-user-manual/bitbake-user-manual-fetching:git +fetcher (\`\`git://\`\`)` section. + +.. note:: + + You must clean a recipe when switching between '``git://``' and + '``gitsm://``' URLs. + + The Git Submodules fetcher is not a complete fetcher implementation. + The fetcher has known issues where it does not use the normal source + mirroring infrastructure properly. Further, the submodule sources it + fetches are not visible to the licensing and source archiving + infrastructures. + +.. _clearcase-fetcher: + +ClearCase Fetcher (``ccrc://``) +------------------------------- + +This fetcher submodule fetches code from a +`ClearCase <http://en.wikipedia.org/wiki/Rational_ClearCase>`__ +repository. + +To use this fetcher, make sure your recipe has proper +:term:`SRC_URI`, :term:`SRCREV`, and +:term:`PV` settings. Here is an example: :: + + SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module" + SRCREV = "EXAMPLE_CLEARCASE_TAG" + PV = "${@d.getVar("SRCREV", False).replace("/", "+")}" + +The fetcher uses the ``rcleartool`` or +``cleartool`` remote client, depending on which one is available. + +Following are options for the ``SRC_URI`` statement: + +- *vob*: The name, which must include the prepending "/" character, + of the ClearCase VOB. This option is required. + +- *module*: The module, which must include the prepending "/" + character, in the selected VOB. + + .. note:: + + The module and vob options are combined to create the load rule in the + view config spec. As an example, consider the vob and module values from + the SRC_URI statement at the start of this section. Combining those values + results in the following: :: + + load /example_vob/example_module + +- *proto*: The protocol, which can be either ``http`` or ``https``. + +By default, the fetcher creates a configuration specification. If you +want this specification written to an area other than the default, use +the ``CCASE_CUSTOM_CONFIG_SPEC`` variable in your recipe to define where +the specification is written. + +.. note:: + + the SRCREV loses its functionality if you specify this variable. However, + SRCREV is still used to label the archive after a fetch even though it does + not define what is fetched. + +Here are a couple of other behaviors worth mentioning: + +- When using ``cleartool``, the login of ``cleartool`` is handled by + the system. The login require no special steps. + +- In order to use ``rcleartool`` with authenticated users, an + "rcleartool login" is necessary before using the fetcher. + +.. _perforce-fetcher: + +Perforce Fetcher (``p4://``) +---------------------------- + +This fetcher submodule fetches code from the +`Perforce <https://www.perforce.com/>`__ source control system. The +executable used is specified by ``FETCHCMD_p4``, which defaults to "p4". +The fetcher's temporary working directory is set by +:term:`P4DIR`, which defaults to "DL_DIR/p4". +The fetcher does not make use of a perforce client, instead it +relies on ``p4 files`` to retrieve a list of +files and ``p4 print`` to transfer the content +of those files locally. + +To use this fetcher, make sure your recipe has proper +:term:`SRC_URI`, :term:`SRCREV`, and +:term:`PV` values. The p4 executable is able to use the +config file defined by your system's ``P4CONFIG`` environment variable +in order to define the Perforce server URL and port, username, and +password if you do not wish to keep those values in a recipe itself. If +you choose not to use ``P4CONFIG``, or to explicitly set variables that +``P4CONFIG`` can contain, you can specify the ``P4PORT`` value, which is +the server's URL and port number, and you can specify a username and +password directly in your recipe within ``SRC_URI``. + +Here is an example that relies on ``P4CONFIG`` to specify the server URL +and port, username, and password, and fetches the Head Revision: :: + + SRC_URI = "p4://example-depot/main/source/..." + SRCREV = "${AUTOREV}" + PV = "p4-${SRCPV}" + S = "${WORKDIR}/p4" + +Here is an example that specifies the server URL and port, username, and +password, and fetches a Revision based on a Label: :: + + P4PORT = "tcp:p4server.example.net:1666" + SRC_URI = "p4://user:passwd@example-depot/main/source/..." + SRCREV = "release-1.0" + PV = "p4-${SRCPV}" + S = "${WORKDIR}/p4" + +.. note:: + + You should always set S to "${WORKDIR}/p4" in your recipe. + +By default, the fetcher strips the depot location from the local file paths. In +the above example, the content of ``example-depot/main/source/`` will be placed +in ``${WORKDIR}/p4``. For situations where preserving parts of the remote depot +paths locally is desirable, the fetcher supports two parameters: + +- *"module":* + The top-level depot location or directory to fetch. The value of this + parameter can also point to a single file within the depot, in which case + the local file path will include the module path. +- *"remotepath":* + When used with the value "``keep``", the fetcher will mirror the full depot + paths locally for the specified location, even in combination with the + ``module`` parameter. + +Here is an example use of the the ``module`` parameter: :: + + SRC_URI = "p4://user:passwd@example-depot/main;module=source/..." + +In this case, the content of the top-level directory ``source/`` will be fetched +to ``${P4DIR}``, including the directory itself. The top-level directory will +be accesible at ``${P4DIR}/source/``. + +Here is an example use of the the ``remotepath`` parameter: :: + + SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep" + +In this case, the content of the top-level directory ``source/`` will be fetched +to ``${P4DIR}``, but the complete depot paths will be mirrored locally. The +top-level directory will be accessible at +``${P4DIR}/example-depot/main/source/``. + +.. _repo-fetcher: + +Repo Fetcher (``repo://``) +-------------------------- + +This fetcher submodule fetches code from ``google-repo`` source control +system. The fetcher works by initiating and syncing sources of the +repository into :term:`REPODIR`, which is usually +``${DL_DIR}/repo``. + +This fetcher supports the following parameters: + +- *"protocol":* Protocol to fetch the repository manifest (default: + git). + +- *"branch":* Branch or tag of repository to get (default: master). + +- *"manifest":* Name of the manifest file (default: ``default.xml``). + +Here are some example URLs: :: + + SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml" + SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml" + +Other Fetchers +-------------- + +Fetch submodules also exist for the following: + +- Bazaar (``bzr://``) + +- Mercurial (``hg://``) + +- npm (``npm://``) + +- OSC (``osc://``) + +- Secure FTP (``sftp://``) + +- Secure Shell (``ssh://``) + +- Trees using Git Annex (``gitannex://``) + +No documentation currently exists for these lesser used fetcher +submodules. However, you might find the code helpful and readable. + +Auto Revisions +============== + +We need to document ``AUTOREV`` and ``SRCREV_FORMAT`` here. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml index d1bfc2336..fe4372ade 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml @@ -736,6 +736,10 @@ The fetcher's temporary working directory is set by <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>, which defaults to "DL_DIR/p4". + The fetcher does not make use of a perforce client, instead it + relies on <filename>p4 files</filename> to retrieve a list of + files and <filename>p4 print</filename> to transfer the content + of those files locally. </para> <para> @@ -783,6 +787,62 @@ to <filename>"${WORKDIR}/p4"</filename> in your recipe. </note> </para> + + <para> + By default, the fetcher strips the depot location from the + local file paths. In the above example, the content of + <filename>example-depot/main/source/</filename> + will be placed in <filename>${WORKDIR}/p4</filename>. + For situations where preserving parts of the remote depot paths + locally is desirable, the fetcher supports two parameters: + + <itemizedlist> + <listitem><para> + <emphasis>"module":</emphasis> + The top-level depot location or directory to fetch. The + value of this parameter can also point to a single file + within the depot, in which case the local file path will + include the module path. + </para></listitem> + <listitem><para> + <emphasis>"remotepath":</emphasis> + When used with the value "<filename>keep</filename>", + the fetcher will mirror the full depot paths locally + for the specified location, even in combination with + the <filename>module</filename> parameter. + </para></listitem> + </itemizedlist> + </para> + + <para> + Here is an example use of the the <filename>module</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/..." + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, including the directory itself. + The top-level directory will be accesible at + <filename>${P4DIR}/source/</filename>. + </para> + + <para> + Here is an example use of the the <filename>remotepath</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep" + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, but the complete depot paths will + be mirrored locally. The top-level directory will be accessible + at <filename>${P4DIR}/example-depot/main/source/</filename>. + </para> </section> <section id='repo-fetcher'> diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst new file mode 100644 index 000000000..e3fd32158 --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst @@ -0,0 +1,415 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +=================== +Hello World Example +=================== + +BitBake Hello World +=================== + +The simplest example commonly used to demonstrate any new programming +language or tool is the "`Hello +World <http://en.wikipedia.org/wiki/Hello_world_program>`__" example. +This appendix demonstrates, in tutorial form, Hello World within the +context of BitBake. The tutorial describes how to create a new project +and the applicable metadata files necessary to allow BitBake to build +it. + +Obtaining BitBake +================= + +See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for +information on how to obtain BitBake. Once you have the source code on +your machine, the BitBake directory appears as follows: :: + + $ ls -al + total 100 + drwxrwxr-x. 9 wmat wmat 4096 Jan 31 13:44 . + drwxrwxr-x. 3 wmat wmat 4096 Feb 4 10:45 .. + -rw-rw-r--. 1 wmat wmat 365 Nov 26 04:55 AUTHORS + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 bin + drwxrwxr-x. 4 wmat wmat 4096 Jan 31 13:44 build + -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 classes + drwxrwxr-x. 2 wmat wmat 4096 Nov 26 04:55 conf + drwxrwxr-x. 3 wmat wmat 4096 Nov 26 04:55 contrib + -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING + drwxrwxr-x. 3 wmat wmat 4096 Nov 26 04:55 doc + -rw-rw-r--. 1 wmat wmat 69 Nov 26 04:55 .gitignore + -rw-rw-r--. 1 wmat wmat 849 Nov 26 04:55 HEADER + drwxrwxr-x. 5 wmat wmat 4096 Jan 31 13:44 lib + -rw-rw-r--. 1 wmat wmat 195 Nov 26 04:55 MANIFEST.in + -rw-rw-r--. 1 wmat wmat 2887 Nov 26 04:55 TODO + +At this point, you should have BitBake cloned to a directory that +matches the previous listing except for dates and user names. + +Setting Up the BitBake Environment +================================== + +First, you need to be sure that you can run BitBake. Set your working +directory to where your local BitBake files are and run the following +command: :: + + $ ./bin/bitbake --version + BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0 + +The console output tells you what version +you are running. + +The recommended method to run BitBake is from a directory of your +choice. To be able to run BitBake from any directory, you need to add +the executable binary to your binary to your shell's environment +``PATH`` variable. First, look at your current ``PATH`` variable by +entering the following: :: + + $ echo $PATH + +Next, add the directory location +for the BitBake binary to the ``PATH``. Here is an example that adds the +``/home/scott-lenovo/bitbake/bin`` directory to the front of the +``PATH`` variable: :: + + $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH + +You should now be able to enter the ``bitbake`` command from the command +line while working from any directory. + +The Hello World Example +======================= + +The overall goal of this exercise is to build a complete "Hello World" +example utilizing task and layer concepts. Because this is how modern +projects such as OpenEmbedded and the Yocto Project utilize BitBake, the +example provides an excellent starting point for understanding BitBake. + +To help you understand how to use BitBake to build targets, the example +starts with nothing but the ``bitbake`` command, which causes BitBake to +fail and report problems. The example progresses by adding pieces to the +build to eventually conclude with a working, minimal "Hello World" +example. + +While every attempt is made to explain what is happening during the +example, the descriptions cannot cover everything. You can find further +information throughout this manual. Also, you can actively participate +in the :oe_lists:`/g/bitbake-devel` +discussion mailing list about the BitBake build tool. + +.. note:: + + This example was inspired by and drew heavily from + `Mailing List post - The BitBake equivalent of "Hello, World!" + <http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html>`_. + +As stated earlier, the goal of this example is to eventually compile +"Hello World". However, it is unknown what BitBake needs and what you +have to provide in order to achieve that goal. Recall that BitBake +utilizes three types of metadata files: +:ref:`bitbake-user-manual/bitbake-user-manual-intro:configuration files`, +:ref:`bitbake-user-manual/bitbake-user-manual-intro:classes`, and +:ref:`bitbake-user-manual/bitbake-user-manual-intro:recipes`. +But where do they go? How does BitBake find +them? BitBake's error messaging helps you answer these types of +questions and helps you better understand exactly what is going on. + +Following is the complete "Hello World" example. + +#. **Create a Project Directory:** First, set up a directory for the + "Hello World" project. Here is how you can do so in your home + directory: :: + + $ mkdir ~/hello + $ cd ~/hello + + This is the directory that + BitBake will use to do all of its work. You can use this directory + to keep all the metafiles needed by BitBake. Having a project + directory is a good way to isolate your project. + +#. **Run BitBake:** At this point, you have nothing but a project + directory. Run the ``bitbake`` command and see what it does: :: + + $ bitbake + The BBPATH variable is not set and bitbake did not + find a conf/bblayers.conf file in the expected location. + Maybe you accidentally invoked bitbake from the wrong directory? + DEBUG: Removed the following variables from the environment: + GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP, + GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy, + XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL, + MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR, + GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID, + XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS, + _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH, + UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS + + The majority of this output is specific to environment variables that + are not directly relevant to BitBake. However, the very first + message regarding the ``BBPATH`` variable and the + ``conf/bblayers.conf`` file is relevant. + + When you run BitBake, it begins looking for metadata files. The + :term:`BBPATH` variable is what tells BitBake where + to look for those files. ``BBPATH`` is not set and you need to set + it. Without ``BBPATH``, BitBake cannot find any configuration files + (``.conf``) or recipe files (``.bb``) at all. BitBake also cannot + find the ``bitbake.conf`` file. + +#. **Setting BBPATH:** For this example, you can set ``BBPATH`` in + the same manner that you set ``PATH`` earlier in the appendix. You + should realize, though, that it is much more flexible to set the + ``BBPATH`` variable up in a configuration file for each project. + + From your shell, enter the following commands to set and export the + ``BBPATH`` variable: :: + + $ BBPATH="projectdirectory" + $ export BBPATH + + Use your actual project directory in the command. BitBake uses that + directory to find the metadata it needs for your project. + + .. note:: + + When specifying your project directory, do not use the tilde + ("~") character as BitBake does not expand that character as the + shell would. + +#. **Run BitBake:** Now that you have ``BBPATH`` defined, run the + ``bitbake`` command again: :: + + $ bitbake + ERROR: Traceback (most recent call last): + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped + return func(fn, *args) + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file + return bb.parse.handle(fn, data, include) + File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle + return h['handle'](fn, data, include) + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle + abs_fn = resolve_file(fn, data) + File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file + raise IOError("file %s not found in %s" % (fn, bbpath)) + IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello + + ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello + + This sample output shows that BitBake could not find the + ``conf/bitbake.conf`` file in the project directory. This file is + the first thing BitBake must find in order to build a target. And, + since the project directory for this example is empty, you need to + provide a ``conf/bitbake.conf`` file. + +#. **Creating conf/bitbake.conf:** The ``conf/bitbake.conf`` includes + a number of configuration variables BitBake uses for metadata and + recipe files. For this example, you need to create the file in your + project directory and define some key BitBake variables. For more + information on the ``bitbake.conf`` file, see + http://git.openembedded.org/bitbake/tree/conf/bitbake.conf. + + Use the following commands to create the ``conf`` directory in the + project directory: :: + + $ mkdir conf + + From within the ``conf`` directory, + use some editor to create the ``bitbake.conf`` so that it contains + the following: :: + + PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + + TMPDIR = "${TOPDIR}/tmp" + CACHE = "${TMPDIR}/cache" + STAMP = "${TMPDIR}/${PN}/stamps" + T = "${TMPDIR}/${PN}/work" + B = "${TMPDIR}/${PN}" + + .. note:: + + Without a value for PN , the variables STAMP , T , and B , prevent more + than one recipe from working. You can fix this by either setting PN to + have a value similar to what OpenEmbedded and BitBake use in the default + bitbake.conf file (see previous example). Or, by manually updating each + recipe to set PN . You will also need to include PN as part of the STAMP + , T , and B variable definitions in the local.conf file. + + The ``TMPDIR`` variable establishes a directory that BitBake uses + for build output and intermediate files other than the cached + information used by the + :ref:`bitbake-user-manual/bitbake-user-manual-execution:setscene` + process. Here, the ``TMPDIR`` directory is set to ``hello/tmp``. + + .. tip:: + + You can always safely delete the tmp directory in order to rebuild a + BitBake target. The build process creates the directory for you when you + run BitBake. + + For information about each of the other variables defined in this + example, check :term:`PN`, :term:`TOPDIR`, :term:`CACHE`, :term:`STAMP`, + :term:`T` or :term:`B` to take you to the definitions in the + glossary. + +#. **Run BitBake:** After making sure that the ``conf/bitbake.conf`` file + exists, you can run the ``bitbake`` command again: :: + + $ bitbake + ERROR: Traceback (most recent call last): + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped + return func(fn, *args) + File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit + bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit + include(fn, file, lineno, d, "inherit") + File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include + raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno) + ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + + ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass + + In the sample output, + BitBake could not find the ``classes/base.bbclass`` file. You need + to create that file next. + +#. **Creating classes/base.bbclass:** BitBake uses class files to + provide common code and functionality. The minimally required class + for BitBake is the ``classes/base.bbclass`` file. The ``base`` class + is implicitly inherited by every recipe. BitBake looks for the class + in the ``classes`` directory of the project (i.e ``hello/classes`` + in this example). + + Create the ``classes`` directory as follows: :: + + $ cd $HOME/hello + $ mkdir classes + + Move to the ``classes`` directory and then create the + ``base.bbclass`` file by inserting this single line: addtask build + The minimal task that BitBake runs is the ``do_build`` task. This is + all the example needs in order to build the project. Of course, the + ``base.bbclass`` can have much more depending on which build + environments BitBake is supporting. + +#. **Run BitBake:** After making sure that the ``classes/base.bbclass`` + file exists, you can run the ``bitbake`` command again: :: + + $ bitbake + Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information. + + BitBake is finally reporting + no errors. However, you can see that it really does not have + anything to do. You need to create a recipe that gives BitBake + something to do. + +#. **Creating a Layer:** While it is not really necessary for such a + small example, it is good practice to create a layer in which to + keep your code separate from the general metadata used by BitBake. + Thus, this example creates and uses a layer called "mylayer". + + .. note:: + + You can find additional information on layers in the + ":ref:`bitbake-user-manual/bitbake-user-manual-intro:Layers`" section. + + Minimally, you need a recipe file and a layer configuration file in + your layer. The configuration file needs to be in the ``conf`` + directory inside the layer. Use these commands to set up the layer + and the ``conf`` directory: :: + + $ cd $HOME + $ mkdir mylayer + $ cd mylayer + $ mkdir conf + + Move to the ``conf`` directory and create a ``layer.conf`` file that has the + following: :: + + BBPATH .= ":${LAYERDIR}" + BBFILES += "${LAYERDIR}/\*.bb" + BBFILE_COLLECTIONS += "mylayer" + `BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/" + + For information on these variables, click on :term:`BBFILES`, + :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>` + to go to the definitions in the glossary. + + You need to create the recipe file next. Inside your layer at the + top-level, use an editor and create a recipe file named + ``printhello.bb`` that has the following: :: + + DESCRIPTION = "Prints Hello World" + PN = 'printhello' + PV = '1' + + python do_build() { + bb.plain("********************"); + bb.plain("* *"); + bb.plain("* Hello, World! *"); + bb.plain("* *"); + bb.plain("********************"); + } + + The recipe file simply provides + a description of the recipe, the name, version, and the ``do_build`` + task, which prints out "Hello World" to the console. For more + information on :term:`DESCRIPTION`, :term:`PN` or :term:`PV` + follow the links to the glossary. + +#. **Run BitBake With a Target:** Now that a BitBake target exists, run + the command and provide that target: :: + + $ cd $HOME/hello + $ bitbake printhello + ERROR: no recipe files to build, check your BBPATH and BBFILES? + + Summary: There was 1 ERROR message shown, returning a non-zero exit code. + + We have created the layer with the recipe and + the layer configuration file but it still seems that BitBake cannot + find the recipe. BitBake needs a ``conf/bblayers.conf`` that lists + the layers for the project. Without this file, BitBake cannot find + the recipe. + +#. **Creating conf/bblayers.conf:** BitBake uses the + ``conf/bblayers.conf`` file to locate layers needed for the project. + This file must reside in the ``conf`` directory of the project (i.e. + ``hello/conf`` for this example). + + Set your working directory to the ``hello/conf`` directory and then + create the ``bblayers.conf`` file so that it contains the following: :: + + BBLAYERS ?= " \ + /home/<you>/mylayer \ + " + + You need to provide your own information for ``you`` in the file. + +#. **Run BitBake With a Target:** Now that you have supplied the + ``bblayers.conf`` file, run the ``bitbake`` command and provide the + target: :: + + $ bitbake printhello + Parsing recipes: 100% |##################################################################################| + Time: 00:00:00 + Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors. + NOTE: Resolving any missing task queue dependencies + NOTE: Preparing RunQueue + NOTE: Executing RunQueue Tasks + ******************** + * * + * Hello, World! * + * * + ******************** + NOTE: Tasks Summary: Attempted 1 tasks of which 0 didn't need to be rerun and all succeeded. + + .. note:: + + After the first execution, re-running bitbake printhello again will not + result in a BitBake run that prints the same console output. The reason + for this is that the first time the printhello.bb recipe's do_build task + executes successfully, BitBake writes a stamp file for the task. Thus, + the next time you attempt to run the task using that same bitbake + command, BitBake notices the stamp and therefore determines that the task + does not need to be re-run. If you delete the tmp directory or run + bitbake -c clean printhello and then re-run the build, the "Hello, + World!" message will be printed again. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst new file mode 100644 index 000000000..77dc9668a --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst @@ -0,0 +1,651 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +======== +Overview +======== + +| + +Welcome to the BitBake User Manual. This manual provides information on +the BitBake tool. The information attempts to be as independent as +possible regarding systems that use BitBake, such as OpenEmbedded and +the Yocto Project. In some cases, scenarios or examples within the +context of a build system are used in the manual to help with +understanding. For these cases, the manual clearly states the context. + +.. _intro: + +Introduction +============ + +Fundamentally, BitBake is a generic task execution engine that allows +shell and Python tasks to be run efficiently and in parallel while +working within complex inter-task dependency constraints. One of +BitBake's main users, OpenEmbedded, takes this core and builds embedded +Linux software stacks using a task-oriented approach. + +Conceptually, BitBake is similar to GNU Make in some regards but has +significant differences: + +- BitBake executes tasks according to provided metadata that builds up + the tasks. Metadata is stored in recipe (``.bb``) and related recipe + "append" (``.bbappend``) files, configuration (``.conf``) and + underlying include (``.inc``) files, and in class (``.bbclass``) + files. The metadata provides BitBake with instructions on what tasks + to run and the dependencies between those tasks. + +- BitBake includes a fetcher library for obtaining source code from + various places such as local files, source control systems, or + websites. + +- The instructions for each unit to be built (e.g. a piece of software) + are known as "recipe" files and contain all the information about the + unit (dependencies, source file locations, checksums, description and + so on). + +- BitBake includes a client/server abstraction and can be used from a + command line or used as a service over XML-RPC and has several + different user interfaces. + +History and Goals +================= + +BitBake was originally a part of the OpenEmbedded project. It was +inspired by the Portage package management system used by the Gentoo +Linux distribution. On December 7, 2004, OpenEmbedded project team +member Chris Larson split the project into two distinct pieces: + +- BitBake, a generic task executor + +- OpenEmbedded, a metadata set utilized by BitBake + +Today, BitBake is the primary basis of the +`OpenEmbedded <http://www.openembedded.org/>`__ project, which is being +used to build and maintain Linux distributions such as the `Angstrom +Distribution <http://www.angstrom-distribution.org/>`__, and which is +also being used as the build tool for Linux projects such as the `Yocto +Project <http://www.yoctoproject.org>`__. + +Prior to BitBake, no other build tool adequately met the needs of an +aspiring embedded Linux distribution. All of the build systems used by +traditional desktop Linux distributions lacked important functionality, +and none of the ad hoc Buildroot-based systems, prevalent in the +embedded space, were scalable or maintainable. + +Some important original goals for BitBake were: + +- Handle cross-compilation. + +- Handle inter-package dependencies (build time on target architecture, + build time on native architecture, and runtime). + +- Support running any number of tasks within a given package, + including, but not limited to, fetching upstream sources, unpacking + them, patching them, configuring them, and so forth. + +- Be Linux distribution agnostic for both build and target systems. + +- Be architecture agnostic. + +- Support multiple build and target operating systems (e.g. Cygwin, the + BSDs, and so forth). + +- Be self-contained, rather than tightly integrated into the build + machine's root filesystem. + +- Handle conditional metadata on the target architecture, operating + system, distribution, and machine. + +- Be easy to use the tools to supply local metadata and packages + against which to operate. + +- Be easy to use BitBake to collaborate between multiple projects for + their builds. + +- Provide an inheritance mechanism to share common metadata between + many packages. + +Over time it became apparent that some further requirements were +necessary: + +- Handle variants of a base recipe (e.g. native, sdk, and multilib). + +- Split metadata into layers and allow layers to enhance or override + other layers. + +- Allow representation of a given set of input variables to a task as a + checksum. Based on that checksum, allow acceleration of builds with + prebuilt components. + +BitBake satisfies all the original requirements and many more with +extensions being made to the basic functionality to reflect the +additional requirements. Flexibility and power have always been the +priorities. BitBake is highly extensible and supports embedded Python +code and execution of any arbitrary tasks. + +.. _Concepts: + +Concepts +======== + +BitBake is a program written in the Python language. At the highest +level, BitBake interprets metadata, decides what tasks are required to +run, and executes those tasks. Similar to GNU Make, BitBake controls how +software is built. GNU Make achieves its control through "makefiles", +while BitBake uses "recipes". + +BitBake extends the capabilities of a simple tool like GNU Make by +allowing for the definition of much more complex tasks, such as +assembling entire embedded Linux distributions. + +The remainder of this section introduces several concepts that should be +understood in order to better leverage the power of BitBake. + +Recipes +------- + +BitBake Recipes, which are denoted by the file extension ``.bb``, are +the most basic metadata files. These recipe files provide BitBake with +the following: + +- Descriptive information about the package (author, homepage, license, + and so on) + +- The version of the recipe + +- Existing dependencies (both build and runtime dependencies) + +- Where the source code resides and how to fetch it + +- Whether the source code requires any patches, where to find them, and + how to apply them + +- How to configure and compile the source code + +- How to assemble the generated artifacts into one or more installable + packages + +- Where on the target machine to install the package or packages + created + +Within the context of BitBake, or any project utilizing BitBake as its +build system, files with the ``.bb`` extension are referred to as +recipes. + +.. note:: + + The term "package" is also commonly used to describe recipes. + However, since the same word is used to describe packaged output from + a project, it is best to maintain a single descriptive term - + "recipes". Put another way, a single "recipe" file is quite capable + of generating a number of related but separately installable + "packages". In fact, that ability is fairly common. + +Configuration Files +------------------- + +Configuration files, which are denoted by the ``.conf`` extension, +define various configuration variables that govern the project's build +process. These files fall into several areas that define machine +configuration, distribution configuration, possible compiler tuning, +general common configuration, and user configuration. The main +configuration file is the sample ``bitbake.conf`` file, which is located +within the BitBake source tree ``conf`` directory. + +Classes +------- + +Class files, which are denoted by the ``.bbclass`` extension, contain +information that is useful to share between metadata files. The BitBake +source tree currently comes with one class metadata file called +``base.bbclass``. You can find this file in the ``classes`` directory. +The ``base.bbclass`` class files is special since it is always included +automatically for all recipes and classes. This class contains +definitions for standard basic tasks such as fetching, unpacking, +configuring (empty by default), compiling (runs any Makefile present), +installing (empty by default) and packaging (empty by default). These +tasks are often overridden or extended by other classes added during the +project development process. + +Layers +------ + +Layers allow you to isolate different types of customizations from each +other. While you might find it tempting to keep everything in one layer +when working on a single project, the more modular your metadata, the +easier it is to cope with future changes. + +To illustrate how you can use layers to keep things modular, consider +customizations you might make to support a specific target machine. +These types of customizations typically reside in a special layer, +rather than a general layer, called a Board Support Package (BSP) layer. +Furthermore, the machine customizations should be isolated from recipes +and metadata that support a new GUI environment, for example. This +situation gives you a couple of layers: one for the machine +configurations and one for the GUI environment. It is important to +understand, however, that the BSP layer can still make machine-specific +additions to recipes within the GUI environment layer without polluting +the GUI layer itself with those machine-specific changes. You can +accomplish this through a recipe that is a BitBake append +(``.bbappend``) file. + +.. _append-bbappend-files: + +Append Files +------------ + +Append files, which are files that have the ``.bbappend`` file +extension, extend or override information in an existing recipe file. + +BitBake expects every append file to have a corresponding recipe file. +Furthermore, the append file and corresponding recipe file must use the +same root filename. The filenames can differ only in the file type +suffix used (e.g. ``formfactor_0.0.bb`` and +``formfactor_0.0.bbappend``). + +Information in append files extends or overrides the information in the +underlying, similarly-named recipe files. + +When you name an append file, you can use the "``%``" wildcard character +to allow for matching recipe names. For example, suppose you have an +append file named as follows: :: + + busybox_1.21.%.bbappend + +That append file +would match any ``busybox_1.21.``\ x\ ``.bb`` version of the recipe. So, +the append file would match the following recipe names: :: + + busybox_1.21.1.bb + busybox_1.21.2.bb + busybox_1.21.3.bb + +.. note:: + + The use of the " % " character is limited in that it only works directly in + front of the .bbappend portion of the append file's name. You cannot use the + wildcard character in any other location of the name. + +If the ``busybox`` recipe was updated to ``busybox_1.3.0.bb``, the +append name would not match. However, if you named the append file +``busybox_1.%.bbappend``, then you would have a match. + +In the most general case, you could name the append file something as +simple as ``busybox_%.bbappend`` to be entirely version independent. + +Obtaining BitBake +================= + +You can obtain BitBake several different ways: + +- **Cloning BitBake:** Using Git to clone the BitBake source code + repository is the recommended method for obtaining BitBake. Cloning + the repository makes it easy to get bug fixes and have access to + stable branches and the master branch. Once you have cloned BitBake, + you should use the latest stable branch for development since the + master branch is for BitBake development and might contain less + stable changes. + + You usually need a version of BitBake that matches the metadata you + are using. The metadata is generally backwards compatible but not + forward compatible. + + Here is an example that clones the BitBake repository: :: + + $ git clone git://git.openembedded.org/bitbake + + This command clones the BitBake + Git repository into a directory called ``bitbake``. Alternatively, + you can designate a directory after the ``git clone`` command if you + want to call the new directory something other than ``bitbake``. Here + is an example that names the directory ``bbdev``: :: + + $ git clone git://git.openembedded.org/bitbake bbdev + +- **Installation using your Distribution Package Management System:** + This method is not recommended because the BitBake version that is + provided by your distribution, in most cases, is several releases + behind a snapshot of the BitBake repository. + +- **Taking a snapshot of BitBake:** Downloading a snapshot of BitBake + from the source code repository gives you access to a known branch or + release of BitBake. + + .. note:: + + Cloning the Git repository, as described earlier, is the preferred + method for getting BitBake. Cloning the repository makes it easier + to update as patches are added to the stable branches. + + The following example downloads a snapshot of BitBake version 1.17.0: :: + + $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz + $ tar zxpvf bitbake-1.17.0.tar.gz + + After extraction of the tarball using + the tar utility, you have a directory entitled ``bitbake-1.17.0``. + +- **Using the BitBake that Comes With Your Build Checkout:** A final + possibility for getting a copy of BitBake is that it already comes + with your checkout of a larger BitBake-based build system, such as + Poky. Rather than manually checking out individual layers and gluing + them together yourself, you can check out an entire build system. The + checkout will already include a version of BitBake that has been + thoroughly tested for compatibility with the other components. For + information on how to check out a particular BitBake-based build + system, consult that build system's supporting documentation. + +.. _bitbake-user-manual-command: + +The BitBake Command +=================== + +The ``bitbake`` command is the primary interface to the BitBake tool. +This section presents the BitBake command syntax and provides several +execution examples. + +Usage and syntax +---------------- + +Following is the usage and syntax for BitBake: :: + + $ bitbake -h + Usage: bitbake [options] [recipename/target recipe:do_task ...] + + Executes the specified task (default is 'build') for a given set of target recipes (.bb files). + It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which + will provide the layer, BBFILES and other configuration information. + + Options: + --version show program's version number and exit + -h, --help show this help message and exit + -b BUILDFILE, --buildfile=BUILDFILE + Execute tasks from a specific .bb recipe directly. + WARNING: Does not handle any dependencies from other + recipes. + -k, --continue Continue as much as possible after an error. While the + target that failed and anything depending on it cannot + be built, as much as possible will be built before + stopping. + -f, --force Force the specified targets/task to run (invalidating + any existing stamp file). + -c CMD, --cmd=CMD Specify the task to execute. The exact options + available depend on the metadata. Some examples might + be 'compile' or 'populate_sysroot' or 'listtasks' may + give a list of the tasks available. + -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP + Invalidate the stamp for the specified task such as + 'compile' and then run the default task for the + specified target(s). + -r PREFILE, --read=PREFILE + Read the specified file before bitbake.conf. + -R POSTFILE, --postread=POSTFILE + Read the specified file after bitbake.conf. + -v, --verbose Enable tracing of shell tasks (with 'set -x'). Also + print bb.note(...) messages to stdout (in addition to + writing them to ${T}/log.do_<task>). + -D, --debug Increase the debug level. You can specify this more + than once. -D sets the debug level to 1, where only + bb.debug(1, ...) messages are printed to stdout; -DD + sets the debug level to 2, where both bb.debug(1, ...) + and bb.debug(2, ...) messages are printed; etc. + Without -D, no debug messages are printed. Note that + -D only affects output to stdout. All debug messages + are written to ${T}/log.do_taskname, regardless of the + debug level. + -q, --quiet Output less log message data to the terminal. You can + specify this more than once. + -n, --dry-run Don't execute, just go through the motions. + -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER + Dump out the signature construction information, with + no task execution. The SIGNATURE_HANDLER parameter is + passed to the handler. Two common values are none and + printdiff but the handler may define more/less. none + means only dump the signature, printdiff means compare + the dumped signature with the cached one. + -p, --parse-only Quit after parsing the BB recipes. + -s, --show-versions Show current and preferred versions of all recipes. + -e, --environment Show the global or per-recipe environment complete + with information about where variables were + set/changed. + -g, --graphviz Save dependency tree information for the specified + targets in the dot syntax. + -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED + Assume these dependencies don't exist and are already + provided (equivalent to ASSUME_PROVIDED). Useful to + make dependency graphs more appealing + -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS + Show debug logging for the specified logging domains + -P, --profile Profile the command and save reports. + -u UI, --ui=UI The user interface to use (knotty, ncurses or taskexp + - default knotty). + --token=XMLRPCTOKEN Specify the connection token to be used when + connecting to a remote server. + --revisions-changed Set the exit code depending on whether upstream + floating revisions have changed or not. + --server-only Run bitbake without a UI, only starting a server + (cooker) process. + -B BIND, --bind=BIND The name/address for the bitbake xmlrpc server to bind + to. + -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT + Set timeout to unload bitbake server due to + inactivity, set to -1 means no unload, default: + Environment variable BB_SERVER_TIMEOUT. + --no-setscene Do not run any setscene tasks. sstate will be ignored + and everything needed, built. + --setscene-only Only run setscene tasks, don't run any real tasks. + --remote-server=REMOTE_SERVER + Connect to the specified server. + -m, --kill-server Terminate any running bitbake server. + --observe-only Connect to a server as an observing-only client. + --status-only Check the status of the remote bitbake server. + -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG + Writes the event log of the build to a bitbake event + json file. Use '' (empty string) to assign the name + automatically. + --runall=RUNALL Run the specified task for any recipe in the taskgraph + of the specified target (even if it wouldn't otherwise + have run). + --runonly=RUNONLY Run only the specified task within the taskgraph of + the specified targets (and any task dependencies those + tasks may have). + +.. _bitbake-examples: + +Examples +-------- + +This section presents some examples showing how to use BitBake. + +.. _example-executing-a-task-against-a-single-recipe: + +Executing a Task Against a Single Recipe +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Executing tasks for a single recipe file is relatively simple. You +specify the file in question, and BitBake parses it and executes the +specified task. If you do not specify a task, BitBake executes the +default task, which is "build”. BitBake obeys inter-task dependencies +when doing so. + +The following command runs the build task, which is the default task, on +the ``foo_1.0.bb`` recipe file: :: + + $ bitbake -b foo_1.0.bb + +The following command runs the clean task on the ``foo.bb`` recipe file: :: + + $ bitbake -b foo.bb -c clean + +.. note:: + + The "-b" option explicitly does not handle recipe dependencies. Other + than for debugging purposes, it is instead recommended that you use + the syntax presented in the next section. + +Executing Tasks Against a Set of Recipe Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are a number of additional complexities introduced when one wants +to manage multiple ``.bb`` files. Clearly there needs to be a way to +tell BitBake what files are available and, of those, which you want to +execute. There also needs to be a way for each recipe to express its +dependencies, both for build-time and runtime. There must be a way for +you to express recipe preferences when multiple recipes provide the same +functionality, or when there are multiple versions of a recipe. + +The ``bitbake`` command, when not using "--buildfile" or "-b" only +accepts a "PROVIDES". You cannot provide anything else. By default, a +recipe file generally "PROVIDES" its "packagename" as shown in the +following example: :: + + $ bitbake foo + +This next example "PROVIDES" the +package name and also uses the "-c" option to tell BitBake to just +execute the ``do_clean`` task: :: + + $ bitbake -c clean foo + +Executing a List of Task and Recipe Combinations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The BitBake command line supports specifying different tasks for +individual targets when you specify multiple targets. For example, +suppose you had two targets (or recipes) ``myfirstrecipe`` and +``mysecondrecipe`` and you needed BitBake to run ``taskA`` for the first +recipe and ``taskB`` for the second recipe: :: + + $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB + +Generating Dependency Graphs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +BitBake is able to generate dependency graphs using the ``dot`` syntax. +You can convert these graphs into images using the ``dot`` tool from +`Graphviz <http://www.graphviz.org>`__. + +When you generate a dependency graph, BitBake writes two files to the +current working directory: + +- ``task-depends.dot``: Shows dependencies between tasks. These + dependencies match BitBake's internal task execution list. + +- ``pn-buildlist``: Shows a simple list of targets that are to be + built. + +To stop depending on common depends, use the "-I" depend option and +BitBake omits them from the graph. Leaving this information out can +produce more readable graphs. This way, you can remove from the graph +``DEPENDS`` from inherited classes such as ``base.bbclass``. + +Here are two examples that create dependency graphs. The second example +omits depends common in OpenEmbedded from the graph: :: + + $ bitbake -g foo + + $ bitbake -g -I virtual/kernel -I eglibc foo + +Executing a Multiple Configuration Build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +BitBake is able to build multiple images or packages using a single +command where the different targets require different configurations +(multiple configuration builds). Each target, in this scenario, is +referred to as a "multiconfig". + +To accomplish a multiple configuration build, you must define each +target's configuration separately using a parallel configuration file in +the build directory. The location for these multiconfig configuration +files is specific. They must reside in the current build directory in a +sub-directory of ``conf`` named ``multiconfig``. Following is an example +for two separate targets: + +.. image:: figures/bb_multiconfig_files.png + :align: center + +The reason for this required file hierarchy is because the ``BBPATH`` +variable is not constructed until the layers are parsed. Consequently, +using the configuration file as a pre-configuration file is not possible +unless it is located in the current working directory. + +Minimally, each configuration file must define the machine and the +temporary directory BitBake uses for the build. Suggested practice +dictates that you do not overlap the temporary directories used during +the builds. + +Aside from separate configuration files for each target, you must also +enable BitBake to perform multiple configuration builds. Enabling is +accomplished by setting the +:term:`BBMULTICONFIG` variable in the +``local.conf`` configuration file. As an example, suppose you had +configuration files for ``target1`` and ``target2`` defined in the build +directory. The following statement in the ``local.conf`` file both +enables BitBake to perform multiple configuration builds and specifies +the two extra multiconfigs: :: + + BBMULTICONFIG = "target1 target2" + +Once the target configuration files are in place and BitBake has been +enabled to perform multiple configuration builds, use the following +command form to start the builds: :: + + $ bitbake [mc:multiconfigname:]target [[[mc:multiconfigname:]target] ... ] + +Here is an example for two extra multiconfigs: ``target1`` and ``target2``: :: + + $ bitbake mc::target mc:target1:target mc:target2:target + +.. _bb-enabling-multiple-configuration-build-dependencies: + +Enabling Multiple Configuration Build Dependencies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes dependencies can exist between targets (multiconfigs) in a +multiple configuration build. For example, suppose that in order to +build an image for a particular architecture, the root filesystem of +another build for a different architecture needs to exist. In other +words, the image for the first multiconfig depends on the root +filesystem of the second multiconfig. This dependency is essentially +that the task in the recipe that builds one multiconfig is dependent on +the completion of the task in the recipe that builds another +multiconfig. + +To enable dependencies in a multiple configuration build, you must +declare the dependencies in the recipe using the following statement +form: :: + + task_or_package[mcdepends] = "mc:from_multiconfig:to_multiconfig:recipe_name:task_on_which_to_depend" + +To better show how to use this statement, consider an example with two +multiconfigs: ``target1`` and ``target2``: :: + + image_task[mcdepends] = "mc:target1:target2:image2:rootfs_task" + +In this example, the +``from_multiconfig`` is "target1" and the ``to_multiconfig`` is "target2". The +task on which the image whose recipe contains image_task depends on the +completion of the rootfs_task used to build out image2, which is +associated with the "target2" multiconfig. + +Once you set up this dependency, you can build the "target1" multiconfig +using a BitBake command as follows: :: + + $ bitbake mc:target1:image1 + +This command executes all the tasks needed to create ``image1`` for the "target1" +multiconfig. Because of the dependency, BitBake also executes through +the ``rootfs_task`` for the "target2" multiconfig build. + +Having a recipe depend on the root filesystem of another build might not +seem that useful. Consider this change to the statement in the image1 +recipe: :: + + image_task[mcdepends] = "mc:target1:target2:image2:image_task" + +In this case, BitBake must create ``image2`` for the "target2" build since +the "target1" build depends on it. + +Because "target1" and "target2" are enabled for multiple configuration +builds and have separate configuration files, BitBake places the +artifacts for each build in the respective temporary build directories. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst new file mode 100644 index 000000000..ec5d08590 --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -0,0 +1,1969 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +==================== +Syntax and Operators +==================== + +| + +BitBake files have their own syntax. The syntax has similarities to +several other languages but also has some unique features. This section +describes the available syntax and operators as well as provides +examples. + +Basic Syntax +============ + +This section provides some basic syntax examples. + +Basic Variable Setting +---------------------- + +The following example sets ``VARIABLE`` to "value". This assignment +occurs immediately as the statement is parsed. It is a "hard" +assignment. :: + + VARIABLE = "value" + +As expected, if you include leading or +trailing spaces as part of an assignment, the spaces are retained: :: + + VARIABLE = " value" + VARIABLE = "value " + +Setting ``VARIABLE`` to "" sets +it to an empty string, while setting the variable to " " sets it to a +blank space (i.e. these are not the same values). :: + + VARIABLE = "" + VARIABLE = " " + +You can use single quotes instead of double quotes when setting a +variable's value. Doing so allows you to use values that contain the +double quote character: :: + + VARIABLE = 'I have a " in my value' + +.. note:: + + Unlike in Bourne shells, single quotes work identically to double + quotes in all other ways. They do not suppress variable expansions. + +Modifying Existing Variables +---------------------------- + +Sometimes you need to modify existing variables. Following are some +cases where you might find you want to modify an existing variable: + +- Customize a recipe that uses the variable. + +- Change a variable's default value used in a ``*.bbclass`` file. + +- Change the variable in a ``*.bbappend`` file to override the variable + in the original recipe. + +- Change the variable in a configuration file so that the value + overrides an existing configuration. + +Changing a variable value can sometimes depend on how the value was +originally assigned and also on the desired intent of the change. In +particular, when you append a value to a variable that has a default +value, the resulting value might not be what you expect. In this case, +the value you provide might replace the value rather than append to the +default value. + +If after you have changed a variable's value and something unexplained +occurs, you can use BitBake to check the actual value of the suspect +variable. You can make these checks for both configuration and recipe +level changes: + +- For configuration changes, use the following: :: + + $ bitbake -e + + This + command displays variable values after the configuration files (i.e. + ``local.conf``, ``bblayers.conf``, ``bitbake.conf`` and so forth) + have been parsed. + + .. note:: + + Variables that are exported to the environment are preceded by the + string "export" in the command's output. + +- For recipe changes, use the following: :: + + $ bitbake recipe -e \| grep VARIABLE=" + + This command checks to see if the variable actually makes + it into a specific recipe. + +Line Joining +------------ + +Outside of :ref:`functions <bitbake-user-manual/bitbake-user-manual-metadata:functions>`, +BitBake joins any line ending in +a backslash character ("\") with the following line before parsing +statements. The most common use for the "\" character is to split +variable assignments over multiple lines, as in the following example: :: + + FOO = "bar \ + baz \ + qaz" + +Both the "\" character and the newline +character that follow it are removed when joining lines. Thus, no +newline characters end up in the value of ``FOO``. + +Consider this additional example where the two assignments both assign +"barbaz" to ``FOO``: :: + + FOO = "barbaz" + FOO = "bar\ + baz" + +.. note:: + + BitBake does not interpret escape sequences like "\n" in variable + values. For these to have an effect, the value must be passed to some + utility that interprets escape sequences, such as + ``printf`` or ``echo -n``. + +Variable Expansion +------------------ + +Variables can reference the contents of other variables using a syntax +that is similar to variable expansion in Bourne shells. The following +assignments result in A containing "aval" and B evaluating to +"preavalpost". :: + + A = "aval" + B = "pre${A}post" + +.. note:: + + Unlike in Bourne shells, the curly braces are mandatory: Only ``${FOO}`` and not + ``$FOO`` is recognized as an expansion of ``FOO``. + +The "=" operator does not immediately expand variable references in the +right-hand side. Instead, expansion is deferred until the variable +assigned to is actually used. The result depends on the current values +of the referenced variables. The following example should clarify this +behavior: :: + + A = "${B} baz" + B = "${C} bar" + C = "foo" + *At this point, ${A} equals "foo bar baz"* + C = "qux" + *At this point, ${A} equals "qux bar baz"* + B = "norf" + *At this point, ${A} equals "norf baz"\* + +Contrast this behavior with the +:ref:`bitbake-user-manual/bitbake-user-manual-metadata:immediate variable +expansion (:=)` operator. + +If the variable expansion syntax is used on a variable that does not +exist, the string is kept as is. For example, given the following +assignment, ``BAR`` expands to the literal string "${FOO}" as long as +``FOO`` does not exist. :: + + BAR = "${FOO}" + +Setting a default value (?=) +---------------------------- + +You can use the "?=" operator to achieve a "softer" assignment for a +variable. This type of assignment allows you to define a variable if it +is undefined when the statement is parsed, but to leave the value alone +if the variable has a value. Here is an example: :: + + A ?= "aval" + +If ``A`` is +set at the time this statement is parsed, the variable retains its +value. However, if ``A`` is not set, the variable is set to "aval". + +.. note:: + + This assignment is immediate. Consequently, if multiple "?=" + assignments to a single variable exist, the first of those ends up + getting used. + +Setting a weak default value (??=) +---------------------------------- + +It is possible to use a "weaker" assignment than in the previous section +by using the "??=" operator. This assignment behaves identical to "?=" +except that the assignment is made at the end of the parsing process +rather than immediately. Consequently, when multiple "??=" assignments +exist, the last one is used. Also, any "=" or "?=" assignment will +override the value set with "??=". Here is an example: :: + + A ??= "somevalue" + A ??= "someothervalue" + +If ``A`` is set before the above statements are +parsed, the variable retains its value. If ``A`` is not set, the +variable is set to "someothervalue". + +Again, this assignment is a "lazy" or "weak" assignment because it does +not occur until the end of the parsing process. + +Immediate variable expansion (:=) +--------------------------------- + +The ":=" operator results in a variable's contents being expanded +immediately, rather than when the variable is actually used: :: + + T = "123" + A := "test ${T}" + T = "456" + B := "${T} ${C}" + C = "cval" + C := "${C}append" + +In this example, ``A`` contains "test 123", even though the final value +of ``T`` is "456". The variable ``B`` will end up containing "456 +cvalappend". This is because references to undefined variables are +preserved as is during (immediate)expansion. This is in contrast to GNU +Make, where undefined variables expand to nothing. The variable ``C`` +contains "cvalappend" since ``${C}`` immediately expands to "cval". + +.. _appending-and-prepending: + +Appending (+=) and prepending (=+) With Spaces +---------------------------------------------- + +Appending and prepending values is common and can be accomplished using +the "+=" and "=+" operators. These operators insert a space between the +current value and prepended or appended value. + +These operators take immediate effect during parsing. Here are some +examples: :: + + B = "bval" + B += "additionaldata" + C = "cval" + C =+ "test" + +The variable ``B`` contains "bval additionaldata" and ``C`` contains "test +cval". + +.. _appending-and-prepending-without-spaces: + +Appending (.=) and Prepending (=.) Without Spaces +------------------------------------------------- + +If you want to append or prepend values without an inserted space, use +the ".=" and "=." operators. + +These operators take immediate effect during parsing. Here are some +examples: :: + + B = "bval" + B .= "additionaldata" + C = "cval" + C =. "test" + +The variable ``B`` contains "bvaladditionaldata" and ``C`` contains +"testcval". + +Appending and Prepending (Override Style Syntax) +------------------------------------------------ + +You can also append and prepend a variable's value using an override +style syntax. When you use this syntax, no spaces are inserted. + +These operators differ from the ":=", ".=", "=.", "+=", and "=+" +operators in that their effects are applied at variable expansion time +rather than being immediately applied. Here are some examples: :: + + B = "bval" + B_append = " additional data" + C = "cval" + C_prepend = "additional data " + D = "dval" + D_append = "additional data" + +The variable ``B`` +becomes "bval additional data" and ``C`` becomes "additional data cval". +The variable ``D`` becomes "dvaladditional data". + +.. note:: + + You must control all spacing when you use the override syntax. + +It is also possible to append and prepend to shell functions and +BitBake-style Python functions. See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:shell functions`" and ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:bitbake-style python functions`" +sections for examples. + +.. _removing-override-style-syntax: + +Removal (Override Style Syntax) +------------------------------- + +You can remove values from lists using the removal override style +syntax. Specifying a value for removal causes all occurrences of that +value to be removed from the variable. + +When you use this syntax, BitBake expects one or more strings. +Surrounding spaces and spacing are preserved. Here is an example: :: + + FOO = "123 456 789 123456 123 456 123 456" + FOO_remove = "123" + FOO_remove = "456" + FOO2 = " abc def ghi abcdef abc def abc def def" + FOO2_remove = "\ + def \ + abc \ + ghi \ + " + +The variable ``FOO`` becomes +" 789 123456 " and ``FOO2`` becomes " abcdef ". + +Like "_append" and "_prepend", "_remove" is applied at variable +expansion time. + +Override Style Operation Advantages +----------------------------------- + +An advantage of the override style operations "_append", "_prepend", and +"_remove" as compared to the "+=" and "=+" operators is that the +override style operators provide guaranteed operations. For example, +consider a class ``foo.bbclass`` that needs to add the value "val" to +the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows: :: + + inherit foo + FOO = "initial" + +If ``foo.bbclass`` uses the "+=" operator, +as follows, then the final value of ``FOO`` will be "initial", which is +not what is desired: :: + + FOO += "val" + +If, on the other hand, ``foo.bbclass`` +uses the "_append" operator, then the final value of ``FOO`` will be +"initial val", as intended: :: + + FOO_append = " val" + +.. note:: + + It is never necessary to use "+=" together with "_append". The following + sequence of assignments appends "barbaz" to FOO: :: + + FOO_append = "bar" + FOO_append = "baz" + + + The only effect of changing the second assignment in the previous + example to use "+=" would be to add a space before "baz" in the + appended value (due to how the "+=" operator works). + +Another advantage of the override style operations is that you can +combine them with other overrides as described in the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:conditional syntax (overrides)`" section. + +Variable Flag Syntax +-------------------- + +Variable flags are BitBake's implementation of variable properties or +attributes. It is a way of tagging extra information onto a variable. +You can find more out about variable flags in general in the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" section. + +You can define, append, and prepend values to variable flags. All the +standard syntax operations previously mentioned work for variable flags +except for override style syntax (i.e. "_prepend", "_append", and +"_remove"). + +Here are some examples showing how to set variable flags: :: + + FOO[a] = "abc" + FOO[b] = "123" + FOO[a] += "456" + +The variable ``FOO`` has two flags: +``[a]`` and ``[b]``. The flags are immediately set to "abc" and "123", +respectively. The ``[a]`` flag becomes "abc 456". + +No need exists to pre-define variable flags. You can simply start using +them. One extremely common application is to attach some brief +documentation to a BitBake variable as follows: :: + + CACHE[doc] = "The directory holding the cache of the metadata." + +Inline Python Variable Expansion +-------------------------------- + +You can use inline Python variable expansion to set variables. Here is +an example: :: + + DATE = "${@time.strftime('%Y%m%d',time.gmtime())}" + +This example results in the ``DATE`` variable being set to the current date. + +Probably the most common use of this feature is to extract the value of +variables from BitBake's internal data dictionary, ``d``. The following +lines select the values of a package name and its version number, +respectively: :: + + PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}" + PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}" + +.. note:: + + Inline Python expressions work just like variable expansions insofar as the + "=" and ":=" operators are concerned. Given the following assignment, foo() + is called each time FOO is expanded: :: + + FOO = "${@foo()}" + + Contrast this with the following immediate assignment, where foo() is only + called once, while the assignment is parsed: :: + + FOO := "${@foo()}" + +For a different way to set variables with Python code during parsing, +see the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:anonymous python functions`" section. + +Unsetting variables +------------------- + +It is possible to completely remove a variable or a variable flag from +BitBake's internal data dictionary by using the "unset" keyword. Here is +an example: :: + + unset DATE + unset do_fetch[noexec] + +These two statements remove the ``DATE`` and the ``do_fetch[noexec]`` flag. + +Providing Pathnames +------------------- + +When specifying pathnames for use with BitBake, do not use the tilde +("~") character as a shortcut for your home directory. Doing so might +cause BitBake to not recognize the path since BitBake does not expand +this character in the same way a shell would. + +Instead, provide a fuller path as the following example illustrates: :: + + BBLAYERS ?= " \ + /home/scott-lenovo/LayerA \ + " + +Exporting Variables to the Environment +====================================== + +You can export variables to the environment of running tasks by using +the ``export`` keyword. For example, in the following example, the +``do_foo`` task prints "value from the environment" when run: :: + + export ENV_VARIABLE + ENV_VARIABLE = "value from the environment" + + do_foo() { + bbplain "$ENV_VARIABLE" + } + +.. note:: + + BitBake does not expand ``$ENV_VARIABLE`` in this case because it lacks the + obligatory ``{}`` . Rather, ``$ENV_VARIABLE`` is expanded by the shell. + +It does not matter whether ``export ENV_VARIABLE`` appears before or +after assignments to ``ENV_VARIABLE``. + +It is also possible to combine ``export`` with setting a value for the +variable. Here is an example: :: + + export ENV_VARIABLE = "variable-value" + +In the output of ``bitbake -e``, variables that are exported to the +environment are preceded by "export". + +Among the variables commonly exported to the environment are ``CC`` and +``CFLAGS``, which are picked up by many build systems. + +Conditional Syntax (Overrides) +============================== + +BitBake uses :term:`OVERRIDES` to control what +variables are overridden after BitBake parses recipes and configuration +files. This section describes how you can use ``OVERRIDES`` as +conditional metadata, talks about key expansion in relationship to +``OVERRIDES``, and provides some examples to help with understanding. + +Conditional Metadata +-------------------- + +You can use ``OVERRIDES`` to conditionally select a specific version of +a variable and to conditionally append or prepend the value of a +variable. + +.. note:: + + Overrides can only use lower-case characters. Additionally, + underscores are not permitted in override names as they are used to + separate overrides from each other and from the variable name. + +- *Selecting a Variable:* The ``OVERRIDES`` variable is a + colon-character-separated list that contains items for which you want + to satisfy conditions. Thus, if you have a variable that is + conditional on “arm”, and “arm” is in ``OVERRIDES``, then the + “arm”-specific version of the variable is used rather than the + non-conditional version. Here is an example: :: + + OVERRIDES = "architecture:os:machine" + TEST = "default" + TEST_os = "osspecific" + TEST_nooverride = "othercondvalue" + + In this example, the ``OVERRIDES`` + variable lists three overrides: "architecture", "os", and "machine". + The variable ``TEST`` by itself has a default value of "default". You + select the os-specific version of the ``TEST`` variable by appending + the "os" override to the variable (i.e. ``TEST_os``). + + To better understand this, consider a practical example that assumes + an OpenEmbedded metadata-based Linux kernel recipe file. The + following lines from the recipe file first set the kernel branch + variable ``KBRANCH`` to a default value, then conditionally override + that value based on the architecture of the build: :: + + KBRANCH = "standard/base" + KBRANCH_qemuarm = "standard/arm-versatile-926ejs" + KBRANCH_qemumips = "standard/mti-malta32" + KBRANCH_qemuppc = "standard/qemuppc" + KBRANCH_qemux86 = "standard/common-pc/base" + KBRANCH_qemux86-64 = "standard/common-pc-64/base" + KBRANCH_qemumips64 = "standard/mti-malta64" + +- *Appending and Prepending:* BitBake also supports append and prepend + operations to variable values based on whether a specific item is + listed in ``OVERRIDES``. Here is an example: :: + + DEPENDS = "glibc ncurses" + OVERRIDES = "machine:local" + DEPENDS_append_machine = "libmad" + + In this example, ``DEPENDS`` becomes "glibc ncurses libmad". + + Again, using an OpenEmbedded metadata-based kernel recipe file as an + example, the following lines will conditionally append to the + ``KERNEL_FEATURES`` variable based on the architecture: :: + + KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}" + KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc" + KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc" + +- *Setting a Variable for a Single Task:* BitBake supports setting a + variable just for the duration of a single task. Here is an example: :: + + FOO_task-configure = "val 1" + FOO_task-compile = "val 2" + + In the + previous example, ``FOO`` has the value "val 1" while the + ``do_configure`` task is executed, and the value "val 2" while the + ``do_compile`` task is executed. + + Internally, this is implemented by prepending the task (e.g. + "task-compile:") to the value of + :term:`OVERRIDES` for the local datastore of the + ``do_compile`` task. + + You can also use this syntax with other combinations (e.g. + "``_prepend``") as shown in the following example: :: + + EXTRA_OEMAKE_prepend_task-compile = "${PARALLEL_MAKE} " + +Key Expansion +------------- + +Key expansion happens when the BitBake datastore is finalized. To better +understand this, consider the following example: :: + + A${B} = "X" + B = "2" + A2 = "Y" + +In this case, after all the parsing is complete, BitBake expands +``${B}`` into "2". This expansion causes ``A2``, which was set to "Y" +before the expansion, to become "X". + +.. _variable-interaction-worked-examples: + +Examples +-------- + +Despite the previous explanations that show the different forms of +variable definitions, it can be hard to work out exactly what happens +when variable operators, conditional overrides, and unconditional +overrides are combined. This section presents some common scenarios +along with explanations for variable interactions that typically confuse +users. + +There is often confusion concerning the order in which overrides and +various "append" operators take effect. Recall that an append or prepend +operation using "_append" and "_prepend" does not result in an immediate +assignment as would "+=", ".=", "=+", or "=.". Consider the following +example: :: + + OVERRIDES = "foo" + A = "Z" + A_foo_append = "X" + +For this case, +``A`` is unconditionally set to "Z" and "X" is unconditionally and +immediately appended to the variable ``A_foo``. Because overrides have +not been applied yet, ``A_foo`` is set to "X" due to the append and +``A`` simply equals "Z". + +Applying overrides, however, changes things. Since "foo" is listed in +``OVERRIDES``, the conditional variable ``A`` is replaced with the "foo" +version, which is equal to "X". So effectively, ``A_foo`` replaces +``A``. + +This next example changes the order of the override and the append: :: + + OVERRIDES = "foo" + A = "Z" + A_append_foo = "X" + +For this case, before +overrides are handled, ``A`` is set to "Z" and ``A_append_foo`` is set +to "X". Once the override for "foo" is applied, however, ``A`` gets +appended with "X". Consequently, ``A`` becomes "ZX". Notice that spaces +are not appended. + +This next example has the order of the appends and overrides reversed +back as in the first example: :: + + OVERRIDES = "foo" + A = "Y" + A_foo_append = "Z" + A_foo_append = "X" + +For this case, before any overrides are resolved, +``A`` is set to "Y" using an immediate assignment. After this immediate +assignment, ``A_foo`` is set to "Z", and then further appended with "X" +leaving the variable set to "ZX". Finally, applying the override for +"foo" results in the conditional variable ``A`` becoming "ZX" (i.e. +``A`` is replaced with ``A_foo``). + +This final example mixes in some varying operators: :: + + A = "1" + A_append = "2" + A_append = "3" + A += "4" + A .= "5" + +For this case, the type of append +operators are affecting the order of assignments as BitBake passes +through the code multiple times. Initially, ``A`` is set to "1 45" +because of the three statements that use immediate operators. After +these assignments are made, BitBake applies the "_append" operations. +Those operations result in ``A`` becoming "1 4523". + +Sharing Functionality +===================== + +BitBake allows for metadata sharing through include files (``.inc``) and +class files (``.bbclass``). For example, suppose you have a piece of +common functionality such as a task definition that you want to share +between more than one recipe. In this case, creating a ``.bbclass`` file +that contains the common functionality and then using the ``inherit`` +directive in your recipes to inherit the class would be a common way to +share the task. + +This section presents the mechanisms BitBake provides to allow you to +share functionality between recipes. Specifically, the mechanisms +include ``include``, ``inherit``, ``INHERIT``, and ``require`` +directives. + +Locating Include and Class Files +-------------------------------- + +BitBake uses the :term:`BBPATH` variable to locate +needed include and class files. Additionally, BitBake searches the +current directory for ``include`` and ``require`` directives. + +.. note:: + + The BBPATH variable is analogous to the environment variable PATH . + +In order for include and class files to be found by BitBake, they need +to be located in a "classes" subdirectory that can be found in +``BBPATH``. + +``inherit`` Directive +--------------------- + +When writing a recipe or class file, you can use the ``inherit`` +directive to inherit the functionality of a class (``.bbclass``). +BitBake only supports this directive when used within recipe and class +files (i.e. ``.bb`` and ``.bbclass``). + +The ``inherit`` directive is a rudimentary means of specifying +functionality contained in class files that your recipes require. For +example, you can easily abstract out the tasks involved in building a +package that uses Autoconf and Automake and put those tasks into a class +file and then have your recipe inherit that class file. + +As an example, your recipes could use the following directive to inherit +an ``autotools.bbclass`` file. The class file would contain common +functionality for using Autotools that could be shared across recipes: :: + + inherit autotools + +In this case, BitBake would search for the directory +``classes/autotools.bbclass`` in ``BBPATH``. + +.. note:: + + You can override any values and functions of the inherited class + within your recipe by doing so after the "inherit" statement. + +If you want to use the directive to inherit multiple classes, separate +them with spaces. The following example shows how to inherit both the +``buildhistory`` and ``rm_work`` classes: :: + + inherit buildhistory rm_work + +An advantage with the inherit directive as compared to both the +:ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>` and :ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>` +directives is that you can inherit class files conditionally. You can +accomplish this by using a variable expression after the ``inherit`` +statement. Here is an example: :: + + inherit ${VARNAME} + +If ``VARNAME`` is +going to be set, it needs to be set before the ``inherit`` statement is +parsed. One way to achieve a conditional inherit in this case is to use +overrides: :: + + VARIABLE = "" + VARIABLE_someoverride = "myclass" + +Another method is by using anonymous Python. Here is an example: :: + + python () { + if condition == value: + d.setVar('VARIABLE', 'myclass') + else: + d.setVar('VARIABLE', '') + } + +Alternatively, you could use an in-line Python expression in the +following form: :: + + inherit ${@'classname' if condition else ''} + inherit ${@functionname(params)} + +In all cases, if the expression evaluates to an +empty string, the statement does not trigger a syntax error because it +becomes a no-op. + +``include`` Directive +--------------------- + +BitBake understands the ``include`` directive. This directive causes +BitBake to parse whatever file you specify, and to insert that file at +that location. The directive is much like its equivalent in Make except +that if the path specified on the include line is a relative path, +BitBake locates the first file it can find within ``BBPATH``. + +The include directive is a more generic method of including +functionality as compared to the :ref:`inherit <bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` directive>` +directive, which is restricted to class (i.e. ``.bbclass``) files. The +include directive is applicable for any other kind of shared or +encapsulated functionality or configuration that does not suit a +``.bbclass`` file. + +As an example, suppose you needed a recipe to include some self-test +definitions: :: + + include test_defs.inc + +.. note:: + + The include directive does not produce an error when the file cannot be + found. Consequently, it is recommended that if the file you are including is + expected to exist, you should use :ref:`require <require-inclusion>` instead + of include . Doing so makes sure that an error is produced if the file cannot + be found. + +.. _require-inclusion: + +``require`` Directive +--------------------- + +BitBake understands the ``require`` directive. This directive behaves +just like the ``include`` directive with the exception that BitBake +raises a parsing error if the file to be included cannot be found. Thus, +any file you require is inserted into the file that is being parsed at +the location of the directive. + +The require directive, like the include directive previously described, +is a more generic method of including functionality as compared to the +:ref:`inherit <bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` directive>` directive, which is restricted to class +(i.e. ``.bbclass``) files. The require directive is applicable for any +other kind of shared or encapsulated functionality or configuration that +does not suit a ``.bbclass`` file. + +Similar to how BitBake handles :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`, if +the path specified on the require line is a relative path, BitBake +locates the first file it can find within ``BBPATH``. + +As an example, suppose you have two versions of a recipe (e.g. +``foo_1.2.2.bb`` and ``foo_2.0.0.bb``) where each version contains some +identical functionality that could be shared. You could create an +include file named ``foo.inc`` that contains the common definitions +needed to build "foo". You need to be sure ``foo.inc`` is located in the +same directory as your two recipe files as well. Once these conditions +are set up, you can share the functionality using a ``require`` +directive from within each recipe: :: + + require foo.inc + +``INHERIT`` Configuration Directive +----------------------------------- + +When creating a configuration file (``.conf``), you can use the +:term:`INHERIT` configuration directive to inherit a +class. BitBake only supports this directive when used within a +configuration file. + +As an example, suppose you needed to inherit a class file called +``abc.bbclass`` from a configuration file as follows: :: + + INHERIT += "abc" + +This configuration directive causes the named class to be inherited at +the point of the directive during parsing. As with the ``inherit`` +directive, the ``.bbclass`` file must be located in a "classes" +subdirectory in one of the directories specified in ``BBPATH``. + +.. note:: + + Because .conf files are parsed first during BitBake's execution, using + INHERIT to inherit a class effectively inherits the class globally (i.e. for + all recipes). + +If you want to use the directive to inherit multiple classes, you can +provide them on the same line in the ``local.conf`` file. Use spaces to +separate the classes. The following example shows how to inherit both +the ``autotools`` and ``pkgconfig`` classes: :: + + INHERIT += "autotools pkgconfig" + +Functions +========= + +As with most languages, functions are the building blocks that are used +to build up operations into tasks. BitBake supports these types of +functions: + +- *Shell Functions:* Functions written in shell script and executed + either directly as functions, tasks, or both. They can also be called + by other shell functions. + +- *BitBake-Style Python Functions:* Functions written in Python and + executed by BitBake or other Python functions using + ``bb.build.exec_func()``. + +- *Python Functions:* Functions written in Python and executed by + Python. + +- *Anonymous Python Functions:* Python functions executed automatically + during parsing. + +Regardless of the type of function, you can only define them in class +(``.bbclass``) and recipe (``.bb`` or ``.inc``) files. + +Shell Functions +--------------- + +Functions written in shell script and executed either directly as +functions, tasks, or both. They can also be called by other shell +functions. Here is an example shell function definition: :: + + some_function () { + echo "Hello World" + } + +When you create these types of functions in +your recipe or class files, you need to follow the shell programming +rules. The scripts are executed by ``/bin/sh``, which may not be a bash +shell but might be something such as ``dash``. You should not use +Bash-specific script (bashisms). + +Overrides and override-style operators like ``_append`` and ``_prepend`` +can also be applied to shell functions. Most commonly, this application +would be used in a ``.bbappend`` file to modify functions in the main +recipe. It can also be used to modify functions inherited from classes. + +As an example, consider the following: :: + + do_foo() { + bbplain first + fn + } + + fn_prepend() { + bbplain second + } + + fn() { + bbplain third + } + + do_foo_append() { + bbplain fourth + } + +Running ``do_foo`` prints the following: :: + + recipename do_foo: first + recipename do_foo: second + recipename do_foo: third + recipename do_foo: fourth + +.. note:: + + Overrides and override-style operators can be applied to any shell + function, not just :ref:`tasks <bitbake-user-manual/bitbake-user-manual-metadata:tasks>`. + +You can use the ``bitbake -e`` recipename command to view the final +assembled function after all overrides have been applied. + +BitBake-Style Python Functions +------------------------------ + +These functions are written in Python and executed by BitBake or other +Python functions using ``bb.build.exec_func()``. + +An example BitBake function is: :: + + python some_python_function () { + d.setVar("TEXT", "Hello World") + print d.getVar("TEXT") + } + +Because the +Python "bb" and "os" modules are already imported, you do not need to +import these modules. Also in these types of functions, the datastore +("d") is a global variable and is always automatically available. + +.. note:: + + Variable expressions (e.g. ``${X}`` ) are no longer expanded within Python + functions. This behavior is intentional in order to allow you to freely set + variable values to expandable expressions without having them expanded + prematurely. If you do wish to expand a variable within a Python function, + use ``d.getVar("X")`` . Or, for more complicated expressions, use ``d.expand()``. + +Similar to shell functions, you can also apply overrides and +override-style operators to BitBake-style Python functions. + +As an example, consider the following: :: + + python do_foo_prepend() { + bb.plain("first") + } + + python do_foo() { + bb.plain("second") + } + + python do_foo_append() { + bb.plain("third") + } + +Running ``do_foo`` prints the following: :: + + recipename do_foo: first + recipename do_foo: second + recipename do_foo: third + +You can use the ``bitbake -e`` recipename command to view +the final assembled function after all overrides have been applied. + +Python Functions +---------------- + +These functions are written in Python and are executed by other Python +code. Examples of Python functions are utility functions that you intend +to call from in-line Python or from within other Python functions. Here +is an example: :: + + def get_depends(d): + if d.getVar('SOMECONDITION'): + return "dependencywithcond" + else: + return "dependency" + + SOMECONDITION = "1" + DEPENDS = "${@get_depends(d)}" + +This would result in ``DEPENDS`` containing ``dependencywithcond``. + +Here are some things to know about Python functions: + +- Python functions can take parameters. + +- The BitBake datastore is not automatically available. Consequently, + you must pass it in as a parameter to the function. + +- The "bb" and "os" Python modules are automatically available. You do + not need to import them. + +BitBake-Style Python Functions Versus Python Functions +------------------------------------------------------ + +Following are some important differences between BitBake-style Python +functions and regular Python functions defined with "def": + +- Only BitBake-style Python functions can be :ref:`tasks <bitbake-user-manual/bitbake-user-manual-metadata:tasks>`. + +- Overrides and override-style operators can only be applied to + BitBake-style Python functions. + +- Only regular Python functions can take arguments and return values. + +- :ref:`Variable flags <bitbake-user-manual/bitbake-user-manual-metadata:variable flags>` such as + ``[dirs]``, ``[cleandirs]``, and ``[lockfiles]`` can be used on BitBake-style + Python functions, but not on regular Python functions. + +- BitBake-style Python functions generate a separate + ``${``\ :term:`T`\ ``}/run.``\ function-name\ ``.``\ pid + script that is executed to run the function, and also generate a log + file in ``${T}/log.``\ function-name\ ``.``\ pid if they are executed + as tasks. + + Regular Python functions execute "inline" and do not generate any + files in ``${T}``. + +- Regular Python functions are called with the usual Python syntax. + BitBake-style Python functions are usually tasks and are called + directly by BitBake, but can also be called manually from Python code + by using the ``bb.build.exec_func()`` function. Here is an example: :: + + bb.build.exec_func("my_bitbake_style_function", d) + + .. note:: + + ``bb.build.exec_func()`` can also be used to run shell functions from Python + code. If you want to run a shell function before a Python function within + the same task, then you can use a parent helper Python function that + starts by running the shell function with ``bb.build.exec_func()`` and then + runs the Python code. + + To detect errors from functions executed with + ``bb.build.exec_func()``, you can catch the ``bb.build.FuncFailed`` + exception. + + .. note:: + + Functions in metadata (recipes and classes) should not themselves raise + ``bb.build.FuncFailed``. Rather, ``bb.build.FuncFailed`` should be viewed as a + general indicator that the called function failed by raising an + exception. For example, an exception raised by ``bb.fatal()`` will be caught + inside ``bb.build.exec_func()``, and a ``bb.build.FuncFailed`` will be raised in + response. + +Due to their simplicity, you should prefer regular Python functions over +BitBake-style Python functions unless you need a feature specific to +BitBake-style Python functions. Regular Python functions in metadata are +a more recent invention than BitBake-style Python functions, and older +code tends to use ``bb.build.exec_func()`` more often. + +Anonymous Python Functions +-------------------------- + +Sometimes it is useful to set variables or perform other operations +programmatically during parsing. To do this, you can define special +Python functions, called anonymous Python functions, that run at the end +of parsing. For example, the following conditionally sets a variable +based on the value of another variable: :: + + python () { + if d.getVar('SOMEVAR') == 'value': + d.setVar('ANOTHERVAR', 'value2') + } + +An equivalent way to mark a function as an anonymous function is to give it +the name "__anonymous", rather than no name. + +Anonymous Python functions always run at the end of parsing, regardless +of where they are defined. If a recipe contains many anonymous +functions, they run in the same order as they are defined within the +recipe. As an example, consider the following snippet: :: + + python () { + d.setVar('FOO', 'foo 2') + } + + FOO = "foo 1" + + python () { + d.appendVar('BAR',' bar 2') + } + + BAR = "bar 1" + +The previous example is conceptually +equivalent to the following snippet: :: + + FOO = "foo 1" + BAR = "bar 1" + FOO = "foo 2" + BAR += "bar 2" + +``FOO`` ends up with the value "foo 2", and +``BAR`` with the value "bar 1 bar 2". Just as in the second snippet, the +values set for the variables within the anonymous functions become +available to tasks, which always run after parsing. + +Overrides and override-style operators such as "``_append``" are applied +before anonymous functions run. In the following example, ``FOO`` ends +up with the value "foo from anonymous": :: + + FOO = "foo" + FOO_append = " from outside" + + python () { + d.setVar("FOO", "foo from anonymous") + } + +For methods +you can use with anonymous Python functions, see the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:functions you can call from within python`" +section. For a different method to run Python code during parsing, see +the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:inline python variable expansion`" section. + +Flexible Inheritance for Class Functions +---------------------------------------- + +Through coding techniques and the use of ``EXPORT_FUNCTIONS``, BitBake +supports exporting a function from a class such that the class function +appears as the default implementation of the function, but can still be +called if a recipe inheriting the class needs to define its own version +of the function. + +To understand the benefits of this feature, consider the basic scenario +where a class defines a task function and your recipe inherits the +class. In this basic scenario, your recipe inherits the task function as +defined in the class. If desired, your recipe can add to the start and +end of the function by using the "_prepend" or "_append" operations +respectively, or it can redefine the function completely. However, if it +redefines the function, there is no means for it to call the class +version of the function. ``EXPORT_FUNCTIONS`` provides a mechanism that +enables the recipe's version of the function to call the original +version of the function. + +To make use of this technique, you need the following things in place: + +- The class needs to define the function as follows: :: + + classname_functionname + + For example, if you have a class file + ``bar.bbclass`` and a function named ``do_foo``, the class must + define the function as follows: :: + + bar_do_foo + +- The class needs to contain the ``EXPORT_FUNCTIONS`` statement as + follows: :: + + EXPORT_FUNCTIONS functionname + + For example, continuing with + the same example, the statement in the ``bar.bbclass`` would be as + follows: :: + + EXPORT_FUNCTIONS do_foo + +- You need to call the function appropriately from within your recipe. + Continuing with the same example, if your recipe needs to call the + class version of the function, it should call ``bar_do_foo``. + Assuming ``do_foo`` was a shell function and ``EXPORT_FUNCTIONS`` was + used as above, the recipe's function could conditionally call the + class version of the function as follows: :: + + do_foo() { + if [ somecondition ] ; then + bar_do_foo + else + # Do something else + fi + } + + To call your modified version of the function as defined in your recipe, + call it as ``do_foo``. + +With these conditions met, your single recipe can freely choose between +the original function as defined in the class file and the modified +function in your recipe. If you do not set up these conditions, you are +limited to using one function or the other. + +Tasks +===== + +Tasks are BitBake execution units that make up the steps that BitBake +can run for a given recipe. Tasks are only supported in recipes and +classes (i.e. in ``.bb`` files and files included or inherited from +``.bb`` files). By convention, tasks have names that start with "do\_". + +Promoting a Function to a Task +------------------------------ + +Tasks are either :ref:`shell functions <bitbake-user-manual/bitbake-user-manual-metadata:shell functions>` or +:ref:`BitBake-style Python functions <bitbake-user-manual/bitbake-user-manual-metadata:bitbake-style python functions>` +that have been promoted to tasks by using the ``addtask`` command. The +``addtask`` command can also optionally describe dependencies between +the task and other tasks. Here is an example that shows how to define a +task and declare some dependencies: :: + + python do_printdate () { + import time + print time.strftime('%Y%m%d', time.gmtime()) + } + addtask printdate after do_fetch before do_build + +The first argument to ``addtask`` is the name +of the function to promote to a task. If the name does not start with +"do\_", "do\_" is implicitly added, which enforces the convention that all +task names start with "do\_". + +In the previous example, the ``do_printdate`` task becomes a dependency +of the ``do_build`` task, which is the default task (i.e. the task run +by the ``bitbake`` command unless another task is specified explicitly). +Additionally, the ``do_printdate`` task becomes dependent upon the +``do_fetch`` task. Running the ``do_build`` task results in the +``do_printdate`` task running first. + +.. note:: + + If you try out the previous example, you might see that the + ``do_printdate`` + task is only run the first time you build the recipe with the + ``bitbake`` + command. This is because BitBake considers the task "up-to-date" + after that initial run. If you want to force the task to always be + rerun for experimentation purposes, you can make BitBake always + consider the task "out-of-date" by using the + :ref:`[nostamp] <bitbake-user-manual/bitbake-user-manual-metadata:Variable Flags>` + variable flag, as follows: :: + + do_printdate[nostamp] = "1" + + You can also explicitly run the task and provide the + -f option as follows: :: + + $ bitbake recipe -c printdate -f + + When manually selecting a task to run with the bitbake ``recipe + -c task`` command, you can omit the "do\_" prefix as part of the task + name. + +You might wonder about the practical effects of using ``addtask`` +without specifying any dependencies as is done in the following example: :: + + addtask printdate + +In this example, assuming dependencies have not been +added through some other means, the only way to run the task is by +explicitly selecting it with ``bitbake`` recipe ``-c printdate``. You +can use the ``do_listtasks`` task to list all tasks defined in a recipe +as shown in the following example: :: + + $ bitbake recipe -c listtasks + +For more information on task dependencies, see the +":ref:`bitbake-user-manual/bitbake-user-manual-execution:dependencies`" section. + +See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" section for information +on variable flags you can use with tasks. + +Deleting a Task +--------------- + +As well as being able to add tasks, you can delete them. Simply use the +``deltask`` command to delete a task. For example, to delete the example +task used in the previous sections, you would use: :: + + deltask printdate + +If you delete a task using the ``deltask`` command and the task has +dependencies, the dependencies are not reconnected. For example, suppose +you have three tasks named ``do_a``, ``do_b``, and ``do_c``. +Furthermore, ``do_c`` is dependent on ``do_b``, which in turn is +dependent on ``do_a``. Given this scenario, if you use ``deltask`` to +delete ``do_b``, the implicit dependency relationship between ``do_c`` +and ``do_a`` through ``do_b`` no longer exists, and ``do_c`` +dependencies are not updated to include ``do_a``. Thus, ``do_c`` is free +to run before ``do_a``. + +If you want dependencies such as these to remain intact, use the +``[noexec]`` varflag to disable the task instead of using the +``deltask`` command to delete it: :: + + do_b[noexec] = "1" + +Passing Information Into the Build Task Environment +--------------------------------------------------- + +When running a task, BitBake tightly controls the shell execution +environment of the build tasks to make sure unwanted contamination from +the build machine cannot influence the build. + +.. note:: + + By default, BitBake cleans the environment to include only those + things exported or listed in its whitelist to ensure that the build + environment is reproducible and consistent. You can prevent this + "cleaning" by setting the :term:`BB_PRESERVE_ENV` variable. + +Consequently, if you do want something to get passed into the build task +environment, you must take these two steps: + +#. Tell BitBake to load what you want from the environment into the + datastore. You can do so through the + :term:`BB_ENV_WHITELIST` and + :term:`BB_ENV_EXTRAWHITE` variables. For + example, assume you want to prevent the build system from accessing + your ``$HOME/.ccache`` directory. The following command "whitelists" + the environment variable ``CCACHE_DIR`` causing BitBake to allow that + variable into the datastore: :: + + export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR" + +#. Tell BitBake to export what you have loaded into the datastore to the + task environment of every running task. Loading something from the + environment into the datastore (previous step) only makes it + available in the datastore. To export it to the task environment of + every running task, use a command similar to the following in your + local configuration file ``local.conf`` or your distribution + configuration file: :: + + export CCACHE_DIR + + .. note:: + + A side effect of the previous steps is that BitBake records the + variable as a dependency of the build process in things like the + setscene checksums. If doing so results in unnecessary rebuilds of + tasks, you can whitelist the variable so that the setscene code + ignores the dependency when it creates checksums. + +Sometimes, it is useful to be able to obtain information from the +original execution environment. BitBake saves a copy of the original +environment into a special variable named :term:`BB_ORIGENV`. + +The ``BB_ORIGENV`` variable returns a datastore object that can be +queried using the standard datastore operators such as +``getVar(, False)``. The datastore object is useful, for example, to +find the original ``DISPLAY`` variable. Here is an example: :: + + origenv = d.getVar("BB_ORIGENV", False) + bar = origenv.getVar("BAR", False) + +The previous example returns ``BAR`` from the original execution +environment. + +Variable Flags +============== + +Variable flags (varflags) help control a task's functionality and +dependencies. BitBake reads and writes varflags to the datastore using +the following command forms: :: + + variable = d.getVarFlags("variable") + self.d.setVarFlags("FOO", {"func": True}) + +When working with varflags, the same syntax, with the exception of +overrides, applies. In other words, you can set, append, and prepend +varflags just like variables. See the +":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flag syntax`" section for details. + +BitBake has a defined set of varflags available for recipes and classes. +Tasks support a number of these flags which control various +functionality of the task: + +- ``[cleandirs]``: Empty directories that should be created before + the task runs. Directories that already exist are removed and + recreated to empty them. + +- ``[depends]``: Controls inter-task dependencies. See the + :term:`DEPENDS` variable and the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:inter-task + dependencies`" section for more information. + +- ``[deptask]``: Controls task build-time dependencies. See the + :term:`DEPENDS` variable and the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:build dependencies`" section for more information. + +- ``[dirs]``: Directories that should be created before the task + runs. Directories that already exist are left as is. The last + directory listed is used as the current working directory for the + task. + +- ``[lockfiles]``: Specifies one or more lockfiles to lock while the + task executes. Only one task may hold a lockfile, and any task that + attempts to lock an already locked file will block until the lock is + released. You can use this variable flag to accomplish mutual + exclusion. + +- ``[noexec]``: When set to "1", marks the task as being empty, with + no execution required. You can use the ``[noexec]`` flag to set up + tasks as dependency placeholders, or to disable tasks defined + elsewhere that are not needed in a particular recipe. + +- ``[nostamp]``: When set to "1", tells BitBake to not generate a + stamp file for a task, which implies the task should always be + executed. + + .. caution:: + + Any task that depends (possibly indirectly) on a ``[nostamp]`` task will + always be executed as well. This can cause unnecessary rebuilding if you + are not careful. + +- ``[number_threads]``: Limits tasks to a specific number of + simultaneous threads during execution. This varflag is useful when + your build host has a large number of cores but certain tasks need to + be rate-limited due to various kinds of resource constraints (e.g. to + avoid network throttling). ``number_threads`` works similarly to the + :term:`BB_NUMBER_THREADS` variable but is task-specific. + + Set the value globally. For example, the following makes sure the + ``do_fetch`` task uses no more than two simultaneous execution + threads: do_fetch[number_threads] = "2" + + .. warning:: + + - Setting the varflag in individual recipes rather than globally + can result in unpredictable behavior. + + - Setting the varflag to a value greater than the value used in + the ``BB_NUMBER_THREADS`` variable causes ``number_threads`` to + have no effect. + +- ``[postfuncs]``: List of functions to call after the completion of + the task. + +- ``[prefuncs]``: List of functions to call before the task executes. + +- ``[rdepends]``: Controls inter-task runtime dependencies. See the + :term:`RDEPENDS` variable, the + :term:`RRECOMMENDS` variable, and the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:inter-task dependencies`" section for + more information. + +- ``[rdeptask]``: Controls task runtime dependencies. See the + :term:`RDEPENDS` variable, the + :term:`RRECOMMENDS` variable, and the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:runtime dependencies`" section for more + information. + +- ``[recideptask]``: When set in conjunction with ``recrdeptask``, + specifies a task that should be inspected for additional + dependencies. + +- ``[recrdeptask]``: Controls task recursive runtime dependencies. + See the :term:`RDEPENDS` variable, the + :term:`RRECOMMENDS` variable, and the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:recursive dependencies`" section for + more information. + +- ``[stamp-extra-info]``: Extra stamp information to append to the + task's stamp. As an example, OpenEmbedded uses this flag to allow + machine-specific tasks. + +- ``[umask]``: The umask to run the task under. + +Several varflags are useful for controlling how signatures are +calculated for variables. For more information on this process, see the +":ref:`bitbake-user-manual/bitbake-user-manual-execution:checksums (signatures)`" section. + +- ``[vardeps]``: Specifies a space-separated list of additional + variables to add to a variable's dependencies for the purposes of + calculating its signature. Adding variables to this list is useful, + for example, when a function refers to a variable in a manner that + does not allow BitBake to automatically determine that the variable + is referred to. + +- ``[vardepsexclude]``: Specifies a space-separated list of variables + that should be excluded from a variable's dependencies for the + purposes of calculating its signature. + +- ``[vardepvalue]``: If set, instructs BitBake to ignore the actual + value of the variable and instead use the specified value when + calculating the variable's signature. + +- ``[vardepvalueexclude]``: Specifies a pipe-separated list of + strings to exclude from the variable's value when calculating the + variable's signature. + +Events +====== + +BitBake allows installation of event handlers within recipe and class +files. Events are triggered at certain points during operation, such as +the beginning of operation against a given recipe (i.e. ``*.bb``), the +start of a given task, a task failure, a task success, and so forth. The +intent is to make it easy to do things like email notification on build +failures. + +Following is an example event handler that prints the name of the event +and the content of the ``FILE`` variable: :: + + addhandler myclass_eventhandler + python myclass_eventhandler() { + from bb.event import getName + print("The name of the Event is %s" % getName(e)) + print("The file we run for is %s" % d.getVar('FILE')) + } + myclass_eventhandler[eventmask] = "bb.event.BuildStarted + bb.event.BuildCompleted" + +In the previous example, an eventmask has been +set so that the handler only sees the "BuildStarted" and +"BuildCompleted" events. This event handler gets called every time an +event matching the eventmask is triggered. A global variable "e" is +defined, which represents the current event. With the ``getName(e)`` +method, you can get the name of the triggered event. The global +datastore is available as "d". In legacy code, you might see "e.data" +used to get the datastore. However, realize that "e.data" is deprecated +and you should use "d" going forward. + +The context of the datastore is appropriate to the event in question. +For example, "BuildStarted" and "BuildCompleted" events run before any +tasks are executed so would be in the global configuration datastore +namespace. No recipe-specific metadata exists in that namespace. The +"BuildStarted" and "BuildCompleted" events also run in the main +cooker/server process rather than any worker context. Thus, any changes +made to the datastore would be seen by other cooker/server events within +the current build but not seen outside of that build or in any worker +context. Task events run in the actual tasks in question consequently +have recipe-specific and task-specific contents. These events run in the +worker context and are discarded at the end of task execution. + +During a standard build, the following common events might occur. The +following events are the most common kinds of events that most metadata +might have an interest in viewing: + +- ``bb.event.ConfigParsed()``: Fired when the base configuration; which + consists of ``bitbake.conf``, ``base.bbclass`` and any global + ``INHERIT`` statements; has been parsed. You can see multiple such + events when each of the workers parse the base configuration or if + the server changes configuration and reparses. Any given datastore + only has one such event executed against it, however. If + ```BB_INVALIDCONF`` <#>`__ is set in the datastore by the event + handler, the configuration is reparsed and a new event triggered, + allowing the metadata to update configuration. + +- ``bb.event.HeartbeatEvent()``: Fires at regular time intervals of one + second. You can configure the interval time using the + ``BB_HEARTBEAT_EVENT`` variable. The event's "time" attribute is the + ``time.time()`` value when the event is triggered. This event is + useful for activities such as system state monitoring. + +- ``bb.event.ParseStarted()``: Fired when BitBake is about to start + parsing recipes. This event's "total" attribute represents the number + of recipes BitBake plans to parse. + +- ``bb.event.ParseProgress()``: Fired as parsing progresses. This + event's "current" attribute is the number of recipes parsed as well + as the "total" attribute. + +- ``bb.event.ParseCompleted()``: Fired when parsing is complete. This + event's "cached", "parsed", "skipped", "virtuals", "masked", and + "errors" attributes provide statistics for the parsing results. + +- ``bb.event.BuildStarted()``: Fired when a new build starts. BitBake + fires multiple "BuildStarted" events (one per configuration) when + multiple configuration (multiconfig) is enabled. + +- ``bb.build.TaskStarted()``: Fired when a task starts. This event's + "taskfile" attribute points to the recipe from which the task + originates. The "taskname" attribute, which is the task's name, + includes the ``do_`` prefix, and the "logfile" attribute point to + where the task's output is stored. Finally, the "time" attribute is + the task's execution start time. + +- ``bb.build.TaskInvalid()``: Fired if BitBake tries to execute a task + that does not exist. + +- ``bb.build.TaskFailedSilent()``: Fired for setscene tasks that fail + and should not be presented to the user verbosely. + +- ``bb.build.TaskFailed()``: Fired for normal tasks that fail. + +- ``bb.build.TaskSucceeded()``: Fired when a task successfully + completes. + +- ``bb.event.BuildCompleted()``: Fired when a build finishes. + +- ``bb.cooker.CookerExit()``: Fired when the BitBake server/cooker + shuts down. This event is usually only seen by the UIs as a sign they + should also shutdown. + +This next list of example events occur based on specific requests to the +server. These events are often used to communicate larger pieces of +information from the BitBake server to other parts of BitBake such as +user interfaces: + +- ``bb.event.TreeDataPreparationStarted()`` +- ``bb.event.TreeDataPreparationProgress()`` +- ``bb.event.TreeDataPreparationCompleted()`` +- ``bb.event.DepTreeGenerated()`` +- ``bb.event.CoreBaseFilesFound()`` +- ``bb.event.ConfigFilePathFound()`` +- ``bb.event.FilesMatchingFound()`` +- ``bb.event.ConfigFilesFound()`` +- ``bb.event.TargetsTreeGenerated()`` + +.. _variants-class-extension-mechanism: + +Variants - Class Extension Mechanism +==================================== + +BitBake supports two features that facilitate creating from a single +recipe file multiple incarnations of that recipe file where all +incarnations are buildable. These features are enabled through the +:term:`BBCLASSEXTEND` and :term:`BBVERSIONS` variables. + +.. note:: + + The mechanism for this class extension is extremely specific to the + implementation. Usually, the recipe's :term:`PROVIDES` , :term:`PN` , and + :term:`DEPENDS` variables would need to be modified by the extension + class. For specific examples, see the OE-Core native , nativesdk , and + multilib classes. + +- ``BBCLASSEXTEND``: This variable is a space separated list of + classes used to "extend" the recipe for each variant. Here is an + example that results in a second incarnation of the current recipe + being available. This second incarnation will have the "native" class + inherited. :: + + BBCLASSEXTEND = "native" + +- ``BBVERSIONS``: This variable allows a single recipe to build + multiple versions of a project from a single recipe file. You can + also specify conditional metadata (using the + :term:`OVERRIDES` mechanism) for a single + version, or an optionally named range of versions. Here is an + example: :: + + BBVERSIONS = "1.0 2.0 git" + SRC_URI_git = "git://someurl/somepath.git" + + BBVERSIONS = "1.0.[0-6]:1.0.0+ 1.0.[7-9]:1.0.7+" + SRC_URI_append_1.0.7+ = "file://some_patch_which_the_new_versions_need.patch;patch=1" + + The name of the range defaults to the original version of the recipe. For + example, in OpenEmbedded, the recipe file ``foo_1.0.0+.bb`` creates a default + name range of ``1.0.0+``. This is useful because the range name is not only + placed into overrides, but it is also made available for the metadata to use + in the variable that defines the base recipe versions for use in ``file://`` + search paths (:term:`FILESPATH`). + +Dependencies +============ + +To allow for efficient parallel processing, BitBake handles dependencies +at the task level. Dependencies can exist both between tasks within a +single recipe and between tasks in different recipes. Following are +examples of each: + +- For tasks within a single recipe, a recipe's ``do_configure`` task + might need to complete before its ``do_compile`` task can run. + +- For tasks in different recipes, one recipe's ``do_configure`` task + might require another recipe's ``do_populate_sysroot`` task to finish + first such that the libraries and headers provided by the other + recipe are available. + +This section describes several ways to declare dependencies. Remember, +even though dependencies are declared in different ways, they are all +simply dependencies between tasks. + +.. _dependencies-internal-to-the-bb-file: + +Dependencies Internal to the ``.bb`` File +----------------------------------------- + +BitBake uses the ``addtask`` directive to manage dependencies that are +internal to a given recipe file. You can use the ``addtask`` directive +to indicate when a task is dependent on other tasks or when other tasks +depend on that recipe. Here is an example: :: + + addtask printdate after do_fetch before do_build + +In this example, the ``do_printdate`` task +depends on the completion of the ``do_fetch`` task, and the ``do_build`` +task depends on the completion of the ``do_printdate`` task. + +.. note:: + + For a task to run, it must be a direct or indirect dependency of some + other task that is scheduled to run. + + For illustration, here are some examples: + + - The directive ``addtask mytask before do_configure`` causes + ``do_mytask`` to run before ``do_configure`` runs. Be aware that + ``do_mytask`` still only runs if its :ref:`input + checksum <bitbake-user-manual/bitbake-user-manual-execution:checksums (signatures)>` has changed since the last time it was + run. Changes to the input checksum of ``do_mytask`` also + indirectly cause ``do_configure`` to run. + + - The directive ``addtask mytask after do_configure`` by itself + never causes ``do_mytask`` to run. ``do_mytask`` can still be run + manually as follows: :: + + $ bitbake recipe -c mytask + + Declaring ``do_mytask`` as a dependency of some other task that is + scheduled to run also causes it to run. Regardless, the task runs after + ``do_configure``. + +Build Dependencies +------------------ + +BitBake uses the :term:`DEPENDS` variable to manage +build time dependencies. The ``[deptask]`` varflag for tasks signifies +the task of each item listed in ``DEPENDS`` that must complete before +that task can be executed. Here is an example: :: + + do_configure[deptask] = "do_populate_sysroot" + +In this example, the ``do_populate_sysroot`` task +of each item in ``DEPENDS`` must complete before ``do_configure`` can +execute. + +Runtime Dependencies +-------------------- + +BitBake uses the :term:`PACKAGES`, :term:`RDEPENDS`, and :term:`RRECOMMENDS` +variables to manage runtime dependencies. + +The ``PACKAGES`` variable lists runtime packages. Each of those packages +can have ``RDEPENDS`` and ``RRECOMMENDS`` runtime dependencies. The +``[rdeptask]`` flag for tasks is used to signify the task of each item +runtime dependency which must have completed before that task can be +executed. :: + + do_package_qa[rdeptask] = "do_packagedata" + +In the previous +example, the ``do_packagedata`` task of each item in ``RDEPENDS`` must +have completed before ``do_package_qa`` can execute. +Although ``RDEPENDS`` contains entries from the +runtime dependency namespace, BitBake knows how to map them back +to the build-time dependency namespace, in which the tasks are defined. + +Recursive Dependencies +---------------------- + +BitBake uses the ``[recrdeptask]`` flag to manage recursive task +dependencies. BitBake looks through the build-time and runtime +dependencies of the current recipe, looks through the task's inter-task +dependencies, and then adds dependencies for the listed task. Once +BitBake has accomplished this, it recursively works through the +dependencies of those tasks. Iterative passes continue until all +dependencies are discovered and added. + +The ``[recrdeptask]`` flag is most commonly used in high-level recipes +that need to wait for some task to finish "globally". For example, +``image.bbclass`` has the following: :: + + do_rootfs[recrdeptask] += "do_packagedata" + +This statement says that the ``do_packagedata`` task of +the current recipe and all recipes reachable (by way of dependencies) +from the image recipe must run before the ``do_rootfs`` task can run. + +BitBake allows a task to recursively depend on itself by +referencing itself in the task list: :: + + do_a[recrdeptask] = "do_a do_b" + +In the same way as before, this means that the ``do_a`` +and ``do_b`` tasks of the current recipe and all +recipes reachable (by way of dependencies) from the recipe +must run before the ``do_a`` task can run. In this +case BitBake will ignore the current recipe's ``do_a`` +task circular dependency on itself. + +Inter-Task Dependencies +----------------------- + +BitBake uses the ``[depends]`` flag in a more generic form to manage +inter-task dependencies. This more generic form allows for +inter-dependency checks for specific tasks rather than checks for the +data in ``DEPENDS``. Here is an example: :: + + do_patch[depends] = "quilt-native:do_populate_sysroot" + +In this example, the ``do_populate_sysroot`` task of the target ``quilt-native`` +must have completed before the ``do_patch`` task can execute. + +The ``[rdepends]`` flag works in a similar way but takes targets in the +runtime namespace instead of the build-time dependency namespace. + +Functions You Can Call From Within Python +========================================= + +BitBake provides many functions you can call from within Python +functions. This section lists the most commonly used functions, and +mentions where to find others. + +Functions for Accessing Datastore Variables +------------------------------------------- + +It is often necessary to access variables in the BitBake datastore using +Python functions. The BitBake datastore has an API that allows you this +access. Here is a list of available operations: + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - *Operation* + - *Description* + * - ``d.getVar("X", expand)`` + - Returns the value of variable "X". Using "expand=True" expands the + value. Returns "None" if the variable "X" does not exist. + * - ``d.setVar("X", "value")`` + - Sets the variable "X" to "value" + * - ``d.appendVar("X", "value")`` + - Adds "value" to the end of the variable "X". Acts like ``d.setVar("X", + "value")`` if the variable "X" does not exist. + * - ``d.prependVar("X", "value")`` + - Adds "value" to the start of the variable "X". Acts like + ``d.setVar("X","value")`` if the variable "X" does not exist. + * - ``d.delVar("X")`` + - Deletes the variable "X" from the datastore. Does nothing if the variable + "X" does not exist. + * - ``d.renameVar("X", "Y")`` + - Renames the variable "X" to "Y". Does nothing if the variable "X" does + not exist. + * - ``d.getVarFlag("X", flag, expand)`` + - Returns the value of variable "X". Using "expand=True" expands the + value. Returns "None" if either the variable "X" or the named flag does + not exist. + * - ``d.setVarFlag("X", flag, "value")`` + - Sets the named flag for variable "X" to "value". + * - ``d.appendVarFlag("X", flag, "value")`` + - Appends "value" to the named flag on the variable "X". Acts like + ``d.setVarFlag("X", flag, "value")`` if the named flag does not exist. + * - ``d.prependVarFlag("X", flag, "value")`` + - Prepends "value" to the named flag on the variable "X". Acts like + ``d.setVarFlag("X", flag, "value")`` if the named flag does not exist. + * - ``d.delVarFlag("X", flag)`` + - Deletes the named flag on the variable "X" from the datastore. + * - ``d.setVarFlags("X", flagsdict)`` + - Sets the flags specified in the ``flagsdict()`` + parameter. ``setVarFlags`` does not clear previous flags. Think of this + operation as ``addVarFlags``. + * - ``d.getVarFlags("X")`` + - Returns a ``flagsdict`` of the flags for the variable "X". Returns "None" + if the variable "X" does not exist. + * - ``d.delVarFlags("X")`` + - Deletes all the flags for the variable "X". Does nothing if the variable + "X" does not exist. + * - ``d.expand(expression)`` + - Expands variable references in the specified string + expression. References to variables that do not exist are left as is. For + example, ``d.expand("foo ${X}")`` expands to the literal string "foo + ${X}" if the variable "X" does not exist. + +Other Functions +--------------- + +You can find many other functions that can be called from Python by +looking at the source code of the ``bb`` module, which is in +``bitbake/lib/bb``. For example, ``bitbake/lib/bb/utils.py`` includes +the commonly used functions ``bb.utils.contains()`` and +``bb.utils.mkdirhier()``, which come with docstrings. + +Task Checksums and Setscene +=========================== + +BitBake uses checksums (or signatures) along with the setscene to +determine if a task needs to be run. This section describes the process. +To help understand how BitBake does this, the section assumes an +OpenEmbedded metadata-based example. + +These checksums are stored in :term:`STAMP`. You can +examine the checksums using the following BitBake command: :: + + $ bitbake-dumpsigs + +This command returns the signature data in a readable +format that allows you to examine the inputs used when the OpenEmbedded +build system generates signatures. For example, using +``bitbake-dumpsigs`` allows you to examine the ``do_compile`` task's +“sigdata” for a C application (e.g. ``bash``). Running the command also +reveals that the “CC” variable is part of the inputs that are hashed. +Any changes to this variable would invalidate the stamp and cause the +``do_compile`` task to run. + +The following list describes related variables: + +- :term:`BB_HASHCHECK_FUNCTION`: + Specifies the name of the function to call during the "setscene" part + of the task's execution in order to validate the list of task hashes. + +- :term:`BB_SETSCENE_DEPVALID`: + Specifies a function BitBake calls that determines whether BitBake + requires a setscene dependency to be met. + +- :term:`BB_SETSCENE_VERIFY_FUNCTION2`: + Specifies a function to call that verifies the list of planned task + execution before the main task execution happens. + +- :term:`BB_STAMP_POLICY`: Defines the mode + for comparing timestamps of stamp files. + +- :term:`BB_STAMP_WHITELIST`: Lists stamp + files that are looked at when the stamp policy is "whitelist". + +- :term:`BB_TASKHASH`: Within an executing task, + this variable holds the hash of the task as returned by the currently + enabled signature generator. + +- :term:`STAMP`: The base path to create stamp files. + +- :term:`STAMPCLEAN`: Again, the base path to + create stamp files but can use wildcards for matching a range of + files for clean operations. + +Wildcard Support in Variables +============================= + +Support for wildcard use in variables varies depending on the context in +which it is used. For example, some variables and file names allow +limited use of wildcards through the "``%``" and "``*``" characters. +Other variables or names support Python's +`glob <https://docs.python.org/3/library/glob.html>`_ syntax, +`fnmatch <https://docs.python.org/3/library/fnmatch.html#module-fnmatch>`_ +syntax, or +`Regular Expression (re) <https://docs.python.org/3/library/re.html>`_ +syntax. + +For variables that have wildcard suport, the documentation describes +which form of wildcard, its use, and its limitations. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml index 95a8b95b1..0ca532161 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml @@ -432,7 +432,7 @@ The variable <filename>FOO</filename> becomes " 789 123456 " and <filename>FOO2</filename> becomes - " jkl abcdef ". + " abcdef ". </para> <para> @@ -2565,15 +2565,17 @@ </para> <para> - You might want to not only have BitBake look for - dependencies of those tasks, but also have BitBake look - for build-time and runtime dependencies of the dependent - tasks as well. - If that is the case, you need to reference the task name - itself in the task list: + BitBake allows a task to recursively depend on itself by + referencing itself in the task list: <literallayout class='monospaced'> do_a[recrdeptask] = "do_a do_b" </literallayout> + In the same way as before, this means that the <filename>do_a</filename> + and <filename>do_b</filename> tasks of the current recipe and all + recipes reachable (by way of dependencies) from the recipe + must run before the <filename>do_a</filename> task can run. In this + case BitBake will ignore the current recipe's <filename>do_a</filename> + task circular dependency on itself. </para> </section> diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst new file mode 100644 index 000000000..fb1b353c8 --- /dev/null +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -0,0 +1,1372 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +================== +Variables Glossary +================== + +| + +This chapter lists common variables used by BitBake and gives an +overview of their function and contents. + +.. note:: + + Following are some points regarding the variables listed in this + glossary: + + - The variables listed in this glossary are specific to BitBake. + Consequently, the descriptions are limited to that context. + + - Also, variables exist in other systems that use BitBake (e.g. The + Yocto Project and OpenEmbedded) that have names identical to those + found in this glossary. For such cases, the variables in those + systems extend the functionality of the variable as it is + described here in this glossary. + + - Finally, there are variables mentioned in this glossary that do + not appear in the BitBake glossary. These other variables are + variables used in systems that use BitBake. + +.. glossary:: + + ASSUME_PROVIDED + Lists recipe names (:term:`PN` values) BitBake does not + attempt to build. Instead, BitBake assumes these recipes have already + been built. + + In OpenEmbedded-Core, ``ASSUME_PROVIDED`` mostly specifies native + tools that should not be built. An example is ``git-native``, which + when specified allows for the Git binary from the host to be used + rather than building ``git-native``. + + B + The directory in which BitBake executes functions during a recipe's + build process. + + BB_ALLOWED_NETWORKS + Specifies a space-delimited list of hosts that the fetcher is allowed + to use to obtain the required source code. Following are + considerations surrounding this variable: + + - This host list is only used if + :term:`BB_NO_NETWORK` is either not set or + set to "0". + + - Limited support for the "``*``" wildcard character for matching + against the beginning of host names exists. For example, the + following setting matches ``git.gnu.org``, ``ftp.gnu.org``, and + ``foo.git.gnu.org``. :: + + BB_ALLOWED_NETWORKS = "\*.gnu.org" + + .. important:: + + The use of the "``*``" character only works at the beginning of + a host name and it must be isolated from the remainder of the + host name. You cannot use the wildcard character in any other + location of the name or combined with the front part of the + name. + + For example, ``*.foo.bar`` is supported, while ``*aa.foo.bar`` + is not. + + - Mirrors not in the host list are skipped and logged in debug. + + - Attempts to access networks not in the host list cause a failure. + + Using ``BB_ALLOWED_NETWORKS`` in conjunction with + :term:`PREMIRRORS` is very useful. Adding the + host you want to use to ``PREMIRRORS`` results in the source code + being fetched from an allowed location and avoids raising an error + when a host that is not allowed is in a + :term:`SRC_URI` statement. This is because the + fetcher does not attempt to use the host listed in ``SRC_URI`` after + a successful fetch from the ``PREMIRRORS`` occurs. + + BB_CONSOLELOG + Specifies the path to a log file into which BitBake's user interface + writes output during the build. + + BB_CURRENTTASK + Contains the name of the currently running task. The name does not + include the ``do_`` prefix. + + BB_DANGLINGAPPENDS_WARNONLY + Defines how BitBake handles situations where an append file + (``.bbappend``) has no corresponding recipe file (``.bb``). This + condition often occurs when layers get out of sync (e.g. ``oe-core`` + bumps a recipe version and the old recipe no longer exists and the + other layer has not been updated to the new version of the recipe + yet). + + The default fatal behavior is safest because it is the sane reaction + given something is out of sync. It is important to realize when your + changes are no longer being applied. + + BB_DEFAULT_TASK + The default task to use when none is specified (e.g. with the ``-c`` + command line option). The task name specified should not include the + ``do_`` prefix. + + BB_DISKMON_DIRS + Monitors disk space and available inodes during the build and allows + you to control the build based on these parameters. + + Disk space monitoring is disabled by default. When setting this + variable, use the following form: :: + + BB_DISKMON_DIRS = "<action>,<dir>,<threshold> [...]" + + where: + + <action> is: + ABORT: Immediately abort the build when + a threshold is broken. + STOPTASKS: Stop the build after the currently + executing tasks have finished when + a threshold is broken. + WARN: Issue a warning but continue the + build when a threshold is broken. + Subsequent warnings are issued as + defined by the + BB_DISKMON_WARNINTERVAL variable, + which must be defined. + + <dir> is: + Any directory you choose. You can specify one or + more directories to monitor by separating the + groupings with a space. If two directories are + on the same device, only the first directory + is monitored. + + <threshold> is: + Either the minimum available disk space, + the minimum number of free inodes, or + both. You must specify at least one. To + omit one or the other, simply omit the value. + Specify the threshold using G, M, K for Gbytes, + Mbytes, and Kbytes, respectively. If you do + not specify G, M, or K, Kbytes is assumed by + default. Do not use GB, MB, or KB. + + Here are some examples: :: + + BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K" + BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G" + BB_DISKMON_DIRS = "ABORT,${TMPDIR},,100K" + + The first example works only if you also set the + :term:`BB_DISKMON_WARNINTERVAL` + variable. This example causes the build system to immediately abort + when either the disk space in ``${TMPDIR}`` drops below 1 Gbyte or + the available free inodes drops below 100 Kbytes. Because two + directories are provided with the variable, the build system also + issues a warning when the disk space in the ``${SSTATE_DIR}`` + directory drops below 1 Gbyte or the number of free inodes drops + below 100 Kbytes. Subsequent warnings are issued during intervals as + defined by the ``BB_DISKMON_WARNINTERVAL`` variable. + + The second example stops the build after all currently executing + tasks complete when the minimum disk space in the ``${TMPDIR}`` + directory drops below 1 Gbyte. No disk monitoring occurs for the free + inodes in this case. + + The final example immediately aborts the build when the number of + free inodes in the ``${TMPDIR}`` directory drops below 100 Kbytes. No + disk space monitoring for the directory itself occurs in this case. + + BB_DISKMON_WARNINTERVAL + Defines the disk space and free inode warning intervals. + + If you are going to use the ``BB_DISKMON_WARNINTERVAL`` variable, you + must also use the :term:`BB_DISKMON_DIRS` + variable and define its action as "WARN". During the build, + subsequent warnings are issued each time disk space or number of free + inodes further reduces by the respective interval. + + If you do not provide a ``BB_DISKMON_WARNINTERVAL`` variable and you + do use ``BB_DISKMON_DIRS`` with the "WARN" action, the disk + monitoring interval defaults to the following: + BB_DISKMON_WARNINTERVAL = "50M,5K" + + When specifying the variable in your configuration file, use the + following form: :: + + BB_DISKMON_WARNINTERVAL = "<disk_space_interval>,<disk_inode_interval>" + + where: + + <disk_space_interval> is: + An interval of memory expressed in either + G, M, or K for Gbytes, Mbytes, or Kbytes, + respectively. You cannot use GB, MB, or KB. + + <disk_inode_interval> is: + An interval of free inodes expressed in either + G, M, or K for Gbytes, Mbytes, or Kbytes, + respectively. You cannot use GB, MB, or KB. + + Here is an example: :: + + BB_DISKMON_DIRS = "WARN,${SSTATE_DIR},1G,100K" + BB_DISKMON_WARNINTERVAL = "50M,5K" + + These variables cause BitBake to + issue subsequent warnings each time the available disk space further + reduces by 50 Mbytes or the number of free inodes further reduces by + 5 Kbytes in the ``${SSTATE_DIR}`` directory. Subsequent warnings + based on the interval occur each time a respective interval is + reached beyond the initial warning (i.e. 1 Gbytes and 100 Kbytes). + + BB_ENV_WHITELIST + Specifies the internal whitelist of variables to allow through from + the external environment into BitBake's datastore. If the value of + this variable is not specified (which is the default), the following + list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`, + :term:`BB_ENV_WHITELIST`, and :term:`BB_ENV_EXTRAWHITE`. + + .. note:: + + You must set this variable in the external environment in order + for it to work. + + BB_ENV_EXTRAWHITE + Specifies an additional set of variables to allow through (whitelist) + from the external environment into BitBake's datastore. This list of + variables are on top of the internal list set in + :term:`BB_ENV_WHITELIST`. + + .. note:: + + You must set this variable in the external environment in order + for it to work. + + BB_FETCH_PREMIRRORONLY + When set to "1", causes BitBake's fetcher module to only search + :term:`PREMIRRORS` for files. BitBake will not + search the main :term:`SRC_URI` or + :term:`MIRRORS`. + + BB_FILENAME + Contains the filename of the recipe that owns the currently running + task. For example, if the ``do_fetch`` task that resides in the + ``my-recipe.bb`` is executing, the ``BB_FILENAME`` variable contains + "/foo/path/my-recipe.bb". + + BBFILES_DYNAMIC + Activates content depending on presence of identified layers. You + identify the layers by the collections that the layers define. + + Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose + corresponding ``.bb`` file is in a layer that attempts to modify other + layers through ``.bbappend`` but does not want to introduce a hard + dependency on those other layers. + + Additionally you can prefix the rule with "!" to add ``.bbappend`` and + ``.bb`` files in case a layer is not present. Use this avoid hard + dependency on those other layers. + + Use the following form for ``BBFILES_DYNAMIC``: :: + + collection_name:filename_pattern + + The following example identifies two collection names and two filename + patterns: :: + + BBFILES_DYNAMIC += "\ + clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \ + core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \ + " + + When the collection name is prefixed with "!" it will add the file pattern in case + the layer is absent: :: + + BBFILES_DYNAMIC += "\ + !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \ + " + + This next example shows an error message that occurs because invalid + entries are found, which cause parsing to abort: :: + + ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not: + /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend + /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend + + BB_GENERATE_MIRROR_TARBALLS + Causes tarballs of the Git repositories, including the Git metadata, + to be placed in the :term:`DL_DIR` directory. Anyone + wishing to create a source mirror would want to enable this variable. + + For performance reasons, creating and placing tarballs of the Git + repositories is not the default action by BitBake. :: + + BB_GENERATE_MIRROR_TARBALLS = "1" + + BB_HASHCONFIG_WHITELIST + Lists variables that are excluded from base configuration checksum, + which is used to determine if the cache can be reused. + + One of the ways BitBake determines whether to re-parse the main + metadata is through checksums of the variables in the datastore of + the base configuration data. There are variables that you typically + want to exclude when checking whether or not to re-parse and thus + rebuild the cache. As an example, you would usually exclude ``TIME`` + and ``DATE`` because these variables are always changing. If you did + not exclude them, BitBake would never reuse the cache. + + BB_HASHBASE_WHITELIST + Lists variables that are excluded from checksum and dependency data. + Variables that are excluded can therefore change without affecting + the checksum mechanism. A common example would be the variable for + the path of the build. BitBake's output should not (and usually does + not) depend on the directory in which it was built. + + BB_HASHCHECK_FUNCTION + Specifies the name of the function to call during the "setscene" part + of the task's execution in order to validate the list of task hashes. + The function returns the list of setscene tasks that should be + executed. + + At this point in the execution of the code, the objective is to + quickly verify if a given setscene function is likely to work or not. + It's easier to check the list of setscene functions in one pass than + to call many individual tasks. The returned list need not be + completely accurate. A given setscene task can still later fail. + However, the more accurate the data returned, the more efficient the + build will be. + + BB_INVALIDCONF + Used in combination with the ``ConfigParsed`` event to trigger + re-parsing the base metadata (i.e. all the recipes). The + ``ConfigParsed`` event can set the variable to trigger the re-parse. + You must be careful to avoid recursive loops with this functionality. + + BB_LOGCONFIG + Specifies the name of a config file that contains the user logging + configuration. See + :ref:`bitbake-user-manual/bitbake-user-manual-execution:logging` + for additional information + + BB_LOGFMT + Specifies the name of the log files saved into + ``${``\ :term:`T`\ ``}``. By default, the ``BB_LOGFMT`` + variable is undefined and the log file names get created using the + following form: :: + + log.{task}.{pid} + + If you want to force log files to take a specific name, you can set this + variable in a configuration file. + + BB_NICE_LEVEL + Allows BitBake to run at a specific priority (i.e. nice level). + System permissions usually mean that BitBake can reduce its priority + but not raise it again. See :term:`BB_TASK_NICE_LEVEL` for + additional information. + + BB_NO_NETWORK + Disables network access in the BitBake fetcher modules. With this + access disabled, any command that attempts to access the network + becomes an error. + + Disabling network access is useful for testing source mirrors, + running builds when not connected to the Internet, and when operating + in certain kinds of firewall environments. + + BB_NUMBER_THREADS + The maximum number of tasks BitBake should run in parallel at any one + time. If your host development system supports multiple cores, a good + rule of thumb is to set this variable to twice the number of cores. + + BB_NUMBER_PARSE_THREADS + Sets the number of threads BitBake uses when parsing. By default, the + number of threads is equal to the number of cores on the system. + + BB_ORIGENV + Contains a copy of the original external environment in which BitBake + was run. The copy is taken before any whitelisted variable values are + filtered into BitBake's datastore. + + .. note:: + + The contents of this variable is a datastore object that can be + queried using the normal datastore operations. + + BB_PRESERVE_ENV + Disables whitelisting and instead allows all variables through from + the external environment into BitBake's datastore. + + .. note:: + + You must set this variable in the external environment in order + for it to work. + + BB_RUNFMT + Specifies the name of the executable script files (i.e. run files) + saved into ``${``\ :term:`T`\ ``}``. By default, the + ``BB_RUNFMT`` variable is undefined and the run file names get + created using the following form: :: + + run.{task}.{pid} + + If you want to force run files to take a specific name, you can set this + variable in a configuration file. + + BB_RUNTASK + Contains the name of the currently executing task. The value includes + the "do\_" prefix. For example, if the currently executing task is + ``do_config``, the value is "do_config". + + BB_SCHEDULER + Selects the name of the scheduler to use for the scheduling of + BitBake tasks. Three options exist: + + - *basic* - The basic framework from which everything derives. Using + this option causes tasks to be ordered numerically as they are + parsed. + + - *speed* - Executes tasks first that have more tasks depending on + them. The "speed" option is the default. + + - *completion* - Causes the scheduler to try to complete a given + recipe once its build has started. + + BB_SCHEDULERS + Defines custom schedulers to import. Custom schedulers need to be + derived from the ``RunQueueScheduler`` class. + + For information how to select a scheduler, see the + :term:`BB_SCHEDULER` variable. + + BB_SETSCENE_DEPVALID + Specifies a function BitBake calls that determines whether BitBake + requires a setscene dependency to be met. + + When running a setscene task, BitBake needs to know which + dependencies of that setscene task also need to be run. Whether + dependencies also need to be run is highly dependent on the metadata. + The function specified by this variable returns a "True" or "False" + depending on whether the dependency needs to be met. + + BB_SETSCENE_VERIFY_FUNCTION2 + Specifies a function to call that verifies the list of planned task + execution before the main task execution happens. The function is + called once BitBake has a list of setscene tasks that have run and + either succeeded or failed. + + The function allows for a task list check to see if they make sense. + Even if BitBake was planning to skip a task, the returned value of + the function can force BitBake to run the task, which is necessary + under certain metadata defined circumstances. + + BB_SIGNATURE_EXCLUDE_FLAGS + Lists variable flags (varflags) that can be safely excluded from + checksum and dependency data for keys in the datastore. When + generating checksum or dependency data for keys in the datastore, the + flags set against that key are normally included in the checksum. + + For more information on varflags, see the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" + section. + + BB_SIGNATURE_HANDLER + Defines the name of the signature handler BitBake uses. The signature + handler defines the way stamp files are created and handled, if and + how the signature is incorporated into the stamps, and how the + signature itself is generated. + + A new signature handler can be added by injecting a class derived + from the ``SignatureGenerator`` class into the global namespace. + + BB_SRCREV_POLICY + Defines the behavior of the fetcher when it interacts with source + control systems and dynamic source revisions. The + ``BB_SRCREV_POLICY`` variable is useful when working without a + network. + + The variable can be set using one of two policies: + + - *cache* - Retains the value the system obtained previously rather + than querying the source control system each time. + + - *clear* - Queries the source controls system every time. With this + policy, there is no cache. The "clear" policy is the default. + + BB_STAMP_POLICY + Defines the mode used for how timestamps of stamp files are compared. + You can set the variable to one of the following modes: + + - *perfile* - Timestamp comparisons are only made between timestamps + of a specific recipe. This is the default mode. + + - *full* - Timestamp comparisons are made for all dependencies. + + - *whitelist* - Identical to "full" mode except timestamp + comparisons are made for recipes listed in the + :term:`BB_STAMP_WHITELIST` variable. + + .. note:: + + Stamp policies are largely obsolete with the introduction of + setscene tasks. + + BB_STAMP_WHITELIST + Lists files whose stamp file timestamps are compared when the stamp + policy mode is set to "whitelist". For information on stamp policies, + see the :term:`BB_STAMP_POLICY` variable. + + BB_STRICT_CHECKSUM + Sets a more strict checksum mechanism for non-local URLs. Setting + this variable to a value causes BitBake to report an error if it + encounters a non-local URL that does not have at least one checksum + specified. + + BB_TASK_IONICE_LEVEL + Allows adjustment of a task's Input/Output priority. During + Autobuilder testing, random failures can occur for tasks due to I/O + starvation. These failures occur during various QEMU runtime + timeouts. You can use the ``BB_TASK_IONICE_LEVEL`` variable to adjust + the I/O priority of these tasks. + + .. note:: + + This variable works similarly to the :term:`BB_TASK_NICE_LEVEL` + variable except with a task's I/O priorities. + + Set the variable as follows: :: + + BB_TASK_IONICE_LEVEL = "class.prio" + + For *class*, the default value is "2", which is a best effort. You can use + "1" for realtime and "3" for idle. If you want to use realtime, you + must have superuser privileges. + + For *prio*, you can use any value from "0", which is the highest + priority, to "7", which is the lowest. The default value is "4". You + do not need any special privileges to use this range of priority + values. + + .. note:: + + In order for your I/O priority settings to take effect, you need the + Completely Fair Queuing (CFQ) Scheduler selected for the backing block + device. To select the scheduler, use the following command form where + device is the device (e.g. sda, sdb, and so forth): :: + + $ sudo sh -c “echo cfq > /sys/block/device/queu/scheduler + + BB_TASK_NICE_LEVEL + Allows specific tasks to change their priority (i.e. nice level). + + You can use this variable in combination with task overrides to raise + or lower priorities of specific tasks. For example, on the `Yocto + Project <http://www.yoctoproject.org>`__ autobuilder, QEMU emulation + in images is given a higher priority as compared to build tasks to + ensure that images do not suffer timeouts on loaded systems. + + BB_TASKHASH + Within an executing task, this variable holds the hash of the task as + returned by the currently enabled signature generator. + + BB_VERBOSE_LOGS + Controls how verbose BitBake is during builds. If set, shell scripts + echo commands and shell script output appears on standard out + (stdout). + + BB_WORKERCONTEXT + Specifies if the current context is executing a task. BitBake sets + this variable to "1" when a task is being executed. The value is not + set when the task is in server context during parsing or event + handling. + + BBCLASSEXTEND + Allows you to extend a recipe so that it builds variants of the + software. Some examples of these variants for recipes from the + OpenEmbedded-Core metadata are "natives" such as ``quilt-native``, + which is a copy of Quilt built to run on the build system; "crosses" + such as ``gcc-cross``, which is a compiler built to run on the build + machine but produces binaries that run on the target ``MACHINE``; + "nativesdk", which targets the SDK machine instead of ``MACHINE``; + and "mulitlibs" in the form "``multilib:``\ multilib_name". + + To build a different variant of the recipe with a minimal amount of + code, it usually is as simple as adding the variable to your recipe. + Here are two examples. The "native" variants are from the + OpenEmbedded-Core metadata: :: + + BBCLASSEXTEND =+ "native nativesdk" + BBCLASSEXTEND =+ "multilib:multilib_name" + + .. note:: + + Internally, the ``BBCLASSEXTEND`` mechanism generates recipe + variants by rewriting variable values and applying overrides such + as ``_class-native``. For example, to generate a native version of + a recipe, a :term:`DEPENDS` on "foo" is + rewritten to a ``DEPENDS`` on "foo-native". + + Even when using ``BBCLASSEXTEND``, the recipe is only parsed once. + Parsing once adds some limitations. For example, it is not + possible to include a different file depending on the variant, + since ``include`` statements are processed when the recipe is + parsed. + + BBDEBUG + Sets the BitBake debug output level to a specific value as + incremented by the ``-D`` command line option. + + .. note:: + + You must set this variable in the external environment in order + for it to work. + + BBFILE_COLLECTIONS + Lists the names of configured layers. These names are used to find + the other ``BBFILE_*`` variables. Typically, each layer appends its + name to this variable in its ``conf/layer.conf`` file. + + BBFILE_PATTERN + Variable that expands to match files from + :term:`BBFILES` in a particular layer. This + variable is used in the ``conf/layer.conf`` file and must be suffixed + with the name of the specific layer (e.g. + ``BBFILE_PATTERN_emenlow``). + + BBFILE_PRIORITY + Assigns the priority for recipe files in each layer. + + This variable is useful in situations where the same recipe appears + in more than one layer. Setting this variable allows you to + prioritize a layer against other layers that contain the same recipe + - effectively letting you control the precedence for the multiple + layers. The precedence established through this variable stands + regardless of a recipe's version (:term:`PV` variable). + For example, a layer that has a recipe with a higher ``PV`` value but + for which the ``BBFILE_PRIORITY`` is set to have a lower precedence + still has a lower precedence. + + A larger value for the ``BBFILE_PRIORITY`` variable results in a + higher precedence. For example, the value 6 has a higher precedence + than the value 5. If not specified, the ``BBFILE_PRIORITY`` variable + is set based on layer dependencies (see the ``LAYERDEPENDS`` variable + for more information. The default priority, if unspecified for a + layer with no dependencies, is the lowest defined priority + 1 (or 1 + if no priorities are defined). + + .. tip:: + + You can use the command bitbake-layers show-layers to list all + configured layers along with their priorities. + + BBFILES + A space-separated list of recipe files BitBake uses to build + software. + + When specifying recipe files, you can pattern match using Python's + `glob <https://docs.python.org/3/library/glob.html>`_ syntax. + For details on the syntax, see the documentation by following the + previous link. + + BBINCLUDED + Contains a space-separated list of all of all files that BitBake's + parser included during parsing of the current file. + + BBINCLUDELOGS + If set to a value, enables printing the task log when reporting a + failed task. + + BBINCLUDELOGS_LINES + If :term:`BBINCLUDELOGS` is set, specifies + the maximum number of lines from the task log file to print when + reporting a failed task. If you do not set ``BBINCLUDELOGS_LINES``, + the entire log is printed. + + BBLAYERS + Lists the layers to enable during the build. This variable is defined + in the ``bblayers.conf`` configuration file in the build directory. + Here is an example: :: + + BBLAYERS = " \ + /home/scottrif/poky/meta \ + /home/scottrif/poky/meta-yocto \ + /home/scottrif/poky/meta-yocto-bsp \ + /home/scottrif/poky/meta-mykernel \ + " + + This example enables four layers, one of which is a custom, user-defined + layer named ``meta-mykernel``. + + BBLAYERS_FETCH_DIR + Sets the base location where layers are stored. This setting is used + in conjunction with ``bitbake-layers layerindex-fetch`` and tells + ``bitbake-layers`` where to place the fetched layers. + + BBMASK + Prevents BitBake from processing recipes and recipe append files. + + You can use the ``BBMASK`` variable to "hide" these ``.bb`` and + ``.bbappend`` files. BitBake ignores any recipe or recipe append + files that match any of the expressions. It is as if BitBake does not + see them at all. Consequently, matching files are not parsed or + otherwise used by BitBake. + + The values you provide are passed to Python's regular expression + compiler. Consequently, the syntax follows Python's Regular + Expression (re) syntax. The expressions are compared against the full + paths to the files. For complete syntax information, see Python's + documentation at http://docs.python.org/3/library/re.html. + + The following example uses a complete regular expression to tell + BitBake to ignore all recipe and recipe append files in the + ``meta-ti/recipes-misc/`` directory: :: + + BBMASK = "meta-ti/recipes-misc/" + + If you want to mask out multiple directories or recipes, you can + specify multiple regular expression fragments. This next example + masks out multiple directories and individual recipes: :: + + BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/" + BBMASK += "/meta-oe/recipes-support/" + BBMASK += "/meta-foo/.*/openldap" + BBMASK += "opencv.*\.bbappend" + BBMASK += "lzma" + + .. note:: + + When specifying a directory name, use the trailing slash character + to ensure you match just that directory name. + + BBMULTICONFIG + Enables BitBake to perform multiple configuration builds and lists + each separate configuration (multiconfig). You can use this variable + to cause BitBake to build multiple targets where each target has a + separate configuration. Define ``BBMULTICONFIG`` in your + ``conf/local.conf`` configuration file. + + As an example, the following line specifies three multiconfigs, each + having a separate configuration file: :: + + BBMULTIFONFIG = "configA configB configC" + + Each configuration file you use must reside in the + build directory within a directory named ``conf/multiconfig`` (e.g. + build_directory\ ``/conf/multiconfig/configA.conf``). + + For information on how to use ``BBMULTICONFIG`` in an environment + that supports building targets with multiple configurations, see the + ":ref:`bitbake-user-manual/bitbake-user-manual-intro:executing a multiple configuration build`" + section. + + BBPATH + Used by BitBake to locate class (``.bbclass``) and configuration + (``.conf``) files. This variable is analogous to the ``PATH`` + variable. + + If you run BitBake from a directory outside of the build directory, + you must be sure to set ``BBPATH`` to point to the build directory. + Set the variable as you would any environment variable and then run + BitBake: :: + + $ BBPATH="build_directory" + $ export BBPATH + $ bitbake target + + BBSERVER + Points to the server that runs memory-resident BitBake. The variable + is only used when you employ memory-resident BitBake. + + BBTARGETS + Allows you to use a configuration file to add to the list of + command-line target recipes you want to build. + + BBVERSIONS + Allows a single recipe to build multiple versions of a project from a + single recipe file. You also able to specify conditional metadata + using the :term:`OVERRIDES` mechanism for a + single version or for an optionally named range of versions. + + For more information on ``BBVERSIONS``, see the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variants - class extension mechanism`" + section. + + BITBAKE_UI + Used to specify the UI module to use when running BitBake. Using this + variable is equivalent to using the ``-u`` command-line option. + + .. note:: + + You must set this variable in the external environment in order + for it to work. + + BUILDNAME + A name assigned to the build. The name defaults to a datetime stamp + of when the build was started but can be defined by the metadata. + + BZRDIR + The directory in which files checked out of a Bazaar system are + stored. + + CACHE + Specifies the directory BitBake uses to store a cache of the metadata + so it does not need to be parsed every time BitBake is started. + + CVSDIR + The directory in which files checked out under the CVS system are + stored. + + DEFAULT_PREFERENCE + Specifies a weak bias for recipe selection priority. + + The most common usage of this is variable is to set it to "-1" within + a recipe for a development version of a piece of software. Using the + variable in this way causes the stable version of the recipe to build + by default in the absence of ``PREFERRED_VERSION`` being used to + build the development version. + + .. note:: + + The bias provided by DEFAULT_PREFERENCE is weak and is overridden by + :term:`BBFILE_PRIORITY` if that variable is different between two + layers that contain different versions of the same recipe. + + DEPENDS + Lists a recipe's build-time dependencies (i.e. other recipe files). + + Consider this simple example for two recipes named "a" and "b" that + produce similarly named packages. In this example, the ``DEPENDS`` + statement appears in the "a" recipe: :: + + DEPENDS = "b" + + Here, the dependency is such that the ``do_configure`` task for recipe "a" + depends on the ``do_populate_sysroot`` task of recipe "b". This means + anything that recipe "b" puts into sysroot is available when recipe "a" is + configuring itself. + + For information on runtime dependencies, see the :term:`RDEPENDS` + variable. + + DESCRIPTION + A long description for the recipe. + + DL_DIR + The central download directory used by the build process to store + downloads. By default, ``DL_DIR`` gets files suitable for mirroring for + everything except Git repositories. If you want tarballs of Git + repositories, use the :term:`BB_GENERATE_MIRROR_TARBALLS` variable. + + EXCLUDE_FROM_WORLD + Directs BitBake to exclude a recipe from world builds (i.e. + ``bitbake world``). During world builds, BitBake locates, parses and + builds all recipes found in every layer exposed in the + ``bblayers.conf`` configuration file. + + To exclude a recipe from a world build using this variable, set the + variable to "1" in the recipe. + + .. note:: + + Recipes added to ``EXCLUDE_FROM_WORLD`` may still be built during a world + build in order to satisfy dependencies of other recipes. Adding a + recipe to ``EXCLUDE_FROM_WORLD`` only ensures that the recipe is not + explicitly added to the list of build targets in a world build. + + FAKEROOT + Contains the command to use when running a shell script in a fakeroot + environment. The ``FAKEROOT`` variable is obsolete and has been + replaced by the other ``FAKEROOT*`` variables. See these entries in + the glossary for more information. + + FAKEROOTBASEENV + Lists environment variables to set when executing the command defined + by :term:`FAKEROOTCMD` that starts the + bitbake-worker process in the fakeroot environment. + + FAKEROOTCMD + Contains the command that starts the bitbake-worker process in the + fakeroot environment. + + FAKEROOTDIRS + Lists directories to create before running a task in the fakeroot + environment. + + FAKEROOTENV + Lists environment variables to set when running a task in the + fakeroot environment. For additional information on environment + variables and the fakeroot environment, see the + :term:`FAKEROOTBASEENV` variable. + + FAKEROOTNOENV + Lists environment variables to set when running a task that is not in + the fakeroot environment. For additional information on environment + variables and the fakeroot environment, see the + :term:`FAKEROOTENV` variable. + + FETCHCMD + Defines the command the BitBake fetcher module executes when running + fetch operations. You need to use an override suffix when you use the + variable (e.g. ``FETCHCMD_git`` or ``FETCHCMD_svn``). + + FILE + Points at the current file. BitBake sets this variable during the + parsing process to identify the file being parsed. BitBake also sets + this variable when a recipe is being executed to identify the recipe + file. + + FILESPATH + Specifies directories BitBake uses when searching for patches and + files. The "local" fetcher module uses these directories when + handling ``file://`` URLs. The variable behaves like a shell ``PATH`` + environment variable. The value is a colon-separated list of + directories that are searched left-to-right in order. + + GITDIR + The directory in which a local copy of a Git repository is stored + when it is cloned. + + HGDIR + The directory in which files checked out of a Mercurial system are + stored. + + HOMEPAGE + Website where more information about the software the recipe is + building can be found. + + INHERIT + Causes the named class or classes to be inherited globally. Anonymous + functions in the class or classes are not executed for the base + configuration and in each individual recipe. The OpenEmbedded build + system ignores changes to ``INHERIT`` in individual recipes. + + For more information on ``INHERIT``, see the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` configuration directive`" + section. + + LAYERDEPENDS + Lists the layers, separated by spaces, upon which this recipe + depends. Optionally, you can specify a specific layer version for a + dependency by adding it to the end of the layer name with a colon, + (e.g. "anotherlayer:3" to be compared against + :term:`LAYERVERSION`\ ``_anotherlayer`` in + this case). BitBake produces an error if any dependency is missing or + the version numbers do not match exactly (if specified). + + You use this variable in the ``conf/layer.conf`` file. You must also + use the specific layer name as a suffix to the variable (e.g. + ``LAYERDEPENDS_mylayer``). + + LAYERDIR + When used inside the ``layer.conf`` configuration file, this variable + provides the path of the current layer. This variable is not + available outside of ``layer.conf`` and references are expanded + immediately when parsing of the file completes. + + LAYERDIR_RE + When used inside the ``layer.conf`` configuration file, this variable + provides the path of the current layer, escaped for use in a regular + expression (:term:`BBFILE_PATTERN`). This + variable is not available outside of ``layer.conf`` and references + are expanded immediately when parsing of the file completes. + + LAYERVERSION + Optionally specifies the version of a layer as a single number. You + can use this variable within + :term:`LAYERDEPENDS` for another layer in + order to depend on a specific version of the layer. + + You use this variable in the ``conf/layer.conf`` file. You must also + use the specific layer name as a suffix to the variable (e.g. + ``LAYERDEPENDS_mylayer``). + + LICENSE + The list of source licenses for the recipe. + + MIRRORS + Specifies additional paths from which BitBake gets source code. When + the build system searches for source code, it first tries the local + download directory. If that location fails, the build system tries + locations defined by :term:`PREMIRRORS`, the + upstream source, and then locations specified by ``MIRRORS`` in that + order. + + MULTI_PROVIDER_WHITELIST + Allows you to suppress BitBake warnings caused when building two + separate recipes that provide the same output. + + BitBake normally issues a warning when building two different recipes + where each provides the same output. This scenario is usually + something the user does not want. However, cases do exist where it + makes sense, particularly in the ``virtual/*`` namespace. You can use + this variable to suppress BitBake's warnings. + + To use the variable, list provider names (e.g. recipe names, + ``virtual/kernel``, and so forth). + + OVERRIDES + BitBake uses ``OVERRIDES`` to control what variables are overridden + after BitBake parses recipes and configuration files. + + Following is a simple example that uses an overrides list based on + machine architectures: OVERRIDES = "arm:x86:mips:powerpc" You can + find information on how to use ``OVERRIDES`` in the + ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:conditional syntax + (overrides)`" section. + + P4DIR + The directory in which a local copy of a Perforce depot is stored + when it is fetched. + + PACKAGES + The list of packages the recipe creates. + + PACKAGES_DYNAMIC + A promise that your recipe satisfies runtime dependencies for + optional modules that are found in other recipes. + ``PACKAGES_DYNAMIC`` does not actually satisfy the dependencies, it + only states that they should be satisfied. For example, if a hard, + runtime dependency (:term:`RDEPENDS`) of another + package is satisfied during the build through the + ``PACKAGES_DYNAMIC`` variable, but a package with the module name is + never actually produced, then the other package will be broken. + + PE + The epoch of the recipe. By default, this variable is unset. The + variable is used to make upgrades possible when the versioning scheme + changes in some backwards incompatible way. + + PERSISTENT_DIR + Specifies the directory BitBake uses to store data that should be + preserved between builds. In particular, the data stored is the data + that uses BitBake's persistent data API and the data used by the PR + Server and PR Service. + + PF + Specifies the recipe or package name and includes all version and + revision numbers (i.e. ``eglibc-2.13-r20+svnr15508/`` and + ``bash-4.2-r1/``). + + PN + The recipe name. + + PR + The revision of the recipe. + + PREFERRED_PROVIDER + Determines which recipe should be given preference when multiple + recipes provide the same item. You should always suffix the variable + with the name of the provided item, and you should set it to the + :term:`PN` of the recipe to which you want to give + precedence. Some examples: :: + + PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto" + PREFERRED_PROVIDER_virtual/xserver = "xserver-xf86" + PREFERRED_PROVIDER_virtual/libgl ?= "mesa" + + PREFERRED_PROVIDERS + Determines which recipe should be given preference for cases where + multiple recipes provide the same item. Functionally, + ``PREFERRED_PROVIDERS`` is identical to + :term:`PREFERRED_PROVIDER`. However, the ``PREFERRED_PROVIDERS`` variable + lets you define preferences for multiple situations using the following + form: :: + + PREFERRED_PROVIDERS = "xxx:yyy aaa:bbb ..." + + This form is a convenient replacement for the following: :: + + PREFERRED_PROVIDER_xxx = "yyy" + PREFERRED_PROVIDER_aaa = "bbb" + + PREFERRED_VERSION + If there are multiple versions of recipes available, this variable + determines which recipe should be given preference. You must always + suffix the variable with the :term:`PN` you want to + select, and you should set :term:`PV` accordingly for + precedence. + + The ``PREFERRED_VERSION`` variable supports limited wildcard use + through the "``%``" character. You can use the character to match any + number of characters, which can be useful when specifying versions + that contain long revision numbers that potentially change. Here are + two examples: :: + + PREFERRED_VERSION_python = "2.7.3" + PREFERRED_VERSION_linux-yocto = "4.12%" + + .. important:: + + The use of the " % " character is limited in that it only works at the + end of the string. You cannot use the wildcard character in any other + location of the string. + + PREMIRRORS + Specifies additional paths from which BitBake gets source code. When + the build system searches for source code, it first tries the local + download directory. If that location fails, the build system tries + locations defined by ``PREMIRRORS``, the upstream source, and then + locations specified by :term:`MIRRORS` in that order. + + Typically, you would add a specific server for the build system to + attempt before any others by adding something like the following to + your configuration: :: + + PREMIRRORS_prepend = "\ + git://.*/.* http://www.yoctoproject.org/sources/ \n \ + ftp://.*/.* http://www.yoctoproject.org/sources/ \n \ + http://.*/.* http://www.yoctoproject.org/sources/ \n \ + https://.*/.* http://www.yoctoproject.org/sources/ \n" + + These changes cause the build system to intercept Git, FTP, HTTP, and + HTTPS requests and direct them to the ``http://`` sources mirror. You can + use ``file://`` URLs to point to local directories or network shares as + well. + + PROVIDES + A list of aliases by which a particular recipe can be known. By + default, a recipe's own ``PN`` is implicitly already in its + ``PROVIDES`` list. If a recipe uses ``PROVIDES``, the additional + aliases are synonyms for the recipe and can be useful satisfying + dependencies of other recipes during the build as specified by + ``DEPENDS``. + + Consider the following example ``PROVIDES`` statement from a recipe + file ``libav_0.8.11.bb``: :: + + PROVIDES += "libpostproc" + + The ``PROVIDES`` statement results in the "libav" recipe also being known + as "libpostproc". + + In addition to providing recipes under alternate names, the + ``PROVIDES`` mechanism is also used to implement virtual targets. A + virtual target is a name that corresponds to some particular + functionality (e.g. a Linux kernel). Recipes that provide the + functionality in question list the virtual target in ``PROVIDES``. + Recipes that depend on the functionality in question can include the + virtual target in :term:`DEPENDS` to leave the + choice of provider open. + + Conventionally, virtual targets have names on the form + "virtual/function" (e.g. "virtual/kernel"). The slash is simply part + of the name and has no syntactical significance. + + PRSERV_HOST + The network based :term:`PR` service host and port. + + Following is an example of how the ``PRSERV_HOST`` variable is set: :: + + PRSERV_HOST = "localhost:0" + + You must set the variable if you want to automatically start a local PR + service. You can set ``PRSERV_HOST`` to other values to use a remote PR + service. + + PV + The version of the recipe. + + RDEPENDS + Lists a package's runtime dependencies (i.e. other packages) that + must be installed in order for the built package to run correctly. If + a package in this list cannot be found during the build, you will get + a build error. + + Because the ``RDEPENDS`` variable applies to packages being built, + you should always use the variable in a form with an attached package + name. For example, suppose you are building a development package + that depends on the ``perl`` package. In this case, you would use the + following ``RDEPENDS`` statement: :: + + RDEPENDS_${PN}-dev += "perl" + + In the example, the development package depends on the ``perl`` package. + Thus, the ``RDEPENDS`` variable has the ``${PN}-dev`` package name as part + of the variable. + + BitBake supports specifying versioned dependencies. Although the + syntax varies depending on the packaging format, BitBake hides these + differences from you. Here is the general syntax to specify versions + with the ``RDEPENDS`` variable: :: + + RDEPENDS_${PN} = "package (operator version)" + + For ``operator``, you can specify the following: :: + + = + < + > + <= + >= + + For example, the following sets up a dependency on version 1.2 or + greater of the package ``foo``: :: + + RDEPENDS_${PN} = "foo (>= 1.2)" + + For information on build-time dependencies, see the :term:`DEPENDS` + variable. + + REPODIR + The directory in which a local copy of a ``google-repo`` directory is + stored when it is synced. + + RPROVIDES + A list of package name aliases that a package also provides. These + aliases are useful for satisfying runtime dependencies of other + packages both during the build and on the target (as specified by + ``RDEPENDS``). + + As with all package-controlling variables, you must always use the + variable in conjunction with a package name override. Here is an + example: :: + + RPROVIDES_${PN} = "widget-abi-2" + + RRECOMMENDS + A list of packages that extends the usability of a package being + built. The package being built does not depend on this list of + packages in order to successfully build, but needs them for the + extended usability. To specify runtime dependencies for packages, see + the ``RDEPENDS`` variable. + + BitBake supports specifying versioned recommends. Although the syntax + varies depending on the packaging format, BitBake hides these + differences from you. Here is the general syntax to specify versions + with the ``RRECOMMENDS`` variable: :: + + RRECOMMENDS_${PN} = "package (operator version)" + + For ``operator``, you can specify the following: :: + + = + < + > + <= + >= + + For example, the following sets up a recommend on version + 1.2 or greater of the package ``foo``: :: + + RRECOMMENDS_${PN} = "foo (>= 1.2)" + + SECTION + The section in which packages should be categorized. + + SRC_URI + The list of source files - local or remote. This variable tells + BitBake which bits to pull for the build and how to pull them. For + example, if the recipe or append file needs to fetch a single tarball + from the Internet, the recipe or append file uses a ``SRC_URI`` entry + that specifies that tarball. On the other hand, if the recipe or + append file needs to fetch a tarball and include a custom file, the + recipe or append file needs an ``SRC_URI`` variable that specifies + all those sources. + + The following list explains the available URI protocols: + + - ``file://`` : Fetches files, which are usually files shipped + with the metadata, from the local machine. The path is relative to + the :term:`FILESPATH` variable. + + - ``bzr://`` : Fetches files from a Bazaar revision control + repository. + + - ``git://`` : Fetches files from a Git revision control + repository. + + - ``osc://`` : Fetches files from an OSC (OpenSUSE Build service) + revision control repository. + + - ``repo://`` : Fetches files from a repo (Git) repository. + + - ``http://`` : Fetches files from the Internet using HTTP. + + - ``https://`` : Fetches files from the Internet using HTTPS. + + - ``ftp://`` : Fetches files from the Internet using FTP. + + - ``cvs://`` : Fetches files from a CVS revision control + repository. + + - ``hg://`` : Fetches files from a Mercurial (``hg``) revision + control repository. + + - ``p4://`` : Fetches files from a Perforce (``p4``) revision + control repository. + + - ``ssh://`` : Fetches files from a secure shell. + + - ``svn://`` : Fetches files from a Subversion (``svn``) revision + control repository. + + Here are some additional options worth mentioning: + + - ``unpack`` : Controls whether or not to unpack the file if it is + an archive. The default action is to unpack the file. + + - ``subdir`` : Places the file (or extracts its contents) into the + specified subdirectory. This option is useful for unusual tarballs + or other archives that do not have their files already in a + subdirectory within the archive. + + - ``name`` : Specifies a name to be used for association with + ``SRC_URI`` checksums when you have more than one file specified + in ``SRC_URI``. + + - ``downloadfilename`` : Specifies the filename used when storing + the downloaded file. + + SRCDATE + The date of the source code used to build the package. This variable + applies only if the source was fetched from a Source Code Manager + (SCM). + + SRCREV + The revision of the source code used to build the package. This + variable applies only when using Subversion, Git, Mercurial and + Bazaar. If you want to build a fixed revision and you want to avoid + performing a query on the remote repository every time BitBake parses + your recipe, you should specify a ``SRCREV`` that is a full revision + identifier and not just a tag. + + SRCREV_FORMAT + Helps construct valid :term:`SRCREV` values when + multiple source controlled URLs are used in + :term:`SRC_URI`. + + The system needs help constructing these values under these + circumstances. Each component in the ``SRC_URI`` is assigned a name + and these are referenced in the ``SRCREV_FORMAT`` variable. Consider + an example with URLs named "machine" and "meta". In this case, + ``SRCREV_FORMAT`` could look like "machine_meta" and those names + would have the SCM versions substituted into each position. Only one + ``AUTOINC`` placeholder is added and if needed. And, this placeholder + is placed at the start of the returned string. + + STAMP + Specifies the base path used to create recipe stamp files. The path + to an actual stamp file is constructed by evaluating this string and + then appending additional information. + + STAMPCLEAN + Specifies the base path used to create recipe stamp files. Unlike the + :term:`STAMP` variable, ``STAMPCLEAN`` can contain + wildcards to match the range of files a clean operation should + remove. BitBake uses a clean operation to remove any other stamps it + should be removing when creating a new stamp. + + SUMMARY + A short summary for the recipe, which is 72 characters or less. + + SVNDIR + The directory in which files checked out of a Subversion system are + stored. + + T + Points to a directory were BitBake places temporary files, which + consist mostly of task logs and scripts, when building a particular + recipe. + + TOPDIR + Points to the build directory. BitBake automatically sets this + variable. diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml index c4bd1f258..4c29b2464 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml @@ -1124,6 +1124,67 @@ </glossdef> </glossentry> + <glossentry id='var-BBFILES_DYNAMIC'><glossterm>BBFILES_DYNAMIC</glossterm> + <info> + BBFILES_DYNAMIC[doc] = "Activates content depending on presence of identified layers." + </info> + <glossdef> + <para role="glossdeffirst"> + Activates content depending on presence of identified layers. + You identify the layers by the collections that the layers + define. + </para> + + <para> + Use the <filename>BBFILES_DYNAMIC</filename> variable to + avoid <filename>.bbappend</filename> files whose + corresponding <filename>.bb</filename> file is in a layer + that attempts to modify other layers through + <filename>.bbappend</filename> but does not want to + introduce a hard dependency on those other layers. + </para> + + <para> + Additionally you can prefix the rule with "!" to add + <filename>.bbappend</filename> and <filename>.bb</filename> files + in case a layer is not present. + Use this avoid hard dependency on those other layers. + </para> + + <para> + Use the following form for + <filename>BBFILES_DYNAMIC</filename>: + <literallayout class='monospaced'> + <replaceable>collection_name</replaceable>:<replaceable>filename_pattern</replaceable> + </literallayout> + The following example identifies two collection names and + two filename patterns: + <literallayout class='monospaced'> + BBFILES_DYNAMIC += "\ + clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \ + core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \ + " + </literallayout> + When the collection name is prefixed with "!" it will add the file pattern in case + the layer is absent: + <literallayout class='monospaced'> + BBFILES_DYNAMIC += "\ + !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \ + " + </literallayout> + + This next example shows an error message that occurs + because invalid entries are found, which cause parsing to + abort: + <literallayout class='monospaced'> + ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not: + /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend + /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend + </literallayout> + </para> + </glossdef> + </glossentry> + <glossentry id='var-bb-BBINCLUDED'><glossterm>BBINCLUDED</glossterm> <glossdef> <para> diff --git a/poky/bitbake/doc/conf.py b/poky/bitbake/doc/conf.py new file mode 100644 index 000000000..197f2757a --- /dev/null +++ b/poky/bitbake/doc/conf.py @@ -0,0 +1,94 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import datetime + +current_version = "dev" + +# String used in sidebar +version = 'Version: ' + current_version +if current_version == 'dev': + version = 'Version: Current Development' +# Version seen in documentation_options.js and hence in js switchers code +release = current_version + +# -- Project information ----------------------------------------------------- + +project = 'Bitbake' +copyright = '2004-%s, Richard Purdie, Chris Larson, and Phil Blundell' \ + % datetime.datetime.now().year +author = 'Richard Purdie, Chris Larson, and Phil Blundell' + +# external links and substitutions +extlinks = { + 'yocto_docs': ('https://docs.yoctoproject.org%s', None), + 'oe_lists': ('https://lists.openembedded.org%s', None), +} + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autosectionlabel', + 'sphinx.ext.extlinks', +] +autosectionlabel_prefix_document = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# master document name. The default changed from contents to index. so better +# set it ourselves. +master_doc = 'index' + +# create substitution for project configuration variables +rst_prolog = """ +.. |project_name| replace:: %s +.. |copyright| replace:: %s +.. |author| replace:: %s +""" % (project, copyright, author) + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['sphinx-static'] + +# Add customm CSS and JS files +html_css_files = ['theme_overrides.css'] +html_js_files = ['switchers.js'] + +# Hide 'Created using Sphinx' text +html_show_sphinx = False + +# Add 'Last updated' on each page +html_last_updated_fmt = '%b %d, %Y' + +# Remove the trailing 'dot' in section numbers +html_secnumber_suffix = " " diff --git a/poky/bitbake/doc/genindex.rst b/poky/bitbake/doc/genindex.rst new file mode 100644 index 000000000..a4af06f65 --- /dev/null +++ b/poky/bitbake/doc/genindex.rst @@ -0,0 +1,3 @@ +===== +Index +===== diff --git a/poky/bitbake/doc/index.rst b/poky/bitbake/doc/index.rst new file mode 100644 index 000000000..3ff8b1580 --- /dev/null +++ b/poky/bitbake/doc/index.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +=================== +BitBake User Manual +=================== + +| + +.. toctree:: + :caption: Table of Contents + :numbered: + + bitbake-user-manual/bitbake-user-manual-intro + bitbake-user-manual/bitbake-user-manual-execution + bitbake-user-manual/bitbake-user-manual-metadata + bitbake-user-manual/bitbake-user-manual-fetching + bitbake-user-manual/bitbake-user-manual-ref-variables + bitbake-user-manual/bitbake-user-manual-hello + +.. toctree:: + :maxdepth: 1 + :hidden: + + genindex + releases + +---- + +.. include:: <xhtml1-lat1.txt> + +| BitBake Community +| Copyright |copy| |copyright| +| <bitbake-devel@lists.openembedded.org> + +This work is licensed under the Creative Commons Attribution License. To view a +copy of this license, visit http://creativecommons.org/licenses/by/2.5/ or send +a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, +California 94041, USA. diff --git a/poky/bitbake/doc/releases.rst b/poky/bitbake/doc/releases.rst new file mode 100644 index 000000000..d68d71599 --- /dev/null +++ b/poky/bitbake/doc/releases.rst @@ -0,0 +1,130 @@ +.. SPDX-License-Identifier: CC-BY-2.5 + +========================= + Current Release Manuals +========================= + +**************************** +3.1 'dunfell' Release Series +**************************** + +- :yocto_docs:`3.1 BitBake User Manual </3.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`3.1.1 BitBake User Manual </3.1.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`3.1.2 BitBake User Manual </3.1.2/bitbake-user-manual/bitbake-user-manual.html>` + +========================== + Previous Release Manuals +========================== + +************************* +3.0 'zeus' Release Series +************************* + +- :yocto_docs:`3.0 BitBake User Manual </3.0/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`3.0.1 BitBake User Manual </3.0.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`3.0.2 BitBake User Manual </3.0.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`3.0.3 BitBake User Manual </3.0.3/bitbake-user-manual/bitbake-user-manual.html>` + +**************************** +2.7 'warrior' Release Series +**************************** + +- :yocto_docs:`2.7 BitBake User Manual </2.7/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.7.1 BitBake User Manual </2.7.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.7.2 BitBake User Manual </2.7.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.7.3 BitBake User Manual </2.7.3/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.7.4 BitBake User Manual </2.7.4/bitbake-user-manual/bitbake-user-manual.html>` + +************************* +2.6 'thud' Release Series +************************* + +- :yocto_docs:`2.6 BitBake User Manual </2.6/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.6.1 BitBake User Manual </2.6.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.6.2 BitBake User Manual </2.6.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.6.3 BitBake User Manual </2.6.3/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.6.4 BitBake User Manual </2.6.4/bitbake-user-manual/bitbake-user-manual.html>` + +************************* +2.5 'sumo' Release Series +************************* + +- :yocto_docs:`2.5 BitBake User Manual </2.5/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.5.1 BitBake User Manual </2.5.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.5.2 BitBake User Manual </2.5.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.5.3 BitBake User Manual </2.5.3/bitbake-user-manual/bitbake-user-manual.html>` + +************************** +2.4 'rocko' Release Series +************************** + +- :yocto_docs:`2.4 BitBake User Manual </2.4/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.4.1 BitBake User Manual </2.4.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.4.2 BitBake User Manual </2.4.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.4.3 BitBake User Manual </2.4.3/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.4.4 BitBake User Manual </2.4.4/bitbake-user-manual/bitbake-user-manual.html>` + +************************* +2.3 'pyro' Release Series +************************* + +- :yocto_docs:`2.3 BitBake User Manual </2.3/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.3.1 BitBake User Manual </2.3.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.3.2 BitBake User Manual </2.3.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.3.3 BitBake User Manual </2.3.3/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.3.4 BitBake User Manual </2.3.4/bitbake-user-manual/bitbake-user-manual.html>` + +************************** +2.2 'morty' Release Series +************************** + +- :yocto_docs:`2.2 BitBake User Manual </2.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.2.1 BitBake User Manual </2.2.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.2.2 BitBake User Manual </2.2.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.2.3 BitBake User Manual </2.2.3/bitbake-user-manual/bitbake-user-manual.html>` + +**************************** +2.1 'krogoth' Release Series +**************************** + +- :yocto_docs:`2.1 BitBake User Manual </2.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.1.1 BitBake User Manual </2.1.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.1.2 BitBake User Manual </2.1.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.1.3 BitBake User Manual </2.1.3/bitbake-user-manual/bitbake-user-manual.html>` + +*************************** +2.0 'jethro' Release Series +*************************** + +- :yocto_docs:`1.9 BitBake User Manual </1.9/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.0 BitBake User Manual </2.0/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.0.1 BitBake User Manual </2.0.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.0.2 BitBake User Manual </2.0.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`2.0.3 BitBake User Manual </2.0.3/bitbake-user-manual/bitbake-user-manual.html>` + +************************* +1.8 'fido' Release Series +************************* + +- :yocto_docs:`1.8 BitBake User Manual </1.8/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.8.1 BitBake User Manual </1.8.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.8.2 BitBake User Manual </1.8.2/bitbake-user-manual/bitbake-user-manual.html>` + +************************** +1.7 'dizzy' Release Series +************************** + +- :yocto_docs:`1.7 BitBake User Manual </1.7/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.7.1 BitBake User Manual </1.7.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.7.2 BitBake User Manual </1.7.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.7.3 BitBake User Manual </1.7.3/bitbake-user-manual/bitbake-user-manual.html>` + +************************** +1.6 'daisy' Release Series +************************** + +- :yocto_docs:`1.6 BitBake User Manual </1.6/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.6.1 BitBake User Manual </1.6.1/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.6.2 BitBake User Manual </1.6.2/bitbake-user-manual/bitbake-user-manual.html>` +- :yocto_docs:`1.6.3 BitBake User Manual </1.6.3/bitbake-user-manual/bitbake-user-manual.html>` + diff --git a/poky/bitbake/doc/sphinx-static/switchers.js b/poky/bitbake/doc/sphinx-static/switchers.js new file mode 100644 index 000000000..32113cfa9 --- /dev/null +++ b/poky/bitbake/doc/sphinx-static/switchers.js @@ -0,0 +1,233 @@ +(function() { + 'use strict'; + + var all_versions = { + 'dev': 'dev (3.2)', + '3.1.2': '3.1.2', + '3.0.3': '3.0.3', + '2.7.4': '2.7.4', + }; + + var all_doctypes = { + 'single': 'Individual Webpages', + 'mega': "All-in-one 'Mega' Manual", + }; + + // Simple version comparision + // Return 1 if a > b + // Return -1 if a < b + // Return 0 if a == b + function ver_compare(a, b) { + if (a == "dev") { + return 1; + } + + if (a === b) { + return 0; + } + + var a_components = a.split("."); + var b_components = b.split("."); + + var len = Math.min(a_components.length, b_components.length); + + // loop while the components are equal + for (var i = 0; i < len; i++) { + // A bigger than B + if (parseInt(a_components[i]) > parseInt(b_components[i])) { + return 1; + } + + // B bigger than A + if (parseInt(a_components[i]) < parseInt(b_components[i])) { + return -1; + } + } + + // If one's a prefix of the other, the longer one is greater. + if (a_components.length > b_components.length) { + return 1; + } + + if (a_components.length < b_components.length) { + return -1; + } + + // Otherwise they are the same. + return 0; + } + + function build_version_select(current_series, current_version) { + var buf = ['<select>']; + + $.each(all_versions, function(version, title) { + var series = version.substr(0, 3); + if (series == current_series) { + if (version == current_version) + buf.push('<option value="' + version + '" selected="selected">' + title + '</option>'); + else + buf.push('<option value="' + version + '">' + title + '</option>'); + + if (version != current_version) + buf.push('<option value="' + current_version + '" selected="selected">' + current_version + '</option>'); + } else { + buf.push('<option value="' + version + '">' + title + '</option>'); + } + }); + + buf.push('</select>'); + return buf.join(''); + } + + function build_doctype_select(current_doctype) { + var buf = ['<select>']; + + $.each(all_doctypes, function(doctype, title) { + if (doctype == current_doctype) + buf.push('<option value="' + doctype + '" selected="selected">' + + all_doctypes[current_doctype] + '</option>'); + else + buf.push('<option value="' + doctype + '">' + title + '</option>'); + }); + if (!(current_doctype in all_doctypes)) { + // In case we're browsing a doctype that is not yet in all_doctypes. + buf.push('<option value="' + current_doctype + '" selected="selected">' + + current_doctype + '</option>'); + all_doctypes[current_doctype] = current_doctype; + } + buf.push('</select>'); + return buf.join(''); + } + + function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + var url = urls.shift(); + + // Web browsers won't redirect file:// urls to file urls using ajax but + // its useful for local testing + if (url.startsWith("file://")) { + window.location.href = url; + return; + } + + if (urls.length == 0) { + window.location.href = url; + return; + } + $.ajax({ + url: url, + success: function() { + window.location.href = url; + }, + error: function() { + navigate_to_first_existing(urls); + } + }); + } + + function get_docroot_url() { + var url = window.location.href; + var root = DOCUMENTATION_OPTIONS.URL_ROOT; + + var urlarray = url.split('/'); + // Trim off anything after '/' + urlarray.pop(); + var depth = (root.match(/\.\.\//g) || []).length; + for (var i = 0; i < depth; i++) { + urlarray.pop(); + } + + return urlarray.join('/') + '/'; + } + + function on_version_switch() { + var selected_version = $(this).children('option:selected').attr('value'); + var url = window.location.href; + var current_version = DOCUMENTATION_OPTIONS.VERSION; + var docroot = get_docroot_url() + + var new_versionpath = selected_version + '/'; + if (selected_version == "dev") + new_versionpath = ''; + + // dev versions have no version prefix + if (current_version == "dev") { + var new_url = docroot + new_versionpath + url.replace(docroot, ""); + var fallback_url = docroot + new_versionpath; + } else { + var new_url = url.replace('/' + current_version + '/', '/' + new_versionpath); + var fallback_url = new_url.replace(url.replace(docroot, ""), ""); + } + + console.log(get_docroot_url()) + console.log(url + " to url " + new_url); + console.log(url + " to fallback " + fallback_url); + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + fallback_url, + 'https://www.yoctoproject.org/docs/', + ]); + } + } + + function on_doctype_switch() { + var selected_doctype = $(this).children('option:selected').attr('value'); + var url = window.location.href; + if (selected_doctype == 'mega') { + var docroot = get_docroot_url() + var current_version = DOCUMENTATION_OPTIONS.VERSION; + // Assume manuals before 3.2 are using old docbook mega-manual + if (ver_compare(current_version, "3.2") < 0) { + var new_url = docroot + "mega-manual/mega-manual.html"; + } else { + var new_url = docroot + "singleindex.html"; + } + } else { + var new_url = url.replace("singleindex.html", "index.html") + } + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + 'https://www.yoctoproject.org/docs/', + ]); + } + } + + // Returns the current doctype based upon the url + function doctype_segment_from_url(url) { + if (url.includes("singleindex") || url.includes("mega-manual")) + return "mega"; + return "single"; + } + + $(document).ready(function() { + var release = DOCUMENTATION_OPTIONS.VERSION; + var current_doctype = doctype_segment_from_url(window.location.href); + var current_series = release.substr(0, 3); + var version_select = build_version_select(current_series, release); + + $('.version_switcher_placeholder').html(version_select); + $('.version_switcher_placeholder select').bind('change', on_version_switch); + + var doctype_select = build_doctype_select(current_doctype); + + $('.doctype_switcher_placeholder').html(doctype_select); + $('.doctype_switcher_placeholder select').bind('change', on_doctype_switch); + + if (ver_compare(release, "3.1") < 0) { + $('#outdated-warning').html('Version ' + release + ' of the project is now considered obsolete, please select and use a more recent version'); + $('#outdated-warning').css('padding', '.5em'); + } else if (release != "dev") { + $.each(all_versions, function(version, title) { + var series = version.substr(0, 3); + if (series == current_series && version != release) { + $('#outdated-warning').html('This document is for outdated version ' + release + ', you should select the latest release version in this series, ' + version + '.'); + $('#outdated-warning').css('padding', '.5em'); + } + }); + } + }); +})(); diff --git a/poky/bitbake/doc/sphinx-static/theme_overrides.css b/poky/bitbake/doc/sphinx-static/theme_overrides.css new file mode 100644 index 000000000..4fd76a152 --- /dev/null +++ b/poky/bitbake/doc/sphinx-static/theme_overrides.css @@ -0,0 +1,164 @@ +/* + SPDX-License-Identifier: CC-BY-2.0-UK +*/ + +body { + font-family: Verdana, Sans, sans-serif; + + min-width: 640px; + margin: 0em auto; + color: #333; +} + +h1,h2,h3,h4,h5,h6,h7 { + font-family: Arial, Sans; + color: #00557D; + clear: both; +} + +h1 { + font-size: 2em; + text-align: left; + padding: 0em 0em 0em 0em; + margin: 2em 0em 0em 0em; +} + +h2.subtitle { + margin: 0.10em 0em 3.0em 0em; + padding: 0em 0em 0em 0em; + font-size: 1.8em; + padding-left: 20%; + font-weight: normal; + font-style: italic; +} + +h2 { + margin: 2em 0em 0.66em 0em; + padding: 0.5em 0em 0em 0em; + font-size: 1.5em; + font-weight: bold; +} + +h3.subtitle { + margin: 0em 0em 1em 0em; + padding: 0em 0em 0em 0em; + font-size: 142.14%; + text-align: right; +} + +h3 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 140%; + font-weight: bold; +} + +h4 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 120%; + font-weight: bold; +} + +h5 { + margin: 1em 0em 0.5em 0em; + padding: 1em 0em 0em 0em; + font-size: 110%; + font-weight: bold; +} + +h6 { + margin: 1em 0em 0em 0em; + padding: 1em 0em 0em 0em; + font-size: 110%; + font-weight: bold; +} + +em { + font-weight: bold; +} + +.pre { + font-size: medium; + font-family: Courier, monospace; +} + +.wy-nav-content a { + text-decoration: underline; + color: #444; + background: transparent; +} + +.wy-nav-content a:hover { + text-decoration: underline; + background-color: #dedede; +} + +.wy-nav-content a:visited { + color: #444; +} + +[alt='Permalink'] { color: #eee; } +[alt='Permalink']:hover { color: black; } + +@media screen { + /* content column + * + * RTD theme's default is 800px as max width for the content, but we have + * tables with tons of columns, which need the full width of the view-port. + */ + + .wy-nav-content{max-width: none; } + + /* inline literal: drop the borderbox, padding and red color */ + code, .rst-content tt, .rst-content code { + color: inherit; + border: none; + padding: unset; + background: inherit; + font-size: 85%; + } + + .rst-content tt.literal,.rst-content tt.literal,.rst-content code.literal { + color: inherit; + } + + /* Admonition should be gray, not blue or green */ + .rst-content .note .admonition-title, + .rst-content .tip .admonition-title, + .rst-content .warning .admonition-title, + .rst-content .caution .admonition-title, + .rst-content .important .admonition-title { + background: #f0f0f2; + color: #00557D; + + } + + .rst-content .note, + .rst-content .tip, + .rst-content .important, + .rst-content .warning, + .rst-content .caution { + background: #f0f0f2; + } + + /* Remove the icon in front of note/tip element, and before the logo */ + .icon-home:before, .rst-content .admonition-title:before { + display: none + } + + /* a custom informalexample container is used in some doc */ + .informalexample { + border: 1px solid; + border-color: #aaa; + margin: 1em 0em; + padding: 1em; + page-break-inside: avoid; + } + + /* Remove the blue background in the top left corner, around the logo */ + .wy-side-nav-search { + background: inherit; + } + +} diff --git a/poky/bitbake/lib/bb/COW.py b/poky/bitbake/lib/bb/COW.py index bc20ce38e..23c22b65e 100644 --- a/poky/bitbake/lib/bb/COW.py +++ b/poky/bitbake/lib/bb/COW.py @@ -3,13 +3,14 @@ # # Copyright (C) 2006 Tim Ansell # -#Please Note: +# Please Note: # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW. # Assign a file to __warn__ to get warnings about slow operations. # import copy + ImmutableTypes = ( bool, complex, @@ -22,9 +23,11 @@ ImmutableTypes = ( MUTABLE = "__mutable__" + class COWMeta(type): pass + class COWDictMeta(COWMeta): __warn__ = False __hasmutable__ = False @@ -33,12 +36,15 @@ class COWDictMeta(COWMeta): def __str__(cls): # FIXME: I have magic numbers! return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3) + __repr__ = __str__ def cow(cls): class C(cls): __count__ = cls.__count__ + 1 + return C + copy = cow __call__ = cow @@ -70,8 +76,9 @@ class COWDictMeta(COWMeta): return value __getmarker__ = [] + def __getreadonly__(cls, key, default=__getmarker__): - """\ + """ Get a value (even if mutable) which you promise not to change. """ return cls.__getitem__(key, default, True) @@ -138,24 +145,29 @@ class COWDictMeta(COWMeta): def iterkeys(cls): return cls.iter("keys") + def itervalues(cls, readonly=False): if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + print("Warning: If you aren't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("values", readonly) + def iteritems(cls, readonly=False): if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False: - print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__) + print("Warning: If you aren't going to change any of the values call with True.", file=cls.__warn__) return cls.iter("items", readonly) + class COWSetMeta(COWDictMeta): def __str__(cls): # FIXME: I have magic numbers! - return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) -3) + return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3) + __repr__ = __str__ def cow(cls): class C(cls): __count__ = cls.__count__ + 1 + return C def add(cls, value): @@ -173,131 +185,11 @@ class COWSetMeta(COWDictMeta): def iteritems(cls): raise TypeError("sets don't have 'items'") + # These are the actual classes you use! -class COWDictBase(object, metaclass = COWDictMeta): +class COWDictBase(metaclass=COWDictMeta): __count__ = 0 -class COWSetBase(object, metaclass = COWSetMeta): - __count__ = 0 -if __name__ == "__main__": - import sys - COWDictBase.__warn__ = sys.stderr - a = COWDictBase() - print("a", a) - - a['a'] = 'a' - a['b'] = 'b' - a['dict'] = {} - - b = a.copy() - print("b", b) - b['c'] = 'b' - - print() - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - b['dict']['a'] = 'b' - b['a'] = 'c' - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(): - print(x) - print() - - try: - b['dict2'] - except KeyError as e: - print("Okay!") - - a['set'] = COWSetBase() - a['set'].add("o1") - a['set'].add("o1") - a['set'].add("o2") - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - b['set'].add('o3') - - print("a", a) - for x in a['set'].itervalues(): - print(x) - print("--") - print("b", b) - for x in b['set'].itervalues(): - print(x) - print() - - a['set2'] = set() - a['set2'].add("o1") - a['set2'].add("o1") - a['set2'].add("o2") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - del b['b'] - try: - print(b['b']) - except KeyError: - print("Yay! deleted key raises error") - - if 'b' in b: - print("Boo!") - else: - print("Yay - has_key with delete works!") - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('b') - - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() - - b.__revertitem__('dict') - print("a", a) - for x in a.iteritems(): - print(x) - print("--") - print("b", b) - for x in b.iteritems(readonly=True): - print(x) - print() +class COWSetBase(metaclass=COWSetMeta): + __count__ = 0 diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py index b96466e65..888dd5ccc 100644 --- a/poky/bitbake/lib/bb/__init__.py +++ b/poky/bitbake/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "1.46.0" +__version__ = "1.47.0" import sys if sys.version_info < (3, 5, 0): @@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1 @@ -60,16 +62,56 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + + if sys.version_info < (3, 6): + # These properties were added in Python 3.6. Add them in older versions + # for compatibility + @property + def manager(self): + return self.logger.manager + + @manager.setter + def manager(self, value): + self.logger.manager = value + + @property + def name(self): + return self.logger.name + + def __repr__(self): + logger = self.logger + level = logger.getLevelName(logger.getEffectiveLevel()) + return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index 23b6ee455..974d2ff06 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -16,7 +16,9 @@ import os import sys import logging import glob +import itertools import time +import re import stat import bb import bb.msg @@ -27,6 +29,9 @@ from bb import data, event, utils bblogger = logging.getLogger('BitBake') logger = logging.getLogger('BitBake.Build') +verboseShellLogging = False +verboseStdoutLogging = False + __mtime_cache = {} def cached_mtime_noerror(f): @@ -303,20 +308,60 @@ def exec_func_python(func, d, runfile, cwd=None): def shell_trap_code(): return '''#!/bin/sh\n +__BITBAKE_LAST_LINE=0 + # Emit a useful diagnostic if something fails: -bb_exit_handler() { +bb_sh_exit_handler() { + ret=$? + if [ "$ret" != 0 ]; then + echo "WARNING: exit code $ret from a shell command." + fi + exit $ret +} + +bb_bash_exit_handler() { ret=$? - case $ret in - 0) ;; - *) case $BASH_VERSION in - "") echo "WARNING: exit code $ret from a shell command.";; - *) echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";; - esac - exit $ret - esac + { set +x; } > /dev/null + trap "" DEBUG + if [ "$ret" != 0 ]; then + echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" + + echo "WARNING: Backtrace (BB generated script): " + for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do + if [ "$i" -eq 1 ]; then + echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" + else + echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" + fi + done + fi + exit $ret } -trap 'bb_exit_handler' 0 -set -e + +bb_bash_debug_handler() { + local line=${BASH_LINENO[0]} + # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it + if [ "$line" -eq 1 ]; then + return + fi + + # Track the line number of commands as they execute. This is so we can have access to the failing line number + # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html + if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then + __BITBAKE_LAST_LINE=$line + fi +} + +case $BASH_VERSION in +"") trap 'bb_sh_exit_handler' 0 + set -e + ;; +*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 + trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG + set -e + shopt -s extdebug + ;; +esac ''' def create_progress_handler(func, progress, logfile, d): @@ -346,7 +391,7 @@ def create_progress_handler(func, progress, logfile, d): cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) if not cls_obj: # Fall-back on __builtins__ - cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__) + cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) if cls_obj: return cls_obj(d, outfile=logfile, otherargs=otherargs) bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) @@ -371,7 +416,7 @@ def exec_func_shell(func, d, runfile, cwd=None): bb.data.emit_func(func, script, d) - if bb.msg.loggerVerboseLogs: + if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): script.write("set -x\n") if cwd: script.write("cd '%s'\n" % cwd) @@ -391,14 +436,20 @@ exit $ret if fakerootcmd: cmd = [fakerootcmd, runfile] - if bb.msg.loggerDefaultVerbose: + if verboseStdoutLogging: logfile = LogTee(logger, StdoutNoopContextManager()) else: logfile = StdoutNoopContextManager() progress = d.getVarFlag(func, 'progress') if progress: - logfile = create_progress_handler(func, progress, logfile, d) + try: + logfile = create_progress_handler(func, progress, logfile, d) + except: + from traceback import format_exc + logger.error("Failed to create progress handler") + logger.error(format_exc()) + raise fifobuffer = bytearray() def readfifo(data): @@ -450,6 +501,62 @@ exit $ret bb.debug(2, "Executing shell function %s" % func) with open(os.devnull, 'r+') as stdin, logfile: bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) + except bb.process.ExecutionError as exe: + # Find the backtrace that the shell trap generated + backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") + stdout_lines = (exe.stdout or "").split("\n") + backtrace_start_line = None + for i, line in enumerate(reversed(stdout_lines)): + if backtrace_marker_regex.search(line): + backtrace_start_line = len(stdout_lines) - i + break + + # Read the backtrace frames, starting at the location we just found + backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" + r"?P<lineno>\d+)") + backtrace_frames = [] + if backtrace_start_line: + for line in itertools.islice(stdout_lines, backtrace_start_line, None): + match = backtrace_entry_regex.search(line) + if match: + backtrace_frames.append(match.groupdict()) + + with open(runfile, "r") as script: + script_lines = [line.rstrip() for line in script.readlines()] + + # For each backtrace frame, search backwards in the script (from the line number called out by the frame), + # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata + # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. + script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") + better_frames = [] + # Skip the very last frame since it's just the call to the shell task in the body of the script + for frame in backtrace_frames[:-1]: + # Check whether the frame corresponds to a function defined in the script vs external script. + if os.path.samefile(frame["file"], runfile): + # Search backwards from the frame lineno to locate the comment that BB injected + i = int(frame["lineno"]) - 1 + while i >= 0: + match = script_metadata_comment_regex.match(script_lines[i]) + if match: + # Calculate the relative line in the function itself + relative_line_in_function = int(frame["lineno"]) - i - 2 + # Calculate line in the function as declared in the metadata + metadata_function_line = relative_line_in_function + int(match["lineno"]) + better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( + frameno=frame["frameno"], + funcname=frame["funcname"], + file=match["file"], + lineno=metadata_function_line + )) + break + i -= 1 + else: + better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) + + if better_frames: + better_frames = ("\t{0}".format(frame) for frame in better_frames) + exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) + raise finally: os.unlink(fifopath) diff --git a/poky/bitbake/lib/bb/cache.py b/poky/bitbake/lib/bb/cache.py index d1be83617..9e0c931a0 100644 --- a/poky/bitbake/lib/bb/cache.py +++ b/poky/bitbake/lib/bb/cache.py @@ -19,16 +19,20 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "152" +__cache_version__ = "153" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other @@ -324,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mc=None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped @@ -337,6 +341,10 @@ class NoCache(object): datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores + if mc is not None: + bb_data = self.databuilder.mcdata[mc].createCopy() + return parse_recipe(bb_data, bbfile, appends, mc) + bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends) @@ -354,14 +362,15 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set() @@ -374,31 +383,47 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) - logger.debug(1, "Cache dir: %s", self.cachedir) + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") + + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - cache_ok = cache_ok and os.path.exists(cachefile) + cachefile = self.getCacheFile(cache_class.cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try: @@ -406,22 +431,31 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass + + return cachesize - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - logger.debug(1, 'Loading cache file: %s' % cachefile) + cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information @@ -429,15 +463,15 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') - return + self.logger.info('Invalid cache, rebuilding...') + return 0 if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') - return + self.logger.info('Cache version mismatch, rebuilding...') + return 0 elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') - return + self.logger.info('Bitbake version mismatch, rebuilding...') + return 0 # Load the rest of the cache file current_progress = 0 @@ -460,29 +494,17 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mc=self.mc) depends = [] variants = [] # Process the "real" fn last so we can store variants list @@ -534,7 +556,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached) @@ -570,21 +592,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False @@ -595,14 +617,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False @@ -614,18 +636,18 @@ class Cache(NoCache): # Have to be careful about spaces and colons in filenames flist = self.filelist_regex.split(fl) for f in flist: - if not f or "*" in f: + if not f: continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False - if appends != info_array[0].appends: - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + if tuple(appends) != tuple(info_array[0].appends): + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False @@ -634,10 +656,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all @@ -645,10 +667,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False @@ -661,10 +683,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self): @@ -677,12 +699,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__) @@ -701,8 +724,18 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): + if self.mc is not None: + (fn, cls, mc) = virtualfn2realfn(filename) + if mc: + self.logger.error("Unexpected multiconfig %s", filename) + return + + vfn = realfn2virtual(fn, cls, self.mc) + else: + vfn = filename + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): - cacheData.add_from_recipeinfo(filename, info_array) + cacheData.add_from_recipeinfo(vfn, info_array) if watcher: watcher(info_array[0].file_depends) @@ -727,6 +760,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """ diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py index 6abf38668..f8c6a03bb 100644 --- a/poky/bitbake/lib/bb/command.py +++ b/poky/bitbake/lib/bb/command.py @@ -54,13 +54,20 @@ class Command: self.cooker = cooker self.cmds_sync = CommandsSync() self.cmds_async = CommandsAsync() - self.remotedatastores = bb.remotedata.RemoteDatastores(cooker) + self.remotedatastores = None # FIXME Add lock for this self.currentAsyncCommand = None def runCommand(self, commandline, ro_only = False): command = commandline.pop(0) + + # Ensure cooker is ready for commands + if command != "updateConfig" and command != "setFeatures": + self.cooker.init_configdata() + if not self.remotedatastores: + self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) + if hasattr(CommandsSync, command): # Can run synchronous commands straight away command_method = getattr(self.cmds_sync, command) @@ -84,7 +91,7 @@ class Command: if command not in CommandsAsync.__dict__: return None, "No such command" self.currentAsyncCommand = (command, commandline) - self.cooker.configuration.server_register_idlecallback(self.cooker.runCommands, self.cooker) + self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker) return True, None def runAsyncCommand(self): @@ -136,13 +143,8 @@ class Command: self.cooker.finishcommand() def reset(self): - self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) - -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) + if self.remotedatastores: + self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) class CommandsSync: """ @@ -232,7 +234,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params): @@ -395,22 +401,38 @@ class CommandsSync: def getSkippedRecipes(self, command, params): # Return list sorted by reverse priority order import bb.cache - skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + def sortkey(x): + vfn, _ = x + realfn, _, mc = bb.cache.virtualfn2realfn(vfn) + return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn) + + skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params): @@ -422,7 +444,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_mc(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True @@ -496,6 +518,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3: @@ -507,7 +530,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server, @@ -517,7 +540,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder) @@ -708,10 +731,10 @@ class CommandsAsync: """ Find signature info files via the signature generator """ - pn = params[0] + (mc, pn) = bb.runqueue.split_mc(params[0]) taskname = params[1] sigs = params[2] - res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.data) - bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.data) + res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc]) + bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc]) command.finishAsyncCommand() findSigInfo.needcache = False diff --git a/poky/bitbake/lib/bb/compat.py b/poky/bitbake/lib/bb/compat.py deleted file mode 100644 index 49356681a..000000000 --- a/poky/bitbake/lib/bb/compat.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# SPDX-License-Identifier: GPL-2.0-only -# - -"""Code pulled from future python versions, here for compatibility""" - -from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict -from functools import total_ordering - - diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index e527e2311..1f4cc1e96 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -148,15 +148,18 @@ class BBCooker: Manages one bitbake build run """ - def __init__(self, configuration, featureSet=None): + def __init__(self, featureSet=None, idleCallBackRegister=None): self.recipecaches = None + self.eventlog = None self.skiplist = {} self.featureset = CookerFeatures() if featureSet: for f in featureSet: self.featureset.setFeature(f) - self.configuration = configuration + self.configuration = bb.cookerdata.CookerConfiguration() + + self.idleCallBackRegister = idleCallBackRegister bb.debug(1, "BBCooker starting %s" % time.time()) sys.stdout.flush() @@ -192,25 +195,13 @@ class BBCooker: self.hashserv = None self.hashservaddr = None - self.initConfigurationData() - - bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) - sys.stdout.flush() - - # we log all events to a file if so directed - if self.configuration.writeeventlog: - # register the log file writer as UI Handler - writer = EventWriter(self, self.configuration.writeeventlog) - EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) - bb.event.register_UIHhandler(EventLogWriteHandler(writer)) - self.inotify_modified_files = [] def _process_inotify_updates(server, cooker, abort): cooker.process_inotify_updates() return 1.0 - self.configuration.server_register_idlecallback(_process_inotify_updates, self) + self.idleCallBackRegister(_process_inotify_updates, self) # TOSTOP must not be set or our children will hang when they output try: @@ -237,6 +228,13 @@ class BBCooker: bb.debug(1, "BBCooker startup complete %s" % time.time()) sys.stdout.flush() + def init_configdata(self): + if not hasattr(self, "data"): + self.initConfigurationData() + bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) + sys.stdout.flush() + self.handlePRServ() + def process_inotify_updates(self): for n in [self.confignotifier, self.notifier]: if n.check_events(timeout=0): @@ -322,7 +320,7 @@ class BBCooker: for feature in features: self.featureset.setFeature(feature) bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset))) - if (original_featureset != list(self.featureset)) and self.state != state.error: + if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"): self.reset() def initConfigurationData(self): @@ -354,7 +352,7 @@ class BBCooker: self.caches_array.append(getattr(module, cache_name)) except ImportError as exc: logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) - sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) + raise bb.BBHandledException() self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False) self.databuilder.parseBaseConfiguration() @@ -411,11 +409,6 @@ class BBCooker: self.data.disableTracking() def parseConfiguration(self): - # Set log file verbosity - verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", False)) - if verboselogs: - bb.msg.loggerVerboseLogs = True - # Change nice level if we're asked to nice = self.data.getVar("BB_NICE_LEVEL") if nice: @@ -449,7 +442,28 @@ class BBCooker: logger.debug(1, "Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) clean = False - setattr(self.configuration, o, options[o]) + if hasattr(self.configuration, o): + setattr(self.configuration, o, options[o]) + + if self.configuration.writeeventlog: + if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog: + bb.event.unregister_UIHhandler(self.eventlog[1]) + if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog: + # we log all events to a file if so directed + # register the log file writer as UI Handler + writer = EventWriter(self, self.configuration.writeeventlog) + EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) + self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer))) + + bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel + bb.msg.loggerDefaultDomains = self.configuration.debug_domains + + if hasattr(self, "data"): + origenv = bb.data.init() + for k in environment: + origenv.setVar(k, environment[k]) + self.data.setVar("BB_ORIGENV", origenv) + for k in bb.utils.approved_variables(): if k in environment and k not in self.configuration.env: logger.debug(1, "Updating new environment variable %s to %s" % (k, environment[k])) @@ -465,6 +479,10 @@ class BBCooker: logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k])) self.configuration.env[k] = environment[k] clean = False + + # Now update all the variables not in the datastore to match + self.configuration.env = environment + if not clean: logger.debug(1, "Base environment change, triggering reparse") self.reset() @@ -525,7 +543,7 @@ class BBCooker: self.parseConfiguration() fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) fn = bb.cache.realfn2virtual(fn, cls, mc) elif len(pkgs_to_build) == 1: mc = mc_base(pkgs_to_build[0]) @@ -541,8 +559,8 @@ class BBCooker: if fn: try: - bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) - envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) + bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) + envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn)) except Exception as e: parselog.exception("Unable to read %s", fn) raise @@ -626,6 +644,7 @@ class BBCooker: current = 0 runlist = [] for k in fulltargetlist: + origk = k mc = "" if k.startswith("mc:"): mc = k.split(":")[1] @@ -635,6 +654,10 @@ class BBCooker: k2 = k.split(":do_") k = k2[0] ktask = k2[1] + + if mc not in self.multiconfigs: + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc)) + taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) current += 1 if not ktask.startswith("do_"): @@ -670,7 +693,7 @@ class BBCooker: l = k.split(':') depmc = l[2] if depmc not in self.multiconfigs: - bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc)) + bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) else: logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3]) taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) @@ -929,26 +952,33 @@ class BBCooker: logger.info("Task dependencies saved to 'task-depends.dot'") def show_appends_with_no_recipes(self): + appends_without_recipes = {} # Determine which bbappends haven't been applied - - # First get list of recipes, including skipped - recipefns = list(self.recipecaches[''].pkg_fn.keys()) - recipefns.extend(self.skiplist.keys()) - - # Work out list of bbappends that have been applied - applied_appends = [] - for fn in recipefns: - applied_appends.extend(self.collection.get_file_appends(fn)) - - appends_without_recipes = [] - for _, appendfn in self.collection.bbappends: - if not appendfn in applied_appends: - appends_without_recipes.append(appendfn) - - if appends_without_recipes: - msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) - warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ - False) or "no" + for mc in self.multiconfigs: + # First get list of recipes, including skipped + recipefns = list(self.recipecaches[mc].pkg_fn.keys()) + recipefns.extend(self.skiplist.keys()) + + # Work out list of bbappends that have been applied + applied_appends = [] + for fn in recipefns: + applied_appends.extend(self.collections[mc].get_file_appends(fn)) + + appends_without_recipes[mc] = [] + for _, appendfn in self.collections[mc].bbappends: + if not appendfn in applied_appends: + appends_without_recipes[mc].append(appendfn) + + msgs = [] + for mc in sorted(appends_without_recipes.keys()): + if appends_without_recipes[mc]: + msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', + '\n '.join(appends_without_recipes[mc]))) + + if msgs: + msg = "\n".join(msgs) + warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ + False) or "no" if warn_only.lower() in ("1", "yes", "true"): bb.warn(msg) else: @@ -1097,7 +1127,7 @@ class BBCooker: from bb import shell except ImportError: parselog.exception("Interactive mode not available") - sys.exit(1) + raise bb.BBHandledException() else: shell.start( self ) @@ -1249,15 +1279,15 @@ class BBCooker: if siggen_cache: bb.parse.siggen.checksum_cache.mtime_cache.clear() - def matchFiles(self, bf): + def matchFiles(self, bf, mc=''): """ Find the .bb files which match the expression in 'buildfile'. """ if bf.startswith("/") or bf.startswith("../"): bf = os.path.abspath(bf) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} + filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) try: os.stat(bf) bf = os.path.abspath(bf) @@ -1270,12 +1300,12 @@ class BBCooker: matches.append(f) return matches - def matchFile(self, buildfile): + def matchFile(self, buildfile, mc=''): """ Find the .bb file which matches the expression in 'buildfile'. Raise an error if multiple files """ - matches = self.matchFiles(buildfile) + matches = self.matchFiles(buildfile, mc) if len(matches) != 1: if matches: msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) @@ -1316,14 +1346,14 @@ class BBCooker: task = "do_%s" % task fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) self.buildSetVars() self.reset_mtime_caches() - bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) + bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) - infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) + infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn)) infos = dict(infos) fn = bb.cache.realfn2virtual(fn, cls, mc) @@ -1411,7 +1441,7 @@ class BBCooker: return True return retval - self.configuration.server_register_idlecallback(buildFileIdle, rq) + self.idleCallBackRegister(buildFileIdle, rq) def buildTargets(self, targets, task): """ @@ -1482,7 +1512,7 @@ class BBCooker: if 'universe' in targets: rq.rqdata.warn_multi_bb = True - self.configuration.server_register_idlecallback(buildTargetsIdle, rq) + self.idleCallBackRegister(buildTargetsIdle, rq) def getAllKeysWithFlags(self, flaglist): @@ -1533,6 +1563,7 @@ class BBCooker: if self.state in (state.shutdown, state.forceshutdown, state.error): if hasattr(self.parser, 'shutdown'): self.parser.shutdown(clean=False, force = True) + self.parser.final_cleanup() raise bb.BBHandledException() if self.state != state.parsing: @@ -1552,14 +1583,24 @@ class BBCooker: for dep in self.configuration.extra_assume_provided: self.recipecaches[mc].ignored_dependencies.add(dep) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {} + + mcfilelist = {} + total_masked = 0 + searchdirs = set() + for mc in self.multiconfigs: + self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) + (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) + + mcfilelist[mc] = filelist + total_masked += masked + searchdirs |= set(search) # Add inotify watches for directories searched for bb/bbappend files for dirent in searchdirs: self.add_filewatch([[dirent]], dirs=True) - self.parser = CookerParser(self, filelist, masked) + self.parser = CookerParser(self, mcfilelist, total_masked) self.parsecache_valid = True self.state = state.parsing @@ -1571,7 +1612,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1592,7 +1633,7 @@ class BBCooker: raise NothingToBuild ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() - for pkg in pkgs_to_build: + for pkg in pkgs_to_build.copy(): if pkg in ignore: parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) if pkg.startswith("multiconfig:"): @@ -1630,17 +1671,16 @@ class BBCooker: return pkgs_to_build def pre_serve(self): - # We now are in our own process so we can call this here. - # PRServ exits if its parent process exits - self.handlePRServ() return def post_serve(self): + self.shutdown(force=True) prserv.serv.auto_shutdown() if self.hashserv: self.hashserv.process.terminate() self.hashserv.process.join() - bb.event.fire(CookerExit(), self.data) + if hasattr(self, "data"): + bb.event.fire(CookerExit(), self.data) def shutdown(self, force = False): if force: @@ -1650,6 +1690,7 @@ class BBCooker: if self.parser: self.parser.shutdown(clean=not force, force=force) + self.parser.final_cleanup() def finishcommand(self): self.state = state.initial @@ -1663,8 +1704,9 @@ class BBCooker: self.finishcommand() self.extraconfigdata = {} self.command.reset() - self.databuilder.reset() - self.data = self.databuilder.data + if hasattr(self, "data"): + self.databuilder.reset() + self.data = self.databuilder.data self.parsecache_valid = False self.baseconfig_valid = False @@ -1679,21 +1721,19 @@ class CookerExit(bb.event.Event): class CookerCollectFiles(object): - def __init__(self, priorities): + def __init__(self, priorities, mc=''): + self.mc = mc self.bbappends = [] # Priorities is a list of tupples, with the second element as the pattern. # We need to sort the list with the longest pattern first, and so on to # the shortest. This allows nested layers to be properly evaluated. self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) - def calc_bbfile_priority( self, filename, matched = None ): + def calc_bbfile_priority(self, filename): for _, _, regex, pri in self.bbfile_config_priorities: if regex.match(filename): - if matched is not None: - if not regex in matched: - matched.add(regex) - return pri - return 0 + return pri, regex + return 0, None def get_bbfiles(self): """Get list of default .bb files by reading out the current directory""" @@ -1723,10 +1763,10 @@ class CookerCollectFiles(object): collectlog.debug(1, "collecting .bb files") files = (config.getVar( "BBFILES") or "").split() - config.setVar("BBFILES", " ".join(files)) # Sort files by priority - files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) + files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) + config.setVar("BBFILES_PRIORITIZED", " ".join(files)) if not len(files): files = self.get_bbfiles() @@ -1846,43 +1886,67 @@ class CookerCollectFiles(object): (bbappend, filename) = b if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): filelist.append(filename) - return filelist + return tuple(filelist) - def collection_priorities(self, pkgfns, d): + def collection_priorities(self, pkgfns, fns, d): + # Return the priorities of the entries in pkgfns + # Also check that all the regexes in self.bbfile_config_priorities are used + # (but to do that we need to ensure skipped recipes aren't counted, nor + # collections in BBFILE_PATTERN_IGNORE_EMPTY) priorities = {} + seen = set() + matched = set() + + matched_regex = set() + unmatched_regex = set() + for _, _, regex, _ in self.bbfile_config_priorities: + unmatched_regex.add(regex) # Calculate priorities for each file - matched = set() for p in pkgfns: realfn, cls, mc = bb.cache.virtualfn2realfn(p) - priorities[p] = self.calc_bbfile_priority(realfn, matched) - - unmatched = set() - for _, _, regex, pri in self.bbfile_config_priorities: - if not regex in matched: - unmatched.add(regex) - - # Don't show the warning if the BBFILE_PATTERN did match .bbappend files - def find_bbappend_match(regex): + priorities[p], regex = self.calc_bbfile_priority(realfn) + if regex in unmatched_regex: + matched_regex.add(regex) + unmatched_regex.remove(regex) + seen.add(realfn) + if regex: + matched.add(realfn) + + if unmatched_regex: + # Account for bbappend files for b in self.bbappends: (bbfile, append) = b - if regex.match(append): - # If the bbappend is matched by already "matched set", return False - for matched_regex in matched: - if matched_regex.match(append): - return False - return True - return False + seen.add(append) + + # Account for skipped recipes + seen.update(fns) - for unmatch in unmatched.copy(): - if find_bbappend_match(unmatch): - unmatched.remove(unmatch) + seen.difference_update(matched) + + def already_matched(fn): + for regex in matched_regex: + if regex.match(fn): + return True + return False + + for unmatch in unmatched_regex.copy(): + for fn in seen: + if unmatch.match(fn): + # If the bbappend or file was already matched by another regex, skip it + # e.g. for a layer within a layer, the outer regex could match, the inner + # regex may match nothing and we should warn about that + if already_matched(fn): + continue + unmatched_regex.remove(unmatch) + break for collection, pattern, regex, _ in self.bbfile_config_priorities: - if regex in unmatched: + if regex in unmatched_regex: if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': - collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', + collection, pattern)) return priorities @@ -1931,7 +1995,8 @@ class Parser(multiprocessing.Process): except queue.Empty: pass else: - self.results.cancel_join_thread() + self.results.close() + self.results.join_thread() break if pending: @@ -1940,6 +2005,8 @@ class Parser(multiprocessing.Process): try: job = self.jobs.pop() except IndexError: + self.results.close() + self.results.join_thread() break result = self.parse(*job) # Clear the siggen cache after parsing to control memory usage, its huge @@ -1949,7 +2016,7 @@ class Parser(multiprocessing.Process): except queue.Full: pending.append(result) - def parse(self, filename, appends): + def parse(self, mc, cache, filename, appends): try: origfilter = bb.event.LogHandler.filter # Record the filename we're parsing into any events generated @@ -1963,7 +2030,7 @@ class Parser(multiprocessing.Process): bb.event.set_class_handlers(self.handlers.copy()) bb.event.LogHandler.filter = parse_filter - return True, self.bb_cache.parse(filename, appends) + return True, mc, cache.parse(filename, appends) except Exception as exc: tb = sys.exc_info()[2] exc.recipe = filename @@ -1978,8 +2045,8 @@ class Parser(multiprocessing.Process): bb.event.LogHandler.filter = origfilter class CookerParser(object): - def __init__(self, cooker, filelist, masked): - self.filelist = filelist + def __init__(self, cooker, mcfilelist, masked): + self.mcfilelist = mcfilelist self.cooker = cooker self.cfgdata = cooker.data self.cfghash = cooker.data_hash @@ -1993,28 +2060,31 @@ class CookerParser(object): self.skipped = 0 self.virtuals = 0 - self.total = len(filelist) self.current = 0 self.process_names = [] - self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) - self.fromcache = [] - self.willparse = [] - for filename in self.filelist: - appends = self.cooker.collection.get_file_appends(filename) - if not self.bb_cache.cacheValid(filename, appends): - self.willparse.append((filename, appends)) - else: - self.fromcache.append((filename, appends)) - self.toparse = self.total - len(self.fromcache) + self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array) + self.fromcache = set() + self.willparse = set() + for mc in self.cooker.multiconfigs: + for filename in self.mcfilelist[mc]: + appends = self.cooker.collections[mc].get_file_appends(filename) + if not self.bb_caches[mc].cacheValid(filename, appends): + self.willparse.add((mc, self.bb_caches[mc], filename, appends)) + else: + self.fromcache.add((mc, self.bb_caches[mc], filename, appends)) + + self.total = len(self.fromcache) + len(self.willparse) + self.toparse = len(self.willparse) self.progress_chunk = int(max(self.toparse / 100, 1)) self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or - multiprocessing.cpu_count()), len(self.willparse)) + multiprocessing.cpu_count()), self.toparse) self.start() self.haveshutdown = False + self.syncthread = None def start(self): self.results = self.load_cached() @@ -2022,7 +2092,9 @@ class CookerParser(object): if self.toparse: bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) def init(): - Parser.bb_cache = self.bb_cache + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_IGN) bb.utils.set_process_name(multiprocessing.current_process().name) multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) @@ -2032,7 +2104,7 @@ class CookerParser(object): def chunkify(lst,n): return [lst[i::n] for i in range(n)] - self.jobs = chunkify(self.willparse, self.num_processes) + self.jobs = chunkify(list(self.willparse), self.num_processes) for i in range(0, self.num_processes): parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) @@ -2056,12 +2128,9 @@ class CookerParser(object): self.total) bb.event.fire(event, self.cfgdata) - for process in self.processes: - self.parser_quit.put(None) - else: - self.parser_quit.cancel_join_thread() - for process in self.processes: - self.parser_quit.put(None) + + for process in self.processes: + self.parser_quit.put(None) # Cleanup the queue before call process.join(), otherwise there might be # deadlocks. @@ -2078,9 +2147,17 @@ class CookerParser(object): else: process.join() - sync = threading.Thread(target=self.bb_cache.sync) + self.parser_quit.close() + # Allow data left in the cancel queue to be discarded + self.parser_quit.cancel_join_thread() + + def sync_caches(): + for c in self.bb_caches.values(): + c.sync() + + sync = threading.Thread(target=sync_caches, name="SyncThread") + self.syncthread = sync sync.start() - multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) bb.codeparser.parser_cache_savemerge() bb.fetch.fetcher_parse_done() if self.cooker.configuration.profile: @@ -2094,10 +2171,14 @@ class CookerParser(object): bb.utils.process_profilelog(profiles, pout = pout) print("Processed parsing statistics saved to %s" % (pout)) + def final_cleanup(self): + if self.syncthread: + self.syncthread.join() + def load_cached(self): - for filename, appends in self.fromcache: - cached, infos = self.bb_cache.load(filename, appends) - yield not cached, infos + for mc, cache, filename, appends in self.fromcache: + cached, infos = cache.load(filename, appends) + yield not cached, mc, infos def parse_generator(self): while True: @@ -2119,7 +2200,7 @@ class CookerParser(object): result = [] parsed = None try: - parsed, result = next(self.results) + parsed, mc, result = next(self.results) except StopIteration: self.shutdown() return False @@ -2175,13 +2256,16 @@ class CookerParser(object): if info_array[0].skipped: self.skipped += 1 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) - (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn) - self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc], + self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], parsed=parsed, watcher = self.cooker.add_filewatch) return True def reparse(self, filename): - infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) - for vfn, info_array in infos: - (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) - self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) + to_reparse = set() + for mc in self.cooker.multiconfigs: + to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) + + for mc, filename, appends in to_reparse: + infos = self.bb_caches[mc].parse(filename, appends) + for vfn, info_array in infos: + self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) diff --git a/poky/bitbake/lib/bb/cookerdata.py b/poky/bitbake/lib/bb/cookerdata.py index 472423fdc..91cc4347f 100644 --- a/poky/bitbake/lib/bb/cookerdata.py +++ b/poky/bitbake/lib/bb/cookerdata.py @@ -58,11 +58,18 @@ class ConfigParameters(object): def updateToServer(self, server, environment): options = {} for o in ["abort", "force", "invalidate_stamp", - "verbose", "debug", "dry_run", "dump_signatures", - "debug_domains", "extra_assume_provided", "profile", - "prefile", "postfile", "server_timeout"]: + "dry_run", "dump_signatures", + "extra_assume_provided", "profile", + "prefile", "postfile", "server_timeout", + "nosetscene", "setsceneonly", "skipsetscene", + "runall", "runonly", "writeeventlog"]: options[o] = getattr(self.options, o) + options['build_verbose_shell'] = self.options.verbose + options['build_verbose_stdout'] = self.options.verbose + options['default_loglevel'] = bb.msg.loggerDefaultLogLevel + options['debug_domains'] = bb.msg.loggerDefaultDomains + ret, error = server.runCommand(["updateConfig", options, environment, sys.argv]) if error: raise Exception("Unable to update the server configuration with local parameters: %s" % error) @@ -111,11 +118,11 @@ class CookerConfiguration(object): """ def __init__(self): - self.debug_domains = [] + self.debug_domains = bb.msg.loggerDefaultDomains + self.default_loglevel = bb.msg.loggerDefaultLogLevel self.extra_assume_provided = [] self.prefile = [] self.postfile = [] - self.debug = 0 self.cmd = None self.abort = True self.force = False @@ -125,34 +132,21 @@ class CookerConfiguration(object): self.skipsetscene = False self.invalidate_stamp = False self.dump_signatures = [] + self.build_verbose_shell = False + self.build_verbose_stdout = False self.dry_run = False self.tracking = False - self.xmlrpcinterface = [] - self.server_timeout = None self.writeeventlog = False - self.server_only = False self.limited_deps = False self.runall = [] self.runonly = [] self.env = {} - def setConfigParameters(self, parameters): - for key in self.__dict__.keys(): - if key in parameters.options.__dict__: - setattr(self, key, parameters.options.__dict__[key]) - self.env = parameters.environment.copy() - - def setServerRegIdleCallback(self, srcb): - self.server_register_idlecallback = srcb - def __getstate__(self): state = {} for key in self.__dict__.keys(): - if key == "server_register_idlecallback": - state[key] = None - else: - state[key] = getattr(self, key) + state[key] = getattr(self, key) return state def __setstate__(self,state): @@ -170,7 +164,7 @@ def catch_parse_error(func): import traceback parselog.critical(traceback.format_exc()) parselog.critical("Unable to parse %s: %s" % (fn, exc)) - sys.exit(1) + raise bb.BBHandledException() except bb.data_smart.ExpansionError as exc: import traceback @@ -182,10 +176,10 @@ def catch_parse_error(func): if not fn.startswith(bbdir): break parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb)) - sys.exit(1) + raise bb.BBHandledException() except bb.parse.ParseError as exc: parselog.critical(str(exc)) - sys.exit(1) + raise bb.BBHandledException() return wrapped @catch_parse_error @@ -306,13 +300,13 @@ class CookerDataBuilder(object): self.data_hash = data_hash.hexdigest() except (SyntaxError, bb.BBHandledException): - raise bb.BBHandledException + raise bb.BBHandledException() except bb.data_smart.ExpansionError as e: logger.error(str(e)) - raise bb.BBHandledException + raise bb.BBHandledException() except Exception: logger.exception("Error parsing configuration files") - raise bb.BBHandledException + raise bb.BBHandledException() # Create a copy so we can reset at a later date when UIs disconnect self.origdata = self.data @@ -361,7 +355,7 @@ class CookerDataBuilder(object): for layer in broken_layers: parselog.critical(" %s", layer) parselog.critical("Please check BBLAYERS in %s" % (layerconf)) - sys.exit(1) + raise bb.BBHandledException() for layer in layers: parselog.debug(2, "Adding layer %s", layer) @@ -387,10 +381,13 @@ class CookerDataBuilder(object): invalid.append(entry) continue l, f = parts - if l in collections: + invert = l[0] == "!" + if invert: + l = l[1:] + if (l in collections and not invert) or (l not in collections and invert): data.appendVar("BBFILES", " " + f) if invalid: - bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) + bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split()) collections_tmp = collections[:] @@ -430,7 +427,7 @@ class CookerDataBuilder(object): handlerfn = data.getVarFlag(var, "filename", False) if not handlerfn: parselog.critical("Undefined event handler function '%s'" % var) - sys.exit(1) + raise bb.BBHandledException() handlerln = int(data.getVarFlag(var, "lineno", False)) bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln) diff --git a/poky/bitbake/lib/bb/daemonize.py b/poky/bitbake/lib/bb/daemonize.py index f01e6ec7c..c187fcfc6 100644 --- a/poky/bitbake/lib/bb/daemonize.py +++ b/poky/bitbake/lib/bb/daemonize.py @@ -14,6 +14,8 @@ import sys import io import traceback +import bb + def createDaemon(function, logfile): """ Detach a process from the controlling terminal and run it in the diff --git a/poky/bitbake/lib/bb/data.py b/poky/bitbake/lib/bb/data.py index b0683c518..97022853c 100644 --- a/poky/bitbake/lib/bb/data.py +++ b/poky/bitbake/lib/bb/data.py @@ -161,6 +161,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False): return True if func: + # Write a comment indicating where the shell function came from (line number and filename) to make it easier + # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata + # location of shell functions. + o.write("# line: {0}, file: {1}\n".format( + d.getVarFlag(var, "lineno", False), + d.getVarFlag(var, "filename", False))) # NOTE: should probably check for unbalanced {} within the var val = val.rstrip('\n') o.write("%s() {\n%s\n}\n" % (varExpanded, val)) diff --git a/poky/bitbake/lib/bb/data_smart.py b/poky/bitbake/lib/bb/data_smart.py index 7f1b6dcb4..c559102cf 100644 --- a/poky/bitbake/lib/bb/data_smart.py +++ b/poky/bitbake/lib/bb/data_smart.py @@ -189,7 +189,7 @@ class IncludeHistory(object): if self.current.parent: self.current = self.current.parent else: - bb.warn("Include log: Tried to finish '%s' at top level." % filename) + bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) return False def emit(self, o, level = 0): diff --git a/poky/bitbake/lib/bb/event.py b/poky/bitbake/lib/bb/event.py index 0e6d9b296..694b47052 100644 --- a/poky/bitbake/lib/bb/event.py +++ b/poky/bitbake/lib/bb/event.py @@ -10,17 +10,17 @@ BitBake build tools. # SPDX-License-Identifier: GPL-2.0-only # -import sys -import pickle -import logging -import atexit -import traceback import ast +import atexit +import collections +import logging +import pickle +import sys import threading +import traceback -import bb.utils -import bb.compat import bb.exceptions +import bb.utils # This is the pid for which we should generate the event. This is set when # the runqueue forks off. @@ -56,7 +56,7 @@ def set_class_handlers(h): _handlers = h def clean_class_handlers(): - return bb.compat.OrderedDict() + return collections.OrderedDict() # Internal _handlers = clean_class_handlers() diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index eb112f069..551bfb70f 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1195,8 +1195,6 @@ def get_checksum_file_list(d): paths = ud.method.localpaths(ud, d) for f in paths: pth = ud.decodedurl - if '*' in pth: - f = os.path.join(os.path.abspath(f), pth) if f.startswith(dl_dir): # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else if os.path.exists(f): @@ -1365,9 +1363,6 @@ class FetchMethod(object): # We cannot compute checksums for directories if os.path.isdir(urldata.localpath): return False - if urldata.localpath.find("*") != -1: - return False - return True def recommends_checksum(self, urldata): @@ -1430,11 +1425,6 @@ class FetchMethod(object): iterate = False file = urldata.localpath - # Localpath can't deal with 'dir/*' entries, so it converts them to '.', - # but it must be corrected back for local files copying - if urldata.basename == '*' and file.endswith('/.'): - file = '%s/%s' % (file.rstrip('/.'), urldata.path) - try: unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) except ValueError as exc: @@ -1530,7 +1520,7 @@ class FetchMethod(object): if urlpath.find("/") != -1: destdir = urlpath.rsplit("/", 1)[0] + '/' bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) - cmd = 'cp -fpPRH %s %s' % (file, destdir) + cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) if not cmd: return @@ -1613,10 +1603,15 @@ class FetchMethod(object): """ if os.path.exists(ud.localpath): return True - if ud.localpath.find("*") != -1: - return True return False + def implicit_urldata(self, ud, d): + """ + Get a list of FetchData objects for any implicit URLs that will also + be downloaded when we fetch the given URL. + """ + return [] + class Fetch(object): def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): if localonly and cache: @@ -1842,6 +1837,24 @@ class Fetch(object): if ud.lockfile: bb.utils.unlockfile(lf) + def expanded_urldata(self, urls=None): + """ + Get an expanded list of FetchData objects covering both the given + URLS and any additional implicit URLs that are added automatically by + the appropriate FetchMethod. + """ + + if not urls: + urls = self.urls + + urldata = [] + for url in urls: + ud = self.ud[url] + urldata.append(ud) + urldata += ud.method.implicit_urldata(ud, self.d) + + return urldata + class FetchConnectionCache(object): """ A class which represents an container for socket connections. diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 5b3793a70..07064c694 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -236,7 +236,7 @@ class Git(FetchMethod): ud.unresolvedrev[name] = ud.revisions[name] ud.revisions[name] = self.latest_revision(ud, d, name) - gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.')) + gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) if gitsrcname.startswith('.'): gitsrcname = gitsrcname[1:] @@ -342,7 +342,7 @@ class Git(FetchMethod): # We do this since git will use a "-l" option automatically for local urls where possible if repourl.startswith("file://"): repourl = repourl[7:] - clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir) + clone_cmd = "LANG=C %s clone --bare --mirror \"%s\" %s --progress" % (ud.basecmd, repourl, ud.clonedir) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, clone_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -354,8 +354,8 @@ class Git(FetchMethod): if "origin" in output: runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + runfetchcmd("%s remote add --mirror=fetch origin \"%s\"" % (ud.basecmd, repourl), d, workdir=ud.clonedir) + fetch_cmd = "LANG=C %s fetch -f --progress \"%s\" refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -475,6 +475,9 @@ class Git(FetchMethod): need_lfs = ud.parm.get("lfs", "1") == "1" + if not need_lfs: + ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd + source_found = False source_error = [] @@ -501,12 +504,12 @@ class Git(FetchMethod): raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) repourl = self._get_repo_url(ud) - runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) + runfetchcmd("%s remote set-url origin \"%s\"" % (ud.basecmd, repourl), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if need_lfs and not self._find_git_lfs(d): raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) - else: + elif not need_lfs: bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) if not ud.nocheckout: @@ -563,8 +566,15 @@ class Git(FetchMethod): """ Check if the repository has 'lfs' (large file) content """ - cmd = "%s grep lfs HEAD:.gitattributes | wc -l" % ( - ud.basecmd) + + if not ud.nobranch: + branchname = ud.branches[ud.names[0]] + else: + branchname = "master" + + cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( + ud.basecmd, ud.branches[ud.names[0]]) + try: output = runfetchcmd(cmd, d, quiet=True, workdir=wd) if int(output) > 0: @@ -613,7 +623,7 @@ class Git(FetchMethod): d.setVar('_BB_GIT_IN_LSREMOTE', '1') try: repourl = self._get_repo_url(ud) - cmd = "%s ls-remote %s %s" % \ + cmd = "%s ls-remote \"%s\" %s" % \ (ud.basecmd, repourl, search) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, cmd, repourl) diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index e7083001d..d6e5c5c05 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -143,12 +143,43 @@ class GitSM(Git): try: # Check for the nugget dropped by the download operation known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ - (ud.basecmd), d, workdir=ud.clonedir) + (ud.basecmd), d, workdir=ud.clonedir) - if ud.revisions[ud.names[0]] not in known_srcrevs.split(): - return True + if ud.revisions[ud.names[0]] in known_srcrevs.split(): + return False except bb.fetch2.FetchError: - # No srcrev nuggets, so this is new and needs to be updated + pass + + need_update_list = [] + def need_update_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + + try: + newfetch = Fetch([url], d, cache=False) + new_ud = newfetch.ud[url] + if new_ud.method.need_update(new_ud, d): + need_update_list.append(modpath) + except Exception as e: + logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) + need_update_result = True + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) + self.process_submodules(ud, tmpdir, need_update_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, need_update_submodule, d) + if len(need_update_list) == 0: + # We already have the required commits of all submodules. Drop + # a nugget so we don't need to check again. + runfetchcmd("%s config --add bitbake.srcrev %s" % \ + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) + + if len(need_update_list) > 0: + logger.debug(1, 'gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) return True return False @@ -163,9 +194,6 @@ class GitSM(Git): try: newfetch = Fetch([url], d, cache=False) newfetch.download() - # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir) except Exception as e: logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) raise @@ -181,6 +209,9 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, download_submodule, d) + # Drop a nugget for the srcrev we've fetched (used by need_update) + runfetchcmd("%s config --add bitbake.srcrev %s" % \ + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) def unpack(self, ud, destdir, d): def unpack_submodules(ud, url, module, modpath, workdir, d): @@ -223,3 +254,24 @@ class GitSM(Git): # up the configuration and checks out the files. The main project config should remain # unmodified, and no download from the internet should occur. runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + + def implicit_urldata(self, ud, d): + import shutil, subprocess, tempfile + + urldata = [] + def add_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + newfetch = Fetch([url], d, cache=False) + urldata.extend(newfetch.expanded_urldata()) + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) + self.process_submodules(ud, tmpdir, add_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, add_submodule, d) + + return urldata diff --git a/poky/bitbake/lib/bb/fetch2/local.py b/poky/bitbake/lib/bb/fetch2/local.py index 01d9ff9f8..25d4557db 100644 --- a/poky/bitbake/lib/bb/fetch2/local.py +++ b/poky/bitbake/lib/bb/fetch2/local.py @@ -17,7 +17,7 @@ import os import urllib.request, urllib.parse, urllib.error import bb import bb.utils -from bb.fetch2 import FetchMethod, FetchError +from bb.fetch2 import FetchMethod, FetchError, ParameterError from bb.fetch2 import logger class Local(FetchMethod): @@ -33,6 +33,8 @@ class Local(FetchMethod): ud.basename = os.path.basename(ud.decodedurl) ud.basepath = ud.decodedurl ud.needdonestamp = False + if "*" in ud.decodedurl: + raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url) return def localpath(self, urldata, d): @@ -55,12 +57,6 @@ class Local(FetchMethod): logger.debug(2, "Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) newpath, hist = bb.utils.which(filespath, path, history=True) searched.extend(hist) - if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1: - # For expressions using '*', best we can do is take the first directory in FILESPATH that exists - newpath, hist = bb.utils.which(filespath, ".", history=True) - searched.extend(hist) - logger.debug(2, "Searching for %s in path: %s" % (path, newpath)) - return searched if not os.path.exists(newpath): dldirfile = os.path.join(d.getVar("DL_DIR"), path) logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path)) @@ -70,8 +66,6 @@ class Local(FetchMethod): return searched def need_update(self, ud, d): - if ud.url.find("*") != -1: - return False if os.path.exists(ud.localpath): return False return True @@ -95,9 +89,6 @@ class Local(FetchMethod): """ Check the status of the url """ - if urldata.localpath.find("*") != -1: - logger.info("URL %s looks like a glob and was therefore not checked.", urldata.url) - return True if os.path.exists(urldata.localpath): return True return False diff --git a/poky/bitbake/lib/bb/fetch2/osc.py b/poky/bitbake/lib/bb/fetch2/osc.py index 8f091efd0..3a6cd2951 100644 --- a/poky/bitbake/lib/bb/fetch2/osc.py +++ b/poky/bitbake/lib/bb/fetch2/osc.py @@ -8,12 +8,15 @@ Based on the svn "Fetch" implementation. """ import logging +import os import bb from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError from bb.fetch2 import MissingParameterError from bb.fetch2 import runfetchcmd +logger = logging.getLogger(__name__) + class Osc(FetchMethod): """Class to fetch a module or modules from Opensuse build server repositories.""" diff --git a/poky/bitbake/lib/bb/fetch2/perforce.py b/poky/bitbake/lib/bb/fetch2/perforce.py index f57c2a4f5..6f3c95b6c 100644 --- a/poky/bitbake/lib/bb/fetch2/perforce.py +++ b/poky/bitbake/lib/bb/fetch2/perforce.py @@ -1,6 +1,20 @@ """ BitBake 'Fetch' implementation for perforce +Supported SRC_URI options are: + +- module + The top-level location to fetch while preserving the remote paths + + The value of module can point to either a directory or a file. The result, + in both cases, is that the fetcher will preserve all file paths starting + from the module path. That is, the top-level directory in the module value + will also be the top-level directory in P4DIR. + +- remotepath + If the value "keep" is given, the full depot location of each file is + preserved in P4DIR. This option overrides the effect of the module option. + """ # Copyright (C) 2003, 2004 Chris Larson @@ -17,6 +31,36 @@ from bb.fetch2 import FetchError from bb.fetch2 import logger from bb.fetch2 import runfetchcmd +class PerforceProgressHandler (bb.progress.BasicProgressHandler): + """ + Implements basic progress information for perforce, based on the number of + files to be downloaded. + + The p4 print command will print one line per file, therefore it can be used + to "count" the number of files already completed and give an indication of + the progress. + """ + def __init__(self, d, num_files): + self._num_files = num_files + self._count = 0 + super(PerforceProgressHandler, self).__init__(d) + + # Send an initial progress event so the bar gets shown + self._fire_progress(-1) + + def write(self, string): + self._count = self._count + 1 + + percent = int(100.0 * float(self._count) / float(self._num_files)) + + # In case something goes wrong, we try to preserve our sanity + if percent > 100: + percent = 100 + + self.update(percent) + + super(PerforceProgressHandler, self).write(string) + class Perforce(FetchMethod): """ Class to fetch from perforce repositories """ def supports(self, ud, d): @@ -58,14 +102,33 @@ class Perforce(FetchMethod): logger.debug(1, 'Determined P4PORT to be: %s' % ud.host) if not ud.host: raise FetchError('Could not determine P4PORT from P4CONFIG') - + + # Fetcher options + ud.module = ud.parm.get('module') + ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep') + if ud.path.find('/...') >= 0: ud.pathisdir = True else: ud.pathisdir = False + # Avoid using the "/..." syntax in SRC_URI when a module value is given + if ud.pathisdir and ud.module: + raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given') + cleanedpath = ud.path.replace('/...', '').replace('/', '.') cleanedhost = ud.host.replace(':', '.') + + # Merge the path and module into the final depot location + if ud.module: + if ud.module.find('/') == 0: + raise FetchError('module cannot begin with /') + ud.path = os.path.join(ud.path, ud.module) + + # Append the module path to the local pkg name + cleanedmodule = ud.module.replace('/...', '').replace('/', '.') + cleanedpath += '--%s' % cleanedmodule + ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath) ud.setup_revisions(d) @@ -95,10 +158,20 @@ class Perforce(FetchMethod): pathnrev = '%s' % (ud.path) if depot_filename: - if ud.pathisdir: # Remove leading path to obtain filename + if ud.keepremotepath: + # preserve everything, remove the leading // + filename = depot_filename.lstrip('/') + elif ud.module: + # remove everything up to the module path + modulepath = ud.module.rstrip('/...') + filename = depot_filename[depot_filename.rfind(modulepath):] + elif ud.pathisdir: + # Remove leading (visible) path to obtain the filepath filename = depot_filename[len(ud.path)-1:] else: + # Remove everything, except the filename filename = depot_filename[depot_filename.rfind('/'):] + filename = filename[:filename.find('#')] # Remove trailing '#rev' if command == 'changes': @@ -150,10 +223,12 @@ class Perforce(FetchMethod): bb.utils.remove(ud.pkgdir, True) bb.utils.mkdirhier(ud.pkgdir) + progresshandler = PerforceProgressHandler(d, len(filelist)) + for afile in filelist: p4fetchcmd = self._buildp4command(ud, d, 'print', afile) bb.fetch2.check_network_access(d, p4fetchcmd, ud.url) - runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir) + runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler) runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir) diff --git a/poky/bitbake/lib/bb/fetch2/ssh.py b/poky/bitbake/lib/bb/fetch2/ssh.py index 5e982ecf3..2c8557e1f 100644 --- a/poky/bitbake/lib/bb/fetch2/ssh.py +++ b/poky/bitbake/lib/bb/fetch2/ssh.py @@ -31,8 +31,7 @@ IETF secsh internet draft: # import re, os -from bb.fetch2 import FetchMethod -from bb.fetch2 import runfetchcmd +from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd __pattern__ = re.compile(r''' @@ -65,7 +64,7 @@ class SSH(FetchMethod): def urldata_init(self, urldata, d): if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git': - raise bb.fetch2.ParameterError( + raise ParameterError( "Invalid protocol - if you wish to fetch from a git " + "repository using ssh, you need to use " + "git:// prefix with protocol=ssh", urldata.url) @@ -105,7 +104,7 @@ class SSH(FetchMethod): dldir ) - bb.fetch2.check_network_access(d, cmd, urldata.url) + check_network_access(d, cmd, urldata.url) runfetchcmd(cmd, d) diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index f7d1de26b..e6d9f528d 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -208,10 +208,7 @@ class Wget(FetchMethod): fetch.connection_cache.remove_connection(h.host, h.port) raise urllib.error.URLError(err) else: - try: - r = h.getresponse(buffering=True) - except TypeError: # buffering kw not supported - r = h.getresponse() + r = h.getresponse() # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. diff --git a/poky/bitbake/lib/bb/main.py b/poky/bitbake/lib/bb/main.py index af2880f8d..7990195ea 100755 --- a/poky/bitbake/lib/bb/main.py +++ b/poky/bitbake/lib/bb/main.py @@ -344,8 +344,6 @@ def bitbake_main(configParams, configuration): except: pass - configuration.setConfigParameters(configParams) - if configParams.server_only and configParams.remote_server: raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ @@ -357,13 +355,13 @@ def bitbake_main(configParams, configuration): if "BBDEBUG" in os.environ: level = int(os.environ["BBDEBUG"]) - if level > configuration.debug: - configuration.debug = level + if level > configParams.debug: + configParams.debug = level - bb.msg.init_msgconfig(configParams.verbose, configuration.debug, - configuration.debug_domains) + bb.msg.init_msgconfig(configParams.verbose, configParams.debug, + configParams.debug_domains) - server_connection, ui_module = setup_bitbake(configParams, configuration) + server_connection, ui_module = setup_bitbake(configParams) # No server connection if server_connection is None: if configParams.status_only: @@ -390,7 +388,7 @@ def bitbake_main(configParams, configuration): return 1 -def setup_bitbake(configParams, configuration, extrafeatures=None): +def setup_bitbake(configParams, extrafeatures=None): # Ensure logging messages get sent to the UI as events handler = bb.event.LogHandler() if not configParams.status_only: @@ -431,11 +429,11 @@ def setup_bitbake(configParams, configuration, extrafeatures=None): logger.info("bitbake server is not running.") lock.close() return None, None - # we start a server with a given configuration + # we start a server with a given featureset logger.info("Starting bitbake server...") # Clear the event queue since we already displayed messages bb.event.ui_queue = [] - server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) + server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface) else: logger.info("Reconnecting to bitbake server...") diff --git a/poky/bitbake/lib/bb/msg.py b/poky/bitbake/lib/bb/msg.py index c0b344e32..6f17b6acc 100644 --- a/poky/bitbake/lib/bb/msg.py +++ b/poky/bitbake/lib/bb/msg.py @@ -14,6 +14,7 @@ import sys import copy import logging import logging.config +import os from itertools import groupby import bb import bb.event @@ -146,18 +147,12 @@ class LogFilterLTLevel(logging.Filter): # loggerDefaultLogLevel = BBLogFormatter.NOTE -loggerDefaultVerbose = False -loggerVerboseLogs = False loggerDefaultDomains = {} def init_msgconfig(verbose, debug, debug_domains=None): """ Set default verbosity and debug levels config the logger """ - bb.msg.loggerDefaultVerbose = verbose - if verbose: - bb.msg.loggerVerboseLogs = True - if debug: bb.msg.loggerDefaultLogLevel = BBLogFormatter.DEBUG - debug + 1 elif verbose: @@ -280,7 +275,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None): logconfig = copy.deepcopy(defaultconfig) if userconfigfile: - with open(userconfigfile, 'r') as f: + with open(os.path.normpath(userconfigfile), 'r') as f: if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'): import yaml userconfig = yaml.load(f) diff --git a/poky/bitbake/lib/bb/namedtuple_with_abc.py b/poky/bitbake/lib/bb/namedtuple_with_abc.py index 646aed6ff..e46dbf084 100644 --- a/poky/bitbake/lib/bb/namedtuple_with_abc.py +++ b/poky/bitbake/lib/bb/namedtuple_with_abc.py @@ -61,17 +61,9 @@ class _NamedTupleABCMeta(ABCMeta): return ABCMeta.__new__(mcls, name, bases, namespace) -exec( - # Python 2.x metaclass declaration syntax - """class _NamedTupleABC(object): - '''The abstract base class + mix-in for named tuples.''' - __metaclass__ = _NamedTupleABCMeta - _fields = abstractproperty()""" if version_info[0] < 3 else - # Python 3.x metaclass declaration syntax - """class _NamedTupleABC(metaclass=_NamedTupleABCMeta): - '''The abstract base class + mix-in for named tuples.''' - _fields = abstractproperty()""" -) +class _NamedTupleABC(metaclass=_NamedTupleABCMeta): + '''The abstract base class + mix-in for named tuples.''' + _fields = abstractproperty() _namedtuple.abc = _NamedTupleABC diff --git a/poky/bitbake/lib/bb/parse/ast.py b/poky/bitbake/lib/bb/parse/ast.py index 785aa974e..0714296af 100644 --- a/poky/bitbake/lib/bb/parse/ast.py +++ b/poky/bitbake/lib/bb/parse/ast.py @@ -244,12 +244,14 @@ class AddTaskNode(AstNode): bb.build.addtask(self.func, self.before, self.after, data) class DelTaskNode(AstNode): - def __init__(self, filename, lineno, func): + def __init__(self, filename, lineno, tasks): AstNode.__init__(self, filename, lineno) - self.func = func + self.tasks = tasks def eval(self, data): - bb.build.deltask(self.func, data) + tasks = data.expand(self.tasks).split() + for task in tasks: + bb.build.deltask(task, data) class BBHandlerNode(AstNode): def __init__(self, filename, lineno, fns): @@ -305,7 +307,7 @@ def handleAddTask(statements, filename, lineno, m): statements.append(AddTaskNode(filename, lineno, func, before, after)) def handleDelTask(statements, filename, lineno, m): - func = m.group("func") + func = m.group(1) if func is None: return diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index 6e216effb..215f940b6 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -26,7 +26,7 @@ __func_start_regexp__ = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*( __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) __addtask_regexp__ = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*") -__deltask_regexp__ = re.compile(r"deltask\s+(?P<func>\w+)(?P<ignores>.*)") +__deltask_regexp__ = re.compile(r"deltask\s+(.+)") __addhandler_regexp__ = re.compile(r"addhandler\s+(.+)" ) __def_regexp__ = re.compile(r"def\s+(\w+).*:" ) __python_func_regexp__ = re.compile(r"(\s+.*)|(^$)|(^#)" ) @@ -238,9 +238,6 @@ def feeder(lineno, s, fn, root, statements, eof=False): m = __deltask_regexp__.match(s) if m: - # Check and warn "for deltask task1 task2" - if m.group('ignores'): - logger.warning('deltask ignored: "%s"' % m.group('ignores')) ast.handleDelTask(statements, fn, lineno, m) return diff --git a/poky/bitbake/lib/bb/persist_data.py b/poky/bitbake/lib/bb/persist_data.py index 7357ab2d4..5f4fbe350 100644 --- a/poky/bitbake/lib/bb/persist_data.py +++ b/poky/bitbake/lib/bb/persist_data.py @@ -12,14 +12,14 @@ currently, providing a key/value store accessed by 'domain'. # import collections +import contextlib +import functools import logging import os.path +import sqlite3 import sys import warnings -from bb.compat import total_ordering from collections import Mapping -import sqlite3 -import contextlib sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): @@ -28,7 +28,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): logger = logging.getLogger("BitBake.PersistData") -@total_ordering +@functools.total_ordering class SQLTable(collections.MutableMapping): class _Decorators(object): @staticmethod diff --git a/poky/bitbake/lib/bb/process.py b/poky/bitbake/lib/bb/process.py index 2dc472a86..7c3995cce 100644 --- a/poky/bitbake/lib/bb/process.py +++ b/poky/bitbake/lib/bb/process.py @@ -7,6 +7,7 @@ import signal import subprocess import errno import select +import bb logger = logging.getLogger('BitBake.Process') @@ -41,6 +42,7 @@ class ExecutionError(CmdError): self.exitcode = exitcode self.stdout = stdout self.stderr = stderr + self.extra_message = None def __str__(self): message = "" @@ -51,7 +53,7 @@ class ExecutionError(CmdError): if message: message = ":\n" + message return (CmdError.__str__(self) + - " with exit code %s" % self.exitcode + message) + " with exit code %s" % self.exitcode + message + (self.extra_message or "")) class Popen(subprocess.Popen): defaults = { diff --git a/poky/bitbake/lib/bb/progress.py b/poky/bitbake/lib/bb/progress.py index 9c755b7f7..d051ba019 100644 --- a/poky/bitbake/lib/bb/progress.py +++ b/poky/bitbake/lib/bb/progress.py @@ -14,7 +14,27 @@ import bb.event import bb.build from bb.build import StdoutNoopContextManager -class ProgressHandler(object): + +# from https://stackoverflow.com/a/14693789/221061 +ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') + + +def filter_color(string): + """ + Filter ANSI escape codes out of |string|, return new string + """ + return ANSI_ESCAPE_REGEX.sub('', string) + + +def filter_color_n(string): + """ + Filter ANSI escape codes out of |string|, returns tuple of + (new string, # of ANSI codes removed) + """ + return ANSI_ESCAPE_REGEX.subn('', string) + + +class ProgressHandler: """ Base class that can pretend to be a file object well enough to be used to build objects to intercept console output and determine the @@ -55,6 +75,7 @@ class ProgressHandler(object): self._lastevent = ts self._progress = progress + class LineFilterProgressHandler(ProgressHandler): """ A ProgressHandler variant that provides the ability to filter out @@ -66,7 +87,7 @@ class LineFilterProgressHandler(ProgressHandler): """ def __init__(self, d, outfile=None): self._linebuffer = '' - super(LineFilterProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) def write(self, string): self._linebuffer += string @@ -80,41 +101,44 @@ class LineFilterProgressHandler(ProgressHandler): lbreakpos = line.rfind('\r') + 1 if lbreakpos: line = line[lbreakpos:] - if self.writeline(line): - super(LineFilterProgressHandler, self).write(line) + if self.writeline(filter_color(line)): + super().write(line) def writeline(self, line): return True + class BasicProgressHandler(ProgressHandler): def __init__(self, d, regex=r'(\d+)%', outfile=None): - super(BasicProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) self._regex = re.compile(regex) # Send an initial progress event so the bar gets shown self._fire_progress(0) def write(self, string): - percs = self._regex.findall(string) + percs = self._regex.findall(filter_color(string)) if percs: progress = int(percs[-1]) self.update(progress) - super(BasicProgressHandler, self).write(string) + super().write(string) + class OutOfProgressHandler(ProgressHandler): def __init__(self, d, regex, outfile=None): - super(OutOfProgressHandler, self).__init__(d, outfile) + super().__init__(d, outfile) self._regex = re.compile(regex) # Send an initial progress event so the bar gets shown self._fire_progress(0) def write(self, string): - nums = self._regex.findall(string) + nums = self._regex.findall(filter_color(string)) if nums: progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100 self.update(progress) - super(OutOfProgressHandler, self).write(string) + super().write(string) + -class MultiStageProgressReporter(object): +class MultiStageProgressReporter: """ Class which allows reporting progress without the caller having to know where they are in the overall sequence. Useful @@ -199,6 +223,7 @@ class MultiStageProgressReporter(object): value is considered to be out of stage_total, otherwise it should be a percentage value from 0 to 100. """ + progress = None if self._stage_total: stage_progress = (float(stage_progress) / self._stage_total) * 100 if self._stage < 0: @@ -207,9 +232,10 @@ class MultiStageProgressReporter(object): progress = self._base_progress + (stage_progress * self._stage_weights[self._stage]) else: progress = self._base_progress - if progress > 100: - progress = 100 - self._fire_progress(progress) + if progress: + if progress > 100: + progress = 100 + self._fire_progress(progress) def finish(self): if self._finished: @@ -230,6 +256,7 @@ class MultiStageProgressReporter(object): out.append('Up to finish: %d' % stage_weight) bb.warn('Stage times:\n %s' % '\n '.join(out)) + class MultiStageProcessProgressReporter(MultiStageProgressReporter): """ Version of MultiStageProgressReporter intended for use with @@ -238,7 +265,7 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter): def __init__(self, d, processname, stage_weights, debug=False): self._processname = processname self._started = False - MultiStageProgressReporter.__init__(self, d, stage_weights, debug) + super().__init__(d, stage_weights, debug) def start(self): if not self._started: @@ -255,13 +282,14 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter): MultiStageProgressReporter.finish(self) bb.event.fire(bb.event.ProcessFinished(self._processname), self._data) + class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter): """ MultiStageProcessProgressReporter that takes the calls and does nothing with them (to avoid a bunch of "if progress_reporter:" checks) """ def __init__(self): - MultiStageProcessProgressReporter.__init__(self, "", None, []) + super().__init__(None, []) def _fire_progress(self, taskprogress, rate=None): pass diff --git a/poky/bitbake/lib/bb/pysh/pyshyacc.py b/poky/bitbake/lib/bb/pysh/pyshyacc.py index de565dc9a..924860a6f 100644 --- a/poky/bitbake/lib/bb/pysh/pyshyacc.py +++ b/poky/bitbake/lib/bb/pysh/pyshyacc.py @@ -570,6 +570,7 @@ def p_linebreak(p): def p_separator_op(p): """separator_op : COMMA + | COMMA COMMA | AMP""" p[0] = p[1] diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py index 16f076f3b..28bdadb45 100644 --- a/poky/bitbake/lib/bb/runqueue.py +++ b/poky/bitbake/lib/bb/runqueue.py @@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_mc(n): + if n.startswith("mc:"): + _, mc, n = n.split(":", 2) + return (mc, n) + return ('', n) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':') @@ -370,7 +376,7 @@ class RunQueueData: self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or "" self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split() - self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData) + self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData, targets) self.setscenewhitelist_checked = False self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1") self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter() @@ -1184,8 +1190,9 @@ class RunQueueData: return len(self.runtaskentries) def prepare_task_hash(self, tid): - bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)]) - self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)]) + dc = bb.parse.siggen.get_data_caches(self.dataCaches, mc_from_tid(tid)) + bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, dc) + self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, dc) self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) def dump_data(self): @@ -1256,8 +1263,8 @@ class RunQueue: "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv, "sigdata" : bb.parse.siggen.get_taskdata(), "logdefaultlevel" : bb.msg.loggerDefaultLogLevel, - "logdefaultverbose" : bb.msg.loggerDefaultVerbose, - "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, + "build_verbose_shell" : self.cooker.configuration.build_verbose_shell, + "build_verbose_stdout" : self.cooker.configuration.build_verbose_stdout, "logdefaultdomain" : bb.msg.loggerDefaultDomains, "prhost" : self.cooker.prhost, "buildname" : self.cfgData.getVar("BUILDNAME"), @@ -1557,7 +1564,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options) @@ -2042,10 +2050,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) @@ -2129,10 +2137,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) @@ -2298,7 +2306,8 @@ class RunQueueExecute: if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): continue orighash = self.rqdata.runtaskentries[tid].hash - newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches[mc_from_tid(tid)]) + dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid)) + newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, dc) origuni = self.rqdata.runtaskentries[tid].unihash newuni = bb.parse.siggen.get_unihash(tid) # FIXME, need to check it can come from sstate at all for determinism? @@ -2958,7 +2967,12 @@ class runQueuePipe(): while index != -1 and self.queue.startswith(b"<event>"): try: event = pickle.loads(self.queue[7:index]) - except ValueError as e: + except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e: + if isinstance(e, pickle.UnpicklingError) and "truncated" in str(e): + # The pickled data could contain "</event>" so search for the next occurance + # unpickling again, this should be the only way an unpickle error could occur + index = self.queue.find(b"</event>", index + 1) + continue bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[7:index])) bb.event.fire_from_worker(event, self.d) if isinstance(event, taskUniHashUpdate): @@ -2970,7 +2984,7 @@ class runQueuePipe(): while index != -1 and self.queue.startswith(b"<exitcode>"): try: task, status = pickle.loads(self.queue[10:index]) - except ValueError as e: + except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e: bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index])) self.rqexec.runqueue_process_waitpid(task, status) found = True @@ -2985,16 +2999,15 @@ class runQueuePipe(): print("Warning, worker left partial message: %s" % self.queue) self.input.close() -def get_setscene_enforce_whitelist(d): +def get_setscene_enforce_whitelist(d, targets): if d.getVar('BB_SETSCENE_ENFORCE') != '1': return None whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split() outlist = [] for item in whitelist[:]: if item.startswith('%:'): - for target in sys.argv[1:]: - if not target.startswith('-'): - outlist.append(target.split(':')[0] + ':' + item.split(':')[1]) + for (mc, target, task, fn) in targets: + outlist.append(target + ':' + item.split(':')[1]) else: outlist.append(item) return outlist diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 69aae626e..c7cb34f0c 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -25,6 +25,7 @@ import subprocess import errno import re import datetime +import pickle import bb.server.xmlrpcserver from bb import daemonize from multiprocessing import queues @@ -34,12 +35,15 @@ logger = logging.getLogger('BitBake') class ProcessTimeout(SystemExit): pass -class ProcessServer(multiprocessing.Process): +def serverlog(msg): + print(str(os.getpid()) + " " + datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg) + sys.stdout.flush() + +class ProcessServer(): profile_filename = "profile.log" profile_processed_filename = "profile.log.processed" - def __init__(self, lock, sock, sockname): - multiprocessing.Process.__init__(self) + def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface): self.command_channel = False self.command_channel_reply = False self.quit = False @@ -47,16 +51,22 @@ class ProcessServer(multiprocessing.Process): self.next_heartbeat = time.time() self.event_handle = None + self.hadanyui = False self.haveui = False - self.lastui = False + self.maxuiwait = 30 self.xmlrpc = False self._idlefuns = {} self.bitbake_lock = lock + self.bitbake_lock_name = lockname self.sock = sock self.sockname = sockname + self.server_timeout = server_timeout + self.timeout = self.server_timeout + self.xmlrpcinterface = xmlrpcinterface + def register_idle_function(self, function, data): """Register a function to be called while the server is idle""" assert hasattr(function, '__call__') @@ -67,22 +77,7 @@ class ProcessServer(multiprocessing.Process): if self.xmlrpcinterface[0]: self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self) - print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port)) - - heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') - if heartbeat_event: - try: - self.heartbeat_seconds = float(heartbeat_event) - except: - bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) - - self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT') - try: - if self.timeout: - self.timeout = float(self.timeout) - except: - bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout) - + serverlog("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port)) try: self.bitbake_lock.seek(0) @@ -93,7 +88,7 @@ class ProcessServer(multiprocessing.Process): self.bitbake_lock.write("%s\n" % (os.getpid())) self.bitbake_lock.flush() except Exception as e: - print("Error writing to lock file: %s" % str(e)) + serverlog("Error writing to lock file: %s" % str(e)) pass if self.cooker.configuration.profile: @@ -107,7 +102,7 @@ class ProcessServer(multiprocessing.Process): prof.dump_stats("profile.log") bb.utils.process_profilelog("profile.log") - print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") + serverlog("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") else: ret = self.main() @@ -126,10 +121,11 @@ class ProcessServer(multiprocessing.Process): fds = [self.sock] if self.xmlrpc: fds.append(self.xmlrpc) - print("Entering server connection loop") + seendata = False + serverlog("Entering server connection loop") def disconnect_client(self, fds): - print("Disconnecting Client") + serverlog("Disconnecting Client") if self.controllersock: fds.remove(self.controllersock) self.controllersock.close() @@ -147,30 +143,31 @@ class ProcessServer(multiprocessing.Process): self.haveui = False ready = select.select(fds,[],[],0)[0] if newconnections: - print("Starting new client") + serverlog("Starting new client") conn = newconnections.pop(-1) fds.append(conn) self.controllersock = conn elif self.timeout is None and not ready: - print("No timeout, exiting.") + serverlog("No timeout, exiting.") self.quit = True + self.lastui = time.time() while not self.quit: if self.sock in ready: while select.select([self.sock],[],[],0)[0]: controllersock, address = self.sock.accept() if self.controllersock: - print("Queuing %s (%s)" % (str(ready), str(newconnections))) + serverlog("Queuing %s (%s)" % (str(ready), str(newconnections))) newconnections.append(controllersock) else: - print("Accepting %s (%s)" % (str(ready), str(newconnections))) + serverlog("Accepting %s (%s)" % (str(ready), str(newconnections))) self.controllersock = controllersock fds.append(controllersock) if self.controllersock in ready: try: - print("Processing Client") + serverlog("Processing Client") ui_fds = recvfds(self.controllersock, 3) - print("Connecting Client") + serverlog("Connecting Client") # Where to write events to writer = ConnectionWriter(ui_fds[0]) @@ -187,13 +184,21 @@ class ProcessServer(multiprocessing.Process): self.command_channel_reply = writer self.haveui = True + self.hadanyui = True except (EOFError, OSError): disconnect_client(self, fds) - if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \ + if not self.timeout == -1.0 and not self.haveui and self.timeout and \ (self.lastui + self.timeout) < time.time(): - print("Server timeout, exiting.") + serverlog("Server timeout, exiting.") + self.quit = True + + # If we don't see a UI connection within maxuiwait, its unlikely we're going to see + # one. We have had issue with processes hanging indefinitely so timing out UI-less + # servers is useful. + if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time(): + serverlog("No UI connection within max timeout, exiting to avoid infinite loop.") self.quit = True if self.command_channel in ready: @@ -208,22 +213,45 @@ class ProcessServer(multiprocessing.Process): self.quit = True continue try: - print("Running command %s" % command) + serverlog("Running command %s" % command) self.command_channel_reply.send(self.cooker.command.runCommand(command)) + serverlog("Command Completed") except Exception as e: logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e))) if self.xmlrpc in ready: self.xmlrpc.handle_requests() + if not seendata and hasattr(self.cooker, "data"): + heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT') + if heartbeat_event: + try: + self.heartbeat_seconds = float(heartbeat_event) + except: + bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event) + + self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT') + try: + if self.timeout: + self.timeout = float(self.timeout) + except: + bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout) + seendata = True + ready = self.idle_commands(.1, fds) - print("Exiting") + if len(threading.enumerate()) != 1: + serverlog("More than one thread left?: " + str(threading.enumerate())) + + serverlog("Exiting") # Remove the socket file so we don't get any more connections to avoid races - os.unlink(self.sockname) + try: + os.unlink(self.sockname) + except: + pass self.sock.close() - try: + try: self.cooker.shutdown(True) self.cooker.notifier.stop() self.cooker.confignotifier.stop() @@ -232,41 +260,73 @@ class ProcessServer(multiprocessing.Process): self.cooker.post_serve() + # Flush logs before we release the lock + sys.stdout.flush() + sys.stderr.flush() + # Finally release the lockfile but warn about other processes holding it open lock = self.bitbake_lock - lockfile = lock.name + lockfile = self.bitbake_lock_name + + def get_lock_contents(lockfile): + try: + with open(lockfile, "r") as f: + return f.readlines() + except FileNotFoundError: + return None + + lockcontents = get_lock_contents(lockfile) + serverlog("Original lockfile contents: " + str(lockcontents)) + lock.close() lock = None while not lock: - with bb.utils.timeout(3): - lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True) - if lock: - # We hold the lock so we can remove the file (hide stale pid data) - bb.utils.remove(lockfile) - bb.utils.unlockfile(lock) - return - + i = 0 + lock = None + while not lock and i < 30: + lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False) if not lock: - # Some systems may not have lsof available - procs = None + newlockcontents = get_lock_contents(lockfile) + if newlockcontents != lockcontents: + # A new server was started, the lockfile contents changed, we can exit + serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents)) + return + time.sleep(0.1) + i += 1 + if lock: + # We hold the lock so we can remove the file (hide stale pid data) + # via unlockfile. + bb.utils.unlockfile(lock) + serverlog("Exiting as we could obtain the lock") + return + + if not lock: + # Some systems may not have lsof available + procs = None + try: + procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # File was deleted? + continue + except OSError as e: + if e.errno != errno.ENOENT: + raise + if procs is None: + # Fall back to fuser if lsof is unavailable try: - procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT) + procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # File was deleted? + continue except OSError as e: if e.errno != errno.ENOENT: raise - if procs is None: - # Fall back to fuser if lsof is unavailable - try: - procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" - if procs: - msg += ":\n%s" % str(procs) - print(msg) + + msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock" + if procs: + msg += ":\n%s" % str(procs.decode("utf-8")) + serverlog(msg) def idle_commands(self, delay, fds=None): nextsleep = delay @@ -304,8 +364,9 @@ class ProcessServer(multiprocessing.Process): self.next_heartbeat += self.heartbeat_seconds if self.next_heartbeat <= now: self.next_heartbeat = now + self.heartbeat_seconds - heartbeat = bb.event.HeartbeatEvent(now) - bb.event.fire(heartbeat, self.cooker.data) + if hasattr(self.cooker, "data"): + heartbeat = bb.event.HeartbeatEvent(now) + bb.event.fire(heartbeat, self.cooker.data) if nextsleep and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for # the heartbeat. @@ -331,8 +392,15 @@ class ServerCommunicator(): def runCommand(self, command): self.connection.send(command) if not self.recv.poll(30): - raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server") - return self.recv.get() + logger.info("No reply from server in 30s") + if not self.recv.poll(30): + raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)") + ret, exc = self.recv.get() + # Should probably turn all exceptions in exc back into exceptions? + # For now, at least handle BBHandledException + if exc and ("BBHandledException" in exc or "SystemExit" in exc): + raise bb.BBHandledException() + return ret, exc def updateFeatureSet(self, featureset): _, error = self.runCommand(["setFeatures", featureset]) @@ -365,39 +433,26 @@ class BitBakeProcessServerConnection(object): self.connection.recv.close() return +start_log_format = '--- Starting bitbake server pid %s at %s ---' +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' + class BitBakeServer(object): - start_log_format = '--- Starting bitbake server pid %s at %s ---' - start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f' - def __init__(self, lock, sockname, configuration, featureset): + def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface): - self.configuration = configuration + self.server_timeout = server_timeout + self.xmlrpcinterface = xmlrpcinterface self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock self.readypipe, self.readypipein = os.pipe() - # Create server control socket - if os.path.exists(sockname): - os.unlink(sockname) - # Place the log in the builddirectory alongside the lock file logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") + self.logfile = logfile - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - # AF_UNIX has path length issues so chdir here to workaround - cwd = os.getcwd() - try: - os.chdir(os.path.dirname(sockname)) - self.sock.bind(os.path.basename(sockname)) - finally: - os.chdir(cwd) - self.sock.listen(1) - - os.set_inheritable(self.sock.fileno(), True) startdatetime = datetime.datetime.now() bb.daemonize.createDaemon(self._startServer, logfile) - self.sock.close() self.bitbake_lock.close() os.close(self.readypipein) @@ -416,7 +471,7 @@ class BitBakeServer(object): ready.close() bb.error("Unable to start bitbake server (%s)" % str(r)) if os.path.exists(logfile): - logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) + logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] lastlines = [] @@ -426,9 +481,9 @@ class BitBakeServer(object): lines.append(line) else: lastlines.append(line) - res = logstart_re.match(line.rstrip()) + res = logstart_re.search(line.rstrip()) if res: - ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) + ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) @@ -449,26 +504,53 @@ class BitBakeServer(object): ready.close() def _startServer(self): - print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format))) - sys.stdout.flush() - - server = ProcessServer(self.bitbake_lock, self.sock, self.sockname) - self.configuration.setServerRegIdleCallback(server.register_idle_function) os.close(self.readypipe) - writer = ConnectionWriter(self.readypipein) + os.set_inheritable(self.bitbake_lock.fileno(), True) + os.set_inheritable(self.readypipein, True) + serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") + os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) + +def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface): + + import bb.cookerdata + import bb.cooker + + serverlog(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format))) + + try: + bitbake_lock = os.fdopen(lockfd, "w") + + # Create server control socket + if os.path.exists(sockname): + os.unlink(sockname) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + # AF_UNIX has path length issues so chdir here to workaround + cwd = os.getcwd() + try: + os.chdir(os.path.dirname(sockname)) + sock.bind(os.path.basename(sockname)) + finally: + os.chdir(cwd) + sock.listen(1) + + server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface) + writer = ConnectionWriter(readypipeinfd) try: - self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset) + featureset = [] + cooker = bb.cooker.BBCooker(featureset, server.register_idle_function) except bb.BBHandledException: return None writer.send("r") writer.close() - server.cooker = self.cooker - server.server_timeout = self.configuration.server_timeout - server.xmlrpcinterface = self.configuration.xmlrpcinterface - print("Started bitbake server pid %d" % os.getpid()) - sys.stdout.flush() + server.cooker = cooker + serverlog("Started bitbake server pid %d" % os.getpid()) - server.start() + server.run() + finally: + # Flush any ,essages/errors to the logfile before exit + sys.stdout.flush() + sys.stderr.flush() def connectProcessServer(sockname, featureset): # Connect to socket diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 4c8d81c5d..86e0e16f3 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -14,6 +14,7 @@ import simplediff from bb.checksum import FileChecksumCache from bb import runqueue import hashserv +import hashserv.client logger = logging.getLogger('BitBake.SigGen') hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv') @@ -38,6 +39,11 @@ class SignatureGenerator(object): """ name = "noop" + # If the derived class supports multiconfig datacaches, set this to True + # The default is False for backward compatibility with derived signature + # generators that do not understand multiconfig caches + supports_multiconfig_datacaches = False + def __init__(self, data): self.basehash = {} self.taskhash = {} @@ -58,10 +64,10 @@ class SignatureGenerator(object): def get_unihash(self, tid): return self.taskhash[tid] - def prep_taskhash(self, tid, deps, dataCache): + def prep_taskhash(self, tid, deps, dataCaches): return - def get_taskhash(self, tid, deps, dataCache): + def get_taskhash(self, tid, deps, dataCaches): self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest() return self.taskhash[tid] @@ -105,6 +111,38 @@ class SignatureGenerator(object): def set_setscene_tasks(self, setscene_tasks): return + @classmethod + def get_data_caches(cls, dataCaches, mc): + """ + This function returns the datacaches that should be passed to signature + generator functions. If the signature generator supports multiconfig + caches, the entire dictionary of data caches is sent, otherwise a + special proxy is sent that support both index access to all + multiconfigs, and also direct access for the default multiconfig. + + The proxy class allows code in this class itself to always use + multiconfig aware code (to ease maintenance), but derived classes that + are unaware of multiconfig data caches can still access the default + multiconfig as expected. + + Do not override this function in derived classes; it will be removed in + the future when support for multiconfig data caches is mandatory + """ + class DataCacheProxy(object): + def __init__(self): + pass + + def __getitem__(self, key): + return dataCaches[key] + + def __getattr__(self, name): + return getattr(dataCaches[mc], name) + + if cls.supports_multiconfig_datacaches: + return dataCaches + + return DataCacheProxy() + class SignatureGeneratorBasic(SignatureGenerator): """ """ @@ -200,7 +238,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.lookupcache = {} self.taskdeps = {} - def rundep_check(self, fn, recipename, task, dep, depname, dataCache): + def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): # Return True if we should keep the dependency, False to drop it # We only manipulate the dependencies for packages not in the whitelist if self.twl and not self.twl.search(recipename): @@ -218,37 +256,40 @@ class SignatureGeneratorBasic(SignatureGenerator): pass return taint - def prep_taskhash(self, tid, deps, dataCache): + def prep_taskhash(self, tid, deps, dataCaches): (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) - self.basehash[tid] = dataCache.basetaskhash[tid] + self.basehash[tid] = dataCaches[mc].basetaskhash[tid] self.runtaskdeps[tid] = [] self.file_checksum_values[tid] = [] - recipename = dataCache.pkg_fn[fn] + recipename = dataCaches[mc].pkg_fn[fn] self.tidtopn[tid] = recipename for dep in sorted(deps, key=clean_basepath): - (depmc, _, deptaskname, depfn) = bb.runqueue.split_tid_mcfn(dep) - if mc != depmc: + (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep) + depname = dataCaches[depmc].pkg_fn[depmcfn] + if not self.supports_multiconfig_datacaches and mc != depmc: + # If the signature generator doesn't understand multiconfig + # data caches, any dependency not in the same multiconfig must + # be skipped for backward compatibility continue - depname = dataCache.pkg_fn[depfn] - if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): + if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches): continue if dep not in self.taskhash: bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) self.runtaskdeps[tid].append(dep) - if task in dataCache.file_checksums[fn]: + if task in dataCaches[mc].file_checksums[fn]: if self.checksum_cache: - checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude) + checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude) else: - checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude) + checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude) for (f,cs) in checksums: self.file_checksum_values[tid].append((f,cs)) - taskdep = dataCache.task_deps[fn] + taskdep = dataCaches[mc].task_deps[fn] if 'nostamp' in taskdep and task in taskdep['nostamp']: # Nostamp tasks need an implicit taint so that they force any dependent tasks to run if tid in self.taints and self.taints[tid].startswith("nostamp:"): @@ -259,14 +300,14 @@ class SignatureGeneratorBasic(SignatureGenerator): taint = str(uuid.uuid4()) self.taints[tid] = "nostamp:" + taint - taint = self.read_taint(fn, task, dataCache.stamp[fn]) + taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn]) if taint: self.taints[tid] = taint logger.warning("%s is tainted from a forced run" % tid) return - def get_taskhash(self, tid, deps, dataCache): + def get_taskhash(self, tid, deps, dataCaches): data = self.basehash[tid] for dep in self.runtaskdeps[tid]: @@ -317,7 +358,8 @@ class SignatureGeneratorBasic(SignatureGenerator): else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid] - bb.utils.mkdirhier(os.path.dirname(sigfile)) + with bb.utils.umask(0o002): + bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task @@ -640,6 +682,12 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG self.server = data.getVar('BB_HASHSERVE') self.method = "sstate_output_hash" +# +# Dummy class used for bitbake-selftest +# +class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash): + name = "TestMulticonfigDepends" + supports_multiconfig_datacaches = True def dump_this_task(outfile, d): import bb.parse @@ -699,16 +747,26 @@ def list_inline_diff(oldlist, newlist, colors=None): ret.append(item) return '[%s]' % (', '.join(ret)) -def clean_basepath(a): - mc = None - if a.startswith("mc:"): - _, mc, a = a.split(":", 2) - b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] - if a.startswith("virtual:"): - b = b + ":" + a.rsplit(":", 1)[0] - if mc: - b = b + ":mc:" + mc - return b +def clean_basepath(basepath): + basepath, dir, recipe_task = basepath.rsplit("/", 2) + cleaned = dir + '/' + recipe_task + + if basepath[0] == '/': + return cleaned + + if basepath.startswith("mc:"): + mc, mc_name, basepath = basepath.split(":", 2) + mc_suffix = ':mc:' + mc_name + else: + mc_suffix = '' + + # mc stuff now removed from basepath. Whatever was next, if present will be the first + # suffix. ':/', recipe path start, marks the end of this. Something like + # 'virtual:a[:b[:c]]:/path...' (b and c being optional) + if basepath[0] != '/': + cleaned += ':' + basepath.split(':/', 1)[0] + + return cleaned + mc_suffix def clean_basepaths(a): b = {} diff --git a/poky/bitbake/lib/bb/taskdata.py b/poky/bitbake/lib/bb/taskdata.py index d13a12498..ffbaf362e 100644 --- a/poky/bitbake/lib/bb/taskdata.py +++ b/poky/bitbake/lib/bb/taskdata.py @@ -21,8 +21,13 @@ def re_match_strings(target, strings): Whether or not the string 'target' matches any one string of the strings which can be regular expression string """ - return any(name == target or re.match(name, target) - for name in strings) + for name in strings: + if name.startswith("^") or name.endswith("$"): + if re.match(name, target): + return True + elif name == target: + return True + return False class TaskEntry: def __init__(self): diff --git a/poky/bitbake/lib/bb/tests/color.py b/poky/bitbake/lib/bb/tests/color.py new file mode 100644 index 000000000..bf03750c6 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/color.py @@ -0,0 +1,95 @@ +# +# BitBake Test for ANSI color code filtering +# +# Copyright (C) 2020 Agilent Technologies, Inc. +# Author: Chris Laplante <chris.laplante@agilent.com> +# +# SPDX-License-Identifier: MIT +# + +import unittest +import bb.progress +import bb.data +import bb.event +from bb.progress import filter_color, filter_color_n +import io +import re + + +class ProgressWatcher: + def __init__(self): + self._reports = [] + + def handle_event(self, event): + self._reports.append((event.progress, event.rate)) + + def reports(self): + return self._reports + + +class ColorCodeTests(unittest.TestCase): + def setUp(self): + self.d = bb.data.init() + self._progress_watcher = ProgressWatcher() + bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event) + + def tearDown(self): + bb.event.remove("bb.build.TaskProgress", None) + + def test_filter_color(self): + input_string = "[01;35m[K~~~~~~~~~~~~^~~~~~~~[m[K" + filtered = filter_color(input_string) + self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~") + + def test_filter_color_n(self): + input_string = "[01;35m[K~~~~~~~~~~~~^~~~~~~~[m[K" + filtered, code_count = filter_color_n(input_string) + self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~") + self.assertEqual(code_count, 4) + + def test_LineFilterProgressHandler_color_filtering(self): + class CustomProgressHandler(bb.progress.LineFilterProgressHandler): + PROGRESS_REGEX = re.compile(r"Progress: (?P<progress>\d+)%") + + def writeline(self, line): + match = self.PROGRESS_REGEX.match(line) + if match: + self.update(int(match.group("progress"))) + return False + return True + + buffer = io.StringIO() + handler = CustomProgressHandler(self.d, buffer) + handler.write("Program output!\n") + handler.write("More output!\n") + handler.write("Progress: [01;35m[K10[m[K%\n") # 10% + handler.write("Even more\n") + handler.write("[01;35m[KProgress: 50[m[K%\n") # 50% + handler.write("[01;35m[KProgress: 60[m[K%\n") # 60% + handler.write("Pro[01;35m[Kgress: [m[K100%\n") # 100% + + expected = [(10, None), (50, None), (60, None), (100, None)] + self.assertEqual(self._progress_watcher.reports(), expected) + + self.assertEqual(buffer.getvalue(), "Program output!\nMore output!\nEven more\n") + + def test_BasicProgressHandler_color_filtering(self): + buffer = io.StringIO() + handler = bb.progress.BasicProgressHandler(self.d, outfile=buffer) + handler.write("[01;35m[K1[m[K%\n") # 1% + handler.write("[01;35m[K2[m[K%\n") # 2% + handler.write("[01;35m[K10[m[K%\n") # 10% + handler.write("[01;35m[K100[m[K%\n") # 100% + + expected = [(0, None), (1, None), (2, None), (10, None), (100, None)] + self.assertListEqual(self._progress_watcher.reports(), expected) + + def test_OutOfProgressHandler_color_filtering(self): + buffer = io.StringIO() + handler = bb.progress.OutOfProgressHandler(self.d, r'(\d+) of (\d+)', outfile=buffer) + handler.write("[01;35m[KText text 1 of[m[K 5") # 1/5 + handler.write("[01;35m[KText text 3 of[m[K 5") # 3/5 + handler.write("[01;35m[KText text 5 of[m[K 5") # 5/5 + + expected = [(0, None), (20.0, None), (60.0, None), (100.0, None)] + self.assertListEqual(self._progress_watcher.reports(), expected) diff --git a/poky/bitbake/lib/bb/tests/cooker.py b/poky/bitbake/lib/bb/tests/cooker.py index 74c903f01..c82d4b7b8 100644 --- a/poky/bitbake/lib/bb/tests/cooker.py +++ b/poky/bitbake/lib/bb/tests/cooker.py @@ -60,7 +60,7 @@ class CookerTest(unittest.TestCase): log_handler = LogHandler() logger.addHandler(log_handler) collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities) - collection.collection_priorities(pkgfns, self.d) + collection.collection_priorities(pkgfns, pkgfns, self.d) logger.removeHandler(log_handler) # Should be empty (no generated messages) diff --git a/poky/bitbake/lib/bb/tests/cow.py b/poky/bitbake/lib/bb/tests/cow.py index bf6e79fce..75142649c 100644 --- a/poky/bitbake/lib/bb/tests/cow.py +++ b/poky/bitbake/lib/bb/tests/cow.py @@ -4,9 +4,17 @@ # SPDX-License-Identifier: GPL-2.0-only # # Copyright 2006 Holger Freyther <freyther@handhelds.org> +# Copyright (C) 2020 Agilent Technologies, Inc. # +import io +import re +import sys import unittest +import contextlib +import collections + +from bb.COW import COWDictBase, COWSetBase, COWDictMeta, COWSetMeta class COWTestCase(unittest.TestCase): @@ -14,11 +22,61 @@ class COWTestCase(unittest.TestCase): Test case for the COW module from mithro """ + def setUp(self): + self._track_warnings = False + self._warning_file = io.StringIO() + self._unhandled_warnings = collections.deque() + COWDictBase.__warn__ = self._warning_file + + def tearDown(self): + COWDictBase.__warn__ = sys.stderr + if self._track_warnings: + self._checkAllWarningsRead() + + def trackWarnings(self): + self._track_warnings = True + + def _collectWarnings(self): + self._warning_file.seek(0) + for warning in self._warning_file: + self._unhandled_warnings.append(warning.rstrip("\n")) + self._warning_file.truncate(0) + self._warning_file.seek(0) + + def _checkAllWarningsRead(self): + self._collectWarnings() + self.assertSequenceEqual(self._unhandled_warnings, []) + + @contextlib.contextmanager + def checkReportsWarning(self, expected_warning): + self._checkAllWarningsRead() + yield + self._collectWarnings() + warning = self._unhandled_warnings.popleft() + self.assertEqual(warning, expected_warning) + + def checkStrOutput(self, obj, expected_levels, expected_keys): + if obj.__class__ is COWDictMeta: + expected_class_name = "COWDict" + elif obj.__class__ is COWSetMeta: + expected_class_name = "COWSet" + else: + self.fail("obj is of unknown type {0}".format(type(obj))) + s = str(obj) + regex = re.compile(r"<(\w+) Level: (\d+) Current Keys: (\d+)>") + match = regex.match(s) + self.assertIsNotNone(match, "bad str output: '{0}'".format(s)) + class_name = match.group(1) + self.assertEqual(class_name, expected_class_name) + levels = int(match.group(2)) + self.assertEqual(levels, expected_levels, "wrong # levels in str: '{0}'".format(s)) + keys = int(match.group(3)) + self.assertEqual(keys, expected_keys, "wrong # keys in str: '{0}'".format(s)) + def testGetSet(self): """ Test and set """ - from bb.COW import COWDictBase a = COWDictBase.copy() self.assertEqual(False, 'a' in a) @@ -27,16 +85,14 @@ class COWTestCase(unittest.TestCase): a['b'] = 'b' self.assertEqual(True, 'a' in a) self.assertEqual(True, 'b' in a) - self.assertEqual('a', a['a'] ) - self.assertEqual('b', a['b'] ) + self.assertEqual('a', a['a']) + self.assertEqual('b', a['b']) def testCopyCopy(self): """ Test the copy of copies """ - from bb.COW import COWDictBase - # create two COW dict 'instances' b = COWDictBase.copy() c = COWDictBase.copy() @@ -94,30 +150,168 @@ class COWTestCase(unittest.TestCase): self.assertEqual(False, 'e' in b_2) def testCow(self): - from bb.COW import COWDictBase + self.trackWarnings() + c = COWDictBase.copy() c['123'] = 1027 c['other'] = 4711 - c['d'] = { 'abc' : 10, 'bcd' : 20 } + c['d'] = {'abc': 10, 'bcd': 20} copy = c.copy() self.assertEqual(1027, c['123']) self.assertEqual(4711, c['other']) - self.assertEqual({'abc':10, 'bcd':20}, c['d']) + self.assertEqual({'abc': 10, 'bcd': 20}, c['d']) self.assertEqual(1027, copy['123']) self.assertEqual(4711, copy['other']) - self.assertEqual({'abc':10, 'bcd':20}, copy['d']) + with self.checkReportsWarning("Warning: Doing a copy because d is a mutable type."): + self.assertEqual({'abc': 10, 'bcd': 20}, copy['d']) # cow it now copy['123'] = 1028 copy['other'] = 4712 copy['d']['abc'] = 20 - self.assertEqual(1027, c['123']) self.assertEqual(4711, c['other']) - self.assertEqual({'abc':10, 'bcd':20}, c['d']) + self.assertEqual({'abc': 10, 'bcd': 20}, c['d']) self.assertEqual(1028, copy['123']) self.assertEqual(4712, copy['other']) - self.assertEqual({'abc':20, 'bcd':20}, copy['d']) + self.assertEqual({'abc': 20, 'bcd': 20}, copy['d']) + + def testOriginalTestSuite(self): + # This test suite is a port of the original one from COW.py + self.trackWarnings() + + a = COWDictBase.copy() + self.checkStrOutput(a, 1, 0) + + a['a'] = 'a' + a['b'] = 'b' + a['dict'] = {} + self.checkStrOutput(a, 1, 4) # 4th member is dict__mutable__ + + b = a.copy() + self.checkStrOutput(b, 2, 0) + b['c'] = 'b' + self.checkStrOutput(b, 2, 1) + + with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."): + self.assertListEqual(list(a.iteritems()), + [('a', 'a'), + ('b', 'b'), + ('dict', {}) + ]) + + with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."): + b_gen = b.iteritems() + self.assertTupleEqual(next(b_gen), ('a', 'a')) + self.assertTupleEqual(next(b_gen), ('b', 'b')) + self.assertTupleEqual(next(b_gen), ('c', 'b')) + with self.checkReportsWarning("Warning: Doing a copy because dict is a mutable type."): + self.assertTupleEqual(next(b_gen), ('dict', {})) + with self.assertRaises(StopIteration): + next(b_gen) + + b['dict']['a'] = 'b' + b['a'] = 'c' + + self.checkStrOutput(a, 1, 4) + self.checkStrOutput(b, 2, 3) + + with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."): + self.assertListEqual(list(a.iteritems()), + [('a', 'a'), + ('b', 'b'), + ('dict', {}) + ]) + + with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."): + b_gen = b.iteritems() + self.assertTupleEqual(next(b_gen), ('a', 'c')) + self.assertTupleEqual(next(b_gen), ('b', 'b')) + self.assertTupleEqual(next(b_gen), ('c', 'b')) + self.assertTupleEqual(next(b_gen), ('dict', {'a': 'b'})) + with self.assertRaises(StopIteration): + next(b_gen) + + with self.assertRaises(KeyError): + print(b["dict2"]) + + a['set'] = COWSetBase() + a['set'].add("o1") + a['set'].add("o1") + a['set'].add("o2") + self.assertSetEqual(set(a['set'].itervalues()), {"o1", "o2"}) + self.assertSetEqual(set(b['set'].itervalues()), {"o1", "o2"}) + + b['set'].add('o3') + self.assertSetEqual(set(a['set'].itervalues()), {"o1", "o2"}) + self.assertSetEqual(set(b['set'].itervalues()), {"o1", "o2", "o3"}) + + a['set2'] = set() + a['set2'].add("o1") + a['set2'].add("o1") + a['set2'].add("o2") + + # We don't expect 'a' to change anymore + def check_a(): + with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."): + a_gen = a.iteritems() + self.assertTupleEqual(next(a_gen), ('a', 'a')) + self.assertTupleEqual(next(a_gen), ('b', 'b')) + self.assertTupleEqual(next(a_gen), ('dict', {})) + self.assertTupleEqual(next(a_gen), ('set2', {'o1', 'o2'})) + a_sub_set = next(a_gen) + self.assertEqual(a_sub_set[0], 'set') + self.checkStrOutput(a_sub_set[1], 1, 2) + self.assertSetEqual(set(a_sub_set[1].itervalues()), {'o1', 'o2'}) + + check_a() + + b_gen = b.iteritems(readonly=True) + self.assertTupleEqual(next(b_gen), ('a', 'c')) + self.assertTupleEqual(next(b_gen), ('b', 'b')) + self.assertTupleEqual(next(b_gen), ('c', 'b')) + self.assertTupleEqual(next(b_gen), ('dict', {'a': 'b'})) + self.assertTupleEqual(next(b_gen), ('set2', {'o1', 'o2'})) + b_sub_set = next(b_gen) + self.assertEqual(b_sub_set[0], 'set') + self.checkStrOutput(b_sub_set[1], 2, 1) + self.assertSetEqual(set(b_sub_set[1].itervalues()), {'o1', 'o2', 'o3'}) + + del b['b'] + with self.assertRaises(KeyError): + print(b['b']) + self.assertFalse('b' in b) + + check_a() + + b.__revertitem__('b') + check_a() + self.assertEqual(b['b'], 'b') + self.assertTrue('b' in b) + + b.__revertitem__('dict') + check_a() + + b_gen = b.iteritems(readonly=True) + self.assertTupleEqual(next(b_gen), ('a', 'c')) + self.assertTupleEqual(next(b_gen), ('b', 'b')) + self.assertTupleEqual(next(b_gen), ('c', 'b')) + self.assertTupleEqual(next(b_gen), ('dict', {})) + self.assertTupleEqual(next(b_gen), ('set2', {'o1', 'o2'})) + b_sub_set = next(b_gen) + self.assertEqual(b_sub_set[0], 'set') + self.checkStrOutput(b_sub_set[1], 2, 1) + self.assertSetEqual(set(b_sub_set[1].itervalues()), {'o1', 'o2', 'o3'}) + + self.checkStrOutput(a, 1, 6) + self.checkStrOutput(b, 2, 3) + + def testSetMethods(self): + s = COWSetBase() + with self.assertRaises(TypeError): + print(s.iteritems()) + with self.assertRaises(TypeError): + print(s.iterkeys()) diff --git a/poky/bitbake/lib/bb/tests/data.py b/poky/bitbake/lib/bb/tests/data.py index 5f195047d..1d4a64b10 100644 --- a/poky/bitbake/lib/bb/tests/data.py +++ b/poky/bitbake/lib/bb/tests/data.py @@ -12,6 +12,7 @@ import bb import bb.data import bb.parse import logging +import os class LogRecord(): def __enter__(self): diff --git a/poky/bitbake/lib/bb/tests/event.py b/poky/bitbake/lib/bb/tests/event.py index 9229b63d4..9ca7e9bc8 100644 --- a/poky/bitbake/lib/bb/tests/event.py +++ b/poky/bitbake/lib/bb/tests/event.py @@ -6,17 +6,18 @@ # SPDX-License-Identifier: GPL-2.0-only # -import unittest -import bb -import logging -import bb.compat -import bb.event +import collections import importlib +import logging +import pickle import threading import time -import pickle +import unittest from unittest.mock import Mock from unittest.mock import call + +import bb +import bb.event from bb.msg import BBLogFormatter @@ -75,7 +76,7 @@ class EventHandlingTest(unittest.TestCase): def _create_test_handlers(self): """ Method used to create a test handler ordered dictionary """ - test_handlers = bb.compat.OrderedDict() + test_handlers = collections.OrderedDict() test_handlers["handler1"] = self._test_process.handler1 test_handlers["handler2"] = self._test_process.handler2 return test_handlers @@ -96,7 +97,7 @@ class EventHandlingTest(unittest.TestCase): def test_clean_class_handlers(self): """ Test clean_class_handlers method """ - cleanDict = bb.compat.OrderedDict() + cleanDict = collections.OrderedDict() self.assertEqual(cleanDict, bb.event.clean_class_handlers()) diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index d0c161a7f..5a4db9ca4 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -223,6 +223,21 @@ class URITest(unittest.TestCase): 'query': {}, 'relative': False }, + "git://tfs-example.org:22/tfs/example%20path/example.git": { + 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', + 'scheme': 'git', + 'hostname': 'tfs-example.org', + 'port': 22, + 'hostport': 'tfs-example.org:22', + 'path': '/tfs/example path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, "http://somesite.net;someparam=1": { 'uri': 'http://somesite.net;someparam=1', 'scheme': 'http', @@ -584,6 +599,7 @@ class FetcherLocalTest(FetcherTest): touch(os.path.join(self.localsrcdir, 'dir', 'd')) os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) + touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) self.d.setVar("FILESPATH", self.localsrcdir) def fetchUnpack(self, uris): @@ -601,9 +617,13 @@ class FetcherLocalTest(FetcherTest): tree = self.fetchUnpack(['file://a', 'file://dir/c']) self.assertEqual(tree, ['a', 'dir/c']) + def test_local_backslash(self): + tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) + self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) + def test_local_wildcard(self): - tree = self.fetchUnpack(['file://a', 'file://dir/*']) - self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) + with self.assertRaises(bb.fetch2.ParameterError): + tree = self.fetchUnpack(['file://a', 'file://dir/*']) def test_local_dir(self): tree = self.fetchUnpack(['file://a', 'file://dir']) @@ -1031,7 +1051,7 @@ class SVNTest(FetcherTest): bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) # Github will emulate SVN. Use this to check if we're downloding... - bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .", + bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) bb.process.run("svn commit --non-interactive -m 'Add external'", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) @@ -1152,10 +1172,12 @@ class FetchLatestVersionTest(FetcherTest): ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") : "1.99.4", # version pattern "vX.Y" - ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") + # mirror of git.infradead.org since network issues interfered with testing + ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") : "1.5.0", # version pattern "pkg_name-X.Y" - ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") + # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing + ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") : "1.0", # version pattern "pkg_name-vX.Y.Z" ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") @@ -1169,7 +1191,8 @@ class FetchLatestVersionTest(FetcherTest): ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "") : "20120614", # packages with a valid UPSTREAM_CHECK_GITTAGREGEX - ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))") + # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing + ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))") : "0.4.3", ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))") : "11.0.0", @@ -1261,9 +1284,7 @@ class FetchLatestVersionTest(FetcherTest): class FetchCheckStatusTest(FetcherTest): - test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", - "http://www.cups.org/", - "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", + test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", "https://yoctoproject.org/", @@ -2079,6 +2100,38 @@ class GitLfsTest(FetcherTest): shutil.rmtree(self.gitdir, ignore_errors=True) fetcher.unpack(self.d.getVar('WORKDIR')) +class GitURLWithSpacesTest(FetcherTest): + test_git_urls = { + "git://tfs-example.org:22/tfs/example%20path/example.git" : { + 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git', + 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', + 'path': '/tfs/example path/example.git' + }, + "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : { + 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git', + 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', + 'path': '/tfs/example path/example repo.git' + } + } + + def test_urls(self): + + # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo + self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') + + for test_git_url, ref in self.test_git_urls.items(): + + fetcher = bb.fetch.Fetch([test_git_url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertEqual(ud.url, ref['url']) + self.assertEqual(ud.path, ref['path']) + self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) + self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) + class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 9afd1b208..9e21e1842 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -178,7 +178,10 @@ python () { addtask do_patch after do_foo after do_unpack before do_configure before do_compile addtask do_fetch do_patch -deltask do_fetch do_patch +MYVAR = "do_patch" +EMPTYVAR = "" +deltask do_fetch ${MYVAR} ${EMPTYVAR} +deltask ${EMPTYVAR} """ def test_parse_addtask_deltask(self): import sys @@ -189,6 +192,5 @@ deltask do_fetch do_patch self.assertTrue("addtask contained multiple 'before' keywords" in stdout) self.assertTrue("addtask contained multiple 'after' keywords" in stdout) self.assertTrue('addtask ignored: " do_patch"' in stdout) - self.assertTrue('deltask ignored: " do_patch"' in stdout) #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout) diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 5e451fc2c..efebf001a 100644 --- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf index ecf23e1c7..f34b8dccc 100644 --- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf @@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf index eef338e4c..c3360fc5c 100644 --- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf @@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb new file mode 100644 index 000000000..d45a4cff5 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb @@ -0,0 +1 @@ +do_install[mcdepends] = "mc:mc1:mc2:a1:do_build" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file mode 100644 index 000000000..17a181fff --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb @@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} + diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file mode 100644 index 000000000..cc69e7b82 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb @@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +} diff --git a/poky/bitbake/lib/bb/tests/runqueue.py b/poky/bitbake/lib/bb/tests/runqueue.py index 4ba12a077..d3d62b98f 100644 --- a/poky/bitbake/lib/bb/tests/runqueue.py +++ b/poky/bitbake/lib/bb/tests/runqueue.py @@ -232,6 +232,51 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) + + def test_multiconfig_mcdepends(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True) + expected = ["mc1:f1:%s" % t for t in self.alltasks] + \ + ["mc2:a1:%s" % t for t in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + + # A rebuild does nothing + tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True) + self.assertEqual(set(tasks), set()) + + # Test that a signature change in the dependent task causes + # mcdepends to rebuild + tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True) + expected = ["mc2:a1:compile"] + self.assertEqual(set(tasks), set(expected)) + + rerun_tasks = self.alltasks[:] + for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"): + rerun_tasks.remove(x) + tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True) + expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \ + ["mc2:a1:%s" % t for t in rerun_tasks] + self.assertEqual(set(tasks), set(expected)) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self): diff --git a/poky/bitbake/lib/bb/tests/siggen.py b/poky/bitbake/lib/bb/tests/siggen.py new file mode 100644 index 000000000..c21ab4e4f --- /dev/null +++ b/poky/bitbake/lib/bb/tests/siggen.py @@ -0,0 +1,91 @@ +# +# BitBake Test for lib/bb/siggen.py +# +# Copyright (C) 2020 Jean-François Dagenais +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import logging +import bb +import time + +logger = logging.getLogger('BitBake.TestSiggen') + +import bb.siggen + +class SiggenTest(unittest.TestCase): + + def test_clean_basepath_simple_target_basepath(self): + basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_basic_virtual_basepath(self): + basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_basepath(self): + basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_virtual_long_prefix_basepath(self): + basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_virtual_basepath(self): + basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_virtual_long_prefix_basepath(self): + basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + + # def test_clean_basepath_performance(self): + # input_basepaths = [ + # 'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # ] + + # time_start = time.time() + + # i = 2000000 + # while i >= 0: + # for basepath in input_basepaths: + # bb.siggen.clean_basepath(basepath) + # i -= 1 + + # elapsed = time.time() - time_start + # print('{} ({}s)'.format(self.id(), round(elapsed, 3))) + + # self.assertTrue(False) diff --git a/poky/bitbake/lib/bb/tinfoil.py b/poky/bitbake/lib/bb/tinfoil.py index 8c9b6b8ca..2fb1bb7d2 100644 --- a/poky/bitbake/lib/bb/tinfoil.py +++ b/poky/bitbake/lib/bb/tinfoil.py @@ -22,7 +22,6 @@ import bb.taskdata import bb.utils import bb.command import bb.remotedata -from bb.cookerdata import CookerConfiguration from bb.main import setup_bitbake, BitBakeConfigParameters import bb.fetch2 @@ -117,15 +116,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) @@ -185,10 +185,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() + self.collections = {} self.recipecaches = {} - self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) - for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + for mc in self.multiconfigs: + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name): @@ -379,18 +380,13 @@ class Tinfoil: if not config_params: config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) - cookerconfig = CookerConfiguration() - cookerconfig.setConfigParameters(config_params) - if not config_only: # Disable local loggers because the UI module is going to set up its own for handler in self.localhandlers: self.logger.handlers.remove(handler) self.localhandlers = [] - self.server_connection, ui_module = setup_bitbake(config_params, - cookerconfig, - extrafeatures) + self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures) self.ui_module = ui_module @@ -492,11 +488,11 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc=''): """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) def get_skipped_recipes(self): """ @@ -534,11 +530,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc=''): """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """ @@ -736,7 +732,7 @@ class Tinfoil: continue if helper.eventHandler(event): if isinstance(event, bb.build.TaskFailedSilent): - logger.warning("Logfile for failed setscene task is %s" % event.logfile) + self.logger.warning("Logfile for failed setscene task is %s" % event.logfile) elif isinstance(event, bb.build.TaskFailed): bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter) continue @@ -810,18 +806,22 @@ class Tinfoil: prepare() has been called, or use a with... block when you create the tinfoil object which will ensure that it gets called. """ - if self.server_connection: - self.run_command('clientComplete') - _server_connections.remove(self.server_connection) - bb.event.ui_queue = [] - self.server_connection.terminate() - self.server_connection = None - - # Restore logging handlers to how it looked when we started - if self.oldhandlers: - for handler in self.logger.handlers: - if handler not in self.oldhandlers: - self.logger.handlers.remove(handler) + try: + if self.server_connection: + try: + self.run_command('clientComplete') + finally: + _server_connections.remove(self.server_connection) + bb.event.ui_queue = [] + self.server_connection.terminate() + self.server_connection = None + + finally: + # Restore logging handlers to how it looked when we started + if self.oldhandlers: + for handler in self.logger.handlers: + if handler not in self.oldhandlers: + self.logger.handlers.remove(handler) def _reconvert_type(self, obj, origtypename): """ diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index 87e873d64..a91e4fd15 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -144,7 +144,7 @@ class TerminalFilter(object): pass if not cr: try: - cr = (env['LINES'], env['COLUMNS']) + cr = (os.environ['LINES'], os.environ['COLUMNS']) except: cr = (25, 80) return cr @@ -380,14 +380,27 @@ _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.Lo "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent", "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"] +def drain_events_errorhandling(eventHandler): + # We don't have logging setup, we do need to show any events we see before exiting + event = True + logger = bb.msg.logger_create('bitbake', sys.stdout) + while event: + event = eventHandler.waitEvent(0) + if isinstance(event, logging.LogRecord): + logger.handle(event) + def main(server, eventHandler, params, tf = TerminalFilter): - if not params.observe_only: - params.updateToServer(server, os.environ.copy()) + try: + if not params.observe_only: + params.updateToServer(server, os.environ.copy()) - includelogs, loglines, consolelogfile, logconfigfile = _log_settings_from_server(server, params.observe_only) + includelogs, loglines, consolelogfile, logconfigfile = _log_settings_from_server(server, params.observe_only) - loglevel, _ = bb.msg.constructLogOptions() + loglevel, _ = bb.msg.constructLogOptions() + except bb.BBHandledException: + drain_events_errorhandling(eventHandler) + return 1 if params.options.quiet == 0: console_loglevel = loglevel diff --git a/poky/bitbake/lib/bb/ui/ncurses.py b/poky/bitbake/lib/bb/ui/ncurses.py index da4fbeabb..cf1c876a5 100644 --- a/poky/bitbake/lib/bb/ui/ncurses.py +++ b/poky/bitbake/lib/bb/ui/ncurses.py @@ -48,6 +48,8 @@ import bb import xmlrpc.client from bb.ui import uihelper +logger = logging.getLogger(__name__) + parsespin = itertools.cycle( r'|/-\\' ) X = 0 diff --git a/poky/bitbake/lib/bb/ui/taskexp.py b/poky/bitbake/lib/bb/ui/taskexp.py index 8fff24423..2b246710c 100644 --- a/poky/bitbake/lib/bb/ui/taskexp.py +++ b/poky/bitbake/lib/bb/ui/taskexp.py @@ -8,9 +8,16 @@ # import sys -import gi -gi.require_version('Gtk', '3.0') -from gi.repository import Gtk, Gdk, GObject + +try: + import gi + gi.require_version('Gtk', '3.0') + from gi.repository import Gtk, Gdk, GObject +except ValueError: + sys.exit("FATAL: Gtk version needs to be 3.0") +except ImportError: + sys.exit("FATAL: Gtk ui could not load the required gi python module") + import threading from xmlrpc import client import bb @@ -51,7 +58,12 @@ class PackageReverseDepView(Gtk.TreeView): self.current = None self.filter_model = model.filter_new() self.filter_model.set_visible_func(self._filter) - self.sort_model = self.filter_model.sort_new_with_model() + # The introspected API was fixed but we can't rely on a pygobject that hides this. + # https://gitlab.gnome.org/GNOME/pygobject/-/commit/9cdbc56fbac4db2de78dc080934b8f0a7efc892a + if hasattr(Gtk.TreeModelSort, "new_with_model"): + self.sort_model = Gtk.TreeModelSort.new_with_model(self.filter_model) + else: + self.sort_model = self.filter_model.sort_new_with_model() self.sort_model.set_sort_column_id(COL_DEP_PARENT, Gtk.SortType.ASCENDING) self.set_model(self.sort_model) self.append_column(Gtk.TreeViewColumn(label, Gtk.CellRendererText(), text=COL_DEP_PARENT)) diff --git a/poky/bitbake/lib/bb/ui/teamcity.py b/poky/bitbake/lib/bb/ui/teamcity.py index 1854292fa..fca46c287 100644 --- a/poky/bitbake/lib/bb/ui/teamcity.py +++ b/poky/bitbake/lib/bb/ui/teamcity.py @@ -167,8 +167,6 @@ def main(server, eventHandler, params): forcelevel = bb.msg.BBLogFormatter.ERROR else: forcelevel = bb.msg.BBLogFormatter.WARNING - bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel) - bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr) console.setFormatter(format) errconsole.setFormatter(format) if not bb.msg.has_console_handler(logger): diff --git a/poky/bitbake/lib/bb/ui/uievent.py b/poky/bitbake/lib/bb/ui/uievent.py index 13d0d4a04..8607d0523 100644 --- a/poky/bitbake/lib/bb/ui/uievent.py +++ b/poky/bitbake/lib/bb/ui/uievent.py @@ -11,9 +11,13 @@ server and queue them for the UI to process. This process must be used to avoid client/server deadlocks. """ -import socket, threading, pickle, collections +import collections, logging, pickle, socket, threading from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import bb + +logger = logging.getLogger(__name__) + class BBUIEventQueue: def __init__(self, BBServer, clientinfo=("localhost, 0")): diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index 5f5767c1d..f73d31fb7 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -402,8 +402,8 @@ def better_exec(code, context, text = None, realfile = "<code>", pythonexception (t, value, tb) = sys.exc_info() try: _print_exception(t, value, tb, realfile, text, context) - except Exception as e: - logger.error("Exception handler error: %s" % str(e)) + except Exception as e2: + logger.error("Exception handler error: %s" % str(e2)) e = bb.BBHandledException(e) raise e @@ -433,20 +433,6 @@ def fileslocked(files): for lock in locks: bb.utils.unlockfile(lock) -@contextmanager -def timeout(seconds): - def timeout_handler(signum, frame): - pass - - original_handler = signal.signal(signal.SIGALRM, timeout_handler) - - try: - signal.alarm(seconds) - yield - finally: - signal.alarm(0) - signal.signal(signal.SIGALRM, original_handler) - def lockfile(name, shared=False, retry=True, block=False): """ Use the specified file as a lock file, return when the lock has @@ -580,7 +566,6 @@ def preserved_envvars_exported(): 'PATH', 'PWD', 'SHELL', - 'TERM', 'USER', 'LC_ALL', 'BBSERVER', @@ -959,6 +944,17 @@ def which(path, item, direction = 0, history = False, executable=False): return "", hist return "" +@contextmanager +def umask(new_mask): + """ + Context manager to set the umask to a specific mask, and restore it afterwards. + """ + current_mask = os.umask(new_mask) + try: + yield + finally: + os.umask(current_mask) + def to_boolean(string, default=None): if not string: return default @@ -1086,21 +1082,20 @@ def process_profilelog(fn, pout = None): # Either call with a list of filenames and set pout or a filename and optionally pout. if not pout: pout = fn + '.processed' - pout = open(pout, 'w') - - import pstats - if isinstance(fn, list): - p = pstats.Stats(*fn, stream=pout) - else: - p = pstats.Stats(fn, stream=pout) - p.sort_stats('time') - p.print_stats() - p.print_callers() - p.sort_stats('cumulative') - p.print_stats() - pout.flush() - pout.close() + with open(pout, 'w') as pout: + import pstats + if isinstance(fn, list): + p = pstats.Stats(*fn, stream=pout) + else: + p = pstats.Stats(fn, stream=pout) + p.sort_stats('time') + p.print_stats() + p.print_callers() + p.sort_stats('cumulative') + p.print_stats() + + pout.flush() # # Was present to work around multiprocessing pool bugs in python < 2.7.3 @@ -1473,14 +1468,20 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): return (notadded, notremoved) - -def get_file_layer(filename, d): - """Determine the collection (as defined by a layer's layer.conf file) containing the specified file""" +def get_collection_res(d): collections = (d.getVar('BBFILE_COLLECTIONS') or '').split() collection_res = {} for collection in collections: collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or '' + return collection_res + + +def get_file_layer(filename, d, collection_res={}): + """Determine the collection (as defined by a layer's layer.conf file) containing the specified file""" + if not collection_res: + collection_res = get_collection_res(d) + def path_to_layer(path): # Use longest path so we handle nested layers matchlen = 0 @@ -1492,12 +1493,13 @@ def get_file_layer(filename, d): return match result = None - bbfiles = (d.getVar('BBFILES') or '').split() + bbfiles = (d.getVar('BBFILES_PRIORITIZED') or '').split() bbfilesmatch = False for bbfilesentry in bbfiles: - if fnmatch.fnmatch(filename, bbfilesentry): + if fnmatch.fnmatchcase(filename, bbfilesentry): bbfilesmatch = True result = path_to_layer(bbfilesentry) + break if not bbfilesmatch: # Probably a bbclass diff --git a/poky/bitbake/lib/bblayers/action.py b/poky/bitbake/lib/bblayers/action.py index d6459d661..5b78195ad 100644 --- a/poky/bitbake/lib/bblayers/action.py +++ b/poky/bitbake/lib/bblayers/action.py @@ -143,11 +143,12 @@ build results (as the layer priority order has effectively changed). applied_appends = [] for layer in layers: - overlayed = [] - for f in self.tinfoil.cooker.collection.overlayed.keys(): - for of in self.tinfoil.cooker.collection.overlayed[f]: - if of.startswith(layer): - overlayed.append(of) + overlayed = set() + for mc in self.tinfoil.cooker.multiconfigs: + for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): + for of in self.tinfoil.cooker.collections[mc].overlayed[f]: + if of.startswith(layer): + overlayed.add(of) logger.plain('Copying files from %s...' % layer ) for root, dirs, files in os.walk(layer): @@ -174,14 +175,21 @@ build results (as the layer priority order has effectively changed). logger.warning('Overwriting file %s', fdest) bb.utils.copyfile(f1full, fdest) if ext == '.bb': - for append in self.tinfoil.cooker.collection.get_file_appends(f1full): + appends = set() + for mc in self.tinfoil.cooker.multiconfigs: + appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) + for append in appends: if layer_path_match(append): logger.plain(' Applying append %s to %s' % (append, fdest)) self.apply_append(append, fdest) applied_appends.append(append) # Take care of when some layers are excluded and yet we have included bbappends for those recipes - for b in self.tinfoil.cooker.collection.bbappends: + bbappends = set() + for mc in self.tinfoil.cooker.multiconfigs: + bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) + + for b in bbappends: (recipename, appendname) = b if appendname not in applied_appends: first_append = None diff --git a/poky/bitbake/lib/bblayers/query.py b/poky/bitbake/lib/bblayers/query.py index e2cc31053..f5e3c8474 100644 --- a/poky/bitbake/lib/bblayers/query.py +++ b/poky/bitbake/lib/bblayers/query.py @@ -21,6 +21,10 @@ def plugin_init(plugins): class QueryPlugin(LayerPlugin): + def __init__(self): + super(QueryPlugin, self).__init__() + self.collection_res = {} + def do_show_layers(self, args): """show current configured layers.""" logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority")) @@ -222,7 +226,6 @@ skipped recipes will also be listed, with a " (skipped)" suffix. multilayer = True if prov[0] != pref[0]: same_ver = False - if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): if not items_listed: logger.plain('=== %s ===' % title) @@ -243,8 +246,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix. else: return '?' + def get_collection_res(self): + if not self.collection_res: + self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data) + return self.collection_res + def get_file_layerdir(self, filename): - layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data) + layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res()) return self.bbfile_collections.get(layer, None) def remove_layer_prefix(self, f): @@ -320,12 +328,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else: diff --git a/poky/bitbake/lib/hashserv/__init__.py b/poky/bitbake/lib/hashserv/__init__.py index c3318620f..f95e8f43f 100644 --- a/poky/bitbake/lib/hashserv/__init__.py +++ b/poky/bitbake/lib/hashserv/__init__.py @@ -6,12 +6,20 @@ from contextlib import closing import re import sqlite3 +import itertools +import json UNIX_PREFIX = "unix://" ADDR_TYPE_UNIX = 0 ADDR_TYPE_TCP = 1 +# The Python async server defaults to a 64K receive buffer, so we hardcode our +# maximum chunk size. It would be better if the client and server reported to +# each other what the maximum chunk sizes were, but that will slow down the +# connection setup with a round trip delay so I'd rather not do that unless it +# is necessary +DEFAULT_MAX_CHUNK = 32 * 1024 def setup_database(database, sync=True): db = sqlite3.connect(database) @@ -66,6 +74,20 @@ def parse_address(addr): return (ADDR_TYPE_TCP, (host, int(port))) +def chunkify(msg, max_chunk): + if len(msg) < max_chunk - 1: + yield ''.join((msg, "\n")) + else: + yield ''.join((json.dumps({ + 'chunk-stream': None + }), "\n")) + + args = [iter(msg)] * (max_chunk - 1) + for m in map(''.join, itertools.zip_longest(*args, fillvalue='')): + yield ''.join(itertools.chain(m, "\n")) + yield "\n" + + def create_server(addr, dbname, *, sync=True): from . import server db = setup_database(dbname, sync=sync) diff --git a/poky/bitbake/lib/hashserv/client.py b/poky/bitbake/lib/hashserv/client.py index 46085d641..a29af836d 100644 --- a/poky/bitbake/lib/hashserv/client.py +++ b/poky/bitbake/lib/hashserv/client.py @@ -7,6 +7,7 @@ import json import logging import socket import os +from . import chunkify, DEFAULT_MAX_CHUNK logger = logging.getLogger('hashserv.client') @@ -25,6 +26,7 @@ class Client(object): self.reader = None self.writer = None self.mode = self.MODE_NORMAL + self.max_chunk = DEFAULT_MAX_CHUNK def connect_tcp(self, address, port): def connect_sock(): @@ -58,7 +60,7 @@ class Client(object): self.reader = self._socket.makefile('r', encoding='utf-8') self.writer = self._socket.makefile('w', encoding='utf-8') - self.writer.write('OEHASHEQUIV 1.0\n\n') + self.writer.write('OEHASHEQUIV 1.1\n\n') self.writer.flush() # Restore mode if the socket is being re-created @@ -91,18 +93,35 @@ class Client(object): count += 1 def send_message(self, msg): + def get_line(): + line = self.reader.readline() + if not line: + raise HashConnectionError('Connection closed') + + if not line.endswith('\n'): + raise HashConnectionError('Bad message %r' % message) + + return line + def proc(): - self.writer.write('%s\n' % json.dumps(msg)) + for c in chunkify(json.dumps(msg), self.max_chunk): + self.writer.write(c) self.writer.flush() - l = self.reader.readline() - if not l: - raise HashConnectionError('Connection closed') + l = get_line() - if not l.endswith('\n'): - raise HashConnectionError('Bad message %r' % message) + m = json.loads(l) + if 'chunk-stream' in m: + lines = [] + while True: + l = get_line().rstrip('\n') + if not l: + break + lines.append(l) - return json.loads(l) + m = json.loads(''.join(lines)) + + return m return self._send_wrapper(proc) @@ -155,6 +174,14 @@ class Client(object): m['unihash'] = unihash return self.send_message({'report-equiv': m}) + def get_taskhash(self, method, taskhash, all_properties=False): + self._set_mode(self.MODE_NORMAL) + return self.send_message({'get': { + 'taskhash': taskhash, + 'method': method, + 'all': all_properties + }}) + def get_stats(self): self._set_mode(self.MODE_NORMAL) return self.send_message({'get-stats': None}) diff --git a/poky/bitbake/lib/hashserv/server.py b/poky/bitbake/lib/hashserv/server.py index cc7e48233..81050715e 100644 --- a/poky/bitbake/lib/hashserv/server.py +++ b/poky/bitbake/lib/hashserv/server.py @@ -13,6 +13,7 @@ import os import signal import socket import time +from . import chunkify, DEFAULT_MAX_CHUNK logger = logging.getLogger('hashserv.server') @@ -107,12 +108,29 @@ class Stats(object): return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')} +class ClientError(Exception): + pass + class ServerClient(object): + FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1' + ALL_QUERY = 'SELECT * FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1' + def __init__(self, reader, writer, db, request_stats): self.reader = reader self.writer = writer self.db = db self.request_stats = request_stats + self.max_chunk = DEFAULT_MAX_CHUNK + + self.handlers = { + 'get': self.handle_get, + 'report': self.handle_report, + 'report-equiv': self.handle_equivreport, + 'get-stream': self.handle_get_stream, + 'get-stats': self.handle_get_stats, + 'reset-stats': self.handle_reset_stats, + 'chunk-stream': self.handle_chunk, + } async def process_requests(self): try: @@ -125,7 +143,11 @@ class ServerClient(object): return (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split() - if proto_name != 'OEHASHEQUIV' or proto_version != '1.0': + if proto_name != 'OEHASHEQUIV': + return + + proto_version = tuple(int(v) for v in proto_version.split('.')) + if proto_version < (1, 0) or proto_version > (1, 1): return # Read headers. Currently, no headers are implemented, so look for @@ -140,40 +162,34 @@ class ServerClient(object): break # Handle messages - handlers = { - 'get': self.handle_get, - 'report': self.handle_report, - 'report-equiv': self.handle_equivreport, - 'get-stream': self.handle_get_stream, - 'get-stats': self.handle_get_stats, - 'reset-stats': self.handle_reset_stats, - } - while True: d = await self.read_message() if d is None: break - - for k in handlers.keys(): - if k in d: - logger.debug('Handling %s' % k) - if 'stream' in k: - await handlers[k](d[k]) - else: - with self.request_stats.start_sample() as self.request_sample, \ - self.request_sample.measure(): - await handlers[k](d[k]) - break - else: - logger.warning("Unrecognized command %r" % d) - break - + await self.dispatch_message(d) await self.writer.drain() + except ClientError as e: + logger.error(str(e)) finally: self.writer.close() + async def dispatch_message(self, msg): + for k in self.handlers.keys(): + if k in msg: + logger.debug('Handling %s' % k) + if 'stream' in k: + await self.handlers[k](msg[k]) + else: + with self.request_stats.start_sample() as self.request_sample, \ + self.request_sample.measure(): + await self.handlers[k](msg[k]) + return + + raise ClientError("Unrecognized command %r" % msg) + def write_message(self, msg): - self.writer.write(('%s\n' % json.dumps(msg)).encode('utf-8')) + for c in chunkify(json.dumps(msg), self.max_chunk): + self.writer.write(c.encode('utf-8')) async def read_message(self): l = await self.reader.readline() @@ -191,14 +207,38 @@ class ServerClient(object): logger.error('Bad message from client: %r' % message) raise e + async def handle_chunk(self, request): + lines = [] + try: + while True: + l = await self.reader.readline() + l = l.rstrip(b"\n").decode("utf-8") + if not l: + break + lines.append(l) + + msg = json.loads(''.join(lines)) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.error('Bad message from client: %r' % message) + raise e + + if 'chunk-stream' in msg: + raise ClientError("Nested chunks are not allowed") + + await self.dispatch_message(msg) + async def handle_get(self, request): method = request['method'] taskhash = request['taskhash'] - row = self.query_equivalent(method, taskhash) + if request.get('all', False): + row = self.query_equivalent(method, taskhash, self.ALL_QUERY) + else: + row = self.query_equivalent(method, taskhash, self.FAST_QUERY) + if row is not None: logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) - d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + d = {k: row[k] for k in row.keys()} self.write_message(d) else: @@ -228,7 +268,7 @@ class ServerClient(object): (method, taskhash) = l.split() #logger.debug('Looking up %s %s' % (method, taskhash)) - row = self.query_equivalent(method, taskhash) + row = self.query_equivalent(method, taskhash, self.FAST_QUERY) if row is not None: msg = ('%s\n' % row['unihash']).encode('utf-8') #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) @@ -328,7 +368,7 @@ class ServerClient(object): # Fetch the unihash that will be reported for the taskhash. If the # unihash matches, it means this row was inserted (or the mapping # was already valid) - row = self.query_equivalent(data['method'], data['taskhash']) + row = self.query_equivalent(data['method'], data['taskhash'], self.FAST_QUERY) if row['unihash'] == data['unihash']: logger.info('Adding taskhash equivalence for %s with unihash %s', @@ -354,12 +394,11 @@ class ServerClient(object): self.request_stats.reset() self.write_message(d) - def query_equivalent(self, method, taskhash): + def query_equivalent(self, method, taskhash, query): # This is part of the inner loop and must be as fast as possible try: cursor = self.db.cursor() - cursor.execute('SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1', - {'method': method, 'taskhash': taskhash}) + cursor.execute(query, {'method': method, 'taskhash': taskhash}) return cursor.fetchone() except: cursor.close() diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py index a5472a996..4566f2473 100644 --- a/poky/bitbake/lib/hashserv/tests.py +++ b/poky/bitbake/lib/hashserv/tests.py @@ -9,10 +9,12 @@ from . import create_server, create_client import hashlib import logging import multiprocessing +import os import sys import tempfile import threading import unittest +import socket class TestHashEquivalenceServer(object): @@ -99,6 +101,29 @@ class TestHashEquivalenceServer(object): result = self.client.get_unihash(self.METHOD, taskhash) self.assertEqual(result, unihash) + def test_huge_message(self): + # Simple test that hashes can be created + taskhash = 'c665584ee6817aa99edfc77a44dd853828279370' + outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44' + unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824' + + result = self.client.get_unihash(self.METHOD, taskhash) + self.assertIsNone(result, msg='Found unexpected task, %r' % result) + + siginfo = "0" * (self.client.max_chunk * 4) + + result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash, { + 'outhash_siginfo': siginfo + }) + self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') + + result = self.client.get_taskhash(self.METHOD, taskhash, True) + self.assertEqual(result['taskhash'], taskhash) + self.assertEqual(result['unihash'], unihash) + self.assertEqual(result['method'], self.METHOD) + self.assertEqual(result['outhash'], outhash) + self.assertEqual(result['outhash_siginfo'], siginfo) + def test_stress(self): def query_server(failures): client = Client(self.server.address) @@ -139,4 +164,8 @@ class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase): def get_server_addr(self): - return "localhost:0" + # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled. + # If IPv6 is enabled, it should be safe to use localhost directly, in general + # case it is more reliable to resolve the IP address explicitly. + return socket.gethostbyname("localhost") + ":0" + diff --git a/poky/bitbake/lib/layerindexlib/__init__.py b/poky/bitbake/lib/layerindexlib/__init__.py index 77196b408..45157b668 100644 --- a/poky/bitbake/lib/layerindexlib/__init__.py +++ b/poky/bitbake/lib/layerindexlib/__init__.py @@ -7,6 +7,7 @@ import datetime import logging import imp +import os from collections import OrderedDict from layerindexlib.plugin import LayerIndexPluginUrlError @@ -70,7 +71,7 @@ class LayerIndex(): if self.__class__ != newIndex.__class__ or \ other.__class__ != newIndex.__class__: - raise TypeException("Can not add different types.") + raise TypeError("Can not add different types.") for indexEnt in self.indexes: newIndex.indexes.append(indexEnt) @@ -266,8 +267,8 @@ will write out the individual elements split by layer and related components. logger.debug(1, "Store not implemented in %s" % plugin.type) pass else: - logger.debug(1, "No plugins support %s" % url) - raise LayerIndexException("No plugins support %s" % url) + logger.debug(1, "No plugins support %s" % indexURI) + raise LayerIndexException("No plugins support %s" % indexURI) def is_empty(self): @@ -657,7 +658,7 @@ class LayerIndexObj(): if obj.id in self._index[indexname]: if self._index[indexname][obj.id] == obj: continue - raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id)) + raise LayerIndexException('Conflict adding object %s(%s) to index' % (indexname, obj.id)) self._index[indexname][obj.id] = obj def add_raw_element(self, indexname, objtype, rawobjs): @@ -842,11 +843,11 @@ class LayerIndexObj(): def _resolve_dependencies(layerbranches, ignores, dependencies, invalid): for layerbranch in layerbranches: - if ignores and layerBranch.layer.name in ignores: + if ignores and layerbranch.layer.name in ignores: continue - for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]: - deplayerbranch = layerDependency.dependency_layerBranch + for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: + deplayerbranch = layerdependency.dependency_layerBranch if ignores and deplayerbranch.layer.name in ignores: continue diff --git a/poky/bitbake/lib/layerindexlib/cooker.py b/poky/bitbake/lib/layerindexlib/cooker.py index 65b23d087..21ec438a2 100644 --- a/poky/bitbake/lib/layerindexlib/cooker.py +++ b/poky/bitbake/lib/layerindexlib/cooker.py @@ -4,6 +4,7 @@ # import logging +import os from collections import defaultdict @@ -73,7 +74,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin): d = self.layerindex.data if not branches: - raise LayerIndexFetchError("No branches specified for _load_bblayers!") + raise layerindexlib.LayerIndexFetchError("No branches specified for _load_bblayers!") index = layerindexlib.LayerIndexObj() @@ -202,7 +203,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin): try: depDict = bb.utils.explode_dep_versions2(deps) except bb.utils.VersionStringException as vse: - bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (collection, str(vse))) for dep, oplist in list(depDict.items()): # We need to search ourselves, so use the _ version... @@ -268,7 +269,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin): layer = bb.utils.get_file_layer(realfn[0], self.config_data) - depBranchId = collection_layerbranch[layer] + depBranchId = collection[layer] recipeId += 1 recipe = layerindexlib.Recipe(index, None) diff --git a/poky/bitbake/lib/layerindexlib/restapi.py b/poky/bitbake/lib/layerindexlib/restapi.py index 21fd14414..7023f42f2 100644 --- a/poky/bitbake/lib/layerindexlib/restapi.py +++ b/poky/bitbake/lib/layerindexlib/restapi.py @@ -5,9 +5,13 @@ import logging import json +import os + from urllib.parse import unquote from urllib.parse import urlparse +import bb + import layerindexlib import layerindexlib.plugin @@ -163,7 +167,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin): parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) logger.debug(1, "%s: retry successful.") else: - raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) + raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) return parsed diff --git a/poky/bitbake/lib/layerindexlib/tests/restapi.py b/poky/bitbake/lib/layerindexlib/tests/restapi.py index e5ccafe5c..4646d01f9 100644 --- a/poky/bitbake/lib/layerindexlib/tests/restapi.py +++ b/poky/bitbake/lib/layerindexlib/tests/restapi.py @@ -112,7 +112,7 @@ class LayerIndexWebRestApiTest(LayersTest): break else: self.logger.debug(1, "meta-python was not found") - self.assetTrue(False) + raise self.failureException # Only check the first element... break diff --git a/poky/bitbake/lib/ply/lex.py b/poky/bitbake/lib/ply/lex.py index 267ec100f..182f2e837 100644 --- a/poky/bitbake/lib/ply/lex.py +++ b/poky/bitbake/lib/ply/lex.py @@ -705,11 +705,7 @@ class LexerReflect(object): # Sort the functions by line number for f in self.funcsym.values(): - if sys.version_info[0] < 3: - f.sort(lambda x,y: cmp(func_code(x[1]).co_firstlineno,func_code(y[1]).co_firstlineno)) - else: - # Python 3.0 - f.sort(key=lambda x: func_code(x[1]).co_firstlineno) + f.sort(key=lambda x: func_code(x[1]).co_firstlineno) # Sort the strings by regular expression length for s in self.strsym.values(): diff --git a/poky/bitbake/lib/ply/yacc.py b/poky/bitbake/lib/ply/yacc.py index 561784f2f..46e7dc96f 100644 --- a/poky/bitbake/lib/ply/yacc.py +++ b/poky/bitbake/lib/ply/yacc.py @@ -1205,7 +1205,7 @@ class Production(object): # Precompute the list of productions immediately following. Hack. Remove later try: - p.lr_after = Prodnames[p.prod[n+1]] + p.lr_after = self.Prodnames[p.prod[n+1]] except (IndexError,KeyError): p.lr_after = [] try: diff --git a/poky/bitbake/lib/toaster/tests/functional/functional_helpers.py b/poky/bitbake/lib/toaster/tests/functional/functional_helpers.py index 455c408e9..5c4ea7179 100644 --- a/poky/bitbake/lib/toaster/tests/functional/functional_helpers.py +++ b/poky/bitbake/lib/toaster/tests/functional/functional_helpers.py @@ -75,7 +75,7 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase): try: table_element = self.get_table_element(table_id) element = table_element.find_element_by_link_text(link_text) - except NoSuchElementException as e: + except self.NoSuchElementException: print('no element found') raise return element @@ -86,7 +86,7 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase): element_xpath = "//*[@id='" + table_id + "']" try: element = self.driver.find_element_by_xpath(element_xpath) - except NoSuchElementException as e: + except self.NoSuchElementException: raise return element row = coordinate[0] @@ -96,7 +96,7 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase): element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]" try: element = self.driver.find_element_by_xpath(element_xpath) - except NoSuchElementException as e: + except self.NoSuchElementException: return False return element #now we are looking for an element with specified X and Y @@ -105,6 +105,6 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase): element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]/td[" + str(column) + "]" try: element = self.driver.find_element_by_xpath(element_xpath) - except NoSuchElementException as e: + except self.NoSuchElementException: return False return element diff --git a/poky/bitbake/toaster-requirements.txt b/poky/bitbake/toaster-requirements.txt index a682b085d..735b61454 100644 --- a/poky/bitbake/toaster-requirements.txt +++ b/poky/bitbake/toaster-requirements.txt @@ -1,3 +1,3 @@ -Django>1.8,<1.12 +Django>2.2,<2.3 beautifulsoup4>=4.4.0 pytz |