diff options
Diffstat (limited to 'poky/bitbake')
| -rwxr-xr-x | poky/bitbake/bin/bitbake | 2 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml | 2 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml | 60 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml | 61 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/__init__.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/cache.py | 4 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/cookerdata.py | 7 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/gitsm.py | 45 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/perforce.py | 81 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 3 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/utils.py | 1 |
11 files changed, 249 insertions, 19 deletions
diff --git a/poky/bitbake/bin/bitbake b/poky/bitbake/bin/bitbake index 6c73710cf..61db6b70f 100755 --- a/poky/bitbake/bin/bitbake +++ b/poky/bitbake/bin/bitbake @@ -26,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException if sys.getfilesystemencoding() != "utf-8": sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.") -__version__ = "1.46.0" +__version__ = "1.47.0" if __name__ == "__main__": if __version__ != bb.__version__: diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml index e4251dff5..04c5a26b9 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml @@ -765,7 +765,7 @@ the concept: <literallayout class='monospaced'> BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \ - SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \ + SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \ USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX" diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml index d1bfc2336..fe4372ade 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml @@ -736,6 +736,10 @@ The fetcher's temporary working directory is set by <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>, which defaults to "DL_DIR/p4". + The fetcher does not make use of a perforce client, instead it + relies on <filename>p4 files</filename> to retrieve a list of + files and <filename>p4 print</filename> to transfer the content + of those files locally. </para> <para> @@ -783,6 +787,62 @@ to <filename>"${WORKDIR}/p4"</filename> in your recipe. </note> </para> + + <para> + By default, the fetcher strips the depot location from the + local file paths. In the above example, the content of + <filename>example-depot/main/source/</filename> + will be placed in <filename>${WORKDIR}/p4</filename>. + For situations where preserving parts of the remote depot paths + locally is desirable, the fetcher supports two parameters: + + <itemizedlist> + <listitem><para> + <emphasis>"module":</emphasis> + The top-level depot location or directory to fetch. The + value of this parameter can also point to a single file + within the depot, in which case the local file path will + include the module path. + </para></listitem> + <listitem><para> + <emphasis>"remotepath":</emphasis> + When used with the value "<filename>keep</filename>", + the fetcher will mirror the full depot paths locally + for the specified location, even in combination with + the <filename>module</filename> parameter. + </para></listitem> + </itemizedlist> + </para> + + <para> + Here is an example use of the the <filename>module</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/..." + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, including the directory itself. + The top-level directory will be accesible at + <filename>${P4DIR}/source/</filename>. + </para> + + <para> + Here is an example use of the the <filename>remotepath</filename> + parameter: + + <literallayout class='monospaced'> + SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep" + </literallayout> + + In this case, the content of the top-level directory + <filename>source/</filename> will be fetched to + <filename>${P4DIR}</filename>, but the complete depot paths will + be mirrored locally. The top-level directory will be accessible + at <filename>${P4DIR}/example-depot/main/source/</filename>. + </para> </section> <section id='repo-fetcher'> diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml index c4bd1f258..4c29b2464 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml @@ -1124,6 +1124,67 @@ </glossdef> </glossentry> + <glossentry id='var-BBFILES_DYNAMIC'><glossterm>BBFILES_DYNAMIC</glossterm> + <info> + BBFILES_DYNAMIC[doc] = "Activates content depending on presence of identified layers." + </info> + <glossdef> + <para role="glossdeffirst"> + Activates content depending on presence of identified layers. + You identify the layers by the collections that the layers + define. + </para> + + <para> + Use the <filename>BBFILES_DYNAMIC</filename> variable to + avoid <filename>.bbappend</filename> files whose + corresponding <filename>.bb</filename> file is in a layer + that attempts to modify other layers through + <filename>.bbappend</filename> but does not want to + introduce a hard dependency on those other layers. + </para> + + <para> + Additionally you can prefix the rule with "!" to add + <filename>.bbappend</filename> and <filename>.bb</filename> files + in case a layer is not present. + Use this avoid hard dependency on those other layers. + </para> + + <para> + Use the following form for + <filename>BBFILES_DYNAMIC</filename>: + <literallayout class='monospaced'> + <replaceable>collection_name</replaceable>:<replaceable>filename_pattern</replaceable> + </literallayout> + The following example identifies two collection names and + two filename patterns: + <literallayout class='monospaced'> + BBFILES_DYNAMIC += "\ + clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \ + core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \ + " + </literallayout> + When the collection name is prefixed with "!" it will add the file pattern in case + the layer is absent: + <literallayout class='monospaced'> + BBFILES_DYNAMIC += "\ + !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \ + " + </literallayout> + + This next example shows an error message that occurs + because invalid entries are found, which cause parsing to + abort: + <literallayout class='monospaced'> + ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not: + /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend + /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend + </literallayout> + </para> + </glossdef> + </glossentry> + <glossentry id='var-bb-BBINCLUDED'><glossterm>BBINCLUDED</glossterm> <glossdef> <para> diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py index 4e2f97b23..2c94e10c8 100644 --- a/poky/bitbake/lib/bb/__init__.py +++ b/poky/bitbake/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "1.46.0" +__version__ = "1.47.0" import sys if sys.version_info < (3, 5, 0): diff --git a/poky/bitbake/lib/bb/cache.py b/poky/bitbake/lib/bb/cache.py index be5ea6a8b..b819a0c2d 100644 --- a/poky/bitbake/lib/bb/cache.py +++ b/poky/bitbake/lib/bb/cache.py @@ -26,7 +26,7 @@ import re logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "152" +__cache_version__ = "153" def getCacheFile(path, filename, mc, data_hash): mcspec = '' @@ -727,7 +727,7 @@ class Cache(NoCache): if self.mc is not None: (fn, cls, mc) = virtualfn2realfn(filename) if mc: - self.logger.error("Unexpected multiconfig %s", virtualfn) + self.logger.error("Unexpected multiconfig %s", filename) return vfn = realfn2virtual(fn, cls, self.mc) diff --git a/poky/bitbake/lib/bb/cookerdata.py b/poky/bitbake/lib/bb/cookerdata.py index 472423fdc..24bf09c56 100644 --- a/poky/bitbake/lib/bb/cookerdata.py +++ b/poky/bitbake/lib/bb/cookerdata.py @@ -387,10 +387,13 @@ class CookerDataBuilder(object): invalid.append(entry) continue l, f = parts - if l in collections: + invert = l[0] == "!" + if invert: + l = l[1:] + if (l in collections and not invert) or (l not in collections and invert): data.appendVar("BBFILES", " " + f) if invalid: - bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) + bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split()) collections_tmp = collections[:] diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index 56bd5f048..d6e5c5c05 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -143,12 +143,43 @@ class GitSM(Git): try: # Check for the nugget dropped by the download operation known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ - (ud.basecmd), d, workdir=ud.clonedir) + (ud.basecmd), d, workdir=ud.clonedir) - if ud.revisions[ud.names[0]] not in known_srcrevs.split(): - return True + if ud.revisions[ud.names[0]] in known_srcrevs.split(): + return False except bb.fetch2.FetchError: - # No srcrev nuggets, so this is new and needs to be updated + pass + + need_update_list = [] + def need_update_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + + try: + newfetch = Fetch([url], d, cache=False) + new_ud = newfetch.ud[url] + if new_ud.method.need_update(new_ud, d): + need_update_list.append(modpath) + except Exception as e: + logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) + need_update_result = True + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) + self.process_submodules(ud, tmpdir, need_update_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, need_update_submodule, d) + if len(need_update_list) == 0: + # We already have the required commits of all submodules. Drop + # a nugget so we don't need to check again. + runfetchcmd("%s config --add bitbake.srcrev %s" % \ + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) + + if len(need_update_list) > 0: + logger.debug(1, 'gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) return True return False @@ -163,9 +194,6 @@ class GitSM(Git): try: newfetch = Fetch([url], d, cache=False) newfetch.download() - # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir) except Exception as e: logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) raise @@ -181,6 +209,9 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, download_submodule, d) + # Drop a nugget for the srcrev we've fetched (used by need_update) + runfetchcmd("%s config --add bitbake.srcrev %s" % \ + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) def unpack(self, ud, destdir, d): def unpack_submodules(ud, url, module, modpath, workdir, d): diff --git a/poky/bitbake/lib/bb/fetch2/perforce.py b/poky/bitbake/lib/bb/fetch2/perforce.py index f57c2a4f5..6f3c95b6c 100644 --- a/poky/bitbake/lib/bb/fetch2/perforce.py +++ b/poky/bitbake/lib/bb/fetch2/perforce.py @@ -1,6 +1,20 @@ """ BitBake 'Fetch' implementation for perforce +Supported SRC_URI options are: + +- module + The top-level location to fetch while preserving the remote paths + + The value of module can point to either a directory or a file. The result, + in both cases, is that the fetcher will preserve all file paths starting + from the module path. That is, the top-level directory in the module value + will also be the top-level directory in P4DIR. + +- remotepath + If the value "keep" is given, the full depot location of each file is + preserved in P4DIR. This option overrides the effect of the module option. + """ # Copyright (C) 2003, 2004 Chris Larson @@ -17,6 +31,36 @@ from bb.fetch2 import FetchError from bb.fetch2 import logger from bb.fetch2 import runfetchcmd +class PerforceProgressHandler (bb.progress.BasicProgressHandler): + """ + Implements basic progress information for perforce, based on the number of + files to be downloaded. + + The p4 print command will print one line per file, therefore it can be used + to "count" the number of files already completed and give an indication of + the progress. + """ + def __init__(self, d, num_files): + self._num_files = num_files + self._count = 0 + super(PerforceProgressHandler, self).__init__(d) + + # Send an initial progress event so the bar gets shown + self._fire_progress(-1) + + def write(self, string): + self._count = self._count + 1 + + percent = int(100.0 * float(self._count) / float(self._num_files)) + + # In case something goes wrong, we try to preserve our sanity + if percent > 100: + percent = 100 + + self.update(percent) + + super(PerforceProgressHandler, self).write(string) + class Perforce(FetchMethod): """ Class to fetch from perforce repositories """ def supports(self, ud, d): @@ -58,14 +102,33 @@ class Perforce(FetchMethod): logger.debug(1, 'Determined P4PORT to be: %s' % ud.host) if not ud.host: raise FetchError('Could not determine P4PORT from P4CONFIG') - + + # Fetcher options + ud.module = ud.parm.get('module') + ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep') + if ud.path.find('/...') >= 0: ud.pathisdir = True else: ud.pathisdir = False + # Avoid using the "/..." syntax in SRC_URI when a module value is given + if ud.pathisdir and ud.module: + raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given') + cleanedpath = ud.path.replace('/...', '').replace('/', '.') cleanedhost = ud.host.replace(':', '.') + + # Merge the path and module into the final depot location + if ud.module: + if ud.module.find('/') == 0: + raise FetchError('module cannot begin with /') + ud.path = os.path.join(ud.path, ud.module) + + # Append the module path to the local pkg name + cleanedmodule = ud.module.replace('/...', '').replace('/', '.') + cleanedpath += '--%s' % cleanedmodule + ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath) ud.setup_revisions(d) @@ -95,10 +158,20 @@ class Perforce(FetchMethod): pathnrev = '%s' % (ud.path) if depot_filename: - if ud.pathisdir: # Remove leading path to obtain filename + if ud.keepremotepath: + # preserve everything, remove the leading // + filename = depot_filename.lstrip('/') + elif ud.module: + # remove everything up to the module path + modulepath = ud.module.rstrip('/...') + filename = depot_filename[depot_filename.rfind(modulepath):] + elif ud.pathisdir: + # Remove leading (visible) path to obtain the filepath filename = depot_filename[len(ud.path)-1:] else: + # Remove everything, except the filename filename = depot_filename[depot_filename.rfind('/'):] + filename = filename[:filename.find('#')] # Remove trailing '#rev' if command == 'changes': @@ -150,10 +223,12 @@ class Perforce(FetchMethod): bb.utils.remove(ud.pkgdir, True) bb.utils.mkdirhier(ud.pkgdir) + progresshandler = PerforceProgressHandler(d, len(filelist)) + for afile in filelist: p4fetchcmd = self._buildp4command(ud, d, 'print', afile) bb.fetch2.check_network_access(d, p4fetchcmd, ud.url) - runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir) + runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler) runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir) diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index d0c161a7f..4697ef59b 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -1152,7 +1152,8 @@ class FetchLatestVersionTest(FetcherTest): ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") : "1.99.4", # version pattern "vX.Y" - ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") + # mirror of git.infradead.org since network issues interfered with testing + ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") : "1.5.0", # version pattern "pkg_name-X.Y" ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index 5f5767c1d..50032e50c 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -580,7 +580,6 @@ def preserved_envvars_exported(): 'PATH', 'PWD', 'SHELL', - 'TERM', 'USER', 'LC_ALL', 'BBSERVER', |