diff options
Diffstat (limited to 'poky/bitbake')
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst | 78 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 55 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 18 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/gitsm.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/local.py | 8 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/parse/parse_py/BBHandler.py | 4 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py | 11 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/server/process.py | 7 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 32 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/parse.py | 23 |
11 files changed, 195 insertions, 45 deletions
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index f9d9e617f3..4dfb7889e8 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -688,6 +688,8 @@ Here is an example URL:: It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable. +.. _crate-fetcher: + Crate Fetcher (``crate://``) ---------------------------- @@ -704,6 +706,80 @@ Here is an example URL:: SRC_URI = "crate://crates.io/glob/0.2.11" +.. _npm-fetcher: + +NPM Fetcher (``npm://``) +------------------------ + +This submodule fetches source code from an +`NPM <https://en.wikipedia.org/wiki/Npm_(software)>`__ +Javascript package registry. + +The format for the :term:`SRC_URI` setting must be:: + + SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." + +This fetcher supports the following parameters: + +- *"package":* The NPM package name. This is a mandatory parameter. + +- *"version":* The NPM package version. This is a mandatory parameter. + +- *"downloadfilename":* Specifies the filename used when storing the downloaded file. + +- *"destsuffix":* Specifies the directory to use to unpack the package (default: ``npm``). + +Note that NPM fetcher only fetches the package source itself. The dependencies +can be fetched through the `npmsw-fetcher`_. + +Here is an example URL with both fetchers:: + + SRC_URI = " \ + npm://registry.npmjs.org/;package=cute-files;version=${PV} \ + npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \ + " + +See :yocto_docs:`Creating Node Package Manager (NPM) Packages +</dev-manual/common-tasks.html#creating-node-package-manager-npm-packages>` +in the Yocto Project manual for details about using +:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>` +to automatically create a recipe from an NPM URL. + +.. _npmsw-fetcher: + +NPM shrinkwrap Fetcher (``npmsw://``) +------------------------------------- + +This submodule fetches source code from an +`NPM shrinkwrap <https://docs.npmjs.com/cli/v8/commands/npm-shrinkwrap>`__ +description file, which lists the dependencies +of an NPM package while locking their versions. + +The format for the :term:`SRC_URI` setting must be:: + + SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..." + +This fetcher supports the following parameters: + +- *"dev":* Set this parameter to ``1`` to install "devDependencies". + +- *"destsuffix":* Specifies the directory to use to unpack the dependencies + (``${S}`` by default). + +Note that the shrinkwrap file can also be provided by the recipe for +the package which has such dependencies, for example:: + + SRC_URI = " \ + npm://registry.npmjs.org/;package=cute-files;version=${PV} \ + npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \ + " + +Such a file can automatically be generated using +:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>` +as described in the :yocto_docs:`Creating Node Package Manager (NPM) Packages +</dev-manual/common-tasks.html#creating-node-package-manager-npm-packages>` +section of the Yocto Project. + Other Fetchers -------------- @@ -713,8 +789,6 @@ Fetch submodules also exist for the following: - Mercurial (``hg://``) -- npm (``npm://``) - - OSC (``osc://``) - Secure FTP (``sftp://``) diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index af4b135867..337821612c 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -195,22 +195,45 @@ value. However, if ``A`` is not set, the variable is set to "aval". Setting a weak default value (??=) ---------------------------------- -It is possible to use a "weaker" assignment than in the previous section -by using the "??=" operator. This assignment behaves identical to "?=" -except that the assignment is made at the end of the parsing process -rather than immediately. Consequently, when multiple "??=" assignments -exist, the last one is used. Also, any "=" or "?=" assignment will -override the value set with "??=". Here is an example:: - - A ??= "somevalue" - A ??= "someothervalue" - -If ``A`` is set before the above statements are -parsed, the variable retains its value. If ``A`` is not set, the -variable is set to "someothervalue". - -Again, this assignment is a "lazy" or "weak" assignment because it does -not occur until the end of the parsing process. +The weak default value of a variable is the value which that variable +will expand to if no value has been assigned to it via any of the other +assignment operators. The "??=" operator takes effect immediately, replacing +any previously defined weak default value. Here is an example:: + + W ??= "x" + A := "${W}" # Immediate variable expansion + W ??= "y" + B := "${W}" # Immediate variable expansion + W ??= "z" + C = "${W}" + W ?= "i" + +After parsing we will have:: + + A = "x" + B = "y" + C = "i" + W = "i" + +Appending and prepending non-override style will not substitute the weak +default value, which means that after parsing:: + + W ??= "x" + W += "y" + +we will have:: + + W = " y" + +On the other hand, override-style appends/prepends/removes are applied after +any active weak default value has been substituted:: + + W ??= "x" + W:append = "y" + +After parsing we will have:: + + W = "xy" Immediate variable expansion (:=) --------------------------------- diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index ac557176d7..0fb718b23e 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1097,6 +1097,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): def ensure_symlink(target, link_name): if not os.path.exists(link_name): + dirname = os.path.dirname(link_name) + bb.utils.mkdirhier(dirname) if os.path.islink(link_name): # Broken symbolic link os.unlink(link_name) @@ -1220,23 +1222,21 @@ def get_checksum_file_list(d): SRC_URI as a space-separated string """ fetch = Fetch([], d, cache = False, localonly = True) - - dl_dir = d.getVar('DL_DIR') filelist = [] for u in fetch.urls: ud = fetch.ud[u] - if ud and isinstance(ud.method, local.Local): + found = False paths = ud.method.localpaths(ud, d) for f in paths: pth = ud.decodedurl - if f.startswith(dl_dir): - # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else - if os.path.exists(f): - bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f))) - else: - bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f))) + if os.path.exists(f): + found = True filelist.append(f + ":" + str(os.path.exists(f))) + if not found: + bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found" + "\nThe following paths were searched:" + "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths))) return " ".join(filelist) diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 07b3d9a7ff..4534bd7580 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -738,7 +738,7 @@ class Git(FetchMethod): Compute the HEAD revision for the url """ if not d.getVar("__BBSEENSRCREV"): - raise bb.fetch2.FetchError("Recipe uses a floating tag/branch without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE).") + raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path)) # Ensure we mark as not cached bb.fetch2.get_autorev(d) diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index c5c23d5260..c1950e4819 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -88,7 +88,7 @@ class GitSM(Git): subrevision[m] = module_hash.split()[2] # Convert relative to absolute uri based on parent uri - if uris[m].startswith('..'): + if uris[m].startswith('..') or uris[m].startswith('./'): newud = copy.copy(ud) newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) uris[m] = Git._get_repo_url(self, newud) diff --git a/poky/bitbake/lib/bb/fetch2/local.py b/poky/bitbake/lib/bb/fetch2/local.py index e7d1c8c58f..0bb987c644 100644 --- a/poky/bitbake/lib/bb/fetch2/local.py +++ b/poky/bitbake/lib/bb/fetch2/local.py @@ -57,12 +57,6 @@ class Local(FetchMethod): logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) newpath, hist = bb.utils.which(filespath, path, history=True) searched.extend(hist) - if not os.path.exists(newpath): - dldirfile = os.path.join(d.getVar("DL_DIR"), path) - logger.debug2("Defaulting to %s for %s" % (dldirfile, path)) - bb.utils.mkdirhier(os.path.dirname(dldirfile)) - searched.append(dldirfile) - return searched return searched def need_update(self, ud, d): @@ -78,8 +72,6 @@ class Local(FetchMethod): filespath = d.getVar('FILESPATH') if filespath: locations = filespath.split(":") - locations.append(d.getVar("DL_DIR")) - msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n " + "\n ".join(locations) raise FetchError(msg) diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index ee9bd760ce..68415735fd 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -178,10 +178,10 @@ def feeder(lineno, s, fn, root, statements, eof=False): if s and s[0] == '#': if len(__residue__) != 0 and __residue__[0][0] != "#": - bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s)) + bb.fatal("There is a comment on line %s of file %s:\n'''\n%s\n'''\nwhich is in the middle of a multiline expression. This syntax is invalid, please correct it." % (lineno, fn, s)) if len(__residue__) != 0 and __residue__[0][0] == "#" and (not s or s[0] != "#"): - bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + bb.fatal("There is a confusing multiline partially commented expression on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (lineno - len(__residue__), fn, "\n".join(__residue__))) if s and s[-1] == '\\': __residue__.append(s[:-1]) diff --git a/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py index b895d5b5ef..451e68dd66 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -125,16 +125,21 @@ def handle(fn, data, include): s = f.readline() if not s: break + origlineno = lineno + origline = s w = s.strip() # skip empty lines if not w: continue s = s.rstrip() while s[-1] == '\\': - s2 = f.readline().rstrip() + line = f.readline() + origline += line + s2 = line.rstrip() lineno = lineno + 1 if (not s2 or s2 and s2[0] != "#") and s[0] == "#" : - bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + bb.fatal("There is a confusing multiline, partially commented expression starting on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (origlineno, fn, origline)) + s = s[:-1] + s2 # skip comments if s[0] == '#': @@ -147,8 +152,6 @@ def handle(fn, data, include): if oldfile: data.setVar('FILE', oldfile) - f.close() - for f in confFilters: f(fn, data) diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 43790b6631..5d02c0b9f5 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -756,8 +756,11 @@ class ConnectionWriter(object): process.queue_signals = True self._send(obj) process.queue_signals = False - for sig in process.signal_received.pop(): - process.handle_sig(sig, None) + try: + for sig in process.signal_received.pop(): + process.handle_sig(sig, None) + except IndexError: + pass else: self._send(obj) diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 622c46a05d..7fcf57e7ea 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -468,6 +468,7 @@ class MirrorUriTest(FetcherTest): "http://.*/.* file:///someotherpath/downloads/" def test_urireplace(self): + self.d.setVar("FILESPATH", ".") for k, v in self.replaceuris.items(): ud = bb.fetch.FetchData(k[0], self.d) ud.setup_localpath(self.d) @@ -693,6 +694,11 @@ class FetcherLocalTest(FetcherTest): flst.sort() return flst + def test_local_checksum_fails_no_file(self): + self.d.setVar("SRC_URI", "file://404") + with self.assertRaises(bb.BBHandledException): + bb.fetch.get_checksum_file_list(self.d) + def test_local(self): tree = self.fetchUnpack(['file://a', 'file://dir/c']) self.assertEqual(tree, ['a', 'dir/c']) @@ -920,6 +926,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_file_mirror_of_mirror(self): + self.d.setVar("FILESPATH", ".") self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) os.mkdir(self.dldir + "/some2where") @@ -2895,3 +2902,28 @@ class FetchPremirroronlyNetworkTest(FetcherTest): fetcher = bb.fetch.Fetch([self.recipe_url], self.d) with self.assertRaises(bb.fetch2.NetworkAccess): fetcher.download() + +class FetchPremirroronlyBrokenTarball(FetcherTest): + + def setUp(self): + super(FetchPremirroronlyBrokenTarball, self).setUp() + self.mirrordir = os.path.join(self.tempdir, "mirrors") + os.mkdir(self.mirrordir) + self.reponame = "bitbake" + self.gitdir = os.path.join(self.tempdir, "git", self.reponame) + self.recipe_url = "git://git.fake.repo/bitbake" + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "1") + self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") + self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" + with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: + targz.write("This is not tar.gz file!") + + def test_mirror_broken_download(self): + import sys + self.d.setVar("SRCREV", "0"*40) + fetcher = bb.fetch.Fetch([self.recipe_url], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + stdout = sys.stdout.getvalue() + self.assertFalse(" not a git repository (or any parent up to mount point /)" in stdout) diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py index 2898f9bb14..1a3b74934d 100644 --- a/poky/bitbake/lib/bb/tests/parse.py +++ b/poky/bitbake/lib/bb/tests/parse.py @@ -194,3 +194,26 @@ deltask ${EMPTYVAR} self.assertTrue('addtask ignored: " do_patch"' in stdout) #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout) + broken_multiline_comment = """ +# First line of comment \\ +# Second line of comment \\ + +""" + def test_parse_broken_multiline_comment(self): + f = self.parsehelper(self.broken_multiline_comment) + with self.assertRaises(bb.BBHandledException): + d = bb.parse.handle(f.name, self.d)[''] + + + comment_in_var = """ +VAR = " \\ + SOMEVAL \\ +# some comment \\ + SOMEOTHERVAL \\ +" +""" + def test_parse_comment_in_var(self): + f = self.parsehelper(self.comment_in_var) + with self.assertRaises(bb.BBHandledException): + d = bb.parse.handle(f.name, self.d)[''] + |