diff options
Diffstat (limited to 'poky/bitbake/lib/bb')
-rw-r--r-- | poky/bitbake/lib/bb/cooker.py | 3 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 2 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 12 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/siggen.py | 33 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 52 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/tests/siggen.py | 91 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/ui/taskexp.py | 7 | ||||
-rw-r--r-- | poky/bitbake/lib/bb/utils.py | 11 |
8 files changed, 191 insertions, 20 deletions
diff --git a/poky/bitbake/lib/bb/cooker.py b/poky/bitbake/lib/bb/cooker.py index 5442f7d22..1f4cc1e96 100644 --- a/poky/bitbake/lib/bb/cooker.py +++ b/poky/bitbake/lib/bb/cooker.py @@ -1679,7 +1679,8 @@ class BBCooker: if self.hashserv: self.hashserv.process.terminate() self.hashserv.process.join() - bb.event.fire(CookerExit(), self.data) + if hasattr(self, "data"): + bb.event.fire(CookerExit(), self.data) def shutdown(self, force = False): if force: diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 7ec1fea5d..551bfb70f 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1520,7 +1520,7 @@ class FetchMethod(object): if urlpath.find("/") != -1: destdir = urlpath.rsplit("/", 1)[0] + '/' bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) - cmd = 'cp -fpPRH %s %s' % (file, destdir) + cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) if not cmd: return diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 644ba9238..07064c694 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -236,7 +236,7 @@ class Git(FetchMethod): ud.unresolvedrev[name] = ud.revisions[name] ud.revisions[name] = self.latest_revision(ud, d, name) - gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.')) + gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) if gitsrcname.startswith('.'): gitsrcname = gitsrcname[1:] @@ -342,7 +342,7 @@ class Git(FetchMethod): # We do this since git will use a "-l" option automatically for local urls where possible if repourl.startswith("file://"): repourl = repourl[7:] - clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir) + clone_cmd = "LANG=C %s clone --bare --mirror \"%s\" %s --progress" % (ud.basecmd, repourl, ud.clonedir) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, clone_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -354,8 +354,8 @@ class Git(FetchMethod): if "origin" in output: runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + runfetchcmd("%s remote add --mirror=fetch origin \"%s\"" % (ud.basecmd, repourl), d, workdir=ud.clonedir) + fetch_cmd = "LANG=C %s fetch -f --progress \"%s\" refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -504,7 +504,7 @@ class Git(FetchMethod): raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) repourl = self._get_repo_url(ud) - runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) + runfetchcmd("%s remote set-url origin \"%s\"" % (ud.basecmd, repourl), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if need_lfs and not self._find_git_lfs(d): @@ -623,7 +623,7 @@ class Git(FetchMethod): d.setVar('_BB_GIT_IN_LSREMOTE', '1') try: repourl = self._get_repo_url(ud) - cmd = "%s ls-remote %s %s" % \ + cmd = "%s ls-remote \"%s\" %s" % \ (ud.basecmd, repourl, search) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, cmd, repourl) diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index ad49d1e2a..86e0e16f3 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -358,7 +358,8 @@ class SignatureGeneratorBasic(SignatureGenerator): else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid] - bb.utils.mkdirhier(os.path.dirname(sigfile)) + with bb.utils.umask(0o002): + bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task @@ -746,16 +747,26 @@ def list_inline_diff(oldlist, newlist, colors=None): ret.append(item) return '[%s]' % (', '.join(ret)) -def clean_basepath(a): - mc = None - if a.startswith("mc:"): - _, mc, a = a.split(":", 2) - b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] - if a.startswith("virtual:"): - b = b + ":" + a.rsplit(":", 2)[0] - if mc: - b = b + ":mc:" + mc - return b +def clean_basepath(basepath): + basepath, dir, recipe_task = basepath.rsplit("/", 2) + cleaned = dir + '/' + recipe_task + + if basepath[0] == '/': + return cleaned + + if basepath.startswith("mc:"): + mc, mc_name, basepath = basepath.split(":", 2) + mc_suffix = ':mc:' + mc_name + else: + mc_suffix = '' + + # mc stuff now removed from basepath. Whatever was next, if present will be the first + # suffix. ':/', recipe path start, marks the end of this. Something like + # 'virtual:a[:b[:c]]:/path...' (b and c being optional) + if basepath[0] != '/': + cleaned += ':' + basepath.split(':/', 1)[0] + + return cleaned + mc_suffix def clean_basepaths(a): b = {} diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 0ecf044f3..5a4db9ca4 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -223,6 +223,21 @@ class URITest(unittest.TestCase): 'query': {}, 'relative': False }, + "git://tfs-example.org:22/tfs/example%20path/example.git": { + 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', + 'scheme': 'git', + 'hostname': 'tfs-example.org', + 'port': 22, + 'hostport': 'tfs-example.org:22', + 'path': '/tfs/example path/example.git', + 'userinfo': '', + 'userinfo': '', + 'username': '', + 'password': '', + 'params': {}, + 'query': {}, + 'relative': False + }, "http://somesite.net;someparam=1": { 'uri': 'http://somesite.net;someparam=1', 'scheme': 'http', @@ -584,6 +599,7 @@ class FetcherLocalTest(FetcherTest): touch(os.path.join(self.localsrcdir, 'dir', 'd')) os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) + touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) self.d.setVar("FILESPATH", self.localsrcdir) def fetchUnpack(self, uris): @@ -601,6 +617,10 @@ class FetcherLocalTest(FetcherTest): tree = self.fetchUnpack(['file://a', 'file://dir/c']) self.assertEqual(tree, ['a', 'dir/c']) + def test_local_backslash(self): + tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) + self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) + def test_local_wildcard(self): with self.assertRaises(bb.fetch2.ParameterError): tree = self.fetchUnpack(['file://a', 'file://dir/*']) @@ -2080,6 +2100,38 @@ class GitLfsTest(FetcherTest): shutil.rmtree(self.gitdir, ignore_errors=True) fetcher.unpack(self.d.getVar('WORKDIR')) +class GitURLWithSpacesTest(FetcherTest): + test_git_urls = { + "git://tfs-example.org:22/tfs/example%20path/example.git" : { + 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git', + 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', + 'path': '/tfs/example path/example.git' + }, + "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : { + 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git', + 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', + 'path': '/tfs/example path/example repo.git' + } + } + + def test_urls(self): + + # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo + self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') + + for test_git_url, ref in self.test_git_urls.items(): + + fetcher = bb.fetch.Fetch([test_git_url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertEqual(ud.url, ref['url']) + self.assertEqual(ud.path, ref['path']) + self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) + self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) + self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) + class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil diff --git a/poky/bitbake/lib/bb/tests/siggen.py b/poky/bitbake/lib/bb/tests/siggen.py new file mode 100644 index 000000000..c21ab4e4f --- /dev/null +++ b/poky/bitbake/lib/bb/tests/siggen.py @@ -0,0 +1,91 @@ +# +# BitBake Test for lib/bb/siggen.py +# +# Copyright (C) 2020 Jean-François Dagenais +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import logging +import bb +import time + +logger = logging.getLogger('BitBake.TestSiggen') + +import bb.siggen + +class SiggenTest(unittest.TestCase): + + def test_clean_basepath_simple_target_basepath(self): + basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_basic_virtual_basepath(self): + basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_basepath(self): + basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_virtual_long_prefix_basepath(self): + basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_virtual_basepath(self): + basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + def test_clean_basepath_mc_virtual_long_prefix_basepath(self): + basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask' + expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X' + + actual_cleaned = bb.siggen.clean_basepath(basepath) + + self.assertEqual(actual_cleaned, expected_cleaned) + + + # def test_clean_basepath_performance(self): + # input_basepaths = [ + # 'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # 'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask', + # ] + + # time_start = time.time() + + # i = 2000000 + # while i >= 0: + # for basepath in input_basepaths: + # bb.siggen.clean_basepath(basepath) + # i -= 1 + + # elapsed = time.time() - time_start + # print('{} ({}s)'.format(self.id(), round(elapsed, 3))) + + # self.assertTrue(False) diff --git a/poky/bitbake/lib/bb/ui/taskexp.py b/poky/bitbake/lib/bb/ui/taskexp.py index 05e32338c..2b246710c 100644 --- a/poky/bitbake/lib/bb/ui/taskexp.py +++ b/poky/bitbake/lib/bb/ui/taskexp.py @@ -58,7 +58,12 @@ class PackageReverseDepView(Gtk.TreeView): self.current = None self.filter_model = model.filter_new() self.filter_model.set_visible_func(self._filter) - self.sort_model = self.filter_model.sort_new_with_model() + # The introspected API was fixed but we can't rely on a pygobject that hides this. + # https://gitlab.gnome.org/GNOME/pygobject/-/commit/9cdbc56fbac4db2de78dc080934b8f0a7efc892a + if hasattr(Gtk.TreeModelSort, "new_with_model"): + self.sort_model = Gtk.TreeModelSort.new_with_model(self.filter_model) + else: + self.sort_model = self.filter_model.sort_new_with_model() self.sort_model.set_sort_column_id(COL_DEP_PARENT, Gtk.SortType.ASCENDING) self.set_model(self.sort_model) self.append_column(Gtk.TreeViewColumn(label, Gtk.CellRendererText(), text=COL_DEP_PARENT)) diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index 0b79f92e2..f73d31fb7 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -944,6 +944,17 @@ def which(path, item, direction = 0, history = False, executable=False): return "", hist return "" +@contextmanager +def umask(new_mask): + """ + Context manager to set the umask to a specific mask, and restore it afterwards. + """ + current_mask = os.umask(new_mask) + try: + yield + finally: + os.umask(current_mask) + def to_boolean(string, default=None): if not string: return default |