15 files changed, 528 insertions, 33 deletions
diff --git a/poky/bitbake/lib/bb/tests/color.py b/poky/bitbake/lib/bb/tests/color.py
new file mode 100644
index 000000000..bf03750c6
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/color.py
@@ -0,0 +1,95 @@
+#
+# BitBake Test for ANSI color code filtering
+#
+# Copyright (C) 2020  Agilent Technologies, Inc.
+# Author: Chris Laplante <chris.laplante@agilent.com>
+#
+# SPDX-License-Identifier: MIT
+#
+
+import unittest
+import bb.progress
+import bb.data
+import bb.event
+from bb.progress import filter_color, filter_color_n
+import io
+import re
+
+
+class ProgressWatcher:
+    def __init__(self):
+        self._reports = []
+
+    def handle_event(self, event):
+        self._reports.append((event.progress, event.rate))
+
+    def reports(self):
+        return self._reports
+
+
+class ColorCodeTests(unittest.TestCase):
+    def setUp(self):
+        self.d = bb.data.init()
+        self._progress_watcher = ProgressWatcher()
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+
+    def tearDown(self):
+        bb.event.remove("bb.build.TaskProgress", None)
+
+    def test_filter_color(self):
+        input_string = "[01;35m[K~~~~~~~~~~~~^~~~~~~~[m[K"
+        filtered = filter_color(input_string)
+        self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~")
+
+    def test_filter_color_n(self):
+        input_string = "[01;35m[K~~~~~~~~~~~~^~~~~~~~[m[K"
+        filtered, code_count = filter_color_n(input_string)
+        self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~")
+        self.assertEqual(code_count, 4)
+
+    def test_LineFilterProgressHandler_color_filtering(self):
+        class CustomProgressHandler(bb.progress.LineFilterProgressHandler):
+            PROGRESS_REGEX = re.compile(r"Progress: (?P<progress>\d+)%")
+
+            def writeline(self, line):
+                match = self.PROGRESS_REGEX.match(line)
+                if match:
+                    self.update(int(match.group("progress")))
+                    return False
+                return True
+
+        buffer = io.StringIO()
+        handler = CustomProgressHandler(self.d, buffer)
+        handler.write("Program output!\n")
+        handler.write("More output!\n")
+        handler.write("Progress: [01;35m[K10[m[K%\n") # 10%
+        handler.write("Even more\n")
+        handler.write("[01;35m[KProgress: 50[m[K%\n") # 50%
+        handler.write("[01;35m[KProgress: 60[m[K%\n") # 60%
+        handler.write("Pro[01;35m[Kgress: [m[K100%\n") # 100%
+
+        expected = [(10, None), (50, None), (60, None), (100, None)]
+        self.assertEqual(self._progress_watcher.reports(), expected)
+
+        self.assertEqual(buffer.getvalue(), "Program output!\nMore output!\nEven more\n")
+
+    def test_BasicProgressHandler_color_filtering(self):
+        buffer = io.StringIO()
+        handler = bb.progress.BasicProgressHandler(self.d, outfile=buffer)
+        handler.write("[01;35m[K1[m[K%\n") # 1%
+        handler.write("[01;35m[K2[m[K%\n") # 2%
+        handler.write("[01;35m[K10[m[K%\n") # 10%
+        handler.write("[01;35m[K100[m[K%\n") # 100%
+
+        expected = [(0, None), (1, None), (2, None), (10, None), (100, None)]
+        self.assertListEqual(self._progress_watcher.reports(), expected)
+
+    def test_OutOfProgressHandler_color_filtering(self):
+        buffer = io.StringIO()
+        handler = bb.progress.OutOfProgressHandler(self.d, r'(\d+) of (\d+)', outfile=buffer)
+        handler.write("[01;35m[KText text 1 of[m[K 5") # 1/5
+        handler.write("[01;35m[KText text 3 of[m[K 5") # 3/5
+        handler.write("[01;35m[KText text 5 of[m[K 5") # 5/5
+
+        expected = [(0, None), (20.0, None), (60.0, None), (100.0, None)]
+        self.assertListEqual(self._progress_watcher.reports(), expected)
diff --git a/poky/bitbake/lib/bb/tests/cooker.py b/poky/bitbake/lib/bb/tests/cooker.py
index 74c903f01..c82d4b7b8 100644
--- a/poky/bitbake/lib/bb/tests/cooker.py
+++ b/poky/bitbake/lib/bb/tests/cooker.py
@@ -60,7 +60,7 @@ class CookerTest(unittest.TestCase):
         log_handler = LogHandler()
         logger.addHandler(log_handler)
         collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
-        collection.collection_priorities(pkgfns, self.d)
+        collection.collection_priorities(pkgfns, pkgfns, self.d)
         logger.removeHandler(log_handler)
 
         # Should be empty (no generated messages)
diff --git a/poky/bitbake/lib/bb/tests/cow.py b/poky/bitbake/lib/bb/tests/cow.py
index bf6e79fce..75142649c 100644
--- a/poky/bitbake/lib/bb/tests/cow.py
+++ b/poky/bitbake/lib/bb/tests/cow.py
@@ -4,9 +4,17 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # Copyright 2006 Holger Freyther <freyther@handhelds.org>
+# Copyright (C) 2020  Agilent Technologies, Inc.
 #
 
+import io
+import re
+import sys
 import unittest
+import contextlib
+import collections
+
+from bb.COW import COWDictBase, COWSetBase, COWDictMeta, COWSetMeta
 
 
 class COWTestCase(unittest.TestCase):
@@ -14,11 +22,61 @@ class COWTestCase(unittest.TestCase):
     Test case for the COW module from mithro
     """
 
+    def setUp(self):
+        self._track_warnings = False
+        self._warning_file = io.StringIO()
+        self._unhandled_warnings = collections.deque()
+        COWDictBase.__warn__ = self._warning_file
+
+    def tearDown(self):
+        COWDictBase.__warn__ = sys.stderr
+        if self._track_warnings:
+            self._checkAllWarningsRead()
+
+    def trackWarnings(self):
+        self._track_warnings = True
+
+    def _collectWarnings(self):
+        self._warning_file.seek(0)
+        for warning in self._warning_file:
+            self._unhandled_warnings.append(warning.rstrip("\n"))
+        self._warning_file.truncate(0)
+        self._warning_file.seek(0)
+
+    def _checkAllWarningsRead(self):
+        self._collectWarnings()
+        self.assertSequenceEqual(self._unhandled_warnings, [])
+
+    @contextlib.contextmanager
+    def checkReportsWarning(self, expected_warning):
+        self._checkAllWarningsRead()
+        yield
+        self._collectWarnings()
+        warning = self._unhandled_warnings.popleft()
+        self.assertEqual(warning, expected_warning)
+
+    def checkStrOutput(self, obj, expected_levels, expected_keys):
+        if obj.__class__ is COWDictMeta:
+            expected_class_name = "COWDict"
+        elif obj.__class__ is COWSetMeta:
+            expected_class_name = "COWSet"
+        else:
+            self.fail("obj is of unknown type {0}".format(type(obj)))
+        s = str(obj)
+        regex = re.compile(r"<(\w+) Level: (\d+) Current Keys: (\d+)>")
+        match = regex.match(s)
+        self.assertIsNotNone(match, "bad str output: '{0}'".format(s))
+        class_name = match.group(1)
+        self.assertEqual(class_name, expected_class_name)
+        levels = int(match.group(2))
+        self.assertEqual(levels, expected_levels, "wrong # levels in str: '{0}'".format(s))
+        keys = int(match.group(3))
+        self.assertEqual(keys, expected_keys, "wrong # keys in str: '{0}'".format(s))
+
     def testGetSet(self):
         """
         Test and set
         """
-        from bb.COW import COWDictBase
         a = COWDictBase.copy()
 
         self.assertEqual(False, 'a' in a)
@@ -27,16 +85,14 @@ class COWTestCase(unittest.TestCase):
         a['b'] = 'b'
         self.assertEqual(True, 'a' in a)
         self.assertEqual(True, 'b' in a)
-        self.assertEqual('a', a['a'] )
-        self.assertEqual('b', a['b'] )
+        self.assertEqual('a', a['a'])
+        self.assertEqual('b', a['b'])
 
     def testCopyCopy(self):
         """
         Test the copy of copies
         """
 
-        from bb.COW import COWDictBase
-
         # create two COW dict 'instances'
         b = COWDictBase.copy()
         c = COWDictBase.copy()
@@ -94,30 +150,168 @@ class COWTestCase(unittest.TestCase):
         self.assertEqual(False, 'e' in b_2)
 
     def testCow(self):
-        from bb.COW import COWDictBase
+        self.trackWarnings()
+
         c = COWDictBase.copy()
         c['123'] = 1027
         c['other'] = 4711
-        c['d'] = { 'abc' : 10, 'bcd' : 20 }
+        c['d'] = {'abc': 10, 'bcd': 20}
 
         copy = c.copy()
 
         self.assertEqual(1027, c['123'])
         self.assertEqual(4711, c['other'])
-        self.assertEqual({'abc':10, 'bcd':20}, c['d'])
+        self.assertEqual({'abc': 10, 'bcd': 20}, c['d'])
         self.assertEqual(1027, copy['123'])
         self.assertEqual(4711, copy['other'])
-        self.assertEqual({'abc':10, 'bcd':20}, copy['d'])
+        with self.checkReportsWarning("Warning: Doing a copy because d is a mutable type."):
+            self.assertEqual({'abc': 10, 'bcd': 20}, copy['d'])
 
         # cow it now
         copy['123'] = 1028
         copy['other'] = 4712
         copy['d']['abc'] = 20
 
-
         self.assertEqual(1027, c['123'])
         self.assertEqual(4711, c['other'])
-        self.assertEqual({'abc':10, 'bcd':20}, c['d'])
+        self.assertEqual({'abc': 10, 'bcd': 20}, c['d'])
         self.assertEqual(1028, copy['123'])
         self.assertEqual(4712, copy['other'])
-        self.assertEqual({'abc':20, 'bcd':20}, copy['d'])
+        self.assertEqual({'abc': 20, 'bcd': 20}, copy['d'])
+
+    def testOriginalTestSuite(self):
+        # This test suite is a port of the original one from COW.py
+        self.trackWarnings()
+
+        a = COWDictBase.copy()
+        self.checkStrOutput(a, 1, 0)
+
+        a['a'] = 'a'
+        a['b'] = 'b'
+        a['dict'] = {}
+        self.checkStrOutput(a, 1, 4)  # 4th member is dict__mutable__
+
+        b = a.copy()
+        self.checkStrOutput(b, 2, 0)
+        b['c'] = 'b'
+        self.checkStrOutput(b, 2, 1)
+
+        with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."):
+            self.assertListEqual(list(a.iteritems()),
+                                 [('a', 'a'),
+                                  ('b', 'b'),
+                                  ('dict', {})
+                                  ])
+
+        with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."):
+            b_gen = b.iteritems()
+        self.assertTupleEqual(next(b_gen), ('a', 'a'))
+        self.assertTupleEqual(next(b_gen), ('b', 'b'))
+        self.assertTupleEqual(next(b_gen), ('c', 'b'))
+        with self.checkReportsWarning("Warning: Doing a copy because dict is a mutable type."):
+            self.assertTupleEqual(next(b_gen), ('dict', {}))
+        with self.assertRaises(StopIteration):
+            next(b_gen)
+
+        b['dict']['a'] = 'b'
+        b['a'] = 'c'
+
+        self.checkStrOutput(a, 1, 4)
+        self.checkStrOutput(b, 2, 3)
+
+        with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."):
+            self.assertListEqual(list(a.iteritems()),
+                                 [('a', 'a'),
+                                  ('b', 'b'),
+                                  ('dict', {})
+                                  ])
+
+        with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."):
+            b_gen = b.iteritems()
+        self.assertTupleEqual(next(b_gen), ('a', 'c'))
+        self.assertTupleEqual(next(b_gen), ('b', 'b'))
+        self.assertTupleEqual(next(b_gen), ('c', 'b'))
+        self.assertTupleEqual(next(b_gen), ('dict', {'a': 'b'}))
+        with self.assertRaises(StopIteration):
+            next(b_gen)
+
+        with self.assertRaises(KeyError):
+            print(b["dict2"])
+
+        a['set'] = COWSetBase()
+        a['set'].add("o1")
+        a['set'].add("o1")
+        a['set'].add("o2")
+        self.assertSetEqual(set(a['set'].itervalues()), {"o1", "o2"})
+        self.assertSetEqual(set(b['set'].itervalues()), {"o1", "o2"})
+
+        b['set'].add('o3')
+        self.assertSetEqual(set(a['set'].itervalues()), {"o1", "o2"})
+        self.assertSetEqual(set(b['set'].itervalues()), {"o1", "o2", "o3"})
+
+        a['set2'] = set()
+        a['set2'].add("o1")
+        a['set2'].add("o1")
+        a['set2'].add("o2")
+
+        # We don't expect 'a' to change anymore
+        def check_a():
+            with self.checkReportsWarning("Warning: If you aren't going to change any of the values call with True."):
+                a_gen = a.iteritems()
+                self.assertTupleEqual(next(a_gen), ('a', 'a'))
+            self.assertTupleEqual(next(a_gen), ('b', 'b'))
+            self.assertTupleEqual(next(a_gen), ('dict', {}))
+            self.assertTupleEqual(next(a_gen), ('set2', {'o1', 'o2'}))
+            a_sub_set = next(a_gen)
+            self.assertEqual(a_sub_set[0], 'set')
+            self.checkStrOutput(a_sub_set[1], 1, 2)
+            self.assertSetEqual(set(a_sub_set[1].itervalues()), {'o1', 'o2'})
+
+        check_a()
+
+        b_gen = b.iteritems(readonly=True)
+        self.assertTupleEqual(next(b_gen), ('a', 'c'))
+        self.assertTupleEqual(next(b_gen), ('b', 'b'))
+        self.assertTupleEqual(next(b_gen), ('c', 'b'))
+        self.assertTupleEqual(next(b_gen), ('dict', {'a': 'b'}))
+        self.assertTupleEqual(next(b_gen), ('set2', {'o1', 'o2'}))
+        b_sub_set = next(b_gen)
+        self.assertEqual(b_sub_set[0], 'set')
+        self.checkStrOutput(b_sub_set[1], 2, 1)
+        self.assertSetEqual(set(b_sub_set[1].itervalues()), {'o1', 'o2', 'o3'})
+
+        del b['b']
+        with self.assertRaises(KeyError):
+            print(b['b'])
+        self.assertFalse('b' in b)
+
+        check_a()
+
+        b.__revertitem__('b')
+        check_a()
+        self.assertEqual(b['b'], 'b')
+        self.assertTrue('b' in b)
+
+        b.__revertitem__('dict')
+        check_a()
+
+        b_gen = b.iteritems(readonly=True)
+        self.assertTupleEqual(next(b_gen), ('a', 'c'))
+        self.assertTupleEqual(next(b_gen), ('b', 'b'))
+        self.assertTupleEqual(next(b_gen), ('c', 'b'))
+        self.assertTupleEqual(next(b_gen), ('dict', {}))
+        self.assertTupleEqual(next(b_gen), ('set2', {'o1', 'o2'}))
+        b_sub_set = next(b_gen)
+        self.assertEqual(b_sub_set[0], 'set')
+        self.checkStrOutput(b_sub_set[1], 2, 1)
+        self.assertSetEqual(set(b_sub_set[1].itervalues()), {'o1', 'o2', 'o3'})
+
+        self.checkStrOutput(a, 1, 6)
+        self.checkStrOutput(b, 2, 3)
+
+    def testSetMethods(self):
+        s = COWSetBase()
+        with self.assertRaises(TypeError):
+            print(s.iteritems())
+        with self.assertRaises(TypeError):
+            print(s.iterkeys())
diff --git a/poky/bitbake/lib/bb/tests/data.py b/poky/bitbake/lib/bb/tests/data.py
index 5f195047d..1d4a64b10 100644
--- a/poky/bitbake/lib/bb/tests/data.py
+++ b/poky/bitbake/lib/bb/tests/data.py
@@ -12,6 +12,7 @@ import bb
 import bb.data
 import bb.parse
 import logging
+import os
 
 class LogRecord():
     def __enter__(self):
diff --git a/poky/bitbake/lib/bb/tests/event.py b/poky/bitbake/lib/bb/tests/event.py
index 9229b63d4..9ca7e9bc8 100644
--- a/poky/bitbake/lib/bb/tests/event.py
+++ b/poky/bitbake/lib/bb/tests/event.py
@@ -6,17 +6,18 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import unittest
-import bb
-import logging
-import bb.compat
-import bb.event
+import collections
 import importlib
+import logging
+import pickle
 import threading
 import time
-import pickle
+import unittest
 from unittest.mock import Mock
 from unittest.mock import call
+
+import bb
+import bb.event
 from bb.msg import BBLogFormatter
 
 
@@ -75,7 +76,7 @@ class EventHandlingTest(unittest.TestCase):
 
     def _create_test_handlers(self):
         """ Method used to create a test handler ordered dictionary """
-        test_handlers = bb.compat.OrderedDict()
+        test_handlers = collections.OrderedDict()
         test_handlers["handler1"] = self._test_process.handler1
         test_handlers["handler2"] = self._test_process.handler2
         return test_handlers
@@ -96,7 +97,7 @@ class EventHandlingTest(unittest.TestCase):
 
     def test_clean_class_handlers(self):
         """ Test clean_class_handlers method """
-        cleanDict = bb.compat.OrderedDict()
+        cleanDict = collections.OrderedDict()
         self.assertEqual(cleanDict,
                          bb.event.clean_class_handlers())
 
diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py
index d0c161a7f..5a4db9ca4 100644
--- a/poky/bitbake/lib/bb/tests/fetch.py
+++ b/poky/bitbake/lib/bb/tests/fetch.py
@@ -223,6 +223,21 @@ class URITest(unittest.TestCase):
             'query': {},
             'relative': False
         },
+        "git://tfs-example.org:22/tfs/example%20path/example.git": {
+            'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
+            'scheme': 'git',
+            'hostname': 'tfs-example.org',
+            'port': 22,
+            'hostport': 'tfs-example.org:22',
+            'path': '/tfs/example path/example.git',
+            'userinfo': '',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {},
+            'query': {},
+            'relative': False
+        },
         "http://somesite.net;someparam=1": {
             'uri': 'http://somesite.net;someparam=1',
             'scheme': 'http',
@@ -584,6 +599,7 @@ class FetcherLocalTest(FetcherTest):
         touch(os.path.join(self.localsrcdir, 'dir', 'd'))
         os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
         touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
+        touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
         self.d.setVar("FILESPATH", self.localsrcdir)
 
     def fetchUnpack(self, uris):
@@ -601,9 +617,13 @@ class FetcherLocalTest(FetcherTest):
         tree = self.fetchUnpack(['file://a', 'file://dir/c'])
         self.assertEqual(tree, ['a', 'dir/c'])
 
+    def test_local_backslash(self):
+        tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
+        self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
+
     def test_local_wildcard(self):
-        tree = self.fetchUnpack(['file://a', 'file://dir/*'])
-        self.assertEqual(tree, ['a',  'dir/c', 'dir/d', 'dir/subdir/e'])
+        with self.assertRaises(bb.fetch2.ParameterError):
+            tree = self.fetchUnpack(['file://a', 'file://dir/*'])
 
     def test_local_dir(self):
         tree = self.fetchUnpack(['file://a', 'file://dir'])
@@ -1031,7 +1051,7 @@ class SVNTest(FetcherTest):
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
         # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .",
+        bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1152,10 +1172,12 @@ class FetchLatestVersionTest(FetcherTest):
         ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
             : "1.99.4",
         # version pattern "vX.Y"
-        ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
+        # mirror of git.infradead.org since network issues interfered with testing
+        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
             : "1.5.0",
         # version pattern "pkg_name-X.Y"
-        ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
+        # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
+        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
         ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
@@ -1169,7 +1191,8 @@ class FetchLatestVersionTest(FetcherTest):
         ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
-        ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
+                # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
+        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
             : "0.4.3",
         ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
             : "11.0.0",
@@ -1261,9 +1284,7 @@ class FetchLatestVersionTest(FetcherTest):
 
 
 class FetchCheckStatusTest(FetcherTest):
-    test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
-                      "http://www.cups.org/",
-                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
+    test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
                       "https://yoctoproject.org/",
@@ -2079,6 +2100,38 @@ class GitLfsTest(FetcherTest):
         shutil.rmtree(self.gitdir, ignore_errors=True)
         fetcher.unpack(self.d.getVar('WORKDIR'))
 
+class GitURLWithSpacesTest(FetcherTest):
+    test_git_urls = {
+        "git://tfs-example.org:22/tfs/example%20path/example.git" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
+            'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
+            'path': '/tfs/example path/example.git'
+        },
+        "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
+            'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
+            'path': '/tfs/example path/example repo.git'
+        }
+    }
+
+    def test_urls(self):
+
+        # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+        for test_git_url, ref in self.test_git_urls.items():
+
+            fetcher = bb.fetch.Fetch([test_git_url], self.d)
+            ud = fetcher.ud[fetcher.urls[0]]
+
+            self.assertEqual(ud.url, ref['url'])
+            self.assertEqual(ud.path, ref['path'])
+            self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
+            self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
+            self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
+            self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
+            self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
+
 class NPMTest(FetcherTest):
     def skipIfNoNpm():
         import shutil
diff --git a/poky/bitbake/lib/bb/tests/parse.py b/poky/bitbake/lib/bb/tests/parse.py
index 9afd1b208..9e21e1842 100644
--- a/poky/bitbake/lib/bb/tests/parse.py
+++ b/poky/bitbake/lib/bb/tests/parse.py
@@ -178,7 +178,10 @@ python () {
 addtask do_patch after do_foo after do_unpack before do_configure before do_compile
 addtask do_fetch do_patch
 
-deltask do_fetch do_patch
+MYVAR = "do_patch"
+EMPTYVAR = ""
+deltask do_fetch ${MYVAR} ${EMPTYVAR}
+deltask ${EMPTYVAR}
 """
     def test_parse_addtask_deltask(self):
         import sys
@@ -189,6 +192,5 @@ deltask do_fetch do_patch
         self.assertTrue("addtask contained multiple 'before' keywords" in stdout)
         self.assertTrue("addtask contained multiple 'after' keywords" in stdout)
         self.assertTrue('addtask ignored: " do_patch"' in stdout)
-        self.assertTrue('deltask ignored: " do_patch"' in stdout)
         #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout)
 
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 5e451fc2c..efebf001a 100644
--- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -1,7 +1,8 @@
 CACHE = "${TOPDIR}/cache"
 THISDIR = "${@os.path.dirname(d.getVar('FILE'))}"
 COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}"
-BBFILES = "${COREBASE}/recipes/*.bb"
+EXTRA_BBFILES ?= ""
+BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}"
 PROVIDES = "${PN}"
 PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
 PF = "${BB_CURRENT_MC}:${PN}"
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
index ecf23e1c7..f34b8dccc 100644
--- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf
@@ -1 +1,2 @@
 TMPDIR = "${TOPDIR}/mc1/"
+BBMASK += "recipes/fails-mc/fails-mc1.bb"
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
index eef338e4c..c3360fc5c 100644
--- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf
@@ -1 +1,2 @@
 TMPDIR = "${TOPDIR}/mc2/"
+BBMASK += "recipes/fails-mc/fails-mc2.bb"
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
new file mode 100644
index 000000000..d45a4cff5
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
@@ -0,0 +1 @@
+do_install[mcdepends] = "mc:mc1:mc2:a1:do_build"
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb
new file mode 100644
index 000000000..17a181fff
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb
@@ -0,0 +1,5 @@
+python () {
+    if d.getVar("BB_CURRENT_MC") == "mc1":
+        bb.fatal("Multiconfig is mc1")
+}
+
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
new file mode 100644
index 000000000..cc69e7b82
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
@@ -0,0 +1,4 @@
+python () {
+    if d.getVar("BB_CURRENT_MC") == "mc2":
+        bb.fatal("Multiconfig is mc2")
+}
diff --git a/poky/bitbake/lib/bb/tests/runqueue.py b/poky/bitbake/lib/bb/tests/runqueue.py
index 4ba12a077..d3d62b98f 100644
--- a/poky/bitbake/lib/bb/tests/runqueue.py
+++ b/poky/bitbake/lib/bb/tests/runqueue.py
@@ -232,6 +232,51 @@ class RunQueueTests(unittest.TestCase):
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
+    def test_multiconfig_bbmask(self):
+        # This test validates that multiconfigs can independently mask off
+        # recipes they do not want with BBMASK. It works by having recipes
+        # that will fail to parse for mc1 and mc2, then making each multiconfig
+        # build the one that does parse. This ensures that the recipes are in
+        # each multiconfigs BBFILES, but each is masking only the one that
+        # doesn't parse
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "basic",
+                "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
+            }
+            cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
+            self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
+
+    def test_multiconfig_mcdepends(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
+                "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
+            }
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in self.alltasks] + \
+                       ["mc2:a1:%s" % t for t in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+            # A rebuild does nothing
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            self.assertEqual(set(tasks), set())
+
+            # Test that a signature change in the dependent task causes
+            # mcdepends to rebuild
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc2:a1:compile"]
+            self.assertEqual(set(tasks), set(expected))
+
+            rerun_tasks = self.alltasks[:]
+            for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"):
+                rerun_tasks.remove(x)
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \
+                       ["mc2:a1:%s" % t for t in rerun_tasks]
+            self.assertEqual(set(tasks), set(expected))
 
     @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_single(self):
diff --git a/poky/bitbake/lib/bb/tests/siggen.py b/poky/bitbake/lib/bb/tests/siggen.py
new file mode 100644
index 000000000..c21ab4e4f
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/siggen.py
@@ -0,0 +1,91 @@
+#
+# BitBake Test for lib/bb/siggen.py
+#
+# Copyright (C) 2020 Jean-François Dagenais
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import logging
+import bb
+import time
+
+logger = logging.getLogger('BitBake.TestSiggen')
+
+import bb.siggen
+
+class SiggenTest(unittest.TestCase):
+
+    def test_clean_basepath_simple_target_basepath(self):
+        basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_basic_virtual_basepath(self):
+        basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_basepath(self):
+        basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_virtual_long_prefix_basepath(self):
+        basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_virtual_basepath(self):
+        basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_virtual_long_prefix_basepath(self):
+        basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+
+    # def test_clean_basepath_performance(self):
+    #     input_basepaths = [
+    #         'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #     ]
+
+    #     time_start = time.time()
+
+    #     i = 2000000
+    #     while i >= 0:
+    #         for basepath in input_basepaths:
+    #             bb.siggen.clean_basepath(basepath)
+    #         i -= 1
+
+    #     elapsed = time.time() - time_start
+    #     print('{} ({}s)'.format(self.id(), round(elapsed, 3)))
+
+    #     self.assertTrue(False)