diff options
Diffstat (limited to 'poky/bitbake/lib/bb/tests')
15 files changed, 716 insertions, 11 deletions
diff --git a/poky/bitbake/lib/bb/tests/data.py b/poky/bitbake/lib/bb/tests/data.py index 3cf5abec7..3e49984c9 100644 --- a/poky/bitbake/lib/bb/tests/data.py +++ b/poky/bitbake/lib/bb/tests/data.py @@ -381,6 +381,19 @@ class TestOverrides(unittest.TestCase): self.d.setVar("OVERRIDES", "foo:bar:some_val") self.assertEqual(self.d.getVar("TEST"), " testvalue5") + def test_append_and_override_1(self): + self.d.setVar("TEST_append", "testvalue2") + self.d.setVar("TEST_bar", "testvalue3") + self.assertEqual(self.d.getVar("TEST"), "testvalue3testvalue2") + + def test_append_and_override_2(self): + self.d.setVar("TEST_append_bar", "testvalue2") + self.assertEqual(self.d.getVar("TEST"), "testvaluetestvalue2") + + def test_append_and_override_3(self): + self.d.setVar("TEST_bar_append", "testvalue2") + self.assertEqual(self.d.getVar("TEST"), "testvalue2") + # Test an override with _<numeric> in it based on a real world OE issue def test_underscore_override(self): self.d.setVar("TARGET_ARCH", "x86_64") @@ -466,7 +479,7 @@ class TaskHash(unittest.TestCase): tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile") bb.warn(str(lookupcache)) - return basehash["somefile." + taskname] + return basehash["somefile:" + taskname] d = bb.data.init() d.setVar("__BBTASKS", ["mytask"]) diff --git a/poky/bitbake/lib/bb/tests/event.py b/poky/bitbake/lib/bb/tests/event.py index b6e40c6ae..9229b63d4 100644 --- a/poky/bitbake/lib/bb/tests/event.py +++ b/poky/bitbake/lib/bb/tests/event.py @@ -561,14 +561,6 @@ class EventClassesTest(unittest.TestCase): self.assertEqual(event.fn(1), callback(1)) self.assertEqual(event.pid, EventClassesTest._worker_pid) - def test_StampUpdate(self): - targets = ["foo", "bar"] - stampfns = [lambda:"foobar"] - event = bb.event.StampUpdate(targets, stampfns) - self.assertEqual(event.targets, targets) - self.assertEqual(event.stampPrefix, stampfns) - self.assertEqual(event.pid, EventClassesTest._worker_pid) - def test_BuildBase(self): """ Test base class for bitbake build events """ name = "foo" diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 16f975b13..a0b656b61 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -899,6 +899,7 @@ class FetcherNetworkTest(FetcherTest): if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') + @skipIfNoNetwork() def test_git_submodule_dbus_broker(self): # The following external repositories have show failures in fetch and unpack operations # We want to avoid regressions! @@ -916,6 +917,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') + @skipIfNoNetwork() def test_git_submodule_CLI11(self): url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" fetcher = bb.fetch.Fetch([url], self.d) @@ -929,6 +931,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + @skipIfNoNetwork() def test_git_submodule_update_CLI11(self): """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714" @@ -948,6 +951,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + @skipIfNoNetwork() def test_git_submodule_aktualizr(self): url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" fetcher = bb.fetch.Fetch([url], self.d) @@ -964,6 +968,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') + @skipIfNoNetwork() def test_git_submodule_iotedge(self): """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ @@ -1195,8 +1200,8 @@ class FetchLatestVersionTest(FetcherTest): # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") : "2.0.0", - ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz") - : "6.1.19", + ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://ftp.debian.org/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") + : "5.3.10", } @skipIfNoNetwork() @@ -1903,3 +1908,83 @@ class GitShallowTest(FetcherTest): dir = os.listdir(self.unpackdir + "/git/") self.assertIn("fstests.doap", dir) + +class GitLfsTest(FetcherTest): + def setUp(self): + FetcherTest.setUp(self) + + self.gitdir = os.path.join(self.tempdir, 'git') + self.srcdir = os.path.join(self.tempdir, 'gitsource') + + self.d.setVar('WORKDIR', self.tempdir) + self.d.setVar('S', self.gitdir) + self.d.delVar('PREMIRRORS') + self.d.delVar('MIRRORS') + + self.d.setVar('SRCREV', '${AUTOREV}') + self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') + + bb.utils.mkdirhier(self.srcdir) + self.git('init', cwd=self.srcdir) + with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: + attrs.write('*.mp3 filter=lfs -text') + self.git(['add', '.gitattributes'], cwd=self.srcdir) + self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) + + def git(self, cmd, cwd=None): + if isinstance(cmd, str): + cmd = 'git ' + cmd + else: + cmd = ['git'] + cmd + if cwd is None: + cwd = self.gitdir + return bb.process.run(cmd, cwd=cwd)[0] + + def fetch(self, uri=None): + uris = self.d.getVar('SRC_URI').split() + uri = uris[0] + d = self.d + + fetcher = bb.fetch2.Fetch(uris, d) + fetcher.download() + ud = fetcher.ud[uri] + return fetcher, ud + + def test_lfs_enabled(self): + import shutil + + uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir + self.d.setVar('SRC_URI', uri) + + fetcher, ud = self.fetch() + self.assertIsNotNone(ud.method._find_git_lfs) + + # If git-lfs can be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: True + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + # If git-lfs cannot be found, the unpack should throw an error + with self.assertRaises(bb.fetch2.FetchError): + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + def test_lfs_disabled(self): + import shutil + + uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir + self.d.setVar('SRC_URI', uri) + + fetcher, ud = self.fetch() + self.assertIsNotNone(ud.method._find_git_lfs) + + # If git-lfs can be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: True + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + # If git-lfs cannot be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass new file mode 100644 index 000000000..b57650d59 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass @@ -0,0 +1,262 @@ +SLOWTASKS ??= "" +SSTATEVALID ??= "" + +def stamptask(d): + import time + + thistask = d.expand("${PN}:${BB_CURRENTTASK}") + stampname = d.expand("${TOPDIR}/%s.run" % thistask) + with open(stampname, "a+") as f: + f.write(d.getVar("BB_UNIHASH") + "\n") + + if d.getVar("BB_CURRENT_MC") != "default": + thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}") + if thistask in d.getVar("SLOWTASKS").split(): + bb.note("Slowing task %s" % thistask) + time.sleep(0.5) + if d.getVar("BB_HASHSERVE"): + task = d.getVar("BB_CURRENTTASK") + if task in ['package', 'package_qa', 'packagedata', 'package_write_ipk', 'package_write_rpm', 'populate_lic', 'populate_sysroot']: + bb.parse.siggen.report_unihash(os.getcwd(), d.getVar("BB_CURRENTTASK"), d) + + with open(d.expand("${TOPDIR}/task.log"), "a+") as f: + f.write(thistask + "\n") + + +def sstate_output_hash(path, sigfile, task, d): + import hashlib + h = hashlib.sha256() + h.update(d.expand("${PN}:${BB_CURRENTTASK}").encode('utf-8')) + return h.hexdigest() + +python do_fetch() { + # fetch + stamptask(d) +} +python do_unpack() { + # unpack + stamptask(d) +} +python do_patch() { + # patch + stamptask(d) +} +python do_populate_lic() { + # populate_lic + stamptask(d) +} +python do_prepare_recipe_sysroot() { + # prepare_recipe_sysroot + stamptask(d) +} +python do_configure() { + # configure + stamptask(d) +} +python do_compile() { + # compile + stamptask(d) +} +python do_install() { + # install + stamptask(d) +} +python do_populate_sysroot() { + # populate_sysroot + stamptask(d) +} +python do_package() { + # package + stamptask(d) +} +python do_package_write_ipk() { + # package_write_ipk + stamptask(d) +} +python do_package_write_rpm() { + # package_write_rpm + stamptask(d) +} +python do_packagedata() { + # packagedata + stamptask(d) +} +python do_package_qa() { + # package_qa + stamptask(d) +} +python do_build() { + # build + stamptask(d) +} +do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" +do_package[deptask] += "do_packagedata" +do_build[recrdeptask] += "do_deploy" +do_build[recrdeptask] += "do_package_write_ipk" +do_build[recrdeptask] += "do_package_write_rpm" +do_package_qa[rdeptask] = "do_packagedata" +do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy" + +DEBIANRDEP = "do_packagedata" +oo_package_write_ipk[rdeptask] = "${DEBIANRDEP}" +do_package_write_rpm[rdeptask] = "${DEBIANRDEP}" + +addtask fetch +addtask unpack after do_fetch +addtask patch after do_unpack +addtask prepare_recipe_sysroot after do_patch +addtask configure after do_prepare_recipe_sysroot +addtask compile after do_configure +addtask install after do_compile +addtask populate_sysroot after do_install +addtask package after do_install +addtask package_write_ipk after do_packagedata do_package +addtask package_write_rpm after do_packagedata do_package +addtask packagedata after do_package +addtask package_qa after do_package +addtask build after do_package_qa do_package_write_rpm do_package_write_ipk do_populate_sysroot + +python do_package_setscene() { + stamptask(d) +} +python do_package_qa_setscene() { + stamptask(d) +} +python do_package_write_ipk_setscene() { + stamptask(d) +} +python do_package_write_rpm_setscene() { + stamptask(d) +} +python do_packagedata_setscene() { + stamptask(d) +} +python do_populate_lic_setscene() { + stamptask(d) +} +python do_populate_sysroot_setscene() { + stamptask(d) +} + +addtask package_setscene +addtask package_qa_setscene +addtask package_write_ipk_setscene +addtask package_write_rpm_setscene +addtask packagedata_setscene +addtask populate_lic_setscene +addtask populate_sysroot_setscene + +BB_SETSCENE_DEPVALID = "setscene_depvalid" + +def setscene_depvalid(task, taskdependees, notneeded, d, log=None): + # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME] + # task is included in taskdependees too + # Return - False - We need this dependency + # - True - We can skip this dependency + import re + + def logit(msg, log): + if log is not None: + log.append(msg) + else: + bb.debug(2, msg) + + logit("Considering setscene task: %s" % (str(taskdependees[task])), log) + + def isNativeCross(x): + return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross") + + # We only need to trigger populate_lic through direct dependencies + if taskdependees[task][1] == "do_populate_lic": + return True + + # We only need to trigger packagedata through direct dependencies + # but need to preserve packagedata on packagedata links + if taskdependees[task][1] == "do_packagedata": + for dep in taskdependees: + if taskdependees[dep][1] == "do_packagedata": + return False + return True + + for dep in taskdependees: + logit(" considering dependency: %s" % (str(taskdependees[dep])), log) + if task == dep: + continue + if dep in notneeded: + continue + # do_package_write_* and do_package doesn't need do_package + if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']: + continue + # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm']: + return False + # do_package/packagedata/package_qa don't need do_populate_sysroot + if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']: + continue + # Native/Cross packages don't exist and are noexec anyway + if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']: + continue + + # This is due to the [depends] in useradd.bbclass complicating matters + # The logic *is* reversed here due to the way hard setscene dependencies are injected + if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot': + continue + + # Consider sysroot depending on sysroot tasks + if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': + # Native/Cross populate_sysroot need their dependencies + if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]): + return False + # Target populate_sysroot depended on by cross tools need to be installed + if isNativeCross(taskdependees[dep][0]): + return False + # Native/cross tools depended upon by target sysroot are not needed + # Add an exception for shadow-native as required by useradd.bbclass + if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native': + continue + # Target populate_sysroot need their dependencies + return False + + + if taskdependees[dep][1] == "do_populate_lic": + continue + + # Safe fallthrough default + logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log) + return False + return True + +BB_HASHCHECK_FUNCTION = "sstate_checkhashes" + +def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs): + + found = set() + missed = set() + + valid = d.getVar("SSTATEVALID").split() + + for tid in sorted(sq_data['hash']): + n = os.path.basename(bb.runqueue.fn_from_tid(tid)).split(".")[0] + ":do_" + bb.runqueue.taskname_from_tid(tid)[3:] + print(n) + stampfile = d.expand("${TOPDIR}/%s.run" % n.replace("do_", "")) + if n in valid: + bb.note("SState: Found valid sstate for %s" % n) + found.add(tid) + elif n + ":" + sq_data['hash'][tid] in valid: + bb.note("SState: Found valid sstate for %s" % n) + found.add(tid) + elif os.path.exists(stampfile): + with open(stampfile, "r") as f: + hash = f.readline().strip() + if hash == sq_data['hash'][tid]: + bb.note("SState: Found valid sstate for %s (already run)" % n) + found.add(tid) + else: + bb.note("SState: sstate hash didn't match previous run for %s (%s vs %s)" % (n, sq_data['hash'][tid], hash)) + missed.add(tid) + else: + missed.add(tid) + bb.note("SState: Found no valid sstate for %s (%s)" % (n, sq_data['hash'][tid])) + + return found + diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass new file mode 100644 index 000000000..da9ff1106 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass @@ -0,0 +1,5 @@ +do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa" +do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa" +do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa +do_rootfs[recrdeptask] += "do_packagedata" +do_rootfs[recrdeptask] += "do_populate_lic" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass new file mode 100644 index 000000000..7eaaee54a --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass @@ -0,0 +1,2 @@ +RECIPERDEPTASK = "do_populate_sysroot" +do_populate_sysroot[rdeptask] = "${RECIPERDEPTASK}" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf new file mode 100644 index 000000000..5e451fc2c --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -0,0 +1,16 @@ +CACHE = "${TOPDIR}/cache" +THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" +COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" +BBFILES = "${COREBASE}/recipes/*.bb" +PROVIDES = "${PN}" +PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" +PF = "${BB_CURRENT_MC}:${PN}" +export PATH +TMPDIR ??= "${TOPDIR}" +STAMP = "${TMPDIR}/stamps/${PN}" +T = "${TMPDIR}/workdir/${PN}/temp" +BB_NUMBER_THREADS = "4" + +BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE" + +include conf/multiconfig/${BB_CURRENT_MC}.conf diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf new file mode 100644 index 000000000..ecf23e1c7 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc1/" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf new file mode 100644 index 000000000..eef338e4c --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc2/" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/a1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/a1.bb new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/a1.bb diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb new file mode 100644 index 000000000..c0b288e5b --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb @@ -0,0 +1 @@ +DEPENDS = "a1"
\ No newline at end of file diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/c1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/c1.bb new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/c1.bb diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb new file mode 100644 index 000000000..5ba197515 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb @@ -0,0 +1,3 @@ +DEPENDS = "a1" + +do_package_setscene[depends] = "a1:do_populate_sysroot_setscene" diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb new file mode 100644 index 000000000..1588bc8a5 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb @@ -0,0 +1 @@ +DEPENDS = "b1"
\ No newline at end of file diff --git a/poky/bitbake/lib/bb/tests/runqueue.py b/poky/bitbake/lib/bb/tests/runqueue.py new file mode 100644 index 000000000..5e6439156 --- /dev/null +++ b/poky/bitbake/lib/bb/tests/runqueue.py @@ -0,0 +1,323 @@ +# +# BitBake Tests for runqueue task processing +# +# Copyright (C) 2019 Richard Purdie +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import bb +import os +import tempfile +import subprocess +import sys +import time + +# +# TODO: +# Add tests on task ordering (X happens before Y after Z) +# + +class RunQueueTests(unittest.TestCase): + + alltasks = ['package', 'fetch', 'unpack', 'patch', 'prepare_recipe_sysroot', 'configure', + 'compile', 'install', 'packagedata', 'package_qa', 'package_write_rpm', 'package_write_ipk', + 'populate_sysroot', 'build'] + a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot" + b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot" + + def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False): + env = os.environ.copy() + env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests")) + env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS" + env["SSTATEVALID"] = sstatevalid + env["SLOWTASKS"] = slowtasks + if extraenv: + for k in extraenv: + env[k] = extraenv[k] + env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k + try: + output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir) + print(output) + except subprocess.CalledProcessError as e: + self.fail("Command %s failed with %s" % (cmd, e.output)) + tasks = [] + tasklog = builddir + "/task.log" + if os.path.exists(tasklog): + with open(tasklog, "r") as f: + tasks = [line.rstrip() for line in f] + if cleanup: + os.remove(tasklog) + return tasks + + def test_no_setscenevalid(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + + def test_single_setscenevalid(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = "a1:do_package" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure', + 'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot', 'a1:build'] + self.assertEqual(set(tasks), set(expected)) + + def test_intermediate_setscenevalid(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = "a1:do_package a1:do_populate_sysroot" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot_setscene', 'a1:build'] + self.assertEqual(set(tasks), set(expected)) + + def test_intermediate_notcovered(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = "a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene', + 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] + self.assertEqual(set(tasks), set(expected)) + + def test_all_setscenevalid(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = self.a1_sstatevalid + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene', + 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] + self.assertEqual(set(tasks), set(expected)) + + def test_no_settasks(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1", "-c", "patch"] + sstatevalid = self.a1_sstatevalid + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:fetch', 'a1:unpack', 'a1:patch'] + self.assertEqual(set(tasks), set(expected)) + + def test_mix_covered_notcovered(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"] + sstatevalid = self.a1_sstatevalid + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene'] + self.assertEqual(set(tasks), set(expected)) + + + # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks + def test_mixed_direct_tasks_setscene_tasks(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "c1:do_patch", "a1"] + sstatevalid = self.a1_sstatevalid + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['c1:fetch', 'c1:unpack', 'c1:patch', 'a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene', + 'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene'] + self.assertEqual(set(tasks), set(expected)) + + # This test slows down the execution of do_package_setscene until after other real tasks have + # started running which tests for a bug where tasks were being lost from the buildable list of real + # tasks if they weren't in tasks_covered or tasks_notcovered + def test_slow_setscene(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + sstatevalid = "a1:do_package" + slowtasks = "a1:package_setscene" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, slowtasks) + expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure', + 'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot', 'a1:build'] + self.assertEqual(set(tasks), set(expected)) + + def test_setscenewhitelist(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "a1"] + extraenv = { + "BB_SETSCENE_ENFORCE" : "1", + "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build" + } + sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv) + expected = ['a1:packagedata_setscene', 'a1:package_qa_setscene', 'a1:package_write_ipk_setscene', + 'a1:populate_sysroot_setscene', 'a1:package_setscene'] + self.assertEqual(set(tasks), set(expected)) + + # Tests for problems with dependencies between setscene tasks + def test_no_setscenevalid_harddeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "d1"] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure', + 'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot', 'd1:package', 'd1:fetch', 'd1:unpack', 'd1:patch', 'd1:prepare_recipe_sysroot', 'd1:configure', + 'd1:compile', 'd1:install', 'd1:packagedata', 'd1:package_qa', 'd1:package_write_rpm', 'd1:package_write_ipk', + 'd1:populate_sysroot', 'd1:build'] + self.assertEqual(set(tasks), set(expected)) + + def test_no_setscenevalid_withdeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "b1"] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + expected.remove('a1:build') + expected.remove('a1:package_qa') + self.assertEqual(set(tasks), set(expected)) + + def test_single_a1_setscenevalid_withdeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "b1"] + sstatevalid = "a1:do_package" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure', + 'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + + def test_single_b1_setscenevalid_withdeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "b1"] + sstatevalid = "b1:do_package" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure', + 'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks] + expected.remove('b1:package') + self.assertEqual(set(tasks), set(expected)) + + def test_intermediate_setscenevalid_withdeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "b1"] + sstatevalid = "a1:do_package a1:do_populate_sysroot b1:do_package" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk', + 'a1:populate_sysroot_setscene', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks] + expected.remove('b1:package') + self.assertEqual(set(tasks), set(expected)) + + def test_all_setscenevalid_withdeps(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + cmd = ["bitbake", "b1"] + sstatevalid = self.a1_sstatevalid + " " + self.b1_sstatevalid + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid) + expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene', + 'b1:build', 'a1:populate_sysroot_setscene', 'b1:package_write_ipk_setscene', 'b1:package_write_rpm_setscene', + 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene'] + self.assertEqual(set(tasks), set(expected)) + + def test_multiconfig_setscene_optimise(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic" + } + cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \ + ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \ + ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \ + ['mc1:b1:build', 'mc2:b1:build'] + for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']: + expected.remove(x) + self.assertEqual(set(tasks), set(expected)) + + + @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') + def test_hashserv_single(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BB_HASHSERVE" : "auto", + "BB_SIGNATURE_HANDLER" : "TestEquivHash" + } + cmd = ["bitbake", "a1", "b1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "-c", "install", "-f"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:install'] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "b1"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:populate_sysroot', 'a1:package', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene', + 'a1:package_write_ipk_setscene', 'a1:package_qa_setscene'] + self.assertEqual(set(tasks), set(expected)) + + self.shutdown(tempdir) + + @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') + def test_hashserv_double(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BB_HASHSERVE" : "auto", + "BB_SIGNATURE_HANDLER" : "TestEquivHash" + } + cmd = ["bitbake", "a1", "b1", "e1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + cmd = ["bitbake", "e1"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot', + 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene', + 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene'] + self.assertEqual(set(tasks), set(expected)) + + self.shutdown(tempdir) + + @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') + def test_hashserv_multiple_setscene(self): + # Runs e1:do_package_setscene twice + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BB_HASHSERVE" : "auto", + "BB_SIGNATURE_HANDLER" : "TestEquivHash" + } + cmd = ["bitbake", "a1", "b1", "e1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + cmd = ["bitbake", "e1"] + sstatevalid = "e1:do_package" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="a1:populate_sysroot b1:populate_sysroot") + expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot', + 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene', + 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene', + 'e1:package_setscene'] + self.assertEqual(set(tasks), set(expected)) + for i in expected: + self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i) + + self.shutdown(tempdir) + + def shutdown(self, tempdir): + # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup + while os.path.exists(tempdir + "/hashserve.sock"): + time.sleep(0.5) + + |