summaryrefslogtreecommitdiff
path: root/poky/bitbake/lib/bb/tests
diff options
context:
space:
mode:
Diffstat (limited to 'poky/bitbake/lib/bb/tests')
-rw-r--r--poky/bitbake/lib/bb/tests/data.py2
-rw-r--r--poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass54
-rw-r--r--poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf2
-rw-r--r--poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb1
-rw-r--r--poky/bitbake/lib/bb/tests/runqueue.py174
5 files changed, 211 insertions, 22 deletions
diff --git a/poky/bitbake/lib/bb/tests/data.py b/poky/bitbake/lib/bb/tests/data.py
index 3cf5abec7..a9b0bdb09 100644
--- a/poky/bitbake/lib/bb/tests/data.py
+++ b/poky/bitbake/lib/bb/tests/data.py
@@ -466,7 +466,7 @@ class TaskHash(unittest.TestCase):
tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
bb.warn(str(lookupcache))
- return basehash["somefile." + taskname]
+ return basehash["somefile:" + taskname]
d = bb.data.init()
d.setVar("__BBTASKS", ["mytask"])
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
index 5b87e20bc..b57650d59 100644
--- a/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -5,18 +5,30 @@ def stamptask(d):
import time
thistask = d.expand("${PN}:${BB_CURRENTTASK}")
- with open(d.expand("${TOPDIR}/%s.run") % thistask, "a+") as f:
- f.write("\n")
+ stampname = d.expand("${TOPDIR}/%s.run" % thistask)
+ with open(stampname, "a+") as f:
+ f.write(d.getVar("BB_UNIHASH") + "\n")
if d.getVar("BB_CURRENT_MC") != "default":
thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
if thistask in d.getVar("SLOWTASKS").split():
bb.note("Slowing task %s" % thistask)
time.sleep(0.5)
+ if d.getVar("BB_HASHSERVE"):
+ task = d.getVar("BB_CURRENTTASK")
+ if task in ['package', 'package_qa', 'packagedata', 'package_write_ipk', 'package_write_rpm', 'populate_lic', 'populate_sysroot']:
+ bb.parse.siggen.report_unihash(os.getcwd(), d.getVar("BB_CURRENTTASK"), d)
with open(d.expand("${TOPDIR}/task.log"), "a+") as f:
f.write(thistask + "\n")
+
+def sstate_output_hash(path, sigfile, task, d):
+ import hashlib
+ h = hashlib.sha256()
+ h.update(d.expand("${PN}:${BB_CURRENTTASK}").encode('utf-8'))
+ return h.hexdigest()
+
python do_fetch() {
# fetch
stamptask(d)
@@ -216,27 +228,35 @@ def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
-def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
+def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
- ret = []
- missed = []
+ found = set()
+ missed = set()
valid = d.getVar("SSTATEVALID").split()
- for task in range(len(sq_fn)):
- n = os.path.basename(sq_fn[task]).rsplit(".", 1)[0] + ":" + sq_task[task]
+ for tid in sorted(sq_data['hash']):
+ n = os.path.basename(bb.runqueue.fn_from_tid(tid)).split(".")[0] + ":do_" + bb.runqueue.taskname_from_tid(tid)[3:]
+ print(n)
+ stampfile = d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))
if n in valid:
bb.note("SState: Found valid sstate for %s" % n)
- ret.append(task)
- elif os.path.exists(d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))):
- bb.note("SState: Found valid sstate for %s (already run)" % n)
- ret.append(task)
+ found.add(tid)
+ elif n + ":" + sq_data['hash'][tid] in valid:
+ bb.note("SState: Found valid sstate for %s" % n)
+ found.add(tid)
+ elif os.path.exists(stampfile):
+ with open(stampfile, "r") as f:
+ hash = f.readline().strip()
+ if hash == sq_data['hash'][tid]:
+ bb.note("SState: Found valid sstate for %s (already run)" % n)
+ found.add(tid)
+ else:
+ bb.note("SState: sstate hash didn't match previous run for %s (%s vs %s)" % (n, sq_data['hash'][tid], hash))
+ missed.add(tid)
else:
- missed.append(task)
- bb.note("SState: Found no valid sstate for %s" % n)
-
- if hasattr(bb.parse.siggen, "checkhashes"):
- bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
+ missed.add(tid)
+ bb.note("SState: Found no valid sstate for %s (%s)" % (n, sq_data['hash'][tid]))
- return ret
+ return found
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index 96ee1cd5e..5e451fc2c 100644
--- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -11,6 +11,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
T = "${TMPDIR}/workdir/${PN}/temp"
BB_NUMBER_THREADS = "4"
-BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE"
include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb
new file mode 100644
index 000000000..1588bc8a5
--- /dev/null
+++ b/poky/bitbake/lib/bb/tests/runqueue-tests/recipes/e1.bb
@@ -0,0 +1 @@
+DEPENDS = "b1" \ No newline at end of file
diff --git a/poky/bitbake/lib/bb/tests/runqueue.py b/poky/bitbake/lib/bb/tests/runqueue.py
index f22ad4bd8..c7f5e5572 100644
--- a/poky/bitbake/lib/bb/tests/runqueue.py
+++ b/poky/bitbake/lib/bb/tests/runqueue.py
@@ -25,7 +25,7 @@ class RunQueueTests(unittest.TestCase):
a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
- def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None):
+ def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
env = os.environ.copy()
env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
@@ -37,11 +37,16 @@ class RunQueueTests(unittest.TestCase):
env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
try:
output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
+ print(output)
except subprocess.CalledProcessError as e:
self.fail("Command %s failed with %s" % (cmd, e.output))
tasks = []
- with open(builddir + "/task.log", "r") as f:
- tasks = [line.rstrip() for line in f]
+ tasklog = builddir + "/task.log"
+ if os.path.exists(tasklog):
+ with open(tasklog, "r") as f:
+ tasks = [line.rstrip() for line in f]
+ if cleanup:
+ os.remove(tasklog)
return tasks
def test_no_setscenevalid(self):
@@ -226,3 +231,166 @@ class RunQueueTests(unittest.TestCase):
expected.remove(x)
self.assertEqual(set(tasks), set(expected))
+
+ def test_hashserv_single(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ cmd = ["bitbake", "a1", "-c", "install", "-f"]
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:install']
+ self.assertEqual(set(tasks), set(expected))
+ cmd = ["bitbake", "a1", "b1"]
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:populate_sysroot', 'a1:package', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+ 'a1:package_write_ipk_setscene', 'a1:package_qa_setscene']
+ self.assertEqual(set(tasks), set(expected))
+
+ def test_hashserv_double(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1", "e1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"]
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ cmd = ["bitbake", "e1"]
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
+ 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
+ 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene']
+ self.assertEqual(set(tasks), set(expected))
+
+
+ def test_hashserv_multiple_setscene(self):
+ # Runs e1:do_package_setscene twice
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1", "e1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + ['e1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ cmd = ["bitbake", "a1", "b1", "-c", "install", "-fn"]
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ cmd = ["bitbake", "e1"]
+ sstatevalid = "e1:do_package"
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="a1:populate_sysroot b1:populate_sysroot")
+ expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
+ 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
+ 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
+ 'e1:package_setscene']
+ self.assertEqual(set(tasks), set(expected))
+ for i in expected:
+ if i in ["e1:package_setscene"]:
+ self.assertEqual(tasks.count(i), 4, "%s not in task list four times" % i)
+ else:
+ self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
+
+ def test_hashserv_partial_match(self):
+ # e1:do_package matches initial built but not second hash value
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
+ f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
+ with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
+ f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
+ cmd = ["bitbake", "e1"]
+ sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d"
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
+ 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
+ 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
+ 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
+ expected.remove('e1:package')
+ self.assertEqual(set(tasks), set(expected))
+
+ def test_hashserv_partial_match2(self):
+ # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
+ f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
+ with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
+ f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
+ cmd = ["bitbake", "e1"]
+ sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d e1:do_populate_sysroot:ef7dc0e2dd55d0534e75cba50731ff42f949818b6f29a65d72bc05856e56711d"
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
+ 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
+ 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
+ 'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata']
+ self.assertEqual(set(tasks), set(expected))
+
+ def test_hashserv_partial_match3(self):
+ # e1:do_package is valid for a1 but not after b1
+ # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run
+ # with none of the intermediate tasks which is a serious bug
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ extraenv = {
+ "BB_HASHSERVE" : "localhost:0",
+ "BB_SIGNATURE_HANDLER" : "TestEquivHash"
+ }
+ cmd = ["bitbake", "a1", "b1"]
+ setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+ 'populate_sysroot_setscene', 'package_qa_setscene']
+ sstatevalid = ""
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
+ f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
+ with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
+ f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
+ cmd = ["bitbake", "e1", "-DD"]
+ sstatevalid = "e1:do_package:af056eae12a733a6a8c4f4da8c6757e588e13565852c94e2aad4d953a3989c13 e1:do_package:a3677703db82b22d28d57c1820a47851dd780104580863f5bd32e66e003a779d"
+ tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch b1:install")
+ expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
+ 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
+ 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
+ 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
+ expected.remove('e1:package')
+ self.assertEqual(set(tasks), set(expected))
+
+
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-19 21:56:13 +0300