diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py')
| -rw-r--r-- | yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py | 301 |
1 files changed, 126 insertions, 175 deletions
diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index b5cf5591f..7def1f3a1 100644 --- a/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -32,7 +32,7 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController import logging logger = logging.getLogger("toaster") @@ -48,16 +48,17 @@ class LocalhostBEController(BuildEnvironmentController): def __init__(self, be): super(LocalhostBEController, self).__init__(be) - self.dburl = settings.getDATABASE_URL() self.pokydirname = None self.islayerset = False - def _shellcmd(self, command, cwd = None): + def _shellcmd(self, command, cwd=None, nowait=False): if cwd is None: cwd = self.be.sourcedir logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command)) p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if nowait: + return (out,err) = p.communicate() p.wait() if p.returncode: @@ -71,115 +72,6 @@ class LocalhostBEController(BuildEnvironmentController): logger.debug("localhostbecontroller: shellcmd success") return out - def _setupBE(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - path = self.be.builddir - if not path: - raise Exception("Invalid path creation specified.") - if not os.path.exists(path): - os.makedirs(path, 0755) - self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, path)) - # delete the templateconf.cfg; it may come from an unsupported layer configuration - os.remove(os.path.join(path, "conf/templateconf.cfg")) - - - def writeConfFile(self, file_name, variable_list = None, raw = None): - filepath = os.path.join(self.be.builddir, file_name) - with open(filepath, "w") as conffile: - if variable_list is not None: - for i in variable_list: - conffile.write("%s=\"%s\"\n" % (i.name, i.value)) - if raw is not None: - conffile.write(raw) - - - def startBBServer(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - assert self.islayerset - - # find our own toasterui listener/bitbake - from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path - - own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) - - assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) - - logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) - - toaster_ui_log_filepath = os.path.join(self.be.builddir, "toaster_ui.log") - # get the file length; we need to detect the _last_ start of the toaster UI, not the first - toaster_ui_log_filelength = 0 - if os.path.exists(toaster_ui_log_filepath): - with open(toaster_ui_log_filepath, "w") as f: - f.seek(0, 2) # jump to the end - toaster_ui_log_filelength = f.tell() - - cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (self.pokydirname, self.be.builddir, self.be.builddir, self.be.builddir) - - port = "-1" - logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd) - - cmdoutput = self._shellcmd(cmd) - with open(self.be.builddir + "/toaster_server.log", "r") as f: - for i in f.readlines(): - if i.startswith("Bitbake server address"): - port = i.split(" ")[-1] - logger.debug("localhostbecontroller: Found bitbake server port %s" % port) - - cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && DATABASE_URL=%s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % (self.pokydirname, self.be.builddir, self.dburl, own_bitbake) - with open(toaster_ui_log_filepath, "a+") as f: - p = subprocess.Popen(cmd, cwd = self.be.builddir, shell=True, stdout=f, stderr=f) - - def _toaster_ui_started(filepath, filepos = 0): - if not os.path.exists(filepath): - return False - with open(filepath, "r") as f: - f.seek(filepos) - for line in f: - if line.startswith("NOTE: ToasterUI waiting for events"): - return True - return False - - retries = 0 - started = False - while not started and retries < 50: - started = _toaster_ui_started(toaster_ui_log_filepath, toaster_ui_log_filelength) - import time - logger.debug("localhostbecontroller: Waiting bitbake server to start") - time.sleep(0.5) - retries += 1 - - if not started: - toaster_ui_log = open(os.path.join(self.be.builddir, "toaster_ui.log"), "r").read() - toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() - raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 25 seconds, aborting (Error: '%s' '%s')" % (toaster_ui_log, toaster_server_log)) - - logger.debug("localhostbecontroller: Started bitbake server") - - while port == "-1": - # the port specification is "autodetect"; read the bitbake.lock file - with open("%s/bitbake.lock" % self.be.builddir, "r") as f: - for line in f.readlines(): - if ":" in line: - port = line.split(":")[1].strip() - logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) - break - - assert self.be.sourcedir and os.path.exists(self.be.builddir) - self.be.bbaddress = "localhost" - self.be.bbport = port - self.be.bbstate = BuildEnvironment.SERVER_STARTED - self.be.save() - - def stopBBServer(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - assert self.islayerset - self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % - (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) - self.be.bbstate = BuildEnvironment.SERVER_STOPPED - self.be.save() - logger.debug("localhostbecontroller: Stopped bitbake server") - def getGitCloneDirectory(self, url, branch): """Construct unique clone directory name out of url and branch.""" if branch != "HEAD": @@ -193,22 +85,22 @@ class LocalhostBEController(BuildEnvironmentController): return local_checkout_path - def setLayers(self, bitbakes, layers, targets): + def setLayers(self, bitbake, layers, targets): """ a word of attention: by convention, the first layer for any build will be poky! """ assert self.be.sourcedir is not None - assert len(bitbakes) == 1 # set layers in the layersource # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} - gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] - gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) + gitrepos[(bitbake.giturl, bitbake.commit)] = [] + gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) ) for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): + # We don't need to git clone the layer for the CustomImageRecipe + # as it's generated by us layer on if needed + if CustomImageRecipe.LAYER_NAME in layer.name: continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] @@ -276,7 +168,7 @@ class LocalhostBEController(BuildEnvironmentController): # make sure we have a working bitbake if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories for name, dirpath in gitrepos[(giturl, commit)]: @@ -290,23 +182,13 @@ class LocalhostBEController(BuildEnvironmentController): logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) - # 4. configure the build environment, so we have a conf/bblayers.conf - assert self.pokydirname is not None - self._setupBE() - - # 5. update the bblayers.conf - bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") - if not os.path.exists(bblayerconf): - raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - - # 6. create custom layer and add custom recipes to it - layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom") - if os.path.isdir(layerpath): - shutil.rmtree(layerpath) # remove leftovers from previous builds + # 5. create custom layer and add custom recipes to it + layerpath = os.path.join(self.be.builddir, + CustomImageRecipe.LAYER_NAME) for target in targets: try: customrecipe = CustomImageRecipe.objects.get(name=target.target, - project=bitbakes[0].req.project) + project=bitbake.req.project) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip @@ -322,60 +204,129 @@ class LocalhostBEController(BuildEnvironmentController): with open(config, "w") as conf: conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') + # Update the Layer_Version dirpath that has our base_recipe in + # to be able to read the base recipe to then generate the + # custom recipe. + br_layer_base_recipe = layers.get( + layer_version=customrecipe.base_recipe.layer_version) + + br_layer_base_dirpath = \ + os.path.join(self.be.sourcedir, + self.getGitCloneDirectory( + br_layer_base_recipe.giturl, + br_layer_base_recipe.commit), + customrecipe.base_recipe.layer_version.dirpath + ) + + customrecipe.base_recipe.layer_version.dirpath = \ + br_layer_base_dirpath + + customrecipe.base_recipe.layer_version.save() + # create recipe - recipe = os.path.join(layerpath, "recipes", "%s.bb" % target.target) - with open(recipe, "w") as recipef: - recipef.write("require %s\n" % customrecipe.base_recipe.recipe.file_path) - packages = [pkg.name for pkg in customrecipe.packages.all()] - if packages: - recipef.write('IMAGE_INSTALL = "%s"\n' % ' '.join(packages)) + recipe_path = \ + os.path.join(layerpath, "recipes", "%s.bb" % target.target) + with open(recipe_path, "w") as recipef: + recipef.write(customrecipe.generate_recipe_file_contents()) + + # Update the layer and recipe objects + customrecipe.layer_version.dirpath = layerpath + customrecipe.layer_version.save() + + customrecipe.file_path = recipe_path + customrecipe.save() # create *Layer* objects needed for build machinery to work - layer = Layer.objects.get_or_create(name="Toaster Custom layer", - summary="Layer for custom recipes", - vcs_url="file://%s" % layerpath)[0] - breq = target.req - lver = Layer_Version.objects.get_or_create(project=breq.project, layer=layer, - dirpath=layerpath, build=breq.build)[0] - ProjectLayer.objects.get_or_create(project=breq.project, layercommit=lver, - optional=False) - BRLayer.objects.get_or_create(req=breq, name=layer.name, dirpath=layerpath, + BRLayer.objects.get_or_create(req=target.req, + name=layer.name, + dirpath=layerpath, giturl="file://%s" % layerpath) if os.path.isdir(layerpath): layerlist.append(layerpath) - BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) - self.islayerset = True - return True + return layerlist def readServerLogFile(self): return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() - def release(self): - assert self.be.sourcedir and os.path.exists(self.be.builddir) - import shutil - shutil.rmtree(os.path.join(self.be.sourcedir, "build")) - assert not os.path.exists(self.be.builddir) - - - def triggerBuild(self, bitbake, layers, variables, targets): - # set up the buid environment with the needed layers - self.setLayers(bitbake, layers, targets) - self.writeConfFile("conf/toaster-pre.conf", variables) - self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"") - - # get the bb server running with the build req id and build env id - bbctrl = self.getBBController() - # trigger the build command - task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets)) - if len(task) == 0: - task = None - - bbctrl.build(list(map(lambda x:x.target, targets)), task) + def triggerBuild(self, bitbake, layers, variables, targets, brbe): + layers = self.setLayers(bitbake, layers, targets) + + # init build environment from the clone + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') + # init build environment + self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), + self.be.sourcedir) + + # update bblayers.conf + bblconfpath = os.path.join(builddir, "conf/bblayers.conf") + conflines = open(bblconfpath, "r").readlines() + skip = False + with open(bblconfpath, 'w') as bblayers: + for line in conflines: + if line.startswith("# line added by toaster"): + skip = True + continue + if skip: + skip = False + else: + bblayers.write(line) + + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + + # run bitbake server from the clone + bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') + self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="" %s --read %s ' + '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init, + builddir, bitbake, confpath), self.be.sourcedir) + + # read port number from bitbake.lock + self.be.bbport = "" + bblock = os.path.join(builddir, 'bitbake.lock') + with open(bblock) as fplock: + for line in fplock: + if ":" in line: + self.be.bbport = line.split(":")[-1].strip() + logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport) + break + + if not self.be.bbport: + raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock) - logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir) + self.be.bbaddress = "localhost" + self.be.bbstate = BuildEnvironment.SERVER_STARTED + self.be.lock = BuildEnvironment.LOCK_RUNNING + self.be.save() - # disconnect from the server - bbctrl.disconnect() + bbtargets = '' + for target in targets: + task = target.task + if task: + if not task.startswith('do_'): + task = 'do_' + task + task = ':%s' % task + bbtargets += '%s%s ' % (target.target, task) + + # run build with local bitbake. stop the server after the build. + log = os.path.join(builddir, 'toaster_ui.log') + local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), + 'bitbake') + self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" ' + '%s %s -u toasterui --token="" >>%s 2>&1;' + 'BITBAKE_UI="" BBSERVER=0.0.0.0:-1 %s -m)&\"' \ + % (brbe, local_bitbake, bbtargets, log, bitbake)], + builddir, nowait=True) + + logger.debug('localhostbecontroller: Build launched, exiting. ' + 'Follow build logs at %s' % log) |