diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands')
4 files changed, 583 insertions, 0 deletions
diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py new file mode 100644 index 000000000..3ccc7c67c --- /dev/null +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -0,0 +1,247 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting, Build +import os +import sys, traceback + +def DN(path): + if path is None: + return "" + else: + return os.path.dirname(path) + + +class Command(NoArgsCommand): + args = "" + help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + + def _find_first_path_for_file(self, startdirectory, filename, level = 0): + if level < 0: + return None + dirs = [] + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isfile(j): + if i == filename: + return startdirectory + elif os.path.isdir(j): + dirs.append(j) + for j in dirs: + ret = self._find_first_path_for_file(j, filename, level - 1) + if ret is not None: + return ret + return None + + def _recursive_list_directories(self, startdirectory, level = 0): + if level < 0: + return [] + dirs = [] + try: + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isdir(j): + dirs.append(j) + except OSError: + pass + for j in dirs: + dirs = dirs + self._recursive_list_directories(j, level - 1) + return dirs + + + def _get_suggested_sourcedir(self, be): + if be.betype != BuildEnvironment.TYPE_LOCAL: + return "" + return DN(DN(DN(self._find_first_path_for_file(self.guesspath, "toasterconf.json", 4)))) + + def _get_suggested_builddir(self, be): + if be.betype != BuildEnvironment.TYPE_LOCAL: + return "" + return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4)) + + + def _verify_artifact_storage_dir(self): + # verify that we have a settings for downloading artifacts + while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: + guessedpath = os.getcwd() + "/toaster_build_artifacts/" + print("\nToaster needs to know in which directory it can download build log files and other artifacts.\nToaster suggests \"%s\"." % guessedpath) + artifacts_storage_dir = raw_input("Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) + if len(artifacts_storage_dir) == 0: + artifacts_storage_dir = guessedpath + if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): + try: + os.makedirs(artifacts_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) + return 0 + + + def _verify_build_environment(self): + # refuse to start if we have no build environments + while BuildEnvironment.objects.count() == 0: + print(" !! No build environments found. Toaster needs at least one build environment in order to be able to run builds.\n" + + "You can manually define build environments in the database table bldcontrol_buildenvironment.\n" + + "Or Toaster can define a simple localhost-based build environment for you.") + + i = raw_input(" -- Do you want to create a basic localhost build environment ? (Y/n) "); + if not len(i) or i.startswith("y") or i.startswith("Y"): + BuildEnvironment.objects.create(pk = 1, betype = 0) + else: + raise Exception("Toaster cannot start without build environments. Aborting.") + + + # we make sure we have builddir and sourcedir for all defined build envionments + for be in BuildEnvironment.objects.all(): + be.needs_import = False + def _verify_be(): + is_changed = False + print("\nVerifying the build environment. If the local build environment is not properly configured, you will be asked to configure it.") + + def _update_sourcedir(): + suggesteddir = self._get_suggested_sourcedir(be) + if len(suggesteddir) > 0: + be.sourcedir = raw_input("This is the directory Toaster uses to check out the source code of the layers you will build. Toaster will create new clones of the layers, so existing content in the chosen directory will not be changed.\nToaster suggests you use \"%s\" as your layers checkout directory. If you select this directory, a layer like \"meta-intel\" will end up in \"%s/meta-intel\".\nPress Enter to select \"%s\" or type the full path to a different directory. If you provide your own directory, it must be a parent of the cloned directory for the sources you are using to run Toaster: " % (suggesteddir, suggesteddir, suggesteddir)) + else: + be.sourcedir = raw_input("Toaster needs to know in which directory it should check out the source code of the layers you will build. The directory should be a parent of the cloned directory for the sources you are using to run Toaster. Toaster will create new clones of the layers, so existing content in the chosen directory will not be changed.\nType the full path to the directory (for example: \"%s\": " % os.environ.get('HOME', '/tmp/')) + if len(be.sourcedir) == 0 and len(suggesteddir) > 0: + be.sourcedir = suggesteddir + return True + + if len(be.sourcedir) == 0: + print "\n -- Validation: The layers checkout directory must be set." + is_changed = _update_sourcedir() + + if not be.sourcedir.startswith("/"): + print "\n -- Validation: The layers checkout directory must be set to an absolute path." + is_changed = _update_sourcedir() + + if not be.sourcedir in DN(__file__): + print "\n -- Validation: The layers checkout directory must be a parent of the current checkout." + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): + suggesteddir = self._get_suggested_builddir(be) + if len(suggesteddir) > 0: + be.builddir = raw_input("Toaster needs to know where your build directory is located.\nThe build directory is where all the artifacts created by your builds will be stored. Toaster suggests \"%s\".\nPress Enter to select \"%s\" or type the full path to a different directory: " % (suggesteddir, suggesteddir)) + else: + be.builddir = raw_input("Toaster needs to know where is your build directory.\nThe build directory is where all the artifacts created by your builds will be stored. Type the full path to the directory (for example: \" %s/build\")" % os.environ.get('HOME','/tmp/')) + if len(be.builddir) == 0 and len(suggesteddir) > 0: + be.builddir = suggesteddir + return True + + if len(be.builddir) == 0: + print "\n -- Validation: The build directory must be set." + is_changed = _update_builddir() + + if not be.builddir.startswith("/"): + print "\n -- Validation: The build directory must to be set to an absolute path." + is_changed = _update_builddir() + + + if is_changed: + print "\nBuild configuration saved" + be.save() + return True + + + if be.needs_import: + print "\nToaster can use a SINGLE predefined configuration file to set up default project settings and layer information sources.\n" + + # find configuration files + config_files = [] + for dirname in self._recursive_list_directories(be.sourcedir,2): + if os.path.exists(os.path.join(dirname, ".templateconf")): + import subprocess + proc = subprocess.Popen('bash -c ". '+os.path.join(dirname, ".templateconf")+'; echo \"\$TEMPLATECONF\""', shell=True, stdout=subprocess.PIPE) + conffilepath, stderroroutput = proc.communicate() + proc.wait() + if proc.returncode != 0: + raise Exception("Failed to source TEMPLATECONF: %s" % stderroroutput) + + conffilepath = os.path.join(conffilepath.strip(), "toasterconf.json") + candidatefilepath = os.path.join(dirname, conffilepath) + if "toaster_cloned" in candidatefilepath: + continue + if os.path.exists(candidatefilepath): + config_files.append(candidatefilepath) + + if len(config_files) > 0: + print "Toaster will list now the configuration files that it found. Select the number to use the desired configuration file." + for cf in config_files: + print " [%d] - %s" % (config_files.index(cf) + 1, cf) + print "\n [0] - Exit without importing any file" + try: + i = raw_input("\nEnter your option: ") + if len(i) and (int(i) - 1 >= 0 and int(i) - 1 < len(config_files)): + print "\nImporting file: %s" % config_files[int(i)-1] + from loadconf import Command as LoadConfigCommand + + LoadConfigCommand()._import_layer_config(config_files[int(i)-1]) + # we run lsupdates after config update + print "\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" + from django.core.management import call_command + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print "Failure while trying to import the toaster config file: %s" % e + traceback.print_exc(e) + else: + print "\nToaster could not find a configuration file. You need to configure Toaster manually using the web interface, or create a configuration file and use\n bitbake/lib/toaster/managepy.py loadconf [filename]\n command to load it. You can use https://wiki.yoctoproject.org/wiki/File:Toasterconf.json.txt.patch as a starting point." + + + + + return is_changed + + while (_verify_be()): + pass + return 0 + + def _verify_default_settings(self): + # verify that default settings are there + if ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').count() != 1: + ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').delete() + ToasterSetting.objects.get_or_create(name = 'DEFAULT_RELEASE', value = '') + return 0 + + def _verify_builds_in_progress(self): + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state = BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req = b, errtype = "toaster", errmsg = "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state = BuildRequest.REQ_INPROGRESS).update(state = BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock = BuildEnvironment.LOCK_FREE) + + # also mark "In Progress builds as failures" + from django.utils import timezone + Build.objects.filter(outcome = Build.IN_PROGRESS).update(outcome = Build.FAILED, completed_on = timezone.now()) + + return 0 + + + + def handle_noargs(self, **options): + retval = 0 + retval += self._verify_artifact_storage_dir() + retval += self._verify_build_environment() + retval += self._verify_default_settings() + retval += self._verify_builds_in_progress() + + return retval diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py new file mode 100644 index 000000000..5022b5940 --- /dev/null +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py @@ -0,0 +1,183 @@ +from django.core.management.base import BaseCommand, CommandError +from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version +from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority +from django.db import IntegrityError +import os + +from checksettings import DN + +import logging +logger = logging.getLogger("toaster") + +def _reduce_canon_path(path): + components = [] + for c in path.split("/"): + if c == "..": + del components[-1] + elif c == ".": + pass + else: + components.append(c) + if len(components) < 2: + components.append('') + return "/".join(components) + +def _get_id_for_sourcetype(s): + for i in LayerSource.SOURCE_TYPE: + if s == i[1]: + return i[0] + raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) + +class Command(BaseCommand): + help = "Loads a toasterconf.json file in the database" + args = "filepath" + + + + def _import_layer_config(self, filepath): + if not os.path.exists(filepath) or not os.path.isfile(filepath): + raise Exception("Failed to find toaster config file %s ." % filepath) + + import json + data = json.loads(open(filepath, "r").read()) + + # verify config file validity before updating settings + for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: + assert i in data + + def _read_git_url_from_local_repository(address): + url = None + # we detect the remote name at runtime + import subprocess + (remote, remote_name) = address.split(":", 1) + cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) + (out,err) = cmd.communicate() + if cmd.returncode != 0: + logging.warning("Error while importing layer vcs_url: git error: %s" % err) + for line in out.split("\n"): + try: + (name, path) = line.split("\t", 1) + if name == remote_name: + url = path.split(" ")[0] + break + except ValueError: + pass + if url == None: + logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) + return url + + + # import bitbake data + for bvi in data['bitbake']: + bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) + if bvi['giturl'].startswith("remote:"): + bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) + if bvo.giturl is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name']) + + if bvo.giturl is None: + bvo.giturl = bvi['giturl'] + bvo.branch = bvi['branch'] + bvo.dirpath = bvi['dirpath'] + bvo.save() + + # set the layer sources + for lsi in data['layersources']: + assert 'sourcetype' in lsi + assert 'apiurl' in lsi + assert 'name' in lsi + assert 'branches' in lsi + + + if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): + apiurl = lsi['apiurl'] + else: + apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) + + assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) + + try: + ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) + ls.name = lsi['name'] + ls.save() + except IntegrityError as e: + logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls)) + + + layerbranches = [] + for branchname in lsi['branches']: + bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) + layerbranches.append(bo) + + if 'layers' in lsi: + for layerinfo in lsi['layers']: + lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) + if layerinfo['local_path'].startswith("/"): + lo.local_path = layerinfo['local_path'] + else: + lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) + + if not os.path.exists(lo.local_path): + logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path) + + if layerinfo['vcs_url'].startswith("remote:"): + lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) + if lo.vcs_url is None: + logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name']) + + if lo.vcs_url is None: + lo.vcs_url = layerinfo['vcs_url'] + + if 'layer_index_url' in layerinfo: + lo.layer_index_url = layerinfo['layer_index_url'] + lo.save() + + for branch in layerbranches: + lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, + up_branch = branch, + commit = branch.name, + layer = lo) + lvo.dirpath = layerinfo['dirpath'] + lvo.save() + # set releases + for ri in data['releases']: + bvo = BitbakeVersion.objects.get(name = ri['bitbake']) + assert bvo is not None + + ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) + ro.description = ri['description'] + ro.helptext = ri['helptext'] + ro.save() + + # save layer source priority for release + for ls_name in ri['layersourcepriority'].keys(): + rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) + rlspo.priority = ri['layersourcepriority'][ls_name] + rlspo.save() + + for dli in ri['defaultlayers']: + # find layers with the same name + ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) + + # set default release + if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: + ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) + else: + ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) + + # set default config variables + for configname in data['config']: + if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: + ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) + else: + ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) + + + def handle(self, *args, **options): + if len(args) == 0: + raise CommandError("Need a path to the toasterconf.json file") + filepath = args[0] + self._import_layer_config(filepath) + + + diff --git a/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py new file mode 100644 index 000000000..c3e9b74c0 --- /dev/null +++ b/yocto-poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -0,0 +1,153 @@ +from django.core.management.base import NoArgsCommand, CommandError +from django.db import transaction +from orm.models import Build, ToasterSetting, LogMessage, Target +from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable +import os +import logging + +logger = logging.getLogger("ToasterScheduler") + +class Command(NoArgsCommand): + args = "" + help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)" + + + @transaction.commit_on_success + def _selectBuildEnvironment(self): + bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) + bec.be.lock = BuildEnvironment.LOCK_LOCK + bec.be.save() + return bec + + @transaction.commit_on_success + def _selectBuildRequest(self): + br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0] + br.state = BuildRequest.REQ_INPROGRESS + br.save() + return br + + def schedule(self): + import traceback + try: + br = None + try: + # select the build environment and the request to build + br = self._selectBuildRequest() + except IndexError as e: + #logger.debug("runbuilds: No build request") + return + try: + bec = self._selectBuildEnvironment() + except IndexError as e: + # we could not find a BEC; postpone the BR + br.state = BuildRequest.REQ_QUEUED + br.save() + logger.debug("runbuilds: No build env") + return + + logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) + + # write the build identification variable + BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) + + # let the build request know where it is being executed + br.environment = bec.be + br.save() + + # this triggers an async build + bec.triggerBuild(br.brbitbake_set.all(), br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all()) + + except Exception as e: + logger.error("runbuilds: Error launching build %s" % e) + traceback.print_exc(e) + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + + BRError.objects.create(req = br, + errtype = str(type(e)), + errmsg = errmsg, + traceback = traceback.format_exc(e)) + br.state = BuildRequest.REQ_FAILED + br.save() + bec.be.lock = BuildEnvironment.LOCK_FREE + bec.be.save() + + def archive(self): + ''' archives data from the builds ''' + artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + # save cooker log + if br.build == None: + br.state = BuildRequest.REQ_FAILED + br.save() + continue + build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) + try: + os.makedirs(build_artifact_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + + file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") + try: + with open(file_name, "w") as f: + f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) + except IOError: + os.unlink(file_name) + + br.state = BuildRequest.REQ_COMPLETED + br.save() + + def cleanup(self): + from django.utils import timezone + from datetime import timedelta + # environments locked for more than 30 seconds - they should be unlocked + BuildEnvironment.objects.filter(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED]).filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE) + + + # update all Builds that failed to start + + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS): + # transpose the launch errors in ToasterExceptions + br.build.outcome = Build.FAILED + for brerror in br.brerror_set.all(): + logger.debug("Saving error %s" % brerror) + LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) + br.build.save() + + # we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock + br.environment.lock = BuildEnvironment.LOCK_FREE + br.environment.save() + + + + # update all BuildRequests without a build created + for br in BuildRequest.objects.filter(build = None): + br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created) + br.build.outcome = Build.FAILED + try: + br.build.machine = br.brvariable_set.get(name='MACHINE').value + except BRVariable.DoesNotExist: + pass + br.save() + # transpose target information + for brtarget in br.brtarget_set.all(): + Target.objects.create(build = br.build, target= brtarget.target) + # transpose the launch errors in ToasterExceptions + for brerror in br.brerror_set.all(): + LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) + + br.build.save() + pass + + + def handle_noargs(self, **options): + self.cleanup() + self.archive() + self.schedule() |