diff options
author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-12-17 04:11:34 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2019-01-09 02:21:44 +0300 |
commit | 1a4b7ee28bf7413af6513fb45ad0d0736048f866 (patch) | |
tree | 79f6d8ea698cab8f2eaf4f54b793d2ca7a1451ce /poky/bitbake/lib/toaster/toastermain | |
parent | 5b9ede0403237c7dace972affa65cf64a1aadd0e (diff) | |
download | openbmc-1a4b7ee28bf7413af6513fb45ad0d0736048f866.tar.xz |
reset upstream subtrees to yocto 2.6
Reset the following subtrees on thud HEAD:
poky: 87e3a9739d
meta-openembedded: 6094ae18c8
meta-security: 31dc4e7532
meta-raspberrypi: a48743dc36
meta-xilinx: c42016e2e6
Also re-apply backports that didn't make it into thud:
poky:
17726d0 systemd-systemctl-native: handle Install wildcards
meta-openembedded:
4321a5d libtinyxml2: update to 7.0.1
042f0a3 libcereal: Add native and nativesdk classes
e23284f libcereal: Allow empty package
030e8d4 rsyslog: curl-less build with fmhttp PACKAGECONFIG
179a1b9 gtest: update to 1.8.1
Squashed OpenBMC subtree compatibility updates:
meta-aspeed:
Brad Bishop (1):
aspeed: add yocto 2.6 compatibility
meta-ibm:
Brad Bishop (1):
ibm: prepare for yocto 2.6
meta-ingrasys:
Brad Bishop (1):
ingrasys: set layer compatibility to yocto 2.6
meta-openpower:
Brad Bishop (1):
openpower: set layer compatibility to yocto 2.6
meta-phosphor:
Brad Bishop (3):
phosphor: set layer compatibility to thud
phosphor: libgpg-error: drop patches
phosphor: react to fitimage artifact rename
Ed Tanous (4):
Dropbear: upgrade options for latest upgrade
yocto2.6: update openssl options
busybox: remove upstream watchdog patch
systemd: Rebase CONFIG_CGROUP_BPF patch
Change-Id: I7b1fe71cca880d0372a82d94b5fd785323e3a9e7
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Diffstat (limited to 'poky/bitbake/lib/toaster/toastermain')
-rw-r--r-- | poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py | 6 | ||||
-rw-r--r-- | poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py | 584 |
2 files changed, 589 insertions, 1 deletions
diff --git a/poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py b/poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py index 0bef8d410..bf69a8fb8 100644 --- a/poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py +++ b/poky/bitbake/lib/toaster/toastermain/management/commands/builddelete.py @@ -10,8 +10,12 @@ class Command(BaseCommand): args = '<buildID1 buildID2 .....>' help = "Deletes selected build(s)" + def add_arguments(self, parser): + parser.add_argument('buildids', metavar='N', type=int, nargs='+', + help="Build ID's to delete") + def handle(self, *args, **options): - for bid in args: + for bid in options['buildids']: try: b = Build.objects.get(pk = bid) except ObjectDoesNotExist: diff --git a/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py new file mode 100644 index 000000000..2d57ab557 --- /dev/null +++ b/poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py @@ -0,0 +1,584 @@ +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# BitBake Toaster Implementation +# +# Copyright (C) 2018 Wind River Systems +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# buildimport: import a project for project specific configuration +# +# Usage: +# (a) Set up Toaster environent +# +# (b) Call buildimport +# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ +# --name=$PROJECTNAME \ +# --path=$BUILD_DIRECTORY \ +# --callback="$CALLBACK_SCRIPT" \ +# --command="configure|reconfigure|import" +# +# (c) Return is "|Default_image=%s|Project_id=%d" +# +# (d) Open Toaster to this project using for example: +# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id +# +# (e) To delete a project: +# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ +# --name=$PROJECTNAME --delete-project +# + + +# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import + +from django.core.management.base import BaseCommand, CommandError +from django.core.exceptions import ObjectDoesNotExist +from orm.models import ProjectManager, Project, Release, ProjectVariable +from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer +from toastergui.api import scan_layer_content +from django.db import OperationalError + +import os +import re +import os.path +import subprocess +import shutil + +# Toaster variable section delimiters +TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ===' +TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ===' + +# quick development/debugging support +verbose = 2 +def _log(msg): + if 1 == verbose: + print(msg) + elif 2 == verbose: + f1=open('/tmp/toaster.log', 'a') + f1.write("|" + msg + "|\n" ) + f1.close() + + +__config_regexp__ = re.compile( r""" + ^ + (?P<exp>export\s+)? + (?P<var>[a-zA-Z0-9\-_+.${}/~]+?) + (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])? + + \s* ( + (?P<colon>:=) | + (?P<lazyques>\?\?=) | + (?P<ques>\?=) | + (?P<append>\+=) | + (?P<prepend>=\+) | + (?P<predot>=\.) | + (?P<postdot>\.=) | + = + ) \s* + + (?!'[^']*'[^']*'$) + (?!\"[^\"]*\"[^\"]*\"$) + (?P<apo>['\"]) + (?P<value>.*) + (?P=apo) + $ + """, re.X) + +class Command(BaseCommand): + args = "<name> <path> <release>" + help = "Import a command line build directory" + vars = {} + toaster_vars = {} + + def add_arguments(self, parser): + parser.add_argument( + '--name', dest='name', required=True, + help='name of the project', + ) + parser.add_argument( + '--path', dest='path', required=True, + help='path to the project', + ) + parser.add_argument( + '--release', dest='release', required=False, + help='release for the project', + ) + parser.add_argument( + '--callback', dest='callback', required=False, + help='callback for project config update', + ) + parser.add_argument( + '--delete-project', dest='delete_project', required=False, + help='delete this project from the database', + ) + parser.add_argument( + '--command', dest='command', required=False, + help='command (configure,reconfigure,import)', + ) + + # Extract the bb variables from a conf file + def scan_conf(self,fn): + vars = self.vars + toaster_vars = self.toaster_vars + + #_log("scan_conf:%s" % fn) + if not os.path.isfile(fn): + return + f = open(fn, 'r') + + #statements = ast.StatementGroup() + lineno = 0 + is_toaster_section = False + while True: + lineno = lineno + 1 + s = f.readline() + if not s: + break + w = s.strip() + # skip empty lines + if not w: + continue + # evaluate Toaster sections + if w.startswith(TOASTER_PROLOG): + is_toaster_section = True + continue + if w.startswith(TOASTER_EPILOG): + is_toaster_section = False + continue + s = s.rstrip() + while s[-1] == '\\': + s2 = f.readline().strip() + lineno = lineno + 1 + if (not s2 or s2 and s2[0] != "#") and s[0] == "#" : + echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) + s = s[:-1] + s2 + # skip comments + if s[0] == '#': + continue + # process the line for just assignments + m = __config_regexp__.match(s) + if m: + groupd = m.groupdict() + var = groupd['var'] + value = groupd['value'] + + if groupd['lazyques']: + if not var in vars: + vars[var] = value + continue + if groupd['ques']: + if not var in vars: + vars[var] = value + continue + # preset empty blank for remaining operators + if not var in vars: + vars[var] = '' + if groupd['append']: + vars[var] += value + elif groupd['prepend']: + vars[var] = "%s%s" % (value,vars[var]) + elif groupd['predot']: + vars[var] = "%s %s" % (value,vars[var]) + elif groupd['postdot']: + vars[var] = "%s %s" % (vars[var],value) + else: + vars[var] = "%s" % (value) + # capture vars in a Toaster section + if is_toaster_section: + toaster_vars[var] = vars[var] + + # DONE WITH PARSING + f.close() + self.vars = vars + self.toaster_vars = toaster_vars + + # Update the scanned project variables + def update_project_vars(self,project,name): + pv, create = ProjectVariable.objects.get_or_create(project = project, name = name) + if (not name in self.vars.keys()) or (not self.vars[name]): + self.vars[name] = pv.value + else: + if pv.value != self.vars[name]: + pv.value = self.vars[name] + pv.save() + + # Find the git version of the installation + def find_layer_dir_version(self,path): + # * rocko ... + + install_version = '' + cwd = os.getcwd() + os.chdir(path) + p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate() + out = out.decode("utf-8") + for branch in out.split('\n'): + if ('*' == branch[0:1]) and ('no branch' not in branch): + install_version = re.sub(' .*','',branch[2:]) + break + if 'remotes/m/master' in branch: + install_version = re.sub('.*base/','',branch) + break + os.chdir(cwd) + return install_version + + # Compute table of the installation's registered layer versions (branch or commit) + def find_layer_dir_versions(self,INSTALL_URL_PREFIX): + lv_dict = {} + layer_versions = Layer_Version.objects.all() + for lv in layer_versions: + layer = Layer.objects.filter(pk=lv.layer.pk)[0] + if layer.vcs_url: + url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'') + else: + url_short = '' + # register the core, branch, and the version variations + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name) + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name) + lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name) + #_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name)) + return lv_dict + + # Apply table of all layer versions + def extract_bblayers(self): + # set up the constants + bblayer_str = self.vars['BBLAYERS'] + TOASTER_DIR = os.environ.get('TOASTER_DIR') + INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/" + TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/" + INSTALL_URL_PREFIX = '' + layers = Layer.objects.filter(name='openembedded-core') + for layer in layers: + if layer.vcs_url: + INSTALL_URL_PREFIX = layer.vcs_url + break + INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/") + INSTALL_VERSION_DIR = TOASTER_DIR + INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_') + INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_') + INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX) + + # get the set of available layer:layer_versions + lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX) + + # compute the layer matches + layers_list = [] + for line in bblayer_str.split(' '): + if not line: + continue + if line.endswith('/local'): + continue + + # isolate the repo + layer_path = line + line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/') + + # isolate the sub-path + path_index = line.rfind('/') + if path_index > 0: + sub_path = line[path_index+1:] + line = line[0:path_index] + else: + sub_path = '' + + # isolate the version + if TOASTER_CLONE_PREFIX in layer_path: + is_toaster_clone = True + # extract version from name syntax + version_index = line.find('_') + if version_index > 0: + version = line[version_index+1:] + line = line[0:version_index] + else: + version = '' + _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version)) + else: + is_toaster_clone = False + # version is from the installation + version = self.find_layer_dir_version(layer_path) + _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version)) + + # capture the layer information into layers_list + layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) ) + return layers_list,lv_dict + + # + def find_import_release(self,layers_list,lv_dict,default_release): + # poky,meta,rocko => 4;openembedded-core + release = default_release + for line,path,version,layer_path,is_toaster_clone in layers_list: + key = "%s,%s,%s" % (line,path,version) + if key in lv_dict: + lv_id = lv_dict[key] + if 'openembedded-core' == lv_id[1]: + _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone)) + # only versions in Toaster managed layers are accepted + if not is_toaster_clone: + break + try: + release = Release.objects.get(name=version) + except: + pass + break + _log("Find_import_release:RELEASE=%s" % release.name) + return release + + # Apply the found conf layers + def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None): + for line,path,version,layer_path,is_toaster_clone in layers_list: + # Assert release promote if present + if release: + version = release + # try to match the key to a layer_version + key = "%s,%s,%s" % (line,path,version) + key_short = "%s,%s,%s" % (line,path,'') + lv_id = '' + if key in lv_dict: + lv_id = lv_dict[key] + lv = Layer_Version.objects.get(pk=int(lv_id[0])) + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=lv) + pl.optional=False + pl.save() + _log(" %s => %s;%s" % (key,lv_id[0],lv_id[1])) + elif key_short in lv_dict: + lv_id = lv_dict[key_short] + lv = Layer_Version.objects.get(pk=int(lv_id[0])) + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=lv) + pl.optional=False + pl.save() + _log(" %s ?> %s" % (key,lv_dict[key_short])) + else: + _log("%s <= %s" % (key,layer_path)) + found = False + # does local layer already exist in this project? + try: + for pl in ProjectLayer.objects.filter(project=project): + if pl.layercommit.layer.local_source_dir == layer_path: + found = True + _log(" Project Local Layer found!") + except Exception as e: + _log("ERROR: Local Layer '%s'" % e) + pass + + if not found: + # Does Layer name+path already exist? + try: + layer_name_base = os.path.basename(layer_path) + _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path)) + layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path) + # Found! Attach layer_version and ProjectLayer + layer_version = Layer_Version.objects.create( + layer=layer, + project=project, + layer_source=LayerSource.TYPE_IMPORTED) + layer_version.save() + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=layer_version) + pl.optional=False + pl.save() + found = True + # add layer contents to this layer version + scan_layer_content(layer,layer_version) + _log(" Parent Local Layer found in db!") + except Exception as e: + _log("Layer_exists_test_failed: Local Layer '%s'" % e) + pass + + if not found: + # Insure that layer path exists, in case of user typo + if not os.path.isdir(layer_path): + _log("ERROR:Layer path '%s' not found" % layer_path) + continue + # Add layer to db and attach project to it + layer_name_base = os.path.basename(layer_path) + # generate a unique layer name + layer_name_matches = {} + for layer in Layer.objects.filter(name__contains=layer_name_base): + layer_name_matches[layer.name] = '1' + layer_name_idx = 0 + layer_name_test = layer_name_base + while layer_name_test in layer_name_matches.keys(): + layer_name_idx += 1 + layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx) + # create the layer and layer_verion objects + layer = Layer.objects.create(name=layer_name_test) + layer.local_source_dir = layer_path + layer_version = Layer_Version.objects.create( + layer=layer, + project=project, + layer_source=LayerSource.TYPE_IMPORTED) + layer.save() + layer_version.save() + pl,created = ProjectLayer.objects.get_or_create(project=project, + layercommit=layer_version) + pl.optional=False + pl.save() + # register the layer's content + _log(" Local Layer Add content") + scan_layer_content(layer,layer_version) + _log(" Local Layer Added '%s'!" % layer_name_test) + + # Scan the project's conf files (if any) + def scan_conf_variables(self,project_path): + # scan the project's settings, add any new layers or variables + if os.path.isfile("%s/conf/local.conf" % project_path): + self.scan_conf("%s/conf/local.conf" % project_path) + self.scan_conf("%s/conf/bblayers.conf" % project_path) + # Import then disable old style Toaster conf files (before 'merged_attr') + old_toaster_local = "%s/conf/toaster.conf" % project_path + if os.path.isfile(old_toaster_local): + self.scan_conf(old_toaster_local) + shutil.move(old_toaster_local, old_toaster_local+"_old") + old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path + if os.path.isfile(old_toaster_layer): + self.scan_conf(old_toaster_layer) + shutil.move(old_toaster_layer, old_toaster_layer+"_old") + + # Scan the found conf variables (if any) + def apply_conf_variables(self,project,layers_list,lv_dict,release=None): + if self.vars: + # Catch vars relevant to Toaster (in case no Toaster section) + self.update_project_vars(project,'DISTRO') + self.update_project_vars(project,'MACHINE') + self.update_project_vars(project,'IMAGE_INSTALL_append') + self.update_project_vars(project,'IMAGE_FSTYPES') + self.update_project_vars(project,'PACKAGE_CLASSES') + # These vars are typically only assigned by Toaster + #self.update_project_vars(project,'DL_DIR') + #self.update_project_vars(project,'SSTATE_DIR') + + # Assert found Toaster vars + for var in self.toaster_vars.keys(): + pv, create = ProjectVariable.objects.get_or_create(project = project, name = var) + pv.value = self.toaster_vars[var] + _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value)) + pv.save() + + # Assert found BBLAYERS + if 0 < verbose: + for pl in ProjectLayer.objects.filter(project=project): + release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name + print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) + self.apply_conf_bblayers(layers_list,lv_dict,project,release) + if 0 < verbose: + for pl in ProjectLayer.objects.filter(project=project): + release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name + print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) + + + def handle(self, *args, **options): + project_name = options['name'] + project_path = options['path'] + project_callback = options['callback'] if options['callback'] else '' + release_name = options['release'] if options['release'] else '' + + # + # Delete project + # + + if options['delete_project']: + try: + print("Project '%s' delete from Toaster database" % (project_name)) + project = Project.objects.get(name=project_name) + # TODO: deep project delete + project.delete() + print("Project '%s' Deleted" % (project_name)) + return + except Exception as e: + print("Project '%s' not found, not deleted (%s)" % (project_name,e)) + return + + # + # Create/Update/Import project + # + + # See if project (by name) exists + project = None + try: + # Project already exists + project = Project.objects.get(name=project_name) + except Exception as e: + pass + + # Find the installation's default release + default_release = Release.objects.get(id=1) + + # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import' + if ("reconfigure" == options['command']) and (None == project): + options['command'] = 'import' + + # 'Configure': + if "configure" == options['command']: + # Note: ignore any existing conf files + # create project, SANITY: reuse any project of same name + project = Project.objects.create_project(project_name,default_release,project) + + # 'Re-configure': + if "reconfigure" == options['command']: + # Scan the directory's conf files + self.scan_conf_variables(project_path) + # Scan the layer list + layers_list,lv_dict = self.extract_bblayers() + # Apply any new layers or variables + self.apply_conf_variables(project,layers_list,lv_dict) + + # 'Import': + if "import" == options['command']: + # Scan the directory's conf files + self.scan_conf_variables(project_path) + # Remove these Toaster controlled variables + for var in ('DL_DIR','SSTATE_DIR'): + self.vars.pop(var, None) + self.toaster_vars.pop(var, None) + # Scan the layer list + layers_list,lv_dict = self.extract_bblayers() + # Find the directory's release, and promote to default_release if local paths + release = self.find_import_release(layers_list,lv_dict,default_release) + # create project, SANITY: reuse any project of same name + project = Project.objects.create_project(project_name,release,project) + # Apply any new layers or variables + self.apply_conf_variables(project,layers_list,lv_dict,release) + # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific' + project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1') + + # Set up the project's meta data + project.builddir = project_path + project.merged_attr = True + project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback) + project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT) + if ("configure" == options['command']) or ("import" == options['command']): + # preset the mode and default image recipe + project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW) + project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal") + # Assert any extended/custom actions or variables for new non-Toaster projects + if not len(self.toaster_vars): + pass + else: + project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE) + + # Save the updated Project + project.save() + + _log("Buildimport:project='%s' at '%d'" % (project_name,project.id)) + + if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']): + print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id)) + else: + print("|Project_id=%d" % (project.id)) + |