diff options
Diffstat (limited to 'yocto-poky/scripts/lib/devtool/sdk.py')
-rw-r--r-- | yocto-poky/scripts/lib/devtool/sdk.py | 295 |
1 files changed, 233 insertions, 62 deletions
diff --git a/yocto-poky/scripts/lib/devtool/sdk.py b/yocto-poky/scripts/lib/devtool/sdk.py index f15a6a9ed..46fd12bdb 100644 --- a/yocto-poky/scripts/lib/devtool/sdk.py +++ b/yocto-poky/scripts/lib/devtool/sdk.py @@ -1,4 +1,19 @@ # Development tool - sdk-update command plugin +# +# Copyright (C) 2015-2016 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import subprocess @@ -7,14 +22,12 @@ import glob import shutil import errno import sys -from devtool import exec_build_env_command, setup_tinfoil, DevtoolError +import tempfile +import re +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError logger = logging.getLogger('devtool') -def plugin_init(pluginlist): - """Plugin initialization""" - pass - def parse_locked_sigs(sigfile_path): """Return <pn:task>:<hash> dictionary""" sig_dict = {} @@ -78,13 +91,26 @@ def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): logger.debug("Copying %s to %s" % (sb, dst)) shutil.copy(sb, dst) +def check_manifest(fn, basepath): + import bb.utils + changedfiles = [] + with open(fn, 'r') as f: + for line in f: + splitline = line.split() + if len(splitline) > 1: + chksum = splitline[0] + fpath = splitline[1] + curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath)) + if chksum != curr_chksum: + logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum)) + changedfiles.append(fpath) + return changedfiles + def sdk_update(args, config, basepath, workspace): # Fetch locked-sigs.inc file from remote/local destination updateserver = args.updateserver if not updateserver: updateserver = config.get('SDK', 'updateserver', '') - if not updateserver: - raise DevtoolError("Update server not specified in config file, you must specify it on the command line") logger.debug("updateserver: %s" % updateserver) # Make sure we are using sdk-update from within SDK @@ -101,6 +127,18 @@ def sdk_update(args, config, basepath, workspace): else: is_remote = False + layers_dir = os.path.join(basepath, 'layers') + conf_dir = os.path.join(basepath, 'conf') + + # Grab variable values + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + try: + stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True) + sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True) + site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True) + finally: + tinfoil.shutdown() + if not is_remote: # devtool sdk-update /local/path/to/latest/sdk new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc') @@ -111,6 +149,7 @@ def sdk_update(args, config, basepath, workspace): logger.debug("Found conf/locked-sigs.inc in %s" % updateserver) update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) logger.debug("update_dict = %s" % update_dict) + newsdk_path = updateserver sstate_dir = os.path.join(newsdk_path, 'sstate-cache') if not os.path.exists(sstate_dir): logger.error("sstate-cache directory not found under %s" % newsdk_path) @@ -124,72 +163,204 @@ def sdk_update(args, config, basepath, workspace): install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath) logger.info("Updating configuration files") new_conf_dir = os.path.join(updateserver, 'conf') - old_conf_dir = os.path.join(basepath, 'conf') - shutil.rmtree(old_conf_dir) - shutil.copytree(new_conf_dir, old_conf_dir) + shutil.rmtree(conf_dir) + shutil.copytree(new_conf_dir, conf_dir) logger.info("Updating layers") new_layers_dir = os.path.join(updateserver, 'layers') - old_layers_dir = os.path.join(basepath, 'layers') - shutil.rmtree(old_layers_dir) - ret = subprocess.call("cp -a %s %s" % (new_layers_dir, old_layers_dir), shell=True) + shutil.rmtree(layers_dir) + ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True) if ret != 0: - logger.error("Copying %s to %s failed" % (new_layers_dir, old_layers_dir)) + logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir)) return ret else: # devtool sdk-update http://myhost/sdk - tmpsdk_dir = '/tmp/sdk-ext' - if os.path.exists(tmpsdk_dir): + tmpsdk_dir = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(tmpsdk_dir, 'conf')) + new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') + # Fetch manifest from server + tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest') + ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True) + changedfiles = check_manifest(tmpmanifest, basepath) + if not changedfiles: + logger.info("Already up-to-date") + return 0 + # Update metadata + logger.debug("Updating metadata via git ...") + #Check for the status before doing a fetch and reset + if os.path.exists(os.path.join(basepath, 'layers/.git')): + out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir) + if not out: + ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir) + else: + logger.error("Failed to update metadata as there have been changes made to it. Aborting."); + logger.error("Changed files:\n%s" % out); + return -1 + else: + ret = -1 + if ret != 0: + ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir) + if ret != 0: + logger.error("Updating metadata via git failed") + return ret + logger.debug("Updating conf files ...") + for changedfile in changedfiles: + ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir) + if ret != 0: + logger.error("Updating %s failed" % changedfile) + return ret + + # Check if UNINATIVE_CHECKSUM changed + uninative = False + if 'conf/local.conf' in changedfiles: + def read_uninative_checksums(fn): + chksumitems = [] + with open(fn, 'r') as f: + for line in f: + if line.startswith('UNINATIVE_CHECKSUM'): + splitline = re.split(r'[\[\]"\']', line) + if len(splitline) > 3: + chksumitems.append((splitline[1], splitline[3])) + return chksumitems + + oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf')) + newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf')) + if oldsums != newsums: + uninative = True + for buildarch, chksum in newsums: + uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch) + mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file))) + ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir) + + # Ok, all is well at this point - move everything over + tmplayers_dir = os.path.join(tmpsdk_dir, 'layers') + if os.path.exists(tmplayers_dir): + shutil.rmtree(layers_dir) + shutil.move(tmplayers_dir, layers_dir) + for changedfile in changedfiles: + destfile = os.path.join(basepath, changedfile) + os.remove(destfile) + shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile) + os.remove(os.path.join(conf_dir, 'sdk-conf-manifest')) + shutil.move(tmpmanifest, conf_dir) + if uninative: + shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative')) + shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads')) + + if not sstate_mirrors: + with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: + f.write('SCONF_VERSION = "%s"\n' % site_conf_version) + f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) + finally: shutil.rmtree(tmpsdk_dir) - os.makedirs(tmpsdk_dir) - os.makedirs(os.path.join(tmpsdk_dir, 'conf')) - # Fetch locked-sigs.inc from update server - ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s/locked-sigs.inc" % (updateserver, os.path.join(tmpsdk_dir, 'conf')), shell=True) - if ret != 0: - logger.error("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc failed" % (updateserver, os.path.join(tmpsdk_dir, 'conf'))) - return ret - else: - logger.info("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc succeeded" % (updateserver, os.path.join(tmpsdk_dir, 'conf'))) - new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf/locked-sigs.inc') - update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) - logger.debug("update_dict = %s" % update_dict) - if len(update_dict) == 0: - logger.info("No need to update.") - return 0 - # Update metadata - logger.debug("Updating meta data via git ...") - # Try using 'git pull', if failed, use 'git clone' - if os.path.exists(os.path.join(basepath, 'layers/.git')): - ret = subprocess.call("cd layers && git pull %s/layers/.git" % updateserver, shell=True) + + if not args.skip_prepare: + # Find all potentially updateable tasks + sdk_update_targets = [] + tasks = ['do_populate_sysroot', 'do_packagedata'] + for root, _, files in os.walk(stamps_dir): + for fn in files: + if not '.sigdata.' in fn: + for task in tasks: + if '.%s.' % task in fn or '.%s_setscene.' % task in fn: + sdk_update_targets.append('%s:%s' % (os.path.basename(root), task)) + # Run bitbake command for the whole SDK + logger.info("Preparing build system... (This may take some time.)") + try: + exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) + runlines = [] + for line in output.splitlines(): + if 'Running task ' in line: + runlines.append(line) + if runlines: + logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines)) + return -1 + except bb.process.ExecutionError as e: + logger.error('Preparation failed:\n%s' % e.stdout) + return -1 + return 0 + +def sdk_install(args, config, basepath, workspace): + """Entry point for the devtool sdk-install command""" + + import oe.recipeutils + import bb.process + + for recipe in args.recipename: + if recipe in workspace: + raise DevtoolError('recipe %s is a recipe in your workspace' % recipe) + + tasks = ['do_populate_sysroot', 'do_packagedata'] + stampprefixes = {} + def checkstamp(recipe): + stampprefix = stampprefixes[recipe] + stamps = glob.glob(stampprefix + '*') + for stamp in stamps: + if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')): + return True else: - ret = -1 - if ret != 0: - ret = subprocess.call("rm -rf layers && git clone %s/layers/.git" % updateserver, shell=True) - if ret != 0: - logger.error("Updating meta data via git failed") - return ret - logger.debug("Updating conf files ...") - conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'locked-sigs.inc'] - for conf in conf_files: - ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (updateserver, conf, conf), shell=True) - if ret != 0: - logger.error("Update %s failed" % conf) - return ret - with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f: - f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) - - # Run bitbake command for the whole SDK - sdk_targets = config.get('SDK', 'sdk_targets') - logger.info("Executing 'bitbake %s' ... (This may take some time.)" % sdk_targets) + return False + + install_recipes = [] + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) try: - exec_build_env_command(config.init_path, basepath, 'bitbake %s' % sdk_targets) - except: - logger.error('bitbake %s failed' % sdk_targets) - return -1 - return 0 + for recipe in args.recipename: + rd = parse_recipe(config, tinfoil, recipe, True) + if not rd: + return 1 + stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0]) + if checkstamp(recipe): + logger.info('%s is already installed' % recipe) + else: + install_recipes.append(recipe) + finally: + tinfoil.shutdown() + + if install_recipes: + logger.info('Installing %s...' % ', '.join(install_recipes)) + install_tasks = [] + for recipe in install_recipes: + for task in tasks: + if recipe.endswith('-native') and 'package' in task: + continue + install_tasks.append('%s:%s' % (recipe, task)) + options = '' + if not args.allow_build: + options += ' --setscene-only' + try: + exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True) + except bb.process.ExecutionError as e: + raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e))) + failed = False + for recipe in install_recipes: + if checkstamp(recipe): + logger.info('Successfully installed %s' % recipe) + else: + raise DevtoolError('Failed to install %s - unavailable' % recipe) + failed = True + if failed: + return 2 def register_commands(subparsers, context): """Register devtool subcommands from the sdk plugin""" if context.fixed_setup: - parser_sdk = subparsers.add_parser('sdk-update', help='Update SDK components from a nominated location') - parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from', nargs='?') + parser_sdk = subparsers.add_parser('sdk-update', + help='Update SDK components', + description='Updates installed SDK components from a remote server', + group='sdk') + updateserver = context.config.get('SDK', 'updateserver', '') + if updateserver: + parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?') + else: + parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from') + parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)') parser_sdk.set_defaults(func=sdk_update) + + parser_sdk_install = subparsers.add_parser('sdk-install', + help='Install additional SDK components', + description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)', + group='sdk') + parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+') + parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true') + parser_sdk_install.set_defaults(func=sdk_install) |