diff options
Diffstat (limited to 'yocto-poky/scripts/lib/devtool/sdk.py')
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/sdk.py | 366 |
1 files changed, 0 insertions, 366 deletions
diff --git a/yocto-poky/scripts/lib/devtool/sdk.py b/yocto-poky/scripts/lib/devtool/sdk.py deleted file mode 100644 index 46fd12bdb..000000000 --- a/yocto-poky/scripts/lib/devtool/sdk.py +++ /dev/null @@ -1,366 +0,0 @@ -# Development tool - sdk-update command plugin -# -# Copyright (C) 2015-2016 Intel Corporation -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -import os -import subprocess -import logging -import glob -import shutil -import errno -import sys -import tempfile -import re -from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError - -logger = logging.getLogger('devtool') - -def parse_locked_sigs(sigfile_path): - """Return <pn:task>:<hash> dictionary""" - sig_dict = {} - with open(sigfile_path) as f: - lines = f.readlines() - for line in lines: - if ':' in line: - taskkey, _, hashval = line.rpartition(':') - sig_dict[taskkey.strip()] = hashval.split()[0] - return sig_dict - -def generate_update_dict(sigfile_new, sigfile_old): - """Return a dict containing <pn:task>:<hash> which indicates what need to be updated""" - update_dict = {} - sigdict_new = parse_locked_sigs(sigfile_new) - sigdict_old = parse_locked_sigs(sigfile_old) - for k in sigdict_new: - if k not in sigdict_old: - update_dict[k] = sigdict_new[k] - continue - if sigdict_new[k] != sigdict_old[k]: - update_dict[k] = sigdict_new[k] - continue - return update_dict - -def get_sstate_objects(update_dict, sstate_dir): - """Return a list containing sstate objects which are to be installed""" - sstate_objects = [] - for k in update_dict: - files = set() - hashval = update_dict[k] - p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz' - files |= set(glob.glob(p)) - p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz' - files |= set(glob.glob(p)) - files = list(files) - if len(files) == 1: - sstate_objects.extend(files) - elif len(files) > 1: - logger.error("More than one matching sstate object found for %s" % hashval) - - return sstate_objects - -def mkdir(d): - try: - os.makedirs(d) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - -def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): - """Install sstate objects into destination SDK""" - sstate_dir = os.path.join(dest_sdk, 'sstate-cache') - if not os.path.exists(sstate_dir): - logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk) - raise - for sb in sstate_objects: - dst = sb.replace(src_sdk, dest_sdk) - destdir = os.path.dirname(dst) - mkdir(destdir) - logger.debug("Copying %s to %s" % (sb, dst)) - shutil.copy(sb, dst) - -def check_manifest(fn, basepath): - import bb.utils - changedfiles = [] - with open(fn, 'r') as f: - for line in f: - splitline = line.split() - if len(splitline) > 1: - chksum = splitline[0] - fpath = splitline[1] - curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath)) - if chksum != curr_chksum: - logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum)) - changedfiles.append(fpath) - return changedfiles - -def sdk_update(args, config, basepath, workspace): - # Fetch locked-sigs.inc file from remote/local destination - updateserver = args.updateserver - if not updateserver: - updateserver = config.get('SDK', 'updateserver', '') - logger.debug("updateserver: %s" % updateserver) - - # Make sure we are using sdk-update from within SDK - logger.debug("basepath = %s" % basepath) - old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc') - if not os.path.exists(old_locked_sig_file_path): - logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option") - return -1 - else: - logger.debug("Found conf/locked-sigs.inc in %s" % basepath) - - if ':' in updateserver: - is_remote = True - else: - is_remote = False - - layers_dir = os.path.join(basepath, 'layers') - conf_dir = os.path.join(basepath, 'conf') - - # Grab variable values - tinfoil = setup_tinfoil(config_only=True, basepath=basepath) - try: - stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR', True) - sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS', True) - site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION', True) - finally: - tinfoil.shutdown() - - if not is_remote: - # devtool sdk-update /local/path/to/latest/sdk - new_locked_sig_file_path = os.path.join(updateserver, 'conf/locked-sigs.inc') - if not os.path.exists(new_locked_sig_file_path): - logger.error("%s doesn't exist or is not an extensible SDK" % updateserver) - return -1 - else: - logger.debug("Found conf/locked-sigs.inc in %s" % updateserver) - update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) - logger.debug("update_dict = %s" % update_dict) - newsdk_path = updateserver - sstate_dir = os.path.join(newsdk_path, 'sstate-cache') - if not os.path.exists(sstate_dir): - logger.error("sstate-cache directory not found under %s" % newsdk_path) - return 1 - sstate_objects = get_sstate_objects(update_dict, sstate_dir) - logger.debug("sstate_objects = %s" % sstate_objects) - if len(sstate_objects) == 0: - logger.info("No need to update.") - return 0 - logger.info("Installing sstate objects into %s", basepath) - install_sstate_objects(sstate_objects, updateserver.rstrip('/'), basepath) - logger.info("Updating configuration files") - new_conf_dir = os.path.join(updateserver, 'conf') - shutil.rmtree(conf_dir) - shutil.copytree(new_conf_dir, conf_dir) - logger.info("Updating layers") - new_layers_dir = os.path.join(updateserver, 'layers') - shutil.rmtree(layers_dir) - ret = subprocess.call("cp -a %s %s" % (new_layers_dir, layers_dir), shell=True) - if ret != 0: - logger.error("Copying %s to %s failed" % (new_layers_dir, layers_dir)) - return ret - else: - # devtool sdk-update http://myhost/sdk - tmpsdk_dir = tempfile.mkdtemp() - try: - os.makedirs(os.path.join(tmpsdk_dir, 'conf')) - new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') - # Fetch manifest from server - tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest') - ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True) - changedfiles = check_manifest(tmpmanifest, basepath) - if not changedfiles: - logger.info("Already up-to-date") - return 0 - # Update metadata - logger.debug("Updating metadata via git ...") - #Check for the status before doing a fetch and reset - if os.path.exists(os.path.join(basepath, 'layers/.git')): - out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir) - if not out: - ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir) - else: - logger.error("Failed to update metadata as there have been changes made to it. Aborting."); - logger.error("Changed files:\n%s" % out); - return -1 - else: - ret = -1 - if ret != 0: - ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir) - if ret != 0: - logger.error("Updating metadata via git failed") - return ret - logger.debug("Updating conf files ...") - for changedfile in changedfiles: - ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir) - if ret != 0: - logger.error("Updating %s failed" % changedfile) - return ret - - # Check if UNINATIVE_CHECKSUM changed - uninative = False - if 'conf/local.conf' in changedfiles: - def read_uninative_checksums(fn): - chksumitems = [] - with open(fn, 'r') as f: - for line in f: - if line.startswith('UNINATIVE_CHECKSUM'): - splitline = re.split(r'[\[\]"\']', line) - if len(splitline) > 3: - chksumitems.append((splitline[1], splitline[3])) - return chksumitems - - oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf')) - newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf')) - if oldsums != newsums: - uninative = True - for buildarch, chksum in newsums: - uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch) - mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file))) - ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir) - - # Ok, all is well at this point - move everything over - tmplayers_dir = os.path.join(tmpsdk_dir, 'layers') - if os.path.exists(tmplayers_dir): - shutil.rmtree(layers_dir) - shutil.move(tmplayers_dir, layers_dir) - for changedfile in changedfiles: - destfile = os.path.join(basepath, changedfile) - os.remove(destfile) - shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile) - os.remove(os.path.join(conf_dir, 'sdk-conf-manifest')) - shutil.move(tmpmanifest, conf_dir) - if uninative: - shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative')) - shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads')) - - if not sstate_mirrors: - with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: - f.write('SCONF_VERSION = "%s"\n' % site_conf_version) - f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) - finally: - shutil.rmtree(tmpsdk_dir) - - if not args.skip_prepare: - # Find all potentially updateable tasks - sdk_update_targets = [] - tasks = ['do_populate_sysroot', 'do_packagedata'] - for root, _, files in os.walk(stamps_dir): - for fn in files: - if not '.sigdata.' in fn: - for task in tasks: - if '.%s.' % task in fn or '.%s_setscene.' % task in fn: - sdk_update_targets.append('%s:%s' % (os.path.basename(root), task)) - # Run bitbake command for the whole SDK - logger.info("Preparing build system... (This may take some time.)") - try: - exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) - output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) - runlines = [] - for line in output.splitlines(): - if 'Running task ' in line: - runlines.append(line) - if runlines: - logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines)) - return -1 - except bb.process.ExecutionError as e: - logger.error('Preparation failed:\n%s' % e.stdout) - return -1 - return 0 - -def sdk_install(args, config, basepath, workspace): - """Entry point for the devtool sdk-install command""" - - import oe.recipeutils - import bb.process - - for recipe in args.recipename: - if recipe in workspace: - raise DevtoolError('recipe %s is a recipe in your workspace' % recipe) - - tasks = ['do_populate_sysroot', 'do_packagedata'] - stampprefixes = {} - def checkstamp(recipe): - stampprefix = stampprefixes[recipe] - stamps = glob.glob(stampprefix + '*') - for stamp in stamps: - if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')): - return True - else: - return False - - install_recipes = [] - tinfoil = setup_tinfoil(config_only=False, basepath=basepath) - try: - for recipe in args.recipename: - rd = parse_recipe(config, tinfoil, recipe, True) - if not rd: - return 1 - stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP', True), tasks[0]) - if checkstamp(recipe): - logger.info('%s is already installed' % recipe) - else: - install_recipes.append(recipe) - finally: - tinfoil.shutdown() - - if install_recipes: - logger.info('Installing %s...' % ', '.join(install_recipes)) - install_tasks = [] - for recipe in install_recipes: - for task in tasks: - if recipe.endswith('-native') and 'package' in task: - continue - install_tasks.append('%s:%s' % (recipe, task)) - options = '' - if not args.allow_build: - options += ' --setscene-only' - try: - exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True) - except bb.process.ExecutionError as e: - raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e))) - failed = False - for recipe in install_recipes: - if checkstamp(recipe): - logger.info('Successfully installed %s' % recipe) - else: - raise DevtoolError('Failed to install %s - unavailable' % recipe) - failed = True - if failed: - return 2 - -def register_commands(subparsers, context): - """Register devtool subcommands from the sdk plugin""" - if context.fixed_setup: - parser_sdk = subparsers.add_parser('sdk-update', - help='Update SDK components', - description='Updates installed SDK components from a remote server', - group='sdk') - updateserver = context.config.get('SDK', 'updateserver', '') - if updateserver: - parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?') - else: - parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from') - parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)') - parser_sdk.set_defaults(func=sdk_update) - - parser_sdk_install = subparsers.add_parser('sdk-install', - help='Install additional SDK components', - description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)', - group='sdk') - parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+') - parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true') - parser_sdk_install.set_defaults(func=sdk_install) |