diff options
Diffstat (limited to 'yocto-poky/scripts/lib/devtool')
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/__init__.py | 138 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/build-image.py | 91 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/build.py | 77 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/deploy.py | 149 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/package.py | 61 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/sdk.py | 197 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/standard.py | 930 | ||||
| -rw-r--r-- | yocto-poky/scripts/lib/devtool/upgrade.py | 354 |
8 files changed, 1997 insertions, 0 deletions
diff --git a/yocto-poky/scripts/lib/devtool/__init__.py b/yocto-poky/scripts/lib/devtool/__init__.py new file mode 100644 index 000000000..404d3e664 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/__init__.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +# Development tool - utility functions for plugins +# +# Copyright (C) 2014 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugins module""" + +import os +import sys +import subprocess +import logging + +logger = logging.getLogger('devtool') + + +class DevtoolError(Exception): + """Exception for handling devtool errors""" + pass + + +def exec_build_env_command(init_path, builddir, cmd, watch=False, **options): + """Run a program in bitbake build context""" + import bb + if not 'cwd' in options: + options["cwd"] = builddir + if init_path: + # As the OE init script makes use of BASH_SOURCE to determine OEROOT, + # and can't determine it when running under dash, we need to set + # the executable to bash to correctly set things up + if not 'executable' in options: + options['executable'] = 'bash' + logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path)) + init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir) + else: + logger.debug('Executing command "%s"' % cmd) + init_prefix = '' + if watch: + if sys.stdout.isatty(): + # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly) + cmd = 'script -e -q -c "%s" /dev/null' % cmd + return exec_watch('%s%s' % (init_prefix, cmd), **options) + else: + return bb.process.run('%s%s' % (init_prefix, cmd), **options) + +def exec_watch(cmd, **options): + """Run program with stdout shown on sys.stdout""" + import bb + if isinstance(cmd, basestring) and not "shell" in options: + options["shell"] = True + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options + ) + + buf = '' + while True: + out = process.stdout.read(1) + if out: + sys.stdout.write(out) + sys.stdout.flush() + buf += out + elif out == '' and process.poll() != None: + break + + if process.returncode != 0: + raise bb.process.ExecutionError(cmd, process.returncode, buf, None) + + return buf, None + +def exec_fakeroot(d, cmd, **kwargs): + """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" + # Grab the command and check it actually exists + fakerootcmd = d.getVar('FAKEROOTCMD', True) + if not os.path.exists(fakerootcmd): + logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') + return 2 + # Set up the appropriate environment + newenv = dict(os.environ) + fakerootenv = d.getVar('FAKEROOTENV', True) + for varvalue in fakerootenv.split(): + if '=' in varvalue: + splitval = varvalue.split('=', 1) + newenv[splitval[0]] = splitval[1] + return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs) + +def setup_tinfoil(config_only=False): + """Initialize tinfoil api from bitbake""" + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + sys.exit(1) + + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only) + tinfoil.logger.setLevel(logger.getEffectiveLevel()) + return tinfoil + +def get_recipe_file(cooker, pn): + """Find recipe file corresponding a package name""" + import oe.recipeutils + recipefile = oe.recipeutils.pn_to_recipe(cooker, pn) + if not recipefile: + skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn) + if skipreasons: + logger.error('\n'.join(skipreasons)) + else: + logger.error("Unable to find any recipe file matching %s" % pn) + return recipefile + +def parse_recipe(config, tinfoil, pn, appends): + """Parse recipe of a package""" + import oe.recipeutils + recipefile = get_recipe_file(tinfoil.cooker, pn) + if not recipefile: + # Error already logged + return None + if appends: + append_files = tinfoil.cooker.collection.get_file_appends(recipefile) + # Filter out appends from the workspace + append_files = [path for path in append_files if + not path.startswith(config.workspace_path)] + return oe.recipeutils.parse_recipe(recipefile, append_files, + tinfoil.config_data) diff --git a/yocto-poky/scripts/lib/devtool/build-image.py b/yocto-poky/scripts/lib/devtool/build-image.py new file mode 100644 index 000000000..2c014289f --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/build-image.py @@ -0,0 +1,91 @@ +# Development tool - build-image plugin +# +# Copyright (C) 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool plugin containing the build-image subcommand.""" + +import os +import logging + +from bb.process import ExecutionError +from devtool import exec_build_env_command, setup_tinfoil, parse_recipe + +logger = logging.getLogger('devtool') + +def _get_recipes(workspace, config): + """Get list of target recipes from the workspace.""" + result = [] + tinfoil = setup_tinfoil() + for recipe in workspace: + data = parse_recipe(config, tinfoil, recipe, True) + if 'class-target' in data.getVar('OVERRIDES', True).split(':'): + if recipe in data.getVar('PACKAGES', True): + result.append(recipe) + else: + logger.warning("Skipping recipe %s as it doesn't produce " + "package with the same name", recipe) + tinfoil.shutdown() + return result + +def build_image(args, config, basepath, workspace): + """Entry point for the devtool 'build-image' subcommand.""" + image = args.recipe + appendfile = os.path.join(config.workspace_path, 'appends', + '%s.bbappend' % image) + + # remove <image>.bbapend to make sure setup_tinfoil doesn't + # breake because of it + if os.path.isfile(appendfile): + os.unlink(appendfile) + + recipes = _get_recipes(workspace, config) + if recipes: + with open(appendfile, 'w') as afile: + # include selected recipes into the image + afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(recipes)) + + # Generate notification callback devtool_warn_image_extended + afile.write('do_rootfs[prefuncs] += "devtool_warn_image_extended"\n\n') + afile.write("python devtool_warn_image_extended() {\n") + afile.write(" bb.plain('NOTE: %%s: building with additional '\n" + " 'packages due to \"devtool build-image\"'" + " %% d.getVar('PN', True))\n" + " bb.plain('NOTE: delete %%s to clear this' %% \\\n" + " '%s')\n" % os.path.relpath(appendfile, basepath)) + afile.write("}\n") + + logger.info('Building image %s with the following ' + 'additional packages: %s', image, ' '.join(recipes)) + else: + logger.warning('No recipes in workspace, building image %s unmodified', image) + + # run bitbake to build image + try: + exec_build_env_command(config.init_path, basepath, + 'bitbake %s' % image, watch=True) + except ExecutionError as err: + return err.exitcode + + logger.info('Successfully built %s', image) + +def register_commands(subparsers, context): + """Register devtool subcommands from the build-image plugin""" + parser = subparsers.add_parser('build-image', + help='Build image including workspace recipe packages', + description='Builds an image, extending it to include ' + 'packages from recipes in the workspace') + parser.add_argument('recipe', help='Image recipe to build') + parser.set_defaults(func=build_image) diff --git a/yocto-poky/scripts/lib/devtool/build.py b/yocto-poky/scripts/lib/devtool/build.py new file mode 100644 index 000000000..335aff549 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/build.py @@ -0,0 +1,77 @@ +# Development tool - build command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool build plugin""" + +import os +import bb +import logging +import argparse +import tempfile +from devtool import exec_build_env_command, DevtoolError + +logger = logging.getLogger('devtool') + +def plugin_init(pluginlist): + """Plugin initialization""" + pass + +def _create_conf_file(values, conf_file=None): + if not conf_file: + fd, conf_file = tempfile.mkstemp(suffix='.conf') + elif not os.path.exists(os.path.dirname(conf_file)): + logger.debug("Creating folder %s" % os.path.dirname(conf_file)) + bb.utils.mkdirhier(os.path.dirname(conf_file)) + with open(conf_file, 'w') as f: + for key, value in values.iteritems(): + f.write('%s = "%s"\n' % (key, value)) + return conf_file + +def build(args, config, basepath, workspace): + """Entry point for the devtool 'build' subcommand""" + if not args.recipename in workspace: + raise DevtoolError("no recipe named %s in your workspace" % + args.recipename) + + build_task = config.get('Build', 'build_task', 'populate_sysroot') + + postfile_param = "" + postfile = "" + if args.disable_parallel_make: + logger.info("Disabling 'make' parallelism") + postfile = os.path.join(basepath, 'conf', 'disable_parallelism.conf') + _create_conf_file({'PARALLEL_MAKE':''}, postfile) + postfile_param = "-R %s" % postfile + try: + exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s %s' % (build_task, postfile_param, args.recipename), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + finally: + if postfile: + logger.debug('Removing postfile') + os.remove(postfile) + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_build = subparsers.add_parser('build', help='Build a recipe', + description='Builds the specified recipe using bitbake', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser_build.add_argument('recipename', help='Recipe to build') + parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism') + parser_build.set_defaults(func=build) diff --git a/yocto-poky/scripts/lib/devtool/deploy.py b/yocto-poky/scripts/lib/devtool/deploy.py new file mode 100644 index 000000000..fa93adf18 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/deploy.py @@ -0,0 +1,149 @@ +# Development tool - deploy/undeploy command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugin containing the deploy subcommands""" + +import os +import subprocess +import logging +from devtool import exec_fakeroot, setup_tinfoil, DevtoolError + +logger = logging.getLogger('devtool') + +def deploy(args, config, basepath, workspace): + """Entry point for the devtool 'deploy' subcommand""" + import re + import oe.recipeutils + + if not args.recipename in workspace: + raise DevtoolError("no recipe named %s in your workspace" % + args.recipename) + try: + host, destdir = args.target.split(':') + except ValueError: + destdir = '/' + else: + args.target = host + + deploy_dir = os.path.join(basepath, 'target_deploy', args.target) + deploy_file = os.path.join(deploy_dir, args.recipename + '.list') + + tinfoil = setup_tinfoil() + try: + rd = oe.recipeutils.parse_recipe_simple(tinfoil.cooker, args.recipename, tinfoil.config_data) + except Exception as e: + raise DevtoolError('Exception parsing recipe %s: %s' % + (args.recipename, e)) + recipe_outdir = rd.getVar('D', True) + if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): + raise DevtoolError('No files to deploy - have you built the %s ' + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) + + if args.dry_run: + print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) + for root, _, files in os.walk(recipe_outdir): + for fn in files: + print(' %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)) + return 0 + + if os.path.exists(deploy_file): + if undeploy(args, config, basepath, workspace): + # Error already shown + return 1 + + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if args.show_status: + tarextractopts = 'xv' + else: + tarextractopts = 'x' + extraoptions += ' -q' + # We cannot use scp here, because it doesn't preserve symlinks + ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'tar %s -C %s -f -\'' % (extraoptions, args.target, tarextractopts, destdir), cwd=recipe_outdir, shell=True) + if ret != 0: + raise DevtoolError('Deploy failed - rerun with -s to get a complete ' + 'error message') + + logger.info('Successfully deployed %s' % recipe_outdir) + + if not os.path.exists(deploy_dir): + os.makedirs(deploy_dir) + + files_list = [] + for root, _, files in os.walk(recipe_outdir): + for filename in files: + filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) + files_list.append(os.path.join(destdir, filename)) + + with open(deploy_file, 'w') as fobj: + fobj.write('\n'.join(files_list)) + + return 0 + +def undeploy(args, config, basepath, workspace): + """Entry point for the devtool 'undeploy' subcommand""" + deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list') + if not os.path.exists(deploy_file): + raise DevtoolError('%s has not been deployed' % args.recipename) + + if args.dry_run: + print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target)) + with open(deploy_file, 'r') as f: + for line in f: + print(' %s' % line.rstrip()) + return 0 + + extraoptions = '' + if args.no_host_check: + extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + if not args.show_status: + extraoptions += ' -q' + + ret = subprocess.call("scp %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True) + if ret != 0: + raise DevtoolError('Failed to copy file list to %s - rerun with -s to ' + 'get a complete error message' % args.target) + + ret = subprocess.call("ssh %s %s 'xargs -n1 rm -f </tmp/%s'" % (extraoptions, args.target, os.path.basename(deploy_file)), shell=True) + if ret == 0: + logger.info('Successfully undeployed %s' % args.recipename) + os.remove(deploy_file) + else: + raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' + 'error message') + + return ret + + +def register_commands(subparsers, context): + """Register devtool subcommands from the deploy plugin""" + parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine') + parser_deploy.add_argument('recipename', help='Recipe to deploy') + parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') + parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') + parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') + parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') + parser_deploy.set_defaults(func=deploy) + + parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine') + parser_undeploy.add_argument('recipename', help='Recipe to undeploy') + parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') + parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') + parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') + parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_undeploy.set_defaults(func=undeploy) diff --git a/yocto-poky/scripts/lib/devtool/package.py b/yocto-poky/scripts/lib/devtool/package.py new file mode 100644 index 000000000..3a7a36b60 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/package.py @@ -0,0 +1,61 @@ +# Development tool - package command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool plugin containing the package subcommands""" + +import os +import subprocess +import logging +from bb.process import ExecutionError +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError + +logger = logging.getLogger('devtool') + +def plugin_init(pluginlist): + """Plugin initialization""" + pass + +def package(args, config, basepath, workspace): + """Entry point for the devtool 'package' subcommand""" + if not args.recipename in workspace: + raise DevtoolError("no recipe named %s in your workspace" % + args.recipename) + + image_pkgtype = config.get('Package', 'image_pkgtype', '') + if not image_pkgtype: + tinfoil = setup_tinfoil() + try: + tinfoil.prepare(config_only=True) + image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE', True) + finally: + tinfoil.shutdown() + + package_task = config.get('Package', 'package_task', 'package_write_%s' % image_pkgtype) + try: + exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (package_task, args.recipename), watch=True) + except bb.process.ExecutionError as e: + # We've already seen the output since watch=True, so just ensure we return something to the user + return e.exitcode + logger.info('Your packages are in %s/tmp/deploy/%s' % (basepath, image_pkgtype)) + + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from the package plugin""" + if context.fixed_setup: + parser_package = subparsers.add_parser('package', help='Build packages for a recipe', description='Builds packages for a recipe\'s output files') + parser_package.add_argument('recipename', help='Recipe to package') + parser_package.set_defaults(func=package) diff --git a/yocto-poky/scripts/lib/devtool/sdk.py b/yocto-poky/scripts/lib/devtool/sdk.py new file mode 100644 index 000000000..2f416b36f --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/sdk.py @@ -0,0 +1,197 @@ +# Development tool - sdk-update command plugin + +import os +import subprocess +import logging +import glob +import shutil +import errno +import sys +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError + +logger = logging.getLogger('devtool') + +def plugin_init(pluginlist): + """Plugin initialization""" + pass + +def parse_locked_sigs(sigfile_path): + """Return <pn:task>:<hash> dictionary""" + sig_dict = {} + with open(sigfile_path) as f: + lines = f.readlines() + for line in lines: + if ':' in line: + taskkey, _, hashval = line.rpartition(':') + sig_dict[taskkey.strip()] = hashval.split()[0] + return sig_dict + +def generate_update_dict(sigfile_new, sigfile_old): + """Return a dict containing <pn:task>:<hash> which indicates what need to be updated""" + update_dict = {} + sigdict_new = parse_locked_sigs(sigfile_new) + sigdict_old = parse_locked_sigs(sigfile_old) + for k in sigdict_new: + if k not in sigdict_old: + update_dict[k] = sigdict_new[k] + continue + if sigdict_new[k] != sigdict_old[k]: + update_dict[k] = sigdict_new[k] + continue + return update_dict + +def get_sstate_objects(update_dict, newsdk_path): + """Return a list containing sstate objects which are to be installed""" + sstate_objects = [] + # Ensure newsdk_path points to an extensible SDK + sstate_dir = os.path.join(newsdk_path, 'sstate-cache') + if not os.path.exists(sstate_dir): + logger.error("sstate-cache directory not found under %s" % newsdk_path) + raise + for k in update_dict: + files = set() + hashval = update_dict[k] + p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz' + files |= set(glob.glob(p)) + p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz' + files |= set(glob.glob(p)) + files = list(files) + if len(files) == 1: + sstate_objects.extend(files) + elif len(files) > 1: + logger.error("More than one matching sstate object found for %s" % hashval) + + return sstate_objects + +def mkdir(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + +def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): + """Install sstate objects into destination SDK""" + sstate_dir = os.path.join(dest_sdk, 'sstate-cache') + if not os.path.exists(sstate_dir): + logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk) + raise + for sb in sstate_objects: + dst = sb.replace(src_sdk, dest_sdk) + destdir = os.path.dirname(dst) + mkdir(destdir) + logger.debug("Copying %s to %s" % (sb, dst)) + shutil.copy(sb, dst) + +def sdk_update(args, config, basepath, workspace): + # Fetch locked-sigs.inc file from remote/local destination + from ConfigParser import NoSectionError + updateserver = args.updateserver + if not updateserver: + try: + updateserver = config.get('SDK', 'updateserver', None) + except NoSectionError: + pass + if not updateserver: + raise DevtoolError("Update server not specified in config file, you must specify it on the command line") + logger.debug("updateserver: %s" % args.updateserver) + + # Make sure we are using sdk-update from within SDK + logger.debug("basepath = %s" % basepath) + old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc') + if not os.path.exists(old_locked_sig_file_path): + logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option") + return -1 + else: + logger.debug("Found conf/locked-sigs.inc in %s" % basepath) + + if ':' in args.updateserver: + is_remote = True + else: + is_remote = False + + if not is_remote: + # devtool sdk-update /local/path/to/latest/sdk + new_locked_sig_file_path = os.path.join(args.updateserver, 'conf/locked-sigs.inc') + if not os.path.exists(new_locked_sig_file_path): + logger.error("%s doesn't exist or is not an extensible SDK" % args.updateserver) + return -1 + else: + logger.debug("Found conf/locked-sigs.inc in %s" % args.updateserver) + update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) + logger.debug("update_dict = %s" % update_dict) + sstate_objects = get_sstate_objects(update_dict, args.updateserver) + logger.debug("sstate_objects = %s" % sstate_objects) + if len(sstate_objects) == 0: + logger.info("No need to update.") + return 0 + logger.info("Installing sstate objects into %s", basepath) + install_sstate_objects(sstate_objects, args.updateserver.rstrip('/'), basepath) + logger.info("Updating configuration files") + new_conf_dir = os.path.join(args.updateserver, 'conf') + old_conf_dir = os.path.join(basepath, 'conf') + shutil.rmtree(old_conf_dir) + shutil.copytree(new_conf_dir, old_conf_dir) + logger.info("Updating layers") + new_layers_dir = os.path.join(args.updateserver, 'layers') + old_layers_dir = os.path.join(basepath, 'layers') + shutil.rmtree(old_layers_dir) + shutil.copytree(new_layers_dir, old_layers_dir) + else: + # devtool sdk-update http://myhost/sdk + tmpsdk_dir = '/tmp/sdk-ext' + if os.path.exists(tmpsdk_dir): + shutil.rmtree(tmpsdk_dir) + os.makedirs(tmpsdk_dir) + os.makedirs(os.path.join(tmpsdk_dir, 'conf')) + # Fetch locked-sigs.inc from update server + ret = subprocess.call("wget -q -O - %s/conf/locked-sigs.inc > %s/locked-sigs.inc" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf')), shell=True) + if ret != 0: + logger.error("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc failed" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf'))) + return ret + else: + logger.info("Fetching conf/locked-sigs.inc from %s to %s/locked-sigs.inc succeeded" % (args.updateserver, os.path.join(tmpsdk_dir, 'conf'))) + new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf/locked-sigs.inc') + update_dict = generate_update_dict(new_locked_sig_file_path, old_locked_sig_file_path) + logger.debug("update_dict = %s" % update_dict) + if len(update_dict) == 0: + logger.info("No need to update.") + return 0 + # Update metadata + logger.debug("Updating meta data via git ...") + # Try using 'git pull', if failed, use 'git clone' + if os.path.exists(os.path.join(basepath, 'layers/.git')): + ret = subprocess.call("cd layers && git pull", shell=True) + else: + ret = -1 + if ret != 0: + ret = subprocess.call("rm -rf layers && git clone %s/layers" % args.updateserver, shell=True) + if ret != 0: + logger.error("Updating meta data via git failed") + return ret + logger.debug("Updating conf files ...") + conf_files = ['local.conf', 'bblayers.conf', 'devtool.conf', 'work-config.inc', 'locked-sigs.inc'] + for conf in conf_files: + ret = subprocess.call("wget -q -O - %s/conf/%s > conf/%s" % (args.updateserver, conf, conf), shell=True) + if ret != 0: + logger.error("Update %s failed" % conf) + return ret + with open(os.path.join(basepath, 'conf/local.conf'), 'a') as f: + f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % args.updateserver) + + # Run bitbake command for the whole SDK + sdk_targets = config.get('SDK', 'sdk_targets') + logger.info("Executing 'bitbake %s' ... (This may take some time.)" % sdk_targets) + try: + exec_build_env_command(config.init_path, basepath, 'bitbake %s' % sdk_targets) + except: + logger.error('bitbake %s failed' % sdk_targets) + return -1 + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from the sdk plugin""" + if context.fixed_setup: + parser_sdk = subparsers.add_parser('sdk-update', help='Update SDK components from a nominated location') + parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from', nargs='?') + parser_sdk.set_defaults(func=sdk_update) diff --git a/yocto-poky/scripts/lib/devtool/standard.py b/yocto-poky/scripts/lib/devtool/standard.py new file mode 100644 index 000000000..d5900b4f8 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/standard.py @@ -0,0 +1,930 @@ +# Development tool - standard commands plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Devtool standard plugins""" + +import os +import sys +import re +import shutil +import tempfile +import logging +import argparse +import scriptutils +import errno +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError +from devtool import parse_recipe + +logger = logging.getLogger('devtool') + + +def add(args, config, basepath, workspace): + """Entry point for the devtool 'add' subcommand""" + import bb + import oe.recipeutils + + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % + args.recipename) + + reason = oe.recipeutils.validate_pn(args.recipename) + if reason: + raise DevtoolError(reason) + + srctree = os.path.abspath(args.srctree) + if os.path.exists(srctree): + if args.fetch: + if not os.path.isdir(srctree): + raise DevtoolError("Cannot fetch into source tree path %s as " + "it exists and is not a directory" % + srctree) + elif os.listdir(srctree): + raise DevtoolError("Cannot fetch into source tree path %s as " + "it already exists and is non-empty" % + srctree) + elif not args.fetch: + raise DevtoolError("Specified source tree %s could not be found" % + srctree) + + appendpath = os.path.join(config.workspace_path, 'appends') + if not os.path.exists(appendpath): + os.makedirs(appendpath) + + recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename) + bb.utils.mkdirhier(recipedir) + rfv = None + if args.version: + if '_' in args.version or ' ' in args.version: + raise DevtoolError('Invalid version string "%s"' % args.version) + rfv = args.version + if args.fetch: + if args.fetch.startswith('git://'): + rfv = 'git' + elif args.fetch.startswith('svn://'): + rfv = 'svn' + elif args.fetch.startswith('hg://'): + rfv = 'hg' + if rfv: + bp = "%s_%s" % (args.recipename, rfv) + else: + bp = args.recipename + recipefile = os.path.join(recipedir, "%s.bb" % bp) + if sys.stdout.isatty(): + color = 'always' + else: + color = args.color + extracmdopts = '' + if args.fetch: + source = args.fetch + extracmdopts = '-x %s' % srctree + else: + source = srctree + if args.version: + extracmdopts += ' -V %s' % args.version + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s" %s' % (color, recipefile, source, extracmdopts)) + logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) + except bb.process.ExecutionError as e: + raise DevtoolError('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + _add_md5(config, args.recipename, recipefile) + + initial_rev = None + if os.path.exists(os.path.join(srctree, '.git')): + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) + initial_rev = stdout.rstrip() + + appendfile = os.path.join(appendpath, '%s.bbappend' % bp) + with open(appendfile, 'w') as f: + f.write('inherit externalsrc\n') + f.write('EXTERNALSRC = "%s"\n' % srctree) + if args.same_dir: + f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) + if initial_rev: + f.write('\n# initial_rev: %s\n' % initial_rev) + + _add_md5(config, args.recipename, appendfile) + + return 0 + + +def _check_compatible_recipe(pn, d): + """Check if the recipe is supported by devtool""" + if pn == 'perf': + raise DevtoolError("The perf recipe does not actually check out " + "source and thus cannot be supported by this tool") + + if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): + raise DevtoolError("The %s recipe is not supported by this tool" % pn) + + if bb.data.inherits_class('image', d): + raise DevtoolError("The %s recipe is an image, and therefore is not " + "supported by this tool" % pn) + + if bb.data.inherits_class('populate_sdk', d): + raise DevtoolError("The %s recipe is an SDK, and therefore is not " + "supported by this tool" % pn) + + if bb.data.inherits_class('packagegroup', d): + raise DevtoolError("The %s recipe is a packagegroup, and therefore is " + "not supported by this tool" % pn) + + if bb.data.inherits_class('meta', d): + raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " + "not supported by this tool" % pn) + + if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True): + raise DevtoolError("externalsrc is currently enabled for the %s " + "recipe. This prevents the normal do_patch task " + "from working. You will need to disable this " + "first." % pn) + +def _ls_tree(directory): + """Recursive listing of files in a directory""" + ret = [] + for root, dirs, files in os.walk(directory): + ret.extend([os.path.relpath(os.path.join(root, fname), directory) for + fname in files]) + return ret + + +def extract(args, config, basepath, workspace): + """Entry point for the devtool 'extract' subcommand""" + import bb + + tinfoil = _prep_extract_operation(config, basepath, args.recipename) + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + srctree = os.path.abspath(args.srctree) + initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd) + logger.info('Source tree extracted to %s' % srctree) + + if initial_rev: + return 0 + else: + return 1 + +class BbTaskExecutor(object): + """Class for executing bitbake tasks for a recipe + + FIXME: This is very awkward. Unfortunately it's not currently easy to + properly execute tasks outside of bitbake itself, until then this has to + suffice if we are to handle e.g. linux-yocto's extra tasks + """ + + def __init__(self, rdata): + self.rdata = rdata + self.executed = [] + + def exec_func(self, func, report): + """Run bitbake task function""" + if not func in self.executed: + deps = self.rdata.getVarFlag(func, 'deps') + if deps: + for taskdepfunc in deps: + self.exec_func(taskdepfunc, True) + if report: + logger.info('Executing %s...' % func) + fn = self.rdata.getVar('FILE', True) + localdata = bb.build._task_data(fn, func, self.rdata) + bb.build.exec_func(func, localdata) + self.executed.append(func) + + +def _prep_extract_operation(config, basepath, recipename): + """HACK: Ugly workaround for making sure that requirements are met when + trying to extract a package. Returns the tinfoil instance to be used.""" + tinfoil = setup_tinfoil() + rd = parse_recipe(config, tinfoil, recipename, True) + + if bb.data.inherits_class('kernel-yocto', rd): + tinfoil.shutdown() + try: + stdout, _ = exec_build_env_command(config.init_path, basepath, + 'bitbake kern-tools-native') + tinfoil = setup_tinfoil() + except bb.process.ExecutionError as err: + raise DevtoolError("Failed to build kern-tools-native:\n%s" % + err.stdout) + return tinfoil + + +def _extract_source(srctree, keep_temp, devbranch, d): + """Extract sources of a recipe""" + import bb.event + import oe.recipeutils + + def eventfilter(name, handler, event, d): + """Bitbake event filter for devtool extract operation""" + if name == 'base_eventhandler': + return True + else: + return False + + if hasattr(bb.event, 'set_eventfilter'): + bb.event.set_eventfilter(eventfilter) + + pn = d.getVar('PN', True) + + _check_compatible_recipe(pn, d) + + if os.path.exists(srctree): + if not os.path.isdir(srctree): + raise DevtoolError("output path %s exists and is not a directory" % + srctree) + elif os.listdir(srctree): + raise DevtoolError("output path %s already exists and is " + "non-empty" % srctree) + + # Prepare for shutil.move later on + bb.utils.mkdirhier(srctree) + os.rmdir(srctree) + + # We don't want notes to be printed, they are too verbose + origlevel = bb.logger.getEffectiveLevel() + if logger.getEffectiveLevel() > logging.DEBUG: + bb.logger.setLevel(logging.WARNING) + + initial_rev = None + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + crd = d.createCopy() + # Make a subdir so we guard against WORKDIR==S + workdir = os.path.join(tempdir, 'workdir') + crd.setVar('WORKDIR', workdir) + crd.setVar('T', os.path.join(tempdir, 'temp')) + if not crd.getVar('S', True).startswith(workdir): + # Usually a shared workdir recipe (kernel, gcc) + # Try to set a reasonable default + if bb.data.inherits_class('kernel', d): + crd.setVar('S', '${WORKDIR}/source') + else: + crd.setVar('S', '${WORKDIR}/${BP}') + if bb.data.inherits_class('kernel', d): + # We don't want to move the source to STAGING_KERNEL_DIR here + crd.setVar('STAGING_KERNEL_DIR', '${S}') + + task_executor = BbTaskExecutor(crd) + + crd.setVar('EXTERNALSRC_forcevariable', '') + + logger.info('Fetching %s...' % pn) + task_executor.exec_func('do_fetch', False) + logger.info('Unpacking...') + task_executor.exec_func('do_unpack', False) + if bb.data.inherits_class('kernel-yocto', d): + # Extra step for kernel to populate the source directory + logger.info('Doing kernel checkout...') + task_executor.exec_func('do_kernel_checkout', False) + srcsubdir = crd.getVar('S', True) + if srcsubdir == workdir: + # Find non-patch sources that were "unpacked" to srctree directory + recipe_patches = [os.path.basename(patch) for patch in + oe.recipeutils.get_recipe_patches(crd)] + src_files = [fname for fname in _ls_tree(workdir) if + os.path.basename(fname) not in recipe_patches] + # Force separate S so that patch files can be left out from srctree + srcsubdir = tempfile.mkdtemp(dir=workdir) + crd.setVar('S', srcsubdir) + # Move source files to S + for path in src_files: + tgt_dir = os.path.join(srcsubdir, os.path.dirname(path)) + bb.utils.mkdirhier(tgt_dir) + shutil.move(os.path.join(workdir, path), tgt_dir) + elif os.path.dirname(srcsubdir) != workdir: + # Handle if S is set to a subdirectory of the source + srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) + + scriptutils.git_convert_standalone_clone(srcsubdir) + + patchdir = os.path.join(srcsubdir, 'patches') + haspatches = False + if os.path.exists(patchdir): + if os.listdir(patchdir): + haspatches = True + else: + os.rmdir(patchdir) + + if not os.listdir(srcsubdir): + raise DevtoolError("no source unpacked to S, perhaps the %s " + "recipe doesn't use any source?" % pn) + + if not os.path.exists(os.path.join(srcsubdir, '.git')): + bb.process.run('git init', cwd=srcsubdir) + bb.process.run('git add .', cwd=srcsubdir) + bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir) + + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) + initial_rev = stdout.rstrip() + + bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir) + bb.process.run('git tag -f devtool-base', cwd=srcsubdir) + crd.setVar('PATCHTOOL', 'git') + + logger.info('Patching...') + task_executor.exec_func('do_patch', False) + + bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) + + if os.path.exists(patchdir): + shutil.rmtree(patchdir) + if haspatches: + bb.process.run('git checkout patches', cwd=srcsubdir) + + shutil.move(srcsubdir, srctree) + finally: + bb.logger.setLevel(origlevel) + + if keep_temp: + logger.info('Preserving temporary directory %s' % tempdir) + else: + shutil.rmtree(tempdir) + return initial_rev + +def _add_md5(config, recipename, filename): + """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" + import bb.utils + + def addfile(fn): + md5 = bb.utils.md5_file(fn) + with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f: + f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)) + + if os.path.isdir(filename): + for root, _, files in os.walk(os.path.dirname(filename)): + for f in files: + addfile(os.path.join(root, f)) + else: + addfile(filename) + +def _check_preserve(config, recipename): + """Check if a file was manually changed and needs to be saved in 'attic' + directory""" + import bb.utils + origfile = os.path.join(config.workspace_path, '.devtool_md5') + newfile = os.path.join(config.workspace_path, '.devtool_md5_new') + preservepath = os.path.join(config.workspace_path, 'attic') + with open(origfile, 'r') as f: + with open(newfile, 'w') as tf: + for line in f.readlines(): + splitline = line.rstrip().split('|') + if splitline[0] == recipename: + removefile = os.path.join(config.workspace_path, splitline[1]) + try: + md5 = bb.utils.md5_file(removefile) + except IOError as err: + if err.errno == 2: + # File no longer exists, skip it + continue + else: + raise + if splitline[2] != md5: + bb.utils.mkdirhier(preservepath) + preservefile = os.path.basename(removefile) + logger.warn('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) + shutil.move(removefile, os.path.join(preservepath, preservefile)) + else: + os.remove(removefile) + else: + tf.write(line) + os.rename(newfile, origfile) + +def modify(args, config, basepath, workspace): + """Entry point for the devtool 'modify' subcommand""" + import bb + import oe.recipeutils + + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % + args.recipename) + + if not args.extract and not os.path.isdir(args.srctree): + raise DevtoolError("directory %s does not exist or not a directory " + "(specify -x to extract source from recipe)" % + args.srctree) + if args.extract: + tinfoil = _prep_extract_operation(config, basepath, args.recipename) + else: + tinfoil = setup_tinfoil() + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + recipefile = rd.getVar('FILE', True) + appendname = os.path.splitext(os.path.basename(recipefile))[0] + if args.wildcard: + appendname = re.sub(r'_.*', '_%', appendname) + appendpath = os.path.join(config.workspace_path, 'appends') + appendfile = os.path.join(appendpath, appendname + '.bbappend') + if os.path.exists(appendfile): + raise DevtoolError("Another variant of recipe %s is already in your " + "workspace (only one variant of a recipe can " + "currently be worked on at once)" + % args.recipename) + + _check_compatible_recipe(args.recipename, rd) + + initial_rev = None + commits = [] + srctree = os.path.abspath(args.srctree) + if args.extract: + initial_rev = _extract_source(args.srctree, False, args.branch, rd) + if not initial_rev: + return 1 + logger.info('Source tree extracted to %s' % srctree) + # Get list of commits since this revision + (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree) + commits = stdout.split() + else: + if os.path.exists(os.path.join(args.srctree, '.git')): + # Check if it's a tree previously extracted by us + try: + (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=args.srctree) + except bb.process.ExecutionError: + stdout = '' + for line in stdout.splitlines(): + if line.startswith('*'): + (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=args.srctree) + initial_rev = stdout.rstrip() + if not initial_rev: + # Otherwise, just grab the head revision + (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree) + initial_rev = stdout.rstrip() + + # Check that recipe isn't using a shared workdir + s = os.path.abspath(rd.getVar('S', True)) + workdir = os.path.abspath(rd.getVar('WORKDIR', True)) + if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: + # Handle if S is set to a subdirectory of the source + srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] + srctree = os.path.join(srctree, srcsubdir) + + if not os.path.exists(appendpath): + os.makedirs(appendpath) + with open(appendfile, 'w') as f: + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') + f.write('inherit externalsrc\n') + f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') + f.write('EXTERNALSRC_pn-%s = "%s"\n' % (args.recipename, srctree)) + + b_is_s = True + if args.no_same_dir: + logger.info('using separate build directory since --no-same-dir specified') + b_is_s = False + elif args.same_dir: + logger.info('using source tree as build directory since --same-dir specified') + elif bb.data.inherits_class('autotools-brokensep', rd): + logger.info('using source tree as build directory since original recipe inherits autotools-brokensep') + elif rd.getVar('B', True) == s: + logger.info('using source tree as build directory since that is the default for this recipe') + else: + b_is_s = False + if b_is_s: + f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (args.recipename, srctree)) + + if bb.data.inherits_class('kernel', rd): + f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout do_fetch do_unpack"\n') + if initial_rev: + f.write('\n# initial_rev: %s\n' % initial_rev) + for commit in commits: + f.write('# commit: %s\n' % commit) + + _add_md5(config, args.recipename, appendfile) + + logger.info('Recipe %s now set up to build from %s' % (args.recipename, srctree)) + + return 0 + +def _get_patchset_revs(args, srctree, recipe_path): + """Get initial and update rev of a recipe. These are the start point of the + whole patchset and start point for the patches to be re-generated/updated. + """ + import bb + + if args.initial_rev: + return args.initial_rev, args.initial_rev + + # Parse initial rev from recipe + commits = [] + initial_rev = None + with open(recipe_path, 'r') as f: + for line in f: + if line.startswith('# initial_rev:'): + initial_rev = line.split(':')[-1].strip() + elif line.startswith('# commit:'): + commits.append(line.split(':')[-1].strip()) + + update_rev = initial_rev + if initial_rev: + # Find first actually changed revision + stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % + initial_rev, cwd=srctree) + newcommits = stdout.split() + for i in xrange(min(len(commits), len(newcommits))): + if newcommits[i] == commits[i]: + update_rev = commits[i] + + return initial_rev, update_rev + +def _remove_patch_entries(srcuri, patchlist): + """Remove patch entries from SRC_URI""" + remaining = patchlist[:] + entries = [] + for patch in patchlist: + patchfile = os.path.basename(patch) + for i in xrange(len(srcuri)): + if srcuri[i].startswith('file://') and os.path.basename(srcuri[i].split(';')[0]) == patchfile: + entries.append(srcuri[i]) + remaining.remove(patch) + srcuri.pop(i) + break + return entries, remaining + +def _remove_patch_files(args, patches, destpath): + """Unlink existing patch files""" + for patchfile in patches: + if args.append: + if not destpath: + raise Exception('destpath should be set here') + patchfile = os.path.join(destpath, os.path.basename(patchfile)) + + if os.path.exists(patchfile): + logger.info('Removing patch %s' % patchfile) + # FIXME "git rm" here would be nice if the file in question is + # tracked + # FIXME there's a chance that this file is referred to by + # another recipe, in which case deleting wouldn't be the + # right thing to do + os.remove(patchfile) + # Remove directory if empty + try: + os.rmdir(os.path.dirname(patchfile)) + except OSError as ose: + if ose.errno != errno.ENOTEMPTY: + raise + +def _update_recipe_srcrev(args, srctree, rd, config_data): + """Implement the 'srcrev' mode of update-recipe""" + import bb + import oe.recipeutils + from oe.patch import GitApplyTree + + recipefile = rd.getVar('FILE', True) + logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile)) + + # Get HEAD revision + try: + stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) + except bb.process.ExecutionError as err: + raise DevtoolError('Failed to get HEAD revision in %s: %s' % + (srctree, err)) + srcrev = stdout.strip() + if len(srcrev) != 40: + raise DevtoolError('Invalid hash returned by git: %s' % stdout) + + destpath = None + removepatches = [] + patchfields = {} + patchfields['SRCREV'] = srcrev + orig_src_uri = rd.getVar('SRC_URI', False) or '' + if not args.no_remove: + # Find list of existing patches in recipe file + existing_patches = oe.recipeutils.get_recipe_patches(rd) + + old_srcrev = (rd.getVar('SRCREV', False) or '') + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + GitApplyTree.extractPatches(srctree, old_srcrev, tempdir) + newpatches = os.listdir(tempdir) + for patch in existing_patches: + patchfile = os.path.basename(patch) + if patchfile in newpatches: + removepatches.append(patch) + finally: + shutil.rmtree(tempdir) + + if removepatches: + srcuri = orig_src_uri.split() + removedentries, _ = _remove_patch_entries(srcuri, removepatches) + if removedentries: + patchfields['SRC_URI'] = ' '.join(srcuri) + + if args.append: + _, destpath = oe.recipeutils.bbappend_recipe( + rd, args.append, None, wildcardver=args.wildcard_version, + extralines=patchfields) + else: + oe.recipeutils.patch_recipe(rd, recipefile, patchfields) + + if not 'git://' in orig_src_uri: + logger.info('You will need to update SRC_URI within the recipe to ' + 'point to a git repository where you have pushed your ' + 'changes') + + _remove_patch_files(args, removepatches, destpath) + +def _update_recipe_patch(args, config, srctree, rd, config_data): + """Implement the 'patch' mode of update-recipe""" + import bb + import oe.recipeutils + from oe.patch import GitApplyTree + + recipefile = rd.getVar('FILE', True) + append = os.path.join(config.workspace_path, 'appends', '%s.bbappend' % + os.path.splitext(os.path.basename(recipefile))[0]) + if not os.path.exists(append): + raise DevtoolError('unable to find workspace bbappend for recipe %s' % + args.recipename) + + initial_rev, update_rev = _get_patchset_revs(args, srctree, append) + if not initial_rev: + raise DevtoolError('Unable to find initial revision - please specify ' + 'it with --initial-rev') + + # Find list of existing patches in recipe file + existing_patches = oe.recipeutils.get_recipe_patches(rd) + + removepatches = [] + seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') + if not args.no_remove: + # Get all patches from source tree and check if any should be removed + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + GitApplyTree.extractPatches(srctree, initial_rev, tempdir) + # Strip numbering from patch names. If it's a git sequence named + # patch, the numbers might not match up since we are starting from + # a different revision This does assume that people are using + # unique shortlog values, but they ought to be anyway... + newpatches = [seqpatch_re.match(fname).group(2) for fname in + os.listdir(tempdir)] + for patch in existing_patches: + basename = seqpatch_re.match( + os.path.basename(patch)).group(2) + if basename not in newpatches: + removepatches.append(patch) + finally: + shutil.rmtree(tempdir) + + # Get updated patches from source tree + tempdir = tempfile.mkdtemp(prefix='devtool') + try: + GitApplyTree.extractPatches(srctree, update_rev, tempdir) + + # Match up and replace existing patches with corresponding new patches + updatepatches = False + updaterecipe = False + destpath = None + newpatches = os.listdir(tempdir) + if args.append: + patchfiles = {} + for patch in existing_patches: + patchfile = os.path.basename(patch) + if patchfile in newpatches: + patchfiles[os.path.join(tempdir, patchfile)] = patchfile + newpatches.remove(patchfile) + for patchfile in newpatches: + patchfiles[os.path.join(tempdir, patchfile)] = None + + if patchfiles or removepatches: + removevalues = None + if removepatches: + srcuri = (rd.getVar('SRC_URI', False) or '').split() + removedentries, remaining = _remove_patch_entries( + srcuri, removepatches) + if removedentries or remaining: + remaining = ['file://' + os.path.basename(item) for + item in remaining] + removevalues = {'SRC_URI': removedentries + remaining} + _, destpath = oe.recipeutils.bbappend_recipe( + rd, args.append, patchfiles, + removevalues=removevalues) + else: + logger.info('No patches needed updating') + else: + for patch in existing_patches: + patchfile = os.path.basename(patch) + if patchfile in newpatches: + logger.info('Updating patch %s' % patchfile) + shutil.move(os.path.join(tempdir, patchfile), patch) + newpatches.remove(patchfile) + updatepatches = True + srcuri = (rd.getVar('SRC_URI', False) or '').split() + if newpatches: + # Add any patches left over + patchdir = os.path.join(os.path.dirname(recipefile), + rd.getVar('BPN', True)) + bb.utils.mkdirhier(patchdir) + for patchfile in newpatches: + logger.info('Adding new patch %s' % patchfile) + shutil.move(os.path.join(tempdir, patchfile), + os.path.join(patchdir, patchfile)) + srcuri.append('file://%s' % patchfile) + updaterecipe = True + if removepatches: + removedentries, _ = _remove_patch_entries(srcuri, removepatches) + if removedentries: + updaterecipe = True + if updaterecipe: + logger.info('Updating recipe %s' % os.path.basename(recipefile)) + oe.recipeutils.patch_recipe(rd, recipefile, + {'SRC_URI': ' '.join(srcuri)}) + elif not updatepatches: + # Neither patches nor recipe were updated + logger.info('No patches need updating') + finally: + shutil.rmtree(tempdir) + + _remove_patch_files(args, removepatches, destpath) + +def _guess_recipe_update_mode(srctree, rdata): + """Guess the recipe update mode to use""" + src_uri = (rdata.getVar('SRC_URI', False) or '').split() + git_uris = [uri for uri in src_uri if uri.startswith('git://')] + if not git_uris: + return 'patch' + # Just use the first URI for now + uri = git_uris[0] + # Check remote branch + params = bb.fetch.decodeurl(uri)[5] + upstr_branch = params['branch'] if 'branch' in params else 'master' + # Check if current branch HEAD is found in upstream branch + stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) + head_rev = stdout.rstrip() + stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, + cwd=srctree) + remote_brs = [branch.strip() for branch in stdout.splitlines()] + if 'origin/' + upstr_branch in remote_brs: + return 'srcrev' + + return 'patch' + +def update_recipe(args, config, basepath, workspace): + """Entry point for the devtool 'update-recipe' subcommand""" + if not args.recipename in workspace: + raise DevtoolError("no recipe named %s in your workspace" % + args.recipename) + + if args.append: + if not os.path.exists(args.append): + raise DevtoolError('bbappend destination layer directory "%s" ' + 'does not exist' % args.append) + if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): + raise DevtoolError('conf/layer.conf not found in bbappend ' + 'destination layer "%s"' % args.append) + + tinfoil = setup_tinfoil() + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + srctree = workspace[args.recipename]['srctree'] + if args.mode == 'auto': + mode = _guess_recipe_update_mode(srctree, rd) + else: + mode = args.mode + + if mode == 'srcrev': + _update_recipe_srcrev(args, srctree, rd, tinfoil.config_data) + elif mode == 'patch': + _update_recipe_patch(args, config, srctree, rd, tinfoil.config_data) + else: + raise DevtoolError('update_recipe: invalid mode %s' % mode) + + rf = rd.getVar('FILE', True) + if rf.startswith(config.workspace_path): + logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) + + return 0 + + +def status(args, config, basepath, workspace): + """Entry point for the devtool 'status' subcommand""" + if workspace: + for recipe, value in workspace.iteritems(): + print("%s: %s" % (recipe, value['srctree'])) + else: + logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') + return 0 + + +def reset(args, config, basepath, workspace): + """Entry point for the devtool 'reset' subcommand""" + import bb + if args.recipename: + if args.all: + raise DevtoolError("Recipe cannot be specified if -a/--all is used") + elif not args.recipename in workspace: + raise DevtoolError("no recipe named %s in your workspace" % + args.recipename) + elif not args.all: + raise DevtoolError("Recipe must be specified, or specify -a/--all to " + "reset all recipes") + if args.all: + recipes = workspace + else: + recipes = [args.recipename] + + for pn in recipes: + if not args.no_clean: + logger.info('Cleaning sysroot for recipe %s...' % pn) + try: + exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn) + except bb.process.ExecutionError as e: + raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' + 'wish, you may specify -n/--no-clean to ' + 'skip running this command when resetting' % + (e.command, e.stdout)) + + _check_preserve(config, pn) + + preservepath = os.path.join(config.workspace_path, 'attic', pn) + def preservedir(origdir): + if os.path.exists(origdir): + for root, dirs, files in os.walk(origdir): + for fn in files: + logger.warn('Preserving %s in %s' % (fn, preservepath)) + bb.utils.mkdirhier(preservepath) + shutil.move(os.path.join(origdir, fn), os.path.join(preservepath, fn)) + for dn in dirs: + os.rmdir(os.path.join(root, dn)) + os.rmdir(origdir) + + preservedir(os.path.join(config.workspace_path, 'recipes', pn)) + # We don't automatically create this dir next to appends, but the user can + preservedir(os.path.join(config.workspace_path, 'appends', pn)) + + return 0 + + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_add = subparsers.add_parser('add', help='Add a new recipe', + description='Adds a new recipe') + parser_add.add_argument('recipename', help='Name for new recipe to add') + parser_add.add_argument('srctree', help='Path to external source tree') + parser_add.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree', metavar='URI') + parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') + parser_add.set_defaults(func=add) + + parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', + description='Enables modifying the source for an existing recipe', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser_modify.add_argument('recipename', help='Name for recipe to edit') + parser_modify.add_argument('srctree', help='Path to external source tree') + parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') + parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well') + group = parser_modify.add_mutually_exclusive_group() + group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") + parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)') + parser_modify.set_defaults(func=modify) + + parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', + description='Extracts the source for an existing recipe', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser_extract.add_argument('recipename', help='Name for recipe to extract the source for') + parser_extract.add_argument('srctree', help='Path to where to extract the source tree') + parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') + parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_extract.set_defaults(func=extract) + + parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', + description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)') + parser_update_recipe.add_argument('recipename', help='Name of recipe to update') + parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') + parser_update_recipe.add_argument('--initial-rev', help='Starting revision for patches') + parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') + parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') + parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') + parser_update_recipe.set_defaults(func=update_recipe) + + parser_status = subparsers.add_parser('status', help='Show workspace status', + description='Lists recipes currently in your workspace and the paths to their respective external source trees', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser_status.set_defaults(func=status) + + parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', + description='Removes the specified recipe from your workspace (resetting its state)', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset') + parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') + parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') + parser_reset.set_defaults(func=reset) diff --git a/yocto-poky/scripts/lib/devtool/upgrade.py b/yocto-poky/scripts/lib/devtool/upgrade.py new file mode 100644 index 000000000..86443b073 --- /dev/null +++ b/yocto-poky/scripts/lib/devtool/upgrade.py @@ -0,0 +1,354 @@ +# Development tool - upgrade command plugin +# +# Copyright (C) 2014-2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +"""Devtool upgrade plugin""" + +import os +import sys +import re +import shutil +import tempfile +import logging +import argparse +import scriptutils +import errno +import bb +import oe.recipeutils +from devtool import standard +from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe + +logger = logging.getLogger('devtool') + +def plugin_init(pluginlist): + """Plugin initialization""" + pass + +def _run(cmd, cwd=''): + logger.debug("Running command %s> %s" % (cwd,cmd)) + return bb.process.run('%s' % cmd, cwd=cwd) + +def _get_srctree(tmpdir): + srctree = tmpdir + dirs = os.listdir(tmpdir) + if len(dirs) == 1: + srctree = os.path.join(tmpdir, dirs[0]) + return srctree + +def _copy_source_code(orig, dest): + for path in standard._ls_tree(orig): + dest_dir = os.path.join(dest, os.path.dirname(path)) + bb.utils.mkdirhier(dest_dir) + dest_path = os.path.join(dest, path) + os.rename(os.path.join(orig, path), dest_path) + +def _get_checksums(rf): + import re + checksums = {} + with open(rf) as f: + for line in f: + for cs in ['md5sum', 'sha256sum']: + m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) + if m: + checksums[cs] = m.group(1) + return checksums + +def _replace_checksums(rf, md5, sha256): + if not md5 and not sha256: + return + checksums = {'md5sum':md5, 'sha256sum':sha256} + with open(rf + ".tmp", "w+") as tmprf: + with open(rf) as f: + for line in f: + m = None + for cs in checksums.keys(): + m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line) + if m: + if checksums[cs]: + oldcheck = m.group(1) + newcheck = checksums[cs] + line = line.replace(oldcheck, newcheck) + break + tmprf.write(line) + os.rename(rf + ".tmp", rf) + + +def _remove_patch_dirs(recipefolder): + for root, dirs, files in os.walk(recipefolder): + for d in dirs: + shutil.rmtree(os.path.join(root,d)) + +def _recipe_contains(rf, var): + import re + found = False + with open(rf) as f: + for line in f: + if re.match("^%s.*=.*" % var, line): + found = True + break + return found + +def _rename_recipe_dirs(oldpv, newpv, path): + for root, dirs, files in os.walk(path): + for olddir in dirs: + if olddir.find(oldpv) != -1: + newdir = olddir.replace(oldpv, newpv) + if olddir != newdir: + _run('mv %s %s' % (olddir, newdir)) + +def _rename_recipe_file(bpn, oldpv, newpv, path): + oldrecipe = "%s_%s.bb" % (bpn, oldpv) + newrecipe = "%s_%s.bb" % (bpn, newpv) + if os.path.isfile(os.path.join(path, oldrecipe)): + if oldrecipe != newrecipe: + _run('mv %s %s' % (oldrecipe, newrecipe), cwd=path) + else: + recipe = "%s_git.bb" % bpn + if os.path.isfile(os.path.join(path, recipe)): + newrecipe = recipe + raise DevtoolError("Original recipe not found on workspace") + return os.path.join(path, newrecipe) + +def _rename_recipe_files(bpn, oldpv, newpv, path): + _rename_recipe_dirs(oldpv, newpv, path) + return _rename_recipe_file(bpn, oldpv, newpv, path) + +def _use_external_build(same_dir, no_same_dir, d): + b_is_s = True + if no_same_dir: + logger.info('using separate build directory since --no-same-dir specified') + b_is_s = False + elif same_dir: + logger.info('using source tree as build directory since --same-dir specified') + elif bb.data.inherits_class('autotools-brokensep', d): + logger.info('using source tree as build directory since original recipe inherits autotools-brokensep') + elif d.getVar('B', True) == os.path.abspath(d.getVar('S', True)): + logger.info('using source tree as build directory since that is the default for this recipe') + else: + b_is_s = False + return b_is_s + +def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d): + """Writes an append file""" + if not os.path.exists(rc): + raise DevtoolError("bbappend not created because %s does not exist" % rc) + + appendpath = os.path.join(workspace, 'appends') + if not os.path.exists(appendpath): + bb.utils.mkdirhier(appendpath) + + brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename + + srctree = os.path.abspath(srctree) + pn = d.getVar('PN',True) + af = os.path.join(appendpath, '%s.bbappend' % brf) + with open(af, 'w') as f: + f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') + f.write('inherit externalsrc\n') + f.write(('# NOTE: We use pn- overrides here to avoid affecting' + 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) + f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) + if _use_external_build(same_dir, no_same_dir, d): + f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) + if rev: + f.write('\n# initial_rev: %s\n' % rev) + return af + +def _cleanup_on_error(rf, srctree): + rfp = os.path.split(rf)[0] # recipe folder + rfpp = os.path.split(rfp)[0] # recipes folder + if os.path.exists(rfp): + shutil.rmtree(b) + if not len(os.listdir(rfpp)): + os.rmdir(rfpp) + srctree = os.path.abspath(srctree) + if os.path.exists(srctree): + shutil.rmtree(srctree) + +def _upgrade_error(e, rf, srctree): + if rf: + cleanup_on_error(rf, srctree) + logger.error(e) + raise DevtoolError(e) + +def _get_uri(rd): + srcuris = rd.getVar('SRC_URI', True).split() + if not len(srcuris): + raise DevtoolError('SRC_URI not found on recipe') + srcuri = srcuris[0] # it is assumed, URI is at first position + srcrev = '${AUTOREV}' + if '://' in srcuri: + # Fetch a URL + rev_re = re.compile(';rev=([^;]+)') + res = rev_re.search(srcuri) + if res: + srcrev = res.group(1) + srcuri = rev_re.sub('', srcuri) + return srcuri, srcrev + +def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): + """Extract sources of a recipe with a new version""" + + def __run(cmd): + """Simple wrapper which calls _run with srctree as cwd""" + return _run(cmd, srctree) + + crd = rd.createCopy() + + pv = crd.getVar('PV', True) + crd.setVar('PV', newpv) + + tmpsrctree = None + uri, rev = _get_uri(crd) + if srcrev: + rev = srcrev + if uri.startswith('git://'): + __run('git checkout %s' % rev) + __run('git tag -f devtool-base-new') + md5 = None + sha256 = None + else: + __run('git checkout -b devtool-%s' % newpv) + + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) + except bb.fetch2.FetchError as e: + raise DevtoolError(e) + + tmpsrctree = _get_srctree(tmpdir) + + scrtree = os.path.abspath(srctree) + + _copy_source_code(tmpsrctree, srctree) + + (stdout,_) = __run('git ls-files --modified --others --exclude-standard') + for f in stdout.splitlines(): + __run('git add "%s"' % f) + + __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) + __run('git tag -f devtool-base-%s' % newpv) + + (stdout, _) = __run('git rev-parse HEAD') + rev = stdout.rstrip() + + if no_patch: + patches = oe.recipeutils.get_recipe_patches(crd) + if len(patches): + logger.warn('By user choice, the following patches will NOT be applied') + for patch in patches: + logger.warn("%s" % os.path.basename(patch)) + else: + try: + __run('git checkout devtool-patched -b %s' % branch) + __run('git rebase %s' % rev) + if uri.startswith('git://'): + suffix = 'new' + else: + suffix = newpv + __run('git tag -f devtool-patched-%s' % suffix) + except bb.process.ExecutionError as e: + logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) + + if tmpsrctree: + if keep_temp: + logger.info('Preserving temporary directory %s' % tmpsrctree) + else: + shutil.rmtree(tmpsrctree) + + return (rev, md5, sha256) + +def _create_new_recipe(newpv, md5, sha256, workspace, rd): + """Creates the new recipe under workspace""" + crd = rd.createCopy() + + bpn = crd.getVar('BPN', True) + path = os.path.join(workspace, 'recipes', bpn) + bb.utils.mkdirhier(path) + oe.recipeutils.copy_recipe_files(crd, path) + + oldpv = crd.getVar('PV', True) + if not newpv: + newpv = oldpv + fullpath = _rename_recipe_files(bpn, oldpv, newpv, path) + + if _recipe_contains(fullpath, 'PV') and newpv != oldpv: + oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv}) + + if md5 and sha256: + # Unfortunately, oe.recipeutils.patch_recipe cannot update flags. + # once the latter feature is implemented, we should call patch_recipe + # instead of the following function + _replace_checksums(fullpath, md5, sha256) + + return fullpath + +def upgrade(args, config, basepath, workspace): + """Entry point for the devtool 'upgrade' subcommand""" + + if args.recipename in workspace: + raise DevtoolError("recipe %s is already in your workspace" % args.recipename) + if not args.version and not args.srcrev: + raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option") + + reason = oe.recipeutils.validate_pn(args.recipename) + if reason: + raise DevtoolError(reason) + + tinfoil = setup_tinfoil() + + rd = parse_recipe(config, tinfoil, args.recipename, True) + if not rd: + return 1 + + standard._check_compatible_recipe(args.recipename, rd) + if rd.getVar('PV', True) == args.version and rd.getVar('SRCREV', True) == args.srcrev: + raise DevtoolError("Current and upgrade versions are the same version" % version) + + rf = None + try: + rev1 = standard._extract_source(args.srctree, False, 'devtool-orig', rd) + rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch, + args.srcrev, args.branch, args.keep_temp, + tinfoil, rd) + rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd) + except bb.process.CmdError as e: + _upgrade_error(e, rf, args.srctree) + except DevtoolError as e: + _upgrade_error(e, rf, args.srctree) + standard._add_md5(config, args.recipename, os.path.dirname(rf)) + + af = _write_append(rf, args.srctree, args.same_dir, args.no_same_dir, rev2, + config.workspace_path, rd) + standard._add_md5(config, args.recipename, af) + logger.info('Upgraded source extracted to %s' % args.srctree) + return 0 + +def register_commands(subparsers, context): + """Register devtool subcommands from this plugin""" + parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe', + description='Upgrades an existing recipe to a new upstream version') + parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for') + parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree') + parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)') + parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)') + parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")') + parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code') + group = parser_upgrade.add_mutually_exclusive_group() + group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") + group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") + parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_upgrade.set_defaults(func=upgrade) |