diff options
Diffstat (limited to 'poky/scripts/lib/devtool')
| -rw-r--r-- | poky/scripts/lib/devtool/build.py | 13 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/menuconfig.py | 79 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/standard.py | 180 | ||||
| -rw-r--r-- | poky/scripts/lib/devtool/upgrade.py | 29 |
4 files changed, 257 insertions, 44 deletions
diff --git a/poky/scripts/lib/devtool/build.py b/poky/scripts/lib/devtool/build.py index 7543398d9..935ffab46 100644 --- a/poky/scripts/lib/devtool/build.py +++ b/poky/scripts/lib/devtool/build.py @@ -11,7 +11,8 @@ import bb import logging import argparse import tempfile -from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError +from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError +from devtool import parse_recipe logger = logging.getLogger('devtool') @@ -43,12 +44,22 @@ def _get_build_tasks(config): def build(args, config, basepath, workspace): """Entry point for the devtool 'build' subcommand""" workspacepn = check_workspace_recipe(workspace, args.recipename, bbclassextend=True) + tinfoil = setup_tinfoil(config_only=False, basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, args.recipename, appends=True, filter_workspace=False) + if not rd: + return 1 + deploytask = 'do_deploy' in rd.getVar('__BBTASKS') + finally: + tinfoil.shutdown() if args.clean: # use clean instead of cleansstate to avoid messing things up in eSDK build_tasks = ['do_clean'] else: build_tasks = _get_build_tasks(config) + if deploytask: + build_tasks.append('do_deploy') bbappend = workspace[workspacepn]['bbappend'] if args.disable_parallel_make: diff --git a/poky/scripts/lib/devtool/menuconfig.py b/poky/scripts/lib/devtool/menuconfig.py new file mode 100644 index 000000000..95384c533 --- /dev/null +++ b/poky/scripts/lib/devtool/menuconfig.py @@ -0,0 +1,79 @@ +# OpenEmbedded Development tool - menuconfig command plugin +# +# Copyright (C) 2018 Xilinx +# Written by: Chandana Kalluri <ckalluri@xilinx.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool menuconfig plugin""" + +import os +import bb +import logging +import argparse +import re +import glob +from devtool import setup_tinfoil, parse_recipe, DevtoolError, standard, exec_build_env_command +from devtool import check_workspace_recipe +logger = logging.getLogger('devtool') + +def menuconfig(args, config, basepath, workspace): + """Entry point for the devtool 'menuconfig' subcommand""" + + rd = "" + kconfigpath = "" + pn_src = "" + localfilesdir = "" + workspace_dir = "" + tinfoil = setup_tinfoil(basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, args.component, appends=True, filter_workspace=False) + if not rd: + return 1 + + check_workspace_recipe(workspace, args.component) + pn = rd.getVar('PN', True) + + if not rd.getVarFlag('do_menuconfig','task'): + raise DevtoolError("This recipe does not support menuconfig option") + + workspace_dir = os.path.join(config.workspace_path,'sources') + kconfigpath = rd.getVar('B') + pn_src = os.path.join(workspace_dir,pn) + + # add check to see if oe_local_files exists or not + localfilesdir = os.path.join(pn_src,'oe-local-files') + if not os.path.exists(localfilesdir): + bb.utils.mkdirhier(localfilesdir) + # Add gitignore to ensure source tree is clean + gitignorefile = os.path.join(localfilesdir,'.gitignore') + with open(gitignorefile, 'w') as f: + f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n') + f.write('*\n') + + finally: + tinfoil.shutdown() + + logger.info('Launching menuconfig') + exec_build_env_command(config.init_path, basepath, 'bitbake -c menuconfig %s' % pn, watch=True) + fragment = os.path.join(localfilesdir, 'devtool-fragment.cfg') + res = standard._create_kconfig_diff(pn_src,rd,fragment) + + return 0 + +def register_commands(subparsers, context): + """register devtool subcommands from this plugin""" + parser_menuconfig = subparsers.add_parser('menuconfig',help='Alter build-time configuration for a recipe', description='Launches the make menuconfig command (for recipes where do_menuconfig is available), allowing users to make changes to the build-time configuration. Creates a config fragment corresponding to changes made.', group='advanced') + parser_menuconfig.add_argument('component', help='compenent to alter config') + parser_menuconfig.set_defaults(func=menuconfig,fixed_setup=context.fixed_setup) diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index aca74b1fc..60c9a046f 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -461,11 +461,37 @@ def sync(args, config, basepath, workspace): finally: tinfoil.shutdown() +def symlink_oelocal_files_srctree(rd,srctree): + import oe.patch + if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): + # If recipe extracts to ${WORKDIR}, symlink the files into the srctree + # (otherwise the recipe won't build as expected) + local_files_dir = os.path.join(srctree, 'oe-local-files') + addfiles = [] + for root, _, files in os.walk(local_files_dir): + relpth = os.path.relpath(root, local_files_dir) + if relpth != '.': + bb.utils.mkdirhier(os.path.join(srctree, relpth)) + for fn in files: + if fn == '.gitignore': + continue + destpth = os.path.join(srctree, relpth, fn) + if os.path.exists(destpth): + os.unlink(destpth) + os.symlink('oe-local-files/%s' % fn, destpth) + addfiles.append(os.path.join(relpth, fn)) + if addfiles: + bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) + useroptions = [] + oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) + bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) + def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): """Extract sources of a recipe""" import oe.recipeutils import oe.patch + import oe.path pn = d.getVar('PN') @@ -562,7 +588,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works with open(preservestampfile, 'w') as f: f.write(d.getVar('STAMP')) try: - if bb.data.inherits_class('kernel-yocto', d): + if is_kernel_yocto: # We need to generate the kernel config task = 'do_configure' else: @@ -589,6 +615,23 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) + # Check if work-shared is empty, if yes + # find source and copy to work-shared + if is_kernel_yocto: + workshareddir = d.getVar('STAGING_KERNEL_DIR') + staging_kerVer = get_staging_kver(workshareddir) + kernelVersion = d.getVar('LINUX_VERSION') + + # handle dangling symbolic link in work-shared: + if os.path.islink(workshareddir): + os.unlink(workshareddir) + + if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): + shutil.rmtree(workshareddir) + oe.path.copyhardlinktree(srcsubdir,workshareddir) + elif not os.path.exists(workshareddir): + oe.path.copyhardlinktree(srcsubdir,workshareddir) + tempdir_localdir = os.path.join(tempdir, 'oe-local-files') srctree_localdir = os.path.join(srctree, 'oe-local-files') @@ -617,29 +660,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works shutil.move(tempdir_localdir, srcsubdir) shutil.move(srcsubdir, srctree) - - if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')): - # If recipe extracts to ${WORKDIR}, symlink the files into the srctree - # (otherwise the recipe won't build as expected) - local_files_dir = os.path.join(srctree, 'oe-local-files') - addfiles = [] - for root, _, files in os.walk(local_files_dir): - relpth = os.path.relpath(root, local_files_dir) - if relpth != '.': - bb.utils.mkdirhier(os.path.join(srctree, relpth)) - for fn in files: - if fn == '.gitignore': - continue - destpth = os.path.join(srctree, relpth, fn) - if os.path.exists(destpth): - os.unlink(destpth) - os.symlink('oe-local-files/%s' % fn, destpth) - addfiles.append(os.path.join(relpth, fn)) - if addfiles: - bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) - useroptions = [] - oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) - bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) + symlink_oelocal_files_srctree(d,srctree) if is_kernel_yocto: logger.info('Copying kernel config to srctree') @@ -707,11 +728,31 @@ def _check_preserve(config, recipename): tf.write(line) os.rename(newfile, origfile) +def get_staging_kver(srcdir): + # Kernel version from work-shared + kerver = [] + staging_kerVer="" + if os.path.exists(srcdir) and os.listdir(srcdir): + with open(os.path.join(srcdir,"Makefile")) as f: + version = [next(f) for x in range(5)][1:4] + for word in version: + kerver.append(word.split('= ')[1].split('\n')[0]) + staging_kerVer = ".".join(kerver) + return staging_kerVer + +def get_staging_kbranch(srcdir): + staging_kbranch = "" + if os.path.exists(srcdir) and os.listdir(srcdir): + (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) + staging_kbranch = "".join(branch.split('\n')[0]) + return staging_kbranch + def modify(args, config, basepath, workspace): """Entry point for the devtool 'modify' subcommand""" import bb import oe.recipeutils import oe.patch + import oe.path if args.recipename in workspace: raise DevtoolError("recipe %s is already in your workspace" % @@ -753,6 +794,59 @@ def modify(args, config, basepath, workspace): initial_rev = None commits = [] check_commits = False + + if bb.data.inherits_class('kernel-yocto', rd): + # Current set kernel version + kernelVersion = rd.getVar('LINUX_VERSION') + srcdir = rd.getVar('STAGING_KERNEL_DIR') + kbranch = rd.getVar('KBRANCH') + + staging_kerVer = get_staging_kver(srcdir) + staging_kbranch = get_staging_kbranch(srcdir) + if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): + oe.path.copyhardlinktree(srcdir,srctree) + workdir = rd.getVar('WORKDIR') + srcsubdir = rd.getVar('S') + localfilesdir = os.path.join(srctree,'oe-local-files') + # Move local source files into separate subdir + recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] + local_files = oe.recipeutils.get_recipe_local_files(rd) + + for key in local_files.copy(): + if key.endswith('scc'): + sccfile = open(local_files[key], 'r') + for l in sccfile: + line = l.split() + if line and line[0] in ('kconf', 'patch'): + cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) + if not cfg in local_files.values(): + local_files[line[-1]] = cfg + shutil.copy2(cfg, workdir) + sccfile.close() + + # Ignore local files with subdir={BP} + srcabspath = os.path.abspath(srcsubdir) + local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] + if local_files: + for fname in local_files: + _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) + with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: + f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') + + symlink_oelocal_files_srctree(rd,srctree) + + task = 'do_configure' + res = tinfoil.build_targets(pn, task, handle_events=True) + + # Copy .config to workspace + kconfpath = rd.getVar('B') + logger.info('Copying kernel config to workspace') + shutil.copy2(os.path.join(kconfpath, '.config'),srctree) + + # Set this to true, we still need to get initial_rev + # by parsing the git repo + args.no_extract = True + if not args.no_extract: initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) if not initial_rev: @@ -844,6 +938,11 @@ def modify(args, config, basepath, workspace): ' cp ${B}/.config ${S}/.config.baseline\n' ' ln -sfT ${B}/.config ${S}/.config.new\n' '}\n') + if rd.getVarFlag('do_menuconfig','task'): + f.write('\ndo_configure_append() {\n' + ' cp ${B}/.config ${S}/.config.baseline\n' + ' ln -sfT ${B}/.config ${S}/.config.new\n' + '}\n') if initial_rev: f.write('\n# initial_rev: %s\n' % initial_rev) for commit in commits: @@ -1520,17 +1619,17 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil patches_dir, changed_revs) logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) if filter_patches: - new_p = {} - upd_p = {k:v for k,v in upd_p.items() if k in filter_patches} + new_p = OrderedDict() + upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) remove_files = [f for f in remove_files if f in filter_patches] updatefiles = False updaterecipe = False destpath = None srcuri = (rd.getVar('SRC_URI', False) or '').split() if appendlayerdir: - files = dict((os.path.join(local_files_dir, key), val) for + files = OrderedDict((os.path.join(local_files_dir, key), val) for key, val in list(upd_f.items()) + list(new_f.items())) - files.update(dict((os.path.join(patches_dir, key), val) for + files.update(OrderedDict((os.path.join(patches_dir, key), val) for key, val in list(upd_p.items()) + list(new_p.items()))) if files or remove_files: removevalues = None @@ -1866,13 +1965,27 @@ def reset(args, config, basepath, workspace): def _get_layer(layername, d): """Determine the base layer path for the specified layer name/path""" layerdirs = d.getVar('BBLAYERS').split() - layers = {os.path.basename(p): p for p in layerdirs} + layers = {} # {basename: layer_paths} + for p in layerdirs: + bn = os.path.basename(p) + if bn not in layers: + layers[bn] = [p] + else: + layers[bn].append(p) # Provide some shortcuts if layername.lower() in ['oe-core', 'openembedded-core']: - layerdir = layers.get('meta', None) + layername = 'meta' + layer_paths = layers.get(layername, None) + if not layer_paths: + return os.path.abspath(layername) + elif len(layer_paths) == 1: + return os.path.abspath(layer_paths[0]) else: - layerdir = layers.get(layername, None) - return os.path.abspath(layerdir or layername) + # multiple layers having the same base name + logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) + logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) + return os.path.abspath(layer_paths[0]) + def finish(args, config, basepath, workspace): """Entry point for the devtool 'finish' subcommand""" @@ -1895,7 +2008,7 @@ def finish(args, config, basepath, workspace): else: raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) - no_clean = False + no_clean = args.no_clean tinfoil = setup_tinfoil(basepath=basepath, tracking=True) try: rd = parse_recipe(config, tinfoil, args.recipename, True) @@ -2169,6 +2282,7 @@ def register_commands(subparsers, context): parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') + parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') diff --git a/poky/scripts/lib/devtool/upgrade.py b/poky/scripts/lib/devtool/upgrade.py index 62ec2f94c..18c5b66a2 100644 --- a/poky/scripts/lib/devtool/upgrade.py +++ b/poky/scripts/lib/devtool/upgrade.py @@ -122,18 +122,22 @@ def _cleanup_on_error(rf, srctree): rfp = os.path.split(rf)[0] # recipe folder rfpp = os.path.split(rfp)[0] # recipes folder if os.path.exists(rfp): - shutil.rmtree(b) + shutil.rmtree(rfp) if not len(os.listdir(rfpp)): os.rmdir(rfpp) srctree = os.path.abspath(srctree) if os.path.exists(srctree): shutil.rmtree(srctree) -def _upgrade_error(e, rf, srctree): - if rf: - cleanup_on_error(rf, srctree) +def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None): + if rf and not keep_failure: + _cleanup_on_error(rf, srctree) logger.error(e) - raise DevtoolError(e) + if extramsg: + logger.error(extramsg) + if keep_failure: + logger.info('Preserving failed upgrade files (--keep-failure)') + sys.exit(1) def _get_uri(rd): srcuris = rd.getVar('SRC_URI').split() @@ -277,6 +281,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) + shutil.rmtree(tmpdir) return (rev, md5, sha256, srcbranch, srcsubdir_rel) @@ -299,7 +304,7 @@ def _add_license_diff_to_recipe(path, diff): f.write("\n#\n\n".encode()) f.write(orig_content) -def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses): +def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" bpn = rd.getVar('BPN') @@ -416,7 +421,10 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src newvalues["LIC_FILES_CHKSUM"] = newlicchksum _add_license_diff_to_recipe(fullpath, license_diff) - rd = tinfoil.parse_recipe_file(fullpath, False) + try: + rd = tinfoil.parse_recipe_file(fullpath, False) + except bb.tinfoil.TinfoilCommandFailed as e: + _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed') oe.recipeutils.patch_recipe(rd, fullpath, newvalues) return fullpath, copied @@ -548,11 +556,11 @@ def upgrade(args, config, basepath, workspace): tinfoil, rd) new_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM')) license_diff = _generate_license_diff(old_licenses, new_licenses) - rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses) + rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except bb.process.CmdError as e: - _upgrade_error(e, rf, srctree) + _upgrade_error(e, rf, srctree, args.keep_failure) except DevtoolError as e: - _upgrade_error(e, rf, srctree) + _upgrade_error(e, rf, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf)) af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2, @@ -623,6 +631,7 @@ def register_commands(subparsers, context): group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)') parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup) parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe', |