diff options
Diffstat (limited to 'poky/scripts')
22 files changed, 316 insertions, 159 deletions
diff --git a/poky/scripts/buildhistory-collect-srcrevs b/poky/scripts/buildhistory-collect-srcrevs index c4d203ff4..340bee78b 100755 --- a/poky/scripts/buildhistory-collect-srcrevs +++ b/poky/scripts/buildhistory-collect-srcrevs @@ -59,6 +59,7 @@ def main(): all_srcrevs = collections.defaultdict(list) for root, dirs, files in os.walk(options.buildhistory_dir): + dirs.sort() if '.git' in dirs: dirs.remove('.git') for fn in files: diff --git a/poky/scripts/buildhistory-diff b/poky/scripts/buildhistory-diff index 833f7c33a..3bd40a2a1 100755 --- a/poky/scripts/buildhistory-diff +++ b/poky/scripts/buildhistory-diff @@ -28,10 +28,12 @@ def get_args_parser(): %(prog)s [options] [from-revision [to-revision]] (if not specified, from-revision defaults to build-minus-1, and to-revision defaults to HEAD)""") + default_dir = os.path.join(os.environ.get('BUILDDIR', '.'), 'buildhistory') + parser.add_argument('-p', '--buildhistory-dir', action='store', dest='buildhistory_dir', - default='buildhistory/', + default=default_dir, help="Specify path to buildhistory directory (defaults to buildhistory/ under cwd)") parser.add_argument('-v', '--report-version', action='store_true', @@ -80,11 +82,6 @@ def main(): parser.print_help() sys.exit(1) - if not os.path.exists(args.buildhistory_dir): - if args.buildhistory_dir == 'buildhistory/': - cwd = os.getcwd() - if os.path.basename(cwd) == 'buildhistory': - args.buildhistory_dir = cwd if not os.path.exists(args.buildhistory_dir): sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % args.buildhistory_dir) diff --git a/poky/scripts/contrib/graph-tool b/poky/scripts/contrib/graph-tool index 6d2e68b82..26488930e 100755 --- a/poky/scripts/contrib/graph-tool +++ b/poky/scripts/contrib/graph-tool @@ -11,6 +11,13 @@ # import sys +import os +import argparse + +scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib')) +sys.path.insert(0, scripts_lib_path) +import argparse_oe + def get_path_networkx(dotfile, fromnode, tonode): try: @@ -34,47 +41,78 @@ def get_path_networkx(dotfile, fromnode, tonode): return networkx.all_simple_paths(graph, source=fromnode, target=tonode) -def find_paths(args, usage): - if len(args) < 3: - usage() - sys.exit(1) - - fromnode = args[1] - tonode = args[2] - +def find_paths(args): path = None - for path in get_path_networkx(args[0], fromnode, tonode): + for path in get_path_networkx(args.dotfile, args.fromnode, args.tonode): print(" -> ".join(map(str, path))) if not path: - print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) - sys.exit(1) + print("ERROR: no path from %s to %s in graph" % (args.fromnode, args.tonode)) + return 1 + + +def filter_graph(args): + import fnmatch + + exclude_tasks = [] + if args.exclude_tasks: + for task in args.exclude_tasks.split(','): + if not task.startswith('do_'): + task = 'do_%s' % task + exclude_tasks.append(task) + + def checkref(strval): + strval = strval.strip().strip('"') + target, taskname = strval.rsplit('.', 1) + if exclude_tasks: + for extask in exclude_tasks: + if fnmatch.fnmatch(taskname, extask): + return False + if strval in args.ref or target in args.ref: + return True + return False + + with open(args.infile, 'r') as f: + for line in f: + line = line.rstrip() + if line.startswith(('digraph', '}')): + print(line) + elif '->' in line: + linesplit = line.split('->') + if checkref(linesplit[0]) and checkref(linesplit[1]): + print(line) + elif (not args.no_nodes) and checkref(line.split()[0]): + print(line) + def main(): - import optparse - parser = optparse.OptionParser( - usage = '''%prog [options] <command> <arguments> + parser = argparse_oe.ArgumentParser(description='Small utility for working with .dot graph files') -Available commands: - find-paths <dotfile> <from> <to> - Find all of the paths between two nodes in a dot graph''') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + subparsers.required = True - #parser.add_option("-d", "--debug", - # help = "Report all SRCREV values, not just ones where AUTOREV has been used", - # action="store_true", dest="debug", default=False) + parser_find_paths = subparsers.add_parser('find-paths', + help='Find all of the paths between two nodes in a dot graph', + description='Finds all of the paths between two nodes in a dot graph') + parser_find_paths.add_argument('dotfile', help='.dot graph to search in') + parser_find_paths.add_argument('fromnode', help='starting node name') + parser_find_paths.add_argument('tonode', help='ending node name') + parser_find_paths.set_defaults(func=find_paths) - options, args = parser.parse_args(sys.argv) - args = args[1:] + parser_filter = subparsers.add_parser('filter', + help='Pare down a task graph to contain only the specified references', + description='Pares down a task-depends.dot graph produced by bitbake -g to contain only the specified references') + parser_filter.add_argument('infile', help='Input file') + parser_filter.add_argument('ref', nargs='+', help='Reference to include (either recipe/target name or full target.taskname specification)') + parser_filter.add_argument('-n', '--no-nodes', action='store_true', help='Skip node formatting lines') + parser_filter.add_argument('-x', '--exclude-tasks', help='Comma-separated list of tasks to exclude (do_ prefix optional, wildcards allowed)') + parser_filter.set_defaults(func=filter_graph) - if len(args) < 1: - parser.print_help() - sys.exit(1) + args = parser.parse_args() - if args[0] == "find-paths": - find_paths(args[1:], parser.print_help) - else: - parser.print_help() - sys.exit(1) + ret = args.func(args) + return ret if __name__ == "__main__": - main() + ret = main() + sys.exit(ret) diff --git a/poky/scripts/install-buildtools b/poky/scripts/install-buildtools index c6b3a1eed..7118f48be 100755 --- a/poky/scripts/install-buildtools +++ b/poky/scripts/install-buildtools @@ -35,6 +35,7 @@ import argparse import logging import os +import platform import re import shutil import shlex @@ -56,9 +57,9 @@ logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools') DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto' -DEFAULT_RELEASE = 'yocto-3.1' -DEFAULT_INSTALLER_VERSION = '3.1' -DEFAULT_BUILDDATE = '' +DEFAULT_RELEASE = 'yocto-3.2_M1' +DEFAULT_INSTALLER_VERSION = '3.1+snapshot' +DEFAULT_BUILDDATE = '20200617' # Python version sanity check if not (sys.version_info.major == 3 and sys.version_info.minor >= 4): @@ -112,6 +113,7 @@ def main(): release = "" buildtools_url = "" install_dir = "" + arch = platform.machine() parser = argparse.ArgumentParser( description="Buildtools installation helper", @@ -152,9 +154,11 @@ def main(): group.add_argument('--without-extended-buildtools', action='store_false', dest='with_extended_buildtools', help='disable extended buildtools (traditional buildtools tarball)') - parser.add_argument('-c', '--check', help='enable md5 checksum checking', - default=True, - action='store_true') + group = parser.add_mutually_exclusive_group() + group.add_argument('-c', '--check', help='enable checksum validation', + default=True, action='store_true') + group.add_argument('-n', '--no-check', help='disable checksum validation', + dest="check", action='store_false') parser.add_argument('-D', '--debug', help='enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='print only errors', @@ -194,19 +198,19 @@ def main(): logger.error("Milestone installers require --build-date") else: if args.with_extended_buildtools: - filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( - args.installer_version, args.build_date) + filename = "%s-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( + arch, args.installer_version, args.build_date) else: - filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % ( - args.installer_version, args.build_date) + filename = "%s-buildtools-nativesdk-standalone-%s-%s.sh" % ( + arch, args.installer_version, args.build_date) safe_filename = quote(filename) buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename) # regular release SDK else: if args.with_extended_buildtools: - filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version + filename = "%s-buildtools-extended-nativesdk-standalone-%s.sh" % (arch, args.installer_version) else: - filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version + filename = "%s-buildtools-nativesdk-standalone-%s.sh" % (arch, args.installer_version) safe_filename = quote(filename) buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename) @@ -225,7 +229,7 @@ def main(): if args.check: logger.info("Fetching buildtools installer checksum") checksum_type = "" - for checksum_type in ["md5sum", "sha256"]: + for checksum_type in ["md5sum", "sha256sum"]: check_url = "{}.{}".format(buildtools_url, checksum_type) checksum_filename = "{}.{}".format(filename, checksum_type) tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) @@ -237,7 +241,7 @@ def main(): if ret != 0: logger.error("Could not download file from %s" % check_url) return ret - regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s\s(?P<path>.*/)?(?P<filename>.*)$") + regex = re.compile(r"^(?P<checksum>[0-9a-f]+)\s+(?P<path>.*/)?(?P<filename>.*)$") with open(tmpbuildtools_checksum, 'rb') as f: original = f.read() m = re.search(regex, original.decode("utf-8")) @@ -258,6 +262,7 @@ def main(): else: logger.error("Checksum %s expected. Actual checksum is %s." % (checksum, checksum_value)) + return 1 # Make installer executable logger.info("Making installer executable") @@ -273,12 +278,13 @@ def main(): ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) if ret != 0: logger.error("Could not run buildtools installer") + return ret # Setup the environment logger.info("Setting up the environment") regex = re.compile(r'^(?P<export>export )?(?P<env_var>[A-Z_]+)=(?P<env_val>.+)$') - with open("%s/environment-setup-x86_64-pokysdk-linux" % - install_dir, 'rb') as f: + with open("%s/environment-setup-%s-pokysdk-linux" % + (install_dir, arch), 'rb') as f: for line in f: match = regex.search(line.decode('utf-8')) logger.debug("export regex: %s" % match) diff --git a/poky/scripts/lib/checklayer/__init__.py b/poky/scripts/lib/checklayer/__init__.py index 113800027..fe545607b 100644 --- a/poky/scripts/lib/checklayer/__init__.py +++ b/poky/scripts/lib/checklayer/__init__.py @@ -59,9 +59,14 @@ def _get_layer_collections(layer_path, lconf=None, data=None): pattern = ldata.getVar('BBFILE_PATTERN_%s' % name) depends = ldata.getVar('LAYERDEPENDS_%s' % name) compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name) + try: + depDict = bb.utils.explode_dep_versions2(depends or "") + except bb.utils.VersionStringException as vse: + bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (name, str(vse))) + collections[name]['priority'] = priority collections[name]['pattern'] = pattern - collections[name]['depends'] = depends + collections[name]['depends'] = ' '.join(depDict.keys()) collections[name]['compat'] = compat return collections @@ -224,6 +229,20 @@ def add_layers(bblayersconf, layers, logger): f.write("\nBBLAYERS += \"%s\"\n" % path) return True +def check_bblayers(bblayersconf, layer_path, logger): + ''' + If layer_path found in BBLAYERS return True + ''' + import bb.parse + import bb.data + + ldata = bb.parse.handle(bblayersconf, bb.data.init(), include=True) + for bblayer in (ldata.getVar('BBLAYERS') or '').split(): + if os.path.normpath(bblayer) == os.path.normpath(layer_path): + return True + + return False + def check_command(error_msg, cmd, cwd=None): ''' Run a command under a shell, capture stdout and stderr in a single stream, diff --git a/poky/scripts/lib/devtool/__init__.py b/poky/scripts/lib/devtool/__init__.py index d39c474fb..6ebe368a9 100644 --- a/poky/scripts/lib/devtool/__init__.py +++ b/poky/scripts/lib/devtool/__init__.py @@ -196,7 +196,7 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None): if not os.path.exists(os.path.join(repodir, '.git')): bb.process.run('git init', cwd=repodir) bb.process.run('git config --local gc.autodetach 0', cwd=repodir) - bb.process.run('git add .', cwd=repodir) + bb.process.run('git add -f -A .', cwd=repodir) commit_cmd = ['git'] oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d) commit_cmd += ['commit', '-q'] diff --git a/poky/scripts/lib/devtool/deploy.py b/poky/scripts/lib/devtool/deploy.py index 6a997735f..e5af2c95a 100644 --- a/poky/scripts/lib/devtool/deploy.py +++ b/poky/scripts/lib/devtool/deploy.py @@ -177,13 +177,19 @@ def deploy(args, config, basepath, workspace): rd.getVar('base_libdir'), rd) filelist = [] + inodes = set({}) ftotalsize = 0 for root, _, files in os.walk(recipe_outdir): for fn in files: + fstat = os.lstat(os.path.join(root, fn)) # Get the size in kiB (since we'll be comparing it to the output of du -k) # MUST use lstat() here not stat() or getfilesize() since we don't want to # dereference symlinks - fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024)) + if fstat.st_ino in inodes: + fsize = 0 + else: + fsize = int(math.ceil(float(fstat.st_size)/1024)) + inodes.add(fstat.st_ino) ftotalsize += fsize # The path as it would appear on the target fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) @@ -330,7 +336,7 @@ def register_commands(subparsers, context): parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') parser_deploy.add_argument('-I', '--key', - help='Specifiy ssh private key for connection to the target') + help='Specify ssh private key for connection to the target') strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) strip_opts.add_argument('-S', '--strip', @@ -355,6 +361,6 @@ def register_commands(subparsers, context): parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') parser_undeploy.add_argument('-I', '--key', - help='Specifiy ssh private key for connection to the target') + help='Specify ssh private key for connection to the target') parser_undeploy.set_defaults(func=undeploy) diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index bab644b83..d140b97de 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -1711,7 +1711,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil def _guess_recipe_update_mode(srctree, rdata): """Guess the recipe update mode to use""" - src_uri = (rdata.getVar('SRC_URI', False) or '').split() + src_uri = (rdata.getVar('SRC_URI') or '').split() git_uris = [uri for uri in src_uri if uri.startswith('git://')] if not git_uris: return 'patch' diff --git a/poky/scripts/lib/devtool/upgrade.py b/poky/scripts/lib/devtool/upgrade.py index f962a71e4..0c1de8cdc 100644 --- a/poky/scripts/lib/devtool/upgrade.py +++ b/poky/scripts/lib/devtool/upgrade.py @@ -235,14 +235,14 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee # Copy in new ones _copy_source_code(tmpsrctree, srctree) - (stdout,_) = __run('git ls-files --modified --others --exclude-standard') + (stdout,_) = __run('git ls-files --modified --others') filelist = stdout.splitlines() pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist)) pbar.start() batchsize = 100 for i in range(0, len(filelist), batchsize): batch = filelist[i:i+batchsize] - __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch])) + __run('git add -f -A %s' % ' '.join(['"%s"' % item for item in batch])) pbar.update(i) pbar.finish() @@ -550,12 +550,12 @@ def upgrade(args, config, basepath, workspace): try: logger.info('Extracting current version source...') rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) - old_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM')) + old_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) logger.info('Extracting upgraded version source...') rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, args.srcrev, args.srcbranch, args.branch, args.keep_temp, tinfoil, rd) - new_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM')) + new_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) license_diff = _generate_license_diff(old_licenses, new_licenses) rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except bb.process.CmdError as e: diff --git a/poky/scripts/lib/recipetool/create.py b/poky/scripts/lib/recipetool/create.py index 6cbf4de67..566c75369 100644 --- a/poky/scripts/lib/recipetool/create.py +++ b/poky/scripts/lib/recipetool/create.py @@ -66,7 +66,7 @@ class RecipeHandler(object): libdir = d.getVar('libdir') base_libdir = d.getVar('base_libdir') libpaths = list(set([base_libdir, libdir])) - libname_re = re.compile('^lib(.+)\.so.*$') + libname_re = re.compile(r'^lib(.+)\.so.*$') pkglibmap = {} for lib, item in shlib_providers.items(): for path, pkg in item.items(): @@ -428,7 +428,7 @@ def create_recipe(args): if scriptutils.is_src_url(source): # Warn about github archive URLs - if re.match('https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): + if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).') # Fetch a URL fetchuri = reformat_git_uri(urldefrag(source)[0]) @@ -460,6 +460,7 @@ def create_recipe(args): logger.error('branch= parameter and -B/--srcbranch option cannot both be specified - use one or the other') sys.exit(1) srcbranch = args.srcbranch + params['branch'] = srcbranch nobranch = params.get('nobranch') if nobranch and srcbranch: logger.error('nobranch= cannot be used if you specify a branch') @@ -829,7 +830,7 @@ def create_recipe(args): elif line.startswith('PV = '): if realpv: # Replace the first part of the PV value - line = re.sub('"[^+]*\+', '"%s+' % realpv, line) + line = re.sub(r'"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: @@ -1065,8 +1066,8 @@ def crunch_license(licfile): import oe.utils # Note: these are carefully constructed! - license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') - license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') + license_title_re = re.compile(r'^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$') + license_statement_re = re.compile(r'^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$') copyright_re = re.compile('^(#+)? *Copyright .*$') crunched_md5sums = {} diff --git a/poky/scripts/lib/recipetool/edit.py b/poky/scripts/lib/recipetool/edit.py index 94bdf7bca..d5b980a1c 100644 --- a/poky/scripts/lib/recipetool/edit.py +++ b/poky/scripts/lib/recipetool/edit.py @@ -34,7 +34,7 @@ def edit(args): recipe_path = tinfoil.get_recipe_file(args.target) appends = tinfoil.get_file_appends(recipe_path) - return scriptutils.run_editor([recipe_path] + appends, logger) + return scriptutils.run_editor([recipe_path] + list(appends), logger) def register_commands(subparsers): diff --git a/poky/scripts/lib/wic/filemap.py b/poky/scripts/lib/wic/filemap.py index c53147c2f..4d9da2817 100644 --- a/poky/scripts/lib/wic/filemap.py +++ b/poky/scripts/lib/wic/filemap.py @@ -142,15 +142,6 @@ class _FilemapBase(object): raise Error("the method is not implemented") - def block_is_unmapped(self, block): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. It returns - 'True' if block number 'block' of the image file is not mapped (hole) - and 'False' otherwise. - """ - - raise Error("the method is not implemented") - def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201 """ This method has has to be implemented by child classes. This is a @@ -164,15 +155,6 @@ class _FilemapBase(object): raise Error("the method is not implemented") - def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. Just like - 'get_mapped_ranges()', but yields unmapped block ranges instead - (holes). - """ - - raise Error("the method is not implemented") - # The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call _SEEK_DATA = 3 @@ -265,15 +247,10 @@ class FilemapSeek(_FilemapBase): % (block, result)) return result - def block_is_unmapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - return not self.block_is_mapped(block) - def _get_ranges(self, start, count, whence1, whence2): """ - This function implements 'get_mapped_ranges()' and - 'get_unmapped_ranges()' depending on what is passed in the 'whence1' - and 'whence2' arguments. + This function implements 'get_mapped_ranges()' depending + on what is passed in the 'whence1' and 'whence2' arguments. """ assert whence1 != whence2 @@ -303,12 +280,6 @@ class FilemapSeek(_FilemapBase): % (start, count, start + count - 1)) return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) - def get_unmapped_ranges(self, start, count): - """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) - # Below goes the FIEMAP ioctl implementation, which is not very readable # because it deals with the rather complex FIEMAP ioctl. To understand the @@ -422,10 +393,6 @@ class FilemapFiemap(_FilemapBase): % (block, result)) return result - def block_is_unmapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - return not self.block_is_mapped(block) - def _unpack_fiemap_extent(self, index): """ Unpack a 'struct fiemap_extent' structure object number 'index' from @@ -502,23 +469,28 @@ class FilemapFiemap(_FilemapBase): % (first_prev, last_prev)) yield (first_prev, last_prev) - def get_unmapped_ranges(self, start, count): +class FilemapNobmap(_FilemapBase): + """ + This class is used when both the 'SEEK_DATA/HOLE' and FIEMAP are not + supported by the filesystem or kernel. + """ + + def __init__(self, image, log=None): """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapFiemap: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - hole_first = start - for first, last in self._do_get_mapped_ranges(start, count): - if first > hole_first: - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, first - 1)) - yield (hole_first, first - 1) - hole_first = last + 1 + # Call the base class constructor first + _FilemapBase.__init__(self, image, log) + self._log.debug("FilemapNobmap: initializing") + + def block_is_mapped(self, block): + """Refer the '_FilemapBase' class for the documentation.""" + return True - if hole_first < start + count: - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, start + count - 1)) - yield (hole_first, start + count - 1) + def get_mapped_ranges(self, start, count): + """Refer the '_FilemapBase' class for the documentation.""" + self._log.debug("FilemapNobmap: get_mapped_ranges(%d, %d(%d))" + % (start, count, start + count - 1)) + yield (start, start + count -1) def filemap(image, log=None): """ @@ -533,7 +505,10 @@ def filemap(image, log=None): try: return FilemapFiemap(image, log) except ErrorNotSupp: - return FilemapSeek(image, log) + try: + return FilemapSeek(image, log) + except ErrorNotSupp: + return FilemapNobmap(image, log) def sparse_copy(src_fname, dst_fname, skip=0, seek=0, length=0, api=None): diff --git a/poky/scripts/lib/wic/ksparser.py b/poky/scripts/lib/wic/ksparser.py index b8befe78e..913e3283d 100644 --- a/poky/scripts/lib/wic/ksparser.py +++ b/poky/scripts/lib/wic/ksparser.py @@ -51,26 +51,39 @@ class KickStartParser(ArgumentParser): def error(self, message): raise ArgumentError(None, message) -def sizetype(arg): - """ - Custom type for ArgumentParser - Converts size string in <num>[K|k|M|G] format into the integer value - """ - if arg.isdigit(): - return int(arg) * 1024 +def sizetype(default, size_in_bytes=False): + def f(arg): + """ + Custom type for ArgumentParser + Converts size string in <num>[S|s|K|k|M|G] format into the integer value + """ + try: + suffix = default + size = int(arg) + except ValueError: + try: + suffix = arg[-1:] + size = int(arg[:-1]) + except ValueError: + raise ArgumentTypeError("Invalid size: %r" % arg) + + + if size_in_bytes: + if suffix == 's' or suffix == 'S': + return size * 512 + mult = 1024 + else: + mult = 1 + + if suffix == "k" or suffix == "K": + return size * mult + if suffix == "M": + return size * mult * 1024 + if suffix == "G": + return size * mult * 1024 * 1024 - if not arg[:-1].isdigit(): raise ArgumentTypeError("Invalid size: %r" % arg) - - size = int(arg[:-1]) - if arg.endswith("k") or arg.endswith("K"): - return size - if arg.endswith("M"): - return size * 1024 - if arg.endswith("G"): - return size * 1024 * 1024 - - raise ArgumentTypeError("Invalid size: %r" % arg) + return f def overheadtype(arg): """ @@ -136,10 +149,11 @@ class KickStart(): part.add_argument('mountpoint', nargs='?') part.add_argument('--active', action='store_true') part.add_argument('--align', type=int) + part.add_argument('--offset', type=sizetype("K", True)) part.add_argument('--exclude-path', nargs='+') part.add_argument('--include-path', nargs='+', action='append') part.add_argument('--change-directory') - part.add_argument("--extra-space", type=sizetype) + part.add_argument("--extra-space", type=sizetype("M")) part.add_argument('--fsoptions', dest='fsopts') part.add_argument('--fstype', default='vfat', choices=('ext2', 'ext3', 'ext4', 'btrfs', @@ -161,8 +175,8 @@ class KickStart(): # --error, but since nesting mutually exclusive groups does not work, # ----extra-space/--overhead-factor are handled later sizeexcl = part.add_mutually_exclusive_group() - sizeexcl.add_argument('--size', type=sizetype, default=0) - sizeexcl.add_argument('--fixed-size', type=sizetype, default=0) + sizeexcl.add_argument('--size', type=sizetype("M"), default=0) + sizeexcl.add_argument('--fixed-size', type=sizetype("M"), default=0) part.add_argument('--source') part.add_argument('--sourceparams') diff --git a/poky/scripts/lib/wic/misc.py b/poky/scripts/lib/wic/misc.py index 91975ba15..4b08d649c 100644 --- a/poky/scripts/lib/wic/misc.py +++ b/poky/scripts/lib/wic/misc.py @@ -138,8 +138,9 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): if pseudo: cmd_and_args = pseudo + cmd_and_args - native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ - (native_sysroot, native_sysroot, native_sysroot) + native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin" % \ + (native_sysroot, native_sysroot, + native_sysroot, native_sysroot) native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ (native_paths, cmd_and_args) diff --git a/poky/scripts/lib/wic/partition.py b/poky/scripts/lib/wic/partition.py index 7d9dd616a..85eb15c8b 100644 --- a/poky/scripts/lib/wic/partition.py +++ b/poky/scripts/lib/wic/partition.py @@ -40,6 +40,7 @@ class Partition(): self.mountpoint = args.mountpoint self.no_table = args.no_table self.num = None + self.offset = args.offset self.overhead_factor = args.overhead_factor self.part_name = args.part_name self.part_type = args.part_type diff --git a/poky/scripts/lib/wic/plugins/imager/direct.py b/poky/scripts/lib/wic/plugins/imager/direct.py index 2d06c242b..55db826e9 100644 --- a/poky/scripts/lib/wic/plugins/imager/direct.py +++ b/poky/scripts/lib/wic/plugins/imager/direct.py @@ -428,6 +428,21 @@ class PartitionedImage(): # increase the offset so we actually start the partition on right alignment self.offset += align_sectors + if part.offset is not None: + offset = part.offset // self.sector_size + + if offset * self.sector_size != part.offset: + raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) + + delta = offset - self.offset + if delta < 0: + raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) + + logger.debug("Skipping %d sectors to place %s%s at offset %dK", + delta, part.disk, self.numpart, part.offset) + + self.offset = offset + part.start = self.offset self.offset += part.size_sec diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py index 2cfdc10ec..cdc72543c 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -13,6 +13,9 @@ import logging import os import shutil +import re + +from glob import glob from wic import WicError from wic.engine import get_custom_config @@ -209,6 +212,57 @@ class BootimgEFIPlugin(SourcePlugin): except KeyError: raise WicError("bootimg-efi requires a loader, none specified") + if get_bitbake_var("IMAGE_EFI_BOOT_FILES") is None: + logger.debug('No boot files defined in IMAGE_EFI_BOOT_FILES') + else: + boot_files = None + for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)): + if fmt: + var = fmt % id + else: + var = "" + + boot_files = get_bitbake_var("IMAGE_EFI_BOOT_FILES" + var) + if boot_files: + break + + logger.debug('Boot files: %s', boot_files) + + # list of tuples (src_name, dst_name) + deploy_files = [] + for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files): + if ';' in src_entry: + dst_entry = tuple(src_entry.split(';')) + if not dst_entry[0] or not dst_entry[1]: + raise WicError('Malformed boot file entry: %s' % src_entry) + else: + dst_entry = (src_entry, src_entry) + + logger.debug('Destination entry: %r', dst_entry) + deploy_files.append(dst_entry) + + cls.install_task = []; + for deploy_entry in deploy_files: + src, dst = deploy_entry + if '*' in src: + # by default install files under their basename + entry_name_fn = os.path.basename + if dst != src: + # unless a target name was given, then treat name + # as a directory and append a basename + entry_name_fn = lambda name: \ + os.path.join(dst, + os.path.basename(name)) + + srcs = glob(os.path.join(kernel_dir, src)) + + logger.debug('Globbed sources: %s', ', '.join(srcs)) + for entry in srcs: + src = os.path.relpath(entry, kernel_dir) + entry_dst_name = entry_name_fn(entry) + cls.install_task.append((src, entry_dst_name)) + else: + cls.install_task.append((src, dst)) @classmethod def do_prepare_partition(cls, part, source_params, creator, cr_workdir, @@ -238,6 +292,12 @@ class BootimgEFIPlugin(SourcePlugin): (staging_kernel_dir, kernel, hdddir, kernel) exec_cmd(install_cmd) + if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): + for src_path, dst_path in cls.install_task: + install_cmd = "install -m 0644 -D %s %s" \ + % (os.path.join(kernel_dir, src_path), + os.path.join(hdddir, dst_path)) + exec_cmd(install_cmd) try: if source_params['loader'] == 'grub-efi': diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py index 138986a71..5dbe2558d 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-partition.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-partition.py @@ -141,7 +141,7 @@ class BootimgPartitionPlugin(SourcePlugin): break if not kernel_name: - raise WicError('No kernel file founded') + raise WicError('No kernel file found') # Compose the extlinux.conf extlinux_conf = "default Yocto\n" diff --git a/poky/scripts/oe-build-perf-report b/poky/scripts/oe-build-perf-report index e781f4f03..7ed86a72f 100755 --- a/poky/scripts/oe-build-perf-report +++ b/poky/scripts/oe-build-perf-report @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # Examine build performance test results # diff --git a/poky/scripts/oe-publish-sdk b/poky/scripts/oe-publish-sdk index 4b70f436b..deb8ae180 100755 --- a/poky/scripts/oe-publish-sdk +++ b/poky/scripts/oe-publish-sdk @@ -94,7 +94,10 @@ def publish(args): logger.error('Failed to unpack %s to %s' % (dest_sdk, destination)) return ret else: - cmd = "ssh %s 'sh %s -p -y -d %s && rm -f %s'" % (host, dest_sdk, destdir, dest_sdk) + rm_or_not = " && rm -f %s" % dest_sdk + if args.keep_orig: + rm_or_not = "" + cmd = "ssh %s 'sh %s -p -y -d %s%s'" % (host, dest_sdk, destdir, rm_or_not) ret = subprocess.call(cmd, shell=True) if ret == 0: logger.info('Successfully unpacked %s to %s' % (dest_sdk, destdir)) @@ -106,7 +109,7 @@ def publish(args): if not is_remote: cmd = 'set -e; mkdir -p %s/layers; cd %s/layers; if [ ! -e .git ]; then git init .; cp .git/hooks/post-update.sample .git/hooks/post-commit; echo "*.pyc\n*.pyo\npyshtables.py" > .gitignore; fi; git add -A .; git config user.email "oe@oe.oe" && git config user.name "OE" && git commit -q -m "init repo" || true' % (destination, destination) else: - cmd = "ssh %s 'set -e; mkdir -p %s/layers; cd %s/layers; if [ ! -e .git ]; then git init .; cp .git/hooks/post-update.sample .git/hooks/post-commit; echo '*.pyc\n*.pyo\npyshtables.py' > .gitignore; fi; git add -A .; git config user.email 'oe@oe.oe' && git config user.name 'OE' && git commit -q -m \"init repo\" || true'" % (host, destdir, destdir) + cmd = "ssh %s 'set -e; mkdir -p %s/layers; cd %s/layers; if [ ! -e .git ]; then git init .; cp .git/hooks/post-update.sample .git/hooks/post-commit; echo '*.pyc' > .gitignore; echo '*.pyo' >> .gitignore; echo 'pyshtables.py' >> .gitignore; fi; git add -A .; git config user.email 'oe@oe.oe' && git config user.name 'OE' && git commit -q -m \"init repo\" || true'" % (host, destdir, destdir) ret = subprocess.call(cmd, shell=True) if ret == 0: logger.info('SDK published successfully') @@ -119,6 +122,7 @@ def main(): parser = argparse_oe.ArgumentParser(description="OpenEmbedded extensible SDK publishing tool - writes server-side data to support the extensible SDK update process to a specified location") parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') + parser.add_argument('-k', '--keep-orig', help='When published to a remote host, the eSDK installer gets deleted by default.', action='store_true') parser.add_argument('sdk', help='Extensible SDK to publish (path to .sh installer file)') parser.add_argument('dest', help='Destination to publish SDK to; can be local path or remote in the form of user@host:/path (in the latter case ssh/scp will be used).') diff --git a/poky/scripts/runqemu b/poky/scripts/runqemu index 21680b49d..e5e66f345 100755 --- a/poky/scripts/runqemu +++ b/poky/scripts/runqemu @@ -456,32 +456,37 @@ class BaseConfig(object): if arg in self.fstypes + self.vmtypes + self.wictypes: self.check_arg_fstype(arg) elif arg == 'nographic': + if ('sdl' in sys.argv): + raise RunQemuError('Option nographic makes no sense alongside the sdl option.' % (arg)) + if ('gtk' in sys.argv): + raise RunQemuError('Option nographic makes no sense alongside the gtk option.' % (arg)) self.qemu_opt_script += ' -nographic' self.kernel_cmdline_script += ' console=ttyS0' elif arg == 'sdl': if 'gl' in sys.argv[1:]: self.set_dri_path() - self.qemu_opt_script += ' -vga virtio -display sdl,gl=on' + self.qemu_opt_script += ' -vga virtio -display sdl,gl=on,show-cursor=on' elif 'gl-es' in sys.argv[1:]: self.set_dri_path() - self.qemu_opt_script += ' -vga virtio -display sdl,gl=es' + self.qemu_opt_script += ' -vga virtio -display sdl,gl=es,show-cursor=on' else: - self.qemu_opt_script += ' -display sdl' + self.qemu_opt_script += ' -display sdl,show-cursor=on' elif arg == 'gtk': if 'gl' in sys.argv[1:]: self.set_dri_path() - self.qemu_opt_script += ' -vga virtio -display gtk,gl=on' + self.qemu_opt_script += ' -vga virtio -display gtk,gl=on,show-cursor=on' elif 'gl-es' in sys.argv[1:]: self.set_dri_path() - self.qemu_opt_script += ' -vga virtio -display gtk,gl=es' + self.qemu_opt_script += ' -vga virtio -display gtk,gl=es,show-cursor=on' else: - self.qemu_opt_script += ' -display gtk' + self.qemu_opt_script += ' -display gtk,show-cursor=on' elif arg == 'gl' or arg == 'gl-es': # These args are handled inside sdl or gtk blocks above - pass + if ('gtk' not in sys.argv) and ('sdl' not in sys.argv): + raise RunQemuError('Option %s also needs gtk or sdl option.' % (arg)) elif arg == 'egl-headless': self.set_dri_path() - self.qemu_opt_script += ' -vga virtio -display egl-headless' + self.qemu_opt_script += ' -vga virtio -display egl-headless,show-cursor=on' elif arg == 'serial': self.kernel_cmdline_script += ' console=ttyS0' self.serialconsole = True @@ -585,10 +590,10 @@ class BaseConfig(object): logger.error("For further help see:") raise RunQemuError(yocto_paravirt_kvm_wiki) - if not os.access(dev_kvm, os.W_OK|os.R_OK): + if not os.access(dev_vhost, os.W_OK|os.R_OK): logger.error("You have no read or write permission on /dev/vhost-net.") logger.error("Please change the ownership of this file as described at:") - raise RunQemuError(yocto_kvm_wiki) + raise RunQemuError(yocto_paravirt_kvm_wiki) def check_fstype(self): """Check and setup FSTYPE""" @@ -1241,7 +1246,9 @@ class BaseConfig(object): vm_drive = '-drive if=virtio,file=%s,format=%s' % (self.rootfs, rootfs_format) # All branches above set vm_drive. - self.rootfs_options = '%s -no-reboot' % vm_drive + self.rootfs_options = vm_drive + if not self.fstype in self.vmtypes: + self.rootfs_options += ' -no-reboot' self.kernel_cmdline = 'root=%s rw' % (self.get('QB_KERNEL_ROOT')) if self.fstype == 'nfs': @@ -1329,7 +1336,7 @@ class BaseConfig(object): if not os.access(qemu_bin, os.X_OK): raise OEPathError("No QEMU binary '%s' could be found" % qemu_bin) - self.qemu_opt = "%s %s %s %s" % (qemu_bin, self.get('NETWORK_CMD'), self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND')) + self.qemu_opt = "%s %s %s %s %s" % (qemu_bin, self.get('NETWORK_CMD'), self.get('QB_RNG'), self.get('ROOTFS_OPTIONS'), self.get('QB_OPT_APPEND')) for ovmf in self.ovmf_bios: format = ovmf.rsplit('.', 1)[-1] @@ -1512,6 +1519,11 @@ def main(): try: config = BaseConfig() + renice = os.path.expanduser("~/bin/runqemu-renice") + if os.path.exists(renice): + logger.info('Using %s to renice' % renice) + subprocess.check_call([renice, str(os.getpid())]) + def sigterm_handler(signum, frame): logger.info("SIGTERM received") os.kill(config.qemupid, signal.SIGTERM) diff --git a/poky/scripts/yocto-check-layer b/poky/scripts/yocto-check-layer index ca6c79bc8..b7c83c8b5 100755 --- a/poky/scripts/yocto-check-layer +++ b/poky/scripts/yocto-check-layer @@ -24,7 +24,7 @@ import scriptpath scriptpath.add_oe_lib_path() scriptpath.add_bitbake_lib_path() -from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_signatures +from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_signatures, check_bblayers from oeqa.utils.commands import get_bb_vars PROGNAME = 'yocto-check-layer' @@ -138,6 +138,13 @@ def main(): layer['type'] == LayerType.ERROR_BSP_DISTRO: continue + if check_bblayers(bblayersconf, layer['path'], logger): + logger.info("%s already in %s. To capture initial signatures, layer under test should not present " + "in BBLAYERS. Please remove %s from BBLAYERS." % (layer['name'], bblayersconf, layer['name'])) + results[layer['name']] = None + results_status[layer['name']] = 'SKIPPED (Layer under test should not present in BBLAYERS)' + continue + logger.info('') logger.info("Setting up for %s(%s), %s" % (layer['name'], layer['type'], layer['path'])) |