diff options
Diffstat (limited to 'import-layers/yocto-poky/scripts/lib/compatlayer/__init__.py')
| -rw-r--r-- | import-layers/yocto-poky/scripts/lib/compatlayer/__init__.py | 392 |
1 files changed, 0 insertions, 392 deletions
diff --git a/import-layers/yocto-poky/scripts/lib/compatlayer/__init__.py b/import-layers/yocto-poky/scripts/lib/compatlayer/__init__.py deleted file mode 100644 index 7197e850e4..0000000000 --- a/import-layers/yocto-poky/scripts/lib/compatlayer/__init__.py +++ /dev/null @@ -1,392 +0,0 @@ -# Yocto Project compatibility layer tool -# -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -import os -import re -import subprocess -from enum import Enum - -import bb.tinfoil - -class LayerType(Enum): - BSP = 0 - DISTRO = 1 - SOFTWARE = 2 - ERROR_NO_LAYER_CONF = 98 - ERROR_BSP_DISTRO = 99 - -def _get_configurations(path): - configs = [] - - for f in os.listdir(path): - file_path = os.path.join(path, f) - if os.path.isfile(file_path) and f.endswith('.conf'): - configs.append(f[:-5]) # strip .conf - return configs - -def _get_layer_collections(layer_path, lconf=None, data=None): - import bb.parse - import bb.data - - if lconf is None: - lconf = os.path.join(layer_path, 'conf', 'layer.conf') - - if data is None: - ldata = bb.data.init() - bb.parse.init_parser(ldata) - else: - ldata = data.createCopy() - - ldata.setVar('LAYERDIR', layer_path) - try: - ldata = bb.parse.handle(lconf, ldata, include=True) - except BaseException as exc: - raise LayerError(exc) - ldata.expandVarref('LAYERDIR') - - collections = (ldata.getVar('BBFILE_COLLECTIONS', True) or '').split() - if not collections: - name = os.path.basename(layer_path) - collections = [name] - - collections = {c: {} for c in collections} - for name in collections: - priority = ldata.getVar('BBFILE_PRIORITY_%s' % name, True) - pattern = ldata.getVar('BBFILE_PATTERN_%s' % name, True) - depends = ldata.getVar('LAYERDEPENDS_%s' % name, True) - collections[name]['priority'] = priority - collections[name]['pattern'] = pattern - collections[name]['depends'] = depends - - return collections - -def _detect_layer(layer_path): - """ - Scans layer directory to detect what type of layer - is BSP, Distro or Software. - - Returns a dictionary with layer name, type and path. - """ - - layer = {} - layer_name = os.path.basename(layer_path) - - layer['name'] = layer_name - layer['path'] = layer_path - layer['conf'] = {} - - if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')): - layer['type'] = LayerType.ERROR_NO_LAYER_CONF - return layer - - machine_conf = os.path.join(layer_path, 'conf', 'machine') - distro_conf = os.path.join(layer_path, 'conf', 'distro') - - is_bsp = False - is_distro = False - - if os.path.isdir(machine_conf): - machines = _get_configurations(machine_conf) - if machines: - is_bsp = True - - if os.path.isdir(distro_conf): - distros = _get_configurations(distro_conf) - if distros: - is_distro = True - - if is_bsp and is_distro: - layer['type'] = LayerType.ERROR_BSP_DISTRO - elif is_bsp: - layer['type'] = LayerType.BSP - layer['conf']['machines'] = machines - elif is_distro: - layer['type'] = LayerType.DISTRO - layer['conf']['distros'] = distros - else: - layer['type'] = LayerType.SOFTWARE - - layer['collections'] = _get_layer_collections(layer['path']) - - return layer - -def detect_layers(layer_directories, no_auto): - layers = [] - - for directory in layer_directories: - directory = os.path.realpath(directory) - if directory[-1] == '/': - directory = directory[0:-1] - - if no_auto: - conf_dir = os.path.join(directory, 'conf') - if os.path.isdir(conf_dir): - layer = _detect_layer(directory) - if layer: - layers.append(layer) - else: - for root, dirs, files in os.walk(directory): - dir_name = os.path.basename(root) - conf_dir = os.path.join(root, 'conf') - if os.path.isdir(conf_dir): - layer = _detect_layer(root) - if layer: - layers.append(layer) - - return layers - -def _find_layer_depends(depend, layers): - for layer in layers: - for collection in layer['collections']: - if depend == collection: - return layer - return None - -def add_layer_dependencies(bblayersconf, layer, layers, logger): - def recurse_dependencies(depends, layer, layers, logger, ret = []): - logger.debug('Processing dependencies %s for layer %s.' % \ - (depends, layer['name'])) - - for depend in depends.split(): - # core (oe-core) is suppose to be provided - if depend == 'core': - continue - - layer_depend = _find_layer_depends(depend, layers) - if not layer_depend: - logger.error('Layer %s depends on %s and isn\'t found.' % \ - (layer['name'], depend)) - ret = None - continue - - # We keep processing, even if ret is None, this allows us to report - # multiple errors at once - if ret is not None and layer_depend not in ret: - ret.append(layer_depend) - - # Recursively process... - if 'collections' not in layer_depend: - continue - - for collection in layer_depend['collections']: - collect_deps = layer_depend['collections'][collection]['depends'] - if not collect_deps: - continue - ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret) - - return ret - - layer_depends = [] - for collection in layer['collections']: - depends = layer['collections'][collection]['depends'] - if not depends: - continue - - layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) - - # Note: [] (empty) is allowed, None is not! - if layer_depends is None: - return False - else: - # Don't add a layer that is already present. - added = set() - output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') - for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): - added.add(path) - - for layer_depend in layer_depends: - name = layer_depend['name'] - path = layer_depend['path'] - if path in added: - continue - else: - added.add(path) - logger.info('Adding layer dependency %s' % name) - with open(bblayersconf, 'a+') as f: - f.write("\nBBLAYERS += \"%s\"\n" % path) - return True - -def add_layer(bblayersconf, layer, layers, logger): - logger.info('Adding layer %s' % layer['name']) - with open(bblayersconf, 'a+') as f: - f.write("\nBBLAYERS += \"%s\"\n" % layer['path']) - - return True - -def check_command(error_msg, cmd): - ''' - Run a command under a shell, capture stdout and stderr in a single stream, - throw an error when command returns non-zero exit code. Returns the output. - ''' - - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output, _ = p.communicate() - if p.returncode: - msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8')) - raise RuntimeError(msg) - return output - -def get_signatures(builddir, failsafe=False, machine=None): - import re - - # some recipes needs to be excluded like meta-world-pkgdata - # because a layer can add recipes to a world build so signature - # will be change - exclude_recipes = ('meta-world-pkgdata',) - - sigs = {} - tune2tasks = {} - - cmd = '' - if machine: - cmd += 'MACHINE=%s ' % machine - cmd += 'bitbake ' - if failsafe: - cmd += '-k ' - cmd += '-S none world' - sigs_file = os.path.join(builddir, 'locked-sigs.inc') - if os.path.exists(sigs_file): - os.unlink(sigs_file) - try: - check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.', - cmd) - except RuntimeError as ex: - if failsafe and os.path.exists(sigs_file): - # Ignore the error here. Most likely some recipes active - # in a world build lack some dependencies. There is a - # separate test_machine_world_build which exposes the - # failure. - pass - else: - raise - - sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") - tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") - current_tune = None - with open(sigs_file, 'r') as f: - for line in f.readlines(): - line = line.strip() - t = tune_regex.search(line) - if t: - current_tune = t.group('tune') - s = sig_regex.match(line) - if s: - exclude = False - for er in exclude_recipes: - (recipe, task) = s.group('task').split(':') - if er == recipe: - exclude = True - break - if exclude: - continue - - sigs[s.group('task')] = s.group('hash') - tune2tasks.setdefault(current_tune, []).append(s.group('task')) - - if not sigs: - raise RuntimeError('Can\'t load signatures from %s' % sigs_file) - - return (sigs, tune2tasks) - -def get_depgraph(targets=['world'], failsafe=False): - ''' - Returns the dependency graph for the given target(s). - The dependency graph is taken directly from DepTreeEvent. - ''' - depgraph = None - with bb.tinfoil.Tinfoil() as tinfoil: - tinfoil.prepare(config_only=False) - tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted']) - if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'): - raise RuntimeError('starting generateDepTreeEvent failed') - while True: - event = tinfoil.wait_event(timeout=1000) - if event: - if isinstance(event, bb.command.CommandFailed): - raise RuntimeError('Generating dependency information failed: %s' % event.error) - elif isinstance(event, bb.command.CommandCompleted): - break - elif isinstance(event, bb.event.NoProvider): - if failsafe: - # The event is informational, we will get information about the - # remaining dependencies eventually and thus can ignore this - # here like we do in get_signatures(), if desired. - continue - if event._reasons: - raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons)) - else: - raise RuntimeError('Nothing provides %s.' % (event._item)) - elif isinstance(event, bb.event.DepTreeGenerated): - depgraph = event._depgraph - - if depgraph is None: - raise RuntimeError('Could not retrieve the depgraph.') - return depgraph - -def compare_signatures(old_sigs, curr_sigs): - ''' - Compares the result of two get_signatures() calls. Returns None if no - problems found, otherwise a string that can be used as additional - explanation in self.fail(). - ''' - # task -> (old signature, new signature) - sig_diff = {} - for task in old_sigs: - if task in curr_sigs and \ - old_sigs[task] != curr_sigs[task]: - sig_diff[task] = (old_sigs[task], curr_sigs[task]) - - if not sig_diff: - return None - - # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures() - # uses <pn>:<taskname>. Need to convert sometimes. The output follows - # the convention from get_signatures() because that seems closer to - # normal bitbake output. - def sig2graph(task): - pn, taskname = task.rsplit(':', 1) - return pn + '.' + taskname - def graph2sig(task): - pn, taskname = task.rsplit('.', 1) - return pn + ':' + taskname - depgraph = get_depgraph(failsafe=True) - depends = depgraph['tdepends'] - - # If a task A has a changed signature, but none of its - # dependencies, then we need to report it because it is - # the one which introduces a change. Any task depending on - # A (directly or indirectly) will also have a changed - # signature, but we don't need to report it. It might have - # its own changes, which will become apparent once the - # issues that we do report are fixed and the test gets run - # again. - sig_diff_filtered = [] - for task, (old_sig, new_sig) in sig_diff.items(): - deps_tainted = False - for dep in depends.get(sig2graph(task), ()): - if graph2sig(dep) in sig_diff: - deps_tainted = True - break - if not deps_tainted: - sig_diff_filtered.append((task, old_sig, new_sig)) - - msg = [] - msg.append('%d signatures changed, initial differences (first hash before, second after):' % - len(sig_diff)) - for diff in sorted(sig_diff_filtered): - recipe, taskname = diff[0].rsplit(':', 1) - cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \ - (recipe, taskname, diff[1], diff[2]) - msg.append(' %s: %s -> %s' % diff) - msg.append(' %s' % cmd) - try: - output = check_command('Determining signature difference failed.', - cmd).decode('utf-8') - except RuntimeError as error: - output = str(error) - if output: - msg.extend([' ' + line for line in output.splitlines()]) - msg.append('') - return '\n'.join(msg) |