diff options
Diffstat (limited to 'yocto-poky/meta/lib')
55 files changed, 2853 insertions, 1316 deletions
diff --git a/yocto-poky/meta/lib/oe/copy_buildsystem.py b/yocto-poky/meta/lib/oe/copy_buildsystem.py index c0e7541c0..7b9a0ee06 100644 --- a/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -8,7 +8,7 @@ def _smart_copy(src, dest): # source is a file or a directory. mode = os.stat(src).st_mode if stat.S_ISDIR(mode): - shutil.copytree(src, dest, symlinks=True) + shutil.copytree(src, dest, symlinks=True, ignore=shutil.ignore_patterns('.git')) else: shutil.copyfile(src, dest) shutil.copymode(src, dest) @@ -18,8 +18,9 @@ class BuildSystem(object): self.d = d self.context = context self.layerdirs = d.getVar('BBLAYERS', True).split() + self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() - def copy_bitbake_and_layers(self, destdir): + def copy_bitbake_and_layers(self, destdir, workspace_name=None): # Copy in all metadata layers + bitbake (as repositories) layers_copied = [] bb.utils.mkdirhier(destdir) @@ -28,6 +29,19 @@ class BuildSystem(object): corebase = self.d.getVar('COREBASE', True) layers.append(corebase) + # Exclude layers + for layer_exclude in self.layers_exclude: + if layer_exclude in layers: + layers.remove(layer_exclude) + + workspace_newname = workspace_name + if workspace_newname: + layernames = [os.path.basename(layer) for layer in layers] + extranum = 0 + while workspace_newname in layernames: + extranum += 1 + workspace_newname = '%s-%d' % (workspace_name, extranum) + corebase_files = self.d.getVar('COREBASE_FILES', True).split() corebase_files = [corebase + '/' +x for x in corebase_files] # Make sure bitbake goes in @@ -36,18 +50,24 @@ class BuildSystem(object): for layer in layers: layerconf = os.path.join(layer, 'conf', 'layer.conf') + layernewname = os.path.basename(layer) + workspace = False if os.path.exists(layerconf): with open(layerconf, 'r') as f: if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): - bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) - continue + if workspace_newname: + layernewname = workspace_newname + workspace = True + else: + bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) + continue # If the layer was already under corebase, leave it there # since layers such as meta have issues when moved. layerdestpath = destdir if corebase == os.path.dirname(layer): layerdestpath += '/' + os.path.basename(corebase) - layerdestpath += '/' + os.path.basename(layer) + layerdestpath += '/' + layernewname layer_relative = os.path.relpath(layerdestpath, destdir) @@ -67,15 +87,47 @@ class BuildSystem(object): else: _smart_copy(layer, layerdestpath) + if workspace: + # Make some adjustments original workspace layer + # Drop sources (recipe tasks will be locked, so we don't need them) + srcdir = os.path.join(layerdestpath, 'sources') + if os.path.isdir(srcdir): + shutil.rmtree(srcdir) + # Drop all bbappends except the one for the image the SDK is being built for + # (because of externalsrc, the workspace bbappends will interfere with the + # locked signatures if present, and we don't need them anyway) + image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE', True)))[0] + '.bbappend' + appenddir = os.path.join(layerdestpath, 'appends') + if os.path.isdir(appenddir): + for fn in os.listdir(appenddir): + if fn == image_bbappend: + continue + else: + os.remove(os.path.join(appenddir, fn)) + # Drop README + readme = os.path.join(layerdestpath, 'README') + if os.path.exists(readme): + os.remove(readme) + # Filter out comments in layer.conf and change layer name + layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') + with open(layerconf, 'r') as f: + origlines = f.readlines() + with open(layerconf, 'w') as f: + for line in origlines: + if line.startswith('#'): + continue + line = line.replace('workspacelayer', workspace_newname) + f.write(line) + return layers_copied def generate_locked_sigs(sigfile, d): bb.utils.mkdirhier(os.path.dirname(sigfile)) - depd = d.getVar('BB_TASKDEPDATA', True) + depd = d.getVar('BB_TASKDEPDATA', False) tasks = ['%s.%s' % (v[2], v[1]) for v in depd.itervalues()] bb.parse.siggen.dump_lockedsigs(sigfile, tasks) -def prune_lockedsigs(allowed_tasks, excluded_targets, lockedsigs, pruned_output): +def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): with open(lockedsigs, 'r') as infile: bb.utils.mkdirhier(os.path.dirname(pruned_output)) with open(pruned_output, 'w') as f: @@ -84,7 +136,7 @@ def prune_lockedsigs(allowed_tasks, excluded_targets, lockedsigs, pruned_output) if invalue: if line.endswith('\\\n'): splitval = line.strip().split(':') - if splitval[1] in allowed_tasks and not splitval[0] in excluded_targets: + if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: f.write(line) else: f.write(line) @@ -93,10 +145,73 @@ def prune_lockedsigs(allowed_tasks, excluded_targets, lockedsigs, pruned_output) invalue = True f.write(line) +def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output): + merged = {} + arch_order = [] + with open(lockedsigs_main, 'r') as f: + invalue = None + for line in f: + if invalue: + if line.endswith('\\\n'): + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + merged[invalue] = [] + arch_order.append(invalue) + + with open(lockedsigs_extra, 'r') as f: + invalue = None + tocopy = {} + for line in f: + if invalue: + if line.endswith('\\\n'): + if not line in merged[invalue]: + target, task = line.strip().split(':')[:2] + if not copy_tasks or task in copy_tasks: + tocopy[invalue].append(line) + merged[invalue].append(line) + else: + invalue = None + elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): + invalue = line[18:].split('=', 1)[0].rstrip() + if not invalue in merged: + merged[invalue] = [] + arch_order.append(invalue) + tocopy[invalue] = [] + + def write_sigs_file(fn, types, sigs): + fulltypes = [] + bb.utils.mkdirhier(os.path.dirname(fn)) + with open(fn, 'w') as f: + for typename in types: + lines = sigs[typename] + if lines: + f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) + for line in lines: + f.write(line) + f.write(' "\n') + fulltypes.append(typename) + f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) + + write_sigs_file(copy_output, tocopy.keys(), tocopy) + if merged_output: + write_sigs_file(merged_output, arch_order, merged) + def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): bb.note('Generating sstate-cache...') - bb.process.run("gen-lockedsig-cache %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache)) + nativelsbstring = d.getVar('NATIVELSBSTRING', True) + bb.process.run("gen-lockedsig-cache %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring)) if fixedlsbstring: - os.rename(output_sstate_cache + '/' + d.getVar('NATIVELSBSTRING', True), - output_sstate_cache + '/' + fixedlsbstring) + nativedir = output_sstate_cache + '/' + nativelsbstring + if os.path.isdir(nativedir): + destdir = os.path.join(output_sstate_cache, fixedlsbstring) + bb.utils.mkdirhier(destdir) + + dirlist = os.listdir(nativedir) + for i in dirlist: + src = os.path.join(nativedir, i) + dest = os.path.join(destdir, i) + os.rename(src, dest) diff --git a/yocto-poky/meta/lib/oe/data.py b/yocto-poky/meta/lib/oe/data.py index 4cc0e0296..e49572177 100644 --- a/yocto-poky/meta/lib/oe/data.py +++ b/yocto-poky/meta/lib/oe/data.py @@ -3,7 +3,7 @@ import oe.maketype def typed_value(key, d): """Construct a value for the specified metadata variable, using its flags to determine the type and parameters for construction.""" - var_type = d.getVarFlag(key, 'type') + var_type = d.getVarFlag(key, 'type', True) flags = d.getVarFlags(key) if flags is not None: flags = dict((flag, d.expand(value)) diff --git a/yocto-poky/meta/lib/oe/distro_check.py b/yocto-poky/meta/lib/oe/distro_check.py index f92cd2e42..8655a6fc1 100644 --- a/yocto-poky/meta/lib/oe/distro_check.py +++ b/yocto-poky/meta/lib/oe/distro_check.py @@ -9,10 +9,12 @@ def create_socket(url, d): socket.close() def get_proxies(d): - import os - proxykeys = ['http', 'https', 'ftp', 'ftps', 'no', 'all'] - proxyvalues = map(lambda key: d.getVar(key+'_proxy', True), proxykeys) - return dict(zip(proxykeys, proxyvalues)) + proxies = {} + for key in ['http', 'https', 'ftp', 'ftps', 'no', 'all']: + proxy = d.getVar(key + '_proxy', True) + if proxy: + proxies[key] = proxy + return proxies def get_links_from_url(url, d): "Return all the href links found on the web location" diff --git a/yocto-poky/meta/lib/oe/gpg_sign.py b/yocto-poky/meta/lib/oe/gpg_sign.py new file mode 100644 index 000000000..b83ee8672 --- /dev/null +++ b/yocto-poky/meta/lib/oe/gpg_sign.py @@ -0,0 +1,116 @@ +"""Helper module for GPG signing""" +import os + +import bb +import oe.utils + +class LocalSigner(object): + """Class for handling local (on the build host) signing""" + def __init__(self, d): + self.gpg_bin = d.getVar('GPG_BIN', True) or \ + bb.utils.which(os.getenv('PATH'), 'gpg') + self.gpg_path = d.getVar('GPG_PATH', True) + self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpm") + + def export_pubkey(self, output_file, keyid, armor=True): + """Export GPG public key to a file""" + cmd = '%s --batch --yes --export -o %s ' % \ + (self.gpg_bin, output_file) + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + if armor: + cmd += "--armor " + cmd += keyid + status, output = oe.utils.getstatusoutput(cmd) + if status: + raise bb.build.FuncFailed('Failed to export gpg public key (%s): %s' % + (keyid, output)) + + def sign_rpms(self, files, keyid, passphrase): + """Sign RPM files""" + + cmd = self.rpm_bin + " --addsign --define '_gpg_name %s' " % keyid + cmd += "--define '_gpg_passphrase %s' " % passphrase + if self.gpg_bin: + cmd += "--define '%%__gpg %s' " % self.gpg_bin + if self.gpg_path: + cmd += "--define '_gpg_path %s' " % self.gpg_path + cmd += ' '.join(files) + + status, output = oe.utils.getstatusoutput(cmd) + if status: + raise bb.build.FuncFailed("Failed to sign RPM packages: %s" % output) + + def detach_sign(self, input_file, keyid, passphrase_file, passphrase=None, armor=True): + """Create a detached signature of a file""" + import subprocess + + if passphrase_file and passphrase: + raise Exception("You should use either passphrase_file of passphrase, not both") + + cmd = [self.gpg_bin, '--detach-sign', '--batch', '--no-tty', '--yes', + '--passphrase-fd', '0', '-u', keyid] + + if self.gpg_path: + cmd += ['--homedir', self.gpg_path] + if armor: + cmd += ['--armor'] + + #gpg > 2.1 supports password pipes only through the loopback interface + #gpg < 2.1 errors out if given unknown parameters + dots = self.get_gpg_version().split('.') + assert len(dots) >= 2 + if int(dots[0]) >= 2 and int(dots[1]) >= 1: + cmd += ['--pinentry-mode', 'loopback'] + + cmd += [input_file] + + try: + if passphrase_file: + with open(passphrase_file) as fobj: + passphrase = fobj.readline(); + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase) + + if job.returncode: + raise bb.build.FuncFailed("GPG exited with code %d: %s" % + (job.returncode, stderr)) + + except IOError as e: + bb.error("IO error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s'" % input_file) + + except OSError as e: + bb.error("OS error (%s): %s" % (e.errno, e.strerror)) + raise Exception("Failed to sign '%s" % input_file) + + + def get_gpg_version(self): + """Return the gpg version""" + import subprocess + try: + return subprocess.check_output((self.gpg_bin, "--version")).split()[2] + except subprocess.CalledProcessError as e: + raise bb.build.FuncFailed("Could not get gpg version: %s" % e) + + + def verify(self, sig_file): + """Verify signature""" + cmd = self.gpg_bin + " --verify " + if self.gpg_path: + cmd += "--homedir %s " % self.gpg_path + cmd += sig_file + status, _ = oe.utils.getstatusoutput(cmd) + ret = False if status else True + return ret + + +def get_signer(d, backend): + """Get signer object for the specified backend""" + # Use local signing by default + if backend == 'local': + return LocalSigner(d) + else: + bb.fatal("Unsupported signing backend '%s'" % backend) + diff --git a/yocto-poky/meta/lib/oe/image.py b/yocto-poky/meta/lib/oe/image.py deleted file mode 100644 index b9eb3de5a..000000000 --- a/yocto-poky/meta/lib/oe/image.py +++ /dev/null @@ -1,418 +0,0 @@ -from oe.utils import execute_pre_post_process -import os -import subprocess -import multiprocessing - - -def generate_image(arg): - (type, subimages, create_img_cmd, sprefix) = arg - - bb.note("Running image creation script for %s: %s ..." % - (type, create_img_cmd)) - - try: - output = subprocess.check_output(create_img_cmd, - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - return("Error: The image creation script '%s' returned %d:\n%s" % - (e.cmd, e.returncode, e.output)) - - bb.note("Script output:\n%s" % output) - - return None - - -""" -This class will help compute IMAGE_FSTYPE dependencies and group them in batches -that can be executed in parallel. - -The next example is for illustration purposes, highly unlikely to happen in real life. -It's just one of the test cases I used to test the algorithm: - -For: -IMAGE_FSTYPES = "i1 i2 i3 i4 i5" -IMAGE_TYPEDEP_i4 = "i2" -IMAGE_TYPEDEP_i5 = "i6 i4" -IMAGE_TYPEDEP_i6 = "i7" -IMAGE_TYPEDEP_i7 = "i2" - -We get the following list of batches that can be executed in parallel, having the -dependencies satisfied: - -[['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] -""" -class ImageDepGraph(object): - def __init__(self, d): - self.d = d - self.graph = dict() - self.deps_array = dict() - - def _construct_dep_graph(self, image_fstypes): - graph = dict() - - def add_node(node): - base_type = self._image_base_type(node) - deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") - base_deps = (self.d.getVar('IMAGE_TYPEDEP_' + base_type, True) or "") - - graph[node] = "" - for dep in deps.split() + base_deps.split(): - if not dep in graph[node]: - if graph[node] != "": - graph[node] += " " - graph[node] += dep - - if not dep in graph: - add_node(dep) - - for fstype in image_fstypes: - add_node(fstype) - - return graph - - def _clean_graph(self): - # Live and VMDK/VDI images will be processed via inheriting - # bbclass and does not get processed here. Remove them from the fstypes - # graph. Their dependencies are already added, so no worries here. - remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split() - - for item in remove_list: - self.graph.pop(item, None) - - def _image_base_type(self, type): - ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() - if type in ["vmdk", "vdi", "qcow2", "live", "iso", "hddimg"]: - type = "ext4" - basetype = type - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - break - - return basetype - - def _compute_dependencies(self): - """ - returns dict object of nodes with [no_of_depends_on, no_of_depended_by] - for each node - """ - deps_array = dict() - for node in self.graph: - deps_array[node] = [0, 0] - - for node in self.graph: - deps = self.graph[node].split() - deps_array[node][0] += len(deps) - for dep in deps: - deps_array[dep][1] += 1 - - return deps_array - - def _sort_graph(self): - sorted_list = [] - group = [] - for node in self.graph: - if node not in self.deps_array: - continue - - depends_on = self.deps_array[node][0] - - if depends_on == 0: - group.append(node) - - if len(group) == 0 and len(self.deps_array) != 0: - bb.fatal("possible fstype circular dependency...") - - sorted_list.append(group) - - # remove added nodes from deps_array - for item in group: - for node in self.graph: - if item in self.graph[node].split(): - self.deps_array[node][0] -= 1 - - self.deps_array.pop(item, None) - - if len(self.deps_array): - # recursive call, to find the next group - sorted_list += self._sort_graph() - - return sorted_list - - def group_fstypes(self, image_fstypes): - self.graph = self._construct_dep_graph(image_fstypes) - - self._clean_graph() - - self.deps_array = self._compute_dependencies() - - alltypes = [node for node in self.graph] - - return (alltypes, self._sort_graph()) - - -class Image(ImageDepGraph): - def __init__(self, d): - self.d = d - - super(Image, self).__init__(d) - - def _get_rootfs_size(self): - """compute the rootfs size""" - rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) - overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True)) - rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True)) - rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) - rootfs_maxsize = self.d.getVar('IMAGE_ROOTFS_MAXSIZE', True) - - output = subprocess.check_output(['du', '-ks', - self.d.getVar('IMAGE_ROOTFS', True)]) - size_kb = int(output.split()[0]) - base_size = size_kb * overhead_factor - base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ - rootfs_extra_space - - if base_size != int(base_size): - base_size = int(base_size + 1) - else: - base_size = int(base_size) - - base_size += rootfs_alignment - 1 - base_size -= base_size % rootfs_alignment - - # Check the rootfs size against IMAGE_ROOTFS_MAXSIZE (if set) - if rootfs_maxsize: - rootfs_maxsize_int = int(rootfs_maxsize) - if base_size > rootfs_maxsize_int: - bb.fatal("The rootfs size %d(K) overrides the max size %d(K)" % \ - (base_size, rootfs_maxsize_int)) - - return base_size - - def _create_symlinks(self, subimages): - """create symlinks to the newly created image""" - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - img_name = self.d.getVar('IMAGE_NAME', True) - link_name = self.d.getVar('IMAGE_LINK_NAME', True) - manifest_name = self.d.getVar('IMAGE_MANIFEST', True) - - os.chdir(deploy_dir) - - if link_name: - for type in subimages: - if os.path.exists(img_name + ".rootfs." + type): - dst = link_name + "." + type - src = img_name + ".rootfs." + type - bb.note("Creating symlink: %s -> %s" % (dst, src)) - os.symlink(src, dst) - - if manifest_name is not None and \ - os.path.exists(manifest_name) and \ - not os.path.exists(link_name + ".manifest"): - os.symlink(os.path.basename(manifest_name), - link_name + ".manifest") - - def _remove_old_symlinks(self): - """remove the symlinks to old binaries""" - - if self.d.getVar('IMAGE_LINK_NAME', True): - deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) - for img in os.listdir(deploy_dir): - if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0: - img = os.path.join(deploy_dir, img) - if os.path.islink(img): - if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \ - os.path.exists(os.path.realpath(img)): - os.remove(os.path.realpath(img)) - - os.remove(img) - - """ - This function will just filter out the compressed image types from the - fstype groups returning a (filtered_fstype_groups, cimages) tuple. - """ - def _filter_out_commpressed(self, fstype_groups): - ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() - cimages = {} - - filtered_groups = [] - for group in fstype_groups: - filtered_group = [] - for type in group: - basetype = None - for ctype in ctypes: - if type.endswith("." + ctype): - basetype = type[:-len("." + ctype)] - if basetype not in filtered_group: - filtered_group.append(basetype) - if basetype not in cimages: - cimages[basetype] = [] - if ctype not in cimages[basetype]: - cimages[basetype].append(ctype) - break - if not basetype and type not in filtered_group: - filtered_group.append(type) - - filtered_groups.append(filtered_group) - - return (filtered_groups, cimages) - - def _get_image_types(self): - """returns a (types, cimages) tuple""" - - alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split()) - - filtered_groups, cimages = self._filter_out_commpressed(fstype_groups) - - return (alltypes, filtered_groups, cimages) - - def _write_script(self, type, cmds, sprefix=""): - tempdir = self.d.getVar('T', True) - script_name = os.path.join(tempdir, sprefix + "create_image." + type) - rootfs_size = self._get_rootfs_size() - - self.d.setVar('img_creation_func', '\n'.join(cmds)) - self.d.setVarFlag('img_creation_func', 'func', 1) - self.d.setVarFlag('img_creation_func', 'fakeroot', 1) - self.d.setVar('ROOTFS_SIZE', str(rootfs_size)) - - with open(script_name, "w+") as script: - script.write("%s" % bb.build.shell_trap_code()) - script.write("export ROOTFS_SIZE=%d\n" % rootfs_size) - bb.data.emit_func('img_creation_func', script, self.d) - script.write("img_creation_func\n") - - os.chmod(script_name, 0775) - - return script_name - - def _get_imagecmds(self, sprefix=""): - old_overrides = self.d.getVar('OVERRIDES', 0) - - alltypes, fstype_groups, cimages = self._get_image_types() - - image_cmd_groups = [] - - bb.note("The image creation groups are: %s" % str(fstype_groups)) - for fstype_group in fstype_groups: - image_cmds = [] - for type in fstype_group: - cmds = [] - subimages = [] - - localdata = bb.data.createCopy(self.d) - localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) - bb.data.update_data(localdata) - localdata.setVar('type', type) - - image_cmd = localdata.getVar("IMAGE_CMD", True) - if image_cmd: - cmds.append("\t" + image_cmd) - else: - bb.fatal("No IMAGE_CMD defined for IMAGE_FSTYPES entry '%s' - possibly invalid type name or missing support class" % type) - cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) - - if type in cimages: - for ctype in cimages[type]: - cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) - subimages.append(type + "." + ctype) - - if type not in alltypes: - cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) - else: - subimages.append(type) - - script_name = self._write_script(type, cmds, sprefix) - - image_cmds.append((type, subimages, script_name, sprefix)) - - image_cmd_groups.append(image_cmds) - - return image_cmd_groups - - def _write_wic_env(self): - """ - Write environment variables used by wic - to tmp/sysroots/<machine>/imgdata/<image>.env - """ - stdir = self.d.getVar('STAGING_DIR_TARGET', True) - outdir = os.path.join(stdir, 'imgdata') - if not os.path.exists(outdir): - os.makedirs(outdir) - basename = self.d.getVar('IMAGE_BASENAME', True) - with open(os.path.join(outdir, basename) + '.env', 'w') as envf: - for var in self.d.getVar('WICVARS', True).split(): - value = self.d.getVar(var, True) - if value: - envf.write('%s="%s"\n' % (var, value.strip())) - - def create(self): - bb.note("###### Generate images #######") - pre_process_cmds = self.d.getVar("IMAGE_PREPROCESS_COMMAND", True) - post_process_cmds = self.d.getVar("IMAGE_POSTPROCESS_COMMAND", True) - - execute_pre_post_process(self.d, pre_process_cmds) - - self._remove_old_symlinks() - - image_cmd_groups = self._get_imagecmds() - - # Process the debug filesystem... - debugfs_d = bb.data.createCopy(self.d) - if self.d.getVar('IMAGE_GEN_DEBUGFS', True) == "1": - bb.note("Processing debugfs image(s) ...") - orig_d = self.d - self.d = debugfs_d - - self.d.setVar('IMAGE_ROOTFS', orig_d.getVar('IMAGE_ROOTFS', True) + '-dbg') - self.d.setVar('IMAGE_NAME', orig_d.getVar('IMAGE_NAME', True) + '-dbg') - self.d.setVar('IMAGE_LINK_NAME', orig_d.getVar('IMAGE_LINK_NAME', True) + '-dbg') - - debugfs_image_fstypes = orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True) - if debugfs_image_fstypes: - self.d.setVar('IMAGE_FSTYPES', orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True)) - - self._remove_old_symlinks() - - image_cmd_groups += self._get_imagecmds("debugfs.") - - self.d = orig_d - - self._write_wic_env() - - for image_cmds in image_cmd_groups: - # create the images in parallel - nproc = multiprocessing.cpu_count() - pool = bb.utils.multiprocessingpool(nproc) - results = list(pool.imap(generate_image, image_cmds)) - pool.close() - pool.join() - - for result in results: - if result is not None: - bb.fatal(result) - - for image_type, subimages, script, sprefix in image_cmds: - if sprefix == 'debugfs.': - bb.note("Creating symlinks for %s debugfs image ..." % image_type) - orig_d = self.d - self.d = debugfs_d - self._create_symlinks(subimages) - self.d = orig_d - else: - bb.note("Creating symlinks for %s image ..." % image_type) - self._create_symlinks(subimages) - - execute_pre_post_process(self.d, post_process_cmds) - - -def create_image(d): - Image(d).create() - -if __name__ == "__main__": - """ - Image creation can be called independent from bitbake environment. - """ - """ - TBD - """ diff --git a/yocto-poky/meta/lib/oe/lsb.py b/yocto-poky/meta/lib/oe/lsb.py index ddfe71b6b..e0bdfba25 100644 --- a/yocto-poky/meta/lib/oe/lsb.py +++ b/yocto-poky/meta/lib/oe/lsb.py @@ -62,6 +62,8 @@ def distro_identifier(adjust_hook=None): """Return a distro identifier string based upon lsb_release -ri, with optional adjustment via a hook""" + import re + lsb_data = release_dict() if lsb_data: distro_id, release = lsb_data['Distributor ID'], lsb_data['Release'] @@ -76,6 +78,9 @@ def distro_identifier(adjust_hook=None): distro_id, release = adjust_hook(distro_id, release) if not distro_id: return "Unknown" + # Filter out any non-alphanumerics + distro_id = re.sub(r'\W', '', distro_id) + if release: id_str = '{0}-{1}'.format(distro_id, release) else: diff --git a/yocto-poky/meta/lib/oe/package.py b/yocto-poky/meta/lib/oe/package.py index f176446b8..288768954 100644 --- a/yocto-poky/meta/lib/oe/package.py +++ b/yocto-poky/meta/lib/oe/package.py @@ -123,3 +123,35 @@ def read_shlib_providers(d): shlib_provider[s[0]] = {} shlib_provider[s[0]][s[1]] = (dep_pkg, s[2]) return shlib_provider + + +def npm_split_package_dirs(pkgdir): + """ + Work out the packages fetched and unpacked by BitBake's npm fetcher + Returns a dict of packagename -> (relpath, package.json) ordered + such that it is suitable for use in PACKAGES and FILES + """ + from collections import OrderedDict + import json + packages = {} + for root, dirs, files in os.walk(pkgdir): + if os.path.basename(root) == 'node_modules': + for dn in dirs: + relpth = os.path.relpath(os.path.join(root, dn), pkgdir) + pkgitems = ['${PN}'] + for pathitem in relpth.split('/'): + if pathitem == 'node_modules': + continue + pkgitems.append(pathitem) + pkgname = '-'.join(pkgitems).replace('_', '-') + pkgfile = os.path.join(root, dn, 'package.json') + data = None + if os.path.exists(pkgfile): + with open(pkgfile, 'r') as f: + data = json.loads(f.read()) + packages[pkgname] = (relpth, data) + # We want the main package for a module sorted *after* its subpackages + # (so that it doesn't otherwise steal the files for the subpackage), so + # this is a cheap way to do that whilst still having an otherwise + # alphabetical sort + return OrderedDict((key, packages[key]) for key in sorted(packages, key=lambda pkg: pkg + '~')) diff --git a/yocto-poky/meta/lib/oe/package_manager.py b/yocto-poky/meta/lib/oe/package_manager.py index b9fa6d879..b4b359a8c 100644 --- a/yocto-poky/meta/lib/oe/package_manager.py +++ b/yocto-poky/meta/lib/oe/package_manager.py @@ -8,7 +8,8 @@ import re import bb import tempfile import oe.utils - +import string +from oe.gpg_sign import get_signer # this can be used by all PM backends to create the index files in parallel def create_index(arg): @@ -109,16 +110,11 @@ class RpmIndexer(Indexer): rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo") if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - pkgfeed_gpg_name = self.d.getVar('PACKAGE_FEED_GPG_NAME', True) - pkgfeed_gpg_pass = self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True) + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND', True)) else: - pkgfeed_gpg_name = None - pkgfeed_gpg_pass = None - gpg_bin = self.d.getVar('GPG_BIN', True) or \ - bb.utils.which(os.getenv('PATH'), "gpg") - + signer = None index_cmds = [] - repo_sign_cmds = [] + repomd_files = [] rpm_dirs_found = False for arch in archs: dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch) @@ -130,15 +126,7 @@ class RpmIndexer(Indexer): index_cmds.append("%s --dbpath %s --update -q %s" % \ (rpm_createrepo, dbpath, arch_dir)) - if pkgfeed_gpg_name: - repomd_file = os.path.join(arch_dir, 'repodata', 'repomd.xml') - gpg_cmd = "%s --detach-sign --armor --batch --no-tty --yes " \ - "--passphrase-file '%s' -u '%s' " % \ - (gpg_bin, pkgfeed_gpg_pass, pkgfeed_gpg_name) - if self.d.getVar('GPG_PATH', True): - gpg_cmd += "--homedir %s " % self.d.getVar('GPG_PATH', True) - gpg_cmd += repomd_file - repo_sign_cmds.append(gpg_cmd) + repomd_files.append(os.path.join(arch_dir, 'repodata', 'repomd.xml')) rpm_dirs_found = True @@ -151,19 +139,14 @@ class RpmIndexer(Indexer): if result: bb.fatal('%s' % ('\n'.join(result))) # Sign repomd - result = oe.utils.multiprocess_exec(repo_sign_cmds, create_index) - if result: - bb.fatal('%s' % ('\n'.join(result))) - # Copy pubkey(s) to repo - distro_version = self.d.getVar('DISTRO_VERSION', True) or "oe.0" - if self.d.getVar('RPM_SIGN_PACKAGES', True) == '1': - shutil.copy2(self.d.getVar('RPM_GPG_PUBKEY', True), - os.path.join(self.deploy_dir, - 'RPM-GPG-KEY-%s' % distro_version)) - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - shutil.copy2(self.d.getVar('PACKAGE_FEED_GPG_PUBKEY', True), - os.path.join(self.deploy_dir, - 'REPODATA-GPG-KEY-%s' % distro_version)) + if signer: + for repomd in repomd_files: + feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE', True) + is_ascii_sig = (feed_sig_type.upper() != "BIN") + signer.detach_sign(repomd, + self.d.getVar('PACKAGE_FEED_GPG_NAME', True), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True), + armor=is_ascii_sig) class OpkgIndexer(Indexer): @@ -173,11 +156,16 @@ class OpkgIndexer(Indexer): "MULTILIB_ARCHS"] opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") + if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': + signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND', True)) + else: + signer = None if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): open(os.path.join(self.deploy_dir, "Packages"), "w").close() - index_cmds = [] + index_cmds = set() + index_sign_files = set() for arch_var in arch_vars: archs = self.d.getVar(arch_var, True) if archs is None: @@ -193,9 +181,11 @@ class OpkgIndexer(Indexer): if not os.path.exists(pkgs_file): open(pkgs_file, "w").close() - index_cmds.append('%s -r %s -p %s -m %s' % + index_cmds.add('%s -r %s -p %s -m %s' % (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) + index_sign_files.add(pkgs_file) + if len(index_cmds) == 0: bb.note("There are no packages in %s!" % self.deploy_dir) return @@ -203,9 +193,15 @@ class OpkgIndexer(Indexer): result = oe.utils.multiprocess_exec(index_cmds, create_index) if result: bb.fatal('%s' % ('\n'.join(result))) - if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': - raise NotImplementedError('Package feed signing not implementd for ipk') + if signer: + feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE', True) + is_ascii_sig = (feed_sig_type.upper() != "BIN") + for f in index_sign_files: + signer.detach_sign(f, + self.d.getVar('PACKAGE_FEED_GPG_NAME', True), + self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE', True), + armor=is_ascii_sig) class DpkgIndexer(Indexer): @@ -247,7 +243,7 @@ class DpkgIndexer(Indexer): if a not in pkg_archs: arch_list.append(a) - all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").split() arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list) apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive") @@ -293,10 +289,61 @@ class PkgsList(object): self.rootfs_dir = rootfs_dir @abstractmethod - def list(self, format=None): + def list_pkgs(self): pass + """ + This method parse the output from the package manager + and return a dictionary with the information of the + installed packages. This is used whne the packages are + in deb or ipk format + """ + def opkg_query(self, cmd_output): + verregex = re.compile(' \([=<>]* [^ )]*\)') + output = dict() + filename = "" + dep = [] + pkg = "" + for line in cmd_output.splitlines(): + line = line.rstrip() + if ':' in line: + if line.startswith("Package: "): + pkg = line.split(": ")[1] + elif line.startswith("Architecture: "): + arch = line.split(": ")[1] + elif line.startswith("Version: "): + ver = line.split(": ")[1] + elif line.startswith("File: "): + filename = line.split(": ")[1] + elif line.startswith("Depends: "): + depends = verregex.sub('', line.split(": ")[1]) + for depend in depends.split(", "): + dep.append(depend) + elif line.startswith("Recommends: "): + recommends = verregex.sub('', line.split(": ")[1]) + for recommend in recommends.split(", "): + dep.append("%s [REC]" % recommend) + else: + # IPK doesn't include the filename + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + if pkg: + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + pkg = "" + filename = "" + dep = [] + + if pkg: + if not filename: + filename = "%s_%s_%s.ipk" % (pkg, ver, arch) + output[pkg] = {"arch":arch, "ver":ver, + "filename":filename, "deps": dep } + + return output + + class RpmPkgsList(PkgsList): def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): super(RpmPkgsList, self).__init__(d, rootfs_dir) @@ -314,7 +361,6 @@ class RpmPkgsList(PkgsList): except subprocess.CalledProcessError as e: bb.fatal("Getting rpm version failed. Command '%s' " "returned %d:\n%s" % (cmd, e.returncode, e.output)) - self.rpm_version = int(output.split()[-1].split('.')[0]) ''' Translate the RPM/Smart format names to the OE multilib format names @@ -362,55 +408,53 @@ class RpmPkgsList(PkgsList): return output - def list(self, format=None): - if format == "deps": - if self.rpm_version == 4: - bb.fatal("'deps' format dependency listings are not supported with rpm 4 since rpmresolve does not work") - return self._list_pkg_deps() - + def list_pkgs(self): cmd = self.rpm_cmd + ' --root ' + self.rootfs_dir cmd += ' -D "_dbpath /var/lib/rpm" -qa' - if self.rpm_version == 4: - cmd += " --qf '[%{NAME} %{ARCH} %{VERSION}\n]'" - else: - cmd += " --qf '[%{NAME} %{ARCH} %{VERSION} %{PACKAGEORIGIN}\n]'" + cmd += " --qf '[%{NAME} %{ARCH} %{VERSION} %{PACKAGEORIGIN}\n]'" try: # bb.note(cmd) tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() - except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d:\n%s" % (cmd, e.returncode, e.output)) - output = list() + output = dict() + deps = dict() + dependencies = self._list_pkg_deps() + + # Populate deps dictionary for better manipulation + for line in dependencies.splitlines(): + try: + pkg, dep = line.split("|") + if not pkg in deps: + deps[pkg] = list() + if not dep in deps[pkg]: + deps[pkg].append(dep) + except: + # Ignore any other lines they're debug or errors + pass + for line in tmp_output.split('\n'): if len(line.strip()) == 0: continue pkg = line.split()[0] arch = line.split()[1] ver = line.split()[2] + dep = deps.get(pkg, []) + # Skip GPG keys if pkg == 'gpg-pubkey': continue - if self.rpm_version == 4: - pkgorigin = "unknown" - else: - pkgorigin = line.split()[3] - new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, arch) - if format == "arch": - output.append('%s %s' % (new_pkg, new_arch)) - elif format == "file": - output.append('%s %s %s' % (new_pkg, pkgorigin, new_arch)) - elif format == "ver": - output.append('%s %s %s' % (new_pkg, new_arch, ver)) - else: - output.append('%s' % (new_pkg)) + pkgorigin = line.split()[3] + new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, arch) - output.sort() + output[new_pkg] = {"arch":new_arch, "ver":ver, + "filename":pkgorigin, "deps":dep} - return '\n'.join(output) + return output class OpkgPkgsList(PkgsList): @@ -421,107 +465,39 @@ class OpkgPkgsList(PkgsList): self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) self.opkg_args += self.d.getVar("OPKG_ARGS", True) - def list(self, format=None): - opkg_query_cmd = bb.utils.which(os.getenv('PATH'), "opkg-query-helper.py") - - if format == "arch": - cmd = "%s %s status | %s -a" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "file": - cmd = "%s %s status | %s -f" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "ver": - cmd = "%s %s status | %s -v" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - elif format == "deps": - cmd = "%s %s status | %s" % \ - (self.opkg_cmd, self.opkg_args, opkg_query_cmd) - else: - cmd = "%s %s list_installed | cut -d' ' -f1" % \ - (self.opkg_cmd, self.opkg_args) - - try: - # bb.note(cmd) - tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() - - except subprocess.CalledProcessError as e: + def list_pkgs(self, format=None): + cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) + + # opkg returns success even when it printed some + # "Collected errors:" report to stderr. Mixing stderr into + # stdout then leads to random failures later on when + # parsing the output. To avoid this we need to collect both + # output streams separately and check for empty stderr. + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + cmd_output, cmd_stderr = p.communicate() + if p.returncode or cmd_stderr: bb.fatal("Cannot get the installed packages list. Command '%s' " - "returned %d:\n%s" % (cmd, e.returncode, e.output)) - - output = list() - for line in tmp_output.split('\n'): - if len(line.strip()) == 0: - continue - if format == "file": - pkg, pkg_file, pkg_arch = line.split() - full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file) - if os.path.exists(full_path): - output.append('%s %s %s' % (pkg, full_path, pkg_arch)) - else: - output.append('%s %s %s' % (pkg, pkg_file, pkg_arch)) - else: - output.append(line) - - output.sort() + "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) - return '\n'.join(output) + return self.opkg_query(cmd_output) class DpkgPkgsList(PkgsList): - def list(self, format=None): + + def list_pkgs(self): cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"), "--admindir=%s/var/lib/dpkg" % self.rootfs_dir, "-W"] - if format == "arch": - cmd.append("-f=${Package} ${PackageArch}\n") - elif format == "file": - cmd.append("-f=${Package} ${Package}_${Version}_${Architecture}.deb ${PackageArch}\n") - elif format == "ver": - cmd.append("-f=${Package} ${PackageArch} ${Version}\n") - elif format == "deps": - cmd.append("-f=Package: ${Package}\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") - else: - cmd.append("-f=${Package}\n") + cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\n\n") try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) - if format == "file": - tmp_output = "" - for line in tuple(output.split('\n')): - if not line.strip(): - continue - pkg, pkg_file, pkg_arch = line.split() - full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file) - if os.path.exists(full_path): - tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch) - else: - tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch) - - output = tmp_output - elif format == "deps": - opkg_query_cmd = bb.utils.which(os.getenv('PATH'), "opkg-query-helper.py") - file_out = tempfile.NamedTemporaryFile() - file_out.write(output) - file_out.flush() - - try: - output = subprocess.check_output("cat %s | %s" % - (file_out.name, opkg_query_cmd), - stderr=subprocess.STDOUT, - shell=True) - except subprocess.CalledProcessError as e: - file_out.close() - bb.fatal("Cannot compute packages dependencies. Command '%s' " - "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) - - file_out.close() - - return output + return self.opkg_query(cmd_output) class PackageManager(object): @@ -535,7 +511,8 @@ class PackageManager(object): self.deploy_dir = None self.deploy_lock = None self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or "" - self.feed_prefix = self.d.getVar('PACKAGE_FEED_PREFIX', True) or "" + self.feed_base_paths = self.d.getVar('PACKAGE_FEED_BASE_PATHS', True) or "" + self.feed_archs = self.d.getVar('PACKAGE_FEED_ARCHS', True) """ Update the package manager package database. @@ -572,7 +549,7 @@ class PackageManager(object): pass @abstractmethod - def list_installed(self, format=None): + def list_installed(self): pass @abstractmethod @@ -592,7 +569,9 @@ class PackageManager(object): installed_pkgs_file = os.path.join(self.d.getVar('WORKDIR', True), "installed_pkgs.txt") with open(installed_pkgs_file, "w+") as installed_pkgs: - installed_pkgs.write(self.list_installed("arch")) + pkgs = self.list_installed() + output = oe.utils.format_pkg_list(pkgs, "arch") + installed_pkgs.write(output) if globs is None: globs = self.d.getVar('IMAGE_INSTALL_COMPLEMENTARY', True) @@ -625,6 +604,7 @@ class PackageManager(object): "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) self.install(complementary_pkgs.split(), attempt_only=True) + os.remove(installed_pkgs_file) def deploy_dir_lock(self): if self.deploy_dir is None: @@ -642,6 +622,25 @@ class PackageManager(object): self.deploy_lock = None + """ + Construct URIs based on the following pattern: uri/base_path where 'uri' + and 'base_path' correspond to each element of the corresponding array + argument leading to len(uris) x len(base_paths) elements on the returned + array + """ + def construct_uris(self, uris, base_paths): + def _append(arr1, arr2, sep='/'): + res = [] + narr1 = map(lambda a: string.rstrip(a, sep), arr1) + narr2 = map(lambda a: string.lstrip(string.rstrip(a, sep), sep), arr2) + for a1 in narr1: + if arr2: + for a2 in narr2: + res.append("%s%s%s" % (a1, sep, a2)) + else: + res.append(a1) + return res + return _append(uris, base_paths) class RpmPM(PackageManager): def __init__(self, @@ -664,8 +663,16 @@ class RpmPM(PackageManager): self.install_dir_path = os.path.join(self.target_rootfs, self.install_dir_name) self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") self.smart_cmd = bb.utils.which(os.getenv('PATH'), "smart") - self.smart_opt = "--log-level=warning --data-dir=" + os.path.join(target_rootfs, - 'var/lib/smart') + # 0 = default, only warnings + # 1 = --log-level=info (includes information about executing scriptlets and their output) + # 2 = --log-level=debug + # 3 = --log-level=debug plus dumps of scriplet content and command invocation + self.debug_level = int(d.getVar('ROOTFS_RPM_DEBUG', True) or "0") + self.smart_opt = "--log-level=%s --data-dir=%s" % \ + ("warning" if self.debug_level == 0 else + "info" if self.debug_level == 1 else + "debug", + os.path.join(target_rootfs, 'var/lib/smart')) self.scriptlet_wrapper = self.d.expand('${WORKDIR}/scriptlet_wrapper') self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % self.task_name) @@ -677,7 +684,6 @@ class RpmPM(PackageManager): self.indexer = RpmIndexer(self.d, self.deploy_dir) self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) - self.rpm_version = self.pkgs_list.rpm_version self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) @@ -685,42 +691,55 @@ class RpmPM(PackageManager): if self.feed_uris == "": return - # List must be prefered to least preferred order - default_platform_extra = set() - platform_extra = set() - bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" - for mlib in self.ml_os_list: - for arch in self.ml_prefix_list[mlib]: - plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] - if mlib == bbextendvariant: - default_platform_extra.add(plt) - else: - platform_extra.add(plt) + arch_list = [] + if self.feed_archs is not None: + # User define feed architectures + arch_list = self.feed_archs.split() + else: + # List must be prefered to least preferred order + default_platform_extra = set() + platform_extra = set() + bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or "" + for mlib in self.ml_os_list: + for arch in self.ml_prefix_list[mlib]: + plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib] + if mlib == bbextendvariant: + default_platform_extra.add(plt) + else: + platform_extra.add(plt) - platform_extra = platform_extra.union(default_platform_extra) + platform_extra = platform_extra.union(default_platform_extra) - arch_list = [] - for canonical_arch in platform_extra: - arch = canonical_arch.split('-')[0] - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - arch_list.append(arch) + for canonical_arch in platform_extra: + arch = canonical_arch.split('-')[0] + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + + feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) uri_iterator = 0 - channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list) + channel_priority = 10 + 5 * len(feed_uris) * (len(arch_list) if arch_list else 1) - for uri in self.feed_uris.split(): - full_uri = uri - if self.feed_prefix: - full_uri = os.path.join(uri, self.feed_prefix) - for arch in arch_list: - bb.note('Note: adding Smart channel url%d%s (%s)' % - (uri_iterator, arch, channel_priority)) - self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y' - % (uri_iterator, arch, full_uri, arch)) - self._invoke_smart('channel --set url%d-%s priority=%d' % - (uri_iterator, arch, channel_priority)) + for uri in feed_uris: + if arch_list: + for arch in arch_list: + bb.note('Note: adding Smart channel url%d%s (%s)' % + (uri_iterator, arch, channel_priority)) + self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/%s -y' + % (uri_iterator, arch, uri, arch)) + self._invoke_smart('channel --set url%d-%s priority=%d' % + (uri_iterator, arch, channel_priority)) + channel_priority -= 5 + else: + bb.note('Note: adding Smart channel url%d (%s)' % + (uri_iterator, channel_priority)) + self._invoke_smart('channel --add url%d type=rpm-md baseurl=%s -y' + % (uri_iterator, uri)) + self._invoke_smart('channel --set url%d priority=%d' % + (uri_iterator, channel_priority)) channel_priority -= 5 + uri_iterator += 1 ''' @@ -886,44 +905,41 @@ class RpmPM(PackageManager): # After change the __db.* cache size, log file will not be # generated automatically, that will raise some warnings, # so touch a bare log for rpm write into it. - if self.rpm_version == 5: - rpmlib_log = os.path.join(self.image_rpmlib, 'log', 'log.0000000001') - if not os.path.exists(rpmlib_log): - bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log')) - open(rpmlib_log, 'w+').close() - - DB_CONFIG_CONTENT = "# ================ Environment\n" \ - "set_data_dir .\n" \ - "set_create_dir .\n" \ - "set_lg_dir ./log\n" \ - "set_tmp_dir ./tmp\n" \ - "set_flags db_log_autoremove on\n" \ - "\n" \ - "# -- thread_count must be >= 8\n" \ - "set_thread_count 64\n" \ - "\n" \ - "# ================ Logging\n" \ - "\n" \ - "# ================ Memory Pool\n" \ - "set_cachesize 0 1048576 0\n" \ - "set_mp_mmapsize 268435456\n" \ - "\n" \ - "# ================ Locking\n" \ - "set_lk_max_locks 16384\n" \ - "set_lk_max_lockers 16384\n" \ - "set_lk_max_objects 16384\n" \ - "mutex_set_max 163840\n" \ - "\n" \ - "# ================ Replication\n" - - db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG') - if not os.path.exists(db_config_dir): - open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT) + rpmlib_log = os.path.join(self.image_rpmlib, 'log', 'log.0000000001') + if not os.path.exists(rpmlib_log): + bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log')) + open(rpmlib_log, 'w+').close() + + DB_CONFIG_CONTENT = "# ================ Environment\n" \ + "set_data_dir .\n" \ + "set_create_dir .\n" \ + "set_lg_dir ./log\n" \ + "set_tmp_dir ./tmp\n" \ + "set_flags db_log_autoremove on\n" \ + "\n" \ + "# -- thread_count must be >= 8\n" \ + "set_thread_count 64\n" \ + "\n" \ + "# ================ Logging\n" \ + "\n" \ + "# ================ Memory Pool\n" \ + "set_cachesize 0 1048576 0\n" \ + "set_mp_mmapsize 268435456\n" \ + "\n" \ + "# ================ Locking\n" \ + "set_lk_max_locks 16384\n" \ + "set_lk_max_lockers 16384\n" \ + "set_lk_max_objects 16384\n" \ + "mutex_set_max 163840\n" \ + "\n" \ + "# ================ Replication\n" + + db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG') + if not os.path.exists(db_config_dir): + open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT) # Create database so that smart doesn't complain (lazy init) opt = "-qa" - if self.rpm_version == 4: - opt = "--initdb" cmd = "%s --root %s --dbpath /var/lib/rpm %s > /dev/null" % ( self.rpm_cmd, self.target_rootfs, opt) try: @@ -1010,12 +1026,19 @@ class RpmPM(PackageManager): # If we ever run into needing more the 899 scripts, we'll have to. # change num to start with 1000. # - if self.rpm_version == 4: - scriptletcmd = "$2 $3 $4\n" - scriptpath = "$3" + scriptletcmd = "$2 $1/$3 $4\n" + scriptpath = "$1/$3" + + # When self.debug_level >= 3, also dump the content of the + # executed scriptlets and how they get invoked. We have to + # replace "exit 1" and "ERR" because printing those as-is + # would trigger a log analysis failure. + if self.debug_level >= 3: + dump_invocation = 'echo "Executing ${name} ${kind} with: ' + scriptletcmd + '"\n' + dump_script = 'cat ' + scriptpath + '| sed -e "s/exit 1/exxxit 1/g" -e "s/ERR/IRR/g"; echo\n' else: - scriptletcmd = "$2 $1/$3 $4\n" - scriptpath = "$1/$3" + dump_invocation = 'echo "Executing ${name} ${kind}"\n' + dump_script = '' SCRIPTLET_FORMAT = "#!/bin/bash\n" \ "\n" \ @@ -1027,19 +1050,25 @@ class RpmPM(PackageManager): "export INTERCEPT_DIR=%s\n" \ "export NATIVE_ROOT=%s\n" \ "\n" \ + "name=`head -1 " + scriptpath + " | cut -d\' \' -f 2`\n" \ + "kind=`head -1 " + scriptpath + " | cut -d\' \' -f 4`\n" \ + + dump_invocation \ + + dump_script \ + scriptletcmd + \ - "if [ $? -ne 0 ]; then\n" \ + "ret=$?\n" \ + "echo Result of ${name} ${kind}: ${ret}\n" \ + "if [ ${ret} -ne 0 ]; then\n" \ " if [ $4 -eq 1 ]; then\n" \ " mkdir -p $1/etc/rpm-postinsts\n" \ " num=100\n" \ " while [ -e $1/etc/rpm-postinsts/${num}-* ]; do num=$((num + 1)); done\n" \ - " name=`head -1 " + scriptpath + " | cut -d\' \' -f 2`\n" \ ' echo "#!$2" > $1/etc/rpm-postinsts/${num}-${name}\n' \ ' echo "# Arg: $4" >> $1/etc/rpm-postinsts/${num}-${name}\n' \ " cat " + scriptpath + " >> $1/etc/rpm-postinsts/${num}-${name}\n" \ " chmod +x $1/etc/rpm-postinsts/${num}-${name}\n" \ + ' echo "Info: deferring ${name} ${kind} install scriptlet to first boot"\n' \ " else\n" \ - ' echo "Error: pre/post remove scriptlet failed"\n' \ + ' echo "Error: ${name} ${kind} remove scriptlet failed"\n' \ " fi\n" \ "fi\n" @@ -1238,8 +1267,8 @@ class RpmPM(PackageManager): self.image_rpmlib, symlinks=True) - def list_installed(self, format=None): - return self.pkgs_list.list(format) + def list_installed(self): + return self.pkgs_list.list_pkgs() ''' If incremental install, we need to determine what we've got, @@ -1397,10 +1426,11 @@ class OpkgPM(PackageManager): if not os.path.exists(self.d.expand('${T}/saved')): bb.utils.mkdirhier(self.d.expand('${T}/saved')) - if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": - self._create_config() - else: + self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") == "1" + if self.from_feeds: self._create_custom_config() + else: + self._create_config() self.indexer = OpkgIndexer(self.d, self.deploy_dir) @@ -1471,14 +1501,14 @@ class OpkgPM(PackageManager): self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True), arch)) - if self.opkg_dir != '/var/lib/opkg': - # There is no command line option for this anymore, we need to add - # info_dir and status_file to config file, if OPKGLIBDIR doesn't have - # the default value of "/var/lib" as defined in opkg: - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" - # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" - cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) - cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) + if self.opkg_dir != '/var/lib/opkg': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" + cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) + cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) def _create_config(self): @@ -1512,21 +1542,26 @@ class OpkgPM(PackageManager): rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' % self.target_rootfs) + feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) + archs = self.pkg_archs.split() if self.feed_archs is None else self.feed_archs.split() + with open(rootfs_config, "w+") as config_file: uri_iterator = 0 - for uri in self.feed_uris.split(): - full_uri = uri - if self.feed_prefix: - full_uri = os.path.join(uri, self.feed_prefix) - - for arch in self.pkg_archs.split(): - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - bb.note('Note: adding opkg feed url-%s-%d (%s)' % - (arch, uri_iterator, full_uri)) + for uri in feed_uris: + if archs: + for arch in archs: + if (self.feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))): + continue + bb.note('Note: adding opkg feed url-%s-%d (%s)' % + (arch, uri_iterator, uri)) + config_file.write("src/gz uri-%s-%d %s/%s\n" % + (arch, uri_iterator, uri, arch)) + else: + bb.note('Note: adding opkg feed url-%d (%s)' % + (uri_iterator, uri)) + config_file.write("src/gz uri-%d %s\n" % + (uri_iterator, uri)) - config_file.write("src/gz uri-%s-%d %s/%s\n" % - (arch, uri_iterator, full_uri, arch)) uri_iterator += 1 def update(self): @@ -1598,8 +1633,12 @@ class OpkgPM(PackageManager): # create the directory back, it's needed by PM lock bb.utils.mkdirhier(self.opkg_dir) - def list_installed(self, format=None): - return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list(format) + def remove_lists(self): + if not self.from_feeds: + bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) + + def list_installed(self): + return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs() def handle_bad_recommendations(self): bad_recommendations = self.d.getVar("BAD_RECOMMENDATIONS", True) or "" @@ -1711,7 +1750,7 @@ class DpkgPM(PackageManager): self.apt_args = d.getVar("APT_ARGS", True) self.all_arch_list = archs.split() - all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split() + all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").split() self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list) self._create_configs(archs, base_archs) @@ -1872,20 +1911,26 @@ class DpkgPM(PackageManager): % self.target_rootfs) arch_list = [] - for arch in self.all_arch_list: - if not os.path.exists(os.path.join(self.deploy_dir, arch)): - continue - arch_list.append(arch) + if self.feed_archs is None: + for arch in self.all_arch_list: + if not os.path.exists(os.path.join(self.deploy_dir, arch)): + continue + arch_list.append(arch) + else: + arch_list = self.feed_archs.split() + + feed_uris = self.construct_uris(self.feed_uris.split(), self.feed_base_paths.split()) with open(sources_conf, "w+") as sources_file: - for uri in self.feed_uris.split(): - full_uri = uri - if self.feed_prefix: - full_uri = os.path.join(uri, self.feed_prefix) - for arch in arch_list: + for uri in feed_uris: + if arch_list: + for arch in arch_list: + bb.note('Note: adding dpkg channel at (%s)' % uri) + sources_file.write("deb %s/%s ./\n" % + (uri, arch)) + else: bb.note('Note: adding dpkg channel at (%s)' % uri) - sources_file.write("deb %s/%s ./\n" % - (full_uri, arch)) + sources_file.write("deb %s ./\n" % uri) def _create_configs(self, archs, base_archs): base_archs = re.sub("_", "-", base_archs) @@ -1896,6 +1941,7 @@ class DpkgPM(PackageManager): bb.utils.mkdirhier(self.apt_conf_dir) bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/") bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/") + bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/") arch_list = [] for arch in self.all_arch_list: @@ -1930,10 +1976,14 @@ class DpkgPM(PackageManager): base_arch_list = base_archs.split() multilib_variants = self.d.getVar("MULTILIB_VARIANTS", True); for variant in multilib_variants.split(): - if variant == "lib32": - base_arch_list.append("i386") - elif variant == "lib64": - base_arch_list.append("amd64") + localdata = bb.data.createCopy(self.d) + variant_tune = localdata.getVar("DEFAULTTUNE_virtclass-multilib-" + variant, False) + orig_arch = localdata.getVar("DPKG_ARCH", True) + localdata.setVar("DEFAULTTUNE", variant_tune) + bb.data.update_data(localdata) + variant_arch = localdata.getVar("DPKG_ARCH", True) + if variant_arch not in base_arch_list: + base_arch_list.append(variant_arch) with open(self.apt_conf_file, "w+") as apt_conf: with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample: @@ -1976,8 +2026,8 @@ class DpkgPM(PackageManager): bb.fatal("Cannot fix broken dependencies. Command '%s' " "returned %d:\n%s" % (cmd, e.returncode, e.output)) - def list_installed(self, format=None): - return DpkgPkgsList(self.d, self.target_rootfs).list() + def list_installed(self): + return DpkgPkgsList(self.d, self.target_rootfs).list_pkgs() def generate_index_files(d): diff --git a/yocto-poky/meta/lib/oe/packagedata.py b/yocto-poky/meta/lib/oe/packagedata.py index cd5f0445f..bc0fd06bc 100644 --- a/yocto-poky/meta/lib/oe/packagedata.py +++ b/yocto-poky/meta/lib/oe/packagedata.py @@ -1,4 +1,5 @@ import codecs +import os def packaged(pkg, d): return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) diff --git a/yocto-poky/meta/lib/oe/packagegroup.py b/yocto-poky/meta/lib/oe/packagegroup.py index 12eb4212f..a6fee5f95 100644 --- a/yocto-poky/meta/lib/oe/packagegroup.py +++ b/yocto-poky/meta/lib/oe/packagegroup.py @@ -3,9 +3,9 @@ import itertools def is_optional(feature, d): packages = d.getVar("FEATURE_PACKAGES_%s" % feature, True) if packages: - return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional")) + return bool(d.getVarFlag("FEATURE_PACKAGES_%s" % feature, "optional", True)) else: - return bool(d.getVarFlag("PACKAGE_GROUP_%s" % feature, "optional")) + return bool(d.getVarFlag("PACKAGE_GROUP_%s" % feature, "optional", True)) def packages(features, d): for feature in features: diff --git a/yocto-poky/meta/lib/oe/patch.py b/yocto-poky/meta/lib/oe/patch.py index 2bf501e9e..9d3617290 100644 --- a/yocto-poky/meta/lib/oe/patch.py +++ b/yocto-poky/meta/lib/oe/patch.py @@ -8,12 +8,14 @@ class NotFoundError(bb.BBHandledException): return "Error: %s not found." % self.path class CmdError(bb.BBHandledException): - def __init__(self, exitstatus, output): + def __init__(self, command, exitstatus, output): + self.command = command self.status = exitstatus self.output = output def __str__(self): - return "Command Error: exit status: %d Output:\n%s" % (self.status, self.output) + return "Command Error: '%s' exited with %d Output:\n%s" % \ + (self.command, self.status, self.output) def runcmd(args, dir = None): @@ -32,7 +34,7 @@ def runcmd(args, dir = None): # print("cmd: %s" % cmd) (exitstatus, output) = oe.utils.getstatusoutput(cmd) if exitstatus != 0: - raise CmdError(exitstatus >> 8, output) + raise CmdError(cmd, exitstatus >> 8, output) return output finally: @@ -212,13 +214,17 @@ class PatchTree(PatchSet): if not force: shellcmd.append('--dry-run') - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + try: + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) - if force: - return + if force: + return - shellcmd.pop(len(shellcmd) - 1) - output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + shellcmd.pop(len(shellcmd) - 1) + output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + except CmdError as err: + raise bb.BBHandledException("Applying '%s' failed:\n%s" % + (os.path.basename(patch['file']), err.output)) if not reverse: self._appendPatchFile(patch['file'], patch['strippath']) @@ -264,6 +270,7 @@ class PatchTree(PatchSet): class GitApplyTree(PatchTree): patch_line_prefix = '%% original patch' + ignore_commit_prefix = '%% ignore' def __init__(self, dir, d): PatchTree.__init__(self, dir, d) @@ -282,33 +289,32 @@ class GitApplyTree(PatchTree): return lines @staticmethod - def prepareCommit(patchfile): - """ - Prepare a git commit command line based on the header from a patch file - (typically this is useful for patches that cannot be applied with "git am" due to formatting) - """ - import tempfile + def decodeAuthor(line): + from email.header import decode_header + authorval = line.split(':', 1)[1].strip().replace('"', '') + return decode_header(authorval)[0][0] + + @staticmethod + def interpretPatchHeader(headerlines): import re author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') - # Process patch header and extract useful information - lines = GitApplyTree.extractPatchHeader(patchfile) outlines = [] author = None date = None - for line in lines: + subject = None + for line in headerlines: if line.startswith('Subject: '): subject = line.split(':', 1)[1] # Remove any [PATCH][oe-core] etc. subject = re.sub(r'\[.+?\]\s*', '', subject) - outlines.insert(0, '%s\n\n' % subject.strip()) continue - if line.startswith('From: ') or line.startswith('Author: '): - authorval = line.split(':', 1)[1].strip().replace('"', '') + elif line.startswith('From: ') or line.startswith('Author: '): + authorval = GitApplyTree.decodeAuthor(line) # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval continue - if line.startswith('Date: '): + elif line.startswith('Date: '): if date is None: dateval = line.split(':', 1)[1].strip() # Very crude check for date format, since git will blow up if it's not in the right @@ -316,12 +322,41 @@ class GitApplyTree(PatchTree): if len(dateval) > 12: date = dateval continue - if line.startswith('Signed-off-by: '): - authorval = line.split(':', 1)[1].strip().replace('"', '') + elif not author and line.lower().startswith('signed-off-by: '): + authorval = GitApplyTree.decodeAuthor(line) # git is fussy about author formatting i.e. it must be Name <email@domain> if author_re.match(authorval): author = authorval outlines.append(line) + return outlines, author, date, subject + + @staticmethod + def prepareCommit(patchfile): + """ + Prepare a git commit command line based on the header from a patch file + (typically this is useful for patches that cannot be applied with "git am" due to formatting) + """ + import tempfile + # Process patch header and extract useful information + lines = GitApplyTree.extractPatchHeader(patchfile) + outlines, author, date, subject = GitApplyTree.interpretPatchHeader(lines) + if not author or not subject: + try: + shellcmd = ["git", "log", "--format=email", "--diff-filter=A", "--", patchfile] + out = runcmd(["sh", "-c", " ".join(shellcmd)], os.path.dirname(patchfile)) + except CmdError: + out = None + if out: + _, newauthor, newdate, newsubject = GitApplyTree.interpretPatchHeader(out.splitlines()) + if not author or not date: + # These really need to go together + author = newauthor + date = newdate + if not subject: + subject = newsubject + if subject: + outlines.insert(0, '%s\n\n' % subject.strip()) + # Write out commit message to a file with tempfile.NamedTemporaryFile('w', delete=False) as tf: tmpfile = tf.name @@ -356,6 +391,8 @@ class GitApplyTree(PatchTree): if line.startswith(GitApplyTree.patch_line_prefix): outfile = line.split()[-1].strip() continue + if line.startswith(GitApplyTree.ignore_commit_prefix): + continue patchlines.append(line) if not outfile: outfile = os.path.basename(srcfile) @@ -383,14 +420,15 @@ class GitApplyTree(PatchTree): reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() if not reporoot: raise Exception("Cannot get repository root for directory %s" % self.dir) - commithook = os.path.join(reporoot, '.git', 'hooks', 'commit-msg') - commithook_backup = commithook + '.devtool-orig' - applyhook = os.path.join(reporoot, '.git', 'hooks', 'applypatch-msg') - applyhook_backup = applyhook + '.devtool-orig' - if os.path.exists(commithook): - shutil.move(commithook, commithook_backup) - if os.path.exists(applyhook): - shutil.move(applyhook, applyhook_backup) + hooks_dir = os.path.join(reporoot, '.git', 'hooks') + hooks_dir_backup = hooks_dir + '.devtool-orig' + if os.path.lexists(hooks_dir_backup): + raise Exception("Git hooks backup directory already exists: %s" % hooks_dir_backup) + if os.path.lexists(hooks_dir): + shutil.move(hooks_dir, hooks_dir_backup) + os.mkdir(hooks_dir) + commithook = os.path.join(hooks_dir, 'commit-msg') + applyhook = os.path.join(hooks_dir, 'applypatch-msg') with open(commithook, 'w') as f: # NOTE: the formatting here is significant; if you change it you'll also need to # change other places which read it back @@ -439,12 +477,9 @@ class GitApplyTree(PatchTree): os.remove(tmpfile) return output finally: - os.remove(commithook) - os.remove(applyhook) - if os.path.exists(commithook_backup): - shutil.move(commithook_backup, commithook) - if os.path.exists(applyhook_backup): - shutil.move(applyhook_backup, applyhook) + shutil.rmtree(hooks_dir) + if os.path.lexists(hooks_dir_backup): + shutil.move(hooks_dir_backup, hooks_dir) class QuiltTree(PatchSet): diff --git a/yocto-poky/meta/lib/oe/qa.py b/yocto-poky/meta/lib/oe/qa.py index d5cdaa0fc..3cfeee737 100644 --- a/yocto-poky/meta/lib/oe/qa.py +++ b/yocto-poky/meta/lib/oe/qa.py @@ -1,3 +1,8 @@ +import os, struct + +class NotELFFileError(Exception): + pass + class ELFFile: EI_NIDENT = 16 @@ -7,6 +12,8 @@ class ELFFile: EI_OSABI = 7 EI_ABIVERSION = 8 + E_MACHINE = 0x12 + # possible values for EI_CLASS ELFCLASSNONE = 0 ELFCLASS32 = 1 @@ -20,10 +27,12 @@ class ELFFile: ELFDATA2LSB = 1 ELFDATA2MSB = 2 + PT_INTERP = 3 + def my_assert(self, expectation, result): if not expectation == result: #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name) - raise Exception("This does not work as expected") + raise NotELFFileError("%s is not an ELF" % self.name) def __init__(self, name, bits = 0): self.name = name @@ -31,10 +40,16 @@ class ELFFile: self.objdump_output = {} def open(self): + if not os.path.isfile(self.name): + raise NotELFFileError("%s is not a normal file" % self.name) + self.file = file(self.name, "r") - self.data = self.file.read(ELFFile.EI_NIDENT+4) + # Read 4k which should cover most of the headers we're after + self.data = self.file.read(4096) + + if len(self.data) < ELFFile.EI_NIDENT + 4: + raise NotELFFileError("%s is not an ELF" % self.name) - self.my_assert(len(self.data), ELFFile.EI_NIDENT+4) self.my_assert(self.data[0], chr(0x7f) ) self.my_assert(self.data[1], 'E') self.my_assert(self.data[2], 'L') @@ -46,24 +61,24 @@ class ELFFile: self.bits = 64 else: # Not 32-bit or 64.. lets assert - raise Exception("ELF but not 32 or 64 bit.") + raise NotELFFileError("ELF but not 32 or 64 bit.") elif self.bits == 32: self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32)) elif self.bits == 64: self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64)) else: - raise Exception("Must specify unknown, 32 or 64 bit size.") + raise NotELFFileError("Must specify unknown, 32 or 64 bit size.") self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) ) self.sex = self.data[ELFFile.EI_DATA] if self.sex == chr(ELFFile.ELFDATANONE): - raise Exception("self.sex == ELFDATANONE") + raise NotELFFileError("self.sex == ELFDATANONE") elif self.sex == chr(ELFFile.ELFDATA2LSB): self.sex = "<" elif self.sex == chr(ELFFile.ELFDATA2MSB): self.sex = ">" else: - raise Exception("Unknown self.sex") + raise NotELFFileError("Unknown self.sex") def osAbi(self): return ord(self.data[ELFFile.EI_OSABI]) @@ -77,17 +92,36 @@ class ELFFile: def isLittleEndian(self): return self.sex == "<" - def isBigEngian(self): + def isBigEndian(self): return self.sex == ">" + def getShort(self, offset): + return struct.unpack_from(self.sex+"H", self.data, offset)[0] + + def getWord(self, offset): + return struct.unpack_from(self.sex+"i", self.data, offset)[0] + + def isDynamic(self): + """ + Return True if there is a .interp segment (therefore dynamically + linked), otherwise False (statically linked). + """ + offset = self.getWord(self.bits == 32 and 0x1C or 0x20) + size = self.getShort(self.bits == 32 and 0x2A or 0x36) + count = self.getShort(self.bits == 32 and 0x2C or 0x38) + + for i in range(0, count): + p_type = self.getWord(offset + i * size) + if p_type == ELFFile.PT_INTERP: + return True + return False + def machine(self): """ We know the sex stored in self.sex and we know the position """ - import struct - (a,) = struct.unpack(self.sex+"H", self.data[18:20]) - return a + return self.getShort(ELFFile.E_MACHINE) def run_objdump(self, cmd, d): import bb.process @@ -109,3 +143,9 @@ class ELFFile: except Exception as e: bb.note("%s %s %s failed: %s" % (objdump, cmd, self.name, e)) return "" + +if __name__ == "__main__": + import sys + elf = ELFFile(sys.argv[1]) + elf.open() + print elf.isDynamic() diff --git a/yocto-poky/meta/lib/oe/recipeutils.py b/yocto-poky/meta/lib/oe/recipeutils.py index 119a68821..6c7adb5bd 100644 --- a/yocto-poky/meta/lib/oe/recipeutils.py +++ b/yocto-poky/meta/lib/oe/recipeutils.py @@ -19,9 +19,9 @@ from collections import OrderedDict, defaultdict # Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy'] +recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package()', 'do_deploy()'] # Variables that sometimes are a bit long but shouldn't be wrapped -nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER'] +nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'SRC_URI[md5sum]', 'SRC_URI[sha256sum]'] list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] @@ -164,85 +164,112 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): Note that some manual inspection/intervention may be required since this cannot handle all situations. """ + + import bb.utils + + recipe_progression_res = [] + recipe_progression_restrs = [] + for item in recipe_progression: + if item.endswith('()'): + key = item[:-2] + else: + key = item + restr = '%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key + if item.endswith('()'): + recipe_progression_restrs.append(restr + '()') + else: + recipe_progression_restrs.append(restr) + recipe_progression_res.append(re.compile('^%s$' % restr)) + + def get_recipe_pos(variable): + for i, p in enumerate(recipe_progression_res): + if p.match(variable): + return i + return -1 + remainingnames = {} for k in values.keys(): - remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1 + remainingnames[k] = get_recipe_pos(k) remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1])) - with tempfile.NamedTemporaryFile('w', delete=False) as tf: - def outputvalue(name): - rawtext = '%s = "%s"\n' % (name, values[name]) - if name in nowrap_vars: - tf.write(rawtext) - elif name in list_vars: - splitvalue = split_var_value(values[name], assignment=False) - if len(splitvalue) > 1: - linesplit = ' \\\n' + (' ' * (len(name) + 4)) - tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) - else: - tf.write(rawtext) + modifying = False + + def outputvalue(name, lines, rewindcomments=False): + if values[name] is None: + return + rawtext = '%s = "%s"\n' % (name, values[name]) + addlines = [] + if name in nowrap_vars: + addlines.append(rawtext) + elif name in list_vars: + splitvalue = split_var_value(values[name], assignment=False) + if len(splitvalue) > 1: + linesplit = ' \\\n' + (' ' * (len(name) + 4)) + addlines.append('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) else: - wrapped = textwrap.wrap(rawtext) - for wrapline in wrapped[:-1]: - tf.write('%s \\\n' % wrapline) - tf.write('%s\n' % wrapped[-1]) - - tfn = tf.name - with open(fn, 'r') as f: - # First runthrough - find existing names (so we know not to insert based on recipe_progression) - # Second runthrough - make the changes - existingnames = [] - for runthrough in [1, 2]: - currname = None - for line in f: - if not currname: - insert = False - for k in remainingnames.keys(): - for p in recipe_progression: - if re.match('^%s(_prepend|_append)*[ ?:=(]' % p, line): - if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames: - outputvalue(k) - del remainingnames[k] - break - for k in remainingnames.keys(): - if re.match('^%s[ ?:=]' % k, line): - currname = k - if runthrough == 1: - existingnames.append(k) - else: - del remainingnames[k] - break - if currname and runthrough > 1: - outputvalue(currname) - - if currname: - sline = line.rstrip() - if not sline.endswith('\\'): - currname = None - continue - if runthrough > 1: - tf.write(line) - f.seek(0) - if remainingnames: - tf.write('\n') - for k in remainingnames.keys(): - outputvalue(k) - - with open(tfn, 'U') as f: - tolines = f.readlines() + addlines.append(rawtext) + else: + wrapped = textwrap.wrap(rawtext) + for wrapline in wrapped[:-1]: + addlines.append('%s \\\n' % wrapline) + addlines.append('%s\n' % wrapped[-1]) + if rewindcomments: + # Ensure we insert the lines before any leading comments + # (that we'd want to ensure remain leading the next value) + for i, ln in reversed(list(enumerate(lines))): + if ln[0] != '#': + lines[i+1:i+1] = addlines + break + else: + lines.extend(addlines) + else: + lines.extend(addlines) + + existingnames = [] + def patch_recipe_varfunc(varname, origvalue, op, newlines): + if modifying: + # Insert anything that should come before this variable + pos = get_recipe_pos(varname) + for k in remainingnames.keys()[:]: + if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: + outputvalue(k, newlines, rewindcomments=True) + del remainingnames[k] + # Now change this variable, if it needs to be changed + if varname in existingnames and op in ['+=', '=', '=+']: + if varname in remainingnames: + outputvalue(varname, newlines) + del remainingnames[varname] + return None, None, 0, True + else: + if varname in values: + existingnames.append(varname) + return origvalue, None, 0, True + + # First run - establish which values we want to set are already in the file + varlist = [re.escape(item) for item in values.keys()] + with open(fn, 'r') as f: + changed, fromlines = bb.utils.edit_metadata(f, varlist, patch_recipe_varfunc) + # Second run - actually set everything + modifying = True + varlist.extend(recipe_progression_restrs) + changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) + + if remainingnames: + if tolines[-1].strip() != '': + tolines.append('\n') + for k in remainingnames.keys(): + outputvalue(k, tolines) + if patch: - with open(fn, 'U') as f: - fromlines = f.readlines() relfn = os.path.relpath(fn, relpath) diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) - os.remove(tfn) return diff else: with open(fn, 'w') as f: f.writelines(tolines) - os.remove(tfn) return None + def localise_file_vars(fn, varfiles, varlist): """Given a list of variables and variable history (fetched with get_var_files()) find where each variable should be set/changed. This handles for example where a @@ -398,6 +425,8 @@ def validate_pn(pn): return 'Recipe name "%s" is invalid: is a reserved keyword' % pn elif pn.startswith('pn-'): return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn + elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): + return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn return '' diff --git a/yocto-poky/meta/lib/oe/rootfs.py b/yocto-poky/meta/lib/oe/rootfs.py index 18df22d9a..a95e1b739 100644 --- a/yocto-poky/meta/lib/oe/rootfs.py +++ b/yocto-poky/meta/lib/oe/rootfs.py @@ -63,6 +63,15 @@ class Rootfs(object): if 'log_check' in line: continue + if hasattr(self, 'log_check_expected_errors_regexes'): + m = None + for ee in self.log_check_expected_errors_regexes: + m = re.search(ee, line) + if m: + break + if m: + continue + m = r.search(line) if m: found_error = 1 @@ -164,6 +173,7 @@ class Rootfs(object): bb.note("###### Generate rootfs #######") pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND", True) post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND", True) + rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True) postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR", True) if not postinst_intercepts_dir: @@ -193,6 +203,8 @@ class Rootfs(object): with open(sysconfdir + "/version", "w+") as ver: ver.write(self.d.getVar('BUILDNAME', True) + "\n") + execute_pre_post_process(self.d, rootfs_post_install_cmds) + self._run_intercepts() execute_pre_post_process(self.d, post_process_cmds) @@ -229,46 +241,28 @@ class Rootfs(object): self.d.getVar('IMAGE_ROOTFS', True), "run-postinsts", "remove"]) - runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", - True, False, self.d) - sysvcompat_in_distro = bb.utils.contains("DISTRO_FEATURES", [ "systemd", "sysvinit" ], - True, False, self.d) image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", - True, False, self.d) - if sysvcompat_in_distro and not image_rorfs: - pkg_to_remove = "" - else: - pkg_to_remove = "update-rc.d" - if not runtime_pkgmanage: - # Remove components that we don't need if we're not going to install - # additional packages at runtime - if delayed_postinsts is None: - installed_pkgs_dir = self.d.expand('${WORKDIR}/installed_pkgs.txt') - pkgs_to_remove = list() - with open(installed_pkgs_dir, "r+") as installed_pkgs: - pkgs_installed = installed_pkgs.read().splitlines() - for pkg_installed in pkgs_installed[:]: - pkg = pkg_installed.split()[0] - if pkg in ["update-rc.d", - "base-passwd", - "shadow", - "update-alternatives", pkg_to_remove, - self.d.getVar("ROOTFS_BOOTSTRAP_INSTALL", True) - ]: - pkgs_to_remove.append(pkg) - pkgs_installed.remove(pkg_installed) - - if len(pkgs_to_remove) > 0: - self.pm.remove(pkgs_to_remove, False) - # Update installed_pkgs.txt - open(installed_pkgs_dir, "w+").write('\n'.join(pkgs_installed)) - - else: - self._save_postinsts() + True, False, self.d) + if image_rorfs: + # Remove components that we don't need if it's a read-only rootfs + unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED", True).split() + pkgs_installed = image_list_installed_packages(self.d) + pkgs_to_remove = [pkg for pkg in pkgs_installed if pkg in unneeded_pkgs] + + if len(pkgs_to_remove) > 0: + self.pm.remove(pkgs_to_remove, False) + + if delayed_postinsts: + self._save_postinsts() + if image_rorfs: + bb.warn("There are post install scripts " + "in a read-only rootfs") post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND", True) execute_pre_post_process(self.d, post_uninstall_cmds) + runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", + True, False, self.d) if not runtime_pkgmanage: # Remove the package manager data files self.pm.remove_packaging_data() @@ -628,6 +622,10 @@ class DpkgRootfs(DpkgOpkgRootfs): def __init__(self, d, manifest_dir): super(DpkgRootfs, self).__init__(d) self.log_check_regex = '^E:' + self.log_check_expected_errors_regexes = \ + [ + "^E: Unmet dependencies." + ] bb.utils.remove(self.image_rootfs, True) bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True) @@ -882,7 +880,6 @@ class OpkgRootfs(DpkgOpkgRootfs): pkgs_to_install = self.manifest.parse_initial_manifest() opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS', True) opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS', True) - rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND', True) # update PM index files, unless users provide their own feeds if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1": @@ -913,7 +910,6 @@ class OpkgRootfs(DpkgOpkgRootfs): self._setup_dbg_rootfs(['/var/lib/opkg']) execute_pre_post_process(self.d, opkg_post_process_cmds) - execute_pre_post_process(self.d, rootfs_post_install_cmds) if self.inc_opkg_image_gen == "1": self.pm.backup_packaging_data() @@ -941,7 +937,7 @@ class OpkgRootfs(DpkgOpkgRootfs): self._log_check_error() def _cleanup(self): - pass + self.pm.remove_lists() def get_class_for_type(imgtype): return {"rpm": RpmRootfs, @@ -968,17 +964,17 @@ def create_rootfs(d, manifest_dir=None): os.environ.update(env_bkp) -def image_list_installed_packages(d, format=None, rootfs_dir=None): +def image_list_installed_packages(d, rootfs_dir=None): if not rootfs_dir: rootfs_dir = d.getVar('IMAGE_ROOTFS', True) img_type = d.getVar('IMAGE_PKGTYPE', True) if img_type == "rpm": - return RpmPkgsList(d, rootfs_dir).list(format) + return RpmPkgsList(d, rootfs_dir).list_pkgs() elif img_type == "ipk": - return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list(format) + return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET", True)).list_pkgs() elif img_type == "deb": - return DpkgPkgsList(d, rootfs_dir).list(format) + return DpkgPkgsList(d, rootfs_dir).list_pkgs() if __name__ == "__main__": """ diff --git a/yocto-poky/meta/lib/oe/sdk.py b/yocto-poky/meta/lib/oe/sdk.py index 3103f4889..f15fbdb36 100644 --- a/yocto-poky/meta/lib/oe/sdk.py +++ b/yocto-poky/meta/lib/oe/sdk.py @@ -219,17 +219,10 @@ class OpkgSdk(Sdk): pm.update() - pkgs = [] - pkgs_attempt = [] - for pkg_type in pkgs_to_install: - if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY: - pkgs_attempt += pkgs_to_install[pkg_type] - else: - pkgs += pkgs_to_install[pkg_type] - - pm.install(pkgs) - - pm.install(pkgs_attempt, True) + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) def _populate(self): bb.note("Installing TARGET packages") @@ -239,11 +232,15 @@ class OpkgSdk(Sdk): execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND", True)) + self.target_pm.remove_packaging_data() + bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND", True)) + self.host_pm.remove_packaging_data() + target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) @@ -302,17 +299,10 @@ class DpkgSdk(Sdk): pm.write_index() pm.update() - pkgs = [] - pkgs_attempt = [] - for pkg_type in pkgs_to_install: - if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY: - pkgs_attempt += pkgs_to_install[pkg_type] - else: - pkgs += pkgs_to_install[pkg_type] - - pm.install(pkgs) - - pm.install(pkgs_attempt, True) + for pkg_type in self.install_order: + if pkg_type in pkgs_to_install: + pm.install(pkgs_to_install[pkg_type], + [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) def _populate(self): bb.note("Installing TARGET packages") @@ -341,7 +331,7 @@ class DpkgSdk(Sdk): -def sdk_list_installed_packages(d, target, format=None, rootfs_dir=None): +def sdk_list_installed_packages(d, target, rootfs_dir=None): if rootfs_dir is None: sdk_output = d.getVar('SDK_OUTPUT', True) target_path = d.getVar('SDKTARGETSYSROOT', True).strip('/') @@ -352,12 +342,12 @@ def sdk_list_installed_packages(d, target, format=None, rootfs_dir=None): if img_type == "rpm": arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] os_var = ["SDK_OS", None][target is True] - return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list(format) + return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list_pkgs() elif img_type == "ipk": conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True] - return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var, True)).list(format) + return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var, True)).list_pkgs() elif img_type == "deb": - return DpkgPkgsList(d, rootfs_dir).list(format) + return DpkgPkgsList(d, rootfs_dir).list_pkgs() def populate_sdk(d, manifest_dir=None): env_bkp = os.environ.copy() diff --git a/yocto-poky/meta/lib/oe/sstatesig.py b/yocto-poky/meta/lib/oe/sstatesig.py index 6d1be3e37..01dce660c 100644 --- a/yocto-poky/meta/lib/oe/sstatesig.py +++ b/yocto-poky/meta/lib/oe/sstatesig.py @@ -40,7 +40,7 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): # Only target packages beyond here # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes - if isPackageGroup(fn) and isAllArch(fn): + if isPackageGroup(fn) and isAllArch(fn) and not isNative(depname): return False # Exclude well defined machine specific configurations which don't change ABI @@ -65,12 +65,13 @@ def sstate_lockedsigs(d): sigs = {} types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split() for t in types: - lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split() + siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t + lockedsigs = (d.getVar(siggen_lockedsigs_var, True) or "").split() for ls in lockedsigs: pn, task, h = ls.split(":", 2) if pn not in sigs: sigs[pn] = {} - sigs[pn][task] = h + sigs[pn][task] = [h, siggen_lockedsigs_var] return sigs class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): @@ -93,6 +94,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): self.lockedhashfn = {} self.machine = data.getVar("MACHINE", True) self.mismatch_msgs = [] + self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES", True) or + "").split() + self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } pass def tasks_resolved(self, virtmap, virtpnmap, dataCache): @@ -135,17 +139,37 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): recipename = dataCache.pkg_fn[fn] self.lockedpnmap[fn] = recipename self.lockedhashfn[fn] = dataCache.hashfn[fn] - if recipename in self.lockedsigs: + + unlocked = False + if recipename in self.unlockedrecipes: + unlocked = True + else: + def recipename_from_dep(dep): + # The dep entry will look something like + # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, + # ... + fn = dep.rsplit('.', 1)[0] + return dataCache.pkg_fn[fn] + + # If any unlocked recipe is in the direct dependencies then the + # current recipe should be unlocked as well. + depnames = [ recipename_from_dep(x) for x in deps ] + if any(x in y for y in depnames for x in self.unlockedrecipes): + self.unlockedrecipes[recipename] = '' + unlocked = True + + if not unlocked and recipename in self.lockedsigs: if task in self.lockedsigs[recipename]: k = fn + "." + task - h_locked = self.lockedsigs[recipename][task] + h_locked = self.lockedsigs[recipename][task][0] + var = self.lockedsigs[recipename][task][1] self.lockedhashes[k] = h_locked self.taskhash[k] = h_locked #bb.warn("Using %s %s %s" % (recipename, task, h)) if h != h_locked: - self.mismatch_msgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead' - % (recipename, task, h, h_locked)) + self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' + % (recipename, task, h, h_locked, var)) return h_locked #bb.warn("%s %s %s" % (recipename, task, h)) @@ -189,18 +213,35 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys()))) def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): - checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True) + warn_msgs = [] + error_msgs = [] + sstate_missing_msgs = [] + for task in range(len(sq_fn)): if task not in ret: for pn in self.lockedsigs: if sq_hash[task] in self.lockedsigs[pn].itervalues(): - self.mismatch_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" + if sq_task[task] == 'do_shared_workdir': + continue + sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task])) - if self.mismatch_msgs and checklevel == 'warn': - bb.warn("\n".join(self.mismatch_msgs)) - elif self.mismatch_msgs and checklevel == 'error': - bb.fatal("\n".join(self.mismatch_msgs)) + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK", True) + if checklevel == 'warn': + warn_msgs += self.mismatch_msgs + elif checklevel == 'error': + error_msgs += self.mismatch_msgs + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK", True) + if checklevel == 'warn': + warn_msgs += sstate_missing_msgs + elif checklevel == 'error': + error_msgs += sstate_missing_msgs + + if warn_msgs: + bb.warn("\n".join(warn_msgs)) + if error_msgs: + bb.fatal("\n".join(error_msgs)) # Insert these classes into siggen's namespace so it can see and select them @@ -236,6 +277,10 @@ def find_siginfo(pn, taskname, taskhashlist, d): localdata.setVar('PR', '*') localdata.setVar('EXTENDPE', '') stamp = localdata.getVar('STAMP', True) + if pn.startswith("gcc-source"): + # gcc-source shared workdir is a special case :( + stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") + filespec = '%s.%s.sigdata.*' % (stamp, taskname) foundall = False import glob diff --git a/yocto-poky/meta/lib/oe/terminal.py b/yocto-poky/meta/lib/oe/terminal.py index 52a891388..634daa903 100644 --- a/yocto-poky/meta/lib/oe/terminal.py +++ b/yocto-poky/meta/lib/oe/terminal.py @@ -83,7 +83,7 @@ class Terminology(XTerminal): priority = 2 class Konsole(XTerminal): - command = 'konsole --nofork -p tabtitle="{title}" -e {command}' + command = 'konsole --nofork --workdir . -p tabtitle="{title}" -e {command}' priority = 2 def __init__(self, sh_cmd, title=None, env=None, d=None): @@ -131,7 +131,7 @@ class TmuxRunning(Terminal): raise UnsupportedTerminal('tmux is not running') if not check_tmux_pane_size('tmux'): - raise UnsupportedTerminal('tmux pane too small') + raise UnsupportedTerminal('tmux pane too small or tmux < 1.9 version is being used') Terminal.__init__(self, sh_cmd, title, env, d) @@ -218,6 +218,12 @@ def spawn(name, sh_cmd, title=None, env=None, d=None): def check_tmux_pane_size(tmux): import subprocess as sub + # On older tmux versions (<1.9), return false. The reason + # is that there is no easy way to get the height of the active panel + # on current window without nested formats (available from version 1.9) + vernum = check_terminal_version("tmux") + if vernum and LooseVersion(vernum) < '1.9': + return False try: p = sub.Popen('%s list-panes -F "#{?pane_active,#{pane_height},}"' % tmux, shell=True,stdout=sub.PIPE,stderr=sub.PIPE) @@ -229,14 +235,18 @@ def check_tmux_pane_size(tmux): return None else: raise - if size/2 >= 19: - return True - return False + + return size/2 >= 19 def check_terminal_version(terminalName): import subprocess as sub try: - p = sub.Popen(['sh', '-c', '%s --version' % terminalName],stdout=sub.PIPE,stderr=sub.PIPE) + cmdversion = '%s --version' % terminalName + if terminalName.startswith('tmux'): + cmdversion = '%s -V' % terminalName + newenv = os.environ.copy() + newenv["LANG"] = "C" + p = sub.Popen(['sh', '-c', cmdversion], stdout=sub.PIPE, stderr=sub.PIPE, env=newenv) out, err = p.communicate() ver_info = out.rstrip().split('\n') except OSError as exc: @@ -251,6 +261,8 @@ def check_terminal_version(terminalName): vernum = ver.split(' ')[-1] if ver.startswith('GNOME Terminal'): vernum = ver.split(' ')[-1] + if ver.startswith('tmux'): + vernum = ver.split()[-1] return vernum def distro_name(): diff --git a/yocto-poky/meta/lib/oe/utils.py b/yocto-poky/meta/lib/oe/utils.py index cee087fdf..30d30629f 100644 --- a/yocto-poky/meta/lib/oe/utils.py +++ b/yocto-poky/meta/lib/oe/utils.py @@ -208,6 +208,28 @@ def squashspaces(string): import re return re.sub("\s+", " ", string).strip() +def format_pkg_list(pkg_dict, ret_format=None): + output = [] + + if ret_format == "arch": + for pkg in sorted(pkg_dict): + output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"])) + elif ret_format == "file": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"])) + elif ret_format == "ver": + for pkg in sorted(pkg_dict): + output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"])) + elif ret_format == "deps": + for pkg in sorted(pkg_dict): + for dep in pkg_dict[pkg]["deps"]: + output.append("%s|%s" % (pkg, dep)) + else: + for pkg in sorted(pkg_dict): + output.append(pkg) + + return '\n'.join(output) + # # Python 2.7 doesn't have threaded pools (just multiprocessing) # so implement a version here @@ -271,3 +293,14 @@ class ThreadedPool: self.tasks.join() for worker in self.workers: worker.join() + +def write_ld_so_conf(d): + # Some utils like prelink may not have the correct target library paths + # so write an ld.so.conf to help them + ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf") + if os.path.exists(ldsoconf): + bb.utils.remove(ldsoconf) + bb.utils.mkdirhier(os.path.dirname(ldsoconf)) + with open(ldsoconf, "w") as f: + f.write(d.getVar("base_libdir", True) + '\n') + f.write(d.getVar("libdir", True) + '\n') diff --git a/yocto-poky/meta/lib/oeqa/oetest.py b/yocto-poky/meta/lib/oeqa/oetest.py index 6f9edec58..3ed5bb8c2 100644 --- a/yocto-poky/meta/lib/oeqa/oetest.py +++ b/yocto-poky/meta/lib/oeqa/oetest.py @@ -7,16 +7,25 @@ # It also has some helper functions and it's responsible for actually starting the tests -import os, re, mmap +import os, re, mmap, sys import unittest import inspect import subprocess +import signal try: import bb except ImportError: pass import logging + +import oeqa.runtime +# Exported test doesn't require sdkext +try: + import oeqa.sdkext +except ImportError: + pass from oeqa.utils.decorators import LogResults, gettag, getResults +from oeqa.utils import avoid_paths_in_environ logger = logging.getLogger("BitBake") @@ -30,7 +39,6 @@ def getVar(obj): def checkTags(tc, tagexp): return eval(tagexp, None, getVar(tc)) - def filterByTagExp(testsuite, tagexp): if not tagexp: return testsuite @@ -43,106 +51,6 @@ def filterByTagExp(testsuite, tagexp): caseList.append(filterByTagExp(each, tagexp)) return testsuite.__class__(caseList) -def loadTests(tc, type="runtime"): - if type == "runtime": - # set the context object passed from the test class - setattr(oeTest, "tc", tc) - # set ps command to use - setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") - # prepare test suite, loader and runner - suite = unittest.TestSuite() - elif type == "sdk": - # set the context object passed from the test class - setattr(oeTest, "tc", tc) - testloader = unittest.TestLoader() - testloader.sortTestMethodsUsing = None - suites = [testloader.loadTestsFromName(name) for name in tc.testslist] - suites = filterByTagExp(suites, getattr(tc, "tagexp", None)) - - def getTests(test): - '''Return all individual tests executed when running the suite.''' - # Unfortunately unittest does not have an API for this, so we have - # to rely on implementation details. This only needs to work - # for TestSuite containing TestCase. - method = getattr(test, '_testMethodName', None) - if method: - # leaf case: a TestCase - yield test - else: - # Look into TestSuite. - tests = getattr(test, '_tests', []) - for t1 in tests: - for t2 in getTests(t1): - yield t2 - - # Determine dependencies between suites by looking for @skipUnlessPassed - # method annotations. Suite A depends on suite B if any method in A - # depends on a method on B. - for suite in suites: - suite.dependencies = [] - suite.depth = 0 - for test in getTests(suite): - methodname = getattr(test, '_testMethodName', None) - if methodname: - method = getattr(test, methodname) - depends_on = getattr(method, '_depends_on', None) - if depends_on: - for dep_suite in suites: - if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]: - if dep_suite not in suite.dependencies and \ - dep_suite is not suite: - suite.dependencies.append(dep_suite) - break - else: - logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % - (test, depends_on)) - # Use brute-force topological sort to determine ordering. Sort by - # depth (higher depth = must run later), with original ordering to - # break ties. - def set_suite_depth(suite): - for dep in suite.dependencies: - new_depth = set_suite_depth(dep) + 1 - if new_depth > suite.depth: - suite.depth = new_depth - return suite.depth - for index, suite in enumerate(suites): - set_suite_depth(suite) - suite.index = index - suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) - return testloader.suiteClass(suites) - -_buffer = "" - -def custom_verbose(msg, *args, **kwargs): - global _buffer - if msg[-1] != "\n": - _buffer += msg - else: - _buffer += msg - try: - bb.plain(_buffer.rstrip("\n"), *args, **kwargs) - except NameError: - logger.info(_buffer.rstrip("\n"), *args, **kwargs) - _buffer = "" - -def runTests(tc, type="runtime"): - - suite = loadTests(tc, type) - logger.info("Test modules %s" % tc.testslist) - if hasattr(tc, "tagexp") and tc.tagexp: - logger.info("Filter test cases by tags: %s" % tc.tagexp) - logger.info("Found %s tests" % suite.countTestCases()) - runner = unittest.TextTestRunner(verbosity=2) - try: - if bb.msg.loggerDefaultVerbose: - runner.stream.write = custom_verbose - except NameError: - # Not in bb environment? - pass - result = runner.run(suite) - - return result - @LogResults class oeTest(unittest.TestCase): @@ -222,7 +130,19 @@ class oeSDKTest(oeTest): return False def _run(self, cmd): - return subprocess.check_output(". %s; " % self.tc.sdkenv + cmd, shell=True) + return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True) + +class oeSDKExtTest(oeSDKTest): + def _run(self, cmd): + # extensible sdk shows a warning if found bitbake in the path + # because can cause contamination, i.e. use devtool from + # poky/scripts instead of eSDK one. + env = os.environ.copy() + paths_to_avoid = ['bitbake/bin', 'poky/scripts'] + env['PATH'] = avoid_paths_in_environ(paths_to_avoid) + + return subprocess.check_output(". %s > /dev/null;"\ + " %s;" % (self.tc.sdkenv, cmd), shell=True, env=env) def getmodule(pos=2): # stack returns a list of tuples containg frame information @@ -250,3 +170,278 @@ def skipModuleUnless(cond, reason): if not cond: skipModule(reason, 3) + +_buffer_logger = "" +def custom_verbose(msg, *args, **kwargs): + global _buffer_logger + if msg[-1] != "\n": + _buffer_logger += msg + else: + _buffer_logger += msg + try: + bb.plain(_buffer_logger.rstrip("\n"), *args, **kwargs) + except NameError: + logger.info(_buffer_logger.rstrip("\n"), *args, **kwargs) + _buffer_logger = "" + +class TestContext(object): + def __init__(self, d): + self.d = d + + self.testsuites = self._get_test_suites() + self.testslist = self._get_tests_list(d.getVar("BBPATH", True).split(':')) + self.testsrequired = self._get_test_suites_required() + + self.filesdir = os.path.join(os.path.dirname(os.path.abspath( + oeqa.runtime.__file__)), "files") + self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() + self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() + + # get testcase list from specified file + # if path is a relative path, then relative to build/conf/ + def _read_testlist(self, fpath, builddir): + if not os.path.isabs(fpath): + fpath = os.path.join(builddir, "conf", fpath) + if not os.path.exists(fpath): + bb.fatal("No such manifest file: ", fpath) + tcs = [] + for line in open(fpath).readlines(): + line = line.strip() + if line and not line.startswith("#"): + tcs.append(line) + return " ".join(tcs) + + # return test list by type also filter if TEST_SUITES is specified + def _get_tests_list(self, bbpath): + testslist = [] + + type = self._get_test_namespace() + + # This relies on lib/ under each directory in BBPATH being added to sys.path + # (as done by default in base.bbclass) + for testname in self.testsuites: + if testname != "auto": + if testname.startswith("oeqa."): + testslist.append(testname) + continue + found = False + for p in bbpath: + if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')): + testslist.append("oeqa." + type + "." + testname) + found = True + break + elif os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname.split(".")[0] + '.py')): + testslist.append("oeqa." + type + "." + testname) + found = True + break + if not found: + bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname) + + if "auto" in self.testsuites: + def add_auto_list(path): + if not os.path.exists(os.path.join(path, '__init__.py')): + bb.fatal('Tests directory %s exists but is missing __init__.py' % path) + files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) + for f in files: + module = 'oeqa.' + type + '.' + f[:-3] + if module not in testslist: + testslist.append(module) + + for p in bbpath: + testpath = os.path.join(p, 'lib', 'oeqa', type) + bb.debug(2, 'Searching for tests in %s' % testpath) + if os.path.exists(testpath): + add_auto_list(testpath) + + return testslist + + def loadTests(self): + setattr(oeTest, "tc", self) + + testloader = unittest.TestLoader() + testloader.sortTestMethodsUsing = None + suites = [testloader.loadTestsFromName(name) for name in self.testslist] + suites = filterByTagExp(suites, getattr(self, "tagexp", None)) + + def getTests(test): + '''Return all individual tests executed when running the suite.''' + # Unfortunately unittest does not have an API for this, so we have + # to rely on implementation details. This only needs to work + # for TestSuite containing TestCase. + method = getattr(test, '_testMethodName', None) + if method: + # leaf case: a TestCase + yield test + else: + # Look into TestSuite. + tests = getattr(test, '_tests', []) + for t1 in tests: + for t2 in getTests(t1): + yield t2 + + # Determine dependencies between suites by looking for @skipUnlessPassed + # method annotations. Suite A depends on suite B if any method in A + # depends on a method on B. + for suite in suites: + suite.dependencies = [] + suite.depth = 0 + for test in getTests(suite): + methodname = getattr(test, '_testMethodName', None) + if methodname: + method = getattr(test, methodname) + depends_on = getattr(method, '_depends_on', None) + if depends_on: + for dep_suite in suites: + if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]: + if dep_suite not in suite.dependencies and \ + dep_suite is not suite: + suite.dependencies.append(dep_suite) + break + else: + logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % + (test, depends_on)) + + # Use brute-force topological sort to determine ordering. Sort by + # depth (higher depth = must run later), with original ordering to + # break ties. + def set_suite_depth(suite): + for dep in suite.dependencies: + new_depth = set_suite_depth(dep) + 1 + if new_depth > suite.depth: + suite.depth = new_depth + return suite.depth + + for index, suite in enumerate(suites): + set_suite_depth(suite) + suite.index = index + suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) + + self.suite = testloader.suiteClass(suites) + + return self.suite + + def runTests(self): + logger.info("Test modules %s" % self.testslist) + if hasattr(self, "tagexp") and self.tagexp: + logger.info("Filter test cases by tags: %s" % self.tagexp) + logger.info("Found %s tests" % self.suite.countTestCases()) + runner = unittest.TextTestRunner(verbosity=2) + if 'bb' in sys.modules: + runner.stream.write = custom_verbose + + return runner.run(self.suite) + +class ImageTestContext(TestContext): + def __init__(self, d, target, host_dumper): + super(ImageTestContext, self).__init__(d) + + self.tagexp = d.getVar("TEST_SUITES_TAGS", True) + + self.target = target + self.host_dumper = host_dumper + + manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), + d.getVar("IMAGE_LINK_NAME", True) + ".manifest") + nomanifest = d.getVar("IMAGE_NO_MANIFEST", True) + if nomanifest is None or nomanifest != "1": + try: + with open(manifest) as f: + self.pkgmanifest = f.read() + except IOError as e: + bb.fatal("No package manifest file found. Did you build the image?\n%s" % e) + else: + self.pkgmanifest = "" + + self.sigterm = False + self.origsigtermhandler = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, self._sigterm_exception) + + def _sigterm_exception(self, signum, stackframe): + bb.warn("TestImage received SIGTERM, shutting down...") + self.sigterm = True + self.target.stop() + + def _get_test_namespace(self): + return "runtime" + + def _get_test_suites(self): + testsuites = [] + + manifests = (self.d.getVar("TEST_SUITES_MANIFEST", True) or '').split() + if manifests: + for manifest in manifests: + testsuites.extend(self._read_testlist(manifest, + self.d.getVar("TOPDIR", True)).split()) + + else: + testsuites = self.d.getVar("TEST_SUITES", True).split() + + return testsuites + + def _get_test_suites_required(self): + return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] + + def loadTests(self): + super(ImageTestContext, self).loadTests() + setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") + +class SDKTestContext(TestContext): + def __init__(self, d, sdktestdir, sdkenv, tcname, *args): + super(SDKTestContext, self).__init__(d) + + self.sdktestdir = sdktestdir + self.sdkenv = sdkenv + self.tcname = tcname + + if not hasattr(self, 'target_manifest'): + self.target_manifest = d.getVar("SDK_TARGET_MANIFEST", True) + try: + with open(self.target_manifest) as f: + self.pkgmanifest = f.read() + except IOError as e: + bb.fatal("No package manifest file found. Did you build the sdk image?\n%s" % e) + + if not hasattr(self, 'host_manifest'): + self.host_manifest = d.getVar("SDK_HOST_MANIFEST", True) + try: + with open(self.host_manifest) as f: + self.hostpkgmanifest = f.read() + except IOError as e: + bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e) + + def _get_test_namespace(self): + return "sdk" + + def _get_test_suites(self): + return (self.d.getVar("TEST_SUITES_SDK", True) or "auto").split() + + def _get_test_suites_required(self): + return [t for t in (self.d.getVar("TEST_SUITES_SDK", True) or \ + "auto").split() if t != "auto"] + +class SDKExtTestContext(SDKTestContext): + def __init__(self, d, sdktestdir, sdkenv, tcname, *args): + self.target_manifest = d.getVar("SDK_EXT_TARGET_MANIFEST", True) + self.host_manifest = d.getVar("SDK_EXT_HOST_MANIFEST", True) + if args: + self.cm = args[0] # Compatibility mode for run SDK tests + else: + self.cm = False + + super(SDKExtTestContext, self).__init__(d, sdktestdir, sdkenv, tcname) + + self.sdkextfilesdir = os.path.join(os.path.dirname(os.path.abspath( + oeqa.sdkext.__file__)), "files") + + def _get_test_namespace(self): + if self.cm: + return "sdk" + else: + return "sdkext" + + def _get_test_suites(self): + return (self.d.getVar("TEST_SUITES_SDK_EXT", True) or "auto").split() + + def _get_test_suites_required(self): + return [t for t in (self.d.getVar("TEST_SUITES_SDK_EXT", True) or \ + "auto").split() if t != "auto"] diff --git a/yocto-poky/meta/lib/oeqa/runexported.py b/yocto-poky/meta/lib/oeqa/runexported.py index dba0d7aec..cc89e13c0 100755 --- a/yocto-poky/meta/lib/oeqa/runexported.py +++ b/yocto-poky/meta/lib/oeqa/runexported.py @@ -30,7 +30,7 @@ except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa"))) -from oeqa.oetest import runTests +from oeqa.oetest import TestContext from oeqa.utils.sshcontrol import SSHControl from oeqa.utils.dump import get_host_dumper @@ -49,7 +49,7 @@ class FakeTarget(object): def exportStart(self): self.sshlog = os.path.join(self.testdir, "ssh_target_log.%s" % self.datetime) sshloglink = os.path.join(self.testdir, "ssh_target_log") - if os.path.exists(sshloglink): + if os.path.lexists(sshloglink): os.remove(sshloglink) os.symlink(self.sshlog, sshloglink) print("SSH log file: %s" % self.sshlog) @@ -69,10 +69,9 @@ class MyDataDict(dict): def getVar(self, key, unused = None): return self.get(key, "") -class TestContext(object): - def __init__(self): - self.d = None - self.target = None +class ExportTestContext(TestContext): + def __init__(self, d): + self.d = d def main(): @@ -121,7 +120,9 @@ def main(): host_dumper.parent_dir = loaded["host_dumper"]["parent_dir"] host_dumper.cmds = loaded["host_dumper"]["cmds"] - tc = TestContext() + target.exportStart() + tc = ExportTestContext(d) + setattr(tc, "d", d) setattr(tc, "target", target) setattr(tc, "host_dumper", host_dumper) @@ -129,8 +130,8 @@ def main(): if key != "d" and key != "target" and key != "host_dumper": setattr(tc, key, loaded[key]) - target.exportStart() - runTests(tc) + tc.loadTests() + tc.runTests() return 0 @@ -140,5 +141,5 @@ if __name__ == "__main__": except Exception: ret = 1 import traceback - traceback.print_exc(5) + traceback.print_exc() sys.exit(ret) diff --git a/yocto-poky/meta/lib/oeqa/runtime/dmesg.py b/yocto-poky/meta/lib/oeqa/runtime/dmesg.py deleted file mode 100644 index 5831471e5..000000000 --- a/yocto-poky/meta/lib/oeqa/runtime/dmesg.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -from oeqa.oetest import oeRuntimeTest -from oeqa.utils.decorators import * - - -class DmesgTest(oeRuntimeTest): - - @testcase(215) - @skipUnlessPassed('test_ssh') - def test_dmesg(self): - (status, output) = self.target.run('dmesg | grep -v mmci-pl18x | grep -v "error changing net interface name" | grep -iv "dma timeout" | grep -v usbhid | grep -i error') - self.assertEqual(status, 1, msg = "Error messages in dmesg log: %s" % output) diff --git a/yocto-poky/meta/lib/oeqa/runtime/logrotate.py b/yocto-poky/meta/lib/oeqa/runtime/logrotate.py index 86d791c30..de300bf55 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/logrotate.py +++ b/yocto-poky/meta/lib/oeqa/runtime/logrotate.py @@ -14,9 +14,9 @@ class LogrotateTest(oeRuntimeTest): @skipUnlessPassed("test_ssh") def test_1_logrotate_setup(self): - (status, output) = self.target.run('mkdir /home/root/logrotate_dir') + (status, output) = self.target.run('mkdir $HOME/logrotate_dir') self.assertEqual(status, 0, msg = "Could not create logrotate_dir. Output: %s" % output) - (status, output) = self.target.run("sed -i 's#wtmp {#wtmp {\\n olddir /home/root/logrotate_dir#' /etc/logrotate.conf") + (status, output) = self.target.run("sed -i \"s#wtmp {#wtmp {\\n olddir $HOME/logrotate_dir#\" /etc/logrotate.conf") self.assertEqual(status, 0, msg = "Could not write to logrotate.conf file. Status and output: %s and %s)" % (status, output)) @testcase(289) @@ -24,5 +24,5 @@ class LogrotateTest(oeRuntimeTest): def test_2_logrotate(self): (status, output) = self.target.run('logrotate -f /etc/logrotate.conf') self.assertEqual(status, 0, msg = "logrotate service could not be reloaded. Status and output: %s and %s" % (status, output)) - output = self.target.run('ls -la /home/root/logrotate_dir/ | wc -l')[1] - self.assertTrue(int(output)>=3, msg = "new logfile could not be created. List of files within log directory: %s" %(self.target.run('ls -la /home/root/logrotate_dir')[1])) + output = self.target.run('ls -la $HOME/logrotate_dir/ | wc -l')[1] + self.assertTrue(int(output)>=3, msg = "new logfile could not be created. List of files within log directory: %s" %(self.target.run('ls -la $HOME/logrotate_dir')[1])) diff --git a/yocto-poky/meta/lib/oeqa/runtime/multilib.py b/yocto-poky/meta/lib/oeqa/runtime/multilib.py index e1bcc428f..593d38502 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/multilib.py +++ b/yocto-poky/meta/lib/oeqa/runtime/multilib.py @@ -10,39 +10,33 @@ def setUpModule(): class MultilibTest(oeRuntimeTest): - def parse(self, s): + def archtest(self, binary, arch): """ - Parse the output of readelf -h and return the binary class, or fail. + Check that ``binary`` has the ELF class ``arch`` (e.g. ELF32/ELF64). """ - l = [l.split()[1] for l in s.split('\n') if "Class:" in l] + + (status, output) = self.target.run("readelf -h %s" % binary) + self.assertEqual(status, 0, "Failed to readelf %s" % binary) + + l = [l.split()[1] for l in output.split('\n') if "Class:" in l] if l: - return l[0] + theclass = l[0] else: self.fail("Cannot parse readelf output\n" + s) + self.assertEqual(theclass, arch, msg="%s isn't %s (is %s)" % (binary, arch, theclass)) + @skipUnlessPassed('test_ssh') def test_check_multilib_libc(self): """ Check that a multilib image has both 32-bit and 64-bit libc in. """ - - (status, output) = self.target.run("readelf -h /lib/libc.so.6") - self.assertEqual(status, 0, "Failed to readelf /lib/libc.so.6") - class32 = self.parse(output) - - (status, output) = self.target.run("readelf -h /lib64/libc.so.6") - self.assertEqual(status, 0, "Failed to readelf /lib64/libc.so.6") - class64 = self.parse(output) - - self.assertEqual(class32, "ELF32", msg="/lib/libc.so.6 isn't ELF32 (is %s)" % class32) - self.assertEqual(class64, "ELF64", msg="/lib64/libc.so.6 isn't ELF64 (is %s)" % class64) + self.archtest("/lib/libc.so.6", "ELF32") + self.archtest("/lib64/libc.so.6", "ELF64") @testcase('279') @skipUnlessPassed('test_check_multilib_libc') def test_file_connman(self): - self.assertTrue(oeRuntimeTest.hasPackage('lib32-connman-gnome'), msg="This test assumes lib32-connman-gnome is installed") + self.assertTrue(oeRuntimeTest.hasPackage('lib32-connman'), msg="This test assumes lib32-connman is installed") - (status, output) = self.target.run("readelf -h /usr/bin/connman-applet") - self.assertEqual(status, 0, "Failed to readelf /usr/bin/connman-applet") - theclass = self.parse(output) - self.assertEqual(theclass, "ELF32", msg="connman-applet isn't ELF32 (is %s)" % theclass) + self.archtest("/usr/sbin/connmand", "ELF32") diff --git a/yocto-poky/meta/lib/oeqa/runtime/parselogs.py b/yocto-poky/meta/lib/oeqa/runtime/parselogs.py index fc2bc3893..dec9ebe87 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/parselogs.py +++ b/yocto-poky/meta/lib/oeqa/runtime/parselogs.py @@ -38,13 +38,23 @@ common_errors = [ 'Online check failed for', 'netlink init failed', 'Fast TSC calibration', + "BAR 0-9", + "Failed to load module \"ati\"", + "controller can't do DEVSLP, turning off", + "stmmac_dvr_probe: warning: cannot get CSR clock", + "error: couldn\'t mount because of unsupported optional features", ] +video_related = [ + "uvesafb", +] + x86_common = [ '[drm:psb_do_init] *ERROR* Debug is', 'wrong ELF class', 'Could not enable PowerButton event', 'probe of LNXPWRBN:00 failed with error -22', + 'pmd_set_huge: Cannot satisfy', ] + common_errors qemux86_common = [ @@ -80,6 +90,7 @@ ignore_errors = { 'qemuarm64' : [ 'Fatal server error:', '(EE) Server terminated with error (1). Closing log file.', + 'dmi: Firmware registration failed.', ] + common_errors, 'emenlow' : [ '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', @@ -99,11 +110,8 @@ ignore_errors = { '(EE) Failed to load module psbdrv', '(EE) open /dev/fb0: No such file or directory', '(EE) AIGLX: reverting to software rendering', - "controller can't do DEVSLP, turning off", ] + x86_common, - 'intel-corei7-64' : [ - "controller can't do DEVSLP, turning off", - ] + common_errors, + 'intel-corei7-64' : x86_common, 'crownbay' : x86_common, 'genericx86' : x86_common, 'genericx86-64' : x86_common, @@ -123,9 +131,24 @@ class ParseLogsTest(oeRuntimeTest): @classmethod def setUpClass(self): self.errors = errors + + # When systemd is enabled we need to notice errors on + # circular dependencies in units. + if self.hasFeature("systemd"): + self.errors.extend([ + 'Found ordering cycle on', + 'Breaking ordering cycle by deleting job', + 'deleted to break ordering cycle', + 'Ordering cycle found, skipping', + ]) + self.ignore_errors = ignore_errors self.log_locations = log_locations self.msg = "" + (is_lsb, location) = oeRuntimeTest.tc.target.run("which LSB_Test.sh") + if is_lsb == 0: + for machine in self.ignore_errors: + self.ignore_errors[machine] = self.ignore_errors[machine] + video_related def getMachine(self): return oeRuntimeTest.tc.d.getVar("MACHINE", True) @@ -200,6 +223,7 @@ class ParseLogsTest(oeRuntimeTest): ignore_error = ignore_error.replace("[", "\[") ignore_error = ignore_error.replace("]", "\]") ignore_error = ignore_error.replace("*", "\*") + ignore_error = ignore_error.replace("0-9", "[0-9]") grepcmd += ignore_error+"|" grepcmd = grepcmd[:-1] grepcmd += "\'" @@ -221,9 +245,8 @@ class ParseLogsTest(oeRuntimeTest): results[log.replace('target_logs/','')] = {} rez = result.splitlines() for xrez in rez: - command = "grep \"\\"+str(xrez)+"\" -B "+str(lines_before)+" -A "+str(lines_after)+" "+str(log) try: - grep_output = subprocess.check_output(command, shell=True) + grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]) except: pass results[log.replace('target_logs/','')][xrez]=grep_output diff --git a/yocto-poky/meta/lib/oeqa/runtime/rpm.py b/yocto-poky/meta/lib/oeqa/runtime/rpm.py index 32aae2423..624c515aa 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/rpm.py +++ b/yocto-poky/meta/lib/oeqa/runtime/rpm.py @@ -52,11 +52,11 @@ class RpmInstallRemoveTest(oeRuntimeTest): @skipUnlessPassed('test_ssh') def test_rpm_query_nonroot(self): (status, output) = self.target.run('useradd test1') - self.assertTrue(status == 0, msg="Failed to create new user") + self.assertTrue(status == 0, msg="Failed to create new user: " + output) (status, output) = self.target.run('sudo -u test1 id') self.assertTrue('(test1)' in output, msg="Failed to execute as new user") (status, output) = self.target.run('sudo -u test1 rpm -qa') - self.assertEqual(status, 0, msg="status: %s. Cannot run rpm -qa" % status) + self.assertEqual(status, 0, msg="status: %s. Cannot run rpm -qa: %s" % (status, output)) @testcase(195) @skipUnlessPassed('test_rpm_install') diff --git a/yocto-poky/meta/lib/oeqa/runtime/smart.py b/yocto-poky/meta/lib/oeqa/runtime/smart.py index e41668d26..126d61463 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/smart.py +++ b/yocto-poky/meta/lib/oeqa/runtime/smart.py @@ -147,7 +147,7 @@ class SmartRepoTest(SmartTest): for i in output.split("\n"): if ("rpmsys" != str(i)) and ("myrpmdir" != str(i)): self.smart('channel --disable '+str(i)) - self.target.run('cd /home/root') + self.target.run('cd $HOME') self.smart('install psplash') for i in output.split("\n"): if ("rpmsys" != str(i)) and ("myrpmdir" != str(i)): @@ -172,4 +172,4 @@ class SmartRepoTest(SmartTest): @skipUnlessPassed('test_smart_channel_add') def test_smart_remove_package(self): self.smart('install -y psplash') - self.smart('remove -y psplash')
\ No newline at end of file + self.smart('remove -y psplash') diff --git a/yocto-poky/meta/lib/oeqa/runtime/systemd.py b/yocto-poky/meta/lib/oeqa/runtime/systemd.py index c74394ca5..2b2f10d71 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/systemd.py +++ b/yocto-poky/meta/lib/oeqa/runtime/systemd.py @@ -21,6 +21,34 @@ class SystemdTest(oeRuntimeTest): self.assertEqual(status, expected, message) return output + #TODO: use pyjournalctl instead + def journalctl(self, args='',l_match_units=[]): + """ + Request for the journalctl output to the current target system + + Arguments: + -args, an optional argument pass through argument + -l_match_units, an optional list of units to filter the output + Returns: + -string output of the journalctl command + Raises: + -AssertionError, on remote commands that fail + -ValueError, on a journalctl call with filtering by l_match_units that + returned no entries + """ + query_units="" + if len(l_match_units): + query_units = ['_SYSTEMD_UNIT='+unit for unit in l_match_units] + query_units = " ".join(query_units) + command = 'journalctl %s %s' %(args, query_units) + status, output = self.target.run(command) + if status: + raise AssertionError("Command '%s' returned non-zero exit \ + code %d:\n%s" % (command, status, output)) + if len(output) == 1 and "-- No entries --" in output: + raise ValueError("List of units to match: %s, returned no entries" + % l_match_units) + return output class SystemdBasicTests(SystemdTest): @@ -99,3 +127,52 @@ class SystemdJournalTests(SystemdTest): def test_systemd_journal(self): (status, output) = self.target.run('journalctl') self.assertEqual(status, 0, output) + + @skipUnlessPassed('test_systemd_basic') + def test_systemd_boot_time(self, systemd_TimeoutStartSec=90): + """ + Get the target boot time from journalctl and log it + + Arguments: + -systemd_TimeoutStartSec, an optional argument containing systemd's + unit start timeout to compare against + """ + + # the expression chain that uniquely identifies the time boot message + expr_items=["Startup finished","kernel", "userspace","\.$"] + try: + output = self.journalctl(args="-o cat --reverse") + except AssertionError: + self.fail("Error occurred while calling journalctl") + if not len(output): + self.fail("Error, unable to get startup time from systemd journal") + + # check for the regular expression items that match the startup time + for line in output.split('\n'): + check_match = "".join(re.findall(".*".join(expr_items), line)) + if check_match: break + # put the startup time in the test log + if check_match: + print "%s" % check_match + else: + self.skipTest("Error at obtaining the boot time from journalctl") + boot_time_sec = 0 + + # get the numeric values from the string and convert them to seconds + # same data will be placed in list and string for manipulation + l_boot_time = check_match.split(" ")[-2:] + s_boot_time = " ".join(l_boot_time) + try: + # Obtain the minutes it took to boot + if l_boot_time[0].endswith('min') and l_boot_time[0][0].isdigit(): + boot_time_min = s_boot_time.split("min")[0] + # convert to seconds and accumulate it + boot_time_sec += int(boot_time_min) * 60 + # Obtain the seconds it took to boot and accumulate + boot_time_sec += float(l_boot_time[1].split("s")[0]) + except ValueError: + self.skipTest("Error when parsing time from boot string") + #Assert the target boot time against systemd's unit start timeout + if boot_time_sec > systemd_TimeoutStartSec: + print "Target boot time %s exceeds systemd's TimeoutStartSec %s"\ + %(boot_time_sec, systemd_TimeoutStartSec) diff --git a/yocto-poky/meta/lib/oeqa/runtime/vnc.py b/yocto-poky/meta/lib/oeqa/runtime/vnc.py deleted file mode 100644 index f31deff30..000000000 --- a/yocto-poky/meta/lib/oeqa/runtime/vnc.py +++ /dev/null @@ -1,20 +0,0 @@ -from oeqa.oetest import oeRuntimeTest, skipModuleUnless -from oeqa.utils.decorators import * -import re - -def setUpModule(): - skipModuleUnless(oeRuntimeTest.hasPackage('x11vnc'), "No x11vnc package in image") - -class VNCTest(oeRuntimeTest): - - @testcase(213) - @skipUnlessPassed('test_ssh') - def test_vnc(self): - (status, output) = self.target.run('x11vnc -display :0 -bg -o x11vnc.log') - self.assertEqual(status, 0, msg="x11vnc server failed to start: %s" % output) - port = re.search('PORT=[0-9]*', output) - self.assertTrue(port, msg="Listening port not specified in command output: %s" %output) - - vncport = port.group(0).split('=')[1] - (status, output) = self.target.run('netstat -ntl | grep ":%s"' % vncport) - self.assertEqual(status, 0, msg="x11vnc server not running on port %s\n\n%s" % (vncport, self.target.run('netstat -ntl; cat x11vnc.log')[1])) diff --git a/yocto-poky/meta/lib/oeqa/sdkext/__init__.py b/yocto-poky/meta/lib/oeqa/sdkext/__init__.py new file mode 100644 index 000000000..4cf3fa76b --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/sdkext/__init__.py @@ -0,0 +1,3 @@ +# Enable other layers to have tests in the same named directory +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/yocto-poky/meta/lib/oeqa/sdkext/devtool.py b/yocto-poky/meta/lib/oeqa/sdkext/devtool.py new file mode 100644 index 000000000..c5bb3102a --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/sdkext/devtool.py @@ -0,0 +1,32 @@ +import shutil + +from oeqa.oetest import oeSDKExtTest +from oeqa.utils.decorators import * + +class DevtoolTest(oeSDKExtTest): + + @classmethod + def setUpClass(self): + self.myapp_src = os.path.join(self.tc.sdkextfilesdir, "myapp") + self.myapp_dst = os.path.join(self.tc.sdktestdir, "myapp") + shutil.copytree(self.myapp_src, self.myapp_dst) + + def test_devtool_location(self): + output = self._run('which devtool') + self.assertEqual(output.startswith(self.tc.sdktestdir), True, \ + msg="Seems that devtool isn't the eSDK one: %s" % output) + + @skipUnlessPassed('test_devtool_location') + def test_devtool_add_reset(self): + self._run('devtool add myapp %s' % self.myapp_dst) + self._run('devtool reset myapp') + + @skipUnlessPassed('test_devtool_location') + def test_devtool_build(self): + self._run('devtool add myapp %s' % self.myapp_dst) + self._run('devtool build myapp') + self._run('devtool reset myapp') + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.myapp_dst) diff --git a/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/Makefile b/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/Makefile new file mode 100644 index 000000000..abd91bea6 --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/Makefile @@ -0,0 +1,10 @@ +all: myapp + +myapp: myapp.o + $(CC) $(LDFLAGS) $< -o $@ + +myapp.o: myapp.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + rm -rf myapp.o myapp diff --git a/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/myapp.c b/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/myapp.c new file mode 100644 index 000000000..f0b63f03f --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/sdkext/files/myapp/myapp.c @@ -0,0 +1,9 @@ +#include <stdio.h> + +int +main(int argc, char *argv[]) +{ + printf("Hello world\n"); + + return 0; +} diff --git a/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py b/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py new file mode 100644 index 000000000..7a2a6fe7c --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/sdkext/sdk_update.py @@ -0,0 +1,39 @@ +import os +import shutil +import subprocess + +from oeqa.oetest import oeSDKExtTest +from oeqa.utils.httpserver import HTTPService + +class SdkUpdateTest(oeSDKExtTest): + + @classmethod + def setUpClass(self): + self.publish_dir = os.path.join(self.tc.sdktestdir, 'esdk_publish') + if os.path.exists(self.publish_dir): + shutil.rmtree(self.publish_dir) + os.mkdir(self.publish_dir) + + tcname_new = self.tc.d.expand( + "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}-new.sh") + if not os.path.exists(tcname_new): + tcname_new = self.tc.tcname + + cmd = 'oe-publish-sdk %s %s' % (tcname_new, self.publish_dir) + subprocess.check_output(cmd, shell=True) + + self.http_service = HTTPService(self.publish_dir) + self.http_service.start() + + self.http_url = "http://127.0.0.1:%d" % self.http_service.port + + def test_sdk_update_http(self): + output = self._run("devtool sdk-update \"%s\"" % self.http_url) + + def test_sdk_update_local(self): + output = self._run("devtool sdk-update \"%s\"" % self.publish_dir) + + @classmethod + def tearDownClass(self): + self.http_service.stop() + shutil.rmtree(self.publish_dir) diff --git a/yocto-poky/meta/lib/oeqa/selftest/base.py b/yocto-poky/meta/lib/oeqa/selftest/base.py index 9bddc23f8..e10455edc 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/base.py +++ b/yocto-poky/meta/lib/oeqa/selftest/base.py @@ -4,7 +4,7 @@ # DESCRIPTION -# Base class inherited by test classes in meta/lib/selftest +# Base class inherited by test classes in meta/lib/oeqa/selftest import unittest import os @@ -16,6 +16,8 @@ import errno import oeqa.utils.ftools as ftools from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer from oeqa.utils.decorators import LogResults +from random import choice +import glob @LogResults class oeSelfTest(unittest.TestCase): @@ -29,9 +31,10 @@ class oeSelfTest(unittest.TestCase): self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc") self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf") self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") + self.machineinc_path = os.path.join(self.builddir, "conf/machine.inc") self.testlayer_path = oeSelfTest.testlayer_path self._extra_tear_down_commands = [] - self._track_for_cleanup = [self.testinc_path] + self._track_for_cleanup = [self.testinc_path, self.testinc_bblayers_path, self.machineinc_path] super(oeSelfTest, self).__init__(methodName) def setUp(self): @@ -47,11 +50,25 @@ class oeSelfTest(unittest.TestCase): for f in files: if f == 'test_recipe.inc': os.remove(os.path.join(root, f)) - try: - os.remove(self.testinc_bblayers_path) - except OSError as e: - if e.errno != errno.ENOENT: - raise + + for incl_file in [self.testinc_bblayers_path, self.machineinc_path]: + try: + os.remove(incl_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + # Get CUSTOMMACHINE from env (set by --machine argument to oe-selftest) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine: + if custommachine == 'random': + machine = get_random_machine() + else: + machine = custommachine + machine_conf = 'MACHINE ??= "%s"\n' % machine + self.set_machine_config(machine_conf) + print 'MACHINE: %s' % machine + # tests might need their own setup # but if they overwrite this one they have to call # super each time, so let's give them an alternative @@ -99,11 +116,21 @@ class oeSelfTest(unittest.TestCase): self.log.debug("Writing to: %s\n%s\n" % (self.testinc_path, data)) ftools.write_file(self.testinc_path, data) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine and 'MACHINE' in data: + machine = get_bb_var('MACHINE') + self.log.warning('MACHINE overridden: %s' % machine) + # append to <builddir>/conf/selftest.inc def append_config(self, data): self.log.debug("Appending to: %s\n%s\n" % (self.testinc_path, data)) ftools.append_file(self.testinc_path, data) + custommachine = os.getenv('CUSTOMMACHINE') + if custommachine and 'MACHINE' in data: + machine = get_bb_var('MACHINE') + self.log.warning('MACHINE overridden: %s' % machine) + # remove data from <builddir>/conf/selftest.inc def remove_config(self, data): self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data)) @@ -151,3 +178,28 @@ class oeSelfTest(unittest.TestCase): def remove_bblayers_config(self, data): self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_bblayers_path, data)) ftools.remove_from_file(self.testinc_bblayers_path, data) + + # write to <builddir>/conf/machine.inc + def set_machine_config(self, data): + self.log.debug("Writing to: %s\n%s\n" % (self.machineinc_path, data)) + ftools.write_file(self.machineinc_path, data) + + +def get_available_machines(): + # Get a list of all available machines + bbpath = get_bb_var('BBPATH').split(':') + machines = [] + + for path in bbpath: + found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf')) + if found_machines: + for i in found_machines: + # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf' + machines.append(os.path.splitext(os.path.basename(i))[0]) + + return machines + + +def get_random_machine(): + # Get a random machine + return choice(get_available_machines()) diff --git a/yocto-poky/meta/lib/oeqa/selftest/bblayers.py b/yocto-poky/meta/lib/oeqa/selftest/bblayers.py index 20c17e46f..d23675e84 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/bblayers.py +++ b/yocto-poky/meta/lib/oeqa/selftest/bblayers.py @@ -23,8 +23,10 @@ class BitbakeLayers(oeSelfTest): @testcase(93) def test_bitbakelayers_showappends(self): + recipe = "xcursor-transparent-theme" + bb_file = self.get_recipe_basename(recipe) result = runCmd('bitbake-layers show-appends') - self.assertTrue('xcursor-transparent-theme_0.1.1.bbappend' in result.output, msg="xcursor-transparent-theme_0.1.1.bbappend file was not recognised. bitbake-layers show-appends output: %s" % result.output) + self.assertTrue(bb_file in result.output, msg="%s file was not recognised. bitbake-layers show-appends output: %s" % (bb_file, result.output)) @testcase(90) def test_bitbakelayers_showoverlayed(self): @@ -33,11 +35,14 @@ class BitbakeLayers(oeSelfTest): @testcase(95) def test_bitbakelayers_flatten(self): + recipe = "xcursor-transparent-theme" + recipe_path = "recipes-graphics/xcursor-transparent-theme" + recipe_file = self.get_recipe_basename(recipe) testoutdir = os.path.join(self.builddir, 'test_bitbakelayers_flatten') self.assertFalse(os.path.isdir(testoutdir), msg = "test_bitbakelayers_flatten should not exist at this point in time") self.track_for_cleanup(testoutdir) result = runCmd('bitbake-layers flatten %s' % testoutdir) - bb_file = os.path.join(testoutdir, 'recipes-graphics/xcursor-transparent-theme/xcursor-transparent-theme_0.1.1.bb') + bb_file = os.path.join(testoutdir, recipe_path, recipe_file) self.assertTrue(os.path.isfile(bb_file), msg = "Cannot find xcursor-transparent-theme_0.1.1.bb in the test_bitbakelayers_flatten local dir.") contents = ftools.read_file(bb_file) find_in_contents = re.search("##### bbappended from meta-selftest #####\n(.*\n)*include test_recipe.inc", contents) @@ -60,3 +65,40 @@ class BitbakeLayers(oeSelfTest): result = runCmd('bitbake-layers remove-layer */meta-skeleton') result = runCmd('bitbake-layers show-layers') self.assertNotIn('meta-skeleton', result.output, msg = "meta-skeleton should have been removed at this step. bitbake-layers show-layers output: %s" % result.output) + + @testcase(1384) + def test_bitbakelayers_showrecipes(self): + result = runCmd('bitbake-layers show-recipes') + self.assertIn('aspell:', result.output) + self.assertIn('mtd-utils:', result.output) + self.assertIn('linux-yocto:', result.output) + self.assertIn('core-image-minimal:', result.output) + result = runCmd('bitbake-layers show-recipes mtd-utils') + self.assertIn('mtd-utils:', result.output) + self.assertNotIn('aspell:', result.output) + result = runCmd('bitbake-layers show-recipes -i kernel') + self.assertIn('linux-yocto:', result.output) + self.assertNotIn('mtd-utils:', result.output) + result = runCmd('bitbake-layers show-recipes -i image') + self.assertIn('core-image-minimal', result.output) + self.assertNotIn('linux-yocto:', result.output) + self.assertNotIn('mtd-utils:', result.output) + result = runCmd('bitbake-layers show-recipes -i cmake,pkgconfig') + self.assertIn('libproxy:', result.output) + self.assertNotIn('mtd-utils:', result.output) # doesn't inherit either + self.assertNotIn('wget:', result.output) # doesn't inherit cmake + self.assertNotIn('waffle:', result.output) # doesn't inherit pkgconfig + result = runCmd('bitbake-layers show-recipes -i nonexistentclass', ignore_status=True) + self.assertNotEqual(result.status, 0, 'bitbake-layers show-recipes -i nonexistentclass should have failed') + self.assertIn('ERROR:', result.output) + + def get_recipe_basename(self, recipe): + recipe_file = "" + result = runCmd("bitbake-layers show-recipes -f %s" % recipe) + for line in result.output.splitlines(): + if recipe in line: + recipe_file = line + break + + self.assertTrue(os.path.isfile(recipe_file), msg = "Can't find recipe file for %s" % recipe) + return os.path.basename(recipe_file) diff --git a/yocto-poky/meta/lib/oeqa/selftest/bbtests.py b/yocto-poky/meta/lib/oeqa/selftest/bbtests.py index 94ca79c03..26728a4b4 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/bbtests.py +++ b/yocto-poky/meta/lib/oeqa/selftest/bbtests.py @@ -8,6 +8,11 @@ from oeqa.utils.decorators import testcase class BitbakeTests(oeSelfTest): + def getline(self, res, line): + for l in res.output.split('\n'): + if line in l: + return l + @testcase(789) def test_run_bitbake_from_dir_1(self): os.chdir(os.path.join(self.builddir, 'conf')) @@ -63,7 +68,8 @@ class BitbakeTests(oeSelfTest): result = bitbake('man -c patch', ignore_status=True) self.delete_recipeinc('man') bitbake('-cclean man') - self.assertTrue("ERROR: Function failed: patch_do_patch" in result.output, msg = "Though no man-1.5h1-make.patch file exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) + line = self.getline(result, "Function failed: patch_do_patch") + self.assertTrue(line and line.startswith("ERROR:"), msg = "Though no man-1.5h1-make.patch file exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) @testcase(1354) def test_force_task_1(self): @@ -135,7 +141,8 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertEqual(result.status, 1, msg="Command succeded when it should have failed. bitbake output: %s" % result.output) self.assertTrue('Fetcher failure: Unable to find file file://invalid anywhere. The paths that were searched were:' in result.output, msg = "\"invalid\" file \ doesn't exist, yet no error message encountered. bitbake output: %s" % result.output) - self.assertTrue('ERROR: Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.' in result.output, msg = "\"invalid\" file \ + line = self.getline(result, 'Function failed: Fetcher failure for URL: \'file://invalid\'. Unable to fetch URL from any source.') + self.assertTrue(line and line.startswith("ERROR:"), msg = "\"invalid\" file \ doesn't exist, yet fetcher didn't report any error. bitbake output: %s" % result.output) @testcase(171) @@ -225,3 +232,41 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.assertEqual(result.status, 0, "Bitbake failed, exit code %s, output %s" % (result.status, result.output)) self.assertFalse(os.path.isfile(os.path.join(self.builddir, 'tmp/deploy/licenses/readline/generic_GPLv3'))) self.assertTrue(os.path.isfile(os.path.join(self.builddir, 'tmp/deploy/licenses/readline/generic_GPLv2'))) + + @testcase(1422) + def test_setscene_only(self): + """ Bitbake option to restore from sstate only within a build (i.e. execute no real tasks, only setscene)""" + test_recipe = 'ed' + + bitbake(test_recipe) + bitbake('-c clean %s' % test_recipe) + ret = bitbake('--setscene-only %s' % test_recipe) + + tasks = re.findall(r'task\s+(do_\S+):', ret.output) + + for task in tasks: + self.assertIn('_setscene', task, 'A task different from _setscene ran: %s.\n' + 'Executed tasks were: %s' % (task, str(tasks))) + + @testcase(1425) + def test_bbappend_order(self): + """ Bitbake should bbappend to recipe in a predictable order """ + test_recipe = 'ed' + test_recipe_summary_before = get_bb_var('SUMMARY', test_recipe) + test_recipe_pv = get_bb_var('PV', test_recipe) + recipe_append_file = test_recipe + '_' + test_recipe_pv + '.bbappend' + expected_recipe_summary = test_recipe_summary_before + + for i in range(5): + recipe_append_dir = test_recipe + '_test_' + str(i) + recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir, recipe_append_file) + os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', recipe_append_dir)) + feature = 'SUMMARY += "%s"\n' % i + ftools.write_file(recipe_append_path, feature) + expected_recipe_summary += ' %s' % i + + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test', + test_recipe + '_test_*')) + + test_recipe_summary_after = get_bb_var('SUMMARY', test_recipe) + self.assertEqual(expected_recipe_summary, test_recipe_summary_after) diff --git a/yocto-poky/meta/lib/oeqa/selftest/buildhistory.py b/yocto-poky/meta/lib/oeqa/selftest/buildhistory.py index d8cae4664..674da6205 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/buildhistory.py +++ b/yocto-poky/meta/lib/oeqa/selftest/buildhistory.py @@ -1,12 +1,10 @@ -import unittest import os import re -import shutil import datetime -import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import Command, runCmd, bitbake, get_bb_var, get_test_layer +from oeqa.utils.commands import bitbake, get_bb_var +from oeqa.utils.decorators import testcase class BuildhistoryBase(oeSelfTest): @@ -40,6 +38,9 @@ class BuildhistoryBase(oeSelfTest): if expect_error: self.assertEqual(result.status, 1, msg="Error expected for global config '%s' and target config '%s'" % (global_config, target_config)) search_for_error = re.search(error_regex, result.output) - self.assertTrue(search_for_error, msg="Could not find desired error in output: %s" % error_regex) + self.assertTrue(search_for_error, msg="Could not find desired error in output: %s (%s)" % (error_regex, result.output)) else: self.assertEqual(result.status, 0, msg="Command 'bitbake %s' has failed unexpectedly: %s" % (target, result.output)) + + # No tests should be added to the base class. + # Please create a new class that inherit this one, or use one of those already available for adding tests. diff --git a/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py b/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py index acf481f7b..35d5dfd29 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py +++ b/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py @@ -1,7 +1,8 @@ import os import re import glob as g - +import shutil +import tempfile from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase from oeqa.utils.commands import runCmd, bitbake, get_bb_var @@ -56,6 +57,11 @@ class ImageOptionsTests(oeSelfTest): res = runCmd("grep ccache %s" % (os.path.join(get_bb_var("WORKDIR","m4"),"temp/log.do_compile")), ignore_status=True) self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % os.path.join(get_bb_var("WORKDIR","m4"),"temp/log.do_compile")) + @testcase(1435) + def test_read_only_image(self): + self.write_config('IMAGE_FEATURES += "read-only-rootfs"') + bitbake("core-image-sato") + # do_image will fail if there are any pending postinsts class DiskMonTest(oeSelfTest): @@ -74,6 +80,10 @@ class DiskMonTest(oeSelfTest): self.assertTrue('WARNING: The free space' in res.output, msg = "A warning should have been displayed for disk monitor is set to WARN: %s" %res.output) class SanityOptionsTest(oeSelfTest): + def getline(self, res, line): + for l in res.output.split('\n'): + if line in l: + return l @testcase(927) def test_options_warnqa_errorqa_switch(self): @@ -85,7 +95,8 @@ class SanityOptionsTest(oeSelfTest): self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') res = bitbake("xcursor-transparent-theme", ignore_status=True) self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("ERROR: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) + line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") + self.assertTrue(line and line.startswith("ERROR:"), msg=res.output) self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output)) self.write_recipeinc('xcursor-transparent-theme', 'PACKAGES += \"${PN}-dbg\"') self.append_config('ERROR_QA_remove = "packages-list"') @@ -93,15 +104,85 @@ class SanityOptionsTest(oeSelfTest): bitbake("xcursor-transparent-theme -ccleansstate") res = bitbake("xcursor-transparent-theme") self.delete_recipeinc('xcursor-transparent-theme') - self.assertTrue("WARNING: QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors." in res.output, msg=res.output) + line = self.getline(res, "QA Issue: xcursor-transparent-theme-dbg is listed in PACKAGES multiple times, this leads to packaging errors.") + self.assertTrue(line and line.startswith("WARNING:"), msg=res.output) @testcase(278) - def test_sanity_userspace_dependency(self): - self.write_config('WARN_QA_append = " unsafe-references-in-binaries unsafe-references-in-scripts"') - bitbake("-ccleansstate gzip nfs-utils") - res = bitbake("gzip nfs-utils") - self.assertTrue("WARNING: QA Issue: gzip" in res.output, "WARNING: QA Issue: gzip message is not present in bitbake's output: %s" % res.output) - self.assertTrue("WARNING: QA Issue: nfs-utils" in res.output, "WARNING: QA Issue: nfs-utils message is not present in bitbake's output: %s" % res.output) + def test_sanity_unsafe_script_references(self): + self.write_config('WARN_QA_append = " unsafe-references-in-scripts"') + + bitbake("-ccleansstate gzip") + res = bitbake("gzip") + line = self.getline(res, "QA Issue: gzip") + self.assertFalse(line, "WARNING: QA Issue: gzip message is present in bitbake's output and shouldn't be: %s" % res.output) + + self.append_config(""" +do_install_append_pn-gzip () { + echo "\n${bindir}/test" >> ${D}${bindir}/zcat +} +""") + res = bitbake("gzip") + line = self.getline(res, "QA Issue: gzip") + self.assertTrue(line and line.startswith("WARNING:"), "WARNING: QA Issue: gzip message is not present in bitbake's output: %s" % res.output) + + @testcase(1434) + def test_sanity_unsafe_binary_references(self): + self.write_config('WARN_QA_append = " unsafe-references-in-binaries"') + + bitbake("-ccleansstate nfs-utils") + #res = bitbake("nfs-utils") + # FIXME when nfs-utils passes this test + #line = self.getline(res, "QA Issue: nfs-utils") + #self.assertFalse(line, "WARNING: QA Issue: nfs-utils message is present in bitbake's output and shouldn't be: %s" % res.output) + +# self.append_config(""" +#do_install_append_pn-nfs-utils () { +# echo "\n${bindir}/test" >> ${D}${base_sbindir}/osd_login +#} +#""") + res = bitbake("nfs-utils") + line = self.getline(res, "QA Issue: nfs-utils") + self.assertTrue(line and line.startswith("WARNING:"), "WARNING: QA Issue: nfs-utils message is not present in bitbake's output: %s" % res.output) + + @testcase(1421) + def test_layer_without_git_dir(self): + """ + Summary: Test that layer git revisions are displayed and do not fail without git repository + Expected: The build to be successful and without "fatal" errors + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + dirpath = tempfile.mkdtemp() + + dummy_layer_name = 'meta-dummy' + dummy_layer_path = os.path.join(dirpath, dummy_layer_name) + dummy_layer_conf_dir = os.path.join(dummy_layer_path, 'conf') + os.makedirs(dummy_layer_conf_dir) + dummy_layer_conf_path = os.path.join(dummy_layer_conf_dir, 'layer.conf') + + dummy_layer_content = 'BBPATH .= ":${LAYERDIR}"\n' \ + 'BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"\n' \ + 'BBFILE_COLLECTIONS += "%s"\n' \ + 'BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' \ + 'BBFILE_PRIORITY_%s = "6"\n' % (dummy_layer_name, dummy_layer_name, dummy_layer_name) + + ftools.write_file(dummy_layer_conf_path, dummy_layer_content) + + bblayers_conf = 'BBLAYERS += "%s"\n' % dummy_layer_path + self.write_bblayers_config(bblayers_conf) + + test_recipe = 'ed' + + ret = bitbake('-n %s' % test_recipe) + + err = 'fatal: Not a git repository' + + shutil.rmtree(dirpath) + + self.assertNotIn(err, ret.output) + class BuildhistoryTests(BuildhistoryBase): @@ -114,10 +195,70 @@ class BuildhistoryTests(BuildhistoryBase): def test_buildhistory_buildtime_pr_backwards(self): self.add_command_to_tearDown('cleanup-workdir') target = 'xcursor-transparent-theme' - error = "ERROR: QA Issue: Package version for package %s went backwards which would break package feeds from (.*-r1 to .*-r0)" % target + error = "ERROR:.*QA Issue: Package version for package %s went backwards which would break package feeds from (.*-r1.* to .*-r0.*)" % target self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) + @testcase(1386) + def test_buildhistory_does_not_change_signatures(self): + """ + Summary: Ensure that buildhistory does not change signatures + Expected: Only 'do_rootfs' task should be rerun + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + tmpdir1_name = 'tmpsig1' + tmpdir2_name = 'tmpsig2' + builddir = os.environ.get('BUILDDIR') + tmpdir1 = os.path.join(builddir, tmpdir1_name) + tmpdir2 = os.path.join(builddir, tmpdir2_name) + + self.track_for_cleanup(tmpdir1) + self.track_for_cleanup(tmpdir2) + + features = 'TMPDIR = "%s"\n' % tmpdir1 + self.write_config(features) + bitbake('core-image-minimal -S none -c rootfs') + + features = 'TMPDIR = "%s"\n' % tmpdir2 + features += 'INHERIT += "buildhistory"\n' + self.write_config(features) + bitbake('core-image-minimal -S none -c rootfs') + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + for name in files: + f.append(os.path.join(root, name)) + return f + + files1 = get_files(tmpdir1 + '/stamps') + files2 = get_files(tmpdir2 + '/stamps') + files2 = [x.replace(tmpdir2_name, tmpdir1_name) for x in files2] + + f1 = set(files1) + f2 = set(files2) + sigdiff = f1 - f2 + + self.assertEqual(len(sigdiff), 1, 'Expected 1 signature differences. Out: %s' % list(sigdiff)) + + unexpected_diff = [] + + # No new signatures should appear apart from do_rootfs + found_do_rootfs_flag = False + + for sig in sigdiff: + if 'do_rootfs' in sig: + found_do_rootfs_flag = True + else: + unexpected_diff.append(sig) + + self.assertTrue(found_do_rootfs_flag, 'Task do_rootfs did not rerun.') + self.assertFalse(unexpected_diff, 'Found unexpected signature differences. Out: %s' % unexpected_diff) + + class BuildImagesTest(oeSelfTest): @testcase(563) def test_directfb(self): @@ -137,7 +278,7 @@ class ArchiverTest(oeSelfTest): Test for archiving the work directory and exporting the source files. """ self.add_command_to_tearDown('cleanup-workdir') - self.write_config("INHERIT = \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") + self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") res = bitbake("xcursor-transparent-theme", ignore_status=True) self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) pkgs_path = g.glob(str(self.builddir) + "/tmp/deploy/sources/allarch*/xcurs*") diff --git a/yocto-poky/meta/lib/oeqa/selftest/devtool.py b/yocto-poky/meta/lib/oeqa/selftest/devtool.py index dcdef5a14..132a73d0e 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/devtool.py +++ b/yocto-poky/meta/lib/oeqa/selftest/devtool.py @@ -15,18 +15,45 @@ class DevtoolBase(oeSelfTest): def _test_recipe_contents(self, recipefile, checkvars, checkinherits): with open(recipefile, 'r') as f: + invar = None + invalue = None for line in f: - if '=' in line: + var = None + if invar: + value = line.strip().strip('"') + if value.endswith('\\'): + invalue += ' ' + value[:-1].strip() + continue + else: + invalue += ' ' + value.strip() + var = invar + value = invalue + invar = None + elif '=' in line: splitline = line.split('=', 1) var = splitline[0].rstrip() value = splitline[1].strip().strip('"') - if var in checkvars: - needvalue = checkvars.pop(var) - self.assertEqual(value, needvalue, 'values for %s do not match' % var) - if line.startswith('inherit '): + if value.endswith('\\'): + invalue = value[:-1].strip() + invar = var + continue + elif line.startswith('inherit '): inherits = line.split()[1:] - self.assertEqual(checkvars, {}, 'Some variables not found: %s' % checkvars) + if var and var in checkvars: + needvalue = checkvars.pop(var) + if needvalue is None: + self.fail('Variable %s should not appear in recipe') + if isinstance(needvalue, set): + value = set(value.split()) + self.assertEqual(value, needvalue, 'values for %s do not match' % var) + + + missingvars = {} + for var, value in checkvars.iteritems(): + if value is not None: + missingvars[var] = value + self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars) for inherit in checkinherits: self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit) @@ -68,6 +95,8 @@ class DevtoolBase(oeSelfTest): filelist = [] for line in output.splitlines(): splitline = line.split() + if len(splitline) < 8: + self.fail('_process_ls_output: invalid output line: %s' % line) # Remove trailing . on perms splitline[0] = splitline[0].rstrip('.') # Remove leading . on paths @@ -172,6 +201,44 @@ class DevtoolTests(DevtoolBase): bindir = bindir[1:] self.assertTrue(os.path.isfile(os.path.join(installdir, bindir, 'pv')), 'pv binary not found in D') + @testcase(1423) + def test_devtool_add_git_local(self): + # Fetch source from a remote URL, but do it outside of devtool + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + pn = 'dbus-wait' + # We choose an https:// git URL here to check rewriting the URL works + url = 'https://git.yoctoproject.org/git/dbus-wait' + # Force fetching to "noname" subdir so we verify we're picking up the name from autoconf + # instead of the directory name + result = runCmd('git clone %s noname' % url, cwd=tempdir) + srcdir = os.path.join(tempdir, 'noname') + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # Don't specify a name since we should be able to auto-detect it + result = runCmd('devtool add %s' % srcdir) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + # Check the recipe name is correct + recipefile = get_bb_var('FILE', pn) + self.assertIn('%s_git.bb' % pn, recipefile, 'Recipe file incorrectly named') + self.assertIn(recipefile, result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertIn(pn, result.output) + self.assertIn(srcdir, result.output) + self.assertIn(recipefile, result.output) + checkvars = {} + checkvars['LICENSE'] = 'GPLv2' + checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' + checkvars['S'] = '${WORKDIR}/git' + checkvars['PV'] = '0.1+git${SRCPV}' + checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/dbus-wait;protocol=https' + checkvars['SRCREV'] = '${AUTOREV}' + checkvars['DEPENDS'] = set(['dbus']) + self._test_recipe_contents(recipefile, checkvars, []) + @testcase(1162) def test_devtool_add_library(self): # We don't have the ability to pick up this dependency automatically yet... @@ -179,15 +246,16 @@ class DevtoolTests(DevtoolBase): # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - url = 'http://www.intra2net.com/en/developer/libftdi/download/libftdi1-1.1.tar.bz2' + version = '1.1' + url = 'https://www.intra2net.com/en/developer/libftdi/download/libftdi1-%s.tar.bz2' % version result = runCmd('wget %s' % url, cwd=tempdir) - result = runCmd('tar xfv libftdi1-1.1.tar.bz2', cwd=tempdir) - srcdir = os.path.join(tempdir, 'libftdi1-1.1') + result = runCmd('tar xfv libftdi1-%s.tar.bz2' % version, cwd=tempdir) + srcdir = os.path.join(tempdir, 'libftdi1-%s' % version) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory') # Test devtool add (and use -V so we test that too) self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') - result = runCmd('devtool add libftdi %s -V 1.1' % srcdir) + result = runCmd('devtool add libftdi %s -V %s' % (srcdir, version)) self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') # Test devtool status result = runCmd('devtool status') @@ -195,6 +263,14 @@ class DevtoolTests(DevtoolBase): self.assertIn(srcdir, result.output) # Clean up anything in the workdir/sysroot/sstate cache (have to do this *after* devtool add since the recipe only exists then) bitbake('libftdi -c cleansstate') + # libftdi's python/CMakeLists.txt is a bit broken, so let's just disable it + # There's also the matter of it installing cmake files to a path we don't + # normally cover, which triggers the installed-vs-shipped QA test we have + # within do_package + recipefile = '%s/recipes/libftdi/libftdi_%s.bb' % (self.workspacedir, version) + result = runCmd('recipetool setvar %s EXTRA_OECMAKE -- \'-DPYTHON_BINDINGS=OFF -DLIBFTDI_CMAKE_CONFIG_DIR=${datadir}/cmake/Modules\'' % recipefile) + with open(recipefile, 'a') as f: + f.write('\nFILES_${PN}-dev += "${datadir}/cmake/Modules"\n') # Test devtool build result = runCmd('devtool build libftdi') staging_libdir = get_bb_var('STAGING_LIBDIR', 'libftdi') @@ -226,21 +302,23 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') + self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') # Test devtool status result = runCmd('devtool status') self.assertIn(testrecipe, result.output) self.assertIn(srcdir, result.output) # Check recipe recipefile = get_bb_var('FILE', testrecipe) - self.assertIn('%s.bb' % testrecipe, recipefile, 'Recipe file incorrectly named') + self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') checkvars = {} - checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver - checkvars['SRC_URI'] = url + checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}' + checkvars['SRC_URI'] = url.replace(testver, '${PV}') self._test_recipe_contents(recipefile, checkvars, []) # Try with version specified result = runCmd('devtool reset -n %s' % testrecipe) shutil.rmtree(srcdir) - result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, testver)) + fakever = '1.9' + result = runCmd('devtool add %s %s -f %s -V %s' % (testrecipe, srcdir, url, fakever)) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') # Test devtool status result = runCmd('devtool status') @@ -248,10 +326,10 @@ class DevtoolTests(DevtoolBase): self.assertIn(srcdir, result.output) # Check recipe recipefile = get_bb_var('FILE', testrecipe) - self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') + self.assertIn('%s_%s.bb' % (testrecipe, fakever), recipefile, 'Recipe file incorrectly named') checkvars = {} - checkvars['S'] = '${WORKDIR}/MarkupSafe-${PV}' - checkvars['SRC_URI'] = url.replace(testver, '${PV}') + checkvars['S'] = '${WORKDIR}/MarkupSafe-%s' % testver + checkvars['SRC_URI'] = url self._test_recipe_contents(recipefile, checkvars, []) @testcase(1161) @@ -279,7 +357,7 @@ class DevtoolTests(DevtoolBase): self.assertIn('_git.bb', recipefile, 'Recipe file incorrectly named') checkvars = {} checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.0+git${SRCPV}' + checkvars['PV'] = '1.11+git${SRCPV}' checkvars['SRC_URI'] = url checkvars['SRCREV'] = '${AUTOREV}' self._test_recipe_contents(recipefile, checkvars, []) @@ -303,6 +381,34 @@ class DevtoolTests(DevtoolBase): checkvars['SRCREV'] = checkrev self._test_recipe_contents(recipefile, checkvars, []) + @testcase(1391) + def test_devtool_add_fetch_simple(self): + # Fetch source from a remote URL, auto-detecting name + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + testver = '1.6.0' + url = 'http://www.ivarch.com/programs/sources/pv-%s.tar.bz2' % testver + testrecipe = 'pv' + srcdir = os.path.join(self.workspacedir, 'sources', testrecipe) + # Test devtool add + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool add %s' % url) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) + self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory') + self.assertTrue(os.path.isdir(os.path.join(srcdir, '.git')), 'git repository for external source tree was not created') + # Test devtool status + result = runCmd('devtool status') + self.assertIn(testrecipe, result.output) + self.assertIn(srcdir, result.output) + # Check recipe + recipefile = get_bb_var('FILE', testrecipe) + self.assertIn('%s_%s.bb' % (testrecipe, testver), recipefile, 'Recipe file incorrectly named') + checkvars = {} + checkvars['S'] = None + checkvars['SRC_URI'] = url.replace(testver, '${PV}') + self._test_recipe_contents(recipefile, checkvars, []) + @testcase(1164) def test_devtool_modify(self): # Clean up anything in the workdir/sysroot/sstate cache @@ -504,7 +610,8 @@ class DevtoolTests(DevtoolBase): self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # (don't bother with cleaning the recipe on teardown, we won't be building it) - result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # We don't use -x here so that we test the behaviour of devtool modify without it + result = runCmd('devtool modify %s %s' % (testrecipe, tempdir)) # Check git repo self._check_src_repo(tempdir) # Add a couple of commits @@ -823,10 +930,11 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + self.append_config('PREFERRED_PROVIDER_virtual/make = "remake"') result = runCmd('devtool extract remake %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) self._check_src_repo(tempdir) @testcase(1379) @@ -834,10 +942,10 @@ class DevtoolTests(DevtoolBase): tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - self.track_for_cleanup(self.workspacedir) - self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool extract virtual/libx11 %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + # devtool extract shouldn't create the workspace + self.assertFalse(os.path.exists(self.workspacedir)) self._check_src_repo(tempdir) @testcase(1168) @@ -920,7 +1028,7 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool deploy-target -n %s root@localhost' % testrecipe) self.assertIn(' %s' % testfile, result.output) # Boot the image - with runqemu(testimage, self) as qemu: + with runqemu(testimage) as qemu: # Now really test deploy-target result = runCmd('devtool deploy-target -c %s root@%s' % (testrecipe, qemu.ip)) # Run a test command to see if it was installed properly @@ -990,14 +1098,18 @@ class DevtoolTests(DevtoolBase): def test_devtool_upgrade(self): # Check preconditions self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # Check parameters result = runCmd('devtool upgrade -h') for param in 'recipename srctree --version -V --branch -b --keep-temp --no-patch'.split(): self.assertIn(param, result.output) # For the moment, we are using a real recipe. - recipe='devtool-upgrade' - version='0.2' + recipe = 'devtool-upgrade-test1' + version = '1.6.0' + oldrecipefile = get_bb_var('FILE', recipe) tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) # Check that recipe is not already under devtool control result = runCmd('devtool status') self.assertNotIn(recipe, result.output) @@ -1005,22 +1117,64 @@ class DevtoolTests(DevtoolBase): # we are downgrading instead of upgrading. result = runCmd('devtool upgrade %s %s -V %s' % (recipe, tempdir, version)) # Check if srctree at least is populated - self.assertTrue(len(os.listdir(tempdir)) > 0, 'scrtree (%s) should be populated with new (%s) source code' % (tempdir, version)) - # Check new recipe folder is present - self.assertTrue(os.path.exists(os.path.join(self.workspacedir,'recipes',recipe)), 'Recipe folder should exist') + self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, version)) + # Check new recipe subdirectory is present + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe, '%s-%s' % (recipe, version))), 'Recipe folder should exist') # Check new recipe file is present - self.assertTrue(os.path.exists(os.path.join(self.workspacedir,'recipes',recipe,"%s_%s.bb" % (recipe,version))), 'Recipe folder should exist') + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, '%s_%s.bb' % (recipe, version)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file should exist after upgrade') # Check devtool status and make sure recipe is present result = runCmd('devtool status') self.assertIn(recipe, result.output) self.assertIn(tempdir, result.output) + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) # Check devtool reset recipe result = runCmd('devtool reset %s -n' % recipe) result = runCmd('devtool status') self.assertNotIn(recipe, result.output) - self.track_for_cleanup(tempdir) + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after resetting') + + @testcase(1433) + def test_devtool_upgrade_git(self): + # Check preconditions + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + recipe = 'devtool-upgrade-test2' + commit = '6cc6077a36fe2648a5f993fe7c16c9632f946517' + oldrecipefile = get_bb_var('FILE', recipe) + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + # Check that recipe is not already under devtool control + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + # Check upgrade + result = runCmd('devtool upgrade %s %s -S %s' % (recipe, tempdir, commit)) + # Check if srctree at least is populated + self.assertTrue(len(os.listdir(tempdir)) > 0, 'srctree (%s) should be populated with new (%s) source code' % (tempdir, commit)) + # Check new recipe file is present + newrecipefile = os.path.join(self.workspacedir, 'recipes', recipe, os.path.basename(oldrecipefile)) + self.assertTrue(os.path.exists(newrecipefile), 'Recipe file should exist after upgrade') + # Check devtool status and make sure recipe is present + result = runCmd('devtool status') + self.assertIn(recipe, result.output) + self.assertIn(tempdir, result.output) + # Check recipe got changed as expected + with open(oldrecipefile + '.upgraded', 'r') as f: + desiredlines = f.readlines() + with open(newrecipefile, 'r') as f: + newlines = f.readlines() + self.assertEqual(desiredlines, newlines) + # Check devtool reset recipe + result = runCmd('devtool reset %s -n' % recipe) + result = runCmd('devtool status') + self.assertNotIn(recipe, result.output) + self.assertFalse(os.path.exists(os.path.join(self.workspacedir, 'recipes', recipe)), 'Recipe directory should not exist after resetting') @testcase(1352) def test_devtool_layer_plugins(self): diff --git a/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py b/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py new file mode 100755 index 000000000..1b36a0d68 --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/selftest/esdk_prepare.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import shutil, tempfile +import sys +import os +import imp +import unittest +try: + from oeqa.utils.commands import get_bb_var +except ImportError: + pass + +# module under test +module_file_name = "ext-sdk-prepare.py" +module_path = "" + +class ExtSdkPrepareTest(unittest.TestCase): + + """ unit test for fix for Yocto #9019 """ + + @classmethod + def setUpClass(self): + # copy module under test to temp dir + self.test_dir = tempfile.mkdtemp() + module_dest_path = os.path.join(self.test_dir, module_file_name) + try: + shutil.copy(module_path, self.test_dir) + # load module under test + self.test_mod = imp.load_source("", module_dest_path) + except: + print "error: unable to copy or load %s [src: %s, dst: %s]" % \ + (module_file_name, module_path, module_dest_path) + sys.exit(1) + + def test_prepare_unexpected(self): + # test data + # note: pathnames have been truncated from the actual bitbake + # output as they are not important for the test. + test_data = ( + 'NOTE: Running noexec task 9 of 6539 (ID: 28, quilt/quilt-native_0.64.bb, do_build)\n' + 'NOTE: Running task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)\n' + 'NOTE: Running task 11 of 6539 (ID: 30, quilt/quilt-native_0.64.bb, do_rm_work)\n' + 'NOTE: Running noexec task 6402 of 6539 (ID: 1, images/core-image-sato.bb, do_patch)\n' + 'NOTE: Running task 6538 of 6539 (ID: 14, images/core-image-sato.bb, do_rm_work)\n' + ) + # expected warning output + expected = [ (' task 10 of 6539 (ID: 29, quilt/quilt-native_0.64.bb, do_package)') ] + # recipe to test, matching test input data + recipes = [ "core-image-sato.bb" ] + + # run the test + output = self.test_mod.check_unexpected(test_data, recipes) + self.assertEqual(output, expected) + + @classmethod + def tearDownClass(self): + # remove temp dir + shutil.rmtree(self.test_dir) + +if __name__ == '__main__': + # running from command line - i.e., not under oe-selftest + # directory containing module under test comes from command line + if len(sys.argv) == 2 and os.path.isdir(sys.argv[1]): + module_path = os.path.join(sys.argv[1], module_file_name) + suite = unittest.TestLoader().loadTestsFromTestCase(ExtSdkPrepareTest) + unittest.TextTestRunner().run(suite) + else: + progname = os.path.basename(sys.argv[0]) + print "%s: missing directory path" % progname + print "usage: %s /path/to/directory-of(ext-sdk-prepare.py)" % progname + sys.exit(1) +else: + # running under oe-selftest + # determine module source dir from COREBASE and expected path + module_path = os.path.join(get_bb_var("COREBASE"), "meta", "files", module_file_name) diff --git a/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py b/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py index 4efb0d92a..8a53899c7 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py +++ b/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py @@ -30,7 +30,7 @@ class ImageFeatures(oeSelfTest): # Build a core-image-minimal bitbake('core-image-minimal') - with runqemu("core-image-minimal", self) as qemu: + with runqemu("core-image-minimal") as qemu: # Attempt to ssh with each user into qemu with empty password for user in [self.root_user, self.test_user]: ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user=user) @@ -56,7 +56,7 @@ class ImageFeatures(oeSelfTest): # Build a core-image-minimal bitbake('core-image-minimal') - with runqemu("core-image-minimal", self) as qemu: + with runqemu("core-image-minimal") as qemu: # Attempt to ssh with each user into qemu with empty password for user in [self.root_user, self.test_user]: ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user=user) @@ -67,66 +67,6 @@ class ImageFeatures(oeSelfTest): self.assertEqual(status, 0, 'ssh to user tester failed with %s' % output) - @testcase(1114) - def test_rpm_version_4_support_on_image(self): - """ - Summary: Check rpm version 4 support on image - Expected: Rpm version must be 4.x - Product: oe-core - Author: Ionut Chisanovici <ionutx.chisanovici@intel.com> - AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> - """ - - features = 'PREFERRED_VERSION_rpm = "4.%"\n' - features += 'PREFERRED_VERSION_rpm-native = "4.%"\n' - # Use openssh in IMAGE_INSTALL instead of ssh-server-openssh in EXTRA_IMAGE_FEATURES as a workaround for bug 8047 - features += 'IMAGE_INSTALL_append = " openssh"\n' - features += 'EXTRA_IMAGE_FEATURES = "empty-root-password allow-empty-password package-management"\n' - features += 'RPMROOTFSDEPENDS_remove = "rpmresolve-native:do_populate_sysroot"' - self.write_config(features) - - # Build a core-image-minimal - bitbake('core-image-minimal') - - # Check the native version of rpm is correct - native_bindir = get_bb_var('STAGING_BINDIR_NATIVE') - result = runCmd(os.path.join(native_bindir, 'rpm') + ' --version') - self.assertIn('version 4.', result.output) - - # Check manifest for the rpm package - deploydir = get_bb_var('DEPLOY_DIR_IMAGE') - imgname = get_bb_var('IMAGE_LINK_NAME', 'core-image-minimal') - with open(os.path.join(deploydir, imgname) + '.manifest', 'r') as f: - for line in f: - splitline = line.split() - if len(splitline) > 2: - rpm_version = splitline[2] - if splitline[0] == 'rpm': - if not rpm_version.startswith('4.'): - self.fail('rpm version %s found in image, expected 4.x' % rpm_version) - break - else: - self.fail('No rpm package found in image') - - # Now do a couple of runtime tests - with runqemu("core-image-minimal", self) as qemu: - command = "rpm --version" - status, output = qemu.run(command) - self.assertEqual(0, status, 'Failed to run command "%s": %s' % (command, output)) - found_rpm_version = output.strip() - - # Make sure the retrieved rpm version is the expected one - if rpm_version not in found_rpm_version: - self.fail('RPM version is not {}, found instead {}.'.format(rpm_version, found_rpm_version)) - - # Test that the rpm database is there and working - command = "rpm -qa" - status, output = qemu.run(command) - self.assertEqual(0, status, 'Failed to run command "%s": %s' % (command, output)) - self.assertIn('packagegroup-core-boot', output) - self.assertIn('busybox', output) - - @testcase(1116) def test_clutter_image_can_be_built(self): """ diff --git a/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py b/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py index bd3b9a1ba..cac6d8445 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py +++ b/yocto-poky/meta/lib/oeqa/selftest/lic-checksum.py @@ -13,7 +13,7 @@ class LicenseTests(oeSelfTest): @testcase(1197) def test_nonmatching_checksum(self): bitbake_cmd = '-c configure emptytest' - error_msg = 'ERROR: emptytest: The new md5 checksum is 8d777f385d3dfec8815d20f7496026dc' + error_msg = 'emptytest: The new md5 checksum is 8d777f385d3dfec8815d20f7496026dc' lic_file, lic_path = tempfile.mkstemp() os.close(lic_file) diff --git a/yocto-poky/meta/lib/oeqa/selftest/prservice.py b/yocto-poky/meta/lib/oeqa/selftest/prservice.py index 4187fbfee..1b9a510fd 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/prservice.py +++ b/yocto-poky/meta/lib/oeqa/selftest/prservice.py @@ -9,9 +9,10 @@ import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var from oeqa.utils.decorators import testcase +from oeqa.utils.network import get_free_port class BitbakePrTests(oeSelfTest): - + def get_pr_version(self, package_name): pkgdata_dir = get_bb_var('PKGDATA_DIR') package_data_file = os.path.join(pkgdata_dir, 'runtime', package_name) @@ -26,7 +27,7 @@ class BitbakePrTests(oeSelfTest): package_stamps_path = "/".join(stampdata[:-1]) stamps = [] for stamp in os.listdir(package_stamps_path): - find_stamp = re.match("%s\.%s\.([a-z0-9]{32})" % (prefix, recipe_task), stamp) + find_stamp = re.match("%s\.%s\.([a-z0-9]{32})" % (re.escape(prefix), recipe_task), stamp) if find_stamp: stamps.append(find_stamp.group(1)) self.assertFalse(len(stamps) == 0, msg="Cound not find stamp for task %s for recipe %s" % (recipe_task, package_name)) @@ -34,7 +35,7 @@ class BitbakePrTests(oeSelfTest): return str(stamps[0]) def increment_package_pr(self, package_name): - inc_data = "do_package_append() {\nbb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() + inc_data = "do_package_append() {\n bb.build.exec_func('do_test_prserv', d)\n}\ndo_test_prserv() {\necho \"The current date is: %s\"\n}" % datetime.datetime.now() self.write_recipeinc(package_name, inc_data) bitbake("-ccleansstate %s" % package_name) res = bitbake(package_name, ignore_status=True) @@ -119,3 +120,13 @@ class BitbakePrTests(oeSelfTest): @testcase(936) def test_pr_service_ipk_arch_indep(self): self.run_test_pr_service('xcursor-transparent-theme', 'ipk', 'do_package') + + @testcase(1419) + def test_stopping_prservice_message(self): + port = get_free_port() + + runCmd('bitbake-prserv --host localhost --port %s --loglevel=DEBUG --start' % port) + ret = runCmd('bitbake-prserv --host localhost --port %s --loglevel=DEBUG --stop' % port) + + self.assertEqual(ret.status, 0) + diff --git a/yocto-poky/meta/lib/oeqa/selftest/recipetool.py b/yocto-poky/meta/lib/oeqa/selftest/recipetool.py index b1f1d2ab9..e72911b0a 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/recipetool.py +++ b/yocto-poky/meta/lib/oeqa/selftest/recipetool.py @@ -383,7 +383,7 @@ class RecipetoolTests(RecipetoolBase): @testcase(1194) def test_recipetool_create_git(self): # Ensure we have the right data in shlibs/pkgdata - bitbake('libpng pango libx11 libxext jpeg') + bitbake('libpng pango libx11 libxext jpeg libxsettings-client libcheck') # Try adding a recipe tempsrc = os.path.join(self.tempdir, 'srctree') os.makedirs(tempsrc) @@ -395,12 +395,52 @@ class RecipetoolTests(RecipetoolBase): checkvars['LICENSE'] = 'LGPLv2.1' checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' checkvars['S'] = '${WORKDIR}/git' - checkvars['PV'] = '1.0+git${SRCPV}' + checkvars['PV'] = '1.11+git${SRCPV}' checkvars['SRC_URI'] = srcuri - checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg' + checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxsettings-client', 'libxext', 'pango']) inherits = ['autotools', 'pkgconfig'] self._test_recipe_contents(recipefile, checkvars, inherits) + @testcase(1392) + def test_recipetool_create_simple(self): + # Try adding a recipe + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + pv = '1.7.3.0' + srcuri = 'http://www.dest-unreach.org/socat/download/socat-%s.tar.bz2' % pv + result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe)) + dirlist = os.listdir(temprecipe) + if len(dirlist) > 1: + self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): + self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) + self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named') + checkvars = {} + checkvars['LICENSE'] = set(['Unknown', 'GPLv2']) + checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263']) + # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot + checkvars['S'] = None + checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}') + inherits = ['autotools'] + self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits) + + @testcase(1418) + def test_recipetool_create_cmake(self): + # Try adding a recipe + temprecipe = os.path.join(self.tempdir, 'recipe') + os.makedirs(temprecipe) + recipefile = os.path.join(temprecipe, 'navit_0.5.0.bb') + srcuri = 'http://downloads.sourceforge.net/project/navit/v0.5.0/navit-0.5.0.tar.gz' + result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) + self.assertTrue(os.path.isfile(recipefile)) + checkvars = {} + checkvars['LICENSE'] = set(['Unknown', 'GPLv2', 'LGPLv2']) + checkvars['SRC_URI'] = 'http://downloads.sourceforge.net/project/navit/v${PV}/navit-${PV}.tar.gz' + checkvars['SRC_URI[md5sum]'] = '242f398e979a6b8c0f3c802b63435b68' + checkvars['SRC_URI[sha256sum]'] = '13353481d7fc01a4f64e385dda460b51496366bba0fd2cc85a89a0747910e94d' + checkvars['DEPENDS'] = set(['freetype', 'zlib', 'openssl', 'glib-2.0', 'virtual/libgl', 'virtual/egl', 'gtk+', 'libpng', 'libsdl', 'freeglut', 'dbus-glib']) + inherits = ['cmake', 'python-dir', 'gettext', 'pkgconfig'] + self._test_recipe_contents(recipefile, checkvars, inherits) class RecipetoolAppendsrcBase(RecipetoolBase): def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): diff --git a/yocto-poky/meta/lib/oeqa/selftest/signing.py b/yocto-poky/meta/lib/oeqa/selftest/signing.py new file mode 100644 index 000000000..1babca07d --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/selftest/signing.py @@ -0,0 +1,186 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import runCmd, bitbake, get_bb_var +import os +import glob +import re +import shutil +import tempfile +from oeqa.utils.decorators import testcase +from oeqa.utils.ftools import write_file + + +class Signing(oeSelfTest): + + gpg_dir = "" + pub_key_name = 'key.pub' + secret_key_name = 'key.secret' + + @classmethod + def setUpClass(cls): + # Import the gpg keys + + cls.gpg_dir = os.path.join(cls.testlayer_path, 'files/signing/') + + # key.secret key.pub are located in gpg_dir + pub_key_location = cls.gpg_dir + cls.pub_key_name + secret_key_location = cls.gpg_dir + cls.secret_key_name + runCmd('gpg --homedir %s --import %s %s' % (cls.gpg_dir, pub_key_location, secret_key_location)) + + @classmethod + def tearDownClass(cls): + # Delete the files generated by 'gpg --import' + + gpg_files = glob.glob(cls.gpg_dir + '*.gpg*') + random_seed_file = cls.gpg_dir + 'random_seed' + gpg_files.append(random_seed_file) + + for gpg_file in gpg_files: + runCmd('rm -f ' + gpg_file) + + @testcase(1362) + def test_signing_packages(self): + """ + Summary: Test that packages can be signed in the package feed + Expected: Package should be signed with the correct key + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + import oe.packagedata + + package_classes = get_bb_var('PACKAGE_CLASSES') + if 'package_rpm' not in package_classes: + self.skipTest('This test requires RPM Packaging.') + + test_recipe = 'ed' + + feature = 'INHERIT += "sign_rpm"\n' + feature += 'RPM_GPG_PASSPHRASE = "test123"\n' + feature += 'RPM_GPG_NAME = "testuser"\n' + feature += 'RPM_GPG_PUBKEY = "%s%s"\n' % (self.gpg_dir, self.pub_key_name) + feature += 'GPG_PATH = "%s"\n' % self.gpg_dir + + self.write_config(feature) + + bitbake('-c cleansstate %s' % test_recipe) + bitbake(test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + pkgdatadir = get_bb_var('PKGDATA_DIR', test_recipe) + pkgdata = oe.packagedata.read_pkgdatafile(pkgdatadir + "/runtime/ed") + if 'PKGE' in pkgdata: + pf = pkgdata['PN'] + "-" + pkgdata['PKGE'] + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] + else: + pf = pkgdata['PN'] + "-" + pkgdata['PKGV'] + '-' + pkgdata['PKGR'] + deploy_dir_rpm = get_bb_var('DEPLOY_DIR_RPM', test_recipe) + package_arch = get_bb_var('PACKAGE_ARCH', test_recipe).replace('-', '_') + staging_bindir_native = get_bb_var('STAGING_BINDIR_NATIVE') + + pkg_deploy = os.path.join(deploy_dir_rpm, package_arch, '.'.join((pf, package_arch, 'rpm'))) + + # Use a temporary rpmdb + rpmdb = tempfile.mkdtemp(prefix='oeqa-rpmdb') + + runCmd('%s/rpm --define "_dbpath %s" --import %s%s' % + (staging_bindir_native, rpmdb, self.gpg_dir, self.pub_key_name)) + + ret = runCmd('%s/rpm --define "_dbpath %s" --checksig %s' % + (staging_bindir_native, rpmdb, pkg_deploy)) + # tmp/deploy/rpm/i586/ed-1.9-r0.i586.rpm: rsa sha1 md5 OK + self.assertIn('rsa sha1 md5 OK', ret.output, 'Package signed incorrectly.') + shutil.rmtree(rpmdb) + + @testcase(1382) + def test_signing_sstate_archive(self): + """ + Summary: Test that sstate archives can be signed + Expected: Package should be signed with the correct key + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + test_recipe = 'ed' + + builddir = os.environ.get('BUILDDIR') + sstatedir = os.path.join(builddir, 'test-sstate') + + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + self.add_command_to_tearDown('bitbake -c cleansstate %s' % test_recipe) + self.add_command_to_tearDown('rm -rf %s' % sstatedir) + + # Determine the pub key signature + ret = runCmd('gpg --homedir %s --list-keys' % self.gpg_dir) + pub_key = re.search(r'^pub\s+\S+/(\S+)\s+', ret.output, re.M) + self.assertIsNotNone(pub_key, 'Failed to determine the public key signature.') + pub_key = pub_key.group(1) + + feature = 'SSTATE_SIG_KEY ?= "%s"\n' % pub_key + feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n' + feature += 'SSTATE_VERIFY_SIG ?= "1"\n' + feature += 'GPG_PATH = "%s"\n' % self.gpg_dir + feature += 'SSTATE_DIR = "%s"\n' % sstatedir + + self.write_config(feature) + + bitbake('-c cleansstate %s' % test_recipe) + bitbake(test_recipe) + + recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig') + recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_package.tgz') + + self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') + self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') + + ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0])) + # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30 + # gpg: Good signature from "testuser (nocomment) <testuser@email.com>" + self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.') + + +class LockedSignatures(oeSelfTest): + + @testcase(1420) + def test_locked_signatures(self): + """ + Summary: Test locked signature mechanism + Expected: Locked signatures will prevent task to run + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + test_recipe = 'ed' + locked_sigs_file = 'locked-sigs.inc' + + self.add_command_to_tearDown('rm -f %s' % os.path.join(self.builddir, locked_sigs_file)) + + bitbake(test_recipe) + # Generate locked sigs include file + bitbake('-S none %s' % test_recipe) + + feature = 'require %s\n' % locked_sigs_file + feature += 'SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n' + self.write_config(feature) + + # Build a locked recipe + bitbake(test_recipe) + + # Make a change that should cause the locked task signature to change + recipe_append_file = test_recipe + '_' + get_bb_var('PV', test_recipe) + '.bbappend' + recipe_append_path = os.path.join(self.testlayer_path, 'recipes-test', test_recipe, recipe_append_file) + feature = 'SUMMARY += "test locked signature"\n' + + os.mkdir(os.path.join(self.testlayer_path, 'recipes-test', test_recipe)) + write_file(recipe_append_path, feature) + + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, 'recipes-test', test_recipe)) + + # Build the recipe again + ret = bitbake(test_recipe) + + # Verify you get the warning and that the real task *isn't* run (i.e. the locked signature has worked) + patt = r'WARNING: The %s:do_package sig is computed to be \S+, but the sig is locked to \S+ in SIGGEN_LOCKEDSIGS\S+' % test_recipe + found_warn = re.search(patt, ret.output) + + self.assertIsNotNone(found_warn, "Didn't find the expected warning message. Output: %s" % ret.output) diff --git a/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py b/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py index 3c230620e..acaf405ac 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py +++ b/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py @@ -4,6 +4,7 @@ import os import re import shutil import glob +import subprocess import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -23,11 +24,20 @@ class SStateTests(SStateBase): bitbake(['-ccleansstate'] + targets) bitbake(targets) - file_tracker = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) + file_tracker = [] + results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) + if distro_nonspecific: + for r in results: + if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo", "_fetch.tgz.siginfo", "_unpack.tgz.siginfo", "_patch.tgz.siginfo")): + continue + file_tracker.append(r) + else: + file_tracker = results + if should_pass: self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets))) else: - self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s" % ', '.join(map(str, targets))) + self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) @testcase(975) def test_sstate_creation_distro_specific_pass(self): @@ -56,14 +66,14 @@ class SStateTests(SStateBase): bitbake(targets) tgz_created = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_created))) siginfo_created = self.search_sstate('|'.join(map(str, [s + '.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) bitbake(['-ccleansstate'] + targets) tgz_removed = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) - self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s" % ', '.join(map(str, targets))) + self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_removed))) @testcase(977) def test_cleansstate_task_distro_specific_nonspecific(self): @@ -87,7 +97,13 @@ class SStateTests(SStateBase): bitbake(['-ccleansstate'] + targets) bitbake(targets) - self.assertTrue(self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) == [], msg="Found distro non-specific sstate for: %s" % ', '.join(map(str, targets))) + results = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) + filtered_results = [] + for r in results: + if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo")): + continue + filtered_results.append(r) + self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) file_tracker_1 = self.search_sstate('|'.join(map(str, [s + '.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) @@ -208,28 +224,28 @@ class SStateTests(SStateBase): def test_sstate_32_64_same_hash(self): """ The sstate checksums for both native and target should not vary whether - they're built on a 32 or 64 bit system. Rather than requiring two different + they're built on a 32 or 64 bit system. Rather than requiring two different build machines and running a builds, override the variables calling uname() manually and check using bitbake -S. - - Also check that SDKMACHINE changing doesn't change any of these stamps. """ topdir = get_bb_var('TOPDIR') targetvendor = get_bb_var('TARGET_VENDOR') self.write_config(""" -TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" -BUILD_ARCH = \"x86_64\" -BUILD_OS = \"linux\" -SDKMACHINE = \"x86_64\" +MACHINE = "qemux86" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash" +BUILD_ARCH = "x86_64" +BUILD_OS = "linux" +SDKMACHINE = "x86_64" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") bitbake("core-image-sato -S none") self.write_config(""" -TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" -BUILD_ARCH = \"i686\" -BUILD_OS = \"linux\" -SDKMACHINE = \"i686\" +MACHINE = "qemux86" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" +BUILD_ARCH = "i686" +BUILD_OS = "linux" +SDKMACHINE = "i686" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") bitbake("core-image-sato -S none") @@ -238,9 +254,10 @@ SDKMACHINE = \"i686\" f = [] for root, dirs, files in os.walk(d): if "core-image-sato" in root: - # SDKMACHINE changing will change do_rootfs/do_testimage/do_build stamps of core-image-sato itself - # which is safe to ignore - continue + # SDKMACHINE changing will change + # do_rootfs/do_testimage/do_build stamps of images which + # is safe to ignore. + continue f.extend(os.path.join(root, name) for name in files) return f files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") @@ -254,7 +271,7 @@ SDKMACHINE = \"i686\" def test_sstate_nativelsbstring_same_hash(self): """ The sstate checksums should be independent of whichever NATIVELSBSTRING is - detected. Rather than requiring two different build machines and running + detected. Rather than requiring two different build machines and running builds, override the variables manually and check using bitbake -S. """ @@ -286,7 +303,7 @@ NATIVELSBSTRING = \"DistroB\" @testcase(1368) def test_sstate_allarch_samesigs(self): """ - The sstate checksums of allarch packages should be independent of whichever + The sstate checksums of allarch packages should be independent of whichever MACHINE is set. Check this using bitbake -S. Also, rather than duplicate the test, check nativesdk stamps are the same between the two MACHINE values. @@ -309,32 +326,32 @@ MACHINE = \"qemuarm\" bitbake("world meta-toolchain -S none") def get_files(d): - f = [] + f = {} for root, dirs, files in os.walk(d): for name in files: if "meta-environment" in root or "cross-canadian" in root: continue if "do_build" not in name: - f.append(os.path.join(root, name)) + # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 + (_, task, _, shash) = name.rsplit(".", 3) + f[os.path.join(os.path.basename(root), task)] = shash return f files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/all" + targetvendor + "-" + targetos) files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/all" + targetvendor + "-" + targetos) - files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None - self.assertItemsEqual(files1, files2) + self.assertEqual(files1, files2) nativesdkdir = os.path.basename(glob.glob(topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) - files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None - self.assertItemsEqual(files1, files2) + self.assertEqual(files1, files2) @testcase(1369) def test_sstate_sametune_samesigs(self): """ - The sstate checksums of two identical machines (using the same tune) should be the + The sstate checksums of two identical machines (using the same tune) should be the same, apart from changes within the machine specific stamps directory. We use the qemux86copy machine to test this. Also include multilibs in the test. """ @@ -377,3 +394,68 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86" files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None self.assertItemsEqual(files1, files2) + + + def test_sstate_noop_samesigs(self): + """ + The sstate checksums of two builds with these variables changed or + classes inherits should be the same. + """ + + topdir = get_bb_var('TOPDIR') + targetvendor = get_bb_var('TARGET_VENDOR') + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash" +BB_NUMBER_THREADS = "1" +PARALLEL_MAKE = "-j 1" +DL_DIR = "${TOPDIR}/download1" +TIME = "111111" +DATE = "20161111" +INHERIT_remove = "buildstats-summary buildhistory" +""") + self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(""" +TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" +BB_NUMBER_THREADS = "2" +PARALLEL_MAKE = "-j 2" +DL_DIR = "${TOPDIR}/download2" +TIME = "222222" +DATE = "20161212" +INHERIT += "buildstats-summary buildhistory" +""") + self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + bitbake("world meta-toolchain -S none") + + def get_files(d): + f = {} + for root, dirs, files in os.walk(d): + for name in files: + name, shash = name.rsplit('.', 1) + # Extract just the machine and recipe name + base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name]) + f[base] = shash + return f + files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") + files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") + # Remove items that are identical in both sets + for k,v in files1.viewitems() & files2.viewitems(): + del files1[k] + del files2[k] + if not files1 and not files2: + # No changes, so we're done + return + + for k in files1.viewkeys() | files2.viewkeys(): + if k in files1 and k in files2: + print "%s differs:" % k + print subprocess.check_output(("bitbake-diffsigs", + topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k], + topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k])) + elif k in files1 and k not in files2: + print "%s in files1" % k + elif k not in files1 and k in files2: + print "%s in files2" % k + else: + assert "shouldn't reach here" + self.fail("sstate hashes not identical.") diff --git a/yocto-poky/meta/lib/oeqa/selftest/wic.py b/yocto-poky/meta/lib/oeqa/selftest/wic.py index ea78e2259..a569fbf74 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/wic.py +++ b/yocto-poky/meta/lib/oeqa/selftest/wic.py @@ -24,13 +24,12 @@ """Test cases for wic.""" import os -import sys from glob import glob from shutil import rmtree from oeqa.selftest.base import oeSelfTest -from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu from oeqa.utils.decorators import testcase @@ -42,7 +41,8 @@ class Wic(oeSelfTest): def setUpLocal(self): """This code is executed before each test method.""" - self.write_config('IMAGE_FSTYPES += " hddimg"\nMACHINE_FEATURES_append = " efi"\n') + self.write_config('IMAGE_FSTYPES += " hddimg"\n' + 'MACHINE_FEATURES_append = " efi"\n') # Do this here instead of in setUpClass as the base setUp does some # clean up which can result in the native tools built earlier in @@ -80,14 +80,14 @@ class Wic(oeSelfTest): @testcase(1212) def test_build_artifacts(self): """Test wic create directdisk providing all artifacts.""" - vars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ + bbvars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', 'STAGING_DIR_NATIVE', 'IMAGE_ROOTFS')) status = runCmd("wic create directdisk " "-b %(staging_datadir)s " "-k %(deploy_dir_image)s " "-n %(staging_dir_native)s " - "-r %(image_rootfs)s" % vars).status + "-r %(image_rootfs)s" % bbvars).status self.assertEqual(0, status) self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) @@ -102,7 +102,7 @@ class Wic(oeSelfTest): def test_unsupported_subcommand(self): """Test unsupported subcommand""" self.assertEqual(1, runCmd('wic unsupported', - ignore_status=True).status) + ignore_status=True).status) @testcase(1214) def test_no_command(self): @@ -172,20 +172,20 @@ class Wic(oeSelfTest): @testcase(1269) def test_rootfs_artifacts(self): """Test usage of rootfs plugin with rootfs paths""" - vars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ + bbvars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', 'STAGING_DIR_NATIVE', 'IMAGE_ROOTFS')) - vars['wks'] = "directdisk-multi-rootfs" + bbvars['wks'] = "directdisk-multi-rootfs" status = runCmd("wic create %(wks)s " "-b %(staging_datadir)s " "-k %(deploy_dir_image)s " "-n %(staging_dir_native)s " "--rootfs-dir rootfs1=%(image_rootfs)s " "--rootfs-dir rootfs2=%(image_rootfs)s" \ - % vars).status + % bbvars).status self.assertEqual(0, status) self.assertEqual(1, len(glob(self.resultdir + \ - "%(wks)s-*.direct" % vars))) + "%(wks)s-*.direct" % bbvars))) @testcase(1346) def test_iso_image(self): @@ -199,6 +199,7 @@ class Wic(oeSelfTest): def test_image_env(self): """Test generation of <image>.env files.""" image = 'core-image-minimal' + self.assertEqual(0, bitbake('%s -c do_rootfs_wicenv' % image).status) stdir = get_bb_var('STAGING_DIR_TARGET', image) imgdatadir = os.path.join(stdir, 'imgdata') @@ -228,7 +229,7 @@ class Wic(oeSelfTest): prefix = os.path.join(deploy_dir, 'wic-image-minimal-%s.' % machine) # check if we have result image and manifests symlinks # pointing to existing files - for suffix in ('wic.bz2', 'manifest'): + for suffix in ('wic', 'manifest'): path = prefix + suffix self.assertTrue(os.path.islink(path)) self.assertTrue(os.path.isfile(os.path.realpath(path))) @@ -256,3 +257,22 @@ class Wic(oeSelfTest): self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ % image).status) self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) + + @testcase(1385) + def test_directdisk_bootloader_config(self): + """Test creation of directdisk-bootloader-config image""" + image = "directdisk-bootloader-config" + self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ + % image).status) + self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) + + @testcase(1422) + def test_qemu(self): + """Test wic-image-minimal under qemu""" + self.assertEqual(0, bitbake('wic-image-minimal').status) + + with runqemu('wic-image-minimal', ssh=False) as qemu: + command = "mount |grep '^/dev/' | cut -f1,3 -d ' '" + status, output = qemu.run_serial(command) + self.assertEqual(1, status, 'Failed to run command "%s": %s' % (command, output)) + self.assertEqual(output, '/dev/root /\r\n/dev/vda3 /mnt') diff --git a/yocto-poky/meta/lib/oeqa/targetcontrol.py b/yocto-poky/meta/lib/oeqa/targetcontrol.py index edc0d01c1..5422a617c 100644 --- a/yocto-poky/meta/lib/oeqa/targetcontrol.py +++ b/yocto-poky/meta/lib/oeqa/targetcontrol.py @@ -68,7 +68,7 @@ class BaseTarget(object): bb.note("SSH log file: %s" % self.sshlog) @abstractmethod - def start(self, params=None): + def start(self, params=None, ssh=True): pass @abstractmethod @@ -113,7 +113,7 @@ class BaseTarget(object): class QemuTarget(BaseTarget): - supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz'] + supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] def __init__(self, d): @@ -176,11 +176,12 @@ class QemuTarget(BaseTarget): bb.note("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() - def start(self, params=None): - if self.runner.start(params): - self.ip = self.runner.ip - self.server_ip = self.runner.server_ip - self.connection = SSHControl(ip=self.ip, logfile=self.sshlog) + def start(self, params=None, ssh=True): + if self.runner.start(params, get_ip=ssh): + if ssh: + self.ip = self.runner.ip + self.server_ip = self.runner.server_ip + self.connection = SSHControl(ip=self.ip, logfile=self.sshlog) else: self.stop() if os.path.exists(self.qemulog): @@ -231,8 +232,9 @@ class SimpleRemoteTarget(BaseTarget): def deploy(self): super(SimpleRemoteTarget, self).deploy() - def start(self, params=None): - self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) + def start(self, params=None, ssh=True): + if ssh: + self.connection = SSHControl(self.ip, logfile=self.sshlog, port=self.port) def stop(self): self.connection = None diff --git a/yocto-poky/meta/lib/oeqa/utils/__init__.py b/yocto-poky/meta/lib/oeqa/utils/__init__.py index 226004602..8f706f363 100644 --- a/yocto-poky/meta/lib/oeqa/utils/__init__.py +++ b/yocto-poky/meta/lib/oeqa/utils/__init__.py @@ -13,3 +13,26 @@ class CommandError(Exception): def __str__(self): return "Command '%s' returned non-zero exit status %d with output: %s" % (self.cmd, self.retcode, self.output) +def avoid_paths_in_environ(paths): + """ + Searches for every path in os.environ['PATH'] + if found remove it. + + Returns new PATH without avoided PATHs. + """ + import os + + new_path = '' + for p in os.environ['PATH'].split(':'): + avoid = False + for pa in paths: + if pa in p: + avoid = True + break + if avoid: + continue + + new_path = new_path + p + ':' + + new_path = new_path[:-1] + return new_path diff --git a/yocto-poky/meta/lib/oeqa/utils/commands.py b/yocto-poky/meta/lib/oeqa/utils/commands.py index 08e2cbb90..48f644129 100644 --- a/yocto-poky/meta/lib/oeqa/utils/commands.py +++ b/yocto-poky/meta/lib/oeqa/utils/commands.py @@ -18,6 +18,11 @@ from oeqa.utils import CommandError from oeqa.utils import ftools import re import contextlib +# Export test doesn't require bb +try: + import bb +except ImportError: + pass class Command(object): def __init__(self, command, bg=False, timeout=None, data=None, **options): @@ -177,7 +182,7 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec= @contextlib.contextmanager -def runqemu(pn, test): +def runqemu(pn, ssh=True): import bb.tinfoil import bb.build @@ -208,10 +213,20 @@ def runqemu(pn, test): # Luckily QemuTarget doesn't need it after the constructor. tinfoil.shutdown() + # Setup bitbake logger as console handler is removed by tinfoil.shutdown + bblogger = logging.getLogger('BitBake') + bblogger.setLevel(logging.INFO) + console = logging.StreamHandler(sys.stdout) + bbformat = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + if sys.stdout.isatty(): + bbformat.enable_color() + console.setFormatter(bbformat) + bblogger.addHandler(console) + try: qemu.deploy() try: - qemu.start() + qemu.start(ssh=ssh) except bb.build.FuncFailed: raise Exception('Failed to start QEMU - see the logs in %s' % logdir) diff --git a/yocto-poky/meta/lib/oeqa/utils/ftools.py b/yocto-poky/meta/lib/oeqa/utils/ftools.py index 1bd9a30a4..a7233d4ca 100644 --- a/yocto-poky/meta/lib/oeqa/utils/ftools.py +++ b/yocto-poky/meta/lib/oeqa/utils/ftools.py @@ -36,10 +36,11 @@ def remove_from_file(path, data): return else: raise - lines = rdata.splitlines() - rmdata = data.strip().splitlines() - for l in rmdata: - for c in range(0, lines.count(l)): - i = lines.index(l) - del(lines[i]) - write_file(path, "\n".join(lines)) + + contents = rdata.strip().splitlines() + for r in data.strip().splitlines(): + try: + contents.remove(r) + except ValueError: + pass + write_file(path, "\n".join(contents)) diff --git a/yocto-poky/meta/lib/oeqa/utils/network.py b/yocto-poky/meta/lib/oeqa/utils/network.py new file mode 100644 index 000000000..2768f6c5d --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/utils/network.py @@ -0,0 +1,8 @@ +import socket + +def get_free_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('', 0)) + addr = s.getsockname() + s.close() + return addr[1] diff --git a/yocto-poky/meta/lib/oeqa/utils/qemurunner.py b/yocto-poky/meta/lib/oeqa/utils/qemurunner.py index bdc6e0a8f..784cf964f 100644 --- a/yocto-poky/meta/lib/oeqa/utils/qemurunner.py +++ b/yocto-poky/meta/lib/oeqa/utils/qemurunner.py @@ -91,12 +91,12 @@ class QemuRunner: self._dump_host() raise SystemExit - def start(self, qemuparams = None): + def start(self, qemuparams = None, get_ip = True): if self.display: os.environ["DISPLAY"] = self.display - else: - logger.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)") - return False + # Set this flag so that Qemu doesn't do any grabs as SDL grabs + # interact badly with screensavers. + os.environ["QEMU_DONT_GRAB"] = "1" if not os.path.exists(self.rootfs): logger.error("Invalid rootfs %s" % self.rootfs) return False @@ -118,10 +118,10 @@ class QemuRunner: logger.error("Failed to create listening socket: %s" % msg[1]) return False - # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact - # badly with screensavers. - os.environ["QEMU_DONT_GRAB"] = "1" - self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8" qemuparams="-serial tcp:127.0.0.1:{}"'.format(threadport) + + self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8 printk.time=1" qemuparams="-serial tcp:127.0.0.1:{}"'.format(threadport) + if not self.display: + self.qemuparams = 'nographic ' + self.qemuparams if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' @@ -178,27 +178,28 @@ class QemuRunner: if self.is_alive(): logger.info("qemu started - qemu procces pid is %s" % self.qemupid) - cmdline = '' - with open('/proc/%s/cmdline' % self.qemupid) as p: - cmdline = p.read() - # It is needed to sanitize the data received - # because is possible to have control characters - cmdline = re_control_char.sub('', cmdline) - try: - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - if not ips or len(ips) != 3: - raise ValueError - else: - self.ip = ips[0] - self.server_ip = ips[1] - except IndexError, ValueError: - logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) - self._dump_host() - self.stop() - return False - logger.info("qemu cmdline used:\n{}".format(cmdline)) - logger.info("Target IP: %s" % self.ip) - logger.info("Server IP: %s" % self.server_ip) + if get_ip: + cmdline = '' + with open('/proc/%s/cmdline' % self.qemupid) as p: + cmdline = p.read() + # It is needed to sanitize the data received + # because is possible to have control characters + cmdline = re_control_char.sub('', cmdline) + try: + ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + if not ips or len(ips) != 3: + raise ValueError + else: + self.ip = ips[0] + self.server_ip = ips[1] + except IndexError, ValueError: + logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) + self._dump_host() + self.stop() + return False + logger.info("qemu cmdline used:\n{}".format(cmdline)) + logger.info("Target IP: %s" % self.ip) + logger.info("Server IP: %s" % self.server_ip) self.thread = LoggingThread(self.log, threadsock, logger) self.thread.start() @@ -366,23 +367,25 @@ class QemuRunner: # We assume target system have echo to get command status if not raw: command = "%s; echo $?\n" % command - self.server_socket.sendall(command) + data = '' status = 0 - stopread = False - endtime = time.time()+5 - while time.time()<endtime and not stopread: + self.server_socket.sendall(command) + keepreading = True + while keepreading: sread, _, _ = select.select([self.server_socket],[],[],5) - for sock in sread: - answer = sock.recv(1024) + if sread: + answer = self.server_socket.recv(1024) if answer: data += answer # Search the prompt to stop if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): - stopread = True - break + keepreading = False else: raise Exception("No data on serial console socket") + else: + keepreading = False + if data: if raw: status = 1 diff --git a/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py b/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py index 00f505105..165874416 100644 --- a/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py +++ b/yocto-poky/meta/lib/oeqa/utils/sshcontrol.py @@ -136,8 +136,7 @@ class SSHControl(object): timeout=0 - no timeout, let command run until it returns """ - # We need to source /etc/profile for a proper PATH on the target - command = self.ssh + [self.ip, ' . /etc/profile; ' + command] + command = self.ssh + [self.ip, 'export PATH=/usr/sbin:/sbin:/usr/bin:/bin; ' + command] if timeout is None: return self._internal_run(command, self.defaulttimeout, self.ignore_status) diff --git a/yocto-poky/meta/lib/oeqa/utils/testexport.py b/yocto-poky/meta/lib/oeqa/utils/testexport.py new file mode 100644 index 000000000..243463bc1 --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/utils/testexport.py @@ -0,0 +1,263 @@ +# Copyright (C) 2015 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides functions to help with exporting binaries obtained from built targets + +import os, re, glob as g, shutil as sh,sys +from time import sleep +from commands import runCmd +from difflib import SequenceMatcher as SM + +try: + import bb +except ImportError: + class my_log(): + def __init__(self): + pass + def plain(self, msg): + if msg: + print msg + def warn(self, msg): + if msg: + print "WARNING: " + msg + def fatal(self, msg): + if msg: + print "FATAL:" + msg + sys.exit(1) + bb = my_log() + + +def determine_if_poky_env(): + """ + used to determine if we are inside the poky env or not. Usefull for remote machine where poky is not present + """ + check_env = True if ("/scripts" and "/bitbake/bin") in os.getenv("PATH") else False + return check_env + + +def get_dest_folder(tune_features, folder_list): + """ + Function to determine what rpm deploy dir to choose for a given architecture based on TUNE_FEATURES + """ + features_list = tune_features.split(" ") + features_list.reverse() + features_list = "_".join(features_list) + match_rate = 0 + best_match = None + for folder in folder_list: + curr_match_rate = SM(None, folder, features_list).ratio() + if curr_match_rate > match_rate: + match_rate = curr_match_rate + best_match = folder + return best_match + + +def process_binaries(d, params): + param_list = params + export_env = d.getVar("TEST_EXPORT_ONLY") + + def extract_binary(pth_to_pkg, dest_pth=None): + cpio_command = runCmd("which cpio") + rpm2cpio_command = runCmd("ls /usr/bin/rpm2cpio") + if (cpio_command.status != 0) and (rpm2cpio_command.status != 0): + bb.fatal("Either \"rpm2cpio\" or \"cpio\" tools are not available on your system." + "All binaries extraction processes will not be available, crashing all related tests." + "Please install them according to your OS recommendations") # will exit here + if dest_pth: + os.chdir(dest_pth) + else: + os.chdir("%s" % os.sep)# this is for native package + extract_bin_command = runCmd("%s %s | %s -idm" % (rpm2cpio_command.output, pth_to_pkg, cpio_command.output)) # semi-hardcoded because of a bug on poky's rpm2cpio + return extract_bin_command + + if determine_if_poky_env(): # machine with poky environment + exportpath = d.getVar("TEST_EXPORT_DIR", True) if export_env else d.getVar("DEPLOY_DIR", True) + rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM", True) + arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(rpm_deploy_dir)) + arch_rpm_dir = os.path.join(rpm_deploy_dir, arch) + extracted_bin_dir = os.path.join(exportpath,"binaries", arch, "extracted_binaries") + packaged_bin_dir = os.path.join(exportpath,"binaries", arch, "packaged_binaries") + # creating necessary directory structure in case testing is done in poky env. + if export_env == "0": + if not os.path.exists(extracted_bin_dir): bb.utils.mkdirhier(extracted_bin_dir) + if not os.path.exists(packaged_bin_dir): bb.utils.mkdirhier(packaged_bin_dir) + + if param_list[3] == "native": + if export_env == "1": #this is a native package and we only need to copy it. no need for extraction + native_rpm_dir = os.path.join(rpm_deploy_dir, get_dest_folder("{} nativesdk".format(d.getVar("BUILD_SYS")), os.listdir(rpm_deploy_dir))) + native_rpm_file_list = [item for item in os.listdir(native_rpm_dir) if re.search("nativesdk-" + param_list[0] + "-([0-9]+\.*)", item)] + if not native_rpm_file_list: + bb.warn("Couldn't find any version of {} native package. Related tests will most probably fail.".format(param_list[0])) + return "" + for item in native_rpm_file_list:# will copy all versions of package. Used version will be selected on remote machine + bb.plain("Copying native package file: %s" % item) + sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries", "native")) + else: # nothing to do here; running tests under bitbake, so we asume native binaries are in sysroots dir. + if param_list[1] or param_list[4]: + bb.warn("Native binary %s %s%s. Running tests under bitbake environment. Version can't be checked except when the test itself does it" + " and binary can't be removed."%(param_list[0],"has assigned ver. " + param_list[1] if param_list[1] else "", + ", is marked for removal" if param_list[4] else "")) + else:# the package is target aka DUT intended and it is either required to be delivered in an extracted form or in a packaged version + target_rpm_file_list = [item for item in os.listdir(arch_rpm_dir) if re.search(param_list[0] + "-([0-9]+\.*)", item)] + if not target_rpm_file_list: + bb.warn("Couldn't find any version of target package %s. Please ensure it was built. " + "Related tests will probably fail." % param_list[0]) + return "" + if param_list[2] == "rpm": # binary should be deployed as rpm; (other, .deb, .ipk? ; in the near future) + for item in target_rpm_file_list: # copying all related rpm packages. "Intuition" reasons, someone may need other versions too. Deciding later on version + bb.plain("Copying target specific packaged file: %s" % item) + sh.copy(os.path.join(arch_rpm_dir, item), packaged_bin_dir) + return "copied" + else: # it is required to extract the binary + if param_list[1]: # the package is versioned + for item in target_rpm_file_list: + if re.match(".*-{}-.*\.rpm".format(param_list[1]), item): + destination = os.path.join(extracted_bin_dir,param_list[0], param_list[1]) + bb.utils.mkdirhier(destination) + extract_binary(os.path.join(arch_rpm_dir, item), destination) + break + else: + bb.warn("Couldn't find the desired version %s for target binary %s. Related test cases will probably fail." % (param_list[1], param_list[0])) + return "" + return "extracted" + else: # no version provided, just extract one binary + destination = os.path.join(extracted_bin_dir,param_list[0], + re.search(".*-([0-9]+\.[0-9]+)-.*rpm", target_rpm_file_list[0]).group(1)) + bb.utils.mkdirhier(destination) + extract_binary(os.path.join(arch_rpm_dir, target_rpm_file_list[0]), destination) + return "extracted" + else: # remote machine + binaries_path = os.getenv("bin_dir")# in order to know where the binaries are, bin_dir is set as env. variable + if param_list[3] == "native": #need to extract the native pkg here + native_rpm_dir = os.path.join(binaries_path, "native") + native_rpm_file_list = os.listdir(native_rpm_dir) + for item in native_rpm_file_list: + if param_list[1] and re.match("nativesdk-{}-{}-.*\.rpm".format(param_list[0], param_list[1]), item): # native package has version + extract_binary(os.path.join(native_rpm_dir, item)) + break + else:# just copy any related native binary + found_version = re.match("nativesdk-{}-([0-9]+\.[0-9]+)-".format(param_list[0]), item).group(1) + if found_version: + extract_binary(os.path.join(native_rpm_dir, item)) + else: + bb.warn("Couldn't find native package %s%s. Related test cases will be influenced." % + (param_list[0], " with version " + param_list[1] if param_list[1] else "")) + return + + else: # this is for target device + if param_list[2] == "rpm": + return "No need to extract, this is an .rpm file" + arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(binaries_path)) + extracted_bin_path = os.path.join(binaries_path, arch, "extracted_binaries") + extracted_bin_list = [item for item in os.listdir(extracted_bin_path)] + packaged_bin_path = os.path.join(binaries_path, arch, "packaged_binaries") + packaged_bin_file_list = os.listdir(packaged_bin_path) + # see if the package is already in the extracted ones; maybe it was deployed when exported the env. + if os.path.exists(os.path.join(extracted_bin_path, param_list[0], param_list[1] if param_list[1] else "")): + return "binary %s is already extracted" % param_list[0] + else: # we need to search for it in the packaged binaries directory. It may have been shipped after export + for item in packaged_bin_file_list: + if param_list[1]: + if re.match("%s-%s.*rpm" % (param_list[0], param_list[1]), item): # package with version + if not os.path.exists(os.path.join(extracted_bin_path, param_list[0],param_list[1])): + os.makedirs(os.path.join(extracted_bin_path, param_list[0], param_list[1])) + extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0],param_list[1])) + bb.plain("Using {} for {}".format(os.path.join(packaged_bin_path, item), param_list[0])) + break + else: + if re.match("%s-.*rpm" % param_list[0], item): + found_version = re.match(".*-([0-9]+\.[0-9]+)-", item).group(1) + if not os.path.exists(os.path.join(extracted_bin_path, param_list[0], found_version)): + os.makedirs(os.path.join(extracted_bin_path, param_list[0], found_version)) + bb.plain("Used ver. %s for %s" % (found_version, param_list[0])) + extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0], found_version)) + break + else: + bb.warn("Couldn't find target package %s%s. Please ensure it is available " + "in either of these directories: extracted_binaries or packaged_binaries. " + "Related tests will probably fail." % (param_list[0], " with version " + param_list[1] if param_list[1] else "")) + return + return "Binary %s extracted successfully." % param_list[0] + + +def files_to_copy(base_dir): + """ + Produces a list of files relative to the base dir path sent as param + :return: the list of relative path files + """ + files_list = [] + dir_list = [base_dir] + count = 1 + dir_count = 1 + while (dir_count == 1 or dir_count != count): + count = dir_count + for dir in dir_list: + for item in os.listdir(dir): + if os.path.isdir(os.path.join(dir, item)) and os.path.join(dir, item) not in dir_list: + dir_list.append(os.path.join(dir, item)) + dir_count = len(dir_list) + elif os.path.join(dir, item) not in files_list and os.path.isfile(os.path.join(dir, item)): + files_list.append(os.path.join(dir, item)) + return files_list + + +def send_bin_to_DUT(d,params): + from oeqa.oetest import oeRuntimeTest + param_list = params + cleanup_list = list() + bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries") if determine_if_poky_env() \ + else os.getenv("bin_dir") + arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(bins_dir)) + arch_rpms_dir = os.path.join(bins_dir, arch, "packaged_binaries") + extracted_bin_dir = os.path.join(bins_dir, arch, "extracted_binaries", param_list[0]) + + def send_extracted_binary(): + bin_local_dir = os.path.join(extracted_bin_dir, param_list[1] if param_list[1] else os.listdir(extracted_bin_dir)[0]) + for item in files_to_copy(bin_local_dir): + split_path = item.split(bin_local_dir)[1] + path_on_DUT = split_path if split_path[0] is "/" else "/" + split_path # create the path as on DUT; eg. /usr/bin/bin_file + (status, output) = oeRuntimeTest.tc.target.copy_to(item, path_on_DUT) + if status != 0: + bb.warn("Failed to copy %s binary file %s on the remote target: %s" % + (param_list[0], "ver. " + param_list[1] if param_list[1] else "", d.getVar("MACHINE"))) + return + if param_list[4] == "rm": + cleanup_list.append(path_on_DUT) + return cleanup_list + + def send_rpm(remote_path): # if it is not required to have an extracted binary, but to send an .rpm file + rpm_to_send = "" + for item in os.listdir(arch_rpms_dir): + if param_list[1] and re.match("%s-%s-.*rpm"%(param_list[0], param_list[1]), item): + rpm_to_send = item + break + elif re.match("%s-[0-9]+\.[0-9]+-.*rpm" % param_list[0], item): + rpm_to_send = item + break + else: + bb.warn("No rpm package found for %s %s in .rpm files dir %s. Skipping deployment." % + (param_list[0], "ver. " + param_list[1] if param_list[1] else "", rpms_file_dir) ) + return + (status, output) = oeRuntimeTest.tc.target.copy_to(os.path.join(arch_rpms_dir, rpm_to_send), remote_path) + if status != 0: + bb.warn("Failed to copy %s on the remote target: %s" %(param_list[0], d.getVar("MACHINE"))) + return + if param_list[4] == "rm": + cleanup_list.append(os.path.join(remote_path, rpm_to_send)) + return cleanup_list + + if param_list[2] == "rpm": # send an .rpm file + return send_rpm("/home/root") # rpms will be sent on home dir of remote machine + else: + return send_extracted_binary() + + +def rm_bin(removal_list): # need to know both if the binary is sent archived and the path where it is sent if archived + from oeqa.oetest import oeRuntimeTest + for item in removal_list: + (status,output) = oeRuntimeTest.tc.target.run("rm " + item) + if status != 0: + bb.warn("Failed to remove: %s. Please ensure connection with the target device is up and running and " + "you have the needed rights." % item) + |