diff options
Diffstat (limited to 'yocto-poky/meta/lib')
34 files changed, 1221 insertions, 466 deletions
diff --git a/yocto-poky/meta/lib/oe/copy_buildsystem.py b/yocto-poky/meta/lib/oe/copy_buildsystem.py index 979578c41..c0e7541c0 100644 --- a/yocto-poky/meta/lib/oe/copy_buildsystem.py +++ b/yocto-poky/meta/lib/oe/copy_buildsystem.py @@ -14,8 +14,9 @@ def _smart_copy(src, dest): shutil.copymode(src, dest) class BuildSystem(object): - def __init__(self, d): + def __init__(self, context, d): self.d = d + self.context = context self.layerdirs = d.getVar('BBLAYERS', True).split() def copy_bitbake_and_layers(self, destdir): @@ -38,7 +39,7 @@ class BuildSystem(object): if os.path.exists(layerconf): with open(layerconf, 'r') as f: if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): - bb.warn("Skipping local workspace layer %s" % layer) + bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) continue # If the layer was already under corebase, leave it there diff --git a/yocto-poky/meta/lib/oe/distro_check.py b/yocto-poky/meta/lib/oe/distro_check.py index 8ed5b0ec8..f92cd2e42 100644 --- a/yocto-poky/meta/lib/oe/distro_check.py +++ b/yocto-poky/meta/lib/oe/distro_check.py @@ -1,7 +1,23 @@ -def get_links_from_url(url): +from contextlib import contextmanager +@contextmanager +def create_socket(url, d): + import urllib + socket = urllib.urlopen(url, proxies=get_proxies(d)) + try: + yield socket + finally: + socket.close() + +def get_proxies(d): + import os + proxykeys = ['http', 'https', 'ftp', 'ftps', 'no', 'all'] + proxyvalues = map(lambda key: d.getVar(key+'_proxy', True), proxykeys) + return dict(zip(proxykeys, proxyvalues)) + +def get_links_from_url(url, d): "Return all the href links found on the web location" - import urllib, sgmllib + import sgmllib class LinksParser(sgmllib.SGMLParser): def parse(self, s): @@ -24,19 +40,18 @@ def get_links_from_url(url): "Return the list of hyperlinks." return self.hyperlinks - sock = urllib.urlopen(url) - webpage = sock.read() - sock.close() + with create_socket(url,d) as sock: + webpage = sock.read() linksparser = LinksParser() linksparser.parse(webpage) return linksparser.get_hyperlinks() -def find_latest_numeric_release(url): +def find_latest_numeric_release(url, d): "Find the latest listed numeric release on the given url" max=0 maxstr="" - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): try: release = float(link) except: @@ -70,7 +85,7 @@ def clean_package_list(package_list): return set.keys() -def get_latest_released_meego_source_package_list(): +def get_latest_released_meego_source_package_list(d): "Returns list of all the name os packages in the latest meego distro" package_names = [] @@ -82,11 +97,11 @@ def get_latest_released_meego_source_package_list(): package_list=clean_package_list(package_names) return "1.0", package_list -def get_source_package_list_from_url(url, section): +def get_source_package_list_from_url(url, section, d): "Return a sectioned list of package names from a URL list" bb.note("Reading %s: %s" % (url, section)) - links = get_links_from_url(url) + links = get_links_from_url(url, d) srpms = filter(is_src_rpm, links) names_list = map(package_name_from_srpm, srpms) @@ -96,44 +111,44 @@ def get_source_package_list_from_url(url, section): return new_pkgs -def get_latest_released_fedora_source_package_list(): +def get_latest_released_fedora_source_package_list(d): "Returns list of all the name os packages in the latest fedora distro" - latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/") + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) - package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main") + package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d) # package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything") - package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates") + package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def get_latest_released_opensuse_source_package_list(): +def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/") + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main") - package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates") + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d) + package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def get_latest_released_mandriva_source_package_list(): +def get_latest_released_mandriva_source_package_list(d): "Returns list of all the name os packages in the latest mandriva distro" - latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/") - package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main") + latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d) + package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d) # package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib") - package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates") + package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def find_latest_debian_release(url): +def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url" releases = [] - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): if link[:6] == "Debian": if ';' not in link: releases.append(link) @@ -143,16 +158,15 @@ def find_latest_debian_release(url): except: return "_NotFound_" -def get_debian_style_source_package_list(url, section): +def get_debian_style_source_package_list(url, section, d): "Return the list of package-names stored in the debian style Sources.gz file" - import urllib - sock = urllib.urlopen(url) - import tempfile - tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) - tmpfilename=tmpfile.name - tmpfile.write(sock.read()) - sock.close() - tmpfile.close() + with create_socket(url,d) as sock: + webpage = sock.read() + import tempfile + tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False) + tmpfilename=tmpfile.name + tmpfile.write(sock.read()) + tmpfile.close() import gzip bb.note("Reading %s: %s" % (url, section)) @@ -165,41 +179,41 @@ def get_debian_style_source_package_list(url, section): return package_names -def get_latest_released_debian_source_package_list(): +def get_latest_released_debian_source_package_list(d): "Returns list of all the name os packages in the latest debian distro" - latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/") + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" - package_names = get_debian_style_source_package_list(url, "main") + package_names = get_debian_style_source_package_list(url, "main", d) # url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" # package_names += get_debian_style_source_package_list(url, "contrib") url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" - package_names += get_debian_style_source_package_list(url, "updates") + package_names += get_debian_style_source_package_list(url, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def find_latest_ubuntu_release(url): +def find_latest_ubuntu_release(url, d): "Find the latest listed ubuntu release on the given url" url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url): + for link in get_links_from_url(url, d): if link[-8:] == "-updates": return link[:-8] return "_NotFound_" -def get_latest_released_ubuntu_source_package_list(): +def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/") + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest - package_names = get_debian_style_source_package_list(url, "main") + package_names = get_debian_style_source_package_list(url, "main", d) # url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest # package_names += get_debian_style_source_package_list(url, "multiverse") # url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest # package_names += get_debian_style_source_package_list(url, "universe") url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest - package_names += get_debian_style_source_package_list(url, "updates") + package_names += get_debian_style_source_package_list(url, "updates", d) package_list=clean_package_list(package_names) return latest, package_list -def create_distro_packages_list(distro_check_dir): +def create_distro_packages_list(distro_check_dir, d): pkglst_dir = os.path.join(distro_check_dir, "package_lists") if not os.path.isdir (pkglst_dir): os.makedirs(pkglst_dir) @@ -220,7 +234,7 @@ def create_distro_packages_list(distro_check_dir): begin = datetime.now() for distro in per_distro_functions: name = distro[0] - release, package_list = distro[1]() + release, package_list = distro[1](d) bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) package_list_file = os.path.join(pkglst_dir, name + "-" + release) f = open(package_list_file, "w+b") @@ -231,7 +245,7 @@ def create_distro_packages_list(distro_check_dir): delta = end - begin bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds) -def update_distro_data(distro_check_dir, datetime): +def update_distro_data(distro_check_dir, datetime, d): """ If distro packages list data is old then rebuild it. The operations has to be protected by a lock so that @@ -258,7 +272,7 @@ def update_distro_data(distro_check_dir, datetime): if saved_datetime[0:8] != datetime[0:8]: bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) bb.note("Regenerating distro package lists") - create_distro_packages_list(distro_check_dir) + create_distro_packages_list(distro_check_dir, d) f.seek(0) f.write(datetime) diff --git a/yocto-poky/meta/lib/oe/image.py b/yocto-poky/meta/lib/oe/image.py index f9e9bfd58..b9eb3de5a 100644 --- a/yocto-poky/meta/lib/oe/image.py +++ b/yocto-poky/meta/lib/oe/image.py @@ -5,7 +5,7 @@ import multiprocessing def generate_image(arg): - (type, subimages, create_img_cmd) = arg + (type, subimages, create_img_cmd, sprefix) = arg bb.note("Running image creation script for %s: %s ..." % (type, create_img_cmd)) @@ -54,14 +54,16 @@ class ImageDepGraph(object): base_type = self._image_base_type(node) deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") base_deps = (self.d.getVar('IMAGE_TYPEDEP_' + base_type, True) or "") - if deps != "" or base_deps != "": - graph[node] = deps - for dep in deps.split() + base_deps.split(): - if not dep in graph: - add_node(dep) - else: - graph[node] = "" + graph[node] = "" + for dep in deps.split() + base_deps.split(): + if not dep in graph[node]: + if graph[node] != "": + graph[node] += " " + graph[node] += dep + + if not dep in graph: + add_node(dep) for fstype in image_fstypes: add_node(fstype) @@ -264,9 +266,9 @@ class Image(ImageDepGraph): return (alltypes, filtered_groups, cimages) - def _write_script(self, type, cmds): + def _write_script(self, type, cmds, sprefix=""): tempdir = self.d.getVar('T', True) - script_name = os.path.join(tempdir, "create_image." + type) + script_name = os.path.join(tempdir, sprefix + "create_image." + type) rootfs_size = self._get_rootfs_size() self.d.setVar('img_creation_func', '\n'.join(cmds)) @@ -284,7 +286,7 @@ class Image(ImageDepGraph): return script_name - def _get_imagecmds(self): + def _get_imagecmds(self, sprefix=""): old_overrides = self.d.getVar('OVERRIDES', 0) alltypes, fstype_groups, cimages = self._get_image_types() @@ -320,9 +322,9 @@ class Image(ImageDepGraph): else: subimages.append(type) - script_name = self._write_script(type, cmds) + script_name = self._write_script(type, cmds, sprefix) - image_cmds.append((type, subimages, script_name)) + image_cmds.append((type, subimages, script_name, sprefix)) image_cmd_groups.append(image_cmds) @@ -355,6 +357,27 @@ class Image(ImageDepGraph): image_cmd_groups = self._get_imagecmds() + # Process the debug filesystem... + debugfs_d = bb.data.createCopy(self.d) + if self.d.getVar('IMAGE_GEN_DEBUGFS', True) == "1": + bb.note("Processing debugfs image(s) ...") + orig_d = self.d + self.d = debugfs_d + + self.d.setVar('IMAGE_ROOTFS', orig_d.getVar('IMAGE_ROOTFS', True) + '-dbg') + self.d.setVar('IMAGE_NAME', orig_d.getVar('IMAGE_NAME', True) + '-dbg') + self.d.setVar('IMAGE_LINK_NAME', orig_d.getVar('IMAGE_LINK_NAME', True) + '-dbg') + + debugfs_image_fstypes = orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True) + if debugfs_image_fstypes: + self.d.setVar('IMAGE_FSTYPES', orig_d.getVar('IMAGE_FSTYPES_DEBUGFS', True)) + + self._remove_old_symlinks() + + image_cmd_groups += self._get_imagecmds("debugfs.") + + self.d = orig_d + self._write_wic_env() for image_cmds in image_cmd_groups: @@ -369,9 +392,16 @@ class Image(ImageDepGraph): if result is not None: bb.fatal(result) - for image_type, subimages, script in image_cmds: - bb.note("Creating symlinks for %s image ..." % image_type) - self._create_symlinks(subimages) + for image_type, subimages, script, sprefix in image_cmds: + if sprefix == 'debugfs.': + bb.note("Creating symlinks for %s debugfs image ..." % image_type) + orig_d = self.d + self.d = debugfs_d + self._create_symlinks(subimages) + self.d = orig_d + else: + bb.note("Creating symlinks for %s image ..." % image_type) + self._create_symlinks(subimages) execute_pre_post_process(self.d, post_process_cmds) diff --git a/yocto-poky/meta/lib/oe/package_manager.py b/yocto-poky/meta/lib/oe/package_manager.py index 292ed4446..b9fa6d879 100644 --- a/yocto-poky/meta/lib/oe/package_manager.py +++ b/yocto-poky/meta/lib/oe/package_manager.py @@ -133,8 +133,11 @@ class RpmIndexer(Indexer): if pkgfeed_gpg_name: repomd_file = os.path.join(arch_dir, 'repodata', 'repomd.xml') gpg_cmd = "%s --detach-sign --armor --batch --no-tty --yes " \ - "--passphrase-file '%s' -u '%s' %s" % (gpg_bin, - pkgfeed_gpg_pass, pkgfeed_gpg_name, repomd_file) + "--passphrase-file '%s' -u '%s' " % \ + (gpg_bin, pkgfeed_gpg_pass, pkgfeed_gpg_name) + if self.d.getVar('GPG_PATH', True): + gpg_cmd += "--homedir %s " % self.d.getVar('GPG_PATH', True) + gpg_cmd += repomd_file repo_sign_cmds.append(gpg_cmd) rpm_dirs_found = True @@ -200,6 +203,8 @@ class OpkgIndexer(Indexer): result = oe.utils.multiprocess_exec(index_cmds, create_index) if result: bb.fatal('%s' % ('\n'.join(result))) + if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': + raise NotImplementedError('Package feed signing not implementd for ipk') @@ -275,6 +280,8 @@ class DpkgIndexer(Indexer): result = oe.utils.multiprocess_exec(index_cmds, create_index) if result: bb.fatal('%s' % ('\n'.join(result))) + if self.d.getVar('PACKAGE_FEED_SIGN', True) == '1': + raise NotImplementedError('Package feed signing not implementd for dpkg') @@ -434,24 +441,30 @@ class OpkgPkgsList(PkgsList): (self.opkg_cmd, self.opkg_args) try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + # bb.note(cmd) + tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip() + except subprocess.CalledProcessError as e: bb.fatal("Cannot get the installed packages list. Command '%s' " "returned %d:\n%s" % (cmd, e.returncode, e.output)) - if output and format == "file": - tmp_output = "" - for line in output.split('\n'): + output = list() + for line in tmp_output.split('\n'): + if len(line.strip()) == 0: + continue + if format == "file": pkg, pkg_file, pkg_arch = line.split() full_path = os.path.join(self.rootfs_dir, pkg_arch, pkg_file) if os.path.exists(full_path): - tmp_output += "%s %s %s\n" % (pkg, full_path, pkg_arch) + output.append('%s %s %s' % (pkg, full_path, pkg_arch)) else: - tmp_output += "%s %s %s\n" % (pkg, pkg_file, pkg_arch) + output.append('%s %s %s' % (pkg, pkg_file, pkg_arch)) + else: + output.append(line) - output = tmp_output + output.sort() - return output + return '\n'.join(output) class DpkgPkgsList(PkgsList): @@ -605,12 +618,12 @@ class PackageManager(object): cmd.extend(['-x', exclude]) try: bb.note("Installing complementary packages ...") + bb.note('Running %s' % cmd) complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: bb.fatal("Could not compute complementary packages list. Command " "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output)) - self.install(complementary_pkgs.split(), attempt_only=True) def deploy_dir_lock(self): @@ -1050,6 +1063,35 @@ class RpmPM(PackageManager): def update(self): self._invoke_smart('update rpmsys') + def get_rdepends_recursively(self, pkgs): + # pkgs will be changed during the loop, so use [:] to make a copy. + for pkg in pkgs[:]: + sub_data = oe.packagedata.read_subpkgdata(pkg, self.d) + sub_rdep = sub_data.get("RDEPENDS_" + pkg) + if not sub_rdep: + continue + done = bb.utils.explode_dep_versions2(sub_rdep).keys() + next = done + # Find all the rdepends on dependency chain + while next: + new = [] + for sub_pkg in next: + sub_data = oe.packagedata.read_subpkgdata(sub_pkg, self.d) + sub_pkg_rdep = sub_data.get("RDEPENDS_" + sub_pkg) + if not sub_pkg_rdep: + continue + for p in bb.utils.explode_dep_versions2(sub_pkg_rdep): + # Already handled, skip it. + if p in done or p in pkgs: + continue + # It's a new dep + if oe.packagedata.has_subpkgdata(p, self.d): + done.append(p) + new.append(p) + next = new + pkgs.extend(done) + return pkgs + ''' Install pkgs with smart, the pkg name is oe format ''' @@ -1059,8 +1101,58 @@ class RpmPM(PackageManager): bb.note("There are no packages to install") return bb.note("Installing the following packages: %s" % ' '.join(pkgs)) + if not attempt_only: + # Pull in multilib requires since rpm may not pull in them + # correctly, for example, + # lib32-packagegroup-core-standalone-sdk-target requires + # lib32-libc6, but rpm may pull in libc6 rather than lib32-libc6 + # since it doesn't know mlprefix (lib32-), bitbake knows it and + # can handle it well, find out the RDEPENDS on the chain will + # fix the problem. Both do_rootfs and do_populate_sdk have this + # issue. + # The attempt_only packages don't need this since they are + # based on the installed ones. + # + # Separate pkgs into two lists, one is multilib, the other one + # is non-multilib. + ml_pkgs = [] + non_ml_pkgs = pkgs[:] + for pkg in pkgs: + for mlib in (self.d.getVar("MULTILIB_VARIANTS", True) or "").split(): + if pkg.startswith(mlib + '-'): + ml_pkgs.append(pkg) + non_ml_pkgs.remove(pkg) + + if len(ml_pkgs) > 0 and len(non_ml_pkgs) > 0: + # Found both foo and lib-foo + ml_pkgs = self.get_rdepends_recursively(ml_pkgs) + non_ml_pkgs = self.get_rdepends_recursively(non_ml_pkgs) + # Longer list makes smart slower, so only keep the pkgs + # which have the same BPN, and smart can handle others + # correctly. + pkgs_new = [] + for pkg in non_ml_pkgs: + for mlib in (self.d.getVar("MULTILIB_VARIANTS", True) or "").split(): + mlib_pkg = mlib + "-" + pkg + if mlib_pkg in ml_pkgs: + pkgs_new.append(pkg) + pkgs_new.append(mlib_pkg) + for pkg in pkgs: + if pkg not in pkgs_new: + pkgs_new.append(pkg) + pkgs = pkgs_new + new_depends = {} + deps = bb.utils.explode_dep_versions2(" ".join(pkgs)) + for depend in deps: + data = oe.packagedata.read_subpkgdata(depend, self.d) + key = "PKG_%s" % depend + if key in data: + new_depend = data[key] + else: + new_depend = depend + new_depends[new_depend] = deps[depend] + pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ') pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) - if not attempt_only: bb.note('to be installed: %s' % ' '.join(pkgs)) cmd = "%s %s install -y %s" % \ @@ -1379,6 +1471,16 @@ class OpkgPM(PackageManager): self.d.getVar('FEED_DEPLOYDIR_BASE_URI', True), arch)) + if self.opkg_dir != '/var/lib/opkg': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" + cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) + cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) + + def _create_config(self): with open(self.config_file, "w+") as config_file: priority = 1 @@ -1394,6 +1496,15 @@ class OpkgPM(PackageManager): config_file.write("src oe-%s file:%s\n" % (arch, pkgs_dir)) + if self.opkg_dir != '/var/lib/opkg': + # There is no command line option for this anymore, we need to add + # info_dir and status_file to config file, if OPKGLIBDIR doesn't have + # the default value of "/var/lib" as defined in opkg: + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR "/var/lib/opkg/info" + # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE "/var/lib/opkg/status" + config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'info')) + config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR', True), 'opkg', 'status')) + def insert_feeds_uris(self): if self.feed_uris == "": return @@ -1433,7 +1544,7 @@ class OpkgPM(PackageManager): self.deploy_dir_unlock() def install(self, pkgs, attempt_only=False): - if attempt_only and len(pkgs) == 0: + if not pkgs: return cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) diff --git a/yocto-poky/meta/lib/oe/patch.py b/yocto-poky/meta/lib/oe/patch.py index 108bf1de5..2bf501e9e 100644 --- a/yocto-poky/meta/lib/oe/patch.py +++ b/yocto-poky/meta/lib/oe/patch.py @@ -337,12 +337,15 @@ class GitApplyTree(PatchTree): return (tmpfile, cmd) @staticmethod - def extractPatches(tree, startcommit, outdir): + def extractPatches(tree, startcommit, outdir, paths=None): import tempfile import shutil tempdir = tempfile.mkdtemp(prefix='oepatch') try: shellcmd = ["git", "format-patch", startcommit, "-o", tempdir] + if paths: + shellcmd.append('--') + shellcmd.extend(paths) out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) if out: for srcfile in out.split(): @@ -407,6 +410,13 @@ class GitApplyTree(PatchTree): runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) except CmdError: pass + # git am won't always clean up after itself, sadly, so... + shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Also need to take care of any stray untracked files + shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"] + runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) + # Fall back to git apply shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']] try: diff --git a/yocto-poky/meta/lib/oe/recipeutils.py b/yocto-poky/meta/lib/oe/recipeutils.py index d4fa72651..119a68821 100644 --- a/yocto-poky/meta/lib/oe/recipeutils.py +++ b/yocto-poky/meta/lib/oe/recipeutils.py @@ -31,9 +31,13 @@ def pn_to_recipe(cooker, pn): import bb.providers if pn in cooker.recipecache.pkg_pn: - filenames = cooker.recipecache.pkg_pn[pn] best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn) return best[3] + elif pn in cooker.recipecache.providers: + filenames = cooker.recipecache.providers[pn] + eligible, foundUnique = bb.providers.filterProviders(filenames, pn, cooker.expanded_data, cooker.recipecache) + filename = eligible[0] + return filename else: return None @@ -72,6 +76,8 @@ def parse_recipe_simple(cooker, pn, d, appends=True): raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn) if appends: appendfiles = cooker.collection.get_file_appends(recipefile) + else: + appendfiles = None return parse_recipe(recipefile, appendfiles, d) @@ -95,6 +101,63 @@ def get_var_files(fn, varlist, d): return varfiles +def split_var_value(value, assignment=True): + """ + Split a space-separated variable's value into a list of items, + taking into account that some of the items might be made up of + expressions containing spaces that should not be split. + Parameters: + value: + The string value to split + assignment: + True to assume that the value represents an assignment + statement, False otherwise. If True, and an assignment + statement is passed in the first item in + the returned list will be the part of the assignment + statement up to and including the opening quote character, + and the last item will be the closing quote. + """ + inexpr = 0 + lastchar = None + out = [] + buf = '' + for char in value: + if char == '{': + if lastchar == '$': + inexpr += 1 + elif char == '}': + inexpr -= 1 + elif assignment and char in '"\'' and inexpr == 0: + if buf: + out.append(buf) + out.append(char) + char = '' + buf = '' + elif char.isspace() and inexpr == 0: + char = '' + if buf: + out.append(buf) + buf = '' + buf += char + lastchar = char + if buf: + out.append(buf) + + # Join together assignment statement and opening quote + outlist = out + if assignment: + assigfound = False + for idx, item in enumerate(out): + if '=' in item: + assigfound = True + if assigfound: + if '"' in item or "'" in item: + outlist = [' '.join(out[:idx+1])] + outlist.extend(out[idx+1:]) + break + return outlist + + def patch_recipe_file(fn, values, patch=False, relpath=''): """Update or insert variable values into a recipe file (assuming you have already identified the exact file you want to update.) @@ -112,7 +175,7 @@ def patch_recipe_file(fn, values, patch=False, relpath=''): if name in nowrap_vars: tf.write(rawtext) elif name in list_vars: - splitvalue = values[name].split() + splitvalue = split_var_value(values[name], assignment=False) if len(splitvalue) > 1: linesplit = ' \\\n' + (' ' * (len(name) + 4)) tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit)) @@ -277,6 +340,22 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True): return remotes +def get_recipe_local_files(d, patches=False): + """Get a list of local files in SRC_URI within a recipe.""" + uris = (d.getVar('SRC_URI', True) or "").split() + fetch = bb.fetch2.Fetch(uris, d) + ret = {} + for uri in uris: + if fetch.ud[uri].type == 'file': + if (not patches and + bb.utils.exec_flat_python_func('patch_path', uri, fetch, '')): + continue + # Skip files that are referenced by absolute path + if not os.path.isabs(fetch.ud[uri].basepath): + ret[fetch.ud[uri].basepath] = fetch.localpath(uri) + return ret + + def get_recipe_patches(d): """Get a list of the patches included in SRC_URI within a recipe.""" patchfiles = [] @@ -518,7 +597,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, instfunclines.append(line) return (instfunclines, None, 4, False) else: - splitval = origvalue.split() + splitval = split_var_value(origvalue, assignment=False) changed = False removevar = varname if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]: @@ -673,11 +752,14 @@ def get_recipe_upstream_version(rd): ru['type'] = 'U' ru['datetime'] = '' + pv = rd.getVar('PV', True) + # XXX: If don't have SRC_URI means that don't have upstream sources so - # returns 1.0. + # returns the current recipe version, so that upstream version check + # declares a match. src_uris = rd.getVar('SRC_URI', True) if not src_uris: - ru['version'] = '1.0' + ru['version'] = pv ru['type'] = 'M' ru['datetime'] = datetime.now() return ru @@ -686,8 +768,6 @@ def get_recipe_upstream_version(rd): src_uri = src_uris.split()[0] uri_type, _, _, _, _, _ = decodeurl(src_uri) - pv = rd.getVar('PV', True) - manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION", True) if manual_upstream_version: # manual tracking of upstream version. diff --git a/yocto-poky/meta/lib/oe/rootfs.py b/yocto-poky/meta/lib/oe/rootfs.py index 3b53fce4a..18df22d9a 100644 --- a/yocto-poky/meta/lib/oe/rootfs.py +++ b/yocto-poky/meta/lib/oe/rootfs.py @@ -66,6 +66,7 @@ class Rootfs(object): m = r.search(line) if m: found_error = 1 + bb.warn('[log_check] In line: [%s]' % line) bb.warn('[log_check] %s: found an error message in the logfile (keyword \'%s\'):\n[log_check] %s' % (self.d.getVar('PN', True), m.group(), line)) @@ -278,6 +279,7 @@ class Rootfs(object): bb.note("Running intercept scripts:") os.environ['D'] = self.image_rootfs + os.environ['STAGING_DIR_NATIVE'] = self.d.getVar('STAGING_DIR_NATIVE', True) for script in os.listdir(intercepts_dir): script_full = os.path.join(intercepts_dir, script) @@ -595,7 +597,11 @@ class DpkgOpkgRootfs(Rootfs): pkg_list = [] - pkgs = self._get_pkgs_postinsts(status_file) + pkgs = None + if not self.d.getVar('PACKAGE_INSTALL', True).strip(): + bb.note("Building empty image") + else: + pkgs = self._get_pkgs_postinsts(status_file) if pkgs: root = "__packagegroup_postinst__" pkgs[root] = pkgs.keys() diff --git a/yocto-poky/meta/lib/oe/sdk.py b/yocto-poky/meta/lib/oe/sdk.py index 53da0f01a..3103f4889 100644 --- a/yocto-poky/meta/lib/oe/sdk.py +++ b/yocto-poky/meta/lib/oe/sdk.py @@ -5,6 +5,7 @@ from oe.package_manager import * import os import shutil import glob +import traceback class Sdk(object): @@ -25,7 +26,7 @@ class Sdk(object): else: self.manifest_dir = manifest_dir - bb.utils.remove(self.sdk_output, True) + self.remove(self.sdk_output, True) self.install_order = Manifest.INSTALL_ORDER @@ -34,29 +35,56 @@ class Sdk(object): pass def populate(self): - bb.utils.mkdirhier(self.sdk_output) + self.mkdirhier(self.sdk_output) # call backend dependent implementation self._populate() # Don't ship any libGL in the SDK - bb.utils.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), - "libGL*")) + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk', True).strip('/'), + "libGL*")) # Fix or remove broken .la files - bb.utils.remove(os.path.join(self.sdk_output, self.sdk_native_path, - self.d.getVar('libdir_nativesdk', True).strip('/'), - "*.la")) + self.remove(os.path.join(self.sdk_output, self.sdk_native_path, + self.d.getVar('libdir_nativesdk', True).strip('/'), + "*.la")) # Link the ld.so.cache file into the hosts filesystem link_name = os.path.join(self.sdk_output, self.sdk_native_path, self.sysconfdir, "ld.so.cache") - bb.utils.mkdirhier(os.path.dirname(link_name)) + self.mkdirhier(os.path.dirname(link_name)) os.symlink("/etc/ld.so.cache", link_name) execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND', True)) + def movefile(self, sourcefile, destdir): + try: + # FIXME: this check of movefile's return code to None should be + # fixed within the function to use only exceptions to signal when + # something goes wrong + if (bb.utils.movefile(sourcefile, destdir) == None): + raise OSError("moving %s to %s failed" + %(sourcefile, destdir)) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.error("unable to place %s in final SDK location" % sourcefile) + + def mkdirhier(self, dirpath): + try: + bb.utils.mkdirhier(dirpath) + except OSError as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.fatal("cannot make dir for SDK: %s" % dirpath) + + def remove(self, path, recurse=False): + try: + bb.utils.remove(path, recurse) + #FIXME: using umbrella exc catching because bb.utils method raises it + except Exception as e: + bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) + bb.warn("cannot remove SDK dir: %s" % path) class RpmSdk(Sdk): def __init__(self, d, manifest_dir=None): @@ -143,15 +171,15 @@ class RpmSdk(Sdk): "lib", "rpm" ) - bb.utils.mkdirhier(native_rpm_state_dir) + self.mkdirhier(native_rpm_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "rpm", "*")): - bb.utils.movefile(f, native_rpm_state_dir) + self.movefile(f, native_rpm_state_dir) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) + self.remove(os.path.join(self.sdk_output, "var"), True) # Move host sysconfig data native_sysconf_dir = os.path.join(self.sdk_output, @@ -159,10 +187,10 @@ class RpmSdk(Sdk): self.d.getVar('sysconfdir', True).strip('/'), ) - bb.utils.mkdirhier(native_sysconf_dir) + self.mkdirhier(native_sysconf_dir) for f in glob.glob(os.path.join(self.sdk_output, "etc", "*")): - bb.utils.movefile(f, native_sysconf_dir) - bb.utils.remove(os.path.join(self.sdk_output, "etc"), True) + self.movefile(f, native_sysconf_dir) + self.remove(os.path.join(self.sdk_output, "etc"), True) class OpkgSdk(Sdk): @@ -219,12 +247,12 @@ class OpkgSdk(Sdk): target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir) host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir) - bb.utils.mkdirhier(target_sysconfdir) + self.mkdirhier(target_sysconfdir) shutil.copy(self.target_conf, target_sysconfdir) os.chmod(os.path.join(target_sysconfdir, os.path.basename(self.target_conf)), 0644) - bb.utils.mkdirhier(host_sysconfdir) + self.mkdirhier(host_sysconfdir) shutil.copy(self.host_conf, host_sysconfdir) os.chmod(os.path.join(host_sysconfdir, os.path.basename(self.host_conf)), 0644) @@ -232,11 +260,11 @@ class OpkgSdk(Sdk): native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, self.d.getVar('localstatedir_nativesdk', True).strip('/'), "lib", "opkg") - bb.utils.mkdirhier(native_opkg_state_dir) + self.mkdirhier(native_opkg_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "opkg", "*")): - bb.utils.movefile(f, native_opkg_state_dir) + self.movefile(f, native_opkg_state_dir) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) + self.remove(os.path.join(self.sdk_output, "var"), True) class DpkgSdk(Sdk): @@ -264,7 +292,7 @@ class DpkgSdk(Sdk): def _copy_apt_dir_to(self, dst_dir): staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE", True) - bb.utils.remove(dst_dir, True) + self.remove(dst_dir, True) shutil.copytree(os.path.join(staging_etcdir_native, "apt"), dst_dir) @@ -306,11 +334,11 @@ class DpkgSdk(Sdk): native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path, "var", "lib", "dpkg") - bb.utils.mkdirhier(native_dpkg_state_dir) + self.mkdirhier(native_dpkg_state_dir) for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "dpkg", "*")): - bb.utils.movefile(f, native_dpkg_state_dir) + self.movefile(f, native_dpkg_state_dir) + self.remove(os.path.join(self.sdk_output, "var"), True) - bb.utils.remove(os.path.join(self.sdk_output, "var"), True) def sdk_list_installed_packages(d, target, format=None, rootfs_dir=None): diff --git a/yocto-poky/meta/lib/oe/sstatesig.py b/yocto-poky/meta/lib/oe/sstatesig.py index cb46712ee..6d1be3e37 100644 --- a/yocto-poky/meta/lib/oe/sstatesig.py +++ b/yocto-poky/meta/lib/oe/sstatesig.py @@ -94,6 +94,26 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): self.machine = data.getVar("MACHINE", True) self.mismatch_msgs = [] pass + + def tasks_resolved(self, virtmap, virtpnmap, dataCache): + # Translate virtual/xxx entries to PN values + newabisafe = [] + for a in self.abisaferecipes: + if a in virtpnmap: + newabisafe.append(virtpnmap[a]) + else: + newabisafe.append(a) + self.abisaferecipes = newabisafe + newsafedeps = [] + for a in self.saferecipedeps: + a1, a2 = a.split("->") + if a1 in virtpnmap: + a1 = virtpnmap[a1] + if a2 in virtpnmap: + a2 = virtpnmap[a2] + newsafedeps.append(a1 + "->" + a2) + self.saferecipedeps = newsafedeps + def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) diff --git a/yocto-poky/meta/lib/oeqa/oetest.py b/yocto-poky/meta/lib/oeqa/oetest.py index a6f89b6a8..6f9edec58 100644 --- a/yocto-poky/meta/lib/oeqa/oetest.py +++ b/yocto-poky/meta/lib/oeqa/oetest.py @@ -11,9 +11,14 @@ import os, re, mmap import unittest import inspect import subprocess -import bb -from oeqa.utils.decorators import LogResults, gettag -from sys import exc_info, exc_clear +try: + import bb +except ImportError: + pass +import logging +from oeqa.utils.decorators import LogResults, gettag, getResults + +logger = logging.getLogger("BitBake") def getVar(obj): #extend form dict, if a variable didn't exists, need find it in testcase @@ -89,7 +94,7 @@ def loadTests(tc, type="runtime"): suite.dependencies.append(dep_suite) break else: - bb.warn("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % + logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % (test, depends_on)) # Use brute-force topological sort to determine ordering. Sort by # depth (higher depth = must run later), with original ordering to @@ -106,14 +111,34 @@ def loadTests(tc, type="runtime"): suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) return testloader.suiteClass(suites) +_buffer = "" + +def custom_verbose(msg, *args, **kwargs): + global _buffer + if msg[-1] != "\n": + _buffer += msg + else: + _buffer += msg + try: + bb.plain(_buffer.rstrip("\n"), *args, **kwargs) + except NameError: + logger.info(_buffer.rstrip("\n"), *args, **kwargs) + _buffer = "" + def runTests(tc, type="runtime"): suite = loadTests(tc, type) - bb.note("Test modules %s" % tc.testslist) + logger.info("Test modules %s" % tc.testslist) if hasattr(tc, "tagexp") and tc.tagexp: - bb.note("Filter test cases by tags: %s" % tc.tagexp) - bb.note("Found %s tests" % suite.countTestCases()) + logger.info("Filter test cases by tags: %s" % tc.tagexp) + logger.info("Found %s tests" % suite.countTestCases()) runner = unittest.TextTestRunner(verbosity=2) + try: + if bb.msg.loggerDefaultVerbose: + runner.stream.write = custom_verbose + except NameError: + # Not in bb environment? + pass result = runner.run(suite) return result @@ -158,17 +183,24 @@ class oeRuntimeTest(oeTest): pass def tearDown(self): - # If a test fails or there is an exception - if not exc_info() == (None, None, None): - exc_clear() - #Only dump for QemuTarget - if (type(self.target).__name__ == "QemuTarget"): - self.tc.host_dumper.create_dir(self._testMethodName) - self.tc.host_dumper.dump_host() - self.target.target_dumper.dump_target( - self.tc.host_dumper.dump_dir) - print ("%s dump data stored in %s" % (self._testMethodName, - self.tc.host_dumper.dump_dir)) + res = getResults() + # If a test fails or there is an exception dump + # for QemuTarget only + if (type(self.target).__name__ == "QemuTarget" and + (self.id() in res.getErrorList() or + self.id() in res.getFailList())): + self.tc.host_dumper.create_dir(self._testMethodName) + self.tc.host_dumper.dump_host() + self.target.target_dumper.dump_target( + self.tc.host_dumper.dump_dir) + print ("%s dump data stored in %s" % (self._testMethodName, + self.tc.host_dumper.dump_dir)) + + self.tearDownLocal() + + # Method to be run after tearDown and implemented by child classes + def tearDownLocal(self): + pass #TODO: use package_manager.py to install packages on any type of image def install_packages(self, packagelist): @@ -190,7 +222,7 @@ class oeSDKTest(oeTest): return False def _run(self, cmd): - return subprocess.check_output(cmd, shell=True) + return subprocess.check_output(". %s; " % self.tc.sdkenv + cmd, shell=True) def getmodule(pos=2): # stack returns a list of tuples containg frame information diff --git a/yocto-poky/meta/lib/oeqa/runexported.py b/yocto-poky/meta/lib/oeqa/runexported.py index 96442b1b2..dba0d7aec 100755 --- a/yocto-poky/meta/lib/oeqa/runexported.py +++ b/yocto-poky/meta/lib/oeqa/runexported.py @@ -21,7 +21,7 @@ import sys import os import time -from optparse import OptionParser +import argparse try: import simplejson as json @@ -49,8 +49,8 @@ class FakeTarget(object): def exportStart(self): self.sshlog = os.path.join(self.testdir, "ssh_target_log.%s" % self.datetime) sshloglink = os.path.join(self.testdir, "ssh_target_log") - if os.path.islink(sshloglink): - os.unlink(sshloglink) + if os.path.exists(sshloglink): + os.remove(sshloglink) os.symlink(self.sshlog, sshloglink) print("SSH log file: %s" % self.sshlog) self.connection = SSHControl(self.ip, logfile=self.sshlog) @@ -76,43 +76,41 @@ class TestContext(object): def main(): - usage = "usage: %prog [options] <json file>" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--target-ip", dest="ip", help="The IP address of the target machine. Use this to \ + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--target-ip", dest="ip", help="The IP address of the target machine. Use this to \ overwrite the value determined from TEST_TARGET_IP at build time") - parser.add_option("-s", "--server-ip", dest="server_ip", help="The IP address of this machine. Use this to \ + parser.add_argument("-s", "--server-ip", dest="server_ip", help="The IP address of this machine. Use this to \ overwrite the value determined from TEST_SERVER_IP at build time.") - parser.add_option("-d", "--deploy-dir", dest="deploy_dir", help="Full path to the package feeds, that this \ + parser.add_argument("-d", "--deploy-dir", dest="deploy_dir", help="Full path to the package feeds, that this \ the contents of what used to be DEPLOY_DIR on the build machine. If not specified it will use the value \ specified in the json if that directory actually exists or it will error out.") - parser.add_option("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ + parser.add_argument("-l", "--log-dir", dest="log_dir", help="This sets the path for TEST_LOG_DIR. If not specified \ the current dir is used. This is used for usually creating a ssh log file and a scp test file.") + parser.add_argument("json", help="The json file exported by the build system", default="testdata.json", nargs='?') - (options, args) = parser.parse_args() - if len(args) != 1: - parser.error("Incorrect number of arguments. The one and only argument should be a json file exported by the build system") + args = parser.parse_args() - with open(args[0], "r") as f: + with open(args.json, "r") as f: loaded = json.load(f) - if options.ip: - loaded["target"]["ip"] = options.ip - if options.server_ip: - loaded["target"]["server_ip"] = options.server_ip + if args.ip: + loaded["target"]["ip"] = args.ip + if args.server_ip: + loaded["target"]["server_ip"] = args.server_ip d = MyDataDict() for key in loaded["d"].keys(): d[key] = loaded["d"][key] - if options.log_dir: - d["TEST_LOG_DIR"] = options.log_dir + if args.log_dir: + d["TEST_LOG_DIR"] = args.log_dir else: d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__)) - if options.deploy_dir: - d["DEPLOY_DIR"] = options.deploy_dir + if args.deploy_dir: + d["DEPLOY_DIR"] = args.deploy_dir else: if not os.path.isdir(d["DEPLOY_DIR"]): - raise Exception("The path to DEPLOY_DIR does not exists: %s" % d["DEPLOY_DIR"]) + print("WARNING: The path to DEPLOY_DIR does not exist: %s" % d["DEPLOY_DIR"]) target = FakeTarget(d) diff --git a/yocto-poky/meta/lib/oeqa/runtime/_ptest.py b/yocto-poky/meta/lib/oeqa/runtime/_ptest.py index 81c9c4386..0621028b8 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/_ptest.py +++ b/yocto-poky/meta/lib/oeqa/runtime/_ptest.py @@ -98,7 +98,7 @@ class PtestRunnerTest(oeRuntimeTest): return complementary_pkgs.split() - def setUp(self): + def setUpLocal(self): self.ptest_log = os.path.join(oeRuntimeTest.tc.d.getVar("TEST_LOG_DIR",True), "ptest-%s.log" % oeRuntimeTest.tc.d.getVar('DATETIME', True)) @skipUnlessPassed('test_ssh') diff --git a/yocto-poky/meta/lib/oeqa/runtime/connman.py b/yocto-poky/meta/lib/oeqa/runtime/connman.py index ee69e5df9..bd9dba3bd 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/connman.py +++ b/yocto-poky/meta/lib/oeqa/runtime/connman.py @@ -29,26 +29,3 @@ class ConnmanTest(oeRuntimeTest): if status != 0: print self.service_status("connman") self.fail("No connmand process running") - - @testcase(223) - def test_only_one_connmand_in_background(self): - """ - Summary: Only one connmand in background - Expected: There will be only one connmand instance in background. - Product: BSPs - Author: Alexandru Georgescu <alexandru.c.georgescu@intel.com> - AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> - """ - - # Make sure that 'connmand' is running in background - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand') - self.assertEqual(0, status, 'Failed to find "connmand" process running in background.') - - # Start a new instance of 'connmand' - (status, output) = self.target.run('connmand') - self.assertEqual(0, status, 'Failed to start a new "connmand" process.') - - # Make sure that only one 'connmand' is running in background - (status, output) = self.target.run(oeRuntimeTest.pscmd + ' | grep [c]onnmand | wc -l') - self.assertEqual(0, status, 'Failed to find "connmand" process running in background.') - self.assertEqual(1, int(output), 'Found {} connmand processes running, expected 1.'.format(output)) diff --git a/yocto-poky/meta/lib/oeqa/runtime/date.py b/yocto-poky/meta/lib/oeqa/runtime/date.py index 3a8fe8481..447987e07 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/date.py +++ b/yocto-poky/meta/lib/oeqa/runtime/date.py @@ -4,11 +4,11 @@ import re class DateTest(oeRuntimeTest): - def setUp(self): + def setUpLocal(self): if oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True) == "systemd": self.target.run('systemctl stop systemd-timesyncd') - def tearDown(self): + def tearDownLocal(self): if oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager", True) == "systemd": self.target.run('systemctl start systemd-timesyncd') diff --git a/yocto-poky/meta/lib/oeqa/runtime/files/testsdkmakefile b/yocto-poky/meta/lib/oeqa/runtime/files/testsdkmakefile new file mode 100644 index 000000000..fb05f822f --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/runtime/files/testsdkmakefile @@ -0,0 +1,5 @@ +test: test.o + $(CC) -o test test.o -lm +test.o: test.c + $(CC) -c test.c + diff --git a/yocto-poky/meta/lib/oeqa/runtime/kernelmodule.py b/yocto-poky/meta/lib/oeqa/runtime/kernelmodule.py index 2e8172032..38ca18454 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/kernelmodule.py +++ b/yocto-poky/meta/lib/oeqa/runtime/kernelmodule.py @@ -10,7 +10,7 @@ def setUpModule(): class KernelModuleTest(oeRuntimeTest): - def setUp(self): + def setUpLocal(self): self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod.c"), "/tmp/hellomod.c") self.target.copy_to(os.path.join(oeRuntimeTest.tc.filesdir, "hellomod_makefile"), "/tmp/Makefile") @@ -30,5 +30,5 @@ class KernelModuleTest(oeRuntimeTest): (status, output) = self.target.run(cmd, 900) self.assertEqual(status, 0, msg="\n".join([cmd, output])) - def tearDown(self): + def tearDownLocal(self): self.target.run('rm -f /tmp/Makefile /tmp/hellomod.c') diff --git a/yocto-poky/meta/lib/oeqa/runtime/parselogs.py b/yocto-poky/meta/lib/oeqa/runtime/parselogs.py index e20947b8b..fc2bc3893 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/parselogs.py +++ b/yocto-poky/meta/lib/oeqa/runtime/parselogs.py @@ -36,6 +36,8 @@ common_errors = [ 'VGA arbiter: cannot open kernel arbiter, no multi-card support', 'Failed to find URL:http://ipv4.connman.net/online/status.html', 'Online check failed for', + 'netlink init failed', + 'Fast TSC calibration', ] x86_common = [ @@ -46,7 +48,6 @@ x86_common = [ ] + common_errors qemux86_common = [ - 'Fast TSC calibration', 'wrong ELF class', "fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.", "can't claim BAR ", @@ -89,7 +90,7 @@ ignore_errors = { '(EE) open /dev/fb0: No such file or directory', '(EE) AIGLX: reverting to software rendering', ] + x86_common, - 'core2_32' : [ + 'intel-core2-32' : [ 'ACPI: No _BQC method, cannot determine initial brightness', '[Firmware Bug]: ACPI: No _BQC method, cannot determine initial brightness', '(EE) Failed to load module "psb"', @@ -98,6 +99,7 @@ ignore_errors = { '(EE) Failed to load module psbdrv', '(EE) open /dev/fb0: No such file or directory', '(EE) AIGLX: reverting to software rendering', + "controller can't do DEVSLP, turning off", ] + x86_common, 'intel-corei7-64' : [ "controller can't do DEVSLP, turning off", @@ -108,13 +110,9 @@ ignore_errors = { 'edgerouter' : [ 'Fatal server error:', ] + common_errors, - 'minnow' : [ - 'netlink init failed', - ] + common_errors, 'jasperforest' : [ 'Activated service \'org.bluez\' failed:', 'Unable to find NFC netlink family', - 'netlink init failed', ] + common_errors, } @@ -233,8 +231,7 @@ class ParseLogsTest(oeRuntimeTest): #get the output of dmesg and write it in a file. This file is added to log_locations. def write_dmesg(self): - (status, dmesg) = self.target.run("dmesg") - (status, dmesg2) = self.target.run("echo \""+str(dmesg)+"\" > /tmp/dmesg_output.log") + (status, dmesg) = self.target.run("dmesg > /tmp/dmesg_output.log") @testcase(1059) @skipUnlessPassed('test_ssh') diff --git a/yocto-poky/meta/lib/oeqa/runtime/scanelf.py b/yocto-poky/meta/lib/oeqa/runtime/scanelf.py index 43a024ab9..67e02ff45 100644 --- a/yocto-poky/meta/lib/oeqa/runtime/scanelf.py +++ b/yocto-poky/meta/lib/oeqa/runtime/scanelf.py @@ -8,7 +8,7 @@ def setUpModule(): class ScanelfTest(oeRuntimeTest): - def setUp(self): + def setUpLocal(self): self.scancmd = 'scanelf --quiet --recursive --mount --ldpath --path' @testcase(966) diff --git a/yocto-poky/meta/lib/oeqa/sdk/gcc.py b/yocto-poky/meta/lib/oeqa/sdk/gcc.py index 67994b9b5..8395b9b90 100644 --- a/yocto-poky/meta/lib/oeqa/sdk/gcc.py +++ b/yocto-poky/meta/lib/oeqa/sdk/gcc.py @@ -14,7 +14,7 @@ class GccCompileTest(oeSDKTest): @classmethod def setUpClass(self): - for f in ['test.c', 'test.cpp', 'testmakefile']: + for f in ['test.c', 'test.cpp', 'testsdkmakefile']: shutil.copyfile(os.path.join(self.tc.filesdir, f), self.tc.sdktestdir + f) def test_gcc_compile(self): @@ -27,10 +27,10 @@ class GccCompileTest(oeSDKTest): self._run('$CXX %s/test.cpp -o %s/test -lm' % (self.tc.sdktestdir, self.tc.sdktestdir)) def test_make(self): - self._run('cd %s; make -f testmakefile' % self.tc.sdktestdir) + self._run('cd %s; make -f testsdkmakefile' % self.tc.sdktestdir) @classmethod def tearDownClass(self): - files = [self.tc.sdktestdir + f for f in ['test.c', 'test.cpp', 'test.o', 'test', 'testmakefile']] + files = [self.tc.sdktestdir + f for f in ['test.c', 'test.cpp', 'test.o', 'test', 'testsdkmakefile']] for f in files: bb.utils.remove(f) diff --git a/yocto-poky/meta/lib/oeqa/selftest/archiver.py b/yocto-poky/meta/lib/oeqa/selftest/archiver.py new file mode 100644 index 000000000..f2030c446 --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/selftest/archiver.py @@ -0,0 +1,50 @@ +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import bitbake, get_bb_var +from oeqa.utils.decorators import testcase +import glob +import os +import shutil + + +class Archiver(oeSelfTest): + + @testcase(1345) + def test_archiver_allows_to_filter_on_recipe_name(self): + """ + Summary: The archiver should offer the possibility to filter on the recipe. (#6929) + Expected: 1. Included recipe (busybox) should be included + 2. Excluded recipe (zlib) should be excluded + Product: oe-core + Author: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com> + """ + + include_recipe = 'busybox' + exclude_recipe = 'zlib' + + features = 'INHERIT += "archiver"\n' + features += 'ARCHIVER_MODE[src] = "original"\n' + features += 'COPYLEFT_PN_INCLUDE = "%s"\n' % include_recipe + features += 'COPYLEFT_PN_EXCLUDE = "%s"\n' % exclude_recipe + + # Update local.conf + self.write_config(features) + + tmp_dir = get_bb_var('TMPDIR') + deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') + target_sys = get_bb_var('TARGET_SYS') + src_path = os.path.join(deploy_dir_src, target_sys) + + # Delete tmp directory + shutil.rmtree(tmp_dir) + + # Build core-image-minimal + bitbake('core-image-minimal') + + # Check that include_recipe was included + is_included = len(glob.glob(src_path + '/%s*' % include_recipe)) + self.assertEqual(1, is_included, 'Recipe %s was not included.' % include_recipe) + + # Check that exclude_recipe was excluded + is_excluded = len(glob.glob(src_path + '/%s*' % exclude_recipe)) + self.assertEqual(0, is_excluded, 'Recipe %s was not excluded.' % exclude_recipe) diff --git a/yocto-poky/meta/lib/oeqa/selftest/base.py b/yocto-poky/meta/lib/oeqa/selftest/base.py index b2faa661e..9bddc23f8 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/base.py +++ b/yocto-poky/meta/lib/oeqa/selftest/base.py @@ -31,7 +31,7 @@ class oeSelfTest(unittest.TestCase): self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") self.testlayer_path = oeSelfTest.testlayer_path self._extra_tear_down_commands = [] - self._track_for_cleanup = [] + self._track_for_cleanup = [self.testinc_path] super(oeSelfTest, self).__init__(methodName) def setUp(self): diff --git a/yocto-poky/meta/lib/oeqa/selftest/bbtests.py b/yocto-poky/meta/lib/oeqa/selftest/bbtests.py index 3d6860f65..94ca79c03 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/bbtests.py +++ b/yocto-poky/meta/lib/oeqa/selftest/bbtests.py @@ -1,8 +1,5 @@ -import unittest import os -import logging import re -import shutil import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -68,15 +65,43 @@ class BitbakeTests(oeSelfTest): bitbake('-cclean man') self.assertTrue("ERROR: Function failed: patch_do_patch" in result.output, msg = "Though no man-1.5h1-make.patch file exists, bitbake didn't output any err. message. bitbake output: %s" % result.output) + @testcase(1354) + def test_force_task_1(self): + # test 1 from bug 5875 + test_recipe = 'zlib' + test_data = "Microsoft Made No Profit From Anyone's Zunes Yo" + image_dir = get_bb_var('D', test_recipe) + pkgsplit_dir = get_bb_var('PKGDEST', test_recipe) + man_dir = get_bb_var('mandir', test_recipe) + + bitbake('-c cleansstate %s' % test_recipe) + bitbake(test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + man_file = os.path.join(image_dir + man_dir, 'man3/zlib.3') + ftools.append_file(man_file, test_data) + bitbake('-c package -f %s' % test_recipe) + + man_split_file = os.path.join(pkgsplit_dir, 'zlib-doc' + man_dir, 'man3/zlib.3') + man_split_content = ftools.read_file(man_split_file) + self.assertIn(test_data, man_split_content, 'The man file has not changed in packages-split.') + + ret = bitbake(test_recipe) + self.assertIn('task do_package_write_rpm:', ret.output, 'Task do_package_write_rpm did not re-executed.') + @testcase(163) - def test_force_task(self): - bitbake('m4-native') - self.add_command_to_tearDown('bitbake -c clean m4-native') - result = bitbake('-C compile m4-native') - look_for_tasks = ['do_compile', 'do_install', 'do_populate_sysroot'] + def test_force_task_2(self): + # test 2 from bug 5875 + test_recipe = 'zlib' + + bitbake('-c cleansstate %s' % test_recipe) + bitbake(test_recipe) + self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe) + + result = bitbake('-C compile %s' % test_recipe) + look_for_tasks = ['do_compile:', 'do_install:', 'do_populate_sysroot:', 'do_package:'] for task in look_for_tasks: - find_task = re.search("m4-native.*%s" % task, result.output) - self.assertTrue(find_task, msg = "Couldn't find %s task. bitbake output %s" % (task, result.output)) + self.assertIn(task, result.output, msg="Couldn't find %s task.") @testcase(167) def test_bitbake_g(self): @@ -101,6 +126,8 @@ class BitbakeTests(oeSelfTest): self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" """) + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) + bitbake('-ccleanall man') result = bitbake('-c fetch man', ignore_status=True) bitbake('-ccleanall man') @@ -116,20 +143,20 @@ doesn't exist, yet fetcher didn't report any error. bitbake output: %s" % result self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" """) + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) + data = 'SRC_URI_append = ";downloadfilename=test-aspell.tar.gz"' self.write_recipeinc('aspell', data) bitbake('-ccleanall aspell') result = bitbake('-c fetch aspell', ignore_status=True) self.delete_recipeinc('aspell') - self.addCleanup(bitbake, '-ccleanall aspell') self.assertEqual(result.status, 0, msg = "Couldn't fetch aspell. %s" % result.output) self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz')), msg = "File rename failed. No corresponding test-aspell.tar.gz file found under %s" % str(get_bb_var("DL_DIR"))) self.assertTrue(os.path.isfile(os.path.join(get_bb_var("DL_DIR"), 'test-aspell.tar.gz.done')), "File rename failed. No corresponding test-aspell.tar.gz.done file found under %s" % str(get_bb_var("DL_DIR"))) @testcase(1028) def test_environment(self): - self.append_config("TEST_ENV=\"localconf\"") - self.addCleanup(self.remove_config, "TEST_ENV=\"localconf\"") + self.write_config("TEST_ENV=\"localconf\"") result = runCmd('bitbake -e | grep TEST_ENV=') self.assertTrue('localconf' in result.output, msg = "bitbake didn't report any value for TEST_ENV variable. To test, run 'bitbake -e | grep TEST_ENV='") @@ -156,8 +183,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" ftools.write_file(preconf ,"TEST_PREFILE=\"prefile\"") result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') self.assertTrue('prefile' in result.output, "Preconfigure file \"prefile.conf\"was not taken into consideration. ") - self.append_config("TEST_PREFILE=\"localconf\"") - self.addCleanup(self.remove_config, "TEST_PREFILE=\"localconf\"") + self.write_config("TEST_PREFILE=\"localconf\"") result = runCmd('bitbake -r conf/prefile.conf -e | grep TEST_PREFILE=') self.assertTrue('localconf' in result.output, "Preconfigure file \"prefile.conf\"was not taken into consideration.") @@ -166,8 +192,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" postconf = os.path.join(self.builddir, 'conf/postfile.conf') self.track_for_cleanup(postconf) ftools.write_file(postconf , "TEST_POSTFILE=\"postfile\"") - self.append_config("TEST_POSTFILE=\"localconf\"") - self.addCleanup(self.remove_config, "TEST_POSTFILE=\"localconf\"") + self.write_config("TEST_POSTFILE=\"localconf\"") result = runCmd('bitbake -R conf/postfile.conf -e | grep TEST_POSTFILE=') self.assertTrue('postfile' in result.output, "Postconfigure file \"postfile.conf\"was not taken into consideration.") @@ -181,6 +206,7 @@ SSTATE_DIR = \"${TOPDIR}/download-selftest\" self.write_config("""DL_DIR = \"${TOPDIR}/download-selftest\" SSTATE_DIR = \"${TOPDIR}/download-selftest\" """) + self.track_for_cleanup(os.path.join(self.builddir, "download-selftest")) self.write_recipeinc('man',"\ndo_fail_task () {\nexit 1 \n}\n\naddtask do_fail_task before do_fetch\n" ) runCmd('bitbake -c cleanall man xcursor-transparent-theme') result = runCmd('bitbake man xcursor-transparent-theme -k', ignore_status=True) diff --git a/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py b/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py index 483803bf8..acf481f7b 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py +++ b/yocto-poky/meta/lib/oeqa/selftest/buildoptions.py @@ -1,9 +1,6 @@ -import unittest import os -import logging import re import glob as g -import pexpect as p from oeqa.selftest.base import oeSelfTest from oeqa.selftest.buildhistory import BuildhistoryBase @@ -42,7 +39,7 @@ class ImageOptionsTests(oeSelfTest): for image_file in deploydir_files: if imagename in image_file and os.path.islink(os.path.join(deploydir, image_file)): track_original_files.append(os.path.realpath(os.path.join(deploydir, image_file))) - self.append_config("RM_OLD_IMAGE = \"1\"") + self.write_config("RM_OLD_IMAGE = \"1\"") bitbake("-C rootfs core-image-minimal") deploydir_files = os.listdir(deploydir) remaining_not_expected = [path for path in track_original_files if os.path.basename(path) in deploydir_files] @@ -100,7 +97,7 @@ class SanityOptionsTest(oeSelfTest): @testcase(278) def test_sanity_userspace_dependency(self): - self.append_config('WARN_QA_append = " unsafe-references-in-binaries unsafe-references-in-scripts"') + self.write_config('WARN_QA_append = " unsafe-references-in-binaries unsafe-references-in-scripts"') bitbake("-ccleansstate gzip nfs-utils") res = bitbake("gzip nfs-utils") self.assertTrue("WARNING: QA Issue: gzip" in res.output, "WARNING: QA Issue: gzip message is not present in bitbake's output: %s" % res.output) @@ -128,7 +125,7 @@ class BuildImagesTest(oeSelfTest): This method is used to test the build of directfb image for arm arch. In essence we build a coreimagedirectfb and test the exitcode of bitbake that in case of success is 0. """ - self.add_command_to_tearDown('cleanupworkdir') + self.add_command_to_tearDown('cleanup-workdir') self.write_config("DISTRO_FEATURES_remove = \"x11\"\nDISTRO_FEATURES_append = \" directfb\"\nMACHINE ??= \"qemuarm\"") res = bitbake("core-image-directfb", ignore_status=True) self.assertEqual(res.status, 0, "\ncoreimagedirectfb failed to build. Please check logs for further details.\nbitbake output %s" % res.output) @@ -139,7 +136,7 @@ class ArchiverTest(oeSelfTest): """ Test for archiving the work directory and exporting the source files. """ - self.add_command_to_tearDown('cleanupworkdir') + self.add_command_to_tearDown('cleanup-workdir') self.write_config("INHERIT = \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") res = bitbake("xcursor-transparent-theme", ignore_status=True) self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) diff --git a/yocto-poky/meta/lib/oeqa/selftest/devtool.py b/yocto-poky/meta/lib/oeqa/selftest/devtool.py index 6e731d677..dcdef5a14 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/devtool.py +++ b/yocto-poky/meta/lib/oeqa/selftest/devtool.py @@ -84,11 +84,44 @@ class DevtoolBase(oeSelfTest): class DevtoolTests(DevtoolBase): + def setUp(self): + """Test case setup function""" + super(DevtoolTests, self).setUp() + self.workspacedir = os.path.join(self.builddir, 'workspace') + self.assertTrue(not os.path.exists(self.workspacedir), + 'This test cannot be run with a workspace directory ' + 'under the build directory') + + def _check_src_repo(self, repo_dir): + """Check srctree git repository""" + self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')), + 'git repository for external source tree not found') + result = runCmd('git status --porcelain', cwd=repo_dir) + self.assertEqual(result.output.strip(), "", + 'Created git repo is not clean') + result = runCmd('git symbolic-ref HEAD', cwd=repo_dir) + self.assertEqual(result.output.strip(), "refs/heads/devtool", + 'Wrong branch in git repo') + + def _check_repo_status(self, repo_dir, expected_status): + """Check the worktree status of a repository""" + result = runCmd('git status . --porcelain', + cwd=repo_dir) + for line in result.output.splitlines(): + for ind, (f_status, fn_re) in enumerate(expected_status): + if re.match(fn_re, line[3:]): + if f_status != line[:2]: + self.fail('Unexpected status in line: %s' % line) + expected_status.pop(ind) + break + else: + self.fail('Unexpected modified file in line: %s' % line) + if expected_status: + self.fail('Missing file changes: %s' % expected_status) + @testcase(1158) def test_create_workspace(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') result = runCmd('bitbake-layers show-layers') self.assertTrue('/workspace' not in result.output, 'This test cannot be run with a workspace layer in bblayers.conf') # Try creating a workspace layer with a specific path @@ -99,19 +132,16 @@ class DevtoolTests(DevtoolBase): result = runCmd('bitbake-layers show-layers') self.assertIn(tempdir, result.output) # Try creating a workspace layer with the default path - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool create-workspace') - self.assertTrue(os.path.isfile(os.path.join(workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output) + self.assertTrue(os.path.isfile(os.path.join(self.workspacedir, 'conf', 'layer.conf')), msg = "No workspace created. devtool output: %s " % result.output) result = runCmd('bitbake-layers show-layers') self.assertNotIn(tempdir, result.output) - self.assertIn(workspacedir, result.output) + self.assertIn(self.workspacedir, result.output) @testcase(1159) def test_devtool_add(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -121,11 +151,11 @@ class DevtoolTests(DevtoolBase): srcdir = os.path.join(tempdir, 'pv-1.5.3') self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure')), 'Unable to find configure script in source directory') # Test devtool add - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate pv') self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool add pv %s' % srcdir) - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') # Test devtool status result = runCmd('devtool status') self.assertIn('pv', result.output) @@ -144,9 +174,6 @@ class DevtoolTests(DevtoolBase): @testcase(1162) def test_devtool_add_library(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # We don't have the ability to pick up this dependency automatically yet... bitbake('libusb1') # Fetch source @@ -158,10 +185,10 @@ class DevtoolTests(DevtoolBase): srcdir = os.path.join(tempdir, 'libftdi1-1.1') self.assertTrue(os.path.isfile(os.path.join(srcdir, 'CMakeLists.txt')), 'Unable to find CMakeLists.txt in source directory') # Test devtool add (and use -V so we test that too) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool add libftdi %s -V 1.1' % srcdir) - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') # Test devtool status result = runCmd('devtool status') self.assertIn('libftdi', result.output) @@ -185,9 +212,6 @@ class DevtoolTests(DevtoolBase): @testcase(1160) def test_devtool_add_fetch(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -196,11 +220,11 @@ class DevtoolTests(DevtoolBase): testrecipe = 'python-markupsafe' srcdir = os.path.join(tempdir, testrecipe) # Test devtool add - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'setup.py')), 'Unable to find setup.py in source directory') # Test devtool status result = runCmd('devtool status') @@ -232,9 +256,6 @@ class DevtoolTests(DevtoolBase): @testcase(1161) def test_devtool_add_fetch_git(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Fetch source tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) @@ -243,11 +264,11 @@ class DevtoolTests(DevtoolBase): testrecipe = 'libmatchbox2' srcdir = os.path.join(tempdir, testrecipe) # Test devtool add - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake -c cleansstate %s' % testrecipe) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool add %s %s -f %s' % (testrecipe, srcdir, url)) - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created: %s' % result.output) self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure.ac in source directory') # Test devtool status result = runCmd('devtool status') @@ -284,32 +305,25 @@ class DevtoolTests(DevtoolBase): @testcase(1164) def test_devtool_modify(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Clean up anything in the workdir/sysroot/sstate cache bitbake('mdadm -c cleansstate') # Try modifying a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean mdadm') result = runCmd('devtool modify mdadm -x %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') - self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') - matches = glob.glob(os.path.join(workspacedir, 'appends', 'mdadm_*.bbappend')) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mdadm_*.bbappend')) self.assertTrue(matches, 'bbappend not created %s' % result.output) # Test devtool status result = runCmd('devtool status') self.assertIn('mdadm', result.output) self.assertIn(tempdir, result.output) # Check git repo - result = runCmd('git status --porcelain', cwd=tempdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempdir) # Try building bitbake('mdadm') # Try making (minor) modifications to the source @@ -336,13 +350,10 @@ class DevtoolTests(DevtoolBase): @testcase(1166) def test_devtool_modify_invalid(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Try modifying some recipes tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') testrecipes = 'perf kernel-devsrc package-index core-image-minimal meta-toolchain packagegroup-core-sdk meta-ide-support'.split() @@ -367,14 +378,14 @@ class DevtoolTests(DevtoolBase): self.assertNotEqual(result.status, 0, 'devtool modify on %s should have failed. devtool output: %s' % (testrecipe, result.output)) self.assertIn('ERROR: ', result.output, 'devtool modify on %s should have given an ERROR' % testrecipe) + @testcase(1365) def test_devtool_modify_native(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Try modifying some recipes tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') bbclassextended = False @@ -400,8 +411,6 @@ class DevtoolTests(DevtoolBase): @testcase(1165) def test_devtool_modify_git(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'mkelfimage' src_uri = get_bb_var('SRC_URI', testrecipe) self.assertIn('git://', src_uri, 'This test expects the %s recipe to be a git recipe' % testrecipe) @@ -410,32 +419,26 @@ class DevtoolTests(DevtoolBase): # Try modifying a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile')), 'Extracted source could not be found') - self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output) - matches = glob.glob(os.path.join(workspacedir, 'appends', 'mkelfimage_*.bbappend')) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created. devtool output: %s' % result.output) + matches = glob.glob(os.path.join(self.workspacedir, 'appends', 'mkelfimage_*.bbappend')) self.assertTrue(matches, 'bbappend not created') # Test devtool status result = runCmd('devtool status') self.assertIn(testrecipe, result.output) self.assertIn(tempdir, result.output) # Check git repo - result = runCmd('git status --porcelain', cwd=tempdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempdir) # Try building bitbake(testrecipe) @testcase(1167) def test_devtool_modify_localfiles(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'lighttpd' src_uri = (get_bb_var('SRC_URI', testrecipe) or '').split() foundlocal = False @@ -449,13 +452,13 @@ class DevtoolTests(DevtoolBase): # Try modifying a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) self.assertTrue(os.path.exists(os.path.join(tempdir, 'configure.ac')), 'Extracted source could not be found') - self.assertTrue(os.path.exists(os.path.join(workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') - matches = glob.glob(os.path.join(workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % testrecipe)) self.assertTrue(matches, 'bbappend not created') # Test devtool status result = runCmd('devtool status') @@ -464,30 +467,46 @@ class DevtoolTests(DevtoolBase): # Try building bitbake(testrecipe) + @testcase(1378) + def test_devtool_modify_virtual(self): + # Try modifying a virtual recipe + virtrecipe = 'virtual/libx11' + realrecipe = 'libx11' + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (virtrecipe, tempdir)) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir, 'conf', 'layer.conf')), 'Workspace directory not created') + matches = glob.glob(os.path.join(self.workspacedir, 'appends', '%s_*.bbappend' % realrecipe)) + self.assertTrue(matches, 'bbappend not created %s' % result.output) + # Test devtool status + result = runCmd('devtool status') + self.assertNotIn(virtrecipe, result.output) + self.assertIn(realrecipe, result.output) + # Check git repo + self._check_src_repo(tempdir) + # This is probably sufficient + + @testcase(1169) def test_devtool_update_recipe(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'minicom' recipefile = get_bb_var('FILE', testrecipe) src_uri = get_bb_var('SRC_URI', testrecipe) self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # (don't bother with cleaning the recipe on teardown, we won't be building it) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) # Check git repo - self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') - result = runCmd('git status --porcelain', cwd=tempdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempdir) # Add a couple of commits # FIXME: this only tests adding, need to also test update and remove result = runCmd('echo "Additional line" >> README', cwd=tempdir) @@ -497,25 +516,14 @@ class DevtoolTests(DevtoolBase): result = runCmd('git commit -m "Add a new file"', cwd=tempdir) self.add_command_to_tearDown('cd %s; rm %s/*.patch; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe) - status = result.output.splitlines() - self.assertEqual(len(status), 3, 'Less/more files modified than expected. Entire status:\n%s' % result.output) - for line in status: - if line.endswith('0001-Change-the-README.patch'): - self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line) - elif line.endswith('0002-Add-a-new-file.patch'): - self.assertEqual(line[:3], '?? ', 'Unexpected status in line: %s' % line) - elif re.search('%s_[^_]*.bb$' % testrecipe, line): - self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line) - else: - raise AssertionError('Unexpected modified file in status: %s' % line) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/0001-Change-the-README.patch$'), + ('??', '.*/0002-Add-a-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) @testcase(1172) def test_devtool_update_recipe_git(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'mtd-utils' recipefile = get_bb_var('FILE', testrecipe) src_uri = get_bb_var('SRC_URI', testrecipe) @@ -525,21 +533,16 @@ class DevtoolTests(DevtoolBase): if entry.startswith('file://') and entry.endswith('.patch'): patches.append(entry[7:].split(';')[0]) self.assertGreater(len(patches), 0, 'The %s recipe does not appear to contain any patches, so this test will not be effective' % testrecipe) - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # (don't bother with cleaning the recipe on teardown, we won't be building it) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) # Check git repo - self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') - result = runCmd('git status --porcelain', cwd=tempdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempdir) # Add a couple of commits # FIXME: this only tests adding, need to also test update and remove result = runCmd('echo "# Additional line" >> Makefile', cwd=tempdir) @@ -549,19 +552,10 @@ class DevtoolTests(DevtoolBase): result = runCmd('git commit -m "Add a new file"', cwd=tempdir) self.add_command_to_tearDown('cd %s; rm -rf %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe -m srcrev %s' % testrecipe) - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertNotEqual(result.output.strip(), "", '%s recipe should be modified' % testrecipe) - status = result.output.splitlines() - for line in status: - for patch in patches: - if line.endswith(patch): - self.assertEqual(line[:3], ' D ', 'Unexpected status in line: %s' % line) - break - else: - if re.search('%s_[^_]*.bb$' % testrecipe, line): - self.assertEqual(line[:3], ' M ', 'Unexpected status in line: %s' % line) - else: - raise AssertionError('Unexpected modified file in status: %s' % line) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile))] + \ + [(' D', '.*/%s$' % patch) for patch in patches] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) addlines = ['SRCREV = ".*"', 'SRC_URI = "git://git.infradead.org/mtd-utils.git"'] srcurilines = src_uri.split() @@ -588,50 +582,33 @@ class DevtoolTests(DevtoolBase): # Now try with auto mode runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) - result = runCmd('git rev-parse --show-toplevel') + result = runCmd('git rev-parse --show-toplevel', cwd=os.path.dirname(recipefile)) topleveldir = result.output.strip() - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - status = result.output.splitlines() relpatchpath = os.path.join(os.path.relpath(os.path.dirname(recipefile), topleveldir), testrecipe) - expectedstatus = [('M', os.path.relpath(recipefile, topleveldir)), - ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath), - ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)] - for line in status: - statusline = line.split(None, 1) - for fstatus, fn in expectedstatus: - if fn == statusline[1]: - if fstatus != statusline[0]: - self.fail('Unexpected status in line: %s' % line) - break - else: - self.fail('Unexpected modified file in line: %s' % line) + expected_status = [(' M', os.path.relpath(recipefile, topleveldir)), + ('??', '%s/0001-Change-the-Makefile.patch' % relpatchpath), + ('??', '%s/0002-Add-a-new-file.patch' % relpatchpath)] + self._check_repo_status(os.path.dirname(recipefile), expected_status) @testcase(1170) def test_devtool_update_recipe_append(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'mdadm' recipefile = get_bb_var('FILE', testrecipe) src_uri = get_bb_var('SRC_URI', testrecipe) self.assertNotIn('git://', src_uri, 'This test expects the %s recipe to NOT be a git recipe' % testrecipe) - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') tempsrcdir = os.path.join(tempdir, 'source') templayerdir = os.path.join(tempdir, 'layer') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # (don't bother with cleaning the recipe on teardown, we won't be building it) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir)) # Check git repo - self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found') - result = runCmd('git status --porcelain', cwd=tempsrcdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempsrcdir) # Add a commit result = runCmd("sed 's!\\(#define VERSION\\W*\"[^\"]*\\)\"!\\1-custom\"!' -i ReadMe.c", cwd=tempsrcdir) result = runCmd('git commit -a -m "Add our custom version"', cwd=tempsrcdir) @@ -642,8 +619,7 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool update-recipe %s -a %s' % (testrecipe, templayerdir)) self.assertNotIn('WARNING:', result.output) # Check recipe is still clean - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # Check bbappend was created splitpath = os.path.dirname(recipefile).split(os.sep) appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) @@ -685,8 +661,6 @@ class DevtoolTests(DevtoolBase): @testcase(1171) def test_devtool_update_recipe_append_git(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') testrecipe = 'mtd-utils' recipefile = get_bb_var('FILE', testrecipe) src_uri = get_bb_var('SRC_URI', testrecipe) @@ -695,23 +669,18 @@ class DevtoolTests(DevtoolBase): if entry.startswith('git://'): git_uri = entry break - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # First, modify a recipe tempdir = tempfile.mkdtemp(prefix='devtoolqa') tempsrcdir = os.path.join(tempdir, 'source') templayerdir = os.path.join(tempdir, 'layer') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') # (don't bother with cleaning the recipe on teardown, we won't be building it) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempsrcdir)) # Check git repo - self.assertTrue(os.path.isdir(os.path.join(tempsrcdir, '.git')), 'git repository for external source tree not found') - result = runCmd('git status --porcelain', cwd=tempsrcdir) - self.assertEqual(result.output.strip(), "", 'Created git repo is not clean') - result = runCmd('git symbolic-ref HEAD', cwd=tempsrcdir) - self.assertEqual(result.output.strip(), "refs/heads/devtool", 'Wrong branch in git repo') + self._check_src_repo(tempsrcdir) # Add a commit result = runCmd('echo "# Additional line" >> Makefile', cwd=tempsrcdir) result = runCmd('git commit -a -m "Change the Makefile"', cwd=tempsrcdir) @@ -731,8 +700,7 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool update-recipe -m srcrev %s -a %s' % (testrecipe, templayerdir)) self.assertNotIn('WARNING:', result.output) # Check recipe is still clean - result = runCmd('git status . --porcelain', cwd=os.path.dirname(recipefile)) - self.assertEqual(result.output.strip(), "", '%s recipe is not clean' % testrecipe) + self._check_repo_status(os.path.dirname(recipefile), []) # Check bbappend was created splitpath = os.path.dirname(recipefile).split(os.sep) appenddir = os.path.join(templayerdir, splitpath[-2], splitpath[-1]) @@ -779,28 +747,104 @@ class DevtoolTests(DevtoolBase): self.assertEqual(expectedlines, f.readlines()) # Deleting isn't expected to work under these circumstances + @testcase(1370) + def test_devtool_update_recipe_local_files(self): + """Check that local source files are copied over instead of patched""" + testrecipe = 'makedevs' + recipefile = get_bb_var('FILE', testrecipe) + # Setup srctree for modifying the recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + # (don't bother with cleaning the recipe on teardown, we won't be + # building it) + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Edit / commit local source + runCmd('echo "/* Foobar */" >> oe-local-files/makedevs.c', cwd=tempdir) + runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) + runCmd('echo "Bar" > new-file', cwd=tempdir) + runCmd('git add new-file', cwd=tempdir) + runCmd('git commit -m "Add new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % + os.path.dirname(recipefile)) + runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + (' M', '.*/makedevs/makedevs.c$'), + ('??', '.*/makedevs/new-local$'), + ('??', '.*/makedevs/0001-Add-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + + @testcase(1371) + def test_devtool_update_recipe_local_files_2(self): + """Check local source files support when oe-local-files is in Git""" + testrecipe = 'lzo' + recipefile = get_bb_var('FILE', testrecipe) + # Setup srctree for modifying the recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + # Check git repo + self._check_src_repo(tempdir) + # Add oe-local-files to Git + runCmd('rm oe-local-files/.gitignore', cwd=tempdir) + runCmd('git add oe-local-files', cwd=tempdir) + runCmd('git commit -m "Add local sources"', cwd=tempdir) + # Edit / commit local sources + runCmd('echo "# Foobar" >> oe-local-files/acinclude.m4', cwd=tempdir) + runCmd('git commit -am "Edit existing file"', cwd=tempdir) + runCmd('git rm oe-local-files/run-ptest', cwd=tempdir) + runCmd('git commit -m"Remove file"', cwd=tempdir) + runCmd('echo "Foo" > oe-local-files/new-local', cwd=tempdir) + runCmd('git add oe-local-files/new-local', cwd=tempdir) + runCmd('git commit -m "Add new local file"', cwd=tempdir) + runCmd('echo "Gar" > new-file', cwd=tempdir) + runCmd('git add new-file', cwd=tempdir) + runCmd('git commit -m "Add new file"', cwd=tempdir) + self.add_command_to_tearDown('cd %s; git clean -fd .; git checkout .' % + os.path.dirname(recipefile)) + # Checkout unmodified file to working copy -> devtool should still pick + # the modified version from HEAD + runCmd('git checkout HEAD^ -- oe-local-files/acinclude.m4', cwd=tempdir) + runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + (' M', '.*/acinclude.m4$'), + (' D', '.*/run-ptest$'), + ('??', '.*/new-local$'), + ('??', '.*/0001-Add-new-file.patch$')] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + @testcase(1163) def test_devtool_extract(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') tempdir = tempfile.mkdtemp(prefix='devtoolqa') # Try devtool extract self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') result = runCmd('devtool extract remake %s' % tempdir) self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') - self.assertTrue(os.path.isdir(os.path.join(tempdir, '.git')), 'git repository for external source tree not found') + self._check_src_repo(tempdir) + + @testcase(1379) + def test_devtool_extract_virtual(self): + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + # Try devtool extract + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool extract virtual/libx11 %s' % tempdir) + self.assertTrue(os.path.exists(os.path.join(tempdir, 'Makefile.am')), 'Extracted source could not be found') + self._check_src_repo(tempdir) @testcase(1168) def test_devtool_reset_all(self): - # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') testrecipe1 = 'mdadm' testrecipe2 = 'cronie' @@ -823,6 +867,7 @@ class DevtoolTests(DevtoolBase): matches2 = glob.glob(stampprefix2 + '*') self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2) + @testcase(1272) def test_devtool_deploy_target(self): # NOTE: Whilst this test would seemingly be better placed as a runtime test, # unfortunately the runtime tests run under bitbake and you can't run @@ -846,8 +891,7 @@ class DevtoolTests(DevtoolBase): break else: self.skipTest('No tap devices found - you must set up tap devices with scripts/runqemu-gen-tapdevs before running this test') - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Definitions testrecipe = 'mdadm' testfile = '/sbin/mdadm' @@ -863,7 +907,7 @@ class DevtoolTests(DevtoolBase): # Try devtool modify tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) @@ -908,18 +952,19 @@ class DevtoolTests(DevtoolBase): result = runCmd('ssh %s root@%s %s' % (sshargs, qemu.ip, testcommand), ignore_status=True) self.assertNotEqual(result, 0, 'undeploy-target did not remove command as it should have') + @testcase(1366) def test_devtool_build_image(self): """Test devtool build-image plugin""" # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') image = 'core-image-minimal' - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') self.add_command_to_tearDown('bitbake -c clean %s' % image) bitbake('%s -c clean' % image) # Add target and native recipes to workspace - for recipe in ('mdadm', 'parted-native'): + recipes = ['mdadm', 'parted-native'] + for recipe in recipes: tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.add_command_to_tearDown('bitbake -c clean %s' % recipe) @@ -927,17 +972,24 @@ class DevtoolTests(DevtoolBase): # Try to build image result = runCmd('devtool build-image %s' % image) self.assertNotEqual(result, 0, 'devtool build-image failed') - # Check if image.bbappend has required content - bbappend = os.path.join(workspacedir, 'appends', image+'.bbappend') - self.assertTrue(os.path.isfile(bbappend), 'bbappend not created %s' % result.output) - # NOTE: native recipe parted-native should not be in IMAGE_INSTALL_append - self.assertTrue('IMAGE_INSTALL_append = " mdadm"\n' in open(bbappend).readlines(), - 'IMAGE_INSTALL_append = " mdadm" not found in %s' % bbappend) + # Check if image contains expected packages + deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE') + image_link_name = get_bb_var('IMAGE_LINK_NAME', image) + reqpkgs = [item for item in recipes if not item.endswith('-native')] + with open(os.path.join(deploy_dir_image, image_link_name + '.manifest'), 'r') as f: + for line in f: + splitval = line.split() + if splitval: + pkg = splitval[0] + if pkg in reqpkgs: + reqpkgs.remove(pkg) + if reqpkgs: + self.fail('The following packages were not present in the image as expected: %s' % ', '.join(reqpkgs)) + @testcase(1367) def test_devtool_upgrade(self): # Check preconditions - workspacedir = os.path.join(self.builddir, 'workspace') - self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') + self.assertTrue(not os.path.exists(self.workspacedir), 'This test cannot be run with a workspace directory under the build directory') # Check parameters result = runCmd('devtool upgrade -h') for param in 'recipename srctree --version -V --branch -b --keep-temp --no-patch'.split(): @@ -955,9 +1007,9 @@ class DevtoolTests(DevtoolBase): # Check if srctree at least is populated self.assertTrue(len(os.listdir(tempdir)) > 0, 'scrtree (%s) should be populated with new (%s) source code' % (tempdir, version)) # Check new recipe folder is present - self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe)), 'Recipe folder should exist') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir,'recipes',recipe)), 'Recipe folder should exist') # Check new recipe file is present - self.assertTrue(os.path.exists(os.path.join(workspacedir,'recipes',recipe,"%s_%s.bb" % (recipe,version))), 'Recipe folder should exist') + self.assertTrue(os.path.exists(os.path.join(self.workspacedir,'recipes',recipe,"%s_%s.bb" % (recipe,version))), 'Recipe folder should exist') # Check devtool status and make sure recipe is present result = runCmd('devtool status') self.assertIn(recipe, result.output) @@ -967,5 +1019,18 @@ class DevtoolTests(DevtoolBase): result = runCmd('devtool status') self.assertNotIn(recipe, result.output) self.track_for_cleanup(tempdir) - self.track_for_cleanup(workspacedir) + self.track_for_cleanup(self.workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + @testcase(1352) + def test_devtool_layer_plugins(self): + """Test that devtool can use plugins from other layers. + + This test executes the selftest-reverse command from meta-selftest.""" + + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + + s = "Microsoft Made No Profit From Anyone's Zunes Yo" + result = runCmd("devtool --quiet selftest-reverse \"%s\"" % s) + self.assertEqual(result.output, s[::-1]) diff --git a/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py b/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py index fcffc423e..4efb0d92a 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py +++ b/yocto-poky/meta/lib/oeqa/selftest/imagefeatures.py @@ -25,9 +25,7 @@ class ImageFeatures(oeSelfTest): features = 'EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password"\n' features += 'INHERIT += "extrausers"\n' features += 'EXTRA_USERS_PARAMS = "useradd -p \'\' {}; usermod -s /bin/sh {};"'.format(self.test_user, self.test_user) - - # Append 'features' to local.conf - self.append_config(features) + self.write_config(features) # Build a core-image-minimal bitbake('core-image-minimal') @@ -53,9 +51,7 @@ class ImageFeatures(oeSelfTest): features = 'EXTRA_IMAGE_FEATURES = "ssh-server-openssh allow-empty-password"\n' features += 'INHERIT += "extrausers"\n' features += 'EXTRA_USERS_PARAMS = "useradd -p \'\' {}; usermod -s /bin/sh {};"'.format(self.test_user, self.test_user) - - # Append 'features' to local.conf - self.append_config(features) + self.write_config(features) # Build a core-image-minimal bitbake('core-image-minimal') @@ -87,9 +83,7 @@ class ImageFeatures(oeSelfTest): features += 'IMAGE_INSTALL_append = " openssh"\n' features += 'EXTRA_IMAGE_FEATURES = "empty-root-password allow-empty-password package-management"\n' features += 'RPMROOTFSDEPENDS_remove = "rpmresolve-native:do_populate_sysroot"' - - # Append 'features' to local.conf - self.append_config(features) + self.write_config(features) # Build a core-image-minimal bitbake('core-image-minimal') @@ -159,9 +153,7 @@ class ImageFeatures(oeSelfTest): features = 'DISTRO_FEATURES_append = " wayland"\n' features += 'CORE_IMAGE_EXTRA_INSTALL += "wayland weston"' - - # Append 'features' to local.conf - self.append_config(features) + self.write_config(features) # Build a core-image-weston bitbake('core-image-weston') diff --git a/yocto-poky/meta/lib/oeqa/selftest/layerappend.py b/yocto-poky/meta/lib/oeqa/selftest/layerappend.py index a82a6c8b9..4de5034a9 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/layerappend.py +++ b/yocto-poky/meta/lib/oeqa/selftest/layerappend.py @@ -46,10 +46,11 @@ FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:" SRC_URI_append += "file://appendtest.txt" """ - layerappend = "BBLAYERS += \"COREBASE/meta-layertest0 COREBASE/meta-layertest1 COREBASE/meta-layertest2\"" + layerappend = '' def tearDownLocal(self): - ftools.remove_from_file(self.builddir + "/conf/bblayers.conf", self.layerappend.replace("COREBASE", self.builddir + "/..")) + if self.layerappend: + ftools.remove_from_file(self.builddir + "/conf/bblayers.conf", self.layerappend) @testcase(1196) def test_layer_appends(self): @@ -79,7 +80,9 @@ SRC_URI_append += "file://appendtest.txt" with open(layer + "/recipes-test/layerappendtest/appendtest.txt", "w") as f: f.write("Layer 2 test") self.track_for_cleanup(layer) - ftools.append_file(self.builddir + "/conf/bblayers.conf", self.layerappend.replace("COREBASE", self.builddir + "/..")) + + self.layerappend = "BBLAYERS += \"{0}/meta-layertest0 {0}/meta-layertest1 {0}/meta-layertest2\"".format(corebase) + ftools.append_file(self.builddir + "/conf/bblayers.conf", self.layerappend) bitbake("layerappendtest") data = ftools.read_file(stagingdir + "/appendtest.txt") self.assertEqual(data, "Layer 2 test") diff --git a/yocto-poky/meta/lib/oeqa/selftest/manifest.py b/yocto-poky/meta/lib/oeqa/selftest/manifest.py new file mode 100644 index 000000000..44d0404c5 --- /dev/null +++ b/yocto-poky/meta/lib/oeqa/selftest/manifest.py @@ -0,0 +1,165 @@ +import unittest +import os + +from oeqa.selftest.base import oeSelfTest +from oeqa.utils.commands import get_bb_var, bitbake +from oeqa.utils.decorators import testcase + +class ManifestEntry: + '''A manifest item of a collection able to list missing packages''' + def __init__(self, entry): + self.file = entry + self.missing = [] + +class VerifyManifest(oeSelfTest): + '''Tests for the manifest files and contents of an image''' + + @classmethod + def check_manifest_entries(self, manifest, path): + manifest_errors = [] + try: + with open(manifest, "r") as mfile: + for line in mfile: + manifest_entry = os.path.join(path, line.split()[0]) + self.log.debug("{}: looking for {}"\ + .format(self.classname, manifest_entry)) + if not os.path.isfile(manifest_entry): + manifest_errors.append(manifest_entry) + self.log.debug("{}: {} not found"\ + .format(self.classname, manifest_entry)) + except OSError as e: + self.log.debug("{}: checking of {} failed"\ + .format(self.classname, manifest)) + raise e + + return manifest_errors + + #this will possibly move from here + @classmethod + def get_dir_from_bb_var(self, bb_var, target = None): + target == self.buildtarget if target == None else target + directory = get_bb_var(bb_var, target); + if not directory or not os.path.isdir(directory): + self.log.debug("{}: {} points to {} when target = {}"\ + .format(self.classname, bb_var, directory, target)) + raise OSError + return directory + + @classmethod + def setUpClass(self): + + self.buildtarget = 'core-image-minimal' + self.classname = 'VerifyManifest' + + self.log.info("{}: doing bitbake {} as a prerequisite of the test"\ + .format(self.classname, self.buildtarget)) + if bitbake(self.buildtarget).status: + self.log.debug("{} Failed to setup {}"\ + .format(self.classname, self.buildtarget)) + unittest.SkipTest("{}: Cannot setup testing scenario"\ + .format(self.classname)) + + @testcase(1380) + def test_SDK_manifest_entries(self): + '''Verifying the SDK manifest entries exist, this may take a build''' + + # the setup should bitbake core-image-minimal and here it is required + # to do an additional setup for the sdk + sdktask = '-c populate_sdk' + bbargs = sdktask + ' ' + self.buildtarget + self.log.debug("{}: doing bitbake {} as a prerequisite of the test"\ + .format(self.classname, bbargs)) + if bitbake(bbargs).status: + self.log.debug("{} Failed to bitbake {}"\ + .format(self.classname, bbargs)) + unittest.SkipTest("{}: Cannot setup testing scenario"\ + .format(self.classname)) + + + pkgdata_dir = reverse_dir = {} + mfilename = mpath = m_entry = {} + # get manifest location based on target to query about + d_target= dict(target = self.buildtarget, + host = 'nativesdk-packagegroup-sdk-host') + try: + mdir = self.get_dir_from_bb_var('SDK_DEPLOY', self.buildtarget) + for k in d_target.keys(): + mfilename[k] = "{}-toolchain-{}.{}.manifest".format( + get_bb_var("SDK_NAME", self.buildtarget), + get_bb_var("SDK_VERSION", self.buildtarget), + k) + mpath[k] = os.path.join(mdir, mfilename[k]) + if not os.path.isfile(mpath[k]): + self.log.debug("{}: {} does not exist".format( + self.classname, mpath[k])) + raise IOError + m_entry[k] = ManifestEntry(mpath[k]) + + pkgdata_dir[k] = self.get_dir_from_bb_var('PKGDATA_DIR', + d_target[k]) + reverse_dir[k] = os.path.join(pkgdata_dir[k], + 'runtime-reverse') + if not os.path.exists(reverse_dir[k]): + self.log.debug("{}: {} does not exist".format( + self.classname, reverse_dir[k])) + raise IOError + except OSError: + raise unittest.SkipTest("{}: Error in obtaining manifest dirs"\ + .format(self.classname)) + except IOError: + msg = "{}: Error cannot find manifests in the specified dir:\n{}"\ + .format(self.classname, mdir) + self.fail(msg) + + for k in d_target.keys(): + self.log.debug("{}: Check manifest {}".format( + self.classname, m_entry[k].file)) + + m_entry[k].missing = self.check_manifest_entries(\ + m_entry[k].file,reverse_dir[k]) + if m_entry[k].missing: + msg = '{}: {} Error has the following missing entries'\ + .format(self.classname, m_entry[k].file) + logmsg = msg+':\n'+'\n'.join(m_entry[k].missing) + self.log.debug(logmsg) + self.log.info(msg) + self.fail(logmsg) + + @testcase(1381) + def test_image_manifest_entries(self): + '''Verifying the image manifest entries exist''' + + # get manifest location based on target to query about + try: + mdir = self.get_dir_from_bb_var('DEPLOY_DIR_IMAGE', + self.buildtarget) + mfilename = get_bb_var("IMAGE_LINK_NAME", self.buildtarget)\ + + ".manifest" + mpath = os.path.join(mdir, mfilename) + if not os.path.isfile(mpath): raise IOError + m_entry = ManifestEntry(mpath) + + pkgdata_dir = {} + pkgdata_dir = self.get_dir_from_bb_var('PKGDATA_DIR', + self.buildtarget) + revdir = os.path.join(pkgdata_dir, 'runtime-reverse') + if not os.path.exists(revdir): raise IOError + except OSError: + raise unittest.SkipTest("{}: Error in obtaining manifest dirs"\ + .format(self.classname)) + except IOError: + msg = "{}: Error cannot find manifests in dir:\n{}"\ + .format(self.classname, mdir) + self.fail(msg) + + self.log.debug("{}: Check manifest {}"\ + .format(self.classname, m_entry.file)) + m_entry.missing = self.check_manifest_entries(\ + m_entry.file, revdir) + if m_entry.missing: + msg = '{}: {} Error has the following missing entries'\ + .format(self.classname, m_entry.file) + logmsg = msg+':\n'+'\n'.join(m_entry.missing) + self.log.debug(logmsg) + self.log.info(msg) + self.fail(logmsg) diff --git a/yocto-poky/meta/lib/oeqa/selftest/recipetool.py b/yocto-poky/meta/lib/oeqa/selftest/recipetool.py index c34ad6887..b1f1d2ab9 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/recipetool.py +++ b/yocto-poky/meta/lib/oeqa/selftest/recipetool.py @@ -492,9 +492,12 @@ class RecipetoolAppendsrcBase(RecipetoolBase): class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): + + @testcase(1273) def test_recipetool_appendsrcfile_basic(self): self._test_appendsrcfile('base-files', 'a-file') + @testcase(1274) def test_recipetool_appendsrcfile_basic_wildcard(self): testrecipe = 'base-files' self._test_appendsrcfile(testrecipe, 'a-file', options='-w') @@ -502,12 +505,15 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe) + @testcase(1281) def test_recipetool_appendsrcfile_subdir_basic(self): self._test_appendsrcfile('base-files', 'a-file', 'tmp') + @testcase(1282) def test_recipetool_appendsrcfile_subdir_basic_dirdest(self): self._test_appendsrcfile('base-files', destdir='tmp') + @testcase(1280) def test_recipetool_appendsrcfile_srcdir_basic(self): testrecipe = 'bash' srcdir = get_bb_var('S', testrecipe) @@ -515,12 +521,14 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): subdir = os.path.relpath(srcdir, workdir) self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) + @testcase(1275) def test_recipetool_appendsrcfile_existing_in_src_uri(self): testrecipe = 'base-files' filepath = self._get_first_file_uri(testrecipe) self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) + @testcase(1276) def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): testrecipe = 'base-files' subdir = 'tmp' @@ -530,6 +538,7 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) self.assertTrue(any('with different parameters' in l for l in output)) + @testcase(1277) def test_recipetool_appendsrcfile_replace_file_srcdir(self): testrecipe = 'bash' filepath = 'Makefile.in' @@ -541,6 +550,7 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): bitbake('%s:do_unpack' % testrecipe) self.assertEqual(open(self.testfile, 'r').read(), open(os.path.join(srcdir, filepath), 'r').read()) + @testcase(1278) def test_recipetool_appendsrcfiles_basic(self, destdir=None): newfiles = [self.testfile] for i in range(1, 5): @@ -550,5 +560,6 @@ class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): newfiles.append(testfile) self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W') + @testcase(1279) def test_recipetool_appendsrcfiles_basic_subdir(self): self.test_recipetool_appendsrcfiles_basic(destdir='testdir') diff --git a/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py b/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py index c4efc47fe..3c230620e 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py +++ b/yocto-poky/meta/lib/oeqa/selftest/sstatetests.py @@ -34,7 +34,7 @@ class SStateTests(SStateBase): targetarch = get_bb_var('TUNE_ARCH') self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) - @testcase(975) + @testcase(1374) def test_sstate_creation_distro_specific_fail(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_sstate_creation(['binutils-cross-'+ targetarch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) @@ -43,7 +43,7 @@ class SStateTests(SStateBase): def test_sstate_creation_distro_nonspecific_pass(self): self.run_test_sstate_creation(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) - @testcase(976) + @testcase(1375) def test_sstate_creation_distro_nonspecific_fail(self): self.run_test_sstate_creation(['glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) @@ -70,11 +70,11 @@ class SStateTests(SStateBase): targetarch = get_bb_var('TUNE_ARCH') self.run_test_cleansstate_task(['binutils-cross-' + targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) - @testcase(977) + @testcase(1376) def test_cleansstate_task_distro_nonspecific(self): self.run_test_cleansstate_task(['glibc-initial'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) - @testcase(977) + @testcase(1377) def test_cleansstate_task_distro_specific(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_cleansstate_task(['binutils-cross-'+ targetarch, 'binutils-native', 'glibc-initial'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) @@ -111,12 +111,12 @@ class SStateTests(SStateBase): targetarch = get_bb_var('TUNE_ARCH') self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch, 'binutils-native'], temp_sstate_location=True) - @testcase(175) + @testcase(1372) def test_rebuild_distro_specific_sstate_cross_target(self): targetarch = get_bb_var('TUNE_ARCH') self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + targetarch], temp_sstate_location=True) - @testcase(175) + @testcase(1373) def test_rebuild_distro_specific_sstate_native_target(self): self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) @@ -211,6 +211,8 @@ class SStateTests(SStateBase): they're built on a 32 or 64 bit system. Rather than requiring two different build machines and running a builds, override the variables calling uname() manually and check using bitbake -S. + + Also check that SDKMACHINE changing doesn't change any of these stamps. """ topdir = get_bb_var('TOPDIR') @@ -219,6 +221,7 @@ class SStateTests(SStateBase): TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" BUILD_ARCH = \"x86_64\" BUILD_OS = \"linux\" +SDKMACHINE = \"x86_64\" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash") bitbake("core-image-sato -S none") @@ -226,6 +229,7 @@ BUILD_OS = \"linux\" TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" BUILD_ARCH = \"i686\" BUILD_OS = \"linux\" +SDKMACHINE = \"i686\" """) self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") bitbake("core-image-sato -S none") @@ -233,11 +237,16 @@ BUILD_OS = \"linux\" def get_files(d): f = [] for root, dirs, files in os.walk(d): + if "core-image-sato" in root: + # SDKMACHINE changing will change do_rootfs/do_testimage/do_build stamps of core-image-sato itself + # which is safe to ignore + continue f.extend(os.path.join(root, name) for name in files) return f files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + targetvendor + "-linux", "x86_64" + targetvendor + "-linux", ) for x in files2] + self.maxDiff = None self.assertItemsEqual(files1, files2) @@ -271,11 +280,13 @@ NATIVELSBSTRING = \"DistroB\" files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None self.assertItemsEqual(files1, files2) + @testcase(1368) def test_sstate_allarch_samesigs(self): """ - The sstate checksums off allarch packages should be independent of whichever + The sstate checksums of allarch packages should be independent of whichever MACHINE is set. Check this using bitbake -S. Also, rather than duplicate the test, check nativesdk stamps are the same between the two MACHINE values. @@ -319,4 +330,50 @@ MACHINE = \"qemuarm\" files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] self.maxDiff = None self.assertItemsEqual(files1, files2) - + + @testcase(1369) + def test_sstate_sametune_samesigs(self): + """ + The sstate checksums of two identical machines (using the same tune) should be the + same, apart from changes within the machine specific stamps directory. We use the + qemux86copy machine to test this. Also include multilibs in the test. + """ + + topdir = get_bb_var('TOPDIR') + targetos = get_bb_var('TARGET_OS') + targetvendor = get_bb_var('TARGET_VENDOR') + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\" +MACHINE = \"qemux86\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +""") + self.track_for_cleanup(topdir + "/tmp-sstatesamehash") + bitbake("world meta-toolchain -S none") + self.write_config(""" +TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" +MACHINE = \"qemux86copy\" +require conf/multilib.conf +MULTILIBS = "multilib:lib32" +DEFAULTTUNE_virtclass-multilib-lib32 = "x86" +""") + self.track_for_cleanup(topdir + "/tmp-sstatesamehash2") + bitbake("world meta-toolchain -S none") + + def get_files(d): + f = [] + for root, dirs, files in os.walk(d): + for name in files: + if "meta-environment" in root or "cross-canadian" in root: + continue + if "qemux86copy-" in root or "qemux86-" in root: + continue + if "do_build" not in name and "do_populate_sdk" not in name: + f.append(os.path.join(root, name)) + return f + files1 = get_files(topdir + "/tmp-sstatesamehash/stamps") + files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps") + files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] + self.maxDiff = None + self.assertItemsEqual(files1, files2) diff --git a/yocto-poky/meta/lib/oeqa/selftest/wic.py b/yocto-poky/meta/lib/oeqa/selftest/wic.py index 3dc54a4c6..ea78e2259 100644 --- a/yocto-poky/meta/lib/oeqa/selftest/wic.py +++ b/yocto-poky/meta/lib/oeqa/selftest/wic.py @@ -31,50 +31,54 @@ from shutil import rmtree from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from oeqa.utils.decorators import testcase + class Wic(oeSelfTest): """Wic test class.""" resultdir = "/var/tmp/wic/build/" + image_is_ready = False - @classmethod - def setUpClass(cls): - """Build wic runtime dependencies.""" - bitbake('syslinux syslinux-native parted-native gptfdisk-native ' - 'dosfstools-native mtools-native') - Wic.image_is_ready = False - - def setUp(self): + def setUpLocal(self): """This code is executed before each test method.""" + self.write_config('IMAGE_FSTYPES += " hddimg"\nMACHINE_FEATURES_append = " efi"\n') + + # Do this here instead of in setUpClass as the base setUp does some + # clean up which can result in the native tools built earlier in + # setUpClass being unavailable. if not Wic.image_is_ready: - # build core-image-minimal with required features - features = 'IMAGE_FSTYPES += " hddimg"\nMACHINE_FEATURES_append = " efi"\n' - self.append_config(features) + bitbake('syslinux syslinux-native parted-native gptfdisk-native ' + 'dosfstools-native mtools-native') bitbake('core-image-minimal') - # set this class variable to avoid buiding image many times Wic.image_is_ready = True rmtree(self.resultdir, ignore_errors=True) - def test01_help(self): + @testcase(1208) + def test_help(self): """Test wic --help""" self.assertEqual(0, runCmd('wic --help').status) - def test02_createhelp(self): + @testcase(1209) + def test_createhelp(self): """Test wic create --help""" self.assertEqual(0, runCmd('wic create --help').status) - def test03_listhelp(self): + @testcase(1210) + def test_listhelp(self): """Test wic list --help""" self.assertEqual(0, runCmd('wic list --help').status) - def test04_build_image_name(self): + @testcase(1211) + def test_build_image_name(self): """Test wic create directdisk --image-name core-image-minimal""" self.assertEqual(0, runCmd("wic create directdisk " "--image-name core-image-minimal").status) self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - def test05_build_artifacts(self): + @testcase(1212) + def test_build_artifacts(self): """Test wic create directdisk providing all artifacts.""" vars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', @@ -87,34 +91,41 @@ class Wic(oeSelfTest): self.assertEqual(0, status) self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - def test06_gpt_image(self): + @testcase(1157) + def test_gpt_image(self): """Test creation of core-image-minimal with gpt table and UUID boot""" self.assertEqual(0, runCmd("wic create directdisk-gpt " "--image-name core-image-minimal").status) self.assertEqual(1, len(glob(self.resultdir + "directdisk-*.direct"))) - def test07_unsupported_subcommand(self): + @testcase(1213) + def test_unsupported_subcommand(self): """Test unsupported subcommand""" self.assertEqual(1, runCmd('wic unsupported', ignore_status=True).status) - def test08_no_command(self): + @testcase(1214) + def test_no_command(self): """Test wic without command""" self.assertEqual(1, runCmd('wic', ignore_status=True).status) - def test09_help_kickstart(self): + @testcase(1215) + def test_help_overview(self): """Test wic help overview""" self.assertEqual(0, runCmd('wic help overview').status) - def test10_help_plugins(self): + @testcase(1216) + def test_help_plugins(self): """Test wic help plugins""" self.assertEqual(0, runCmd('wic help plugins').status) - def test11_help_kickstart(self): + @testcase(1217) + def test_help_kickstart(self): """Test wic help kickstart""" self.assertEqual(0, runCmd('wic help kickstart').status) - def test12_compress_gzip(self): + @testcase(1264) + def test_compress_gzip(self): """Test compressing an image with gzip""" self.assertEqual(0, runCmd("wic create directdisk " "--image-name core-image-minimal " @@ -122,7 +133,8 @@ class Wic(oeSelfTest): self.assertEqual(1, len(glob(self.resultdir + \ "directdisk-*.direct.gz"))) - def test13_compress_gzip(self): + @testcase(1265) + def test_compress_bzip2(self): """Test compressing an image with bzip2""" self.assertEqual(0, runCmd("wic create directdisk " "--image-name core-image-minimal " @@ -130,7 +142,8 @@ class Wic(oeSelfTest): self.assertEqual(1, len(glob(self.resultdir + \ "directdisk-*.direct.bz2"))) - def test14_compress_gzip(self): + @testcase(1266) + def test_compress_xz(self): """Test compressing an image with xz""" self.assertEqual(0, runCmd("wic create directdisk " "--image-name core-image-minimal " @@ -138,13 +151,15 @@ class Wic(oeSelfTest): self.assertEqual(1, len(glob(self.resultdir + \ "directdisk-*.direct.xz"))) - def test15_wrong_compressor(self): + @testcase(1267) + def test_wrong_compressor(self): """Test how wic breaks if wrong compressor is provided""" self.assertEqual(2, runCmd("wic create directdisk " "--image-name core-image-minimal " "-c wrong", ignore_status=True).status) - def test16_rootfs_indirect_recipes(self): + @testcase(1268) + def test_rootfs_indirect_recipes(self): """Test usage of rootfs plugin with rootfs recipes""" wks = "directdisk-multi-rootfs" self.assertEqual(0, runCmd("wic create %s " @@ -154,7 +169,8 @@ class Wic(oeSelfTest): % wks).status) self.assertEqual(1, len(glob(self.resultdir + "%s*.direct" % wks))) - def test17_rootfs_artifacts(self): + @testcase(1269) + def test_rootfs_artifacts(self): """Test usage of rootfs plugin with rootfs paths""" vars = dict((var.lower(), get_bb_var(var, 'core-image-minimal')) \ for var in ('STAGING_DATADIR', 'DEPLOY_DIR_IMAGE', @@ -171,14 +187,16 @@ class Wic(oeSelfTest): self.assertEqual(1, len(glob(self.resultdir + \ "%(wks)s-*.direct" % vars))) - def test18_iso_image(self): - """Test creation of hybrid iso imagewith legacy and EFI boot""" + @testcase(1346) + def test_iso_image(self): + """Test creation of hybrid iso image with legacy and EFI boot""" self.assertEqual(0, runCmd("wic create mkhybridiso " "--image-name core-image-minimal").status) self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.direct"))) self.assertEqual(1, len(glob(self.resultdir + "HYBRID_ISO_IMG-*.iso"))) - def test19_image_env(self): + @testcase(1347) + def test_image_env(self): """Test generation of <image>.env files.""" image = 'core-image-minimal' stdir = get_bb_var('STAGING_DIR_TARGET', image) @@ -200,7 +218,8 @@ class Wic(oeSelfTest): self.assertTrue(var in content, "%s is not in .env file" % var) self.assertTrue(content[var]) - def test20_wic_image_type(self): + @testcase(1351) + def test_wic_image_type(self): """Test building wic images by bitbake""" self.assertEqual(0, bitbake('wic-image-minimal').status) @@ -214,21 +233,24 @@ class Wic(oeSelfTest): self.assertTrue(os.path.islink(path)) self.assertTrue(os.path.isfile(os.path.realpath(path))) - def test21_qemux86_directdisk(self): + @testcase(1348) + def test_qemux86_directdisk(self): """Test creation of qemux-86-directdisk image""" image = "qemux86-directdisk" self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ % image).status) self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - def test22_mkgummidisk(self): + @testcase(1349) + def test_mkgummidisk(self): """Test creation of mkgummidisk image""" image = "mkgummidisk" self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ % image).status) self.assertEqual(1, len(glob(self.resultdir + "%s-*direct" % image))) - def test23_mkefidisk(self): + @testcase(1350) + def test_mkefidisk(self): """Test creation of mkefidisk image""" image = "mkefidisk" self.assertEqual(0, runCmd("wic create %s -e core-image-minimal" \ diff --git a/yocto-poky/meta/lib/oeqa/utils/decorators.py b/yocto-poky/meta/lib/oeqa/utils/decorators.py index b6adcb184..0d79223a2 100644 --- a/yocto-poky/meta/lib/oeqa/utils/decorators.py +++ b/yocto-poky/meta/lib/oeqa/utils/decorators.py @@ -33,6 +33,10 @@ class getResults(object): ret.append(s.replace("setUpModule (", "").replace(")","")) else: ret.append(s) + # Append also the test without the full path + testname = s.split('.')[-1] + if testname: + ret.append(testname) return ret self.faillist = handleList(upperf.f_locals['result'].failures) self.errorlist = handleList(upperf.f_locals['result'].errors) @@ -53,11 +57,11 @@ class skipIfFailure(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in (res.getFailList() or res.getErrorList()): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ return wrapped_f @@ -67,11 +71,11 @@ class skipIfSkipped(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ return wrapped_f @@ -81,13 +85,13 @@ class skipUnlessPassed(object): self.testcase = testcase def __call__(self,f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): res = getResults() if self.testcase in res.getSkipList() or \ self.testcase in res.getFailList() or \ self.testcase in res.getErrorList(): raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) - return f(*args) + return f(*args, **kwargs) wrapped_f.__name__ = f.__name__ wrapped_f._depends_on = self.testcase return wrapped_f @@ -98,8 +102,8 @@ class testcase(object): self.test_case = test_case def __call__(self, func): - def wrapped_f(*args): - return func(*args) + def wrapped_f(*args, **kwargs): + return func(*args, **kwargs) wrapped_f.test_case = self.test_case wrapped_f.__name__ = func.__name__ return wrapped_f @@ -111,6 +115,12 @@ class NoParsingFilter(logging.Filter): def LogResults(original_class): orig_method = original_class.run + from time import strftime, gmtime + caller = os.path.basename(sys.argv[0]) + timestamp = strftime('%Y%m%d%H%M%S',gmtime()) + logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log') + linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log') + #rewrite the run method of unittest.TestCase to add testcase logging def run(self, result, *args, **kws): orig_method(self, result, *args, **kws) @@ -127,14 +137,13 @@ def LogResults(original_class): #create custom logging level for filtering. custom_log_level = 100 logging.addLevelName(custom_log_level, 'RESULTS') - caller = os.path.basename(sys.argv[0]) def results(self, message, *args, **kws): if self.isEnabledFor(custom_log_level): self.log(custom_log_level, message, *args, **kws) logging.Logger.results = results - logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'), + logging.basicConfig(filename=logfile, filemode='w', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S', @@ -162,7 +171,13 @@ def LogResults(original_class): if passed: local_log.results("Testcase "+str(test_case)+": PASSED") + # Create symlink to the current log + if os.path.exists(linkfile): + os.remove(linkfile) + os.symlink(logfile, linkfile) + original_class.run = run + return original_class class TimeOut(BaseException): diff --git a/yocto-poky/meta/lib/oeqa/utils/dump.py b/yocto-poky/meta/lib/oeqa/utils/dump.py index 4ae871c65..63a591d36 100644 --- a/yocto-poky/meta/lib/oeqa/utils/dump.py +++ b/yocto-poky/meta/lib/oeqa/utils/dump.py @@ -16,9 +16,20 @@ class BaseDumper(object): def __init__(self, cmds, parent_dir): self.cmds = [] - self.parent_dir = parent_dir + # Some testing doesn't inherit testimage, so it is needed + # to set some defaults. + self.parent_dir = parent_dir or "/tmp/oe-saved-tests" + dft_cmds = """ top -bn1 + iostat -x -z -N -d -p ALL 20 2 + ps -ef + free + df + memstat + dmesg + ip -s link + netstat -an""" if not cmds: - return + cmds = dft_cmds for cmd in cmds.split('\n'): cmd = cmd.lstrip() if not cmd or cmd[0] == '#': diff --git a/yocto-poky/meta/lib/oeqa/utils/ftools.py b/yocto-poky/meta/lib/oeqa/utils/ftools.py index 64ebe3d21..1bd9a30a4 100644 --- a/yocto-poky/meta/lib/oeqa/utils/ftools.py +++ b/yocto-poky/meta/lib/oeqa/utils/ftools.py @@ -1,12 +1,19 @@ import os import re +import errno def write_file(path, data): + # In case data is None, return immediately + if data is None: + return wdata = data.rstrip() + "\n" with open(path, "w") as f: f.write(wdata) def append_file(path, data): + # In case data is None, return immediately + if data is None: + return wdata = data.rstrip() + "\n" with open(path, "a") as f: f.write(wdata) @@ -18,7 +25,18 @@ def read_file(path): return data def remove_from_file(path, data): - lines = read_file(path).splitlines() + # In case data is None, return immediately + if data is None: + return + try: + rdata = read_file(path) + except IOError as e: + # if file does not exit, just quit, otherwise raise an exception + if e.errno == errno.ENOENT: + return + else: + raise + lines = rdata.splitlines() rmdata = data.strip().splitlines() for l in rmdata: for c in range(0, lines.count(l)): diff --git a/yocto-poky/meta/lib/oeqa/utils/qemurunner.py b/yocto-poky/meta/lib/oeqa/utils/qemurunner.py index d32c9db46..bdc6e0a8f 100644 --- a/yocto-poky/meta/lib/oeqa/utils/qemurunner.py +++ b/yocto-poky/meta/lib/oeqa/utils/qemurunner.py @@ -13,12 +13,20 @@ import re import socket import select import errno +import string import threading +import codecs from oeqa.utils.dump import HostDumper import logging logger = logging.getLogger("BitBake.QemuRunner") +# Get Unicode non printable control chars +control_range = range(0,32)+range(127,160) +control_chars = [unichr(x) for x in control_range + if unichr(x) not in string.printable] +re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) + class QemuRunner: def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds): @@ -61,7 +69,10 @@ class QemuRunner: def log(self, msg): if self.logfile: - with open(self.logfile, "a") as f: + # It is needed to sanitize the data received from qemu + # because is possible to have control characters + msg = re_control_char.sub('', unicode(msg, 'utf-8')) + with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) def getOutput(self, o): @@ -170,6 +181,9 @@ class QemuRunner: cmdline = '' with open('/proc/%s/cmdline' % self.qemupid) as p: cmdline = p.read() + # It is needed to sanitize the data received + # because is possible to have control characters + cmdline = re_control_char.sub('', cmdline) try: ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) if not ips or len(ips) != 3: @@ -186,7 +200,6 @@ class QemuRunner: logger.info("Target IP: %s" % self.ip) logger.info("Server IP: %s" % self.server_ip) - logger.info("Starting logging thread") self.thread = LoggingThread(self.log, threadsock, logger) self.thread.start() if not self.thread.connection_established.wait(self.boottime): @@ -197,6 +210,7 @@ class QemuRunner: self.stop_thread() return False + logger.info("Output from runqemu:\n%s", self.getOutput(output)) logger.info("Waiting at most %d seconds for login banner" % self.boottime) endtime = time.time() + self.boottime socklist = [self.server_socket] @@ -259,8 +273,9 @@ class QemuRunner: def stop(self): self.stop_thread() - if self.runqemu: + if hasattr(self, "origchldhandler"): signal.signal(signal.SIGCHLD, self.origchldhandler) + if self.runqemu: os.kill(self.monitorpid, signal.SIGKILL) logger.info("Sending SIGTERM to runqemu") try: @@ -280,7 +295,6 @@ class QemuRunner: self.server_socket = None self.qemupid = None self.ip = None - signal.signal(signal.SIGCHLD, self.origchldhandler) def stop_thread(self): if self.thread and self.thread.is_alive(): @@ -440,9 +454,9 @@ class LoggingThread(threading.Thread): def eventloop(self): poll = select.poll() - eventmask = self.errorevents | self.readevents + event_read_mask = self.errorevents | self.readevents poll.register(self.serversock.fileno()) - poll.register(self.readpipe, eventmask) + poll.register(self.readpipe, event_read_mask) breakout = False self.running = True @@ -466,7 +480,7 @@ class LoggingThread(threading.Thread): self.readsock, _ = self.serversock.accept() self.readsock.setblocking(0) poll.unregister(self.serversock.fileno()) - poll.register(self.readsock.fileno()) + poll.register(self.readsock.fileno(), event_read_mask) self.logger.info("Setting connection established event") self.connection_established.set() |