From eb8dc40360f0cfef56fb6947cc817a547d6d9bc6 Mon Sep 17 00:00:00 2001 From: Dave Cobbley Date: Tue, 14 Aug 2018 10:05:37 -0700 Subject: [Subtree] Removing import-layers directory As part of the move to subtrees, need to bring all the import layers content to the top level. Change-Id: I4a163d10898cbc6e11c27f776f60e1a470049d8f Signed-off-by: Dave Cobbley Signed-off-by: Brad Bishop --- poky/meta/lib/oe/distro_check.py | 308 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 poky/meta/lib/oe/distro_check.py (limited to 'poky/meta/lib/oe/distro_check.py') diff --git a/poky/meta/lib/oe/distro_check.py b/poky/meta/lib/oe/distro_check.py new file mode 100644 index 000000000..e775c3a6e --- /dev/null +++ b/poky/meta/lib/oe/distro_check.py @@ -0,0 +1,308 @@ +def create_socket(url, d): + import urllib + from bb.utils import export_proxies + + export_proxies(d) + return urllib.request.urlopen(url) + +def get_links_from_url(url, d): + "Return all the href links found on the web location" + + from bs4 import BeautifulSoup, SoupStrainer + + soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) + hyperlinks = [] + for line in soup.find_all('a', href=True): + hyperlinks.append(line['href'].strip('/')) + return hyperlinks + +def find_latest_numeric_release(url, d): + "Find the latest listed numeric release on the given url" + max=0 + maxstr="" + for link in get_links_from_url(url, d): + try: + # TODO use LooseVersion + release = float(link) + except: + release = 0 + if release > max: + max = release + maxstr = link + return maxstr + +def is_src_rpm(name): + "Check if the link is pointing to a src.rpm file" + return name.endswith(".src.rpm") + +def package_name_from_srpm(srpm): + "Strip out the package name from the src.rpm filename" + + # ca-certificates-2016.2.7-1.0.fc24.src.rpm + # ^name ^ver ^release^removed + (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) + return name + +def get_source_package_list_from_url(url, section, d): + "Return a sectioned list of package names from a URL list" + + bb.note("Reading %s: %s" % (url, section)) + links = get_links_from_url(url, d) + srpms = filter(is_src_rpm, links) + names_list = map(package_name_from_srpm, srpms) + + new_pkgs = set() + for pkgs in names_list: + new_pkgs.add(pkgs + ":" + section) + return new_pkgs + +def get_source_package_list_from_url_by_letter(url, section, d): + import string + from urllib.error import HTTPError + packages = set() + for letter in (string.ascii_lowercase + string.digits): + # Not all subfolders may exist, so silently handle 404 + try: + packages |= get_source_package_list_from_url(url + "/" + letter, section, d) + except HTTPError as e: + if e.code != 404: raise + return packages + +def get_latest_released_fedora_source_package_list(d): + "Returns list of all the name os packages in the latest fedora distro" + latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) + package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) + package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) + return latest, package_names + +def get_latest_released_opensuse_source_package_list(d): + "Returns list of all the name os packages in the latest opensuse distro" + latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) + + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) + return latest, package_names + +def get_latest_released_clear_source_package_list(d): + latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) + package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) + return latest, package_names + +def find_latest_debian_release(url, d): + "Find the latest listed debian release on the given url" + + releases = [link.replace("Debian", "") + for link in get_links_from_url(url, d) + if link.startswith("Debian")] + releases.sort() + try: + return releases[-1] + except: + return "_NotFound_" + +def get_debian_style_source_package_list(url, section, d): + "Return the list of package-names stored in the debian style Sources.gz file" + import gzip + + package_names = set() + for line in gzip.open(create_socket(url, d), mode="rt"): + if line.startswith("Package:"): + pkg = line.split(":", 1)[1].strip() + package_names.add(pkg + ":" + section) + return package_names + +def get_latest_released_debian_source_package_list(d): + "Returns list of all the name of packages in the latest debian distro" + latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) + url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" + package_names = get_debian_style_source_package_list(url, "main", d) + url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def find_latest_ubuntu_release(url, d): + """ + Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. + + To avoid matching development releases look for distributions that have + updates, so the resulting distro could be any supported release. + """ + url += "?C=M;O=D" # Descending Sort by Last Modified + for link in get_links_from_url(url, d): + if "-updates" in link: + distro = link.replace("-updates", "") + return distro + return "_NotFound_" + +def get_latest_released_ubuntu_source_package_list(d): + "Returns list of all the name os packages in the latest ubuntu distro" + latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) + url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest + package_names = get_debian_style_source_package_list(url, "main", d) + url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest + package_names |= get_debian_style_source_package_list(url, "updates", d) + return latest, package_names + +def create_distro_packages_list(distro_check_dir, d): + import shutil + + pkglst_dir = os.path.join(distro_check_dir, "package_lists") + bb.utils.remove(pkglst_dir, True) + bb.utils.mkdirhier(pkglst_dir) + + per_distro_functions = ( + ("Debian", get_latest_released_debian_source_package_list), + ("Ubuntu", get_latest_released_ubuntu_source_package_list), + ("Fedora", get_latest_released_fedora_source_package_list), + ("openSUSE", get_latest_released_opensuse_source_package_list), + ("Clear", get_latest_released_clear_source_package_list), + ) + + for name, fetcher_func in per_distro_functions: + try: + release, package_list = fetcher_func(d) + except Exception as e: + bb.warn("Cannot fetch packages for %s: %s" % (name, e)) + bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) + if len(package_list) == 0: + bb.error("Didn't fetch any packages for %s %s" % (name, release)) + + package_list_file = os.path.join(pkglst_dir, name + "-" + release) + with open(package_list_file, 'w') as f: + for pkg in sorted(package_list): + f.write(pkg + "\n") + +def update_distro_data(distro_check_dir, datetime, d): + """ + If distro packages list data is old then rebuild it. + The operations has to be protected by a lock so that + only one thread performes it at a time. + """ + if not os.path.isdir (distro_check_dir): + try: + bb.note ("Making new directory: %s" % distro_check_dir) + os.makedirs (distro_check_dir) + except OSError: + raise Exception('Unable to create directory %s' % (distro_check_dir)) + + + datetime_file = os.path.join(distro_check_dir, "build_datetime") + saved_datetime = "_invalid_" + import fcntl + try: + if not os.path.exists(datetime_file): + open(datetime_file, 'w+').close() # touch the file so that the next open won't fail + + f = open(datetime_file, "r+") + fcntl.lockf(f, fcntl.LOCK_EX) + saved_datetime = f.read() + if saved_datetime[0:8] != datetime[0:8]: + bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) + bb.note("Regenerating distro package lists") + create_distro_packages_list(distro_check_dir, d) + f.seek(0) + f.write(datetime) + + except OSError as e: + raise Exception('Unable to open timestamp: %s' % e) + finally: + fcntl.lockf(f, fcntl.LOCK_UN) + f.close() + +def compare_in_distro_packages_list(distro_check_dir, d): + if not os.path.isdir(distro_check_dir): + raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") + + localdata = bb.data.createCopy(d) + pkglst_dir = os.path.join(distro_check_dir, "package_lists") + matching_distros = [] + pn = recipe_name = d.getVar('PN') + bb.note("Checking: %s" % pn) + + if pn.find("-native") != -1: + pnstripped = pn.split("-native") + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + recipe_name = pnstripped[0] + + if pn.startswith("nativesdk-"): + pnstripped = pn.split("nativesdk-") + localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) + recipe_name = pnstripped[1] + + if pn.find("-cross") != -1: + pnstripped = pn.split("-cross") + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + recipe_name = pnstripped[0] + + if pn.find("-initial") != -1: + pnstripped = pn.split("-initial") + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + recipe_name = pnstripped[0] + + bb.note("Recipe: %s" % recipe_name) + + distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) + tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" + for str in tmp.split(): + if str and str.find("=") == -1 and distro_exceptions[str]: + matching_distros.append(str) + + distro_pn_aliases = {} + for str in tmp.split(): + if "=" in str: + (dist, pn_alias) = str.split('=') + distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() + + for file in os.listdir(pkglst_dir): + (distro, distro_release) = file.split("-") + f = open(os.path.join(pkglst_dir, file), "r") + for line in f: + (pkg, section) = line.split(":") + if distro.lower() in distro_pn_aliases: + pn = distro_pn_aliases[distro.lower()] + else: + pn = recipe_name + if pn == pkg: + matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end + f.close() + break + f.close() + + for item in tmp.split(): + matching_distros.append(item) + bb.note("Matching: %s" % matching_distros) + return matching_distros + +def create_log_file(d, logname): + logpath = d.getVar('LOG_DIR') + bb.utils.mkdirhier(logpath) + logfn, logsuffix = os.path.splitext(logname) + logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) + if not os.path.exists(logfile): + slogfile = os.path.join(logpath, logname) + if os.path.exists(slogfile): + os.remove(slogfile) + open(logfile, 'w+').close() + os.symlink(logfile, slogfile) + d.setVar('LOG_FILE', logfile) + return logfile + + +def save_distro_check_result(result, datetime, result_file, d): + pn = d.getVar('PN') + logdir = d.getVar('LOG_DIR') + if not logdir: + bb.error("LOG_DIR variable is not defined, can't write the distro_check results") + return + bb.utils.mkdirhier(logdir) + + line = pn + for i in result: + line = line + "," + i + f = open(result_file, "a") + import fcntl + fcntl.lockf(f, fcntl.LOCK_EX) + f.seek(0, os.SEEK_END) # seek to the end of file + f.write(line + "\n") + fcntl.lockf(f, fcntl.LOCK_UN) + f.close() -- cgit v1.2.3