From 82c905dc58a36aeae40b1b273a12f63fb1973cf4 Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Mon, 13 Apr 2020 13:39:40 -0500 Subject: meta-openembedded and poky: subtree updates Squash of the following due to dependencies among them and OpenBMC changes: meta-openembedded: subtree update:d0748372d2..9201611135 meta-openembedded: subtree update:9201611135..17fd382f34 poky: subtree update:9052e5b32a..2e11d97b6c poky: subtree update:2e11d97b6c..a8544811d7 The change log was too large for the jenkins plugin to handle therefore it has been removed. Here is the first and last commit of each subtree: meta-openembedded:d0748372d2 cppzmq: bump to version 4.6.0 meta-openembedded:17fd382f34 mpv: Remove X11 dependency poky:9052e5b32a package_ipk: Remove pointless comment to trigger rebuild poky:a8544811d7 pbzip2: Fix license warning Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6 Signed-off-by: Andrew Geissler --- poky/scripts/install-buildtools | 339 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100755 poky/scripts/install-buildtools (limited to 'poky/scripts/install-buildtools') diff --git a/poky/scripts/install-buildtools b/poky/scripts/install-buildtools new file mode 100755 index 000000000..c6b3a1eed --- /dev/null +++ b/poky/scripts/install-buildtools @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 + +# Buildtools and buildtools extended installer helper script +# +# Copyright (C) 2017-2020 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# +# NOTE: --with-extended-buildtools is on by default +# +# Example usage (extended buildtools from milestone): +# (1) using --url and --filename +# $ install-buildtools \ +# --url http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools \ +# --filename x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh +# (2) using --base-url, --release, --installer-version and --build-date +# $ install-buildtools \ +# --base-url http://downloads.yoctoproject.org/releases/yocto \ +# --release yocto-3.1_M3 \ +# --installer-version 3.0+snapshot +# --build-date 202000315 +# +# Example usage (standard buildtools from release): +# (3) using --url and --filename +# $ install-buildtools --without-extended-buildtools \ +# --url http://downloads.yoctoproject.org/releases/yocto/yocto-3.0.2/buildtools \ +# --filename x86_64-buildtools-nativesdk-standalone-3.0.2.sh +# (4) using --base-url, --release and --installer-version +# $ install-buildtools --without-extended-buildtools \ +# --base-url http://downloads.yoctoproject.org/releases/yocto \ +# --release yocto-3.0.2 \ +# --installer-version 3.0.2 +# + +import argparse +import logging +import os +import re +import shutil +import shlex +import stat +import subprocess +import sys +import tempfile +from urllib.parse import quote + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptutils +import scriptpath + + +PROGNAME = 'install-buildtools' +logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) + +DEFAULT_INSTALL_DIR = os.path.join(os.path.split(scripts_path)[0],'buildtools') +DEFAULT_BASE_URL = 'http://downloads.yoctoproject.org/releases/yocto' +DEFAULT_RELEASE = 'yocto-3.1' +DEFAULT_INSTALLER_VERSION = '3.1' +DEFAULT_BUILDDATE = '' + +# Python version sanity check +if not (sys.version_info.major == 3 and sys.version_info.minor >= 4): + logger.error("This script requires Python 3.4 or greater") + logger.error("You have Python %s.%s" % + (sys.version_info.major, sys.version_info.minor)) + sys.exit(1) + +# The following three functions are copied directly from +# bitbake/lib/bb/utils.py, in order to allow this script +# to run on versions of python earlier than what bitbake +# supports (e.g. less than Python 3.5 for YP 3.1 release) + +def _hasher(method, filename): + import mmap + + with open(filename, "rb") as f: + try: + with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: + for chunk in iter(lambda: mm.read(8192), b''): + method.update(chunk) + except ValueError: + # You can't mmap() an empty file so silence this exception + pass + return method.hexdigest() + + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + import hashlib + return _hasher(hashlib.md5(), filename) + +def sha256_file(filename): + """ + Return the hex string representation of the 256-bit SHA checksum of + filename. + """ + import hashlib + return _hasher(hashlib.sha256(), filename) + + +def main(): + global DEFAULT_INSTALL_DIR + global DEFAULT_BASE_URL + global DEFAULT_RELEASE + global DEFAULT_INSTALLER_VERSION + global DEFAULT_BUILDDATE + filename = "" + release = "" + buildtools_url = "" + install_dir = "" + + parser = argparse.ArgumentParser( + description="Buildtools installation helper", + add_help=False) + parser.add_argument('-u', '--url', + help='URL from where to fetch buildtools SDK installer, not ' + 'including filename (optional)\n' + 'Requires --filename.', + action='store') + parser.add_argument('-f', '--filename', + help='filename for the buildtools SDK installer to be installed ' + '(optional)\nRequires --url', + action='store') + parser.add_argument('-d', '--directory', + default=DEFAULT_INSTALL_DIR, + help='directory where buildtools SDK will be installed (optional)', + action='store') + parser.add_argument('-r', '--release', + default=DEFAULT_RELEASE, + help='Yocto Project release string for SDK which will be ' + 'installed (optional)', + action='store') + parser.add_argument('-V', '--installer-version', + default=DEFAULT_INSTALLER_VERSION, + help='version string for the SDK to be installed (optional)', + action='store') + parser.add_argument('-b', '--base-url', + default=DEFAULT_BASE_URL, + help='base URL from which to fetch SDK (optional)', action='store') + parser.add_argument('-t', '--build-date', + default=DEFAULT_BUILDDATE, + help='Build date of pre-release SDK (optional)', action='store') + group = parser.add_mutually_exclusive_group() + group.add_argument('--with-extended-buildtools', action='store_true', + dest='with_extended_buildtools', + default=True, + help='enable extended buildtools tarball (on by default)') + group.add_argument('--without-extended-buildtools', action='store_false', + dest='with_extended_buildtools', + help='disable extended buildtools (traditional buildtools tarball)') + parser.add_argument('-c', '--check', help='enable md5 checksum checking', + default=True, + action='store_true') + parser.add_argument('-D', '--debug', help='enable debug output', + action='store_true') + parser.add_argument('-q', '--quiet', help='print only errors', + action='store_true') + + parser.add_argument('-h', '--help', action='help', + default=argparse.SUPPRESS, + help='show this help message and exit') + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.quiet: + logger.setLevel(logging.ERROR) + + if args.url and args.filename: + logger.debug("--url and --filename detected. Ignoring --base-url " + "--release --installer-version arguments.") + filename = args.filename + buildtools_url = "%s/%s" % (args.url, filename) + else: + if args.base_url: + base_url = args.base_url + else: + base_url = DEFAULT_BASE_URL + if args.release: + # check if this is a pre-release "milestone" SDK + m = re.search(r"^(?P[a-zA-Z\-]+)(?P[0-9.]+)(?P_M[1-9])$", + args.release) + logger.debug("milestone regex: %s" % m) + if m and m.group('milestone'): + logger.debug("release[distro]: %s" % m.group('distro')) + logger.debug("release[version]: %s" % m.group('version')) + logger.debug("release[milestone]: %s" % m.group('milestone')) + if not args.build_date: + logger.error("Milestone installers require --build-date") + else: + if args.with_extended_buildtools: + filename = "x86_64-buildtools-extended-nativesdk-standalone-%s-%s.sh" % ( + args.installer_version, args.build_date) + else: + filename = "x86_64-buildtools-nativesdk-standalone-%s-%s.sh" % ( + args.installer_version, args.build_date) + safe_filename = quote(filename) + buildtools_url = "%s/milestones/%s/buildtools/%s" % (base_url, args.release, safe_filename) + # regular release SDK + else: + if args.with_extended_buildtools: + filename = "x86_64-buildtools-extended-nativesdk-standalone-%s.sh" % args.installer_version + else: + filename = "x86_64-buildtools-nativesdk-standalone-%s.sh" % args.installer_version + safe_filename = quote(filename) + buildtools_url = "%s/%s/buildtools/%s" % (base_url, args.release, safe_filename) + + tmpsdk_dir = tempfile.mkdtemp() + try: + # Fetch installer + logger.info("Fetching buildtools installer") + tmpbuildtools = os.path.join(tmpsdk_dir, filename) + ret = subprocess.call("wget -q -O %s %s" % + (tmpbuildtools, buildtools_url), shell=True) + if ret != 0: + logger.error("Could not download file from %s" % buildtools_url) + return ret + + # Verify checksum + if args.check: + logger.info("Fetching buildtools installer checksum") + checksum_type = "" + for checksum_type in ["md5sum", "sha256"]: + check_url = "{}.{}".format(buildtools_url, checksum_type) + checksum_filename = "{}.{}".format(filename, checksum_type) + tmpbuildtools_checksum = os.path.join(tmpsdk_dir, checksum_filename) + ret = subprocess.call("wget -q -O %s %s" % + (tmpbuildtools_checksum, check_url), shell=True) + if ret == 0: + break + else: + if ret != 0: + logger.error("Could not download file from %s" % check_url) + return ret + regex = re.compile(r"^(?P[0-9a-f]+)\s\s(?P.*/)?(?P.*)$") + with open(tmpbuildtools_checksum, 'rb') as f: + original = f.read() + m = re.search(regex, original.decode("utf-8")) + logger.debug("checksum regex match: %s" % m) + logger.debug("checksum: %s" % m.group('checksum')) + logger.debug("path: %s" % m.group('path')) + logger.debug("filename: %s" % m.group('filename')) + if filename != m.group('filename'): + logger.error("Filename does not match name in checksum") + return 1 + checksum = m.group('checksum') + if checksum_type == "md5sum": + checksum_value = md5_file(tmpbuildtools) + else: + checksum_value = sha256_file(tmpbuildtools) + if checksum == checksum_value: + logger.info("Checksum success") + else: + logger.error("Checksum %s expected. Actual checksum is %s." % + (checksum, checksum_value)) + + # Make installer executable + logger.info("Making installer executable") + st = os.stat(tmpbuildtools) + os.chmod(tmpbuildtools, st.st_mode | stat.S_IEXEC) + logger.debug(os.stat(tmpbuildtools)) + if args.directory: + install_dir = args.directory + ret = subprocess.call("%s -d %s -y" % + (tmpbuildtools, install_dir), shell=True) + else: + install_dir = "/opt/poky/%s" % args.installer_version + ret = subprocess.call("%s -y" % tmpbuildtools, shell=True) + if ret != 0: + logger.error("Could not run buildtools installer") + + # Setup the environment + logger.info("Setting up the environment") + regex = re.compile(r'^(?Pexport )?(?P[A-Z_]+)=(?P.+)$') + with open("%s/environment-setup-x86_64-pokysdk-linux" % + install_dir, 'rb') as f: + for line in f: + match = regex.search(line.decode('utf-8')) + logger.debug("export regex: %s" % match) + if match: + env_var = match.group('env_var') + logger.debug("env_var: %s" % env_var) + env_val = match.group('env_val') + logger.debug("env_val: %s" % env_val) + os.environ[env_var] = env_val + + # Test installation + logger.info("Testing installation") + tool = "" + m = re.search("extended", tmpbuildtools) + logger.debug("extended regex: %s" % m) + if args.with_extended_buildtools and not m: + logger.info("Ignoring --with-extended-buildtools as filename " + "does not contain 'extended'") + if args.with_extended_buildtools and m: + tool = 'gcc' + else: + tool = 'tar' + logger.debug("install_dir: %s" % install_dir) + cmd = shlex.split("/usr/bin/which %s" % tool) + logger.debug("cmd: %s" % cmd) + logger.debug("tool: %s" % tool) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + output, errors = proc.communicate() + logger.debug("proc.args: %s" % proc.args) + logger.debug("proc.communicate(): output %s" % output) + logger.debug("proc.communicate(): errors %s" % errors) + which_tool = output.decode('utf-8') + logger.debug("which %s: %s" % (tool, which_tool)) + ret = proc.returncode + if not which_tool.startswith(install_dir): + logger.error("Something went wrong: %s not found in %s" % + (tool, install_dir)) + if ret != 0: + logger.error("Something went wrong: installation failed") + else: + logger.info("Installation successful. Remember to source the " + "environment setup script now and in any new session.") + return ret + + finally: + # cleanup tmp directory + shutil.rmtree(tmpsdk_dir) + + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret) -- cgit v1.2.3