From 0ca19ccf045e022d8a24d26afbf346ab7f2f519f Mon Sep 17 00:00:00 2001 From: Patrick Williams Date: Mon, 16 Aug 2021 14:03:13 -0500 Subject: subtree updates poky: 492205ea83..94dfcaff64: Alejandro Hernandez Samaniego (1): baremetal-helloworld: Enable RISC-V 32 port Alexandre Belloni (1): oeqa/runtime/cases: make date.DateTest.test_date more reliable Anton Blanchard (3): libjpeg-turbo: Handle powerpc64le without Altivec kmod: use nonarch_base_libdir for depmod.d and modprobe.d pixman: Handle PowerPC without Altivec Changqing Li (1): libconvert-asn1-perl: 0.27 -> 0.31 Chen Qi (4): convert-overrides.py: also convert comments without a leading whitespace meta: use new override syntax in comments multilib.bbclass: fix new override syntax for virtclass-multilib util-linux: add back manpages related settings Daniel Gomez (1): docs: fix typo in releases Dmitry Baryshkov (1): linux-firmware: add more Qualcomm firmware packages Dragos-Marian Panait (1): util-linux: fix CVE-2021-37600 Joe Slater (1): terminal.bbclass: force bash for devshell Jon Mason (1): tune-cortexm*: add support for all Arm Cortex-M processors Jose Quaresma (1): sstate.bbclass: fix error handling when sstate mirrors is ro Joshua Watt (2): classes/cve-check: Move get_patches_cves to library lib/packagedata: Fix for new overrides Khem Raj (4): glibc: Upgrade to 2.34 release glibc: Remove obsolete --enable-stackguard-randomization glibc: Drop DUMMY_LOCALE_T define patch glibc: Add missing symlinks for libpthread and librt dev files Michael Halstead (1): releases: update to include 3.1.10 Michael Opdenacker (12): manuals: mention license information in footer manuals: further documentation for cve-check cve-check: remove deprecated CVE_CHECK_CVE_WHITELIST bsp-guide: overrides syntax updates dev-manual: overrides syntax updates kernel-dev manual: overrides syntax updates ref-manual: overrides syntax updates sdk-manual: overrides syntax updates test-manual: overrides syntax updates sdk-manual: reference obsolete reference to ADT Manuals: replace "file name" by "filename" dev-manual: fix grammar in post-install script explanations Nisha Parrakat (1): dbus_%.bbappend: stop using selinux_set_mapping Olaf Mandel (1): kickstart: document which options accept units Patrick Williams (3): pixman: re-disable iwmmxt systemd: add zstd PACKAGECONFIG systemd: set zstd as default PACKAGECONFIG Paul Barker (2): u-boot: Package extlinux.conf separately pypi: Allow override of PyPI archive name Quentin Schulz (3): insane.bbclass: fix new override syntax migration docs: fix new override syntax migration docs: overview-manual: concepts: remove long-gone BBHASHDEPS variable Richard Purdie (6): test-manual: Add extra detail to YP Compatible section migration-3.4: Add extra notes to override syntax changes ruby: Fix DEBUG_PREFIX_MAP in LDFLAGS issue gettext: Fix reproducibility issue with LDFLAGS curl: Fix reproducibility issue with LDFLAGS libtool: Fix lto option passing for reproducible builds Ross Burton (11): e2fsprogs: ensure small images have 256-byte inodes wic: don't forcibly pass -T default parted: drop unneeded ld-is-gold patch parted: update patch status buildtools-tarball: add testsdk task oeqa/sdk: add some buildtools tests bitbake: utils: add environment updating context manager bitbake: fetch2: expose environment variable names that need to be exported bitbake: fetch2/wget: ensure all variables are set when calling urllib bitbake: fetch2/wget: fetch securely by default tar: ignore node-tar CVEs Thomas Perrot (2): kernel-fitimage: images should not be signed with the same keys as the configurations oeqa/selftest/fitimage: update tests to use two keys Tim Orling (3): python3-scons{-native}: upgrade 4.1.0 -> 4.2.0 perl: do_create_rdepends_inc override syntax package.bbclass: FILER* override syntax Tom Rini (2): common-tasks: Add a summary to the end of the bbappend example manuals: Rename the "Using .bbappend Files in Your Layer" section Tony Battersby (2): bitbake.conf: add DEBUG_PREFIX_MAP to TARGET_LDFLAGS ruby: Fix reproducibility issue with LDFLAGS Tony Tascioglu (1): valgrind: skip broken ptests for glibc 2.34 Vyacheslav Yurkov (7): lib/oe: add generic functions for overlayfs overlayfs.bbclass: generate overlayfs mount units rootfs-postcommands: add QA check for overlayfs systemd-machine-units: add bbappend for meta-selftest overlayfs: meta-selftest recipe oeqa/selftest: overlayfs unit tests MAINTAINERS: add overlayfs maintainer Yi Zhao (3): dbus: add PACKAGECONFIG for audit and selinux glib-2.0: add PACKAGECONFIG for selinux shadow: add PACKAGECONFIG for audit and selinux hongxu (1): sdk: fix relocate symlink failed wangmy (1): ell: upgrade 0.41 -> 0.42 meta-raspberrypi: c7f4c739a3..32921fc9bd: Omer Akram (1): linux-firmware-rpidistro: fix wifi driver loading on cm4 Otavio Salvador (1): rpi-config: Allow setting hdmi_cvt meta-openembedded: 3cf2475ea0..a13db91f19: Changqing Li (1): ndpi: fix CVE-2021-36082 Chen Qi (1): Convert to new override syntax using latest convert-overrides.py script Dmitry Baryshkov (1): image_types_sparse: fix sparse image generation Geoff Parker (1): cifs-utils: typo fix fakse --> false Kai Kang (2): libdbi-perl: fix CVE-2014-10402 python3-m2crypto: fix for new overrides syntax Khem Raj (1): packagegroup-meta-oe: Add ttf-ipa Leon Anavi (15): python3-astroid: Upgrade 2.6.5 -> 2.6.6 python3-gast: Upgrade 0.5.1 -> 0.5.2 python3-greenlet: Upgrade 1.1.0 -> 1.1.1 python3-bitarray: Upgrade 2.2.3 -> 2.2.5 python3-send2trash: Upgrade 1.7.1 -> 1.8.0 python3-zeroconf: Upgrade 0.33.2 -> 0.34.3 python3-aiohue: Upgrade 2.5.1 -> 2.6.1 python3-configargparse: Upgrade 1.5.1 -> 1.5.2 python3-pycurl: Upgrade 7.43.0.6 -> 7.44.0 python3-distro: Upgrade 1.5.0 -> 1.6.0 python3-google-api-core: Upgrade 1.30.0 -> 1.31.1 python3-google-auth: Upgrade 1.32.0 -> 1.34.0 python3-google-api-python-client: Upgrade 2.12.0 -> 2.15.0 python3-huey: Upgrade 2.3.2 -> 2.4.0 python3-apply-defaults: Upgrade 0.1.4 -> 0.1.6 Martin Jansa (1): python3-grpcio: make sure that GRPC_CFLAGS is expanded to empty Michael Opdenacker (3): vorbis-tools: update to 1.4.2 (latest in 1.4.x series) bigbuckbunny-1080p: fix sample video URL opus-tools: update to 0.2, move to meta-multimedia and fix license Mingli Yu (3): jemalloc: fix the race during do_install jemalloc: add ptest support jemalloc: improve the ptest output Naveen Saini (1): python3-defusedxml: extend recipe to add native support Philippe Coval (1): mycroft: Install more tools needed by scripts Tony Battersby (3): curlpp: fix QA Issue after LDFLAGS change ldns: fix QA Issue after LDFLAGS change tcsh: fix compile error after LDFLAGS change Yi Zhao (5): audit: upgrade 3.0.3 -> 3.0.4 augeas: rename PACKAGECONFIG[libselinux] to PACKAGECONFIG[selinux] network-manager-applet: add selinux to PACKAGECONFIG if enable selinux distro feature networkmanager: add PACKAGECONFIG for audit and selinux augeas: add selinux to PACKAGECONFIG if enable selinux distro feature leimaohui (1): ttf-ipa: Added a new font. wangmy (1): iwd: upgrade 1.15 -> 1.16 zangrc (1): python3-humanize: upgrade 3.10.0 -> 3.11.0 zhengruoqin (3): python3-engineio: upgrade 4.2.0 -> 4.2.1 python3-ipython: upgrade 7.25.0 -> 7.26.0 python3-isort: upgrade 5.9.2 -> 5.9.3 Signed-off-by: Patrick Williams Change-Id: I7a8bd19709f465db51254ed3fcaf2486fe64dcaf --- poky/bitbake/lib/bb/fetch2/__init__.py | 46 ++++++++++++------------ poky/bitbake/lib/bb/fetch2/wget.py | 64 ++++++++++++++++++++++++++-------- poky/bitbake/lib/bb/tests/utils.py | 18 ++++++++++ poky/bitbake/lib/bb/utils.py | 16 +++++++++ 4 files changed, 107 insertions(+), 37 deletions(-) (limited to 'poky/bitbake/lib') diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index ad898680f..914fa5c02 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -808,6 +808,29 @@ def localpath(url, d): fetcher = bb.fetch2.Fetch([url], d) return fetcher.localpath(url) +# Need to export PATH as binary could be in metadata paths +# rather than host provided +# Also include some other variables. +FETCH_EXPORT_VARS = ['HOME', 'PATH', + 'HTTP_PROXY', 'http_proxy', + 'HTTPS_PROXY', 'https_proxy', + 'FTP_PROXY', 'ftp_proxy', + 'FTPS_PROXY', 'ftps_proxy', + 'NO_PROXY', 'no_proxy', + 'ALL_PROXY', 'all_proxy', + 'GIT_PROXY_COMMAND', + 'GIT_SSH', + 'GIT_SSL_CAINFO', + 'GIT_SMART_HTTP', + 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', + 'SOCKS5_USER', 'SOCKS5_PASSWD', + 'DBUS_SESSION_BUS_ADDRESS', + 'P4CONFIG', + 'SSL_CERT_FILE', + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_DEFAULT_REGION'] + def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): """ Run cmd returning the command output @@ -816,28 +839,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): Optionally remove the files/directories listed in cleanup upon failure """ - # Need to export PATH as binary could be in metadata paths - # rather than host provided - # Also include some other variables. - # FIXME: Should really include all export varaiables? - exportvars = ['HOME', 'PATH', - 'HTTP_PROXY', 'http_proxy', - 'HTTPS_PROXY', 'https_proxy', - 'FTP_PROXY', 'ftp_proxy', - 'FTPS_PROXY', 'ftps_proxy', - 'NO_PROXY', 'no_proxy', - 'ALL_PROXY', 'all_proxy', - 'GIT_PROXY_COMMAND', - 'GIT_SSH', - 'GIT_SSL_CAINFO', - 'GIT_SMART_HTTP', - 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', - 'SOCKS5_USER', 'SOCKS5_PASSWD', - 'DBUS_SESSION_BUS_ADDRESS', - 'P4CONFIG', - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY', - 'AWS_DEFAULT_REGION'] + exportvars = FETCH_EXPORT_VARS if not cleanup: cleanup = [] diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index 784df70c9..29fcfbb3d 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -52,13 +52,19 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler): class Wget(FetchMethod): + """Class to fetch urls via 'wget'""" # CDNs like CloudFlare may do a 'browser integrity test' which can fail # with the standard wget/urllib User-Agent, so pretend to be a modern # browser. user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" - """Class to fetch urls via 'wget'""" + def check_certs(self, d): + """ + Should certificates be checked? + """ + return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0" + def supports(self, ud, d): """ Check to see if a given url can be fetched with wget. @@ -82,7 +88,10 @@ class Wget(FetchMethod): if not ud.localfile: ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" + self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp" + + if not self.check_certs(d): + self.basecmd += " --no-check-certificate" def _runwget(self, ud, d, command, quiet, workdir=None): @@ -282,19 +291,44 @@ class Wget(FetchMethod): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = req.get_method return newreq - exported_proxies = export_proxies(d) - - handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] - if exported_proxies: - handlers.append(urllib.request.ProxyHandler()) - handlers.append(CacheHTTPHandler()) - # Since Python 2.7.9 ssl cert validation is enabled by default - # see PEP-0476, this causes verification errors on some https servers - # so disable by default. - import ssl - if hasattr(ssl, '_create_unverified_context'): - handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) - opener = urllib.request.build_opener(*handlers) + + # We need to update the environment here as both the proxy and HTTPS + # handlers need variables set. The proxy needs http_proxy and friends to + # be set, and HTTPSHandler ends up calling into openssl to load the + # certificates. In buildtools configurations this will be looking at the + # wrong place for certificates by default: we set SSL_CERT_FILE to the + # right location in the buildtools environment script but as BitBake + # prunes prunes the environment this is lost. When binaries are executed + # runfetchcmd ensures these values are in the environment, but this is + # pure Python so we need to update the environment. + # + # Avoid tramping the environment too much by using bb.utils.environment + # to scope the changes to the build_opener request, which is when the + # environment lookups happen. + newenv = {} + for name in bb.fetch2.FETCH_EXPORT_VARS: + value = d.getVar(name) + if not value: + origenv = d.getVar("BB_ORIGENV") + if origenv: + value = origenv.getVar(name) + if value: + newenv[name] = value + + with bb.utils.environment(**newenv): + import ssl + + if self.check_certs(d): + context = ssl.create_default_context() + else: + context = ssl._create_unverified_context() + + handlers = [FixedHTTPRedirectHandler, + HTTPMethodFallback, + urllib.request.ProxyHandler(), + CacheHTTPHandler(), + urllib.request.HTTPSHandler(context=context)] + opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] diff --git a/poky/bitbake/lib/bb/tests/utils.py b/poky/bitbake/lib/bb/tests/utils.py index a7ff33db5..4d5e21b99 100644 --- a/poky/bitbake/lib/bb/tests/utils.py +++ b/poky/bitbake/lib/bb/tests/utils.py @@ -666,3 +666,21 @@ class GetReferencedVars(unittest.TestCase): layers = [{"SRC_URI"}, {"QT_GIT", "QT_MODULE", "QT_MODULE_BRANCH_PARAM", "QT_GIT_PROTOCOL"}, {"QT_GIT_PROJECT", "QT_MODULE_BRANCH", "BPN"}, {"PN", "SPECIAL_PKGSUFFIX"}] self.check_referenced("${SRC_URI}", layers) + + +class EnvironmentTests(unittest.TestCase): + def test_environment(self): + os.environ["A"] = "this is A" + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertNotIn("B", os.environ) + + with bb.utils.environment(B="this is B"): + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertIn("B", os.environ) + self.assertEqual(os.environ["B"], "this is B") + + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertNotIn("B", os.environ) diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index e6e82d111..70634910f 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -1681,3 +1681,19 @@ def rename(src, dst): shutil.move(src, dst) else: raise err + +@contextmanager +def environment(**envvars): + """ + Context manager to selectively update the environment with the specified mapping. + """ + backup = dict(os.environ) + try: + os.environ.update(envvars) + yield + finally: + for var in envvars: + if var in backup: + os.environ[var] = backup[var] + else: + del os.environ[var] -- cgit v1.2.3