diff options
Diffstat (limited to 'poky/bitbake')
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst | 4 | ||||
| -rw-r--r-- | poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 4 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 46 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/wget.py | 64 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/tests/utils.py | 18 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/utils.py | 16 |
6 files changed, 115 insertions, 37 deletions
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 593de61f2..40b245b6d 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -144,6 +144,10 @@ download without a checksum triggers an error message. The make any attempted network access a fatal error, which is useful for checking that mirrors are complete as well as other things. +If :term:`BB_CHECK_SSL_CERTS` is set to ``0`` then SSL certificate checking will +be disabled. This variable defaults to ``1`` so SSL certificates are normally +checked. + .. _bb-the-unpack: The Unpack diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 6283c2654..2392ec425 100644 --- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -93,6 +93,10 @@ overview of their function and contents. fetcher does not attempt to use the host listed in :term:`SRC_URI` after a successful fetch from the :term:`PREMIRRORS` occurs. + :term:`BB_CHECK_SSL_CERTS` + Specifies if SSL certificates should be checked when fetching. The default + value is ``1`` and certificates are not checked if the value is set to ``0``. + :term:`BB_CONSOLELOG` Specifies the path to a log file into which BitBake's user interface writes output during the build. diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index ad898680f..914fa5c02 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -808,6 +808,29 @@ def localpath(url, d): fetcher = bb.fetch2.Fetch([url], d) return fetcher.localpath(url) +# Need to export PATH as binary could be in metadata paths +# rather than host provided +# Also include some other variables. +FETCH_EXPORT_VARS = ['HOME', 'PATH', + 'HTTP_PROXY', 'http_proxy', + 'HTTPS_PROXY', 'https_proxy', + 'FTP_PROXY', 'ftp_proxy', + 'FTPS_PROXY', 'ftps_proxy', + 'NO_PROXY', 'no_proxy', + 'ALL_PROXY', 'all_proxy', + 'GIT_PROXY_COMMAND', + 'GIT_SSH', + 'GIT_SSL_CAINFO', + 'GIT_SMART_HTTP', + 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', + 'SOCKS5_USER', 'SOCKS5_PASSWD', + 'DBUS_SESSION_BUS_ADDRESS', + 'P4CONFIG', + 'SSL_CERT_FILE', + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_DEFAULT_REGION'] + def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): """ Run cmd returning the command output @@ -816,28 +839,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): Optionally remove the files/directories listed in cleanup upon failure """ - # Need to export PATH as binary could be in metadata paths - # rather than host provided - # Also include some other variables. - # FIXME: Should really include all export varaiables? - exportvars = ['HOME', 'PATH', - 'HTTP_PROXY', 'http_proxy', - 'HTTPS_PROXY', 'https_proxy', - 'FTP_PROXY', 'ftp_proxy', - 'FTPS_PROXY', 'ftps_proxy', - 'NO_PROXY', 'no_proxy', - 'ALL_PROXY', 'all_proxy', - 'GIT_PROXY_COMMAND', - 'GIT_SSH', - 'GIT_SSL_CAINFO', - 'GIT_SMART_HTTP', - 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', - 'SOCKS5_USER', 'SOCKS5_PASSWD', - 'DBUS_SESSION_BUS_ADDRESS', - 'P4CONFIG', - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY', - 'AWS_DEFAULT_REGION'] + exportvars = FETCH_EXPORT_VARS if not cleanup: cleanup = [] diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index 784df70c9..29fcfbb3d 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -52,13 +52,19 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler): class Wget(FetchMethod): + """Class to fetch urls via 'wget'""" # CDNs like CloudFlare may do a 'browser integrity test' which can fail # with the standard wget/urllib User-Agent, so pretend to be a modern # browser. user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" - """Class to fetch urls via 'wget'""" + def check_certs(self, d): + """ + Should certificates be checked? + """ + return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0" + def supports(self, ud, d): """ Check to see if a given url can be fetched with wget. @@ -82,7 +88,10 @@ class Wget(FetchMethod): if not ud.localfile: ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" + self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp" + + if not self.check_certs(d): + self.basecmd += " --no-check-certificate" def _runwget(self, ud, d, command, quiet, workdir=None): @@ -282,19 +291,44 @@ class Wget(FetchMethod): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = req.get_method return newreq - exported_proxies = export_proxies(d) - - handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] - if exported_proxies: - handlers.append(urllib.request.ProxyHandler()) - handlers.append(CacheHTTPHandler()) - # Since Python 2.7.9 ssl cert validation is enabled by default - # see PEP-0476, this causes verification errors on some https servers - # so disable by default. - import ssl - if hasattr(ssl, '_create_unverified_context'): - handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) - opener = urllib.request.build_opener(*handlers) + + # We need to update the environment here as both the proxy and HTTPS + # handlers need variables set. The proxy needs http_proxy and friends to + # be set, and HTTPSHandler ends up calling into openssl to load the + # certificates. In buildtools configurations this will be looking at the + # wrong place for certificates by default: we set SSL_CERT_FILE to the + # right location in the buildtools environment script but as BitBake + # prunes prunes the environment this is lost. When binaries are executed + # runfetchcmd ensures these values are in the environment, but this is + # pure Python so we need to update the environment. + # + # Avoid tramping the environment too much by using bb.utils.environment + # to scope the changes to the build_opener request, which is when the + # environment lookups happen. + newenv = {} + for name in bb.fetch2.FETCH_EXPORT_VARS: + value = d.getVar(name) + if not value: + origenv = d.getVar("BB_ORIGENV") + if origenv: + value = origenv.getVar(name) + if value: + newenv[name] = value + + with bb.utils.environment(**newenv): + import ssl + + if self.check_certs(d): + context = ssl.create_default_context() + else: + context = ssl._create_unverified_context() + + handlers = [FixedHTTPRedirectHandler, + HTTPMethodFallback, + urllib.request.ProxyHandler(), + CacheHTTPHandler(), + urllib.request.HTTPSHandler(context=context)] + opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] diff --git a/poky/bitbake/lib/bb/tests/utils.py b/poky/bitbake/lib/bb/tests/utils.py index a7ff33db5..4d5e21b99 100644 --- a/poky/bitbake/lib/bb/tests/utils.py +++ b/poky/bitbake/lib/bb/tests/utils.py @@ -666,3 +666,21 @@ class GetReferencedVars(unittest.TestCase): layers = [{"SRC_URI"}, {"QT_GIT", "QT_MODULE", "QT_MODULE_BRANCH_PARAM", "QT_GIT_PROTOCOL"}, {"QT_GIT_PROJECT", "QT_MODULE_BRANCH", "BPN"}, {"PN", "SPECIAL_PKGSUFFIX"}] self.check_referenced("${SRC_URI}", layers) + + +class EnvironmentTests(unittest.TestCase): + def test_environment(self): + os.environ["A"] = "this is A" + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertNotIn("B", os.environ) + + with bb.utils.environment(B="this is B"): + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertIn("B", os.environ) + self.assertEqual(os.environ["B"], "this is B") + + self.assertIn("A", os.environ) + self.assertEqual(os.environ["A"], "this is A") + self.assertNotIn("B", os.environ) diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py index e6e82d111..70634910f 100644 --- a/poky/bitbake/lib/bb/utils.py +++ b/poky/bitbake/lib/bb/utils.py @@ -1681,3 +1681,19 @@ def rename(src, dst): shutil.move(src, dst) else: raise err + +@contextmanager +def environment(**envvars): + """ + Context manager to selectively update the environment with the specified mapping. + """ + backup = dict(os.environ) + try: + os.environ.update(envvars) + yield + finally: + for var in envvars: + if var in backup: + os.environ[var] = backup[var] + else: + del os.environ[var] |