diff options
Diffstat (limited to 'import-layers/yocto-poky/meta/recipes-devtools/python/python3')
8 files changed, 473 insertions, 330 deletions
diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Do-not-use-the-shell-version-of-python-config-that-w.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Do-not-use-the-shell-version-of-python-config-that-w.patch index b7e0ac6354..8ea3f03fe0 100644 --- a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Do-not-use-the-shell-version-of-python-config-that-w.patch +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Do-not-use-the-shell-version-of-python-config-that-w.patch @@ -1,4 +1,4 @@ -From 045c99b5f1eb6e4e0d8ad1ef9f0ba6574f738150 Mon Sep 17 00:00:00 2001 +From 04df959365e2b54d7503edf0e5534ff094284f2d Mon Sep 17 00:00:00 2001 From: Alexander Kanavin <alex.kanavin@gmail.com> Date: Fri, 23 Oct 2015 12:25:09 +0300 Subject: [PATCH] Do not use the shell version of python-config that was @@ -14,13 +14,13 @@ Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in -index d7fc9a0..47e60bc 100644 +index 236f005..5c4337f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in -@@ -1270,12 +1270,9 @@ python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh +@@ -1348,12 +1348,9 @@ python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh sed -e "s,@EXENAME@,$(BINDIR)/python$(LDVERSION)$(EXE)," < $(srcdir)/Misc/python-config.in >python-config.py # Replace makefile compat. variable references with shell script compat. ones; $(VAR) -> ${VAR} - sed -e 's,\$$(\([A-Za-z0-9_]*\)),\$$\{\1\},g' < Misc/python-config.sh >python-config + LC_ALL=C sed -e 's,\$$(\([A-Za-z0-9_]*\)),\$$\{\1\},g' < Misc/python-config.sh >python-config - # On Darwin, always use the python version of the script, the shell - # version doesn't use the compiler customizations that are provided - # in python (_osx_support.py). @@ -34,5 +34,5 @@ index d7fc9a0..47e60bc 100644 # Install the include files -- -2.1.4 +2.11.0 diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch new file mode 100644 index 0000000000..d1c92e9eed --- /dev/null +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch @@ -0,0 +1,66 @@ +From bcddbf40c7f1b80336268cdddacc17369fb0ccea Mon Sep 17 00:00:00 2001 +From: Libin Dang <libin.dang@windriver.com> +Date: Tue, 11 Apr 2017 14:12:15 +0800 +Subject: [PATCH] Issue #21272: Use _sysconfigdata.py to initialize + distutils.sysconfig + +Backport upstream commit +https://github.com/python/cpython/commit/409482251b06fe75c4ee56e85ffbb4b23d934159 + +Upstream-Status: Backport + +Signed-off-by: Li Zhou <li.zhou@windriver.com> +--- + Lib/distutils/sysconfig.py | 35 ++++------------------------------- + 1 file changed, 4 insertions(+), 31 deletions(-) + +diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py +index 6d5cfd0..9925d24 100644 +--- a/Lib/distutils/sysconfig.py ++++ b/Lib/distutils/sysconfig.py +@@ -424,38 +424,11 @@ _config_vars = None + + def _init_posix(): + """Initialize the module as appropriate for POSIX systems.""" +- g = {} +- # load the installed Makefile: +- try: +- filename = get_makefile_filename() +- parse_makefile(filename, g) +- except OSError as msg: +- my_msg = "invalid Python installation: unable to open %s" % filename +- if hasattr(msg, "strerror"): +- my_msg = my_msg + " (%s)" % msg.strerror +- +- raise DistutilsPlatformError(my_msg) +- +- # load the installed pyconfig.h: +- try: +- filename = get_config_h_filename() +- with open(filename) as file: +- parse_config_h(file, g) +- except OSError as msg: +- my_msg = "invalid Python installation: unable to open %s" % filename +- if hasattr(msg, "strerror"): +- my_msg = my_msg + " (%s)" % msg.strerror +- +- raise DistutilsPlatformError(my_msg) +- +- # On AIX, there are wrong paths to the linker scripts in the Makefile +- # -- these paths are relative to the Python source, but when installed +- # the scripts are in another directory. +- if python_build: +- g['LDSHARED'] = g['BLDSHARED'] +- ++ # _sysconfigdata is generated at build time, see the sysconfig module ++ from _sysconfigdata import build_time_vars + global _config_vars +- _config_vars = g ++ _config_vars = {} ++ _config_vars.update(build_time_vars) + + + def _init_nt(): +-- +1.8.3.1 + diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/000-cross-compile.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch index 2d822218f4..118d75ddc5 100644 --- a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/000-cross-compile.patch +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch @@ -1,27 +1,32 @@ +From 624c029abcc73c724020ccea9a2b4b5b5c00f2a6 Mon Sep 17 00:00:00 2001 +From: Alexander Kanavin <alex.kanavin@gmail.com> +Date: Fri, 31 Mar 2017 15:42:46 +0300 +Subject: [PATCH] cross-compile support + We cross compile python. This patch uses tools from host/native python instead of in-tree tools -Khem Upstream-Status: Inappropriate[Configuration Specific] - +Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> --- - Makefile.pre.in | 25 +++++++++++++------------ - 1 file changed, 13 insertions(+), 12 deletions(-) + Makefile.pre.in | 14 ++++++++------ + 1 file changed, 8 insertions(+), 6 deletions(-) -Index: Python-3.5.2/Makefile.pre.in -=================================================================== ---- Python-3.5.2.orig/Makefile.pre.in -+++ Python-3.5.2/Makefile.pre.in -@@ -220,6 +220,7 @@ LIBOBJS= @LIBOBJS@ +diff --git a/Makefile.pre.in b/Makefile.pre.in +index a88b7d5..7cb8bb3 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -221,6 +221,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -+HOSTPYTHON= $(BUILDPYTHON) ++HOSTPYTHON= $(BUILDPYTHON) - cross_compiling=@cross_compiling@ + PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ -@@ -279,6 +280,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ +@@ -280,6 +281,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ ########################################################################## # Parser PGEN= Parser/pgen$(EXE) @@ -29,7 +34,7 @@ Index: Python-3.5.2/Makefile.pre.in PSRCS= \ Parser/acceler.c \ -@@ -509,7 +511,7 @@ build_all_generate_profile: +@@ -510,7 +512,7 @@ build_all_generate_profile: run_profile_task: : # FIXME: can't run for a cross build @@ -38,16 +43,16 @@ Index: Python-3.5.2/Makefile.pre.in build_all_merge_profile: $(LLVM_PROF_MERGER) -@@ -792,7 +794,7 @@ $(GRAMMAR_H): $(GRAMMAR_INPUT) $(PGEN) +@@ -787,7 +789,7 @@ $(IO_OBJS): $(IO_H) + + $(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) @$(MKDIR_P) Include - # Avoid copying the file onto itself for an in-tree build - if test "$(cross_compiling)" != "yes"; then \ -- $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C); \ -+ $(HOSTPGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C); \ - else \ - cp $(srcdir)/Include/graminit.h $(GRAMMAR_H).tmp; \ - mv $(GRAMMAR_H).tmp $(GRAMMAR_H); \ -@@ -990,7 +992,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/pyth +- $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) ++ $(HOSTPGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) + $(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) + touch $(GRAMMAR_C) + +@@ -976,7 +978,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) ###################################################################### TESTOPTS= $(EXTRATESTOPTS) @@ -56,7 +61,7 @@ Index: Python-3.5.2/Makefile.pre.in TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py TESTTIMEOUT= 3600 -@@ -1481,7 +1483,7 @@ frameworkinstallstructure: $(LDLIBRARY) +@@ -1468,7 +1470,7 @@ frameworkinstallstructure: $(LDLIBRARY) fi; \ done $(LN) -fsn include/python$(LDVERSION) $(DESTDIR)$(prefix)/Headers @@ -65,7 +70,7 @@ Index: Python-3.5.2/Makefile.pre.in $(LN) -fsn $(VERSION) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/Current $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Headers $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers -@@ -1547,7 +1549,7 @@ config.status: $(srcdir)/configure +@@ -1534,7 +1536,7 @@ config.status: $(srcdir)/configure # Run reindent on the library reindent: @@ -74,7 +79,7 @@ Index: Python-3.5.2/Makefile.pre.in # Rerun configure with the same options as it was run last time, # provided the config.status script exists -@@ -1683,7 +1685,7 @@ funny: +@@ -1674,7 +1676,7 @@ funny: # Perform some verification checks on any modified files. patchcheck: all @@ -83,3 +88,6 @@ Index: Python-3.5.2/Makefile.pre.in # Dependencies +-- +2.11.0 + diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch new file mode 100644 index 0000000000..7217c6edea --- /dev/null +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch @@ -0,0 +1,56 @@ +From 62dcf34987b680e95873eb947b3f4d802199c667 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl> +Date: Fri, 10 Feb 2017 00:14:55 -0800 +Subject: [PATCH] Fix #29519: weakref spewing exceptions during interp + finalization + +commit 9cd7e17640a49635d1c1f8c2989578a8fc2c1de6 +from https://github.com/python/cpython + +Upstream-Status: Backport + +Signed-off-by: Lukasz Langa <lukasz@langa.pl> +--- + Lib/weakref.py | 4 ++-- + Misc/NEWS | 3 +++ + 2 files changed, 5 insertions(+), 2 deletions(-) + +diff --git a/Lib/weakref.py b/Lib/weakref.py +index aaebd0c..787e33a 100644 +--- a/Lib/weakref.py ++++ b/Lib/weakref.py +@@ -106,7 +106,7 @@ class WeakValueDictionary(collections.MutableMapping): + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) +- def remove(wr, selfref=ref(self)): ++ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): + self = selfref() + if self is not None: + if self._iterating: +@@ -114,7 +114,7 @@ class WeakValueDictionary(collections.MutableMapping): + else: + # Atomic removal is necessary since this function + # can be called asynchronously by the GC +- _remove_dead_weakref(d, wr.key) ++ _atomic_removal(d, wr.key) + self._remove = remove + # A list of keys to be removed + self._pending_removals = [] +diff --git a/Misc/NEWS b/Misc/NEWS +index 41cfdba..6d89f52 100644 +--- a/Misc/NEWS ++++ b/Misc/NEWS +@@ -5719,6 +5719,9 @@ Core and Builtins + Library + ------- + ++- Issue #29519: Fix weakref spewing exceptions during interpreter shutdown ++ when used with a rare combination of multiprocessing and custom codecs. ++ + - Issue #20154: Deadlock in asyncio.StreamReader.readexactly(). + + - Issue #16113: Remove sha3 module again. +-- +2.7.4 + diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/pass-missing-libraries-to-Extension-for-mul.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/pass-missing-libraries-to-Extension-for-mul.patch new file mode 100644 index 0000000000..5c3af6b626 --- /dev/null +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/pass-missing-libraries-to-Extension-for-mul.patch @@ -0,0 +1,82 @@ +From a784b70d47ba2104afbcfd805e2a66cdc2109ec5 Mon Sep 17 00:00:00 2001 +From: Hongxu Jia <hongxu.jia@windriver.com> +Date: Fri, 4 Aug 2017 11:16:14 +0800 +Subject: [PATCH] setup.py: pass missing libraries to Extension for multiprocessing module + +In the following commit: +... +commit e711cafab13efc9c1fe6c5cd75826401445eb585 +Author: Benjamin Peterson <benjamin@python.org> +Date: Wed Jun 11 16:44:04 2008 +0000 + + Merged revisions 64104,64117 via svnmerge from + svn+ssh://pythondev@svn.python.org/python/trunk +... +(see diff in setup.py) +It assigned libraries for multiprocessing module according +the host_platform, but not pass it to Extension. + +In glibc, the following commit caused two definition of +sem_getvalue are different. +https://sourceware.org/git/?p=glibc.git;a=commit;h=042e1521c794a945edc43b5bfa7e69ad70420524 +(see diff in nptl/sem_getvalue.c for detail) +`__new_sem_getvalue' is the latest sem_getvalue@@GLIBC_2.1 +and `__old_sem_getvalue' is to compat the old version +sem_getvalue@GLIBC_2.0. + +To build python for embedded Linux systems: +http://www.yoctoproject.org/docs/2.3.1/yocto-project-qs/yocto-project-qs.html +If not explicitly link to library pthread (-lpthread), it will +load glibc's sem_getvalue randomly at runtime. + +Such as build python on linux x86_64 host and run the python +on linux x86_32 target. If not link library pthread, it caused +multiprocessing bounded semaphore could not work correctly. +... +>>> import multiprocessing +>>> pool_sema = multiprocessing.BoundedSemaphore(value=1) +>>> pool_sema.acquire() +True +>>> pool_sema.release() +Traceback (most recent call last): + File "<stdin>", line 1, in <module> +ValueError: semaphore or lock released too many times +... + +And the semaphore issue also caused multiprocessing.Queue().put() hung. + +Upstream-Status: Submitted [https://github.com/python/cpython/pull/2999] + +Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com> +--- + setup.py | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index 4f0f522..d05707d 100644 +--- a/setup.py ++++ b/setup.py +@@ -1606,8 +1606,10 @@ class PyBuildExt(build_ext): + elif host_platform.startswith('netbsd'): + macros = dict() + libraries = [] +- +- else: # Linux and other unices ++ elif host_platform.startswith(('linux')): ++ macros = dict() ++ libraries = ['pthread'] ++ else: # Other unices + macros = dict() + libraries = ['rt'] + +@@ -1626,6 +1628,7 @@ class PyBuildExt(build_ext): + if sysconfig.get_config_var('WITH_THREAD'): + exts.append ( Extension('_multiprocessing', multiprocessing_srcs, + define_macros=list(macros.items()), ++ libraries=libraries, + include_dirs=["Modules/_multiprocessing"])) + else: + missing.append('_multiprocessing') +-- +2.7.4 + diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/python3-fix-CVE-2016-1000110.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/python3-fix-CVE-2016-1000110.patch deleted file mode 100644 index ab1b7230ea..0000000000 --- a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/python3-fix-CVE-2016-1000110.patch +++ /dev/null @@ -1,148 +0,0 @@ -From aab3e8c432b90508ac14755128f5a687be2fdf43 Mon Sep 17 00:00:00 2001 -From: Mingli Yu <Mingli.Yu@windriver.com> -Date: Thu, 22 Sep 2016 16:39:49 +0800 -Subject: [PATCH] python3: fix CVE-2016-1000110 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which -indicates that the script is in CGI mode. - -Issue #27568 Reported and patch contributed by Rémi Rampin. [#27568] - -Backport patch from https://hg.python.org/cpython/rev/a0ac52ed8f79 - -Upstream-Status: Backport -CVE: CVE-2016-1000110 -Signed-off-by: Mingli Yu <Mingli.Yu@windriver.com> ---- - Doc/howto/urllib2.rst | 5 +++++ - Doc/library/urllib.request.rst | 17 ++++++++++++++++- - Lib/test/test_urllib.py | 14 +++++++++++++- - Lib/urllib/request.py | 6 ++++++ - Misc/NEWS | 4 ++++ - 5 files changed, 44 insertions(+), 2 deletions(-) - -diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst -index 24a4156..d2c7991 100644 ---- a/Doc/howto/urllib2.rst -+++ b/Doc/howto/urllib2.rst -@@ -538,6 +538,11 @@ setting up a `Basic Authentication`_ handler: :: - through a proxy. However, this can be enabled by extending urllib.request as - shown in the recipe [#]_. - -+.. note:: -+ -+ ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; see -+ the documentation on :func:`~urllib.request.getproxies`. -+ - - Sockets and Layers - ================== -diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst -index 1338906..1291aeb 100644 ---- a/Doc/library/urllib.request.rst -+++ b/Doc/library/urllib.request.rst -@@ -173,6 +173,16 @@ The :mod:`urllib.request` module defines the following functions: - If both lowercase and uppercase environment variables exist (and disagree), - lowercase is preferred. - -+ .. note:: -+ -+ If the environment variable ``REQUEST_METHOD`` is set, which usually -+ indicates your script is running in a CGI environment, the environment -+ variable ``HTTP_PROXY`` (uppercase ``_PROXY``) will be ignored. This is -+ because that variable can be injected by a client using the "Proxy:" HTTP -+ header. If you need to use an HTTP proxy in a CGI environment, either use -+ ``ProxyHandler`` explicitly, or make sure the variable name is in -+ lowercase (or at least the ``_proxy`` suffix). -+ - - The following classes are provided: - -@@ -280,6 +290,11 @@ The following classes are provided: - list of hostname suffixes, optionally with ``:port`` appended, for example - ``cern.ch,ncsa.uiuc.edu,some.host:8080``. - -+ .. note:: -+ -+ ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; -+ see the documentation on :func:`~urllib.request.getproxies`. -+ - - .. class:: HTTPPasswordMgr() - -@@ -1138,7 +1153,7 @@ the returned bytes object to string once it determines or guesses - the appropriate encoding. - - The following W3C document, https://www.w3.org/International/O-charset\ , lists --the various ways in which a (X)HTML or a XML document could have specified its -+the various ways in which an (X)HTML or an XML document could have specified its - encoding information. - - As the python.org website uses *utf-8* encoding as specified in its meta tag, we -diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py -index 5d05f8d..247598a 100644 ---- a/Lib/test/test_urllib.py -+++ b/Lib/test/test_urllib.py -@@ -1,4 +1,4 @@ --"""Regresssion tests for what was in Python 2's "urllib" module""" -+"""Regression tests for what was in Python 2's "urllib" module""" - - import urllib.parse - import urllib.request -@@ -232,6 +232,18 @@ class ProxyTests(unittest.TestCase): - self.assertTrue(urllib.request.proxy_bypass_environment('anotherdomain.com:8888')) - self.assertTrue(urllib.request.proxy_bypass_environment('newdomain.com:1234')) - -+ def test_proxy_cgi_ignore(self): -+ try: -+ self.env.set('HTTP_PROXY', 'http://somewhere:3128') -+ proxies = urllib.request.getproxies_environment() -+ self.assertEqual('http://somewhere:3128', proxies['http']) -+ self.env.set('REQUEST_METHOD', 'GET') -+ proxies = urllib.request.getproxies_environment() -+ self.assertNotIn('http', proxies) -+ finally: -+ self.env.unset('REQUEST_METHOD') -+ self.env.unset('HTTP_PROXY') -+ - def test_proxy_bypass_environment_host_match(self): - bypass = urllib.request.proxy_bypass_environment - self.env.set('NO_PROXY', -diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py -index 1731fe3..3be327d 100644 ---- a/Lib/urllib/request.py -+++ b/Lib/urllib/request.py -@@ -2412,6 +2412,12 @@ def getproxies_environment(): - name = name.lower() - if value and name[-6:] == '_proxy': - proxies[name[:-6]] = value -+ # CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY -+ # (non-all-lowercase) as it may be set from the web server by a "Proxy:" -+ # header from the client -+ # If "proxy" is lowercase, it will still be used thanks to the next block -+ if 'REQUEST_METHOD' in os.environ: -+ proxies.pop('http', None) - for name, value in os.environ.items(): - if name[-6:] == '_proxy': - name = name.lower() -diff --git a/Misc/NEWS b/Misc/NEWS -index 4ad2551..2fcc95b 100644 ---- a/Misc/NEWS -+++ b/Misc/NEWS -@@ -329,6 +329,10 @@ Library - - Issue #26644: Raise ValueError rather than SystemError when a negative - length is passed to SSLSocket.recv() or read(). - -+- Issue #27568: Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the -+ HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates -+ that the script is in CGI mode. -+ - - Issue #23804: Fix SSL recv(0) and read(0) methods to return zero bytes - instead of up to 1024. - --- -2.8.1 - diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/support_SOURCE_DATE_EPOCH_in_py_compile.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/support_SOURCE_DATE_EPOCH_in_py_compile.patch new file mode 100644 index 0000000000..32ecab9fec --- /dev/null +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/support_SOURCE_DATE_EPOCH_in_py_compile.patch @@ -0,0 +1,97 @@ +The compiled .pyc files contain time stamp corresponding to the compile time. +This prevents binary reproducibility. This patch allows to achieve binary +reproducibility by overriding the build time stamp by the value +exported via SOURCE_DATE_EPOCH. + +Upstream-Status: Backport + +Signed-off-by: Juro Bystricky <juro.bystricky@intel.com> + + +From aeab488630fdb1b56a8d0b0c13fa88706b2afe9b Mon Sep 17 00:00:00 2001 +From: "Bernhard M. Wiedemann" <bwiedemann@suse.de> +Date: Sat, 25 Feb 2017 06:42:28 +0100 +Subject: [PATCH] bpo-29708: support SOURCE_DATE_EPOCH env var in py_compile + +to allow for reproducible builds of python packages + +See https://reproducible-builds.org/ for why this is good +and https://reproducible-builds.org/specs/source-date-epoch/ +for the definition of this variable. + +Background: +In some distributions like openSUSE, binary rpms contain precompiled .pyc files. + +And packages like amqp or twisted dynamically generate .py files at build time +so those have the current time and that timestamp gets embedded +into the .pyc file header. +When we then adapt file timestamps in rpms to be constant, +the timestamp in the .pyc header will no more match +the .py timestamp in the filesystem. +The software will still work, but it will not use the .pyc file as it should. +--- + Doc/library/py_compile.rst | 4 ++++ + Lib/py_compile.py | 4 ++++ + Lib/test/test_py_compile.py | 19 +++++++++++++++++++ + 3 files changed, 27 insertions(+) + +diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst +index 0af8fb1..841f3e8 100644 +--- a/Doc/library/py_compile.rst ++++ b/Doc/library/py_compile.rst +@@ -53,6 +53,10 @@ byte-code cache files in the directory containing the source code. + :func:`compile` function. The default of ``-1`` selects the optimization + level of the current interpreter. + ++ If the SOURCE_DATE_EPOCH environment variable is set, the .py file mtime ++ and timestamp entry in .pyc file header, will be limited to this value. ++ See https://reproducible-builds.org/specs/source-date-epoch/ for more info. ++ + .. versionchanged:: 3.2 + Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous + default was *file* + ``'c'`` (``'o'`` if optimization was enabled). +diff --git a/Lib/py_compile.py b/Lib/py_compile.py +index 11c5b50..62dcdc7 100644 +--- a/Lib/py_compile.py ++++ b/Lib/py_compile.py +@@ -137,6 +137,10 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): + except FileExistsError: + pass + source_stats = loader.path_stats(file) ++ sde = os.environ.get('SOURCE_DATE_EPOCH') ++ if sde and source_stats['mtime'] > int(sde): ++ source_stats['mtime'] = int(sde) ++ os.utime(file, (source_stats['mtime'], source_stats['mtime'])) + bytecode = importlib._bootstrap_external._code_to_bytecode( + code, source_stats['mtime'], source_stats['size']) + mode = importlib._bootstrap_external._calc_mode(file) +diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py +index 4a6caa5..3d09963 100644 +--- a/Lib/test/test_py_compile.py ++++ b/Lib/test/test_py_compile.py +@@ -98,6 +98,25 @@ def test_bad_coding(self): + self.assertFalse(os.path.exists( + importlib.util.cache_from_source(bad_coding))) + ++ def test_source_date_epoch(self): ++ testtime = 123456789 ++ orig_sde = os.getenv("SOURCE_DATE_EPOCH") ++ os.environ["SOURCE_DATE_EPOCH"] = str(testtime) ++ py_compile.compile(self.source_path, self.pyc_path) ++ if orig_sde: ++ os.environ["SOURCE_DATE_EPOCH"] = orig_sde ++ else: ++ del os.environ["SOURCE_DATE_EPOCH"] ++ self.assertTrue(os.path.exists(self.pyc_path)) ++ self.assertFalse(os.path.exists(self.cache_path)) ++ statinfo = os.stat(self.source_path) ++ self.assertEqual(statinfo.st_mtime, testtime) ++ f = open(self.pyc_path, "rb") ++ f.read(4) ++ timebytes = f.read(4) # read timestamp from pyc header ++ f.close() ++ self.assertEqual(timebytes, (testtime).to_bytes(4, 'little')) ++ + @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O') + def test_double_dot_no_clobber(self): + # http://bugs.python.org/issue22966 diff --git a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/upstream-random-fixes.patch b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/upstream-random-fixes.patch index 0d9152ccd7..9b40e8ac9f 100644 --- a/import-layers/yocto-poky/meta/recipes-devtools/python/python3/upstream-random-fixes.patch +++ b/import-layers/yocto-poky/meta/recipes-devtools/python/python3/upstream-random-fixes.patch @@ -1,21 +1,7 @@ -This patch updates random.c to match upstream python's code at revision -8125d9a8152b. This addresses various issues around problems with glibc 2.24 -and 2.25 such that python would fail to start with: - -[rpurdie@centos7 ~]$ /tmp/t2/sysroots/x86_64-pokysdk-linux/usr/bin/python3 -Fatal Python error: getentropy() failed -Aborted - -(taken from our buildtools-tarball also breaks eSDK) - -Upstream-Status: Backport - -# HG changeset patch -# User Victor Stinner <victor.stinner@gmail.com> -# Date 1483957133 -3600 -# Node ID 8125d9a8152b79e712cb09c7094b9129b9bcea86 -# Parent 337461574c90281630751b6095c4e1baf380cf7d -Issue #29157: Prefer getrandom() over getentropy() +From 035ba5da3e53e45c712b39fe1f6fb743e697c032 Mon Sep 17 00:00:00 2001 +From: Victor Stinner <victor.stinner@gmail.com> +Date: Mon, 9 Jan 2017 11:18:53 +0100 +Subject: [PATCH] Issue #29157: Prefer getrandom() over getentropy() Copy and then adapt Python/random.c from default branch. Difference between 3.5 and default branches: @@ -26,12 +12,17 @@ and default branches: * Python 3.5 has no _PyOS_URandomNonblock() function: _PyOS_URandom() works in non-blocking mode on Python 3.5 -RP 2017/1/22 +Upstream-Status: Backport [https://github.com/python/cpython/commit/035ba5da3e53e45c712b39fe1f6fb743e697c032] +Signed-off-by: Alexander Kanavin <alexander.kanavin@intel.com> + +--- + Python/random.c | 494 +++++++++++++++++++++++++++++++++----------------------- + 1 file changed, 294 insertions(+), 200 deletions(-) -Index: Python-3.5.2/Python/random.c -=================================================================== ---- Python-3.5.2.orig/Python/random.c -+++ Python-3.5.2/Python/random.c +diff --git a/Python/random.c b/Python/random.c +index d203939..31f61d0 100644 +--- a/Python/random.c ++++ b/Python/random.c @@ -1,6 +1,9 @@ #include "Python.h" #ifdef MS_WINDOWS @@ -42,7 +33,7 @@ Index: Python-3.5.2/Python/random.c #else # include <fcntl.h> # ifdef HAVE_SYS_STAT_H -@@ -36,10 +39,9 @@ win32_urandom_init(int raise) +@@ -37,10 +40,9 @@ win32_urandom_init(int raise) return 0; error: @@ -55,7 +46,7 @@ Index: Python-3.5.2/Python/random.c return -1; } -@@ -52,8 +54,9 @@ win32_urandom(unsigned char *buffer, Py_ +@@ -53,8 +55,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) if (hCryptProv == 0) { @@ -66,7 +57,7 @@ Index: Python-3.5.2/Python/random.c } while (size > 0) -@@ -62,11 +65,9 @@ win32_urandom(unsigned char *buffer, Py_ +@@ -63,11 +66,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer)) { /* CryptGenRandom() failed */ @@ -80,7 +71,7 @@ Index: Python-3.5.2/Python/random.c return -1; } buffer += chunk; -@@ -75,55 +76,29 @@ win32_urandom(unsigned char *buffer, Py_ +@@ -76,58 +77,23 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) return 0; } @@ -129,13 +120,19 @@ Index: Python-3.5.2/Python/random.c #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) #define PY_GETRANDOM 1 +-/* Call getrandom() +/* Call getrandom() to get random bytes: + -+ - Return 1 on success + - Return 1 on success +- - Return 0 if getrandom() syscall is not available (failed with ENOSYS or +- EPERM) or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom +- not initialized yet) and raise=0. + - Return 0 if getrandom() is not available (failed with ENOSYS or EPERM), + or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom not + initialized yet). -+ - Raise an exception (if raise is non-zero) and return -1 on error: + - Raise an exception (if raise is non-zero) and return -1 on error: +- getrandom() failed with EINTR and the Python signal handler raised an +- exception, or getrandom() failed with a different error. */ + if getrandom() failed with EINTR, raise is non-zero and the Python signal + handler raised an exception, or if getrandom() failed with a different + error. @@ -144,26 +141,16 @@ Index: Python-3.5.2/Python/random.c static int py_getrandom(void *buffer, Py_ssize_t size, int raise) { -- /* Is getrandom() supported by the running kernel? -- * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */ -+ /* Is getrandom() supported by the running kernel? Set to 0 if getrandom() -+ failed with ENOSYS or EPERM. Need Linux kernel 3.17 or newer, or Solaris -+ 11.3 or newer */ - static int getrandom_works = 1; - - /* getrandom() on Linux will block if called before the kernel has -@@ -132,84 +107,165 @@ py_getrandom(void *buffer, Py_ssize_t si +@@ -142,16 +108,19 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) * see https://bugs.python.org/issue26839. To avoid this, use the * GRND_NONBLOCK flag. */ const int flags = GRND_NONBLOCK; -- int n; + char *dest; -+ long n; + long n; -- if (!getrandom_works) -+ if (!getrandom_works) { + if (!getrandom_works) { return 0; -+ } + } + dest = buffer; while (0 < size) { @@ -174,11 +161,8 @@ Index: Python-3.5.2/Python/random.c + requested. */ n = Py_MIN(size, 1024); #else -- n = size; -+ n = Py_MIN(size, LONG_MAX); - #endif - - errno = 0; + n = Py_MIN(size, LONG_MAX); +@@ -161,34 +130,35 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) #ifdef HAVE_GETRANDOM if (raise) { Py_BEGIN_ALLOW_THREADS @@ -209,56 +193,45 @@ Index: Python-3.5.2/Python/random.c #endif if (n < 0) { -- if (errno == ENOSYS) { +- /* ENOSYS: getrandom() syscall not supported by the kernel (but +- * maybe supported by the host which built Python). EPERM: +- * getrandom() syscall blocked by SECCOMP or something else. */ + /* ENOSYS: the syscall is not supported by the kernel. + EPERM: the syscall is blocked by a security policy (ex: SECCOMP) + or something else. */ -+ if (errno == ENOSYS || errno == EPERM) { + if (errno == ENOSYS || errno == EPERM) { getrandom_works = 0; return 0; } + if (errno == EAGAIN) { -- /* If we failed with EAGAIN, the entropy pool was -- * uninitialized. In this case, we return failure to fall -- * back to reading from /dev/urandom. -- * -- * Note: In this case the data read will not be random so -- * should not be used for cryptographic purposes. Retaining -- * the existing semantics for practical purposes. */ -+ /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system -+ urandom is not initialiazed yet. In this case, fall back on -+ reading from /dev/urandom. -+ -+ Note: In this case the data read will not be random so -+ should not be used for cryptographic purposes. Retaining -+ the existing semantics for practical purposes. */ - getrandom_works = 0; - return 0; + /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system + urandom is not initialiazed yet. In this case, fall back on +@@ -202,169 +172,225 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) } if (errno == EINTR) { - if (PyErr_CheckSignals()) { -- if (!raise) +- if (!raise) { - Py_FatalError("getrandom() interrupted by a signal"); -- return -1; + if (raise) { + if (PyErr_CheckSignals()) { + return -1; -+ } + } +- return -1; } + - /* retry getrandom() */ -+ + /* retry getrandom() if it was interrupted by a signal */ continue; } -- if (raise) -+ if (raise) { + if (raise) { PyErr_SetFromErrno(PyExc_OSError); -- else + } +- else { - Py_FatalError("getrandom() failed"); -+ } +- } return -1; } @@ -269,12 +242,19 @@ Index: Python-3.5.2/Python/random.c return 1; } -#endif -+ + +-static struct { +- int fd; +- dev_t st_dev; +- ino_t st_ino; +-} urandom_cache = { -1 }; +#elif defined(HAVE_GETENTROPY) +#define PY_GETENTROPY 1 -+ + +/* Fill buffer with size pseudo-random bytes generated by getentropy(): -+ + +-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from +- /dev/urandom if getrandom() is not available. + - Return 1 on success + - Return 0 if getentropy() syscall is not available (failed with ENOSYS or + EPERM). @@ -282,25 +262,47 @@ Index: Python-3.5.2/Python/random.c + if getentropy() failed with EINTR, raise is non-zero and the Python signal + handler raised an exception, or if getentropy() failed with a different + error. -+ + +- Call Py_FatalError() on error. */ +-static void +-dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) + getentropy() is retried if it failed with EINTR: interrupted by a signal. */ +static int +py_getentropy(char *buffer, Py_ssize_t size, int raise) -+{ + { +- int fd; +- Py_ssize_t n; + /* Is getentropy() supported by the running kernel? Set to 0 if + getentropy() failed with ENOSYS or EPERM. */ + static int getentropy_works = 1; -+ + +- assert (0 < size); +- +-#ifdef PY_GETRANDOM +- if (py_getrandom(buffer, size, 0) == 1) { +- return; + if (!getentropy_works) { + return 0; -+ } -+ + } +- /* getrandom() failed with ENOSYS or EPERM, +- fall back on reading /dev/urandom */ +-#endif + +- fd = _Py_open_noraise("/dev/urandom", O_RDONLY); +- if (fd < 0) { +- Py_FatalError("Failed to open /dev/urandom"); +- } + while (size > 0) { + /* getentropy() is limited to returning up to 256 bytes. Call it + multiple times if more bytes are requested. */ + Py_ssize_t len = Py_MIN(size, 256); + int res; -+ + +- while (0 < size) +- { +- do { +- n = read(fd, buffer, (size_t)size); +- } while (n < 0 && errno == EINTR); + if (raise) { + Py_BEGIN_ALLOW_THREADS + res = getentropy(buffer, len); @@ -309,7 +311,11 @@ Index: Python-3.5.2/Python/random.c + else { + res = getentropy(buffer, len); + } -+ + +- if (n <= 0) { +- /* read() failed or returned 0 bytes */ +- Py_FatalError("Failed to read bytes from /dev/urandom"); +- break; + if (res < 0) { + /* ENOSYS: the syscall is not supported by the running kernel. + EPERM: the syscall is blocked by a security policy (ex: SECCOMP) @@ -334,71 +340,44 @@ Index: Python-3.5.2/Python/random.c + PyErr_SetFromErrno(PyExc_OSError); + } + return -1; -+ } + } +- buffer += n; +- size -= n; + + buffer += len; + size -= len; -+ } + } +- close(fd); + return 1; -+} + } +#endif /* defined(HAVE_GETENTROPY) && !defined(sun) */ -+ - static struct { - int fd; -@@ -217,127 +273,123 @@ static struct { - ino_t st_ino; - } urandom_cache = { -1 }; +-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from +- /dev/urandom if getrandom() is not available. +- Return 0 on success. Raise an exception and return -1 on error. */ ++static struct { ++ int fd; ++ dev_t st_dev; ++ ino_t st_ino; ++} urandom_cache = { -1 }; ++ +/* Read random bytes from the /dev/urandom device: - --/* Read size bytes from /dev/urandom into buffer. -- Call Py_FatalError() on error. */ --static void --dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) --{ -- int fd; -- Py_ssize_t n; ++ + - Return 0 on success + - Raise an exception (if raise is non-zero) and return -1 on error - -- assert (0 < size); ++ + Possible causes of errors: - --#ifdef PY_GETRANDOM -- if (py_getrandom(buffer, size, 0) == 1) -- return; -- /* getrandom() is not supported by the running kernel, fall back -- * on reading /dev/urandom */ --#endif ++ + - open() failed with ENOENT, ENXIO, ENODEV, EACCES: the /dev/urandom device + was not found. For example, it was removed manually or not exposed in a + chroot or container. + - open() failed with a different error + - fstat() failed + - read() failed or returned 0 - -- fd = _Py_open_noraise("/dev/urandom", O_RDONLY); -- if (fd < 0) -- Py_FatalError("Failed to open /dev/urandom"); ++ + read() is retried if it failed with EINTR: interrupted by a signal. - -- while (0 < size) -- { -- do { -- n = read(fd, buffer, (size_t)size); -- } while (n < 0 && errno == EINTR); -- if (n <= 0) -- { -- /* stop on error or if read(size) returned 0 */ -- Py_FatalError("Failed to read bytes from /dev/urandom"); -- break; -- } -- buffer += n; -- size -= (Py_ssize_t)n; -- } -- close(fd); --} ++ + The file descriptor of the device is kept open between calls to avoid using + many file descriptors when run in parallel from multiple threads: + see the issue #18756. @@ -406,9 +385,7 @@ Index: Python-3.5.2/Python/random.c + st_dev and st_ino fields of the file descriptor (from fstat()) are cached to + check if the file descriptor was replaced by a different file (which is + likely a bug in the application): see the issue #21207. - --/* Read size bytes from /dev/urandom into buffer. -- Return 0 on success, raise an exception and return -1 on error. */ ++ + If the file descriptor was closed or replaced, open a new file descriptor + but don't close the old file descriptor: it probably points to something + important for some third-party code. */ @@ -422,22 +399,24 @@ Index: Python-3.5.2/Python/random.c -#ifdef PY_GETRANDOM - int res; -#endif - +- - if (size <= 0) - return 0; -+ if (raise) { -+ struct _Py_stat_struct st; -#ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, 1); -- if (res < 0) +- if (res < 0) { - return -1; -- if (res == 1) +- } +- if (res == 1) { - return 0; -- /* getrandom() is not supported by the running kernel, fall back -- * on reading /dev/urandom */ +- } +- /* getrandom() failed with ENOSYS or EPERM, +- fall back on reading /dev/urandom */ -#endif -- ++ if (raise) { ++ struct _Py_stat_struct st; + - if (urandom_cache.fd >= 0) { - /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) @@ -516,8 +495,9 @@ Index: Python-3.5.2/Python/random.c - do { - n = _Py_read(fd, buffer, (size_t)size); -- if (n == -1) +- if (n == -1) { - return -1; +- } - if (n == 0) { - PyErr_Format(PyExc_RuntimeError, - "Failed to read %zi bytes from /dev/urandom", @@ -566,7 +546,7 @@ Index: Python-3.5.2/Python/random.c return 0; } -@@ -349,8 +401,8 @@ dev_urandom_close(void) +@@ -376,8 +402,8 @@ dev_urandom_close(void) urandom_cache.fd = -1; } } @@ -576,7 +556,7 @@ Index: Python-3.5.2/Python/random.c /* Fill buffer with pseudo-random bytes generated by a linear congruent generator (LCG): -@@ -373,29 +425,98 @@ lcg_urandom(unsigned int x0, unsigned ch +@@ -400,31 +426,100 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) } } @@ -661,7 +641,7 @@ Index: Python-3.5.2/Python/random.c #else - return dev_urandom_python((char*)buffer, size); + res = py_getentropy(buffer, size, raise); - #endif ++#endif + if (res < 0) { + return -1; + } @@ -673,9 +653,9 @@ Index: Python-3.5.2/Python/random.c +#endif + + return dev_urandom(buffer, size, raise); -+#endif -+} -+ + #endif + } + +/* Fill buffer with size pseudo-random bytes from the operating system random + number generator (RNG). It is suitable for most cryptographic purposes + except long living private keys for asymmetric encryption. @@ -685,10 +665,12 @@ Index: Python-3.5.2/Python/random.c +_PyOS_URandom(void *buffer, Py_ssize_t size) +{ + return pyurandom(buffer, size, 1); - } - ++} ++ void -@@ -436,13 +557,14 @@ _PyRandom_Init(void) + _PyRandom_Init(void) + { +@@ -463,13 +558,14 @@ _PyRandom_Init(void) } } else { @@ -710,7 +692,7 @@ Index: Python-3.5.2/Python/random.c } } -@@ -454,8 +576,6 @@ _PyRandom_Fini(void) +@@ -481,8 +577,6 @@ _PyRandom_Fini(void) CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } |