From 5f35090dc809872fb7b8f381c1ccf995b75f03a0 Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Fri, 23 Jul 2021 13:09:54 -0400 Subject: poky: subtree update:2834c2f853..17aabc0127 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adam Romanek (3): bitbake: fetch/git: run gc in foreground to avoid race with tar bitbake: fetch2/s3: allow to use credentials from environment variables own-mirrors: Add support for s3:// scheme in SOURCE_MIRROR_URL Alexander Kanavin (44): devtool upgrade: rebase override-only patches as well cmake: update 3.20.3 -> 3.20.4 gnu-config: update to latest revision bash: update 5.1 -> 5.1.8 dnf: updatee 4.7.0 -> 4.8.0 vulkan-samples: update to latest revision bind: upgrade 9.16.16 -> 9.16.18 bluez5: upgrade 5.58 -> 5.59 dejagnu: upgrade 1.6.2 -> 1.6.3 libdnf: upgrade 0.63.0 -> 0.63.1 libpcre: upgrade 8.44 -> 8.45 libxi: upgrade 1.7.10 -> 1.7.99.2 mtools: upgrade 4.0.29 -> 4.0.31 python3-git: upgrade 3.1.17 -> 3.1.18 sqlite3: upgrade 3.35.5 -> 3.36.0 vte: upgrade 0.64.1 -> 0.64.2 x264: upgrade to latest revision python3: apply test skipping patch unconditionally pypi: set SRC_URI with _prepend, not with += man-pages: upgrade 5.11 -> 5.12 rt-tests: update 1.10 -> 2.1 python3-iniparse: update 0.4 -> 0.5 iputils: correct upstream version check texinfo: update 6.7 -> 6.8 xserver-xorg: exclude development snapshots from upstream version checks xwayland: exclude development snapshots from upstream version checks devtool: correctly handle non-standard source tree locations in upgrades llvm: make upgradable via devtool xserver-xorg: update 1.20.11 -> 1.20.12 libmodulemd: update 2.12.1 -> 2.13.0 bluez5: upgrade 5.59 -> 5.60 createrepo-c: upgrade 0.17.3 -> 0.17.4 ethtool: upgrade 5.12 -> 5.13 gtk+3: upgrade 3.24.29 -> 3.24.30 harfbuzz: upgrade 2.8.1 -> 2.8.2 iproute2: upgrade 5.12.0 -> 5.13.0 libgit2: upgrade 1.1.0 -> 1.1.1 mpg123: upgrade 1.28.0 -> 1.28.2 mtools: upgrade 4.0.31 -> 4.0.32 ruby: upgrade 3.0.1 -> 3.0.2 stress-ng: upgrade 0.12.11 -> 0.12.12 webkitgtk: upgrade 2.32.1 -> 2.32.2 xwayland: upgrade 21.1.1 -> 21.1.2 tcl: fix upstream version check Alexey Brodkin (3): dpkg: Add ARC support default-distrovars.inc: Remove seccomp for ARC dhcpcd: add ARC support Andreas Müller (4): libdrm: upgrade 2.4.106 -> 2.4.107 mesa: upgrade 21.1.3 -> 21.1.4 pango: upgrade 1.48.5 -> 1.48.7 mesa: upgrade 21.1.4 -> 21.1.5 Andrej Valek (1): busybox: add tmpdir option into mktemp applet Armin Kuster (1): maintainers.inc: remove myself as a Maintainer Asfak Rahman (1): openssh: Remove temporary keys before generating new ones Bruce Ashfield (24): linux-yocto/5.10: update to v5.10.46 linux-yocto/5.10: features/nft_tables: refresh config options linux-yocto/5.4: update to v5.4.128 linux-yocto/5.10: rcu: Fix stall-warning deadlock due to non-release of rcu_node ->lock linux-yocto/5.10: update to v5.10.47 linux-yocto/5.4: update to v5.4.129 linux-yocto/5.10: scsi-debug needs scsi-disk linux-libc-headers: update to 5.13 kernel-devsrc: fix scripts/prepare for ARM64 kernel-devsrc: fix scripts prepare for powerpc kernel-devsrc: powerpc64 add scripts prepare support linux-yocto: introduce 5.13 recipes linux-yocto/5.13: add devupstream support linux-yocto-rt/5.13: integrate -rt1 linux-yocto: add vfat KERNEL_FEATURE when MACHINE_FEATURES include vfat linux-yocto/5.10: update to v5.10.49 linux-yocto/5.4: update to v5.4.131 linux-yocto/5.13: update to v5.13.2 linux-yocto/5.10: update to v5.10.50 linux-yocto/5.4: update to v5.4.132 linux-yocto/5.13: update to v5.13.3 linux-yocto/5.13: update to v5.13.4 linux-yocto/5.10: update to v5.10.52 linux-yocto/5.4: update to v5.4.134 Changhyeok Bae (1): iputils: Update to 20210202 Changqing Li (1): boost-build-native: workaround one rarely hang problem on fedora34 Christoph Muellner (1): ldconfig-native: Add RISC-V support Damian Wrobel (1): gobject-introspection: Fix the license (add MIT) Denys Dmytriyenko (1): bitbake: providers: replace newly added logger.warn() with logger.warning() Fabio Berton (1): lib/oe/package_manager: Don't ignore installation failures in install_complementary Florian Amstutz (1): devtool: deploy-target: Fix preserving attributes when using --strip Jose Quaresma (3): glslang: upgrade 11.4.0 -> 11.5.0 shaderc: upgrade 2021.0 -> 2021.1 spirv-tools: upgrade 2021.1 -> 2021.2 Joshua Watt (3): ref-manual: Document BUILDHISTORY_PATH_PREFIX_STRIP bitbake: bitbake: Add piping compression library bitbake: server: Fix early parsing errors preventing zombie bitbake Khem Raj (7): glib-2.0: Fix signature of close_range gnome-desktop-testing: Fix non-literal format string warning util-linux: Disable chfn-chsh on non-target builds libseccomp: Update to main branch systemd: Fix libseccomp testcase involving __NR_ppoll util-linux: Fix signature of close_range() gpgme: Use glibc provided closefrom API when available Lee Chee Yang (1): qemu: fix CVE-2021-3527 Marek Vasut (2): pulseaudio: Drop pulseaudio-conf update-rc.d: update SRCREV to pull in fix for non-bash shell support Mark Hatle (1): populate_sdk_ext: Error if trying to generate an eSDK from a mulitconfig Max Krummenacher (1): xwayland: port packageconfig from xserver-xorg recipe Michael Halstead (1): releases: update to include 3.1.9 Michael Ho (1): sstate.bbclass: fix errors about read-only sstate mirrors Mike Crowe (1): licence_image: Add lic-pkgs IMAGE_FEATURE Mingli Yu (4): pulseaudio: check if NEON code can be compiled on arm perlcross: check the file if patched or not perl: correct libpth and glibpth parselogs.py: ignore rndr initialization failure Patrick Williams (1): docs: remove image-mklibs references Paul Barker (1): linux-yocto: Fix devupstream SRCREV assignment Peter Bergin (1): Revert "libubootenv: inherit uboot-config" Quentin Schulz (1): docs: replace remaining ``FOO`` by :term:`FOO` Ralph Siemsen (1): oeqa/manual/toaster: fix small typo Richard Purdie (26): package_pkgdata: Avoid task hash mismatches for generic task changes selftest/fetch: Avoid occasional selftest failure from poor temp file name choice kernel: Fix interaction when packaging disabled kernel-devicetree: Fix interaction when packaging disabled python3-pip/python3-pbr: Drop obsolete md5sum at upgrade oeqa/selftest/runcmd: Tweal test timeouts bash: Fix a rare make race build failure sstate/staging: Handle directory creation race issue oeqa/selftest/archiver: Allow tests to ignore empty directories dwarfsrcfiles: Avoid races over debug-link files oeqa/selftest/multiprocesslauch: Fix test race runqemu: Remove potential lock races around tap device handling glibc-testsuite: Fix build failures when directly running recipe license: Drop adding RRECOMMENDS for license packages report-error: Drop pointless inherit pseudo: Add uninative configuration sanity check pseudo: Update to latest version including statx fix sstate: Drop pseudo exclusion oeqa/qemurunner: Handle files no longer existing gracefully python3: Add a fix for a make install race Add README link to README.poky README.OE-Core/README.qemu: Move to markdown format bitbake: data_smart/parse: Allow ':' characters in variable/function names zstd: Include pzstd in the build buildtools-tarball: Add lz4 and ztsd (including pzstd) build-appliance-image: Update to master head revision Ross Burton (29): glibc: backport MTE improvements from master glibc: re-enable memory tagging libgudev: fix SRC_URI cantarell-fonts: fix SRC_URI shadow: generate alternative links for chfn and chsh util-linux: build chfn and chsh util-linux: add missing ptest dependencies util-linux: backport test suite fixes util-linux: rewrite the ptest integration glib-2.0: fix g-file-into modification time test oeqa/selftest/recipetool: update socat version to fix failing download parted: improve ptest execution tcl: suffix all Tcl man pages instead of using alternatives tcl: EXTRA_OECONF already passes --libdir, don't do it again tcl: mark a patch as upstreamable tcl: use tcl-core instead of patching out packages tcl: use AUTOTOOLS_SCRIPT_PATH tcl: remove redundant file creation tcl: detect tests that error as well as fail tcl: clock.test needs a timezone to be set tcl: fix race in interp.test parted: remove obsolete patch parted: fix ptest RRECOMMENDS busybox: support mounting swap via labels meta: remove redundant ${libdir}/cmake from FILES_${PN}-dev cups: update to 2.3.3op2 parted: skip tests that need vfat support avahi: fix CVE-2021-36217, crash on pinging '.local' parted: add device mappper PACKAGECONFIG Sakib Sajal (3): buildstats.bbclass: log host data on failure to task specific file oe-time-dd-test.sh: add options and refactor scripts/oe-time-dd-test.sh: run "uptime" on each iteration Saul Wold (4): qemurunner: Add info log for qemu startup oeqa/qemurunner: add support qmp cmd args oeqa/dump.py: Add support for QMP command arguments testimage.bbclass: Add dump-guest-memory cmd Scott Weaver (1): meta-skeleton: update to satisfy yocto-check-layer tests Tim Orling (6): python3-importlib-metadata: upgrade 4.5.0 -> 4.6.0 python3-packaging: upgrade 20.9 -> 21.0 python3-hypothesis: upgrade 6.14.0 -> 6.14.1 python3-zipp: upgrade 3.4.1 -> 3.5.0 at-spi2-core: upgrade 2.40.2 -> 2.40.3 python3-hypothesis: upgrade 6.14.1 -> 6.14.3 Tony Tascioglu (1): valgrind: skip flaky ptest fb_test_amd64 Trevor Gamblin (2): python3-pip: upgrade 20.0.2 -> 21.1.2 python3-pip: add multiprocessing to RDEPENDS Yi Zhao (1): libffi: disable use of static exec trampolines Zoltán Böszörményi (1): tzdata: Allow controlling zoneinfo binary format Zqiang (1): ifupdown: Skip wrong test item bkylerussell@gmail.com (1): gstreamer: add libgst packages to PACKAGES_DYNAMIC jbouchard (1): Use the label provided when formating a dos partition wangmy (19): libcap: upgrade 2.50 -> 2.51 libhandy: upgrade 1.2.2 -> 1.2.3 libva: upgrade 2.11.0 -> 2.12.0 libxcrypt: upgrade 4.4.22 -> 4.4.23 python3-numpy: upgrade 1.20.3 -> 1.21.0 python3-pbr: upgrade 5.4.4 -> 5.6.0 cmake: upgrade 3.20.4 -> 3.20.5 gpgme: upgrade 1.15.1 -> 1.16.0 libglu: upgrade 9.0.1 -> 9.0.2 stress-ng: upgrade 0.12.10 -> 0.12.11 xf86-input-libinput: upgrade 1.0.1 -> 1.1.0 vulkan-headers: upgrade 1.2.176 -> 1.2.182 vulkan-loader: upgrade 1.2.176 -> 1.2.182 vulkan-tools: upgrade 1.2.176 -> 1.2.182 gnome-desktop-testing: upgrade 2018.1 -> 2021.1 python3-importlib-metadata: upgrade 4.6.0 -> 4.6.1 u-boot: upgrade 2021.04 -> 2021.07 python3-setuptools: upgrade 57.0.0 -> 57.1.0 btrfs-tools: upgrade 5.12.1 -> 5.13 zangrc (1): python3-pip: upgrade 21.1.2 -> 21.1.3 zhengruoqin (3): libffi: upgrade 3.3 -> 3.4.2 python3: upgrade 3.9.5 -> 3.9.6 python3-pathlib2: upgrade 2.3.5 -> 2.3.6 Signed-off-by: Andrew Geissler Change-Id: If493204235c3a1ce4e7fe65438168512d17a900e --- poky/bitbake/lib/bb/compress/_pipecompress.py | 194 ++++++++++++++++++++++ poky/bitbake/lib/bb/compress/lz4.py | 17 ++ poky/bitbake/lib/bb/compress/zstd.py | 28 ++++ poky/bitbake/lib/bb/data_smart.py | 12 ++ poky/bitbake/lib/bb/fetch2/__init__.py | 5 +- poky/bitbake/lib/bb/fetch2/git.py | 2 +- poky/bitbake/lib/bb/parse/ast.py | 2 + poky/bitbake/lib/bb/parse/parse_py/BBHandler.py | 2 +- poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py | 2 +- poky/bitbake/lib/bb/providers.py | 6 +- poky/bitbake/lib/bb/server/process.py | 2 +- poky/bitbake/lib/bb/tests/compression.py | 98 +++++++++++ 12 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 poky/bitbake/lib/bb/compress/_pipecompress.py create mode 100644 poky/bitbake/lib/bb/compress/lz4.py create mode 100644 poky/bitbake/lib/bb/compress/zstd.py create mode 100644 poky/bitbake/lib/bb/tests/compression.py (limited to 'poky/bitbake/lib/bb') diff --git a/poky/bitbake/lib/bb/compress/_pipecompress.py b/poky/bitbake/lib/bb/compress/_pipecompress.py new file mode 100644 index 0000000000..4b9f662143 --- /dev/null +++ b/poky/bitbake/lib/bb/compress/_pipecompress.py @@ -0,0 +1,194 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Helper library to implement streaming compression and decompression using an +# external process +# +# This library should be used directly by end users; a wrapper library for the +# specific compression tool should be created + +import builtins +import io +import os +import subprocess + + +def open_wrap( + cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs +): + """ + Open a compressed file in binary or text mode. + + Users should not call this directly. A specific compression library can use + this helper to provide it's own "open" command + + The filename argument can be an actual filename (a str or bytes object), or + an existing file object to read from or write to. + + The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for + binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is + "rb". + + For binary mode, this function is equivalent to the cls constructor: + cls(filename, mode). In this case, the encoding, errors and newline + arguments must not be provided. + + For text mode, a cls object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error handling + behavior, and line ending(s). + """ + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + + file_mode = mode.replace("t", "") + if isinstance(filename, (str, bytes, os.PathLike)): + binary_file = cls(filename, file_mode, **kwargs) + elif hasattr(filename, "read") or hasattr(filename, "write"): + binary_file = cls(None, file_mode, fileobj=filename, **kwargs) + else: + raise TypeError("filename must be a str or bytes object, or a file") + + if "t" in mode: + return io.TextIOWrapper( + binary_file, encoding, errors, newline, write_through=True + ) + else: + return binary_file + + +class CompressionError(OSError): + pass + + +class PipeFile(io.RawIOBase): + """ + Class that implements generically piping to/from a compression program + + Derived classes should add the function get_compress() and get_decompress() + that return the required commands. Input will be piped into stdin and the + (de)compressed output should be written to stdout, e.g.: + + class FooFile(PipeCompressionFile): + def get_decompress(self): + return ["fooc", "--decompress", "--stdout"] + + def get_compress(self): + return ["fooc", "--compress", "--stdout"] + + """ + + READ = 0 + WRITE = 1 + + def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None): + if "t" in mode or "U" in mode: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if not "b" in mode: + mode += "b" + + if mode.startswith("r"): + self.mode = self.READ + elif mode.startswith("w"): + self.mode = self.WRITE + else: + raise ValueError("Invalid mode %r" % mode) + + if fileobj is not None: + self.fileobj = fileobj + else: + self.fileobj = builtins.open(filename, mode or "rb") + + if self.mode == self.READ: + self.p = subprocess.Popen( + self.get_decompress(), + stdin=self.fileobj, + stdout=subprocess.PIPE, + stderr=stderr, + close_fds=True, + ) + self.pipe = self.p.stdout + else: + self.p = subprocess.Popen( + self.get_compress(), + stdin=subprocess.PIPE, + stdout=self.fileobj, + stderr=stderr, + close_fds=True, + ) + self.pipe = self.p.stdin + + self.__closed = False + + def _check_process(self): + if self.p is None: + return + + returncode = self.p.wait() + if returncode: + raise CompressionError("Process died with %d" % returncode) + self.p = None + + def close(self): + if self.closed: + return + + self.pipe.close() + if self.p is not None: + self._check_process() + self.fileobj.close() + + self.__closed = True + + @property + def closed(self): + return self.__closed + + def fileno(self): + return self.pipe.fileno() + + def flush(self): + self.pipe.flush() + + def isatty(self): + return self.pipe.isatty() + + def readable(self): + return self.mode == self.READ + + def writable(self): + return self.mode == self.WRITE + + def readinto(self, b): + if self.mode != self.READ: + import errno + + raise OSError( + errno.EBADF, "read() on write-only %s object" % self.__class__.__name__ + ) + size = self.pipe.readinto(b) + if size == 0: + self._check_process() + return size + + def write(self, data): + if self.mode != self.WRITE: + import errno + + raise OSError( + errno.EBADF, "write() on read-only %s object" % self.__class__.__name__ + ) + data = self.pipe.write(data) + + if not data: + self._check_process() + + return data diff --git a/poky/bitbake/lib/bb/compress/lz4.py b/poky/bitbake/lib/bb/compress/lz4.py new file mode 100644 index 0000000000..0f6bc51a5b --- /dev/null +++ b/poky/bitbake/lib/bb/compress/lz4.py @@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb.compress._pipecompress + + +def open(*args, **kwargs): + return bb.compress._pipecompress.open_wrap(LZ4File, *args, **kwargs) + + +class LZ4File(bb.compress._pipecompress.PipeFile): + def get_compress(self): + return ["lz4c", "-z", "-c"] + + def get_decompress(self): + return ["lz4c", "-d", "-c"] diff --git a/poky/bitbake/lib/bb/compress/zstd.py b/poky/bitbake/lib/bb/compress/zstd.py new file mode 100644 index 0000000000..50c42133fb --- /dev/null +++ b/poky/bitbake/lib/bb/compress/zstd.py @@ -0,0 +1,28 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb.compress._pipecompress +import shutil + + +def open(*args, **kwargs): + return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs) + + +class ZstdFile(bb.compress._pipecompress.PipeFile): + def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs): + self.num_threads = num_threads + self.compresslevel = compresslevel + super().__init__(*args, **kwargs) + + def _get_zstd(self): + if self.num_threads == 1 or not shutil.which("pzstd"): + return ["zstd"] + return ["pzstd", "-p", "%d" % self.num_threads] + + def get_compress(self): + return self._get_zstd() + ["-c", "-%d" % self.compresslevel] + + def get_decompress(self): + return self._get_zstd() + ["-d", "-c"] diff --git a/poky/bitbake/lib/bb/data_smart.py b/poky/bitbake/lib/bb/data_smart.py index 2328c334ac..f48726a348 100644 --- a/poky/bitbake/lib/bb/data_smart.py +++ b/poky/bitbake/lib/bb/data_smart.py @@ -481,6 +481,7 @@ class DataSmart(MutableMapping): def setVar(self, var, value, **loginfo): #print("var=" + str(var) + " val=" + str(value)) + var = var.replace(":", "_") self.expand_cache = {} parsing=False if 'parsing' in loginfo: @@ -589,6 +590,8 @@ class DataSmart(MutableMapping): """ Rename the variable key to newkey """ + key = key.replace(":", "_") + newkey = newkey.replace(":", "_") if key == newkey: bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) return @@ -637,6 +640,7 @@ class DataSmart(MutableMapping): self.setVar(var + "_prepend", value, ignore=True, parsing=True) def delVar(self, var, **loginfo): + var = var.replace(":", "_") self.expand_cache = {} loginfo['detail'] = "" @@ -664,6 +668,7 @@ class DataSmart(MutableMapping): override = None def setVarFlag(self, var, flag, value, **loginfo): + var = var.replace(":", "_") self.expand_cache = {} if 'op' not in loginfo: @@ -687,6 +692,7 @@ class DataSmart(MutableMapping): self.dict["__exportlist"]["_content"].add(var) def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): + var = var.replace(":", "_") if flag == "_content": cachename = var else: @@ -814,6 +820,7 @@ class DataSmart(MutableMapping): return value def delVarFlag(self, var, flag, **loginfo): + var = var.replace(":", "_") self.expand_cache = {} local_var, _ = self._findVar(var) @@ -831,6 +838,7 @@ class DataSmart(MutableMapping): del self.dict[var][flag] def appendVarFlag(self, var, flag, value, **loginfo): + var = var.replace(":", "_") loginfo['op'] = 'append' loginfo['flag'] = flag self.varhistory.record(**loginfo) @@ -838,6 +846,7 @@ class DataSmart(MutableMapping): self.setVarFlag(var, flag, newvalue, ignore=True) def prependVarFlag(self, var, flag, value, **loginfo): + var = var.replace(":", "_") loginfo['op'] = 'prepend' loginfo['flag'] = flag self.varhistory.record(**loginfo) @@ -845,6 +854,7 @@ class DataSmart(MutableMapping): self.setVarFlag(var, flag, newvalue, ignore=True) def setVarFlags(self, var, flags, **loginfo): + var = var.replace(":", "_") self.expand_cache = {} infer_caller_details(loginfo) if not var in self.dict: @@ -859,6 +869,7 @@ class DataSmart(MutableMapping): self.dict[var][i] = flags[i] def getVarFlags(self, var, expand = False, internalflags=False): + var = var.replace(":", "_") local_var, _ = self._findVar(var) flags = {} @@ -875,6 +886,7 @@ class DataSmart(MutableMapping): def delVarFlags(self, var, **loginfo): + var = var.replace(":", "_") self.expand_cache = {} if not var in self.dict: self._makeShadowCopy(var) diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index c8e91262a9..0d49e1da37 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -834,7 +834,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 'SOCKS5_USER', 'SOCKS5_PASSWD', 'DBUS_SESSION_BUS_ADDRESS', - 'P4CONFIG'] + 'P4CONFIG', + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_DEFAULT_REGION'] if not cleanup: cleanup = [] diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 5e65c83c63..488f4c7414 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -231,7 +231,7 @@ class Git(FetchMethod): for name in ud.names: ud.unresolvedrev[name] = 'HEAD' - ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" + ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false" write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" ud.write_tarballs = write_tarballs != "0" or ud.rebaseable diff --git a/poky/bitbake/lib/bb/parse/ast.py b/poky/bitbake/lib/bb/parse/ast.py index 50a88f7da7..db2bdc35ec 100644 --- a/poky/bitbake/lib/bb/parse/ast.py +++ b/poky/bitbake/lib/bb/parse/ast.py @@ -97,6 +97,7 @@ class DataNode(AstNode): def eval(self, data): groupd = self.groupd key = groupd["var"] + key = key.replace(":", "_") loginfo = { 'variable': key, 'file': self.filename, @@ -207,6 +208,7 @@ class ExportFuncsNode(AstNode): def eval(self, data): for func in self.n: + func = func.replace(":", "_") calledfunc = self.classname + "_" + func if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): diff --git a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py index f8988b8631..152ef6ab72 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -22,7 +22,7 @@ from .ConfHandler import include, init # For compatibility bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"]) -__func_start_regexp__ = re.compile(r"(((?Ppython(?=(\s|\()))|(?Pfakeroot(?=\s)))\s*)*(?P[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) +__func_start_regexp__ = re.compile(r"(((?Ppython(?=(\s|\()))|(?Pfakeroot(?=\s)))\s*)*(?P[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) __addtask_regexp__ = re.compile(r"addtask\s+(?P\w+)\s*((before\s*(?P((.*(?=after))|(.*))))|(after\s*(?P((.*(?=before))|(.*)))))*") diff --git a/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py index f171c5c932..0834fe3f9b 100644 --- a/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/poky/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -20,7 +20,7 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle __config_regexp__ = re.compile( r""" ^ (?Pexport\s+)? - (?P[a-zA-Z0-9\-_+.${}/~]+?) + (?P[a-zA-Z0-9\-_+.${}/~:]+?) (\[(?P[a-zA-Z0-9\-_+.]+)\])? \s* ( diff --git a/poky/bitbake/lib/bb/providers.py b/poky/bitbake/lib/bb/providers.py index 3ec11a40e1..516d45e4aa 100644 --- a/poky/bitbake/lib/bb/providers.py +++ b/poky/bitbake/lib/bb/providers.py @@ -133,7 +133,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): if required_v is not None: if preferred_v is not None: - logger.warn("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v) + logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v) else: logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr) # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION @@ -173,7 +173,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): pv_str = '%s:%s' % (preferred_e, pv_str) if preferred_file is None: if not required: - logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr) + logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr) available_vers = [] for file_set in pkg_pn: for f in file_set: @@ -185,7 +185,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): available_vers.append(ver_str) if available_vers: available_vers.sort() - logger.warn("versions of %s available: %s", pn, ' '.join(available_vers)) + logger.warning("versions of %s available: %s", pn, ' '.join(available_vers)) if required: logger.error("required version %s of %s not available%s", pv_str, pn, itemstr) else: diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py index 155e8d131f..a0955722e3 100644 --- a/poky/bitbake/lib/bb/server/process.py +++ b/poky/bitbake/lib/bb/server/process.py @@ -147,7 +147,7 @@ class ProcessServer(): conn = newconnections.pop(-1) fds.append(conn) self.controllersock = conn - elif self.timeout is None and not ready: + elif not self.timeout and not ready: serverlog("No timeout, exiting.") self.quit = True diff --git a/poky/bitbake/lib/bb/tests/compression.py b/poky/bitbake/lib/bb/tests/compression.py new file mode 100644 index 0000000000..d3ddf67f1c --- /dev/null +++ b/poky/bitbake/lib/bb/tests/compression.py @@ -0,0 +1,98 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from pathlib import Path +import bb.compress.lz4 +import bb.compress.zstd +import contextlib +import os +import shutil +import tempfile +import unittest +import subprocess + + +class CompressionTests(object): + def setUp(self): + self._t = tempfile.TemporaryDirectory() + self.tmpdir = Path(self._t.name) + self.addCleanup(self._t.cleanup) + + def _file_helper(self, mode_suffix, data): + tmp_file = self.tmpdir / "compressed" + + with self.do_open(tmp_file, mode="w" + mode_suffix) as f: + f.write(data) + + with self.do_open(tmp_file, mode="r" + mode_suffix) as f: + read_data = f.read() + + self.assertEqual(read_data, data) + + def test_text_file(self): + self._file_helper("t", "Hello") + + def test_binary_file(self): + self._file_helper("b", "Hello".encode("utf-8")) + + def _pipe_helper(self, mode_suffix, data): + rfd, wfd = os.pipe() + with open(rfd, "rb") as r, open(wfd, "wb") as w: + with self.do_open(r, mode="r" + mode_suffix) as decompress: + with self.do_open(w, mode="w" + mode_suffix) as compress: + compress.write(data) + read_data = decompress.read() + + self.assertEqual(read_data, data) + + def test_text_pipe(self): + self._pipe_helper("t", "Hello") + + def test_binary_pipe(self): + self._pipe_helper("b", "Hello".encode("utf-8")) + + def test_bad_decompress(self): + tmp_file = self.tmpdir / "compressed" + with tmp_file.open("wb") as f: + f.write(b"\x00") + + with self.assertRaises(OSError): + with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f: + data = f.read() + + +class LZ4Tests(CompressionTests, unittest.TestCase): + def setUp(self): + if shutil.which("lz4c") is None: + self.skipTest("'lz4c' not found") + super().setUp() + + @contextlib.contextmanager + def do_open(self, *args, **kwargs): + with bb.compress.lz4.open(*args, **kwargs) as f: + yield f + + +class ZStdTests(CompressionTests, unittest.TestCase): + def setUp(self): + if shutil.which("zstd") is None: + self.skipTest("'zstd' not found") + super().setUp() + + @contextlib.contextmanager + def do_open(self, *args, **kwargs): + with bb.compress.zstd.open(*args, **kwargs) as f: + yield f + + +class PZStdTests(CompressionTests, unittest.TestCase): + def setUp(self): + if shutil.which("pzstd") is None: + self.skipTest("'pzstd' not found") + super().setUp() + + @contextlib.contextmanager + def do_open(self, *args, **kwargs): + with bb.compress.zstd.open(*args, num_threads=2, **kwargs) as f: + yield f -- cgit v1.2.3