diff options
| author | Andrew Geissler <geissonator@yahoo.com> | 2021-10-29 23:35:00 +0300 |
|---|---|---|
| committer | Andrew Geissler <geissonator@yahoo.com> | 2022-01-24 20:02:44 +0300 |
| commit | eff27476badc5d48b544a07f9f4748a96506c8d7 (patch) | |
| tree | bafd5e712e1ad4e4dc78056f73dbeb6a9ed6a618 /poky/bitbake/lib | |
| parent | 2a947d6075e017a8b50989a0498455752acb0e5d (diff) | |
| download | openbmc-eff27476badc5d48b544a07f9f4748a96506c8d7.tar.xz | |
subtree updates
poky: 80f2b56ad8..d78650b980:
Ahmed Hossam (1):
go.bbclass: Allow adding parameters to go ldflags
Alejandro Hernandez Samaniego (1):
baremetal-image: Fix do_image dependencies
Alexander Kanavin (153):
meson: update 0.58.1 -> 0.59.1
libcap: update 2.51 -> 2.54
lua: add a recipe from meta-oe
lua: update 5.3.6 -> 5.4.3
rpm: update 4.16.1.3 -> 4.17.0
libdnf: fix the rpm sqlite-only target setup
libsolv: disable rpm bdb format support
perl: do not build berkeley db module by default
package_rpm: use zstd instead of xz
qemu: update 6.0.0 -> 6.1.0
runqemu: correct vga-virtio option to keep virgl enabled
gnupg: update 2.3.1 -> 2.3.2
pinentry: update 1.1.1 -> 1.2.0
spirv-tools: update 2021.2 -> 2021.3
glslang: update 11.5.0 -> 11.6.0
shaderc: update 2021.1 -> 2021.2
inetutils: update 2.1 -> 2.2
systemd: update 249.3 -> 249.4
lsof: update 4.91 -> 4.94.0
libpam: update 1.5.1 -> 1.5.2
rt-tests: update 2.1 -> 2.2
libgit2: update 1.1.1 -> 1.2.0
libssh2: update 1.9.0 -> 1.10.0
libhandy: update 1.2.3 -> 1.4.0
qemu: install qmp module without hardcoding the python version in oeqa scripts
lttng-tools: replace ad hoc ptest fixup with upstream fixes
rust: drop PV from include file names
rust: update 1.54.0 -> 1.55.0
librsvg: update 2.40.21 -> 2.52.0 (transition to rust!)
librsvg: do not enable nativesdk
librsvg: add backports to fix big endian targets (e.g. mips)
librsvg: use only the target architecture to determine availability of atomic primitives
librsvg: restore reproducibility
adwaita-icon-theme: update 3.34/38 -> 41.0
gstreamer1.0-plugins-bad: disable rsvg on x32
rust/cargo: exclude UNINATIVE_LOADER from task signature
rust-common.bbclass: rewrite toolchain wrappers in (native) python
rust: do not write ar into target json definitions
rust: generate target definitions from (arch, abi), not just arch
openssl: update 1.1.1l -> 3.0.0
cryptodev-tests: do not use -Werror with openssl 3
serf: add a openssl 3 compatibility fix
ruby: disable openssl extension
glib-2.0: update 2.68.4 -> 2.70.0
glib-networking: update 2.68.2 -> 2.70.0
bison: update 3.7.6 -> 3.8.1
libdnf: update 0.63.1 -> 0.64.0
libexif: update 0.6.22 -> 0.6.23
sudo: update 1.9.7p2 -> 1.9.8p1
wget: update 1.21.1 -> 1.21.2
coreutils: update 8.32 -> 9.0
itstool: update 2.0.6 -> 2.0.7
nghttp2: add recipe from meta-oe
libsoup: add a 3.x recipe
webkitgtk: trim down DEPENDS
epiphany: trim down DEPENDS
webkitgtk: update 2.32.3 -> 2.34.0
epiphany: update 40.3 -> 41.0
python3: update 3.9.7 -> 3.10.0
libjitterentropy: update 3.1.0 -> 3.3.0
kea: update 1.8.2 -> 2.0.0
ghostscript: update 9.54.0 -> 9.55.0
lighttpd: update 1.4.59 -> 1.4.60
bluez5: update 5.61 -> 5.62
ovmf: update 202105 -> 202108
systemd: update 249.4 -> 249.5
meson: update 0.59.1 -> 0.59.2
python3-pip: update 21.2.4 -> 21.3
valgrind: update 3.17.0 -> 3.18.1
librsvg: update 2.52.0 -> 2.52.2
libva: update 2.12.0 -> 2.13.0
liberation-fonts: update 2.1.4 -> 2.1.5
ca-certificates: update 20210119 -> 20211016
curl: update 7.78.0 -> 7.79.1
libgit2: update 1.2.0 -> 1.3.0
libxcrypt: upgrade 4.4.25 -> 4.4.26
bison: upgrade 3.8.1 -> 3.8.2
cmake: update 3.21.2 -> 3.21.3
git: upgrade 2.33.0 -> 2.33.1
tzdata: upgrade 2021a -> 2021d
ofono: upgrade 1.32 -> 1.33
openssh: upgrade 8.7p1 -> 8.8p1
sysvinit: upgrade 2.99 -> 3.00
btrfs-tools: upgrade 5.13.1 -> 5.14.2
ccache: upgrade 4.4 -> 4.4.2
createrepo-c: upgrade 0.17.5 -> 0.17.6
libcomps: upgrade 0.1.17 -> 0.1.18
libedit: upgrade 20210714-3.1 -> 20210910-3.1
librepo: upgrade 1.14.1 -> 1.14.2
python3-jinja2: upgrade 3.0.1 -> 3.0.2
python3-pygobject: upgrade 3.40.1 -> 3.42.0
python3-setuptools: upgrade 58.0.4 -> 58.2.0
vala: upgrade 0.52.5 -> 0.54.2
acpica: upgrade 20210730 -> 20210930
asciidoc: upgrade 9.1.0 -> 9.1.1
libarchive: upgrade 3.5.1 -> 3.5.2
msmtp: upgrade 1.8.15 -> 1.8.17
sudo: upgrade 1.9.8p1 -> 1.9.8p2
gobject-introspection: upgrade 1.68.0 -> 1.70.0
gsettings-desktop-schemas: upgrade 40.0 -> 41.0
json-glib: upgrade 1.6.4 -> 1.6.6
libdazzle: upgrade 3.40.0 -> 3.42.0
harfbuzz: upgrade 2.9.1 -> 3.0.0
pango: upgrade 1.48.9 -> 1.48.10
libinput: upgrade 1.19.0 -> 1.19.1
linux-firmware: upgrade 20210818 -> 20210919
wireless-regdb: upgrade 2021.07.14 -> 2021.08.28
mpg123: upgrade 1.28.2 -> 1.29.0
puzzles: upgrade to latest revision
libwpe: upgrade 1.10.1 -> 1.12.0
diffoscope: upgrade 182 -> 187
fribidi: upgrade 1.0.10 -> 1.0.11
iso-codes: upgrade 4.6.0 -> 4.7.0
libatomic-ops: upgrade 7.6.10 -> 7.6.12
libcap: upgrade 2.54 -> 2.59
libmd: upgrade 1.0.3 -> 1.0.4
libsoup-2.4: upgrade 2.72.0 -> 2.74.0
gnupg: update 2.3.2 -> 2.3.3
libssh2: skip one of the ptests until openssh 8.8 compatibility is fixed
systemd: disable seccomp on mips32
waffle: convert to git, website is down
u-boot: upgrade 2021.07 -> 2021.10
psplash: upgrade to latest revision
stress-ng: convert to git, website is down
json-c: correct upstream version check
createrepo-c: upgrade 0.17.6 -> 0.17.7
python3-numpy: upgrade 1.21.2 -> 1.21.3
python3-pip: upgrade 21.3 -> 21.3.1
python3-setuptools: upgrade 58.2.0 -> 58.3.0
msmtp: upgrade 1.8.17 -> 1.8.18
gi-docgen: upgrade 2021.7 -> 2021.8
piglit: upgrade to latest revision
libinput: upgrade 1.19.1 -> 1.19.2
mpg123: upgrade 1.29.0 -> 1.29.2
puzzles: upgrade to latest revision
webkitgtk: upgrade 2.34.0 -> 2.34.1
wpebackend-fdo: upgrade 1.10.0 -> 1.12.0
diffoscope: upgrade 187 -> 188
libcap: upgrade 2.59 -> 2.60
vte: upgrade 0.64.2 -> 0.66.0
ncurses: update 6.2 -> 6.3
tzdata: update 2021d -> 2021e
asciidoc: update 9.1.1 -> 10.0.0
waffle: update 1.6.1 -> 1.7.0
virgl: skip headless test on specific older distros and fail otherwise
gptfdisk: address ncurses 6.3 compatibility
powertop: address ncurses 6.3 compatibility
systemtap: address ncurses 6.3 compatibility
igt-gpu-tools: address meson 0.60 compatibility
python3-pygobject: do not supply unknown g-i options
gstreamer1.0-python: do not supply unknown g-i options
webkitgtk: drop unnecessary SSE disabling patch
cups: exclude beta/rc versions from version check
Alexandre Belloni (5):
inetutils: drop CVE-2021-40491 patch
oeqa/selftest/sstatetests: fix typo ware -> were
maintainers.inc: fix up rust-cross entry
libevent,btrfs-tools: fix Upstream-Status tag
ruby: fix Upstream-Status
Andreas Müller (1):
webkitgtk: add gperf-native to DEPENDS to fix build
Andrei Gherzan (1):
qemu: Define libnfs PACKAGECONFIG
Andrej Valek (3):
busybox: 1.34.0 -> 1.34.1
featimage: refactor style
kernel-fitimage: use correct kernel image
Andres Beltran (1):
buildhistory: Fix package output files for SDKs
Bruce Ashfield (15):
linux-yocto/5.14: scripts/gcc-plugins: consistently use HOSTCC
linux-yocto/5.14: update to v5.14.8
linux-yocto/5.14: bsp/qemuarm*-gfx: use virtio graphics
linux-yocto/5.10: update to v5.10.69
linux-yocto/5.10: update to v5.10.70
linux-yocto/5.14: update to v5.14.9
kernel-yocto: don't apply config metadata patches twice
linux-yocto/5.14: revert: scripts/gcc-plugins: consistently use HOSTCC
linux-yocto/5.10: update to v5.10.73
linux-yocto/5.14: update to v5.14.12
linux-yocto/5.14: update to v5.14.13
linux-yocto/5.10: update to v5.10.74
linux-yocto/5.14: common-pc: enable CONFIG_ATA_PIIX as built-in
linux-yocto/5.14: update to v5.14.14
linux-yocto/5.10: update to v5.10.75
Caner Altinbasak (2):
bitbake: npmsw: Avoid race condition with multiple npm fetchers
bitbake: fetch2: Do not attempt to verify donestamp if download is not complete
Changhyeok Bae (2):
iproute2: update to 5.14.0
ethtool: update to 5.14
Changqing Li (1):
lttng-ust: fix do_compile failure for arm32 with DEBUG_BUILD enabled
Daiane Angolini (2):
machine/qemuarm*: Fix UBOOT_MACHINE value
ref-manual: Update how to set a useradd password
Daniel McGregor (3):
bison: prevent checking for textstyle.
bitbake.conf: Add gpg-agent as a host tool
sstate: Allow validation of sstate singatures against list of keys
David Joyner (1):
rust: add support for big endian 64-bit PowerPC
Denys Dmytriyenko (2):
wayland-protocols: upgrade 1.21 -> 1.22
wayland-protocols: upgrade 1.22 -> 1.23
Fred Liu (1):
glibc: Drop libcidn package
Henry Kleynhans (1):
sstate: Switch to ZStandard compressor support
Hsia-Jun(Randy) Li (1):
meson: install native file in sdk
Ian Ray (1):
archiver: Configurable tarball compression
Jon Mason (10):
oeqa/manual: Fix no longer valid URLs
bitbake: bitbake:toaster:test: Update SSTATE URL
yocto-bsp/poky: replace http with https for URLs
bitbake: bitbake: replace http with https for URLs
documentation: update sources mirror URL
documentation: replace http with https for URLs
documentation: use YOCTO_DL_URL
dev-manual: remove errant /
runqemu: add DEPLOY_DIR_IMAGE replacement in QB_OPT_APPEND
bitbake: bitbake-user-manual: update sources mirror URL
Jose Quaresma (15):
gstreamer1.0: 1.18.4 -> 1.18.5
gstreamer1.0-plugins-base: 1.18.4 -> 1.18.5
gstreamer1.0-plugins-good: 1.18.4 -> 1.18.5
gstreamer1.0-plugins-bad: 1.18.4 -> 1.18.5
gstreamer1.0-plugins-ugly: 1.18.4 -> 1.18.5
gstreamer1.0-rtsp-server: 1.18.4 -> 1.18.5
gstreamer1.0-libav: 1.18.4 -> 1.18.5
gstreamer1.0-vaapi: 1.18.4 -> 1.18.5
gstreamer1.0-omx: 1.18.4 -> 1.18.5
gstreamer1.0-python: 1.18.4 -> 1.18.5
gst-devtools: 1.18.4 -> 1.18.5
gst-examples: 1.18.4 -> 1.18.5
patch.bbclass: when the patch fails show more info on the fatal error
sstate.bbclass: count the files on mirrors using the pre local files
sstate: fix touching files inside pseudo
Joshua Watt (4):
bitbake: async: Close sync client event loop
bitbake: hashserv: Add tests for diverging reports
bitbake: hashserv: Fix diverging report race condition
classes/populate_sdk_base: Add setscene tasks
Kai Kang (1):
sudo: update multilib patch for sudo.conf
Khem Raj (11):
pkgconfig: Update to latest
libseccomp: Upgrade to 2.5.2 and beyond
openssl: Drop riscv32 upstreamed patches
opensbi-payloads.inc: Use strings for fallback
mesa: Enable svga for x86 only
qemu: Add knob for enabling PMDK pmem support
opensbi-payloads: Add dependency on kernel if fdt is set
librsvg: Fix vendored libc to work on riscv and musl
librsvg: Bump to 2.52.0 -> 2.52.2
rust: Upgrade to 1.56.0
librsvg: Disable 64bit atomics in crossbeam for rv32
Kiran Surendran (1):
ffmpeg: fix CVE-2021-38114
Konrad Weihmann (1):
libical: fix append in DEPENDS
Lukasz Majewski (1):
glibc: Extract common code to build tests to glibc-tests.inc
Marek Vasut (1):
piglit: upgrade to latest revision
Martin Jansa (2):
webkitgtk: add PACKAGECONFIG for opengl-or-es
boost: allow searching for python310
Maximilian Blenk (1):
mount-copybind: add rootcontext mountoption for overlayfs
Michael Halstead (2):
scripts/autobuilder-worker-prereq-tests: jinja2 check
releases: update to include 3.4 honister
Michael Opdenacker (18):
manuals: font fixes
ref-manual: document DEBUG_PREFIX_MAP
manuals: replace "apt-get" by "apt"
Makefile: allow epub and latexpdf outputs to use diagrams from SVG sources
conf.py: use PNG first in EPUB output
overview-manual: SVG diagram for the git workflow
docs: add "make all" to build old types of docs (html, epub, pdf) at once
manuals: introduce 'yocto_sstate' extlink
overview-manual: simplify expression
dev-manual: correct NO_GENERIC_LICENSE section title
dev-manual: warn about license compliance issues when static libraries are used
devpyshell: rename to pydevshell
ref-manual: document TOOLCHAIN_HOST_TASK_ESDK
manuals: rename "devpyshell" to "pydevshell"
ref-manual: document SOURCE_DATE_EPOCH
ref-manual: fix description of SOURCE_DATE_EPOCH
releases.rst: fix release number for 3.3.3
docs: poky.yaml: updates for 3.4
Mike Crowe (2):
lib/oe/qa,insane: Move extra error handling functions to library
insane,license,license_image: Allow treating license problems as errors
Mingli Yu (1):
packagedata.py: silence a DeprecationWarning
Oleh Matiusha (1):
findutils: add ptest
Oleksandr Kravchuk (15):
python3: update to 3.9.7
python3-git: update to 3.1.24
python3-dbusmock: update to 0.24.0
python3-setuptools: update to 58.0.4
python3-setuptools: minor cleanup
xf86-input-libinput: update to 1.2.0
libinput: update to 1.19.0
libxi: update to 1.8
xorgproto: update to 2021.5
xkeyboard-config: update to 2.34
libxkbcommon: update to 1.3.1
mirrors.bbclass: remove dead infozip mirrors
man-pages: update to 5.13
python3-smmap: update to 5.0.0
python3-pyparsing: update to 3.0.0
Pablo Saavedra Rodi?o (1):
mesa: upgrade 21.2.1 -> 21.2.4
Paul Barker (1):
bitbake: doc: bitbake-user-manual: Document BB_GIT_SHALLOW and friends
Paul Eggleton (7):
migration-3.4: tweak overrides change section
ref-manual: remove meta class
poky.yaml: add lz4 and zstd to essential host packages
migration-3.4: add additional migration info
migration: tweak introduction section
poky.yaml: fix lz4 package name for older Ubuntu versions
migration-3.4: add some extra packaging notes
Pavel Zhukov (1):
weston: wrapper for weston modules argument
Peter Hoyes (2):
u-boot: Convert ${UBOOT_ENV}.cmd into ${UBOOT_ENV}.scr
u-boot: Fix syntax error in ${UBOOT_ENV}.scr compilation
Peter Kjellerstedt (2):
meson.bblcass: Remove empty egg-info directories before running meson
qemu.inc: Remove empty egg-info directories before running meson
Petr Vorel (1):
ltp: Update to 20210927
Pgowda (2):
Fix rust-native build issue when debug is enabled
rust-cross: Fix directory not deleted for race glibc vs. musl
Ralph Siemsen (1):
tar: filter CVEs using vendor name
Randy Li (1):
meson: move lang args to the right section
Richard Purdie (54):
sstatesig: Only apply group/other permissions to pseudo files
rpm: Deterministically set vendor macro entry
abi_version/sstate: Bump to fix rpm corruption issues
multilib: Avoid sysroot race issues when multilib enabled
bitbake: knotty/uihelper: Show setscene task progress in summary output
bitbake: bitbake-worker: Handle pseudo shutdown in Ctrl+C case
poky.conf: Update tested distros list with recent changes
bitbake: hashserv: Improve behaviour for better determinism/sstate reuse
poky.conf: Bump version for 3.4 honister release
build-appliance-image: Update to honister head revision
bitbake: bitbake: Bump to version 1.52.0
build-appliance-image: Update to honister head revision
layer.conf: Extend recipes not to install without explict dependencies
libxml2: Use python3targetconfig to fix reproducibility issue
libnewt: Use python3targetconfig to fix reproducibility issue
bootchart2: Don't compile python modules
linux-yocto-dev: Ensure DEPENDS matches recent 5.14 kernel changes
base/insane: Import oe.qa by default
base: Clean up unneeded len() calls
base: Use repr() for printing exceptions
bitbake.conf: Add BB_CURRENTTASK to BB_HASHEXCLUDE
reproducible_build: Drop obsolete sstate workaround
python: Update now reproducibile builds are the default
bitbake: bitbake-worker: Set BB_CURRENTTASK earlier
bitbake: compress: Allow to operate on file descriptor
bitbake: siggen: Change file format of siginfo files to use zstd compressed json
sstate: Fixes for eSDK generation after zstd switch
patch: Use repr() with exceptions instead of str()
reproducible_build: Drop now unneeded compiler warning
reproducible: Move class function code into library
reproducible: Move variable definitions to bitbake.conf
reproducible: Merge code into base.bbclass
kernel: Add KERNEL_DEBUG_TIMESTAMPS variable
reproducible: Drop BUILD_REPRODUCIBLE_BINARIES variable
kernel: Rework kernel make flag to variable mappings
oeqa: Update cleanup code to wait for hashserv exit
poky: Reproducible builds are now the default
bitbake: tests/runqueue: Ensure hashserv exits before deleting files
bitbake: bitbake-worker: Add debug when unpickle fails
bitbake: siggen: Fix type conversion issues
bitbake: test/fetch: Update urls to match upstream branch name changes
libtool: Update patchset to match those submitted upstream
staging: Fix autoconf-native rebuild failure
patch: Fix exception reporting with repr()
bitbake: fetch2/perforce: Fix typo
bitbake: bitbake: Bump to post release verion 1.53.0
poky.conf: Post release version bump
gcc: Merge three related patches together
gcc: Drop sdt (dtrace) header patch
gcc: Drop broken/unneeded patch
bitbake: tests/runqueue: Drop python version test decorators
gcc: Update patches submitted/merged upstream
gcc: Drop testsuite failure fix patch
gcc: Add missing patch Upstream-Status
Ross Burton (21):
bitbake: tests/utils: mark a regex as a raw string
bitbake: tests/fetch: prefix the FetcherTest temporary directory
bitbake: fetch2: clarify the command-no-found error message
bitbake: fetch2/gitsm: remove the 'nugget' SRCREV caching
linux-yocto: add libmpc-native to DEPENDS
bitbake: fetch2: document checkstatus API
curl: fix CVE-2021-22945 through -22947
oe/license: implement ast.NodeVisitor.visit_Constant
license.bbclass: implement ast.NodeVisitor.visit_Constant
bitbake: codegen: implement ast.NodeVisitor.visit_Constant
testimage: fix unclosed testdata file
oe/utils: log exceptions in ThreadedWorker functions
sstate: don't silently handle all exceptions in sstate_checkhashes
gawk: replace AR patch with upstreamed patch
gawk: reduce strictness of the time test
strace: remove obsolete patch
strace: remove TOOLCHAIN assignment
oeqa/runtime: load modules using importlib
oeqa/runtime: search sys.path explicitly for modules
testimage: remove target_modules_path
strace: show test suite log on failure
Sakib Sajal (1):
go: upgrade 1.16.7 -> 1.16.8
Saul Wold (3):
spdx.py: Add annotation to relationship
create-spdx: add create_annotation function
create-spdx: cross recipes are native also
Stefan Herbrechtsmeier (17):
bitbake: fetch2: npm: Support npm archives with missing search directory mode
bitbake: fetch2: npm: Create config npmrc in environment instantiation
bitbake: fetch2: npmsw: Add support for local tarball and link sources
npm: Add support for EXTRA_OENPM arguments
recipetool: Move license md5sums into CSV files
recipetool: Skip common source files in guess_license
recipetool: Change default paramter fallback_licenses of function split_pkg_licenses from None to []
recipetool: ignore empty license files
recipetool: Add logger info for missing license entries
recipetool: Add support for linenumbers to licenses.csv
recipetool: npm: Do not add package.json files to LIC_FILES_CHKSUM
recipetool: npm: Use README as license fallback
npm: Add variable NPM_NODEDIR with default value
npm: Use configs for npm environment and args for npm run command
recipetool: Rework crunch_license to recognize more variants
recipetool: Simplify common source files skip in guess_license
npm: Remove unnecessary configs argument from npm run command
Teoh Jay Shen (1):
oeqa/runtime/parselogs: modified drm error in common errors list
Thomas Perrot (1):
image_types: allow the creation of block devices on top of UBI volumes
Tim Orling (17):
ptest-runner: install -collect-system-data script
python3-hypothesis: upgrade 6.15.0 -> 6.23.2
python3-importlib-metadata: upgrade 4.6.4 -> 4.8.1
python3-more-itertools: upgrade 8.9.0 -> 8.10.0
python3-zipp: upgrade 3.5.0 -> 3.6.0
libconvert-asn1-perl: upgrade 0.31 -> 0.33
python3-pytest: upgrade 6.2.4 -> 6.2.5
at-spi2-core: upgrade 2.40.3 -> 2.42.0
python3-packaging: DEPENDS on python3-setuptools-native
python3-packaging: BBCLASSEXTEND nativesdk
python3-tomli: add recipe for 1.2.1
python3-setuptools-scm: upgrade 6.0.1 -> 6.3.2
python3-setuptools: _distutils/sysconfig fix
python3-pyyaml: upgrade 5.4.1 -> 6.0
python3-manifest: -pprint RDEPENDS on -profile
python3-hypothesis: upgrade 6.23.2 -> 6.24.0
python3-tomli: upgrade 1.2.1 -> 1.2.2
Tobias Kaufmann (1):
mount-copybind: add SELinux support
Yureka (1):
systemd: add missing include for musl
Zoltán Böszörményi (1):
libpam: Fix build with DISTRO_FEATURES usrmerge
hongxu (1):
libcap: fix nativesdk-libcap relocate failure
wangmy (20):
diffoscope: upgrade 181 -> 182
cmake: upgrade 3.21.1 -> 3.21.2
gzip: upgrade 1.10 -> 1.11
harfbuzz: upgrade 2.9.0 -> 2.9.1
vulkan-headers: upgrade 1.2.182 -> 1.2.191
vulkan-loader: upgrade 1.2.182 -> 1.2.191
vulkan-tools: upgrade 1.2.182 -> 1.2.191
help2man: upgrade 1.48.4 -> 1.48.5
python3-more-itertools: upgrade 8.8.0 -> 8.9.0
acpid: upgrade 2.0.32 -> 2.0.33
bind: upgrade 9.16.20 -> 9.16.21
createrepo-c: upgrade 0.17.4 -> 0.17.5
dnf: upgrade 4.8.0 -> 4.9.0
ell: upgrade 0.43 -> 0.44
libical: upgrade 3.0.10 -> 3.0.11
dhcpcd: upgrade 9.4.0 -> 9.4.1
dnf: upgrade 4.9.0 -> 4.10.0
file: upgrade 5.40 -> 5.41
libdnf: upgrade 0.64.0 -> 0.65.0
lttng-tools: upgrade 2.13.0 -> 2.13.1
zhengruoqin (2):
libsolv: upgrade 0.7.19 -> 0.7.20
python3-pycryptodome: upgrade 3.10.1 -> 3.11.0
meta-security: a85fbe980e..e81c15f851:
Armin Kuster (1):
python3-fail2ban: fix build failure and cleanup
Kai Kang (1):
sssd: re-package to fix QA issues
Kristian Klausen (1):
swtpm: update to 0.6.1
Liwei Song (1):
recipes-security/chipsec: platform security assessment framework
meta-openembedded: 23dc4f060f..2f6797d8d6:
Alexander Kanavin (23):
python3-yappi: add python 3.10 fix
python3-gmpy2: update 2.0.8 -> 2.1.0rc1
python3-m2crypto: address build failure with openssl 3.x
lua: remove recipe as it is now in oe-core
nghttp2: remove recipe as it is now in oe-core
kronosnet: update 1.20 -> 1.22
polkit: update 0.116 -> 0.119
openflow: disable openssl PACKAGECONFIG
openipmi: add a python 3.10 compatibility patch
strongswan: disable openssl PACKAGECONFIG
pkcs11-helper: backport a openssl 3.x patch
nodejs: use -fpermissive
nodejs: add a python 3.10 compatibility patch
php: disable openssl PACKAGECONFIG
libsigrokdecode: add a python 3.10 compatibility patch
freerdp: backport openssl 3.x patches
opensc: do not use -Werror
cfengine: blacklist due to openssl 3.x incompatibility
netplan: do not use -Werror
boinc-client: blacklist due to openssl 3.x incompatibility
ldns: blacklist due to openssl 3.x incompatibility
surf: add a webkit 2.34 compatibility patch
mozjs: update 60.9.0 -> 91.1.0
Alexander Thoma (1):
keyutils: fix install path
Armin Kuster (1):
Apache: Several CVE fixes
Chen Qi (1):
ntfs-3g-ntfsprogs: upgrade to 2021.8.22
Gianfranco (2):
vboxguestdrivers: upgrade 6.1.26 -> 6.1.28
vboxguestdrivers: Fix build failure due to the last update.
Ivan Maidanski (1):
bdwgc: upgrade 8.0.4 -> 8.0.6
Jeremy Kerr (1):
mctp: Add MCTP control utilities
Joshua Watt (2):
colord-native: Inherit pkgconfig
mutter: Move gsettings to a separate package
Justin Bronder (2):
python3-mccabe: fix LICENSE and RDEPENDS
python3-pyflakes: fix LICENSE, HOMEPAGE and RDEPENDS
Khem Raj (44):
packagegroup-meta-oe: Add lv-drivers and lvgl
python3-pystache: Remove need for python2x
python3-behave: Switch to using github src_uri
python3-simpleeval: Remove use_2to3
python3-parse-type: Do not ask for 2to3
python3-anyjson: Do not enable 2to3
python3-pybluez: Do not set 2to3
packagegroup-meta-networking: Remove nghttp2
packagegroup-meta-oe: Remove lua
gjs: Inherit pkgconfig
luaposix: Use lua 5.4
devilspie2: Switch to using github repo
devilspie2: Add missing dependency on libxinerama
kronosnet: Correct path to poll.h
luaposix: Upgrade to 35.1
xfce.bbclass: Mark string as raw in a regexp
nodejs: add -fpermissive BUILD_CXXFLAGS
pcimem: Add riscv64 into COMPATIBLE_HOST
luaposix: Depend on libxcrypt-native
libcrypt-openssl-rsa-perl: Upgrade to 0.32
xrdp: Upgrade to 0.9.17
yelp,yelp-xsl,yelp-tools: Upgrade to 41 release
mozjs: Disable JIT on mips
libcrypt-openssl-rsa-perl: Fix build with openssl3+
nodejs: Upgrade to 14.18.1
librest: Use master with libsoup3 support
gnome-online-accounts: Upgrade to 3.43.1
iwd: Fix build with musl
xrdp: Exclude from builds
xorgxrdp: Add to exclusion list for now
bluepy: Set PV and correct syntax for RDEPENDS
python3-kivy: Use new override syntax in RDEPENDS
bluepy: Fix permissions on tarball
nodejs: Upgrade to 16.11.1
mozjs: Fix build on mips/musl
mozjs: Fix musl miscompiles with HAVE_THREAD_TLS_KEYWORD
luaposix: Use virtual/crypt instread of hardcoding libxcrypt
packagegroup-meta-oe: Add libjs-jquery-globalize and libjs-jquery-cookie
mozjs: Add riscv32 support
driverctl: Rename to have PV in recipe name
tbb: Link with libatomic when using clang compiler on rv32
gimp: Disable vector icons on mips/rv32 using clang
mozjs: Update to 91.2.0
mozjs: Fix build for 32bit systems disabling JIT
Kurt Kiefer (2):
mozjs: move large static library into staticdev package
gyp: fix for compatibility with Python 3.10
Leon Anavi (6):
python3-aiohue: Upgrade 2.6.1 -> 2.6.3
python3-paho-mqtt: Upgrade 1.5.1 -> 1.6.1
python3-thrift: Upgrade 0.13.0 -> 0.15.0
python3-prompt-toolkit: Upgrade 3.0.19 -> 3.0.21
python3-traitlets: Upgrade 5.1.0 -> 5.1.1
python3-prettytable: Upgrade 2.1.0 -> 2.2.1
LiweiSong (1):
Revert "chipsec: platform security assessment framework"
Marco Cavallini (1):
bluepy_git.bb: Added recipe for BluePy Python interface to Bluetooth LE on Linux (v2)
Marek Vasut (3):
apitrace: Upgrade to latest version
apitrace: Conditionally enable X11 support
yavta: Upgrade to latest version and drop downstream patch
Martin Jansa (4):
protobuf: import a patch from meta-webosose to fix building protobuf-c with gold linker
grpc: revert GRPCPP_ABSEIL_SYNC change from 1.40.0 to fix undefined references to absl::lts_20210324::Mutex::~Mutex()
luaposix: depend on target libxcrypt instead of native
lv-lib-png: add wayland to REQUIRED_DISTRO_FEATURES
Matthias Klein (1):
gpsd: Install also the generated parts of the Python library
Matthias Schiffer (1):
byacc: switch to official HTTPS mirror
Mingli Yu (1):
opencv: remove setup_vars_opencv4.sh
Oleksandr Kravchuk (72):
iwd: update to 1.18
nano: update to 5.9
ostree: update to 2021.5
pegtl: update to 3.2.1
pmdk: update to 1.11.1
protobuf-c: update to 1.4.0
thingsboard-gateway: update to 2.8
libbpf: fix PV
libcdio-paranoia: update to 10.2+2.0.1
ttf-abyssinica: update to 2.100
ttf-dejavu: add UPSTREAM_CHECK_URI
ttf-vlgothic: update to 20200720
networkmanager: update to 1.32.12
nng: update to 1.4.0
opensaf: update to 5.21.09
python3-absl: update 0.14.1
python3-alembic: update to 1.7.4
python3-astroid: update to 2.8.2
python3-cantools: update to 36.4.0
python3-certifi: update to 2021.10.8
python3-cffi: update to 1.15.0
python3-colorlog: update to 6.5.0
python3-coverage: update to 6.0.2
python3-cryptography-vectors: update to 35.0.0
python3-dateparser: update to 1.1.0
python3-elementpath: update to 2.3.2
python3-eventlet: update to 0.32.0
python3-google-api-core: update to 2.1.1
python3-google-api-python-client: update to 2.26.1
python3-google-auth: update to 2.3.0
python3-grpcio-tools: update to 1.41.0
python3-grpcio: update to 1.41.0
python3-h2: update to 4.1.0
python3-haversine: update to 2.5.1
python3-httplib2: update to 0.20.1
python3-idna: update to 3.3
python3-iso3166: update to 2.0.2
python3-joblib: update to 1.1.0
python3-jsonrpcserver: update to 5.0.3
python3-paramiko: update to 2.8.0
python3-portion: update to 2.2.0
python3-protobuf: update to 3.18.1
python3-pulsectl: update to 21.10.4
python3-pycodestyle: update to 2.8.0
python3-pyephem: update to 4.1
python3-pyflakes: update to 2.4.0
python3-pyjwt: update to 2.2.0
python3-pykickstart: update to 3.34
python3-pyopenssl: update to 21.0.0
python3-pyperf: update to 2.3.0
python3-pytest-timeout: update 2.0.1
python3-pytz: update to 2021.3
python3-regex: update to 2021.10.8
python3-sentry-sdk: update to 1.4.3
python3-sympy: udpate to 1.9
python3-twitter: update to 4.1.0
python3-uritemplate: update to 4.1.1
python3-websockets: update to 10.0
python3-wrapt: update to 1.13.2
python3-xlsxwriter: update to 3.0.1
python3-xmlschema: update to 1.8.0
python3-yarl: update to 1.7.0
python3-typeguard: update to 2.13.0
python3-pyatspi: update to 2.38.1
python3-h5py: update to 3.5.0
python3-pybind11-json: update to 0.2.11
python3-pychromecast: update to 9.3.1
python3-tzlocal: update to 4.0.1
python3-zeroconf: update to 0.36.9
leptonica: update to 1.82.0
redis-plus-plus: update to 1.3.2
hiredis: update to 1.0.2
Peter Bergin (3):
pipewire: remove manpages class
pipewire: also handle pipewire-media-session in systemd class
pipewire: rework PACKAGECONFIG for systemd service files
Philippe Coval (5):
lvgl: Add recipe for Lightweight Graphics Library
lv-drivers: Add recipe for lvgl lib
lv-lib-png: Add recipe for LVGL for handling PNG format
lvgl: Add lv_lib_png to packagegroup
lvgl: Lint recipe
Roland Hieber (1):
rapidjson: override hard-coded lib install dir
Shiping Ji (1):
Add recipe for driverctl
Tim Orling (6):
unicode-ucd: upgrade 12.1.0 -> 14.0.0
gnome-screenshot: add recipe for 40.0
libgweather: upgrade 3.36.2 -> 40.0
colord: inherit pkgconfig
gfbgraph: patch to build with latest rest
gnome-calendar: upgrade 3.38.1 -> 41.0
Trevor Gamblin (3):
python3-posix-ipc: Add recipe for version 1.0.5
python3-pyinotify: Add fcntl, logging to RDEPENDS
python3-cvxopt: upgrade 1.2.6 -> 1.2.7
Vyacheslav Yurkov (3):
grpc: fix cross-compilation of grpc applications
grpc: fix cross-compilation of grpc applications
grpc: upgrade 1.38.1 -> 1.41.0
Wang Mingyu (1):
poppler: upgrade 21.09.0 -> 21.10.0
Yi Zhao (3):
samba: upgrade 4.14.7 -> 4.14.8
audit: upgrade 3.0.5 -> 3.0.6
strongswan: upgrade 5.9.3 -> 5.9.4
Zoltán Böszörményi (1):
python3-jsmin: Upgrade to version 3.0.0
wangmy (11):
openvpn: upgrade 2.5.3 -> 2.5.4
redis: upgrade 6.2.5 -> 6.2.6
span-lite: upgrade 0.10.1 -> 0.10.3
bdwgc: upgrade 8.0.6 -> 8.2.0
icewm: upgrade 2.7.0 -> 2.8.0
ifenslave: upgrade 2.12 -> 2.13
asio: upgrade 1.18.1.bb -> 1.20.0
libgusb: upgrade 0.3.7 -> 0.3.8
libxmlb: upgrade 0.3.2 -> 0.3.3
xdebug: upgrade 2.9.5 -> 3.1.1
cryptsetup: upgrade 2.3.6 -> 2.4.1
zangrc (25):
python3-ipython: upgrade 7.27.0 -> 7.28.0
python3-jdatetime: upgrade 3.6.2 -> 3.6.4
python3-multidict: upgrade 5.1.0 -> 5.2.0
python3-openpyxl: upgrade 3.0.8 -> 3.0.9
python3-pyscaffold: upgrade 4.0.2 -> 4.1
python3-transitions: upgrade 0.8.9 -> 0.8.10
networkmanager-openvpn: upgrade 1.8.14 -> 1.8.16
ser2net: upgrade 4.3.3 -> 4.3.4
python3-humanize: upgrade 3.11.0 -> 3.12.0
python3-nmap: upgrade 1.5.0 -> 1.5.1
libjs-jquery-cookie: Add recipe
libjs-jquery-globalize: Add recipe
python3-cachetools: upgrade 4.2.2 -> 4.2.4
python3-cbor2: upgrade 5.4.1 -> 5.4.2
python3-click: upgrade 8.0.1 -> 8.0.3
python3-configargparse : upgrade 1.5.2 -> 1.5.3
python3-flask: upgrade 2.0.1 -> 2.0.2
python3-pyscaffold: upgrade 4.1 -> 4.1.1
python3-werkzeug: upgrade 2.0.1 -> 2.0.2
python3-absl: upgrade 0.14.1 -> 0.15.0
python3-pandas: upgrade 1.3.3 -> 1.3.4
python3-pulsectl: upgrade 21.10.4 -> 21.10.5
python3-pyjwt: upgrade 2.2.0 -> 2.3.0
python3-pytest-asyncio: upgrade 0.15.1 -> 0.16.0
python3-robotframework: upgrade 4.1.1 -> 4.1.2
zhengruoqin (15):
python3-ujson: upgrade 4.1.0 -> 4.2.0
python3-urllib3: upgrade 1.26.6 -> 1.26.7
python3-watchdog: upgrade 2.1.5 -> 2.1.6
gensio: upgrade 2.2.9 -> 2.3.1
nlohmann-json: upgrade 3.10.2 -> 3.10.4
libencode-perl: upgrade 3.12 -> 3.16
python3-socketio: upgrade 5.4.0 -> 5.4.1
python3-sqlalchemy: upgrade 1.4.23 -> 1.4.26
python3-stevedore: upgrade 3.4.0 -> 3.5.0
autofs: upgrade 5.1.7 -> 5.1.8
links: upgrade 2.22 -> 2.25
atftp: upgrade 0.7.4 -> 0.7.5
python3-gmqtt: upgrade 0.6.10 -> 0.6.11
python3-google-api-python-client: upgrade 2.26.1 -> 2.27.0
python3-greenlet: upgrade 1.1.1 -> 1.1.2
meta-raspberrypi: 9eb4879cf4..90b3ac6fb3:
Zygmunt Krynicki (1):
rpi-config: warn on config.txt lines exceeding 80 bytes
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
Change-Id: I9e75d5bd606a913fbe69e6735c9ecafc436441ba
Diffstat (limited to 'poky/bitbake/lib')
30 files changed, 530 insertions, 274 deletions
diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py index 5c248d3655..e01b8d5256 100644 --- a/poky/bitbake/lib/bb/__init__.py +++ b/poky/bitbake/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "1.51.1" +__version__ = "1.53.0" import sys if sys.version_info < (3, 6, 0): diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py index 50e60d5c31..34960197d1 100644 --- a/poky/bitbake/lib/bb/asyncrpc/client.py +++ b/poky/bitbake/lib/bb/asyncrpc/client.py @@ -7,6 +7,7 @@ import asyncio import json import os import socket +import sys from . import chunkify, DEFAULT_MAX_CHUNK @@ -129,7 +130,7 @@ class Client(object): # required (but harmless) with it. asyncio.set_event_loop(self.loop) - self._add_methods('connect_tcp', 'close', 'ping') + self._add_methods('connect_tcp', 'ping') @abc.abstractmethod def _get_async_client(self): @@ -163,3 +164,9 @@ class Client(object): @max_chunk.setter def max_chunk(self, value): self.client.max_chunk = value + + def close(self): + self.loop.run_until_complete(self.client.close()) + if sys.version_info >= (3, 6): + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() diff --git a/poky/bitbake/lib/bb/build.py b/poky/bitbake/lib/bb/build.py index 7e4ab9f64c..d6418e40b3 100644 --- a/poky/bitbake/lib/bb/build.py +++ b/poky/bitbake/lib/bb/build.py @@ -569,7 +569,6 @@ exit $ret def _task_data(fn, task, d): localdata = bb.data.createCopy(d) localdata.setVar('BB_FILENAME', fn) - localdata.setVar('BB_CURRENTTASK', task[3:]) localdata.setVar('OVERRIDES', 'task-%s:%s' % (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) localdata.finalize() diff --git a/poky/bitbake/lib/bb/compress/_pipecompress.py b/poky/bitbake/lib/bb/compress/_pipecompress.py index 4b9f662143..5de17a82e2 100644 --- a/poky/bitbake/lib/bb/compress/_pipecompress.py +++ b/poky/bitbake/lib/bb/compress/_pipecompress.py @@ -49,7 +49,7 @@ def open_wrap( raise ValueError("Argument 'newline' not supported in binary mode") file_mode = mode.replace("t", "") - if isinstance(filename, (str, bytes, os.PathLike)): + if isinstance(filename, (str, bytes, os.PathLike, int)): binary_file = cls(filename, file_mode, **kwargs) elif hasattr(filename, "read") or hasattr(filename, "write"): binary_file = cls(None, file_mode, fileobj=filename, **kwargs) diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index ee29d89b18..666cc1306a 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -884,7 +884,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) success = True except bb.process.NotFoundError as e: - error_message = "Fetch command %s" % (e.command) + error_message = "Fetch command %s not found" % (e.command) except bb.process.ExecutionError as e: if e.stdout: output = "output:\n%s\n%s" % (e.stdout, e.stderr) @@ -1721,7 +1721,9 @@ class Fetch(object): self.d.setVar("BB_NO_NETWORK", "1") firsterr = None - verified_stamp = m.verify_donestamp(ud, self.d) + verified_stamp = False + if done: + verified_stamp = m.verify_donestamp(ud, self.d) if not done and (not verified_stamp or m.need_update(ud, self.d)): try: if not trusted_network(self.d, ud.url): @@ -1780,7 +1782,11 @@ class Fetch(object): def checkstatus(self, urls=None): """ - Check all urls exist upstream + Check all URLs exist upstream. + + Returns None if the URLs exist, raises FetchError if the check wasn't + successful but there wasn't an error (such as file not found), and + raises other exceptions in error cases. """ if not urls: diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index a4527bf364..a7110a988d 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -140,16 +140,6 @@ class GitSM(Git): if Git.need_update(self, ud, d): return True - try: - # Check for the nugget dropped by the download operation - known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ - (ud.basecmd), d, workdir=ud.clonedir) - - if ud.revisions[ud.names[0]] in known_srcrevs.split(): - return False - except bb.fetch2.FetchError: - pass - need_update_list = [] def need_update_submodule(ud, url, module, modpath, workdir, d): url += ";bareclone=1;nobranch=1" @@ -172,11 +162,6 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, need_update_submodule, d) - if len(need_update_list) == 0: - # We already have the required commits of all submodules. Drop - # a nugget so we don't need to check again. - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) if len(need_update_list) > 0: logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) @@ -209,9 +194,6 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, download_submodule, d) - # Drop a nugget for the srcrev we've fetched (used by need_update) - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) def unpack(self, ud, destdir, d): def unpack_submodules(ud, url, module, modpath, workdir, d): diff --git a/poky/bitbake/lib/bb/fetch2/npm.py b/poky/bitbake/lib/bb/fetch2/npm.py index 47898509ff..e497c38dc7 100644 --- a/poky/bitbake/lib/bb/fetch2/npm.py +++ b/poky/bitbake/lib/bb/fetch2/npm.py @@ -69,17 +69,35 @@ def npm_unpack(tarball, destdir, d): bb.utils.mkdirhier(destdir) cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) cmd += " --no-same-owner" + cmd += " --delay-directory-restore" cmd += " --strip-components=1" runfetchcmd(cmd, d, workdir=destdir) + runfetchcmd("chmod -R +X %s" % (destdir), d, quiet=True, workdir=destdir) class NpmEnvironment(object): """ Using a npm config file seems more reliable than using cli arguments. This class allows to create a controlled environment for npm commands. """ - def __init__(self, d, configs=None): + def __init__(self, d, configs=None, npmrc=None): self.d = d - self.configs = configs + + if configs: + self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) + self.user_config_name = self.user_config.name + for key, value in configs: + self.user_config.write("%s=%s\n" % (key, value)) + else: + self.user_config_name = "/dev/null" + + if npmrc: + self.global_config_name = npmrc + else: + self.global_config_name = "/dev/null" + + def __del__(self): + if self.user_config: + self.user_config.close() def run(self, cmd, args=None, configs=None, workdir=None): """Run npm command in a controlled environment""" @@ -87,23 +105,19 @@ class NpmEnvironment(object): d = bb.data.createCopy(self.d) d.setVar("HOME", tmpdir) - cfgfile = os.path.join(tmpdir, "npmrc") - if not workdir: workdir = tmpdir def _run(cmd): - cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd - cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd + cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config_name) + cmd + cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd return runfetchcmd(cmd, d, workdir=workdir) - if self.configs: - for key, value in self.configs: - _run("npm config set %s %s" % (key, shlex.quote(value))) - if configs: + bb.warn("Use of configs argument of NpmEnvironment.run() function" + " is deprecated. Please use args argument instead.") for key, value in configs: - _run("npm config set %s %s" % (key, shlex.quote(value))) + cmd += " --%s=%s" % (key, shlex.quote(value)) if args: for key, value in args: @@ -165,14 +179,14 @@ class Npm(FetchMethod): def _resolve_proxy_url(self, ud, d): def _npm_view(): - configs = [] - configs.append(("json", "true")) - configs.append(("registry", ud.registry)) + args = [] + args.append(("json", "true")) + args.append(("registry", ud.registry)) pkgver = shlex.quote(ud.package + "@" + ud.version) cmd = ud.basecmd + " view %s" % pkgver env = NpmEnvironment(d) check_network_access(d, cmd, ud.registry) - view_string = env.run(cmd, configs=configs) + view_string = env.run(cmd, args=args) if not view_string: raise FetchError("Unavailable package %s" % pkgver, ud.url) diff --git a/poky/bitbake/lib/bb/fetch2/npmsw.py b/poky/bitbake/lib/bb/fetch2/npmsw.py index 0c3511d8ab..426a139653 100644 --- a/poky/bitbake/lib/bb/fetch2/npmsw.py +++ b/poky/bitbake/lib/bb/fetch2/npmsw.py @@ -24,11 +24,14 @@ import bb from bb.fetch2 import Fetch from bb.fetch2 import FetchMethod from bb.fetch2 import ParameterError +from bb.fetch2 import runfetchcmd from bb.fetch2 import URI from bb.fetch2.npm import npm_integrity from bb.fetch2.npm import npm_localfile from bb.fetch2.npm import npm_unpack from bb.utils import is_semver +from bb.utils import lockfile +from bb.utils import unlockfile def foreach_dependencies(shrinkwrap, callback=None, dev=False): """ @@ -78,6 +81,7 @@ class NpmShrinkWrap(FetchMethod): extrapaths = [] destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] destsuffix = os.path.join(*destsubdirs) + unpack = True integrity = params.get("integrity", None) resolved = params.get("resolved", None) @@ -148,7 +152,12 @@ class NpmShrinkWrap(FetchMethod): url = str(uri) - # local tarball sources and local link sources are unsupported + # Handle local tarball and link sources + elif version.startswith("file"): + localpath = version[5:] + if not version.endswith(".tgz"): + unpack = False + else: raise ParameterError("Unsupported dependency: %s" % name, ud.url) @@ -157,6 +166,7 @@ class NpmShrinkWrap(FetchMethod): "localpath": localpath, "extrapaths": extrapaths, "destsuffix": destsuffix, + "unpack": unpack, }) try: @@ -177,7 +187,7 @@ class NpmShrinkWrap(FetchMethod): # This fetcher resolves multiple URIs from a shrinkwrap file and then # forwards it to a proxy fetcher. The management of the donestamp file, # the lockfile and the checksums are forwarded to the proxy fetcher. - ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) + ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data) ud.needdonestamp = False @staticmethod @@ -187,7 +197,9 @@ class NpmShrinkWrap(FetchMethod): proxy_ud = ud.proxy.ud[proxy_url] proxy_d = ud.proxy.d proxy_ud.setup_localpath(proxy_d) + lf = lockfile(proxy_ud.lockfile) returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) + unlockfile(lf) return returns def verify_donestamp(self, ud, d): @@ -237,7 +249,16 @@ class NpmShrinkWrap(FetchMethod): for dep in manual: depdestdir = os.path.join(destdir, dep["destsuffix"]) - npm_unpack(dep["localpath"], depdestdir, d) + if dep["url"]: + npm_unpack(dep["localpath"], depdestdir, d) + else: + depsrcdir= os.path.join(destdir, dep["localpath"]) + if dep["unpack"]: + npm_unpack(depsrcdir, depdestdir, d) + else: + bb.utils.mkdirhier(depdestdir) + cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) + runfetchcmd(cmd, d, workdir=depdestdir) def clean(self, ud, d): """Clean any existing full or partial download""" diff --git a/poky/bitbake/lib/bb/fetch2/perforce.py b/poky/bitbake/lib/bb/fetch2/perforce.py index e2a41a4a12..3b6fa4b1ec 100644 --- a/poky/bitbake/lib/bb/fetch2/perforce.py +++ b/poky/bitbake/lib/bb/fetch2/perforce.py @@ -134,7 +134,7 @@ class Perforce(FetchMethod): ud.setup_revisions(d) - ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision)) + ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision)) def _buildp4command(self, ud, d, command, depot_filename=None): """ diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py index 625a9cf3bb..578ba5d661 100644 --- a/poky/bitbake/lib/bb/siggen.py +++ b/poky/bitbake/lib/bb/siggen.py @@ -11,6 +11,8 @@ import pickle import bb.data import difflib import simplediff +import json +import bb.compress.zstd from bb.checksum import FileChecksumCache from bb import runqueue import hashserv @@ -19,6 +21,17 @@ import hashserv.client logger = logging.getLogger('BitBake.SigGen') hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv') +class SetEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return dict(_set_object=list(sorted(obj))) + return json.JSONEncoder.default(self, obj) + +def SetDecoder(dct): + if '_set_object' in dct: + return set(dct['_set_object']) + return dct + def init(d): siggens = [obj for obj in globals().values() if type(obj) is type and issubclass(obj, SignatureGenerator)] @@ -398,9 +411,9 @@ class SignatureGeneratorBasic(SignatureGenerator): fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: - with os.fdopen(fd, "wb") as stream: - p = pickle.dump(data, stream, -1) - stream.flush() + with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f: + json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder) + f.flush() os.chmod(tmpfile, 0o664) bb.utils.rename(tmpfile, sigfile) except (OSError, IOError) as err: @@ -794,12 +807,10 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): formatparams.update(values) return formatstr.format(**formatparams) - with open(a, 'rb') as f: - p1 = pickle.Unpickler(f) - a_data = p1.load() - with open(b, 'rb') as f: - p2 = pickle.Unpickler(f) - b_data = p2.load() + with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: + a_data = json.load(f, object_hook=SetDecoder) + with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: + b_data = json.load(f, object_hook=SetDecoder) def dict_diff(a, b, whitelist=set()): sa = set(a.keys()) @@ -815,11 +826,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): def file_checksums_diff(a, b): from collections import Counter - # Handle old siginfo format - if isinstance(a, dict): - a = [(os.path.basename(f), cs) for f, cs in a.items()] - if isinstance(b, dict): - b = [(os.path.basename(f), cs) for f, cs in b.items()] + + # Convert lists back to tuples + a = [(f[0], f[1]) for f in a] + b = [(f[0], f[1]) for f in b] + # Compare lists, ensuring we can handle duplicate filenames if they exist removedcount = Counter(a) removedcount.subtract(b) @@ -902,9 +913,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval)) if not 'file_checksum_values' in a_data: - a_data['file_checksum_values'] = {} + a_data['file_checksum_values'] = [] if not 'file_checksum_values' in b_data: - b_data['file_checksum_values'] = {} + b_data['file_checksum_values'] = [] changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) if changed: @@ -1031,9 +1042,8 @@ def calc_taskhash(sigdata): def dump_sigfile(a): output = [] - with open(a, 'rb') as f: - p1 = pickle.Unpickler(f) - a_data = p1.load() + with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: + a_data = json.load(f, object_hook=SetDecoder) output.append("basewhitelist: %s" % (a_data['basewhitelist'])) diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index af292a2163..8ad1c85990 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -376,7 +376,7 @@ class FetcherTest(unittest.TestCase): def setUp(self): self.origdir = os.getcwd() self.d = bb.data.init() - self.tempdir = tempfile.mkdtemp() + self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") self.dldir = os.path.join(self.tempdir, "download") os.mkdir(self.dldir) self.d.setVar("DL_DIR", self.dldir) @@ -826,12 +826,12 @@ class FetcherNoNetworkTest(FetcherTest): class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch(self): - fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) self.d.setVar("BB_NO_NETWORK", "1") - fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) + fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) fetcher.download() fetcher.unpack(self.unpackdir) self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) @@ -839,21 +839,21 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() def test_fetch_mirror_of_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() def test_fetch_file_mirror_of_mirror(self): - self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) os.mkdir(self.dldir + "/some2where") fetcher.download() @@ -861,20 +861,20 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_premirror(self): - self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() def test_fetch_specify_downloadfilename(self): - fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) @skipIfNoNetwork() def test_fetch_premirror_specify_downloadfilename_regex_uri(self): - self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake/") + self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) @@ -882,7 +882,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() # BZ13039 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): - self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake http://downloads.yoctoproject.org/releases/bitbake") + self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) fetcher.download() self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) @@ -1012,7 +1012,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule_CLI11(self): - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1027,12 +1027,12 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule_update_CLI11(self): """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # CLI11 that pulls in a newer nlohmann-json - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1291,10 +1291,10 @@ class FetchLatestVersionTest(FetcherTest): # # packages with versions only in current directory # - # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 + # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") : "2.19", - # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 + # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") : "20120814", # @@ -1357,13 +1357,13 @@ class FetchLatestVersionTest(FetcherTest): class FetchCheckStatusTest(FetcherTest): - test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", - "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", - "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", + test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", + "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", + "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", "https://yoctoproject.org/", "https://docs.yoctoproject.org", - "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", - "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", + "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", + "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", diff --git a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index efebf001a9..2645c0e985 100644 --- a/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/poky/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}" T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" -BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE" +BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" include conf/multiconfig/${BB_CURRENT_MC}.conf diff --git a/poky/bitbake/lib/bb/tests/runqueue.py b/poky/bitbake/lib/bb/tests/runqueue.py index 3d51779d6c..5b6ada886a 100644 --- a/poky/bitbake/lib/bb/tests/runqueue.py +++ b/poky/bitbake/lib/bb/tests/runqueue.py @@ -278,7 +278,6 @@ class RunQueueTests(unittest.TestCase): ["mc_2:a1:%s" % t for t in rerun_tasks] self.assertEqual(set(tasks), set(expected)) - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self): with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: extraenv = { @@ -304,7 +303,6 @@ class RunQueueTests(unittest.TestCase): self.shutdown(tempdir) - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_double(self): with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: extraenv = { @@ -329,7 +327,6 @@ class RunQueueTests(unittest.TestCase): self.shutdown(tempdir) - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_multiple_setscene(self): # Runs e1:do_package_setscene twice with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: @@ -361,7 +358,7 @@ class RunQueueTests(unittest.TestCase): def shutdown(self, tempdir): # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup - while os.path.exists(tempdir + "/hashserve.sock"): + while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal")): time.sleep(0.5) diff --git a/poky/bitbake/lib/bb/tests/utils.py b/poky/bitbake/lib/bb/tests/utils.py index 4d5e21b99e..c363f62d7d 100644 --- a/poky/bitbake/lib/bb/tests/utils.py +++ b/poky/bitbake/lib/bb/tests/utils.py @@ -418,7 +418,7 @@ MULTILINE = " stuff \\ ['MULTILINE'], handle_var) - testvalue = re.sub('\s+', ' ', value_in_callback.strip()) + testvalue = re.sub(r'\s+', ' ', value_in_callback.strip()) self.assertEqual(expected_value, testvalue) class EditBbLayersConf(unittest.TestCase): diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py index 8df745d130..484545a684 100644 --- a/poky/bitbake/lib/bb/ui/knotty.py +++ b/poky/bitbake/lib/bb/ui/knotty.py @@ -276,11 +276,11 @@ class TerminalFilter(object): print(content) else: if self.quiet: - content = "Running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total) + content = "Running tasks (%s of %s/%s of %s)" % (self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total) elif not len(activetasks): - content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total) + content = "No currently running tasks (%s of %s/%s of %s)" % (self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total) else: - content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total) + content = "Currently %2s running tasks (%s of %s/%s of %s)" % (len(activetasks), self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total) maxtask = self.helper.tasknumber_total if not self.main_progress or self.main_progress.maxval != maxtask: widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()] diff --git a/poky/bitbake/lib/bb/ui/uihelper.py b/poky/bitbake/lib/bb/ui/uihelper.py index 52fdae3fec..82913e0da8 100644 --- a/poky/bitbake/lib/bb/ui/uihelper.py +++ b/poky/bitbake/lib/bb/ui/uihelper.py @@ -50,8 +50,10 @@ class BBUIHelper: removetid(event.pid, tid) self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)}) elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted): - self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + event.stats.setscene_active + 1 + self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed self.tasknumber_total = event.stats.total + self.setscene_current = event.stats.setscene_active + event.stats.setscene_covered + event.stats.setscene_notcovered + self.setscene_total = event.stats.setscene_total self.needUpdate = True elif isinstance(event, bb.build.TaskProgress): if event.pid > 0 and event.pid in self.pidmap: diff --git a/poky/bitbake/lib/codegen.py b/poky/bitbake/lib/codegen.py index 62a6748c47..6955a7ada5 100644 --- a/poky/bitbake/lib/codegen.py +++ b/poky/bitbake/lib/codegen.py @@ -401,6 +401,12 @@ class SourceGenerator(NodeVisitor): def visit_Num(self, node): self.write(repr(node.n)) + def visit_Constant(self, node): + # Python 3.8 deprecated visit_Num(), visit_Str(), visit_Bytes(), + # visit_NameConstant() and visit_Ellipsis(). They can be removed once we + # require 3.8+. + self.write(repr(node.value)) + def visit_Tuple(self, node): self.write('(') idx = -1 diff --git a/poky/bitbake/lib/hashserv/__init__.py b/poky/bitbake/lib/hashserv/__init__.py index 5f2e101e52..9cb3fd57a5 100644 --- a/poky/bitbake/lib/hashserv/__init__.py +++ b/poky/bitbake/lib/hashserv/__init__.py @@ -22,46 +22,68 @@ ADDR_TYPE_TCP = 1 # is necessary DEFAULT_MAX_CHUNK = 32 * 1024 -TABLE_DEFINITION = ( - ("method", "TEXT NOT NULL"), - ("outhash", "TEXT NOT NULL"), - ("taskhash", "TEXT NOT NULL"), - ("unihash", "TEXT NOT NULL"), - ("created", "DATETIME"), +UNIHASH_TABLE_DEFINITION = ( + ("method", "TEXT NOT NULL", "UNIQUE"), + ("taskhash", "TEXT NOT NULL", "UNIQUE"), + ("unihash", "TEXT NOT NULL", ""), +) + +UNIHASH_TABLE_COLUMNS = tuple(name for name, _, _ in UNIHASH_TABLE_DEFINITION) + +OUTHASH_TABLE_DEFINITION = ( + ("method", "TEXT NOT NULL", "UNIQUE"), + ("taskhash", "TEXT NOT NULL", "UNIQUE"), + ("outhash", "TEXT NOT NULL", "UNIQUE"), + ("created", "DATETIME", ""), # Optional fields - ("owner", "TEXT"), - ("PN", "TEXT"), - ("PV", "TEXT"), - ("PR", "TEXT"), - ("task", "TEXT"), - ("outhash_siginfo", "TEXT"), + ("owner", "TEXT", ""), + ("PN", "TEXT", ""), + ("PV", "TEXT", ""), + ("PR", "TEXT", ""), + ("task", "TEXT", ""), + ("outhash_siginfo", "TEXT", ""), ) -TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION) +OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION) + +def _make_table(cursor, name, definition): + cursor.execute(''' + CREATE TABLE IF NOT EXISTS {name} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + {fields} + UNIQUE({unique}) + ) + '''.format( + name=name, + fields=" ".join("%s %s," % (name, typ) for name, typ, _ in definition), + unique=", ".join(name for name, _, flags in definition if "UNIQUE" in flags) + )) + def setup_database(database, sync=True): db = sqlite3.connect(database) db.row_factory = sqlite3.Row with closing(db.cursor()) as cursor: - cursor.execute(''' - CREATE TABLE IF NOT EXISTS tasks_v2 ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - %s - UNIQUE(method, outhash, taskhash) - ) - ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION)) + _make_table(cursor, "unihashes_v2", UNIHASH_TABLE_DEFINITION) + _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION) + cursor.execute('PRAGMA journal_mode = WAL') cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF')) # Drop old indexes cursor.execute('DROP INDEX IF EXISTS taskhash_lookup') cursor.execute('DROP INDEX IF EXISTS outhash_lookup') + cursor.execute('DROP INDEX IF EXISTS taskhash_lookup_v2') + cursor.execute('DROP INDEX IF EXISTS outhash_lookup_v2') + + # TODO: Upgrade from tasks_v2? + cursor.execute('DROP TABLE IF EXISTS tasks_v2') # Create new indexes - cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup_v2 ON tasks_v2 (method, taskhash, created)') - cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup_v2 ON tasks_v2 (method, outhash)') + cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup_v3 ON unihashes_v2 (method, taskhash)') + cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup_v3 ON outhashes_v2 (method, outhash)') return db diff --git a/poky/bitbake/lib/hashserv/client.py b/poky/bitbake/lib/hashserv/client.py index 1a67c6982d..b2aa1026ac 100644 --- a/poky/bitbake/lib/hashserv/client.py +++ b/poky/bitbake/lib/hashserv/client.py @@ -107,11 +107,11 @@ class Client(bb.asyncrpc.Client): super().__init__() self._add_methods( "connect_tcp", - "close", "get_unihash", "report_unihash", "report_unihash_equiv", "get_taskhash", + "get_outhash", "get_stats", "reset_stats", "backfill_wait", diff --git a/poky/bitbake/lib/hashserv/server.py b/poky/bitbake/lib/hashserv/server.py index a059e52115..d40a2ab8f8 100644 --- a/poky/bitbake/lib/hashserv/server.py +++ b/poky/bitbake/lib/hashserv/server.py @@ -5,11 +5,12 @@ from contextlib import closing, contextmanager from datetime import datetime +import enum import asyncio import logging import math import time -from . import create_async_client, TABLE_COLUMNS +from . import create_async_client, UNIHASH_TABLE_COLUMNS, OUTHASH_TABLE_COLUMNS import bb.asyncrpc @@ -106,56 +107,64 @@ class Stats(object): return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')} -def insert_task(cursor, data, ignore=False): +@enum.unique +class Resolve(enum.Enum): + FAIL = enum.auto() + IGNORE = enum.auto() + REPLACE = enum.auto() + + +def insert_table(cursor, table, data, on_conflict): + resolve = { + Resolve.FAIL: "", + Resolve.IGNORE: " OR IGNORE", + Resolve.REPLACE: " OR REPLACE", + }[on_conflict] + keys = sorted(data.keys()) - query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % ( - " OR IGNORE" if ignore else "", - ', '.join(keys), - ', '.join(':' + k for k in keys)) + query = 'INSERT{resolve} INTO {table} ({fields}) VALUES({values})'.format( + resolve=resolve, + table=table, + fields=", ".join(keys), + values=", ".join(":" + k for k in keys), + ) + prevrowid = cursor.lastrowid cursor.execute(query, data) - -async def copy_from_upstream(client, db, method, taskhash): - d = await client.get_taskhash(method, taskhash, True) + logging.debug( + "Inserting %r into %s, %s", + data, + table, + on_conflict + ) + return (cursor.lastrowid, cursor.lastrowid != prevrowid) + +def insert_unihash(cursor, data, on_conflict): + return insert_table(cursor, "unihashes_v2", data, on_conflict) + +def insert_outhash(cursor, data, on_conflict): + return insert_table(cursor, "outhashes_v2", data, on_conflict) + +async def copy_unihash_from_upstream(client, db, method, taskhash): + d = await client.get_taskhash(method, taskhash) if d is not None: - # Filter out unknown columns - d = {k: v for k, v in d.items() if k in TABLE_COLUMNS} - with closing(db.cursor()) as cursor: - insert_task(cursor, d) + insert_unihash( + cursor, + {k: v for k, v in d.items() if k in UNIHASH_TABLE_COLUMNS}, + Resolve.IGNORE, + ) db.commit() - return d -async def copy_outhash_from_upstream(client, db, method, outhash, taskhash): - d = await client.get_outhash(method, outhash, taskhash) - if d is not None: - # Filter out unknown columns - d = {k: v for k, v in d.items() if k in TABLE_COLUMNS} - with closing(db.cursor()) as cursor: - insert_task(cursor, d) - db.commit() +class ServerCursor(object): + def __init__(self, db, cursor, upstream): + self.db = db + self.cursor = cursor + self.upstream = upstream - return d class ServerClient(bb.asyncrpc.AsyncServerConnection): - FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1' - ALL_QUERY = 'SELECT * FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1' - OUTHASH_QUERY = ''' - -- Find tasks with a matching outhash (that is, tasks that - -- are equivalent) - SELECT * FROM tasks_v2 WHERE method=:method AND outhash=:outhash - - -- If there is an exact match on the taskhash, return it. - -- Otherwise return the oldest matching outhash of any - -- taskhash - ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END, - created ASC - - -- Only return one row - LIMIT 1 - ''' - def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only): super().__init__(reader, writer, 'OEHASHEQUIV', logger) self.db = db @@ -210,36 +219,102 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_get(self, request): method = request['method'] taskhash = request['taskhash'] + fetch_all = request.get('all', False) - if request.get('all', False): - row = self.query_equivalent(method, taskhash, self.ALL_QUERY) - else: - row = self.query_equivalent(method, taskhash, self.FAST_QUERY) + with closing(self.db.cursor()) as cursor: + d = await self.get_unihash(cursor, method, taskhash, fetch_all) - if row is not None: - logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) - d = {k: row[k] for k in row.keys()} - elif self.upstream_client is not None: - d = await copy_from_upstream(self.upstream_client, self.db, method, taskhash) + self.write_message(d) + + async def get_unihash(self, cursor, method, taskhash, fetch_all=False): + d = None + + if fetch_all: + cursor.execute( + ''' + SELECT *, unihashes_v2.unihash AS unihash FROM outhashes_v2 + INNER JOIN unihashes_v2 ON unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash + WHERE outhashes_v2.method=:method AND outhashes_v2.taskhash=:taskhash + ORDER BY outhashes_v2.created ASC + LIMIT 1 + ''', + { + 'method': method, + 'taskhash': taskhash, + } + + ) + row = cursor.fetchone() + + if row is not None: + d = {k: row[k] for k in row.keys()} + elif self.upstream_client is not None: + d = await self.upstream_client.get_taskhash(method, taskhash, True) + self.update_unified(cursor, d) + self.db.commit() else: - d = None + row = self.query_equivalent(cursor, method, taskhash) + + if row is not None: + d = {k: row[k] for k in row.keys()} + elif self.upstream_client is not None: + d = await self.upstream_client.get_taskhash(method, taskhash) + d = {k: v for k, v in d.items() if k in UNIHASH_TABLE_COLUMNS} + insert_unihash(cursor, d, Resolve.IGNORE) + self.db.commit() - self.write_message(d) + return d async def handle_get_outhash(self, request): + method = request['method'] + outhash = request['outhash'] + taskhash = request['taskhash'] + with closing(self.db.cursor()) as cursor: - cursor.execute(self.OUTHASH_QUERY, - {k: request[k] for k in ('method', 'outhash', 'taskhash')}) + d = await self.get_outhash(cursor, method, outhash, taskhash) - row = cursor.fetchone() + self.write_message(d) + + async def get_outhash(self, cursor, method, outhash, taskhash): + d = None + cursor.execute( + ''' + SELECT *, unihashes_v2.unihash AS unihash FROM outhashes_v2 + INNER JOIN unihashes_v2 ON unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash + WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash + ORDER BY outhashes_v2.created ASC + LIMIT 1 + ''', + { + 'method': method, + 'outhash': outhash, + } + ) + row = cursor.fetchone() if row is not None: - logger.debug('Found equivalent outhash %s -> %s', (row['outhash'], row['unihash'])) d = {k: row[k] for k in row.keys()} - else: - d = None + elif self.upstream_client is not None: + d = await self.upstream_client.get_outhash(method, outhash, taskhash) + self.update_unified(cursor, d) + self.db.commit() - self.write_message(d) + return d + + def update_unified(self, cursor, data): + if data is None: + return + + insert_unihash( + cursor, + {k: v for k, v in data.items() if k in UNIHASH_TABLE_COLUMNS}, + Resolve.IGNORE + ) + insert_outhash( + cursor, + {k: v for k, v in data.items() if k in OUTHASH_TABLE_COLUMNS}, + Resolve.IGNORE + ) async def handle_get_stream(self, request): self.write_message('ok') @@ -267,7 +342,12 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): (method, taskhash) = l.split() #logger.debug('Looking up %s %s' % (method, taskhash)) - row = self.query_equivalent(method, taskhash, self.FAST_QUERY) + cursor = self.db.cursor() + try: + row = self.query_equivalent(cursor, method, taskhash) + finally: + cursor.close() + if row is not None: msg = ('%s\n' % row['unihash']).encode('utf-8') #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash'])) @@ -294,55 +374,82 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_report(self, data): with closing(self.db.cursor()) as cursor: - cursor.execute(self.OUTHASH_QUERY, - {k: data[k] for k in ('method', 'outhash', 'taskhash')}) + outhash_data = { + 'method': data['method'], + 'outhash': data['outhash'], + 'taskhash': data['taskhash'], + 'created': datetime.now() + } - row = cursor.fetchone() + for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'): + if k in data: + outhash_data[k] = data[k] + + # Insert the new entry, unless it already exists + (rowid, inserted) = insert_outhash(cursor, outhash_data, Resolve.IGNORE) + + if inserted: + # If this row is new, check if it is equivalent to another + # output hash + cursor.execute( + ''' + SELECT outhashes_v2.taskhash AS taskhash, unihashes_v2.unihash AS unihash FROM outhashes_v2 + INNER JOIN unihashes_v2 ON unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash + -- Select any matching output hash except the one we just inserted + WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash AND outhashes_v2.taskhash!=:taskhash + -- Pick the oldest hash + ORDER BY outhashes_v2.created ASC + LIMIT 1 + ''', + { + 'method': data['method'], + 'outhash': data['outhash'], + 'taskhash': data['taskhash'], + } + ) + row = cursor.fetchone() - if row is None and self.upstream_client: - # Try upstream - row = await copy_outhash_from_upstream(self.upstream_client, - self.db, - data['method'], - data['outhash'], - data['taskhash']) - - # If no matching outhash was found, or one *was* found but it - # wasn't an exact match on the taskhash, a new entry for this - # taskhash should be added - if row is None or row['taskhash'] != data['taskhash']: - # If a row matching the outhash was found, the unihash for - # the new taskhash should be the same as that one. - # Otherwise the caller provided unihash is used. - unihash = data['unihash'] if row is not None: + # A matching output hash was found. Set our taskhash to the + # same unihash since they are equivalent unihash = row['unihash'] + resolve = Resolve.IGNORE + else: + # No matching output hash was found. This is probably the + # first outhash to be added. + unihash = data['unihash'] + resolve = Resolve.IGNORE + + # Query upstream to see if it has a unihash we can use + if self.upstream_client is not None: + upstream_data = await self.upstream_client.get_outhash(data['method'], data['outhash'], data['taskhash']) + if upstream_data is not None: + unihash = upstream_data['unihash'] + + + insert_unihash( + cursor, + { + 'method': data['method'], + 'taskhash': data['taskhash'], + 'unihash': unihash, + }, + resolve + ) + + unihash_data = await self.get_unihash(cursor, data['method'], data['taskhash']) + if unihash_data is not None: + unihash = unihash_data['unihash'] + else: + unihash = data['unihash'] - insert_data = { - 'method': data['method'], - 'outhash': data['outhash'], - 'taskhash': data['taskhash'], - 'unihash': unihash, - 'created': datetime.now() - } - - for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'): - if k in data: - insert_data[k] = data[k] - - insert_task(cursor, insert_data) - self.db.commit() - - logger.info('Adding taskhash %s with unihash %s', - data['taskhash'], unihash) + self.db.commit() - d = { - 'taskhash': data['taskhash'], - 'method': data['method'], - 'unihash': unihash - } - else: - d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + d = { + 'taskhash': data['taskhash'], + 'method': data['method'], + 'unihash': unihash, + } self.write_message(d) @@ -350,23 +457,16 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): with closing(self.db.cursor()) as cursor: insert_data = { 'method': data['method'], - 'outhash': "", 'taskhash': data['taskhash'], 'unihash': data['unihash'], - 'created': datetime.now() } - - for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'): - if k in data: - insert_data[k] = data[k] - - insert_task(cursor, insert_data, ignore=True) + insert_unihash(cursor, insert_data, Resolve.IGNORE) self.db.commit() # Fetch the unihash that will be reported for the taskhash. If the # unihash matches, it means this row was inserted (or the mapping # was already valid) - row = self.query_equivalent(data['method'], data['taskhash'], self.FAST_QUERY) + row = self.query_equivalent(cursor, data['method'], data['taskhash']) if row['unihash'] == data['unihash']: logger.info('Adding taskhash equivalence for %s with unihash %s', @@ -399,14 +499,16 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): await self.backfill_queue.join() self.write_message(d) - def query_equivalent(self, method, taskhash, query): + def query_equivalent(self, cursor, method, taskhash): # This is part of the inner loop and must be as fast as possible - try: - cursor = self.db.cursor() - cursor.execute(query, {'method': method, 'taskhash': taskhash}) - return cursor.fetchone() - except: - cursor.close() + cursor.execute( + 'SELECT taskhash, method, unihash FROM unihashes_v2 WHERE method=:method AND taskhash=:taskhash', + { + 'method': method, + 'taskhash': taskhash, + } + ) + return cursor.fetchone() class Server(bb.asyncrpc.AsyncServer): @@ -435,7 +537,7 @@ class Server(bb.asyncrpc.AsyncServer): self.backfill_queue.task_done() break method, taskhash = item - await copy_from_upstream(client, self.db, method, taskhash) + await copy_unihash_from_upstream(client, self.db, method, taskhash) self.backfill_queue.task_done() finally: await client.close() diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py index e851535c59..f6b85aed85 100644 --- a/poky/bitbake/lib/hashserv/tests.py +++ b/poky/bitbake/lib/hashserv/tests.py @@ -19,10 +19,10 @@ import time import signal def server_prefunc(server, idx): - logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w', + logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w', format='%(levelname)s %(filename)s:%(lineno)d %(message)s') server.logger.debug("Running server %d" % idx) - sys.stdout = open('bbhashserv-%d.log' % idx, 'w') + sys.stdout = open('bbhashserv-stdout-%d.log' % idx, 'w') sys.stderr = sys.stdout class HashEquivalenceTestSetup(object): @@ -140,12 +140,17 @@ class HashEquivalenceCommonTests(object): }) self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') - result = self.client.get_taskhash(self.METHOD, taskhash, True) - self.assertEqual(result['taskhash'], taskhash) - self.assertEqual(result['unihash'], unihash) - self.assertEqual(result['method'], self.METHOD) - self.assertEqual(result['outhash'], outhash) - self.assertEqual(result['outhash_siginfo'], siginfo) + result_unihash = self.client.get_taskhash(self.METHOD, taskhash, True) + self.assertEqual(result_unihash['taskhash'], taskhash) + self.assertEqual(result_unihash['unihash'], unihash) + self.assertEqual(result_unihash['method'], self.METHOD) + + result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash) + self.assertEqual(result_outhash['taskhash'], taskhash) + self.assertEqual(result_outhash['method'], self.METHOD) + self.assertEqual(result_outhash['unihash'], unihash) + self.assertEqual(result_outhash['outhash'], outhash) + self.assertEqual(result_outhash['outhash_siginfo'], siginfo) def test_stress(self): def query_server(failures): @@ -260,6 +265,39 @@ class HashEquivalenceCommonTests(object): result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6) self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream') + # Tests read through from server with + taskhash7 = '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74' + outhash7 = '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69' + unihash7 = '05d2a63c81e32f0a36542ca677e8ad852365c538' + self.client.report_unihash(taskhash7, self.METHOD, outhash7, unihash7) + + result = down_client.get_taskhash(self.METHOD, taskhash7, True) + self.assertEqual(result['unihash'], unihash7, 'Server failed to copy unihash from upstream') + self.assertEqual(result['outhash'], outhash7, 'Server failed to copy unihash from upstream') + self.assertEqual(result['taskhash'], taskhash7, 'Server failed to copy unihash from upstream') + self.assertEqual(result['method'], self.METHOD) + + taskhash8 = '86978a4c8c71b9b487330b0152aade10c1ee58aa' + outhash8 = 'ca8c128e9d9e4a28ef24d0508aa20b5cf880604eacd8f65c0e366f7e0cc5fbcf' + unihash8 = 'd8bcf25369d40590ad7d08c84d538982f2023e01' + self.client.report_unihash(taskhash8, self.METHOD, outhash8, unihash8) + + result = down_client.get_outhash(self.METHOD, outhash8, taskhash8) + self.assertEqual(result['unihash'], unihash8, 'Server failed to copy unihash from upstream') + self.assertEqual(result['outhash'], outhash8, 'Server failed to copy unihash from upstream') + self.assertEqual(result['taskhash'], taskhash8, 'Server failed to copy unihash from upstream') + self.assertEqual(result['method'], self.METHOD) + + taskhash9 = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c' + outhash9 = 'afc78172c81880ae10a1fec994b5b4ee33d196a001a1b66212a15ebe573e00b5' + unihash9 = '6662e699d6e3d894b24408ff9a4031ef9b038ee8' + self.client.report_unihash(taskhash9, self.METHOD, outhash9, unihash9) + + result = down_client.get_taskhash(self.METHOD, taskhash9, False) + self.assertEqual(result['unihash'], unihash9, 'Server failed to copy unihash from upstream') + self.assertEqual(result['taskhash'], taskhash9, 'Server failed to copy unihash from upstream') + self.assertEqual(result['method'], self.METHOD) + def test_ro_server(self): (ro_client, ro_server) = self.start_server(dbpath=self.server.dbpath, read_only=True) @@ -287,10 +325,8 @@ class HashEquivalenceCommonTests(object): def test_slow_server_start(self): - """ - Ensures that the server will exit correctly even if it gets a SIGTERM - before entering the main loop - """ + # Ensures that the server will exit correctly even if it gets a SIGTERM + # before entering the main loop event = multiprocessing.Event() @@ -312,6 +348,58 @@ class HashEquivalenceCommonTests(object): server.process.join(300) self.assertIsNotNone(server.process.exitcode, "Server did not exit in a timely manner!") + def test_diverging_report_race(self): + # Tests that a reported task will correctly pick up an updated unihash + + # This is a baseline report added to the database to ensure that there + # is something to match against as equivalent + outhash1 = 'afd11c366050bcd75ad763e898e4430e2a60659b26f83fbb22201a60672019fa' + taskhash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab' + unihash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab' + result = self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + # Add a report that is equivalent to Task 1. It should ignore the + # provided unihash and report the unihash from task 1 + taskhash2 = '6259ae8263bd94d454c086f501c37e64c4e83cae806902ca95b4ab513546b273' + unihash2 = taskhash2 + result = self.client.report_unihash(taskhash2, self.METHOD, outhash1, unihash2) + self.assertEqual(result['unihash'], unihash1) + + # Add another report for Task 2, but with a different outhash (e.g. the + # task is non-deterministic). It should still be marked with the Task 1 + # unihash because it has the Task 2 taskhash, which is equivalent to + # Task 1 + outhash3 = 'd2187ee3a8966db10b34fe0e863482288d9a6185cb8ef58a6c1c6ace87a2f24c' + result = self.client.report_unihash(taskhash2, self.METHOD, outhash3, unihash2) + self.assertEqual(result['unihash'], unihash1) + + + def test_diverging_report_reverse_race(self): + # Same idea as the previous test, but Tasks 2 and 3 are reported in + # reverse order the opposite order + + outhash1 = 'afd11c366050bcd75ad763e898e4430e2a60659b26f83fbb22201a60672019fa' + taskhash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab' + unihash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab' + result = self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1) + + taskhash2 = '6259ae8263bd94d454c086f501c37e64c4e83cae806902ca95b4ab513546b273' + unihash2 = taskhash2 + + # Report Task 3 first. Since there is nothing else in the database it + # will use the client provided unihash + outhash3 = 'd2187ee3a8966db10b34fe0e863482288d9a6185cb8ef58a6c1c6ace87a2f24c' + result = self.client.report_unihash(taskhash2, self.METHOD, outhash3, unihash2) + self.assertEqual(result['unihash'], unihash2) + + # Report Task 2. This is equivalent to Task 1 but there is already a mapping for + # taskhash2 so it will report unihash2 + result = self.client.report_unihash(taskhash2, self.METHOD, outhash1, unihash2) + self.assertEqual(result['unihash'], unihash2) + + # The originally reported unihash for Task 3 should be unchanged even if it + # shares a taskhash with Task 2 + self.assertClientGetHash(self.client, taskhash2, unihash2) class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): def get_server_addr(self, server_idx): diff --git a/poky/bitbake/lib/layerindexlib/__init__.py b/poky/bitbake/lib/layerindexlib/__init__.py index 3159bf2f66..08063c571e 100644 --- a/poky/bitbake/lib/layerindexlib/__init__.py +++ b/poky/bitbake/lib/layerindexlib/__init__.py @@ -198,7 +198,7 @@ The format of the indexURI: For example: - http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index + https://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index cooker:// ''' if reload: @@ -576,7 +576,7 @@ This function is used to implement debugging and provide the user info. # index['config'] - configuration data for this index # index['branches'] - dictionary of Branch objects, by id number # index['layerItems'] - dictionary of layerItem objects, by id number -# ...etc... (See: http://layers.openembedded.org/layerindex/api/) +# ...etc... (See: https://layers.openembedded.org/layerindex/api/) # # The class needs to manage the 'index' entries and allow easily adding # of new items, as well as simply loading of the items. diff --git a/poky/bitbake/lib/layerindexlib/restapi.py b/poky/bitbake/lib/layerindexlib/restapi.py index 26a1c9674e..81d99b02ea 100644 --- a/poky/bitbake/lib/layerindexlib/restapi.py +++ b/poky/bitbake/lib/layerindexlib/restapi.py @@ -31,7 +31,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin): The return value is a LayerIndexObj. url is the url to the rest api of the layer index, such as: - http://layers.openembedded.org/layerindex/api/ + https://layers.openembedded.org/layerindex/api/ Or a local file... """ @@ -138,7 +138,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin): The return value is a LayerIndexObj. ud is the parsed url to the rest api of the layer index, such as: - http://layers.openembedded.org/layerindex/api/ + https://layers.openembedded.org/layerindex/api/ """ def _get_json_response(apiurl=None, username=None, password=None, retry=True): diff --git a/poky/bitbake/lib/layerindexlib/tests/restapi.py b/poky/bitbake/lib/layerindexlib/tests/restapi.py index 33b5c1c4c8..71f0ae8a9d 100644 --- a/poky/bitbake/lib/layerindexlib/tests/restapi.py +++ b/poky/bitbake/lib/layerindexlib/tests/restapi.py @@ -22,7 +22,7 @@ class LayerIndexWebRestApiTest(LayersTest): self.assertFalse(os.environ.get("BB_SKIP_NETTESTS") == "yes", msg="BB_SKIP_NETTESTS set, but we tried to test anyway") LayersTest.setUp(self) self.layerindex = layerindexlib.LayerIndex(self.d) - self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies']) + self.layerindex.load_layerindex('https://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies']) @skipIfNoNetwork() def test_layerindex_is_empty(self): diff --git a/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml index 026d94869a..b01a337012 100644 --- a/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml +++ b/poky/bitbake/lib/toaster/orm/fixtures/oe-core.xml @@ -34,7 +34,7 @@ <field type="CharField" name="description">Openembedded Dunfell</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> <field type="CharField" name="branch_name">dunfell</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=dunfell\">OpenEmbedded Dunfell</a> branch.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=dunfell\">OpenEmbedded Dunfell</a> branch.</field> </object> <object model="orm.release" pk="2"> <field type="CharField" name="name">local</field> @@ -48,14 +48,14 @@ <field type="CharField" name="description">OpenEmbedded core master</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field> <field type="CharField" name="branch_name">master</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch.</field> </object> <object model="orm.release" pk="4"> <field type="CharField" name="name">gatesgarth</field> <field type="CharField" name="description">Openembedded Gatesgarth</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field> <field type="CharField" name="branch_name">gatesgarth</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=gatesgarth\">OpenEmbedded Gatesgarth</a> branch.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=gatesgarth\">OpenEmbedded Gatesgarth</a> branch.</field> </object> <!-- Default layers for each release --> @@ -81,9 +81,9 @@ <object model="orm.layer" pk="1"> <field type="CharField" name="name">openembedded-core</field> <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field> - <field type="CharField" name="vcs_web_url">http://cgit.openembedded.org/openembedded-core</field> - <field type="CharField" name="vcs_web_tree_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field> - <field type="CharField" name="vcs_web_file_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_url">https://cgit.openembedded.org/openembedded-core</field> + <field type="CharField" name="vcs_web_tree_base_url">https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_file_base_url">https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field> </object> <object model="orm.layer_version" pk="1"> <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field> diff --git a/poky/bitbake/lib/toaster/orm/fixtures/poky.xml b/poky/bitbake/lib/toaster/orm/fixtures/poky.xml index a468a54c49..363789d624 100644 --- a/poky/bitbake/lib/toaster/orm/fixtures/poky.xml +++ b/poky/bitbake/lib/toaster/orm/fixtures/poky.xml @@ -39,7 +39,7 @@ <field type="CharField" name="description">Yocto Project 3.1 "Dunfell"</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field> <field type="CharField" name="branch_name">dunfell</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=dunfell">Yocto Project Dunfell branch</a>.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=dunfell">Yocto Project Dunfell branch</a>.</field> </object> <object model="orm.release" pk="2"> <field type="CharField" name="name">local</field> @@ -53,14 +53,14 @@ <field type="CharField" name="description">Yocto Project master</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field> <field type="CharField" name="branch_name">master</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>.</field> </object> <object model="orm.release" pk="4"> <field type="CharField" name="name">gatesgarth</field> <field type="CharField" name="description">Yocto Project 3.2 "Gatesgarth"</field> <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field> <field type="CharField" name="branch_name">gatesgarth</field> - <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=gatesgarth">Yocto Project Gatesgarth branch</a>.</field> + <field type="TextField" name="helptext">Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=gatesgarth">Yocto Project Gatesgarth branch</a>.</field> </object> <!-- Default project layers for each release --> @@ -122,9 +122,9 @@ <field type="CharField" name="name">openembedded-core</field> <field type="CharField" name="layer_index_url"></field> <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field> - <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field> - <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> - <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field> + <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> </object> <object model="orm.layer_version" pk="1"> <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field> @@ -160,9 +160,9 @@ <field type="CharField" name="name">meta-poky</field> <field type="CharField" name="layer_index_url"></field> <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field> - <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field> - <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> - <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field> + <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> </object> <object model="orm.layer_version" pk="5"> <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field> @@ -198,9 +198,9 @@ <field type="CharField" name="name">meta-yocto-bsp</field> <field type="CharField" name="layer_index_url"></field> <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field> - <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field> - <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> - <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field> + <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> + <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field> </object> <object model="orm.layer_version" pk="9"> <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field> diff --git a/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py index 2fbd7be3d2..3b950e6e15 100644 --- a/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py +++ b/poky/bitbake/lib/toaster/orm/management/commands/lsupdates.py @@ -21,7 +21,7 @@ import threading import time logger = logging.getLogger("toaster") -DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" +DEFAULT_LAYERINDEX_SERVER = "https://layers.openembedded.org/layerindex/api/" # Add path to bitbake modules for layerindexlib # lib/toaster/orm/management/commands/lsupdates.py (abspath) diff --git a/poky/bitbake/lib/toaster/tests/builds/buildtest.py b/poky/bitbake/lib/toaster/tests/builds/buildtest.py index 872bbd3775..13b51fb0d8 100644 --- a/poky/bitbake/lib/toaster/tests/builds/buildtest.py +++ b/poky/bitbake/lib/toaster/tests/builds/buildtest.py @@ -119,7 +119,7 @@ class BuildTest(unittest.TestCase): if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"): ProjectVariable.objects.get_or_create( name="SSTATE_MIRRORS", - value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH", + value="file://.* http://sstate.yoctoproject.org/PATH;downloadfilename=PATH", project=project) ProjectTarget.objects.create(project=project, diff --git a/poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html b/poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html index 1e26e31c8b..923ca3bfe4 100644 --- a/poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html +++ b/poky/bitbake/lib/toaster/toastergui/templates/layerdetails.html @@ -355,7 +355,7 @@ {% if layerversion.layer_source == layer_source.TYPE_LAYERINDEX %} <dt>Layer index</dt> <dd> - <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a> + <a href="https://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a> </dd> {% endif %} </dl> diff --git a/poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html b/poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html index 66f8e7f069..a4fcd2aa42 100644 --- a/poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html +++ b/poky/bitbake/lib/toaster/toastergui/templates/package_detail_base.html @@ -127,7 +127,7 @@ {% comment %} # Removed per team meeting of 1/29/2014 until # decision on index search algorithm - <a href="http://layers.openembedded.org" target="_blank"> + <a href="https://layers.openembedded.org" target="_blank"> <i class="glyphicon glyphicon-share get-info"></i> </a> {% endcomment %} |