From 96ff1984133494bf6a3451ddeb7f14548d3697e1 Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Mon, 19 Aug 2019 13:50:42 -0400 Subject: subtree updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit poky: 67266331b0..835f7eac06: Adrian Bunk (9): valgrind: Remove dependency on libx11 bluez5: Remove obsolete dependency on dbus-glib python3-dbus: Remove obsolete dependency on dbus-glib cups: Remove unnecessary dependency on dbus-glib libnotify: Remove obsolete dependency on dbus-glib unfs3: Switch to new upstream location i2c-tools: Add alternative for i2ctransfer meta: Remove remnants of bluez4 support e2fsprogs: Remove patch that disabled 64bit for ext4 by default Adrian Freihofer (1): yocto-bsp: runqemu runs beaglebone-yocto Adrian Ratiu (1): opkg/package/rootfs_ipk: allow overwriting OPKGLIBDIR Alejandro del Castillo (1): opkg: upgrade to version 0.4.1 Alexander Kanavin (3): rt-tests: exclude 1.4 version from upstream check as well gtk-doc: correct the style.css permissions mobile-broadband-provider-info: upgrade 20190116 -> 20190618 Alistair Francis (7): mesa: Add support for the lima PACKAGECONFIG u-boot: Update to 2019.07 packagegroup-core-sdk: Set blank sanitiser for RISC-V 32 opensbi: Update from 0.3 to 0.4 opensbi: Fix installed-vs-shipped warning qemurunner.py: Be more verbose about problems package_manager: Ensure the base-feed directory exists Andrej Valek (2): busybox: 1.30.1 -> 1.31.0 oe/copy_buildsystem: move layer into layers directory Anuj Mittal (25): gstreamer1.0-plugins-bad: depend on vulkan-loader now vulkan-demos: depend on vulkan-loader vulkan: remove binutils: fix CVE-2019-12972 CVE-2019-9071 gnupg: upgrade 2.2.16 -> 2.2.17 libxslt: fix CVE-2019-13117 CVE-2019-13118 libva: upgrade 2.4.1 -> 2.5.0 libva-utils: upgrade 2.4.0 -> 2.5.0 nasm: fix CVE-2018-19755 python: fix CVE-2019-9740 python3: upgrade 3.7.3 -> 3.7.4 binutils: CVE-2019-9070 is same as CVE-2019-9071 qemu: fix CVE-2019-12155 bzip2: upgrade 1.0.7 -> 1.0.8 glib-2.0: upgrade 2.60.4 -> 2.60.5 vte: upgrade 0.56.1 -> 0.56.3 openssl: set CVE vendor to openssl curl: upgrade 7.65.1 -> 7.65.2 rsync: fix CVEs for included zlib glibc: CVE-2018-20796 is same as CVE-2019-9169 unzip: fix CVE-2019-13232 python: include CVE patches for python-native as well gdb: fix CVE-2017-9778 iptables: upgrade 1.8.2 -> 1.8.3 piglit: fix SRC_URI Armin Kuster (1): timezone: update to 2019b Bonnans, Laurent (1): openssl: fix valgrind errors on v1.1.1c Bruce Ashfield (5): linux-yocto/5.0: bsp: add basic xilinx zynqmp support linux-yocto/5.0: make scsi-debug include scsi core configs linux-yocto: bsp/beaglebone: support qemu -machine virt linux-yocto/4.19: update to 4.19.57 and -rt22 package: check PKG_ variables before executing ontarget postinst CHerzig@Gauselmann.de (1): bitbake: fetch2/clearcase: Fix class import errors Changqing Li (5): quilt: run-ptest remove Interactive Input mdadm: fix systemd service start up failure mdam: fix mdmonitor start up failure opkg: make ptest output format align with common style mdadm: make ptest output format align with common style Chee Yang Lee (1): wic: add support for kernel with initramfs bundled Chen Qi (13): target-sdk-provides-dummy: add libperl.so.5 64bit devtool: warn user about multiple layer having the same base name image.bbclass: fix systemd_preset_all devtool.py: track to clean devtool.conf in test_create_workspace grub-efi.bbclass: take into consideration of multilib sysstat: use service file from source codes xmlcatalog: hold libxml2-native dependency oeqa/runtime/rpm: ensure no user process running before deleting user oeqa/runtime/rpm: Move test_rpm_query_nonroot test case to RpmBasicTest qemurunner.py: fix race condition at qemu startup msmtp: use alternatives to manage /usr/lib/sendmail runtime_test.py: use track_for_cleanup for temp dir devtool: remove temp dir in upgrade Fabio Berton (1): mesa: Update 19.1.0 -> 19.1.1 Haiqing Bai (1): sysstat: Use sysstat.service in source for cron with systemd He Zhe (1): ltp: file01: Fix in was not recognized Hongzhi.Song (3): ltp: fix shmctl01 failure when executed. ltp: diotest4: Let kernel pick an address when calling mmap ltp: getrlimit03: adjust-a-bit-of-code-to-compatiable-with mips32 Jason Wessel (5): glibc: Fix multilibs + usrmerge builds psmisc: Fix dependency for USE_NLS=no glibc-locale: Fix build error with PACKAGE_NO_GCONV = "1" glibc/glibc-locale: Fix do_stash_locale to work with usrmerge and multilibs glibc / glibc-locale: Fix stash_locale determinism problems Joe Slater (1): libtool: remove host information from libtool Jon Mason (1): oe_syslog.py: Handle syslogd/klogd restart race Joshua Watt (5): python3: Fix .pyc file reproduciblility oeqa: Test bitbake --skip-setsecene bitbake: bitbake: Add --skip-setscene option classes/icecc: Disable remote pre-processing by default scripts/buildstats-diff: Add option to filter tasks Joël Esponde (1): package.bbclass: fix directories setuid and setgid bits Jun Nie (1): kernel-fitimage: uboot-sign: fix missing signature Kai Kang (4): rng-tools: fix rngd blocks system shutdown openssl: fix multilib files conflict webkitgtk: set incomptible with tune mips defaultsetup.conf: enable select init manager Khem Raj (10): efibootmgr: Pass correct flags to compiler from pkg-config mpeg2dec: Fix PIE build and avoid relocation in text section on ARM Revert "unzip: fix CVE-2019-13232" musl: Upgrade to 1.1.23+ mdadm: Include sys/sysmacros.h for major/minor definitions sysvinit: Include sys/sysmacros.h for major/minor definitions on musl too pam_systemd: Include missing.h for secure_getenv musl-obstack: Add recipe elfutils: Fix eu-* utils builds for musl maintainers: Account for musl-obstack and libssp-nonshared Li Zhou (2): bc: dc: fix exit code of q command iptables: Security Advisory - iptables - CVE-2019-11360 Luca Boccassi (1): bitbake: tests/fetch.py: add missing skipIfNoNetwork tags to tests that try to git clone Matthias Schiffer (1): systemd: backport patch to fix sysctl warning on boot Mike Crowe (4): bitbake.conf: Stop exporting TARGET_ flags variables image.bbclass: Only append to IMAGE_LINK_NAME if it was already set rootfs-postcommands: Cope with empty IMAGE_LINK_NAME in write_image_manifest rootfs-postcommands: Cope with empty IMAGE_LINK_NAME in write_image_test_data Mikko Rapeli (3): busybox: enable unicode support cve-check.bbclass: initialize to_append freetype: add --tag CC to libtool arguments Mingli Yu (2): go.bbclass: separate the ptest logic to go-ptest class mdadm: fix ptest hang Oleksandr Kravchuk (34): mc: update to 4.8.23 encodings: update to 1.0.5 gawk: update to 5.0.1 libinput: update to 1.13.3 libxi: update to 1.7.10 libxt: update to 1.2.0 autoconf-archive: update to 2019.01.06 python3-mako: update to 1.0.12 python3-pbr: update to 5.3.1 python3-pygobject: update to 3.32.2 git: update to 2.22.0 eudev: update to 3.2.8 babeltrace: update to 1.5.7 dpkg: update to 1.19.7 apt: update to 1.2.31 libinput: update to 1.13.4 expat: update to 2.2.7 libsolf: update to 0.7.5 bison: update to 3.4.1 ruby: update to 2.5.5 quilt: update to 0.66 bzip2: update to 1.0.7 python3-mako: update to 1.0.13 ifupdown: update to 0.8.22 libdrm: update to 2.4.99 python3-pbr: update to 5.4.0 linux-firmware: bump to 20190618 iproute2: update to 5.2.0 udev-extraconf: do not mount swap partitions python3-pbr: update to 5.4.1 xinput: update to 1.6.3 python3-scons: update to 3.1.0 python3-docutils: update to 0.15 python3-mako: update to 1.0.14 Pascal Bach (1): cmake: 3.14.1 -> 3.14.5 Paul Eggleton (7): libcap-ng: do not use symlink to share files with libcap-ng-python scripts/contrib/ddimage: fix typo scripts/contrib/ddimage: replace blacklist with mount check scripts/contrib/ddimage: be explicit whether device doesn't exist or isn't writeable list-packageconfig-flags: print PN instead of P recipetool: ignore zero-length setup.py files devtool: upgrade: fix handling of errors parsing upgraded recipe Peter Kjellerstedt (4): glib-2.0: Update to 2.60.4 glibc-package.inc: Do not use bitbake variable syntax for shell variables meson.bbclass: Remove the MESON_*_ARGS variables nativesdk-meson: Remove some unused variables Pierre Le Magourou (10): cve-update-db: Use std library instead of urllib3 cve-update-db: Manage proxy if needed. cve-update-db: do_populate_cve_db depends on do_fetch cve-update-db: Catch request.urlopen errors. cve-check: Depends on cve-update-db-native cve-update-db: Use NVD CPE data to populate PRODUCTS table cve-check: Update unpatched CVE matching cve-update-db-native: Skip recipe when cve-check class is not loaded. cve-check: Replace CVE_CHECK_CVE_WHITELIST by CVE_CHECK_WHITELIST cve-update-db-native: Remove hash column from database. Ricardo Ribalda Delgado (4): nfs-mountd: Add missing dependency on systemd service systemd: Fix interface bring-up on kernels >= 5.2 wic: Fix (again) partition files UIDs on multi rootfs images systemd-bootconf: Mark as machine specific Ricardo Salveti (1): gcc-9.1: add back GLIBC_DYNAMIC_LINKER riscv changes Richard Purdie (58): multilib_global: Fix multilib rebuild issue multilib_global: Fix KERNEL_VERSION expansion problems sysklogd: Fix init script races busybox: Improve syslog restart handling oeqa/runtime/syslog: Improve test debug messages oeqa/runtime/oesyslog: systemd syslog restart doesn't change pid oeqa/runtime/syslog: Add delay to test to avoid failures busybox: Fix typo in syslog initscript pigz: Add debug for autobuilder errors staging: Code cleanup package: Build pkgdata specific to the current recipe Revert "pigz: Add debug for autobuilder errors" grub2: Drop unneeded code bitbake: event: Clear ui_queue after handling it bitbake: main: Ensure log messages are printed when no UI starts bitbake: main: Alter EOFError handling core-image-sato-sdk-ptest: Reduce image padding size due to bootimg 4GB limit oeqa/bbtests: Tweak test bitbake output pattern matching sstate: Add tweak to avoid multiple sstate stats messages bitbake: siggen: Fix default handler bitbake: siggen: Use unique hashes for tasks bitbake: runqueue: Tweak buildable variable handling in scheduler bitbake: runqueue: Drop unused BB_SETSCENE_VERIFY_FUNCTION2 bitbake: runqueue: Remove now uneeded code bitbake: runqueue: Move scenequeue data generation to a separate function bitbake: runqueue: Remove unused function parameter bitbake: runqueue: Factor out the process_setscene_whitelist checks bitbake: runqueue: Uniquely namespace the scenequeue functions bitbake: runqueue: Merge stats handling together for setscene/real tasks bitbake: runqueue: Merge scenequeue and real task queue code together bitbake: runqueue: Fix counter/task updating glitch bitbake: runqueue: Remove RunQueueExecuteScenequeue and RunQueueExecuteTasks bitbake: runqueue: Simplify _execute_runqueue logic bitbake: runqueue: Fold remains of the scenequeue setup into RunQueueExecute bitbake: event/runqueue: Drop StampUpdate event, its pointless/unused bitbake: runqueue: Add covered_tasks (or 'collated_deps') to scenequeue data bitbake: runqueue: Simplify scenequeue unskippable calculation bitbake: runqueue: Tweak comments and debug code bitbake: runqueue: Code simplification bitbake: runqueue: Remove pointless variable bitbake: runqueue: Further scheduler buildable tasks cleanup bitbake: runqueue: Clarify scenequeue_covered vs. tasks_covered bitbake: runqueue: Merge the queues and execute setscene and normal tasks in parallel bitbake: runqueue: Alter setscenewhitelist handling bitbake: runqueue: Complete the merge of scenequeue and normal task execution bitbake: tests: Add initial scenario based test for runqueue bitbake: uihelper: No longer listen to scenequeue task started bitbake: runqueue: Simplify some convoluted logic bitbake: runqueue: Whitespace fix bitbake: runqueue: Abstract hash verification function bitbake: runqueue: Optimise multiconfig with overlapping setscene bitbake: tests/runqueue: Allow common sstate tasks to become valid bitbake: runqueue: Fix non setscene tasks targets being lost staging: Drop clean_recipe_sysroot poky-lsb: Drop features already in poky poky-lsb: Drop libx11 PREFERRED_PROVIDER distro/include: Add poky-distro-alt-test-config.inc bitbake: siggen: Fix handling of tainted sig files Robert Yang (13): update-alternatives.bbclass: run update-alternatives firstly in postinst script busybox: make postinst run firstly before update-alternatives multilib.bbclass: Reduce ALTERNATIVE_PRIORITY for extended recipes bitbake: bitbake: lib: Cleanup /usr/bin/env python bitbake: bitbake: toaster:tests: python -> python3 ksum.py: python -> python3 wic: python2 -> python3 ext-sdk-prepare.py: python2 -> python3 oeqa: Cleanup /usr/bin/env python package_rpm.bbclass: python2 -> python3 bitbake: cache: Remove duplicated lines for provides and rprovides bitbake: cache: Set packages for skipped recipes bitbake: cache: Create a symlink for current cachefile Ross Burton (56): cve-check: be idiomatic gtk-icon-cache: rename intercept to update_gtk_icon_cache fortran-helloworld: add a very dumb Fortran Hello World for testing oeqa/buildoptions: check that Fortran code actually cross-compiles buildhistory: write the contents of the sysroot buildhistory: report sysroot changes perl: fix Upstream-Status tags efivar: ensure that target security flags are not used to build native code multilib_script: fix whitespace buildhistory_analysis: ignore ownership for sysroot diffs insane: use clean_path for the host contamination warnings libsndfile1: disable use of sqlite3 by default libsndfile1: remove redundant autoconf seeding buildhistory: don't output ownership for the sysroot buildhistory: filter out the unexpected prefix for native/cross sysroots alsa-utils: disable tools using GTK+2 packagegroup-core-lsb: remove GTK+ recipetool: add MD5 hash for the line-wrapped MPL-1.1 license oeqa/recipetool: change the CMake test to use taglib gtk+: remove GTK+ 2 gnome-themes-standard: remove Revert "sysstat: use service file from source codes" libpsl: update Upstream-Status grub: build with python 3 qemu: use Python 3 to build ninja: use Python 3 conf/poky: add debian-10 to the supported distribution list tiff: remove redundant patch tiff: fix CVE-2019-6128 tiff: fix CVE-2019-7663 cve-check: remove redundant readline CVE whitelisting cve-check-tool: remove glibc: exclude child recipes from CVE scanning libid3tag: CVE-2017-11551 is the same as CVE-2004-2779 libid3tag: handle unknown encodings (CVE-2017-11550) subversion: set CVE vendor to Apache boost: set CVE vendor to Boost git: set CVE vendor to git-scm ed: set CVE vendor to avoid false positives cve-check: allow comparison of Vendor as well as Product flex: set CVE_PRODUCT to include vendor cve-update-db-native: use SQL placeholders instead of format strings xkeyboard-config: remove redundant intltool dependency piglit: upgrade to latest revision pkgconf: upgrade 1.6.1 -> 1.6.3 conf/poky: add Fedora 30 and Opensuse Leap 15.1 to supported distributions cve-update-db-native: use os.path.join instead of + cve-update-db: actually inherit native cve-update-db-native: use executemany() to optimise CPE insertion cve-update-db-native: improve metadata parsing cve-update-db-native: clean up JSON fetching freetype: upgrade to 2.10.1 unfs3: set upstream tag regex to avoid false-positives meson.bbclass: export STRIP=${BUILD_STRIP} ffmpeg: don't use hardcoded lookup tables ffmpeg: upgrade to 4.1.4 Sai Hari Chandana Kalluri (3): devtool/standard.py: Update devtool modify to copy source from work-shared if its already downloaded devtool/standard.py: Create a copy of kernel source within work-shared if not present devtool: provide support for devtool menuconfig command Scott Rifenbark (5): overview-manual: Fixed manual history table sdk-manual: Updated devtool to talk about oe-local-files. dev-manual: Provided proper link title ref-manual: Fixed typo for BBMULTICONFIG variable. ref-manual: Removed "python2" mention in example. Stefan Agner (1): psplash: create psplash tmpfs mount directory in psplash-init Tim Orling (3): vulkan-headers: add recipe vulkan-loader: add recipe vulkan-tools: add recipe Ulrich Ölmann (1): squashfs-tools: upgrade to commit f95864afe883 William Bourque (2): wic/plugins: Source that support both EFI and BIOS meta/lib/oeqa: Test for bootimg-biosplusefi Source Yi Zhao (2): debianutils: upgrade 4.8.6.1 -> 4.8.6.3 ltp: upgrade 20190115 -> 20190517 Zang Ruochen (9): nss: upgrade 3.44 -> 3.44.1 util-linux:upgrade 2.33.2 -> 2.34 librepo:upgrade 1.10.3 -> 1.10.4 sqlite3: Upgrade 3.28.0 -> 3.29.0 nss: Upgrade 3.44.1 -> 3.45 xauth:upgrade 1.0.10 -> 1.1 libice:upgrade 1.0.9 -> 1.0.10 xwininfo:upgrade 1.1.4 -> 1.1.5 libpciaccess:upgrade 0.14 -> 0.16 meta-phosphor: fe8cee7488..601f253a66: Brad Bishop (1): meta-phosphor: systemd: remove upstreamed patches Change-Id: If591144821cd2e5b990a7aa49a1cf426f6a906de Signed-off-by: Brad Bishop --- poky/scripts/buildstats-diff | 14 +- poky/scripts/contrib/ddimage | 89 +++++++-- poky/scripts/contrib/list-packageconfig-flags.py | 2 +- poky/scripts/lib/buildstats.py | 6 +- poky/scripts/lib/devtool/menuconfig.py | 79 ++++++++ poky/scripts/lib/devtool/standard.py | 169 +++++++++++++--- poky/scripts/lib/devtool/upgrade.py | 29 ++- poky/scripts/lib/recipetool/create.py | 5 +- .../lib/recipetool/create_buildsys_python.py | 9 +- poky/scripts/lib/wic/__init__.py | 2 +- poky/scripts/lib/wic/ksparser.py | 2 +- poky/scripts/lib/wic/pluginbase.py | 2 +- poky/scripts/lib/wic/plugins/imager/direct.py | 37 ++-- .../lib/wic/plugins/source/bootimg-biosplusefi.py | 213 +++++++++++++++++++++ poky/scripts/lib/wic/plugins/source/bootimg-efi.py | 18 +- .../lib/wic/plugins/source/bootimg-pcbios.py | 6 +- .../lib/wic/plugins/source/isoimage-isohybrid.py | 19 +- .../postinst-intercepts/update_gtk_icon_cache | 17 ++ poky/scripts/postinst-intercepts/update_icon_cache | 16 -- poky/scripts/tiny/ksum.py | 44 ++--- 20 files changed, 634 insertions(+), 144 deletions(-) create mode 100644 poky/scripts/lib/devtool/menuconfig.py create mode 100644 poky/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py create mode 100644 poky/scripts/postinst-intercepts/update_gtk_icon_cache delete mode 100644 poky/scripts/postinst-intercepts/update_icon_cache (limited to 'poky/scripts') diff --git a/poky/scripts/buildstats-diff b/poky/scripts/buildstats-diff index c179c93528..2f6498ab67 100755 --- a/poky/scripts/buildstats-diff +++ b/poky/scripts/buildstats-diff @@ -114,7 +114,7 @@ def print_ver_diff(bs1, bs2): print(fmt_str.format(name, field1, field2, maxlen=maxlen)) -def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',)): +def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',), only_tasks=[]): """Diff task execution times""" def val_to_str(val, human_readable=False): """Convert raw value to printable string""" @@ -151,8 +151,9 @@ def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absd """Get cumulative sum of all tasks""" total = 0.0 for recipe_data in buildstats.values(): - for bs_task in recipe_data.tasks.values(): - total += getattr(bs_task, val_type) + for name, bs_task in recipe_data.tasks.items(): + if not only_tasks or name in only_tasks: + total += getattr(bs_task, val_type) return total if min_val: @@ -163,7 +164,7 @@ def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absd val_to_str(min_absdiff, True), val_to_str(min_absdiff))) # Prepare the data - tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff) + tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff, only_tasks) # Sort our list for field in reversed(sort_by): @@ -248,6 +249,8 @@ Script for comparing buildstats of two separate builds.""" parser.add_argument('--multi', action='store_true', help="Read all buildstats from the given paths and " "average over them") + parser.add_argument('--only-task', dest='only_tasks', metavar='TASK', action='append', default=[], + help="Only include TASK in report. May be specified multiple times") parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat") parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat") @@ -266,7 +269,6 @@ Script for comparing buildstats of two separate builds.""" return args - def main(argv=None): """Script entry point""" args = parse_args(argv) @@ -290,7 +292,7 @@ def main(argv=None): print_ver_diff(bs1, bs2) else: print_task_diff(bs1, bs2, args.diff_attr, args.min_val, - args.min_absdiff, sort_by) + args.min_absdiff, sort_by, args.only_tasks) except ScriptError as err: log.error(str(err)) return 1 diff --git a/poky/scripts/contrib/ddimage b/poky/scripts/contrib/ddimage index b577d1ca2a..7f2ad112a6 100755 --- a/poky/scripts/contrib/ddimage +++ b/poky/scripts/contrib/ddimage @@ -3,10 +3,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Default to avoiding the first two disks on typical Linux and Mac OS installs -# Better safe than sorry :-) -BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/disk1 /dev/disk2" - # 1MB blocksize BLOCKSIZE=1048576 @@ -32,7 +28,6 @@ image_details() { } device_details() { - DEV=$1 BLOCK_SIZE=512 echo "Device details" @@ -45,11 +40,17 @@ device_details() { fi # Default / Linux information collection - echo " device: $DEVICE" + ACTUAL_DEVICE=`readlink -f $DEVICE` + DEV=`basename $ACTUAL_DEVICE` + if [ "$ACTUAL_DEVICE" != "$DEVICE" ] ; then + echo " device: $DEVICE -> $ACTUAL_DEVICE" + else + echo " device: $DEVICE" + fi if [ -f "/sys/class/block/$DEV/device/vendor" ]; then echo " vendor: $(cat /sys/class/block/$DEV/device/vendor)" else - echo " vendor: UNKOWN" + echo " vendor: UNKNOWN" fi if [ -f "/sys/class/block/$DEV/device/model" ]; then echo " model: $(cat /sys/class/block/$DEV/device/model)" @@ -64,6 +65,49 @@ device_details() { echo "" } +check_mount_device() { + if cat /proc/self/mounts | awk '{ print $1 }' | grep /dev/ | grep -q -E "^$1$" ; then + return 0 + fi + return 1 +} + +is_mounted() { + if [ "$(uname)" = "Darwin" ]; then + if df | awk '{ print $1 }' | grep /dev/ | grep -q -E "^$1(s[0-9]+)?$" ; then + return 0 + fi + else + if check_mount_device $1 ; then + return 0 + fi + DEV=`basename $1` + if [ -d /sys/class/block/$DEV/ ] ; then + PARENT_BLKDEV=`basename $(readlink -f "/sys/class/block/$DEV/..")` + if [ "$PARENT_BLKDEV" != "block" ] ; then + if check_mount_device $PARENT_BLKDEV ; then + return 0 + fi + fi + for CHILD_BLKDEV in `find /sys/class/block/$DEV/ -mindepth 1 -maxdepth 1 -name "$DEV*" -type d` + do + if check_mount_device /dev/`basename $CHILD_BLKDEV` ; then + return 0 + fi + done + fi + fi + return 1 +} + +is_inuse() { + HOLDERS_DIR="/sys/class/block/`basename $1`/holders" + if [ -d $HOLDERS_DIR ] && [ `ls -A $HOLDERS_DIR` ] ; then + return 0 + fi + return 1 +} + if [ $# -ne 2 ]; then usage exit 1 @@ -78,22 +122,37 @@ if [ ! -e "$IMAGE" ]; then exit 1 fi +if [ ! -e "$DEVICE" ]; then + echo "ERROR: Device $DEVICE does not exist" + usage + exit 1 +fi -for i in ${BLACKLIST_DEVICES}; do - if [ "$i" = "$DEVICE" ]; then - echo "ERROR: Device $DEVICE is blacklisted" - exit 1 - fi -done +if [ "$(uname)" = "Darwin" ]; then + # readlink doesn't support -f on MacOS, just assume it isn't a symlink + ACTUAL_DEVICE=$DEVICE +else + ACTUAL_DEVICE=`readlink -f $DEVICE` +fi +if is_mounted $ACTUAL_DEVICE ; then + echo "ERROR: Device $DEVICE is currently mounted - check if this is the right device, and unmount it first if so" + device_details + exit 1 +fi +if is_inuse $ACTUAL_DEVICE ; then + echo "ERROR: Device $DEVICE is currently in use (possibly part of LVM) - check if this is the right device!" + device_details + exit 1 +fi if [ ! -w "$DEVICE" ]; then - echo "ERROR: Device $DEVICE does not exist or is not writable" + echo "ERROR: Device $DEVICE is not writable - possibly use sudo?" usage exit 1 fi image_details $IMAGE -device_details $(basename $DEVICE) +device_details printf "Write $IMAGE to $DEVICE [y/N]? " read RESPONSE diff --git a/poky/scripts/contrib/list-packageconfig-flags.py b/poky/scripts/contrib/list-packageconfig-flags.py index b1d6c852d8..d6de4dc84d 100755 --- a/poky/scripts/contrib/list-packageconfig-flags.py +++ b/poky/scripts/contrib/list-packageconfig-flags.py @@ -65,7 +65,7 @@ def collect_pkgs(data_dict): for fn in data_dict: pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") pkgconfigflags.pop('doc', None) - pkgname = data_dict[fn].getVar("P") + pkgname = data_dict[fn].getVar("PN") pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) return pkg_dict diff --git a/poky/scripts/lib/buildstats.py b/poky/scripts/lib/buildstats.py index 1adab06edf..c69b5bf4d7 100644 --- a/poky/scripts/lib/buildstats.py +++ b/poky/scripts/lib/buildstats.py @@ -261,13 +261,17 @@ class BuildStats(dict): self[pkg].aggregate(data) -def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None): +def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None, only_tasks=[]): """Compare the tasks of two buildstats""" tasks_diff = [] pkgs = set(bs1.keys()).union(set(bs2.keys())) for pkg in pkgs: tasks1 = bs1[pkg].tasks if pkg in bs1 else {} tasks2 = bs2[pkg].tasks if pkg in bs2 else {} + if only_tasks: + tasks1 = {k: v for k, v in tasks1.items() if k in only_tasks} + tasks2 = {k: v for k, v in tasks2.items() if k in only_tasks} + if not tasks1: pkg_op = '+' elif not tasks2: diff --git a/poky/scripts/lib/devtool/menuconfig.py b/poky/scripts/lib/devtool/menuconfig.py new file mode 100644 index 0000000000..95384c5333 --- /dev/null +++ b/poky/scripts/lib/devtool/menuconfig.py @@ -0,0 +1,79 @@ +# OpenEmbedded Development tool - menuconfig command plugin +# +# Copyright (C) 2018 Xilinx +# Written by: Chandana Kalluri +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +"""Devtool menuconfig plugin""" + +import os +import bb +import logging +import argparse +import re +import glob +from devtool import setup_tinfoil, parse_recipe, DevtoolError, standard, exec_build_env_command +from devtool import check_workspace_recipe +logger = logging.getLogger('devtool') + +def menuconfig(args, config, basepath, workspace): + """Entry point for the devtool 'menuconfig' subcommand""" + + rd = "" + kconfigpath = "" + pn_src = "" + localfilesdir = "" + workspace_dir = "" + tinfoil = setup_tinfoil(basepath=basepath) + try: + rd = parse_recipe(config, tinfoil, args.component, appends=True, filter_workspace=False) + if not rd: + return 1 + + check_workspace_recipe(workspace, args.component) + pn = rd.getVar('PN', True) + + if not rd.getVarFlag('do_menuconfig','task'): + raise DevtoolError("This recipe does not support menuconfig option") + + workspace_dir = os.path.join(config.workspace_path,'sources') + kconfigpath = rd.getVar('B') + pn_src = os.path.join(workspace_dir,pn) + + # add check to see if oe_local_files exists or not + localfilesdir = os.path.join(pn_src,'oe-local-files') + if not os.path.exists(localfilesdir): + bb.utils.mkdirhier(localfilesdir) + # Add gitignore to ensure source tree is clean + gitignorefile = os.path.join(localfilesdir,'.gitignore') + with open(gitignorefile, 'w') as f: + f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n') + f.write('*\n') + + finally: + tinfoil.shutdown() + + logger.info('Launching menuconfig') + exec_build_env_command(config.init_path, basepath, 'bitbake -c menuconfig %s' % pn, watch=True) + fragment = os.path.join(localfilesdir, 'devtool-fragment.cfg') + res = standard._create_kconfig_diff(pn_src,rd,fragment) + + return 0 + +def register_commands(subparsers, context): + """register devtool subcommands from this plugin""" + parser_menuconfig = subparsers.add_parser('menuconfig',help='Alter build-time configuration for a recipe', description='Launches the make menuconfig command (for recipes where do_menuconfig is available), allowing users to make changes to the build-time configuration. Creates a config fragment corresponding to changes made.', group='advanced') + parser_menuconfig.add_argument('component', help='compenent to alter config') + parser_menuconfig.set_defaults(func=menuconfig,fixed_setup=context.fixed_setup) diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index aca74b1fc6..96af488798 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -461,11 +461,37 @@ def sync(args, config, basepath, workspace): finally: tinfoil.shutdown() +def symlink_oelocal_files_srctree(rd,srctree): + import oe.patch + if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): + # If recipe extracts to ${WORKDIR}, symlink the files into the srctree + # (otherwise the recipe won't build as expected) + local_files_dir = os.path.join(srctree, 'oe-local-files') + addfiles = [] + for root, _, files in os.walk(local_files_dir): + relpth = os.path.relpath(root, local_files_dir) + if relpth != '.': + bb.utils.mkdirhier(os.path.join(srctree, relpth)) + for fn in files: + if fn == '.gitignore': + continue + destpth = os.path.join(srctree, relpth, fn) + if os.path.exists(destpth): + os.unlink(destpth) + os.symlink('oe-local-files/%s' % fn, destpth) + addfiles.append(os.path.join(relpth, fn)) + if addfiles: + bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) + useroptions = [] + oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) + bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) + def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): """Extract sources of a recipe""" import oe.recipeutils import oe.patch + import oe.path pn = d.getVar('PN') @@ -562,7 +588,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works with open(preservestampfile, 'w') as f: f.write(d.getVar('STAMP')) try: - if bb.data.inherits_class('kernel-yocto', d): + if is_kernel_yocto: # We need to generate the kernel config task = 'do_configure' else: @@ -589,6 +615,23 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) + # Check if work-shared is empty, if yes + # find source and copy to work-shared + if is_kernel_yocto: + workshareddir = d.getVar('STAGING_KERNEL_DIR') + staging_kerVer = get_staging_kver(workshareddir) + kernelVersion = d.getVar('LINUX_VERSION') + + # handle dangling symbolic link in work-shared: + if os.path.islink(workshareddir): + os.unlink(workshareddir) + + if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): + shutil.rmtree(workshareddir) + oe.path.copyhardlinktree(srcsubdir,workshareddir) + elif not os.path.exists(workshareddir): + oe.path.copyhardlinktree(srcsubdir,workshareddir) + tempdir_localdir = os.path.join(tempdir, 'oe-local-files') srctree_localdir = os.path.join(srctree, 'oe-local-files') @@ -617,29 +660,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works shutil.move(tempdir_localdir, srcsubdir) shutil.move(srcsubdir, srctree) - - if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')): - # If recipe extracts to ${WORKDIR}, symlink the files into the srctree - # (otherwise the recipe won't build as expected) - local_files_dir = os.path.join(srctree, 'oe-local-files') - addfiles = [] - for root, _, files in os.walk(local_files_dir): - relpth = os.path.relpath(root, local_files_dir) - if relpth != '.': - bb.utils.mkdirhier(os.path.join(srctree, relpth)) - for fn in files: - if fn == '.gitignore': - continue - destpth = os.path.join(srctree, relpth, fn) - if os.path.exists(destpth): - os.unlink(destpth) - os.symlink('oe-local-files/%s' % fn, destpth) - addfiles.append(os.path.join(relpth, fn)) - if addfiles: - bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) - useroptions = [] - oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) - bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) + symlink_oelocal_files_srctree(d,srctree) if is_kernel_yocto: logger.info('Copying kernel config to srctree') @@ -707,11 +728,31 @@ def _check_preserve(config, recipename): tf.write(line) os.rename(newfile, origfile) +def get_staging_kver(srcdir): + # Kernel version from work-shared + kerver = [] + staging_kerVer="" + if os.path.exists(srcdir) and os.listdir(srcdir): + with open(os.path.join(srcdir,"Makefile")) as f: + version = [next(f) for x in range(5)][1:4] + for word in version: + kerver.append(word.split('= ')[1].split('\n')[0]) + staging_kerVer = ".".join(kerver) + return staging_kerVer + +def get_staging_kbranch(srcdir): + staging_kbranch = "" + if os.path.exists(srcdir) and os.listdir(srcdir): + (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) + staging_kbranch = "".join(branch.split('\n')[0]) + return staging_kbranch + def modify(args, config, basepath, workspace): """Entry point for the devtool 'modify' subcommand""" import bb import oe.recipeutils import oe.patch + import oe.path if args.recipename in workspace: raise DevtoolError("recipe %s is already in your workspace" % @@ -753,6 +794,59 @@ def modify(args, config, basepath, workspace): initial_rev = None commits = [] check_commits = False + + if bb.data.inherits_class('kernel-yocto', rd): + # Current set kernel version + kernelVersion = rd.getVar('LINUX_VERSION') + srcdir = rd.getVar('STAGING_KERNEL_DIR') + kbranch = rd.getVar('KBRANCH') + + staging_kerVer = get_staging_kver(srcdir) + staging_kbranch = get_staging_kbranch(srcdir) + if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): + oe.path.copyhardlinktree(srcdir,srctree) + workdir = rd.getVar('WORKDIR') + srcsubdir = rd.getVar('S') + localfilesdir = os.path.join(srctree,'oe-local-files') + # Move local source files into separate subdir + recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] + local_files = oe.recipeutils.get_recipe_local_files(rd) + + for key in local_files.copy(): + if key.endswith('scc'): + sccfile = open(local_files[key], 'r') + for l in sccfile: + line = l.split() + if line and line[0] in ('kconf', 'patch'): + cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) + if not cfg in local_files.values(): + local_files[line[-1]] = cfg + shutil.copy2(cfg, workdir) + sccfile.close() + + # Ignore local files with subdir={BP} + srcabspath = os.path.abspath(srcsubdir) + local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] + if local_files: + for fname in local_files: + _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) + with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: + f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') + + symlink_oelocal_files_srctree(rd,srctree) + + task = 'do_configure' + res = tinfoil.build_targets(pn, task, handle_events=True) + + # Copy .config to workspace + kconfpath = rd.getVar('B') + logger.info('Copying kernel config to workspace') + shutil.copy2(os.path.join(kconfpath, '.config'),srctree) + + # Set this to true, we still need to get initial_rev + # by parsing the git repo + args.no_extract = True + if not args.no_extract: initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) if not initial_rev: @@ -844,6 +938,11 @@ def modify(args, config, basepath, workspace): ' cp ${B}/.config ${S}/.config.baseline\n' ' ln -sfT ${B}/.config ${S}/.config.new\n' '}\n') + if rd.getVarFlag('do_menuconfig','task'): + f.write('\ndo_configure_append() {\n' + ' cp ${B}/.config ${S}/.config.baseline\n' + ' ln -sfT ${B}/.config ${S}/.config.new\n' + '}\n') if initial_rev: f.write('\n# initial_rev: %s\n' % initial_rev) for commit in commits: @@ -1866,13 +1965,27 @@ def reset(args, config, basepath, workspace): def _get_layer(layername, d): """Determine the base layer path for the specified layer name/path""" layerdirs = d.getVar('BBLAYERS').split() - layers = {os.path.basename(p): p for p in layerdirs} + layers = {} # {basename: layer_paths} + for p in layerdirs: + bn = os.path.basename(p) + if bn not in layers: + layers[bn] = [p] + else: + layers[bn].append(p) # Provide some shortcuts if layername.lower() in ['oe-core', 'openembedded-core']: - layerdir = layers.get('meta', None) + layername = 'meta' + layer_paths = layers.get(layername, None) + if not layer_paths: + return os.path.abspath(layername) + elif len(layer_paths) == 1: + return os.path.abspath(layer_paths[0]) else: - layerdir = layers.get(layername, None) - return os.path.abspath(layerdir or layername) + # multiple layers having the same base name + logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) + logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) + return os.path.abspath(layer_paths[0]) + def finish(args, config, basepath, workspace): """Entry point for the devtool 'finish' subcommand""" diff --git a/poky/scripts/lib/devtool/upgrade.py b/poky/scripts/lib/devtool/upgrade.py index 62ec2f94cb..18c5b66a29 100644 --- a/poky/scripts/lib/devtool/upgrade.py +++ b/poky/scripts/lib/devtool/upgrade.py @@ -122,18 +122,22 @@ def _cleanup_on_error(rf, srctree): rfp = os.path.split(rf)[0] # recipe folder rfpp = os.path.split(rfp)[0] # recipes folder if os.path.exists(rfp): - shutil.rmtree(b) + shutil.rmtree(rfp) if not len(os.listdir(rfpp)): os.rmdir(rfpp) srctree = os.path.abspath(srctree) if os.path.exists(srctree): shutil.rmtree(srctree) -def _upgrade_error(e, rf, srctree): - if rf: - cleanup_on_error(rf, srctree) +def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None): + if rf and not keep_failure: + _cleanup_on_error(rf, srctree) logger.error(e) - raise DevtoolError(e) + if extramsg: + logger.error(extramsg) + if keep_failure: + logger.info('Preserving failed upgrade files (--keep-failure)') + sys.exit(1) def _get_uri(rd): srcuris = rd.getVar('SRC_URI').split() @@ -277,6 +281,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) + shutil.rmtree(tmpdir) return (rev, md5, sha256, srcbranch, srcsubdir_rel) @@ -299,7 +304,7 @@ def _add_license_diff_to_recipe(path, diff): f.write("\n#\n\n".encode()) f.write(orig_content) -def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses): +def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): """Creates the new recipe under workspace""" bpn = rd.getVar('BPN') @@ -416,7 +421,10 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src newvalues["LIC_FILES_CHKSUM"] = newlicchksum _add_license_diff_to_recipe(fullpath, license_diff) - rd = tinfoil.parse_recipe_file(fullpath, False) + try: + rd = tinfoil.parse_recipe_file(fullpath, False) + except bb.tinfoil.TinfoilCommandFailed as e: + _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed') oe.recipeutils.patch_recipe(rd, fullpath, newvalues) return fullpath, copied @@ -548,11 +556,11 @@ def upgrade(args, config, basepath, workspace): tinfoil, rd) new_licenses = _extract_licenses(srctree, rd.getVar('LIC_FILES_CHKSUM')) license_diff = _generate_license_diff(old_licenses, new_licenses) - rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses) + rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) except bb.process.CmdError as e: - _upgrade_error(e, rf, srctree) + _upgrade_error(e, rf, srctree, args.keep_failure) except DevtoolError as e: - _upgrade_error(e, rf, srctree) + _upgrade_error(e, rf, srctree, args.keep_failure) standard._add_md5(config, pn, os.path.dirname(rf)) af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2, @@ -623,6 +631,7 @@ def register_commands(subparsers, context): group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') + parser_upgrade.add_argument('--keep-failure', action="store_true", help='Keep failed upgrade recipe and associated files (for debugging)') parser_upgrade.set_defaults(func=upgrade, fixed_setup=context.fixed_setup) parser_latest_version = subparsers.add_parser('latest-version', help='Report the latest version of an existing recipe', diff --git a/poky/scripts/lib/recipetool/create.py b/poky/scripts/lib/recipetool/create.py index 98277f74c9..1fb6b55530 100644 --- a/poky/scripts/lib/recipetool/create.py +++ b/poky/scripts/lib/recipetool/create.py @@ -60,7 +60,9 @@ class RecipeHandler(object): if RecipeHandler.recipelibmap: return # First build up library->package mapping - shlib_providers = oe.package.read_shlib_providers(d) + d2 = bb.data.createCopy(d) + d2.setVar("WORKDIR_PKGDATA", "${PKGDATA_DIR}") + shlib_providers = oe.package.read_shlib_providers(d2) libdir = d.getVar('libdir') base_libdir = d.getVar('base_libdir') libpaths = list(set([base_libdir, libdir])) @@ -1053,6 +1055,7 @@ def get_license_md5sums(d, static_only=False): md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0' md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause' + md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1' return md5sums def crunch_license(licfile): diff --git a/poky/scripts/lib/recipetool/create_buildsys_python.py b/poky/scripts/lib/recipetool/create_buildsys_python.py index ac9bc9237c..adfa377956 100644 --- a/poky/scripts/lib/recipetool/create_buildsys_python.py +++ b/poky/scripts/lib/recipetool/create_buildsys_python.py @@ -154,8 +154,13 @@ class PythonRecipeHandler(RecipeHandler): if 'buildsystem' in handled: return False - if not RecipeHandler.checkfiles(srctree, ['setup.py']): - return + # Check for non-zero size setup.py files + setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) + for fn in setupfiles: + if os.path.getsize(fn): + break + else: + return False # setup.py is always parsed to get at certain required information, such as # distutils vs setuptools diff --git a/poky/scripts/lib/wic/__init__.py b/poky/scripts/lib/wic/__init__.py index ba2d614061..85567934ae 100644 --- a/poky/scripts/lib/wic/__init__.py +++ b/poky/scripts/lib/wic/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -tt +#!/usr/bin/env python3 # # Copyright (c) 2007 Red Hat, Inc. # Copyright (c) 2011 Intel, Inc. diff --git a/poky/scripts/lib/wic/ksparser.py b/poky/scripts/lib/wic/ksparser.py index 62048213b3..3e670033ba 100644 --- a/poky/scripts/lib/wic/ksparser.py +++ b/poky/scripts/lib/wic/ksparser.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -tt +#!/usr/bin/env python3 # # Copyright (c) 2016 Intel, Inc. # diff --git a/poky/scripts/lib/wic/pluginbase.py b/poky/scripts/lib/wic/pluginbase.py index bfb73ca615..f74d6430fd 100644 --- a/poky/scripts/lib/wic/pluginbase.py +++ b/poky/scripts/lib/wic/pluginbase.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python -tt +#!/usr/bin/env python3 # # Copyright (c) 2011 Intel, Inc. # diff --git a/poky/scripts/lib/wic/plugins/imager/direct.py b/poky/scripts/lib/wic/plugins/imager/direct.py index 67addeff3e..91fc5e70f6 100644 --- a/poky/scripts/lib/wic/plugins/imager/direct.py +++ b/poky/scripts/lib/wic/plugins/imager/direct.py @@ -49,7 +49,6 @@ class DirectPlugin(ImagerPlugin): # parse possible 'rootfs=name' items self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) - self.replaced_rootfs_paths = {} self.bootimg_dir = bootimg_dir self.kernel_dir = kernel_dir self.native_sysroot = native_sysroot @@ -59,6 +58,7 @@ class DirectPlugin(ImagerPlugin): self.compressor = options.compressor self.bmap = options.bmap self.no_fstab_update = options.no_fstab_update + self.original_fstab = None self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], strftime("%Y%m%d%H%M")) @@ -104,24 +104,13 @@ class DirectPlugin(ImagerPlugin): with open(fstab_path) as fstab: fstab_lines = fstab.readlines() + self.original_fstab = fstab_lines.copy() if self._update_fstab(fstab_lines, self.parts): - # copy rootfs dir to workdir to update fstab - # as rootfs can be used by other tasks and can't be modified - new_pseudo = os.path.realpath(os.path.join(self.workdir, "pseudo")) - from_dir = os.path.join(os.path.join(image_rootfs, ".."), "pseudo") - from_dir = os.path.realpath(from_dir) - copyhardlinktree(from_dir, new_pseudo) - new_rootfs = os.path.realpath(os.path.join(self.workdir, "rootfs_copy")) - copyhardlinktree(image_rootfs, new_rootfs) - fstab_path = os.path.join(new_rootfs, 'etc/fstab') - - os.unlink(fstab_path) - with open(fstab_path, "w") as fstab: fstab.writelines(fstab_lines) - - return new_rootfs + else: + self.original_fstab = None def _update_fstab(self, fstab_lines, parts): """Assume partition order same as in wks""" @@ -170,14 +159,8 @@ class DirectPlugin(ImagerPlugin): filesystems from the artifacts directly and combine them into a partitioned image. """ - if self.no_fstab_update: - new_rootfs = None - else: - new_rootfs = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) - if new_rootfs: - # rootfs was copied to update fstab - self.replaced_rootfs_paths[new_rootfs] = self.rootfs_dir['ROOTFS_DIR'] - self.rootfs_dir['ROOTFS_DIR'] = new_rootfs + if not self.no_fstab_update: + self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) for part in self.parts: # get rootfs size from bitbake variable if it's not set in .ks file @@ -253,8 +236,6 @@ class DirectPlugin(ImagerPlugin): else: suffix = '["%s"]:' % (part.mountpoint or part.label) rootdir = part.rootfs_dir - if rootdir in self.replaced_rootfs_paths: - rootdir = self.replaced_rootfs_paths[rootdir] msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir @@ -292,6 +273,12 @@ class DirectPlugin(ImagerPlugin): if os.path.isfile(path): shutil.move(path, os.path.join(self.outdir, fname)) + #Restore original fstab + if self.original_fstab: + fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab" + with open(fstab_path, "w") as fstab: + fstab.writelines(self.original_fstab) + # remove work directory shutil.rmtree(self.workdir, ignore_errors=True) diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py b/poky/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py new file mode 100644 index 0000000000..5bd7390680 --- /dev/null +++ b/poky/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py @@ -0,0 +1,213 @@ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION +# This implements the 'bootimg-biosplusefi' source plugin class for 'wic' +# +# AUTHORS +# William Bourque + +import types + +from wic.pluginbase import SourcePlugin +from importlib.machinery import SourceFileLoader + +class BootimgBiosPlusEFIPlugin(SourcePlugin): + """ + Create MBR + EFI boot partition + + This plugin creates a boot partition that contains both + legacy BIOS and EFI content. It will be able to boot from both. + This is useful when managing PC fleet with some older machines + without EFI support. + + Note it is possible to create an image that can boot from both + legacy BIOS and EFI by defining two partitions : one with arg + --source bootimg-efi and another one with --source bootimg-pcbios. + However, this method has the obvious downside that it requires TWO + partitions to be created on the storage device. + Both partitions will also be marked as "bootable" which does not work on + most BIOS, has BIOS often uses the "bootable" flag to determine + what to boot. If you have such a BIOS, you need to manually remove the + "bootable" flag from the EFI partition for the drive to be bootable. + Having two partitions also seems to confuse wic : the content of + the first partition will be duplicated into the second, even though it + will not be used at all. + + Also, unlike "isoimage-isohybrid" that also does BIOS and EFI, this plugin + allows you to have more than only a single rootfs partitions and does + not turn the rootfs into an initramfs RAM image. + + This plugin is made to put everything into a single /boot partition so it + does not have the limitations listed above. + + The plugin is made so it does tries not to reimplement what's already + been done in other plugins; as such it imports "bootimg-pcbios" + and "bootimg-efi". + Plugin "bootimg-pcbios" is used to generate legacy BIOS boot. + Plugin "bootimg-efi" is used to generate the UEFI boot. Note that it + requires a --sourceparams argument to know which loader to use; refer + to "bootimg-efi" code/documentation for the list of loader. + + Imports are handled with "SourceFileLoader" from importlib as it is + otherwise very difficult to import module that has hyphen "-" in their + filename. + The SourcePlugin() methods used in the plugins (do_install_disk, + do_configure_partition, do_prepare_partition) are then called on both, + beginning by "bootimg-efi". + + Plugin options, such as "--sourceparams" can still be passed to a + plugin, as long they does not cause issue in the other plugin. + + Example wic configuration: + part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi"\\ + --ondisk sda --label os_boot --active --align 1024 --use-uuid + """ + + name = 'bootimg-biosplusefi' + + __PCBIOS_MODULE_NAME = "bootimg-pcbios" + __EFI_MODULE_NAME = "bootimg-efi" + + __imgEFIObj = None + __imgBiosObj = None + + @classmethod + def __init__(cls): + """ + Constructor (init) + """ + + # XXX + # For some reasons, __init__ constructor is never called. + # Something to do with how pluginbase works? + cls.__instanciateSubClasses() + + @classmethod + def __instanciateSubClasses(cls): + """ + + """ + + # Import bootimg-pcbios (class name "BootimgPcbiosPlugin") + modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)), + cls.__PCBIOS_MODULE_NAME + ".py") + loader = SourceFileLoader(cls.__PCBIOS_MODULE_NAME, modulePath) + mod = types.ModuleType(loader.name) + loader.exec_module(mod) + cls.__imgBiosObj = mod.BootimgPcbiosPlugin() + + # Import bootimg-efi (class name "BootimgEFIPlugin") + modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)), + cls.__EFI_MODULE_NAME + ".py") + loader = SourceFileLoader(cls.__EFI_MODULE_NAME, modulePath) + mod = types.ModuleType(loader.name) + loader.exec_module(mod) + cls.__imgEFIObj = mod.BootimgEFIPlugin() + + @classmethod + def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir, + bootimg_dir, kernel_dir, native_sysroot): + """ + Called after all partitions have been prepared and assembled into a + disk image. + """ + + if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ): + cls.__instanciateSubClasses() + + cls.__imgEFIObj.do_install_disk( + disk, + disk_name, + creator, + workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + native_sysroot) + + cls.__imgBiosObj.do_install_disk( + disk, + disk_name, + creator, + workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + native_sysroot) + + @classmethod + def do_configure_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + native_sysroot): + """ + Called before do_prepare_partition() + """ + + if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ): + cls.__instanciateSubClasses() + + cls.__imgEFIObj.do_configure_partition( + part, + source_params, + creator, + cr_workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + native_sysroot) + + cls.__imgBiosObj.do_configure_partition( + part, + source_params, + creator, + cr_workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + native_sysroot) + + @classmethod + def do_prepare_partition(cls, part, source_params, creator, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + + if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ): + cls.__instanciateSubClasses() + + cls.__imgEFIObj.do_prepare_partition( + part, + source_params, + creator, + cr_workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + rootfs_dir, + native_sysroot) + + cls.__imgBiosObj.do_prepare_partition( + part, + source_params, + creator, + cr_workdir, + oe_builddir, + bootimg_dir, + kernel_dir, + rootfs_dir, + native_sysroot) diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py index 5cc5c8a6b8..2cfdc10ecd 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -74,8 +74,10 @@ class BootimgEFIPlugin(SourcePlugin): grubefi_conf += "menuentry '%s'{\n" % (title if title else "boot") kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) label = source_params.get('label') label_conf = "root=%s" % creator.rootdev @@ -154,8 +156,10 @@ class BootimgEFIPlugin(SourcePlugin): if not custom_cfg: # Create systemd-boot configuration using parameters from wks file kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) title = source_params.get('title') @@ -225,8 +229,10 @@ class BootimgEFIPlugin(SourcePlugin): hdddir = "%s/hdd/boot" % cr_workdir kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) install_cmd = "install -m 0644 %s/%s %s/%s" % \ (staging_kernel_dir, kernel, hdddir, kernel) diff --git a/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py index 670d347747..f2639e7004 100644 --- a/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -150,8 +150,10 @@ class BootimgPcbiosPlugin(SourcePlugin): hdddir = "%s/hdd/boot" % cr_workdir kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) cmds = ("install -m 0644 %s/%s %s/vmlinuz" % (staging_kernel_dir, kernel, hdddir), diff --git a/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index 74d6f14519..24299c1ece 100644 --- a/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -71,8 +71,11 @@ class IsoImagePlugin(SourcePlugin): syslinux_conf += "LABEL boot\n" kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) + syslinux_conf += "KERNEL /" + kernel + "\n" syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \ % bootloader.append @@ -117,8 +120,10 @@ class IsoImagePlugin(SourcePlugin): grubefi_conf += "menuentry 'boot'{\n" kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) grubefi_conf += "linux /%s rootwait %s\n" \ % (kernel, bootloader.append) @@ -273,8 +278,10 @@ class IsoImagePlugin(SourcePlugin): os.remove("%s/initrd.cpio.gz" % cr_workdir) kernel = get_bitbake_var("KERNEL_IMAGETYPE") - if not kernel: - kernel = "bzImage" + if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1": + if get_bitbake_var("INITRAMFS_IMAGE"): + kernel = "%s-%s.bin" % \ + (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) install_cmd = "install -m 0644 %s/%s %s/%s" % \ (kernel_dir, kernel, isodir, kernel) diff --git a/poky/scripts/postinst-intercepts/update_gtk_icon_cache b/poky/scripts/postinst-intercepts/update_gtk_icon_cache new file mode 100644 index 0000000000..99367a2855 --- /dev/null +++ b/poky/scripts/postinst-intercepts/update_gtk_icon_cache @@ -0,0 +1,17 @@ +#!/bin/sh +# +# SPDX-License-Identifier: MIT +# +# Post-install intercept for gtk-icon-cache.bbclass + +set -e + +# Update native pixbuf loaders +$STAGING_DIR_NATIVE/${libdir_native}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache + +for icondir in $D/usr/share/icons/*/ ; do + if [ -d $icondir ] ; then + gtk-update-icon-cache -fqt $icondir + fi +done + diff --git a/poky/scripts/postinst-intercepts/update_icon_cache b/poky/scripts/postinst-intercepts/update_icon_cache deleted file mode 100644 index 212209a9da..0000000000 --- a/poky/scripts/postinst-intercepts/update_icon_cache +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# -# SPDX-License-Identifier: MIT -# - -set -e - -# update native pixbuf loaders -$STAGING_DIR_NATIVE/${libdir_native}/gdk-pixbuf-2.0/gdk-pixbuf-query-loaders --update-cache - -for icondir in $D/usr/share/icons/*/ ; do - if [ -d $icondir ] ; then - gtk-update-icon-cache -fqt $icondir - fi -done - diff --git a/poky/scripts/tiny/ksum.py b/poky/scripts/tiny/ksum.py index f38ae055db..8f0e4c0517 100755 --- a/poky/scripts/tiny/ksum.py +++ b/poky/scripts/tiny/ksum.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (c) 2016, Intel Corporation. # @@ -62,7 +62,7 @@ def is_ko_file(filename): return False def collect_object_files(): - print "Collecting object files recursively from %s..." % os.getcwd() + print("Collecting object files recursively from %s..." % os.getcwd()) for dirpath, dirs, files in os.walk(os.getcwd()): for filename in files: if is_ko_file(filename): @@ -70,7 +70,7 @@ def collect_object_files(): elif is_vmlinux_file(filename): global vmlinux_file vmlinux_file = os.path.join(dirpath, filename) - print "Collecting object files [DONE]" + print("Collecting object files [DONE]") def add_ko_file(filename): p = Popen("size -t " + filename, shell=True, stdout=PIPE, stderr=PIPE) @@ -78,9 +78,9 @@ def add_ko_file(filename): if len(output) > 2: sizes = output[-1].split()[0:4] if verbose: - print " %10d %10d %10d %10d\t" % \ - (int(sizes[0]), int(sizes[1]), int(sizes[2]), int(sizes[3])), - print "%s" % filename[len(os.getcwd()) + 1:] + print(" %10d %10d %10d %10d\t" % \ + (int(sizes[0]), int(sizes[1]), int(sizes[2]), int(sizes[3])), end=' ') + print("%s" % filename[len(os.getcwd()) + 1:]) global n_ko_files, ko_text, ko_data, ko_bss, ko_total ko_text += int(sizes[0]) ko_data += int(sizes[1]) @@ -94,9 +94,9 @@ def get_vmlinux_totals(): if len(output) > 2: sizes = output[-1].split()[0:4] if verbose: - print " %10d %10d %10d %10d\t" % \ - (int(sizes[0]), int(sizes[1]), int(sizes[2]), int(sizes[3])), - print "%s" % vmlinux_file[len(os.getcwd()) + 1:] + print(" %10d %10d %10d %10d\t" % \ + (int(sizes[0]), int(sizes[1]), int(sizes[2]), int(sizes[3])), end=' ') + print("%s" % vmlinux_file[len(os.getcwd()) + 1:]) global vmlinux_text, vmlinux_data, vmlinux_bss, vmlinux_total vmlinux_text += int(sizes[0]) vmlinux_data += int(sizes[1]) @@ -129,20 +129,20 @@ def main(): sum_ko_files() get_vmlinux_totals() - print "\nTotals:" - print "\nvmlinux:" - print " text\tdata\t\tbss\t\ttotal" - print " %-10d\t%-10d\t%-10d\t%-10d" % \ - (vmlinux_text, vmlinux_data, vmlinux_bss, vmlinux_total) - print "\nmodules (%d):" % n_ko_files - print " text\tdata\t\tbss\t\ttotal" - print " %-10d\t%-10d\t%-10d\t%-10d" % \ - (ko_text, ko_data, ko_bss, ko_total) - print "\nvmlinux + modules:" - print " text\tdata\t\tbss\t\ttotal" - print " %-10d\t%-10d\t%-10d\t%-10d" % \ + print("\nTotals:") + print("\nvmlinux:") + print(" text\tdata\t\tbss\t\ttotal") + print(" %-10d\t%-10d\t%-10d\t%-10d" % \ + (vmlinux_text, vmlinux_data, vmlinux_bss, vmlinux_total)) + print("\nmodules (%d):" % n_ko_files) + print(" text\tdata\t\tbss\t\ttotal") + print(" %-10d\t%-10d\t%-10d\t%-10d" % \ + (ko_text, ko_data, ko_bss, ko_total)) + print("\nvmlinux + modules:") + print(" text\tdata\t\tbss\t\ttotal") + print(" %-10d\t%-10d\t%-10d\t%-10d" % \ (vmlinux_text + ko_text, vmlinux_data + ko_data, \ - vmlinux_bss + ko_bss, vmlinux_total + ko_total) + vmlinux_bss + ko_bss, vmlinux_total + ko_total)) if __name__ == "__main__": try: -- cgit v1.2.3