poky: refresh thud: b904775c2b..7c76c5d78b

Update poky to thud HEAD.

Adam Trhon (1):
      icecc-env: don't raise error when icecc not installed

Alexander Kanavin (1):
      openssl10: update to 1.0.2q

Armin Kuster (1):
      perl: add testdepends for ssh

Bruce Ashfield (2):
      linux-yocto/4.18: update to v4.18.26
      linux-yocto/4.18: update to v4.18.27

Changqing Li (1):
      checklayer: generate locked-sigs.inc under builddir

Dan Dedrick (2):
      devtool: remove duplicate overrides
      devtool: improve git repo checks before check_commits logic

Daniel Ammann (1):
      ref-manual: Typo found and fixed.

Douglas Royds (2):
      openssl ptest: Strip build host paths from configdata.pm
      openssl: Strip perl version from installed ptest configdata.pm file

Dustin Bain (1):
      busybox: update to 1.29.3

Jan Kiszka (1):
      oe-git-proxy: Avoid resolving NO_PROXY against local files

Jens Rehsack (1):
      avahi: avoid depending on skipped package

Jonas Bonn (1):
      keymaps: tighten package write dependency

Kai Kang (1):
      selftest/wic: update test case test_qemu

Khem Raj (3):
      openssl10: Fix mutliple include assumptions for bn.h in opensslconf.h
      send-error-report: Use https instead of http protocol
      multilib_header_wrapper.h: Use #pragma once

Leonardo Augusto (1):
      scripts/lib/wic/engine: Fix cp's target path for ext* filesystems

Liu Haitao (1):
      iw: fix parsing of WEP keys

Mingli Yu (1):
      logrotate.py: restore /etc/logrotate.d/wtmp

Otavio Salvador (1):
      linux-firmware: Bump to 710963f revision

Ovidiu Panait (1):
      ghostscript: Fix CVE-2019-6116

Peter Kjellerstedt (1):
      libaio: Extend to native

Richard Purdie (23):
      package: Add pkg_postinst_ontarget to PACKAGEVARS
      oeqa/runtime/ptest: Avoid traceback for tests with no section
      oeqa/utils/logparser: Simplify ptest log parsing code
      oeqa/logparser: Further simplification/clarification
      oeqa/logparser: Reform the ptest results parser
      oeqa/utils/logparser: Add in support for duration, exitcode and logs by section
      oeqa/logparser: Improve results handling
      oeqa/logparser: Various misc cleanups
      oeqa/runtime/ptest: Ensure OOM errors are logged
      scripts/contrib/build-perf-test-wrapper.sh: Improve interaction with autobuilder automation
      scripts/contrib/build-perf-test.sh: Remove it
      oe-build-perf-report: Allow branch without hostname
      oe-build-perf-report: Allow commits from different branches
      oe-build-perf-report: Improve branch comparision handling
      oe-build-perf-report: Fix missing buildstats comparisions
      wic/engine: Fix missing parted autobuilder failures
      lib/buildstats: Improve error message
      scripts/oe-git-archive: Separate out functionality to library function
      oe-build-perf-report/gitarchive: Move common useful functions to library
      bitbake: runqueue: Fix dependency loop analysis 'hangs'
      bitbake: runqueue: Filter out multiconfig dependencies from BB_TASKDEPDATA
      bitbake: siggen: Fix multiconfig corner case
      bitbake: cooker: Tweak multiconfig dependency resolution

Robert Yang (5):
      bluez5: Fix a race issue for tools
      yocto-check-layer-wrapper: Fix path for oe-init-build-env
      checklayer: Avoid adding the layer if it is already present
      runqemu: Let qemuparams override default settings
      runqemu: Make QB_MEM easier to set

Ross Burton (3):
      e2fsprogs: fix file system generation with large files
      linux-firmware: recommend split up packages
      linux-firmware: split out liquidio firmware

Scott Rifenbark (2):
      poky.ent: Updated "meta-intel" version to "10.1"
      overview-manual, mega-manual: Updated Package Feeds diagram

Serhey Popovych (1):
      openssl: Skip assembler optimized code for powerpc64 with musl

William Bourque (1):
      wic/engine.py: Load paths from PATH environment variable

Xulin Sun (1):
      openssl: fix multilib file install conflicts

Zheng Ruoqin (1):
      mdadm: add init and service scripts

Change-Id: Ib14c2fb69d25d84aa3d4bf0a6715bba57d1eb900
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>

2 files changed, 32 insertions, 405 deletions

diff --git a/poky/scripts/contrib/build-perf-test-wrapper.sh b/poky/scripts/contrib/build-perf-test-wrapper.sh
index 19bee1dd0..7cbb5d794 100755
--- a/poky/scripts/contrib/build-perf-test-wrapper.sh
+++ b/poky/scripts/contrib/build-perf-test-wrapper.sh
@@ -33,7 +33,9 @@ Optional arguments:
   -c COMMITISH      test (checkout) this commit, <branch>:<commit> can be
                     specified to test specific commit of certain branch
   -C GIT_REPO       commit results into Git
+  -d DOWNLOAD_DIR   directory to store downloaded sources in
   -E EMAIL_ADDR     send email report
+  -g GLOBALRES_DIR  where to place the globalres file
   -P GIT_REMOTE     push results to a remote Git repository
   -R DEST           rsync reports to a remote destination
   -w WORK_DIR       work dir for this script
@@ -51,19 +53,26 @@ get_os_release_var () {
 commitish=""
 oe_build_perf_test_extra_opts=()
 oe_git_archive_extra_opts=()
-while getopts "ha:c:C:E:P:R:w:x" opt; do
+while getopts "ha:c:C:d:E:g:P:R:w:x" opt; do
     case $opt in
         h)  usage
             exit 0
             ;;
-        a)  archive_dir=`realpath -s "$OPTARG"`
+        a)  mkdir -p "$OPTARG"
+            archive_dir=`realpath -s "$OPTARG"`
             ;;
         c)  commitish=$OPTARG
             ;;
-        C)  results_repo=`realpath -s "$OPTARG"`
+        C)  mkdir -p "$OPTARG"
+            results_repo=`realpath -s "$OPTARG"`
+            ;;
+        d)  download_dir=`realpath -s "$OPTARG"`
             ;;
         E)  email_to="$OPTARG"
             ;;
+        g)  mkdir -p "$OPTARG"
+            globalres_dir=`realpath -s "$OPTARG"`
+            ;;
         P)  oe_git_archive_extra_opts+=("--push" "$OPTARG")
             ;;
         R)  rsync_dst="$OPTARG"
@@ -86,6 +95,17 @@ if [ $# -ne 0 ]; then
     exit 1
 fi
 
+if [ -n "$email_to" ]; then
+    if ! [ -x "$(command -v phantomjs)" ]; then
+        echo "ERROR: Sending email needs phantomjs."
+        exit 1
+    fi
+    if ! [ -x "$(command -v optipng)" ]; then
+        echo "ERROR: Sending email needs optipng."
+        exit 1
+    fi
+fi
+
 # Open a file descriptor for flock and acquire lock
 LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock"
 if ! exec 3> "$LOCK_FILE"; then
@@ -146,11 +166,18 @@ if [ -z "$base_dir" ]; then
 fi
 echo "Using working dir $base_dir"
 
+if [ -z "$download_dir" ]; then
+    download_dir="$base_dir/downloads"
+fi
+if [ -z "$globalres_dir" ]; then
+    globalres_dir="$base_dir"
+fi
+
 timestamp=`date "+%Y%m%d%H%M%S"`
 git_rev=$(git rev-parse --short HEAD)  || exit 1
 build_dir="$base_dir/build-$git_rev-$timestamp"
 results_dir="$base_dir/results-$git_rev-$timestamp"
-globalres_log="$base_dir/globalres.log"
+globalres_log="$globalres_dir/globalres.log"
 machine="qemux86"
 
 mkdir -p "$base_dir"
@@ -161,7 +188,7 @@ auto_conf="$build_dir/conf/auto.conf"
 echo "MACHINE = \"$machine\"" > "$auto_conf"
 echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf"
 echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf"
-echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf"
+echo "DL_DIR = \"$download_dir\"" >> "$auto_conf"
 # Disabling network sanity check slightly reduces the variance of timing results
 echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf"
 # Possibility to define extra settings
diff --git a/poky/scripts/contrib/build-perf-test.sh b/poky/scripts/contrib/build-perf-test.sh
deleted file mode 100755
index 9a091edb0..000000000
--- a/poky/scripts/contrib/build-perf-test.sh
+++ /dev/null
@@ -1,400 +0,0 @@
-#!/bin/bash
-#
-# This script runs a series of tests  (with and without sstate) and reports build time (and tmp/ size)
-# 
-# Build performance test script
-#
-# Copyright 2013 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-#
-# AUTHORS:
-# Stefan Stanacar <stefanx.stanacar@intel.com>
-
-
-ME=$(basename $0)
-
-#
-# usage and setup
-#
-
-usage () {
-cat << EOT
-Usage: $ME [-h]
-       $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>]
-Options:
-        -h
-                Display this help and exit.
-        -c <commit>
-                git checkout <commit> before anything else
-        -v
-                Show bitbake output, don't redirect it to a log.
-        -m <machine>
-                Value for MACHINE. Default is qemux86.
-        -j <val>
-                Value for PARALLEL_MAKE. Default is 8. 
-        -t <val>
-                Value for BB_NUMBER_THREADS. Default is 8.
-        -i <image-name>
-                Instead of timing against core-image-sato, use <image-name>
-        -d <path>
-                Use <path> as DL_DIR
-        -p <githash>
-                Cherry pick githash onto the commit
-                
-Note: current working directory must be inside a poky git clone.
- 
-EOT
-}
-
-
-if clonedir=$(git rev-parse --show-toplevel); then
-        cd $clonedir
-else
-        echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME"
-        exit 1
-fi
-
-IMAGE="core-image-sato"
-verbose=0
-dldir=
-commit=
-pmake=
-cherrypicks=
-while getopts "hvc:m:j:t:i:d:p:" opt; do
-        case $opt in
-                h)      usage
-                        exit 0
-                        ;;
-                v)      verbose=1
-                        ;;
-                c)      commit=$OPTARG
-                        ;;
-                m)      export MACHINE=$OPTARG
-                        ;;
-                j)      pmake=$OPTARG
-                        ;;
-                t)      export BB_NUMBER_THREADS=$OPTARG
-                        ;;
-                i)      IMAGE=$OPTARG
-                        ;;
-                d)      dldir=$OPTARG
-                        ;;
-                p)      cherrypicks="$cherrypicks $OPTARG"
-                        ;;
-                *)      usage
-                        exit 1
-                        ;;                      
-        esac
-done
-
-
-#drop cached credentials and test for sudo access without a password
-sudo -k -n ls > /dev/null 2>&1
-reqpass=$?
-if [ $reqpass -ne 0 ]; then
-    echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)"
-    read -s -p "Please enter your sudo password: " pass
-    echo
-fi
-
-if [ -n "$commit" ]; then
-            echo "git checkout -f $commit"
-            git pull > /dev/null 2>&1
-            git checkout -f $commit || exit 1
-            git pull > /dev/null 2>&1
-fi
-
-if [ -n "$cherrypicks" ]; then
-    for c in $cherrypicks; do
-        git cherry-pick $c
-    done
-fi
-
-rev=$(git rev-parse --short HEAD)  || exit 1
-OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
-BUILDDIR="$OUTDIR/build"
-resultsfile="$OUTDIR/results.log"
-cmdoutput="$OUTDIR/commands.log"
-myoutput="$OUTDIR/output.log"
-globalres="$clonedir/build-perf-test/globalres.log"
-
-mkdir -p $OUTDIR || exit 1
-                                
-log () {
-    local msg="$1"
-    echo "`date`: $msg" | tee -a $myoutput
-}
-
-
-#
-# Config stuff
-#
-
-branch=`git branch 2>&1 | grep "^* " | tr -d "* "`
-gitcommit=$(git rev-parse HEAD)  || exit 1
-log "Running on $branch:$gitcommit"
-
-source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1
-cd $OUTDIR/build
-
-[ -n "$MACHINE" ] || export MACHINE="qemux86"
-[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8"
-
-if [ -n "$pmake" ]; then
-        export PARALLEL_MAKE="-j $pmake"
-else
-        export PARALLEL_MAKE="-j 8"
-fi
-
-if [ -n "$dldir" ]; then
-    echo "DL_DIR = \"$dldir\"" >> conf/local.conf
-else
-    echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf
-fi
-
-# Sometimes I've noticed big differences in timings for the same commit, on the same machine
-# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
-echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf
-
-
-#
-# Functions
-#
-
-declare -a TIMES
-time_count=0
-declare -a SIZES
-size_count=0
-
-time_cmd () {
-    log "   Timing: $*"
-
-    if [ $verbose -eq 0 ]; then 
-        /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
-    else
-        /usr/bin/time -v -o $resultsfile "$@"
-    fi
-    ret=$?
-    if [ $ret -eq 0 ]; then
-        t=`grep wall $resultsfile | sed 's/.*m:ss): //'`
-        log "   TIME: $t"
-        TIMES[(( time_count++ ))]="$t"
-    else
-        log "ERROR: exit status was non-zero, will report time as 0."
-        TIMES[(( time_count++ ))]="0"
-    fi
-    
-    #time by default overwrites the output file and we  want to keep the results
-    #it has an append option but I don't want to clobber the results in the same file
-    i=`ls $OUTDIR/results.log* |wc -l`
-    mv $resultsfile "${resultsfile}.${i}"
-    log "More stats can be found in ${resultsfile}.${i}"    
-}
-
-bbtime () {
-    time_cmd bitbake "$@"
-}
-
-#we don't time bitbake here
-bbnotime () {
-    local arg="$@"
-    log "   Running: bitbake ${arg}"
-    if [ $verbose -eq 0 ]; then
-        bitbake ${arg} >> $cmdoutput
-    else
-        bitbake ${arg}
-    fi
-    ret=$?
-    if [ $ret -eq 0 ]; then
-        log "   Finished bitbake ${arg}"
-    else
-        log "ERROR: exit status was non-zero. Exit.."
-        exit $ret
-    fi
-
-}
-
-do_rmtmp() {
-    log "   Removing tmp"
-    rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp
-}
-do_rmsstate () {
-    log "   Removing sstate-cache"
-    rm -rf sstate-cache
-}
-do_sync () {
-    log "   Syncing and dropping caches"
-    sync; sync
-    if [ $reqpass -eq 0 ]; then
-        sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
-    else
-        echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches"
-        echo
-    fi
-    sleep 3
-}
-
-write_results() {
-    echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres
-    for i in "${TIMES[@]}"; do
-        echo -n "$i," >> $globalres
-    done
-    for i in "${SIZES[@]}"; do
-        echo -n "$i," >> $globalres
-    done
-    echo >> $globalres
-    sed -i '$ s/,$//' $globalres
-}
-
-####
-
-#
-# Test 1
-# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work)
-# Pre: Downloaded sources, no sstate
-# Steps:
-#     Part1:
-#        - fetchall 
-#        - clean build dir
-#        - time bitbake core-image-sato
-#        - collect data
-#     Part2:
-#        - bitbake virtual/kernel -c cleansstate
-#        - time bitbake virtual/kernel
-#     Part3:
-#        - add INHERIT to local.conf
-#        - clean build dir
-#        - build
-#        - report size, remove INHERIT
-
-test1_p1 () {
-    log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir"
-    bbnotime $IMAGE --runall=fetch
-    do_rmtmp
-    do_rmsstate
-    do_sync
-    bbtime $IMAGE
-    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
-    SIZES[(( size_count++ ))]="$s"
-    log "SIZE of tmp dir is: $s"
-    log "Buildstats are saved in $OUTDIR/buildstats-test1"
-    mv tmp/buildstats $OUTDIR/buildstats-test1
-}
-
-
-test1_p2 () {
-    log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel"
-    bbnotime virtual/kernel -c cleansstate
-    do_sync
-    bbtime virtual/kernel
-}
-
-test1_p3 () {
-    log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled"
-    echo "INHERIT += \"rm_work\"" >> conf/local.conf
-    do_rmtmp
-    do_rmsstate
-    do_sync
-    bbtime $IMAGE
-    sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
-    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
-    SIZES[(( size_count++ ))]="$s"
-    log "SIZE of tmp dir is: $s"
-    log "Buildstats are saved in $OUTDIR/buildstats-test13"
-    mv tmp/buildstats $OUTDIR/buildstats-test13
-}
-
-
-#
-# Test 2
-# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir
-# Pre: populated sstate cache
-
-test2 () {
-    # Assuming test 1 has run
-    log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate"
-    do_rmtmp
-    do_sync
-    bbtime $IMAGE -c rootfs
-}
-
-
-# Test 3
-# parsing time metrics
-#
-#  Start with
-#   i) "rm -rf tmp/cache; time bitbake -p"
-#  ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
-# iii) "time bitbake -p"
-
-
-test3 () {
-    log "Running Test 3: Parsing time metrics (bitbake -p)"
-    log "   Removing tmp/cache && cache"
-    rm -rf tmp/cache cache
-    bbtime -p
-    log "   Removing tmp/cache/default-glibc/"
-    rm -rf tmp/cache/default-glibc/
-    bbtime -p
-    bbtime -p
-}
-
-#
-# Test 4 - eSDK
-# Measure: eSDK size and installation time
-test4 () {
-    log "Running Test 4: eSDK size and installation time"
-    bbnotime $IMAGE -c do_populate_sdk_ext
-
-    esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh)
-
-    if [ ${#esdk_installer[*]} -eq 1 ]; then
-        s=$((`stat -c %s "$esdk_installer"` / 1024))
-        SIZES[(( size_count++ ))]="$s"
-        log "Download SIZE of eSDK is: $s kB"
-
-        do_sync
-        time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"
-
-        s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024))
-        SIZES[(( size_count++ ))]="$s"
-        log "Install SIZE of eSDK is: $s kB"
-    else
-        log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
-        SIZES[(( size_count++ ))]="0"
-        TIMES[(( time_count++ ))]="0"
-    fi
-
-}
-
-
-# RUN!
-
-test1_p1
-test1_p2
-test1_p3
-test2
-test3
-test4
-
-# if we got til here write to global results
-write_results
-
-log "All done, cleaning up..."
-
-do_rmtmp
-do_rmsstate