diff options
author | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-02-01 18:27:11 +0300 |
---|---|---|
committer | Brad Bishop <bradleyb@fuzziesquirrel.com> | 2018-03-13 05:51:39 +0300 |
commit | 6e60e8b2b2bab889379b380a28a167a0edd9d1d3 (patch) | |
tree | f12f54d5ba8e74e67e5fad3651a1e125bb8f4191 /import-layers/yocto-poky/scripts/contrib | |
parent | 509842add85b53e13164c1569a1fd43d5b8d91c5 (diff) | |
download | openbmc-6e60e8b2b2bab889379b380a28a167a0edd9d1d3.tar.xz |
Yocto 2.3
Move OpenBMC to Yocto 2.3(pyro).
Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I50744030e771f4850afc2a93a10d3507e76d36bc
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
Resolves: openbmc/openbmc#2461
Diffstat (limited to 'import-layers/yocto-poky/scripts/contrib')
10 files changed, 759 insertions, 78 deletions
diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh new file mode 100755 index 000000000..7e8ae0410 --- /dev/null +++ b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats-plot.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# +# Produces script data to be consumed by gnuplot. There are two possible plots +# depending if either the -S parameter is present or not: +# +# * without -S: Produces a histogram listing top N recipes/tasks versus +# stats. The first stat defined in the -s parameter is the one taken +# into account for ranking +# * -S: Produces a histogram listing tasks versus stats. In this case, +# the value of each stat is the sum for that particular stat in all recipes found. +# Stats values are in descending order defined by the first stat defined on -s +# +# EXAMPLES +# +# 1. Top recipes' tasks taking into account utime +# +# $ buildstats-plot.sh -s utime | gnuplot -p +# +# 2. Tasks versus utime:stime +# +# $ buildstats-plot.sh -s utime:stime -S | gnuplot -p +# +# 3. Tasks versus IO write_bytes:IO read_bytes +# +# $ buildstats-plot.sh -s 'IO write_bytes:IO read_bytes' -S | gnuplot -p +# +# AUTHORS +# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> +# + +set -o nounset +set -o errexit + +BS_DIR="tmp/buildstats" +N=10 +STATS="utime" +SUM="" +OUTDATA_FILE="$PWD/buildstats-plot.out" + +function usage { + CMD=$(basename $0) + cat <<EOM +Usage: $CMD [-b buildstats_dir] [-t do_task] + -b buildstats The path where the folder resides + (default: "$BS_DIR") + -n N Top N recipes to display. Ignored if -S is present + (default: "$N") + -s stats The stats to be matched. If more that one stat, units + should be the same because data is plot as histogram. + (see buildstats.sh -h for all options) or any other defined + (build)stat separated by colons, i.e. stime:utime + (default: "$STATS") + -S Sum values for a particular stat for found recipes + -o Output data file. + (default: "$OUTDATA_FILE") + -h Display this help message +EOM +} + +# Parse and validate arguments +while getopts "b:n:s:o:Sh" OPT; do + case $OPT in + b) + BS_DIR="$OPTARG" + ;; + n) + N="$OPTARG" + ;; + s) + STATS="$OPTARG" + ;; + S) + SUM="y" + ;; + o) + OUTDATA_FILE="$OPTARG" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Get number of stats +IFS=':'; statsarray=(${STATS}); unset IFS +nstats=${#statsarray[@]} + +# Get script folder, use to run buildstats.sh +CD=$(dirname $0) + +# Parse buildstats recipes to produce a single table +OUTBUILDSTATS="$PWD/buildstats.log" +$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS + +# Get headers +HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') + +echo -e "set boxwidth 0.9 relative" +echo -e "set style data histograms" +echo -e "set style fill solid 1.0 border lt -1" +echo -e "set xtics rotate by 45 right" + +# Get output data +if [ -z "$SUM" ]; then + cat $OUTBUILDSTATS | sed -e '1d' | sort -k3 -n -r | head -$N > $OUTDATA_FILE + # include task at recipe column + sed -i -e "1i\ +${HEADERS}" $OUTDATA_FILE + echo -e "set title \"Top task/recipes\"" + echo -e "plot for [COL=3:`expr 3 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(stringcolumn(1).' '.stringcolumn(2)) title columnheader(COL)" +else + + # Construct datatamash sum argument (sum 3 sum 4 ...) + declare -a sumargs + j=0 + for i in `seq $nstats`; do + sumargs[j]=sum; j=$(( $j + 1 )) + sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 )) + done + + # Do the processing with datamash + cat $OUTBUILDSTATS | sed -e '1d' | datamash -t ' ' -g1 ${sumargs[*]} | sort -k2 -n -r > $OUTDATA_FILE + + # Include headers into resulted file, so we can include gnuplot xtics + HEADERS=$(echo $HEADERS | sed -e 's/recipe//1') + sed -i -e "1i\ +${HEADERS}" $OUTDATA_FILE + + # Plot + echo -e "set title \"Sum stats values per task for all recipes\"" + echo -e "plot for [COL=2:`expr 2 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(1) title columnheader(COL)" +fi + diff --git a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh index 96158a965..8d7e2488f 100755 --- a/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh +++ b/import-layers/yocto-poky/scripts/contrib/bb-perf/buildstats.sh @@ -18,24 +18,40 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # DESCRIPTION -# Given a 'buildstats' path (created by bitbake when setting -# USER_CLASSES ?= "buildstats" on local.conf) and task names, outputs -# '<task> <recipe> <elapsed time>' for all recipes. Elapsed times are in -# seconds, and task should be given without the 'do_' prefix. +# Given 'buildstats' data (generate by bitbake when setting +# USER_CLASSES ?= "buildstats" on local.conf), task names and a stats values +# (these are the ones preset on the buildstats files), outputs +# '<task> <recipe> <value_1> <value_2> ... <value_n>'. The units are the ones +# defined at buildstats, which in turn takes data from /proc/[pid] files # # Some useful pipelines # -# 1. Tasks with largest elapsed times -# $ buildstats.sh -b <buildstats> | sort -k3 -n -r | head +# 1. Tasks with largest stime (Amount of time that this process has been scheduled +# in kernel mode) values +# $ buildstats.sh -b <buildstats> -s stime | sort -k3 -n -r | head # -# 2. Min, max, sum per task (in needs GNU datamash) -# $ buildstats.sh -b <buildstats> | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r +# 2. Min, max, sum utime (Amount of time that this process has been scheduled +# in user mode) per task (in needs GNU datamash) +# $ buildstats.sh -b <buildstats> -s utime | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r # # AUTHORS # Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> # + +# Stats, by type +TIME="utime:stime:cutime:cstime" +IO="IO wchar:IO write_bytes:IO syscr:IO read_bytes:IO rchar:IO syscw:IO cancelled_write_bytes" +RUSAGE="rusage ru_utime:rusage ru_stime:rusage ru_maxrss:rusage ru_minflt:rusage ru_majflt:\ +rusage ru_inblock:rusage ru_oublock:rusage ru_nvcsw:rusage ru_nivcsw" + +CHILD_RUSAGE="Child rusage ru_utime:Child rusage ru_stime:Child rusage ru_maxrss:Child rusage ru_minflt:\ +Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rusage ru_nvcsw:\ +Child rusage ru_nivcsw" + BS_DIR="tmp/buildstats" TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" +STATS="$TIME" +HEADER="" # No header by default function usage { CMD=$(basename $0) @@ -45,12 +61,20 @@ Usage: $CMD [-b buildstats_dir] [-t do_task] (default: "$BS_DIR") -t tasks The tasks to be computed (default: "$TASKS") + -s stats The stats to be matched. Options: TIME, IO, RUSAGE, CHILD_RUSAGE + or any other defined buildstat separated by colons, i.e. stime:utime + (default: "$STATS") + Default stat sets: + TIME=$TIME + IO=$IO + RUSAGE=$RUSAGE + CHILD_RUSAGE=$CHILD_RUSAGE -h Display this help message EOM } # Parse and validate arguments -while getopts "b:t:h" OPT; do +while getopts "b:t:s:Hh" OPT; do case $OPT in b) BS_DIR="$OPTARG" @@ -58,6 +82,12 @@ while getopts "b:t:h" OPT; do t) TASKS="$OPTARG" ;; + s) + STATS="$OPTARG" + ;; + H) + HEADER="y" + ;; h) usage exit 0 @@ -76,15 +106,50 @@ if [ ! -d "$BS_DIR" ]; then exit 1 fi -RECIPE_FIELD=1 -TIME_FIELD=4 +stats="" +IFS=":" +for stat in ${STATS}; do + case $stat in + TIME) + stats="${stats}:${TIME}" + ;; + IO) + stats="${stats}:${IO}" + ;; + RUSAGE) + stats="${stats}:${RUSAGE}" + ;; + CHILD_RUSAGE) + stats="${stats}:${CHILD_RUSAGE}" + ;; + *) + stats="${STATS}" + esac +done + +# remove possible colon at the beginning +stats="$(echo "$stats" | sed -e 's/^://1')" + +# Provide a header if required by the user +[ -n "$HEADER" ] && { echo "task:recipe:$stats"; } -tasks=(${TASKS//:/ }) -for task in "${tasks[@]}"; do +for task in ${TASKS}; do task="do_${task}" - for file in $(find ${BS_DIR} -type f -name ${task}); do - recipe=$(sed -n -e "/$task/p" ${file} | cut -d ':' -f${RECIPE_FIELD}) - time=$(sed -n -e "/$task/p" ${file} | cut -d ':' -f${TIME_FIELD} | cut -d ' ' -f2) - echo "${task} ${recipe} ${time}" + for file in $(find ${BS_DIR} -type f -name ${task} | awk 'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); do + recipe="$(basename $(dirname $file))" + times="" + for stat in ${stats}; do + [ -z "$stat" ] && { echo "empty stats"; } + time=$(sed -n -e "s/^\($stat\): \\(.*\\)/\\2/p" $file) + # in case the stat is not present, set the value as NA + [ -z "$time" ] && { time="NA"; } + # Append it to times + if [ -z "$times" ]; then + times="${time}" + else + times="${times} ${time}" + fi + done + echo "${task} ${recipe} ${times}" done done diff --git a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh index e03ea978b..3da32532b 100755 --- a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh +++ b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh @@ -19,6 +19,7 @@ # oe-build-perf-test and archives the results. script=`basename $0` +script_dir=$(realpath $(dirname $0)) archive_dir=~/perf-results/archives usage () { @@ -29,29 +30,44 @@ Optional arguments: -h show this help and exit. -a ARCHIVE_DIR archive results tarball here, give an empty string to disable tarball archiving (default: $archive_dir) - -c COMMITISH test (checkout) this commit + -c COMMITISH test (checkout) this commit, <branch>:<commit> can be + specified to test specific commit of certain branch -C GIT_REPO commit results into Git + -E EMAIL_ADDR send email report + -P GIT_REMOTE push results to a remote Git repository -w WORK_DIR work dir for this script (default: GIT_TOP_DIR/build-perf-test) + -x create xml report (instead of json) EOF } +get_os_release_var () { + ( source /etc/os-release; eval echo '$'$1 ) +} + # Parse command line arguments commitish="" -while getopts "ha:c:C:w:" opt; do +oe_build_perf_test_extra_opts=() +oe_git_archive_extra_opts=() +while getopts "ha:c:C:E:P:w:x" opt; do case $opt in h) usage exit 0 ;; - a) archive_dir=`realpath "$OPTARG"` + a) archive_dir=`realpath -s "$OPTARG"` ;; c) commitish=$OPTARG ;; - C) results_repo=`realpath "$OPTARG"` - commit_results=("--commit-results" "$results_repo") + C) results_repo=`realpath -s "$OPTARG"` + ;; + E) email_to="$OPTARG" ;; - w) base_dir=`realpath "$OPTARG"` + P) oe_git_archive_extra_opts+=("--push" "$OPTARG") + ;; + w) base_dir=`realpath -s "$OPTARG"` + ;; + x) oe_build_perf_test_extra_opts+=("--xml") ;; *) usage exit 1 @@ -67,6 +83,17 @@ if [ $# -ne 0 ]; then exit 1 fi +# Open a file descriptor for flock and acquire lock +LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" +if ! exec 3> "$LOCK_FILE"; then + echo "ERROR: Unable to open lock file" + exit 1 +fi +if ! flock -n 3; then + echo "ERROR: Another instance of this script is running" + exit 1 +fi + echo "Running on `uname -n`" if ! git_topdir=$(git rev-parse --show-toplevel); then echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`" @@ -76,15 +103,33 @@ fi cd "$git_topdir" if [ -n "$commitish" ]; then - # Checkout correct revision - echo "Checking out $commitish" + echo "Running git fetch" git fetch &> /dev/null git checkout HEAD^0 &> /dev/null - git branch -D $commitish &> /dev/null - if ! git checkout -f $commitish &> /dev/null; then - echo "Git checkout failed" + + # Handle <branch>:<commit> format + if echo "$commitish" | grep -q ":"; then + commit=`echo "$commitish" | cut -d":" -f2` + branch=`echo "$commitish" | cut -d":" -f1` + else + commit="$commitish" + branch="$commitish" + fi + + echo "Checking out $commitish" + git branch -D $branch &> /dev/null + if ! git checkout -f $branch &> /dev/null; then + echo "ERROR: Git checkout failed" exit 1 fi + + # Check that the specified branch really contains the commit + commit_hash=`git rev-parse --revs-only $commit --` + if [ -z "$commit_hash" -o "`git merge-base $branch $commit`" != "$commit_hash" ]; then + echo "ERROR: branch $branch does not contain commit $commit" + exit 1 + fi + git reset --hard $commit > /dev/null fi # Setup build environment @@ -119,10 +164,8 @@ fi # Run actual test script oe-build-perf-test --out-dir "$results_dir" \ --globalres-file "$globalres_log" \ - --lock-file "$base_dir/oe-build-perf.lock" \ - "${commit_results[@]}" \ - --commit-results-branch "{tester_host}/{git_branch}/$machine" \ - --commit-results-tag "{tester_host}/{git_branch}/$machine/{git_commit_count}-g{git_commit}/{tag_num}" + "${oe_build_perf_test_extra_opts[@]}" \ + --lock-file "$base_dir/oe-build-perf.lock" case $? in 1) echo "ERROR: oe-build-perf-test script failed!" @@ -132,6 +175,29 @@ case $? in ;; esac +# Commit results to git +if [ -n "$results_repo" ]; then + echo -e "\nArchiving results in $results_repo" + oe-git-archive \ + --git-dir "$results_repo" \ + --branch-name "{hostname}/{branch}/{machine}" \ + --tag-name "{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}" \ + --exclude "buildstats.json" \ + --notes "buildstats/{branch_name}" "$results_dir/buildstats.json" \ + "${oe_git_archive_extra_opts[@]}" \ + "$results_dir" + + # Send email report + if [ -n "$email_to" ]; then + echo -e "\nEmailing test report" + os_name=`get_os_release_var PRETTY_NAME` + oe-build-perf-report -r "$results_repo" > report.txt + oe-build-perf-report -r "$results_repo" --html > report.html + "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text report.txt --html report.html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" + fi +fi + + echo -ne "\n\n-----------------\n" echo "Global results file:" echo -ne "\n" diff --git a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py b/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py index 389fb97f6..7ce718624 100755 --- a/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py +++ b/import-layers/yocto-poky/scripts/contrib/list-packageconfig-flags.py @@ -76,7 +76,7 @@ def collect_pkgs(data_dict): for fn in data_dict: pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") pkgconfigflags.pop('doc', None) - pkgname = data_dict[fn].getVar("P", True) + pkgname = data_dict[fn].getVar("P") pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) return pkg_dict @@ -124,9 +124,9 @@ def display_all(data_dict): ''' Display all pkgs and PACKAGECONFIG information ''' print(str("").ljust(50, '=')) for fn in data_dict: - print('%s' % data_dict[fn].getVar("P", True)) + print('%s' % data_dict[fn].getVar("P")) print(fn) - packageconfig = data_dict[fn].getVar("PACKAGECONFIG", True) or '' + packageconfig = data_dict[fn].getVar("PACKAGECONFIG") or '' if packageconfig.strip() == '': packageconfig = 'None' print('PACKAGECONFIG %s' % packageconfig) diff --git a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh b/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh index d8db3c016..ac4ec9c7f 100755 --- a/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh +++ b/import-layers/yocto-poky/scripts/contrib/mkefidisk.sh @@ -20,6 +20,11 @@ LANG=C +echo +echo "WARNING: This script is deprecated and will be removed soon." +echo "Please consider using wic EFI images instead." +echo + # Set to 1 to enable additional output DEBUG=0 OUT="/dev/null" @@ -379,7 +384,7 @@ EFIDIR="$BOOTFS_MNT/EFI/BOOT" cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy vmlinuz" # Copy the efi loader and configs (booti*.efi and grub.cfg if it exists) cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy EFI dir" -# Silently ignore a missing gummiboot loader dir (we might just be a GRUB image) +# Silently ignore a missing systemd-boot loader dir (we might just be a GRUB image) cp -r $HDDIMG_MNT/loader $BOOTFS_MNT >$OUT 2>&1 # Update the boot loaders configurations for an installed image @@ -405,25 +410,25 @@ if [ -e "$GRUB_CFG" ]; then sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GRUB_CFG fi -# Look for a gummiboot installation -GUMMI_ENTRIES="$BOOTFS_MNT/loader/entries" -GUMMI_CFG="$GUMMI_ENTRIES/boot.conf" -if [ -d "$GUMMI_ENTRIES" ]; then - info "Configuring Gummiboot" +# Look for a systemd-boot installation +SYSTEMD_BOOT_ENTRIES="$BOOTFS_MNT/loader/entries" +SYSTEMD_BOOT_CFG="$SYSTEMD_BOOT_ENTRIES/boot.conf" +if [ -d "$SYSTEMD_BOOT_ENTRIES" ]; then + info "Configuring SystemD-boot" # remove the install target if it exists - rm $GUMMI_ENTRIES/install.conf >$OUT 2>&1 + rm $SYSTEMD_BOOT_ENTRIES/install.conf >$OUT 2>&1 - if [ ! -e "$GUMMI_CFG" ]; then - echo "ERROR: $GUMMI_CFG not found" + if [ ! -e "$SYSTEMD_BOOT_CFG" ]; then + echo "ERROR: $SYSTEMD_BOOT_CFG not found" fi - sed -i "/initrd /d" $GUMMI_CFG - sed -i "s@ root=[^ ]*@ @" $GUMMI_CFG - sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GUMMI_CFG + sed -i "/initrd /d" $SYSTEMD_BOOT_CFG + sed -i "s@ root=[^ ]*@ @" $SYSTEMD_BOOT_CFG + sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $SYSTEMD_BOOT_CFG fi # Ensure we have at least one EFI bootloader configured -if [ ! -e $GRUB_CFG ] && [ ! -e $GUMMI_CFG ]; then +if [ ! -e $GRUB_CFG ] && [ ! -e $SYSTEMD_BOOT_CFG ]; then die "No EFI bootloader configuration found" fi @@ -439,7 +444,7 @@ if [ -d $ROOTFS_MNT/etc/udev/ ] ; then fi # Add startup.nsh script for automated boot -echo "fs0:\EFI\BOOT\bootx64.efi" > $BOOTFS_MNT/startup.nsh +printf "fs0:\%s\BOOT\%s\n" "EFI" "bootx64.efi" > $BOOTFS_MNT/startup.nsh # Call cleanup to unmount devices and images and remove the TMPDIR diff --git a/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py b/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py new file mode 100755 index 000000000..261ca514e --- /dev/null +++ b/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py @@ -0,0 +1,269 @@ +#!/usr/bin/python3 +# +# Send build performance test report emails +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +import argparse +import base64 +import logging +import os +import pwd +import re +import shutil +import smtplib +import socket +import subprocess +import sys +import tempfile +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + + +# Setup logging +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +log = logging.getLogger('oe-build-perf-report') + + +# Find js scaper script +SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', + 'scrape-html-report.js') +if not os.path.isfile(SCRAPE_JS): + log.error("Unableto find oe-build-perf-report-scrape.js") + sys.exit(1) + + +class ReportError(Exception): + """Local errors""" + pass + + +def check_utils(): + """Check that all needed utils are installed in the system""" + missing = [] + for cmd in ('phantomjs', 'optipng'): + if not shutil.which(cmd): + missing.append(cmd) + if missing: + log.error("The following tools are missing: %s", ' '.join(missing)) + sys.exit(1) + + +def parse_args(argv): + """Parse command line arguments""" + description = """Email build perf test report""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=description) + + parser.add_argument('--debug', '-d', action='store_true', + help="Verbose logging") + parser.add_argument('--quiet', '-q', action='store_true', + help="Only print errors") + parser.add_argument('--to', action='append', + help="Recipients of the email") + parser.add_argument('--subject', default="Yocto build perf test report", + help="Email subject") + parser.add_argument('--outdir', '-o', + help="Store files in OUTDIR. Can be used to preserve " + "the email parts") + parser.add_argument('--text', + help="Plain text message") + parser.add_argument('--html', + help="HTML peport generated by oe-build-perf-report") + parser.add_argument('--phantomjs-args', action='append', + help="Extra command line arguments passed to PhantomJS") + + args = parser.parse_args(argv) + + if not args.html and not args.text: + parser.error("Please specify --html and/or --text") + + return args + + +def decode_png(infile, outfile): + """Parse/decode/optimize png data from a html element""" + with open(infile) as f: + raw_data = f.read() + + # Grab raw base64 data + b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) + b64_data = re.sub('">.+$', '', b64_data, 1) + + # Replace file with proper decoded png + with open(outfile, 'wb') as f: + f.write(base64.b64decode(b64_data)) + + subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) + + +def encode_png(pngfile): + """Encode png into a <img> html element""" + with open(pngfile, 'rb') as f: + data = f.read() + + b64_data = base64.b64encode(data) + return '<img src="data:image/png;base64,' + b64_data.decode('utf-8') + '">\n' + + +def mangle_html_report(infile, outfile, pngs): + """Mangle html file into a email compatible format""" + paste = True + png_dir = os.path.dirname(outfile) + with open(infile) as f_in: + with open(outfile, 'w') as f_out: + for line in f_in.readlines(): + stripped = line.strip() + # Strip out scripts + if stripped == '<!--START-OF-SCRIPTS-->': + paste = False + elif stripped == '<!--END-OF-SCRIPTS-->': + paste = True + elif paste: + if re.match('^.+href="data:image/png;base64', stripped): + # Strip out encoded pngs (as they're huge in size) + continue + elif 'www.gstatic.com' in stripped: + # HACK: drop references to external static pages + continue + + # Replace charts with <img> elements + match = re.match('<div id="(?P<id>\w+)"', stripped) + if match and match.group('id') in pngs: + #f_out.write('<img src="{}">\n'.format(match.group('id') + '.png')) + png_file = os.path.join(png_dir, match.group('id') + '.png') + f_out.write(encode_png(png_file)) + else: + f_out.write(line) + + +def scrape_html_report(report, outdir, phantomjs_extra_args=None): + """Scrape html report into a format sendable by email""" + tmpdir = tempfile.mkdtemp(dir='.') + log.debug("Using tmpdir %s for phantomjs output", tmpdir) + + if not os.path.isdir(outdir): + os.mkdir(outdir) + if os.path.splitext(report)[1] not in ('.html', '.htm'): + raise ReportError("Invalid file extension for report, needs to be " + "'.html' or '.htm'") + + try: + log.info("Scraping HTML report with PhangomJS") + extra_args = phantomjs_extra_args if phantomjs_extra_args else [] + subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + + [SCRAPE_JS, report, tmpdir], + stderr=subprocess.STDOUT) + + pngs = [] + attachments = [] + for fname in os.listdir(tmpdir): + base, ext = os.path.splitext(fname) + if ext == '.png': + log.debug("Decoding %s", fname) + decode_png(os.path.join(tmpdir, fname), + os.path.join(outdir, fname)) + pngs.append(base) + attachments.append(fname) + elif ext in ('.html', '.htm'): + report_file = fname + else: + log.warning("Unknown file extension: '%s'", ext) + #shutil.move(os.path.join(tmpdir, fname), outdir) + + log.debug("Mangling html report file %s", report_file) + mangle_html_report(os.path.join(tmpdir, report_file), + os.path.join(outdir, report_file), pngs) + return report_file, attachments + finally: + shutil.rmtree(tmpdir) + +def send_email(text_fn, html_fn, subject, recipients): + """Send email""" + # Generate email message + text_msg = html_msg = None + if text_fn: + with open(text_fn) as f: + text_msg = MIMEText("Yocto build performance test report.\n" + + f.read(), 'plain') + if html_fn: + with open(html_fn) as f: + html_msg = MIMEText(f.read(), 'html') + + if text_msg and html_msg: + msg = MIMEMultipart('alternative') + msg.attach(text_msg) + msg.attach(html_msg) + elif text_msg: + msg = text_msg + elif html_msg: + msg = html_msg + else: + raise ReportError("Neither plain text nor html body specified") + + pw_data = pwd.getpwuid(os.getuid()) + full_name = pw_data.pw_gecos.split(',')[0] + email = os.environ.get('EMAIL', + '{}@{}'.format(pw_data.pw_name, socket.getfqdn())) + msg['From'] = "{} <{}>".format(full_name, email) + msg['To'] = ', '.join(recipients) + msg['Subject'] = subject + + # Send email + with smtplib.SMTP('localhost') as smtp: + smtp.send_message(msg) + + +def main(argv=None): + """Script entry point""" + args = parse_args(argv) + if args.quiet: + log.setLevel(logging.ERROR) + if args.debug: + log.setLevel(logging.DEBUG) + + check_utils() + + if args.outdir: + outdir = args.outdir + if not os.path.exists(outdir): + os.mkdir(outdir) + else: + outdir = tempfile.mkdtemp(dir='.') + + try: + log.debug("Storing email parts in %s", outdir) + html_report = None + if args.html: + scrape_html_report(args.html, outdir, args.phantomjs_args) + html_report = os.path.join(outdir, os.path.basename(args.html)) + + if args.to: + log.info("Sending email to %s", ', '.join(args.to)) + send_email(args.text, html_report, args.subject, args.to) + except subprocess.CalledProcessError as err: + log.error("%s, with output:\n%s", str(err), err.output.decode()) + return 1 + except ReportError as err: + log.error(err) + return 1 + finally: + if not args.outdir: + log.debug("Wiping %s", outdir) + shutil.rmtree(outdir) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py index f2ecf8d3f..8c3655d39 100755 --- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py +++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py @@ -9,10 +9,14 @@ # * Updated to no longer generate special -dbg package, instead use the # single system -dbg # * Update version with ".1" to indicate this change +# +# February 26, 2017 -- Ming Liu <peter.x.liu@external.atlascopco.com> +# * Updated to support generating manifest for native python import os import sys import time +import argparse VERSION = "2.7.2" @@ -21,16 +25,16 @@ __version__ = "20110222.2" class MakefileMaker: - def __init__( self, outfile ): + def __init__( self, outfile, isNative ): """initialize""" self.packages = {} self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.isNative = isNative self.output = outfile self.out( """ # WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. -# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> -# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy -""" % ( sys.argv[0], __version__ ) ) +# Generator: '%s%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +""" % ( sys.argv[0], ' --native' if isNative else '', __version__ ) ) # # helper functions @@ -66,6 +70,20 @@ class MakefileMaker: global VERSION # + # generate rprovides line for native + # + + if self.isNative: + rprovideLine = 'RPROVIDES+="' + for name in sorted(self.packages): + rprovideLine += "%s-native " % name.replace( '${PN}', 'python' ) + rprovideLine += '"' + + self.out( rprovideLine ) + self.out( "" ) + return + + # # generate provides line # @@ -147,17 +165,21 @@ class MakefileMaker: self.doEpilog() if __name__ == "__main__": + parser = argparse.ArgumentParser( description='generate python manifest' ) + parser.add_argument( '-n', '--native', help='generate manifest for native python', action='store_true' ) + parser.add_argument( 'outfile', metavar='OUTPUT_FILE', nargs='?', default='', help='Output file (defaults to stdout)' ) + args = parser.parse_args() - if len( sys.argv ) > 1: + if args.outfile: try: - os.unlink(sys.argv[1]) + os.unlink( args.outfile ) except Exception: sys.exc_clear() - outfile = open( sys.argv[1], "w" ) + outfile = open( args.outfile, "w" ) else: outfile = sys.stdout - m = MakefileMaker( outfile ) + m = MakefileMaker( outfile, args.native ) # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! # Parameters: revision, name, description, dependencies, filenames @@ -171,7 +193,7 @@ if __name__ == "__main__": "UserDict.* UserList.* UserString.* " + "lib-dynload/binascii.so lib-dynload/_struct.so lib-dynload/time.so " + "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* " + - "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "_weakrefset.* sysconfig.* _sysconfigdata.* " + "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h " + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") @@ -185,7 +207,8 @@ if __name__ == "__main__": "${base_libdir}/*.a " + "${base_libdir}/*.o " + "${datadir}/aclocal " + - "${datadir}/pkgconfig " ) + "${datadir}/pkgconfig " + + "config/Makefile ") m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", "${bindir}/2to3 lib2to3" ) # package @@ -360,7 +383,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", "pty.* tty.*" ) - m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-modules", "test" ) # package m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py index 71a71f7af..075860c41 100755 --- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py +++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py @@ -13,9 +13,13 @@ # 2014 Khem Raj <raj.khem@gmail.com> # Added python3 support # +# February 26, 2017 -- Ming Liu <peter.x.liu@external.atlascopco.com> +# * Updated to support generating manifest for native python3 + import os import sys import time +import argparse VERSION = "3.5.0" @@ -24,16 +28,16 @@ __version__ = "20140131" class MakefileMaker: - def __init__( self, outfile ): + def __init__( self, outfile, isNative ): """initialize""" self.packages = {} self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.isNative = isNative self.output = outfile self.out( """ # WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. -# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> -# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy -""" % ( sys.argv[0], __version__ ) ) +# Generator: '%s%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +""" % ( sys.argv[0], ' --native' if isNative else '', __version__ ) ) # # helper functions @@ -59,16 +63,40 @@ class MakefileMaker: for filename in filenames: if filename[0] != "$": fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + fullFilenames.append( "%s%s" % ( self.targetPrefix, + self.pycachePath( filename ) ) ) else: fullFilenames.append( filename ) self.packages[name] = description, dependencies, fullFilenames + def pycachePath( self, filename ): + dirname = os.path.dirname( filename ) + basename = os.path.basename( filename ) + if '.' in basename: + return os.path.join( dirname, '__pycache__', basename ) + else: + return os.path.join( dirname, basename, '__pycache__' ) + def doBody( self ): """generate body of Makefile""" global VERSION # + # generate rprovides line for native + # + + if self.isNative: + rprovideLine = 'RPROVIDES+="' + for name in sorted(self.packages): + rprovideLine += "%s-native " % name.replace( '${PN}', 'python3' ) + rprovideLine += '"' + + self.out( rprovideLine ) + self.out( "" ) + return + + # # generate provides line # @@ -150,17 +178,21 @@ class MakefileMaker: self.doEpilog() if __name__ == "__main__": + parser = argparse.ArgumentParser( description='generate python3 manifest' ) + parser.add_argument( '-n', '--native', help='generate manifest for native python3', action='store_true' ) + parser.add_argument( 'outfile', metavar='OUTPUT_FILE', nargs='?', default='', help='Output file (defaults to stdout)' ) + args = parser.parse_args() - if len( sys.argv ) > 1: + if args.outfile: try: - os.unlink(sys.argv[1]) + os.unlink( args.outfile ) except Exception: sys.exc_clear() - outfile = open( sys.argv[1], "w" ) + outfile = open( args.outfile, "w" ) else: outfile = sys.stdout - m = MakefileMaker( outfile ) + m = MakefileMaker( outfile, args.native ) # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! # Parameters: revision, name, description, dependencies, filenames @@ -174,7 +206,7 @@ if __name__ == "__main__": "UserDict.* UserList.* UserString.* " + "lib-dynload/binascii.*.so lib-dynload/_struct.*.so lib-dynload/time.*.so " + "lib-dynload/xreadlines.*.so types.* platform.* ${bindir}/python* " + - "_weakrefset.* sysconfig.* _sysconfigdata.* config/Makefile " + + "_weakrefset.* sysconfig.* _sysconfigdata.* " + "${includedir}/python${PYTHON_BINABI}/pyconfig*.h " + "${libdir}/python${PYTHON_MAJMIN}/collections " + "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.* " + @@ -191,7 +223,8 @@ if __name__ == "__main__": "${base_libdir}/*.a " + "${base_libdir}/*.o " + "${datadir}/aclocal " + - "${datadir}/pkgconfig " ) + "${datadir}/pkgconfig " + + "config/Makefile ") m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", "lib2to3" ) # package @@ -266,7 +299,7 @@ if __name__ == "__main__": "lib-dynload/fcntl.*.so" ) m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", - "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + "formatter.* htmlentitydefs.* html htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core ${PN}-lang", "importlib imp.*" ) @@ -279,7 +312,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math", "lib-dynload/_socket.*.so lib-dynload/_io.*.so lib-dynload/_ssl.*.so lib-dynload/select.*.so lib-dynload/termios.*.so lib-dynload/cStringIO.*.so " + - "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + "ipaddress.* pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", "json lib-dynload/_json.*.so" ) # package @@ -308,18 +341,18 @@ if __name__ == "__main__": m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", "lib-dynload/_multiprocessing.*.so multiprocessing" ) # package - m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-argparse ${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime ${PN}-html", "*Cookie*.* " + - "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib uuid.* rfc822.* mimetools.*" ) + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* http* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib uuid.* rfc822.* mimetools.*" ) m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", - "cgi.* *HTTPServer.* SocketServer.*" ) + "cgi.* socketserver.* *HTTPServer.* SocketServer.*" ) m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", "decimal.* fractions.* numbers.*" ) m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", - "pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) + "_compat_pickle.* pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", "pkgutil.*") @@ -378,6 +411,9 @@ if __name__ == "__main__": m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", "lib-dynload/_tkinter.*.so lib-tk tkinter" ) # package + m.addPackage( "${PN}-typing", "Python typing support", "${PN}-core", + "typing.*" ) + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", "unittest/" ) @@ -387,7 +423,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", "lib-dynload/_elementtree.*.so lib-dynload/pyexpat.*.so xml xmllib.*" ) # package - m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang ${PN}-pydoc", "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.* xmlrpc" ) m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", diff --git a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py b/import-layers/yocto-poky/scripts/contrib/verify-homepage.py index d39dd1d97..76f1749cf 100755 --- a/import-layers/yocto-poky/scripts/contrib/verify-homepage.py +++ b/import-layers/yocto-poky/scripts/contrib/verify-homepage.py @@ -44,7 +44,7 @@ def verifyHomepage(bbhandler): if realfn in checked: continue data = bbhandler.parse_recipe_file(realfn) - homepage = data.getVar("HOMEPAGE", True) + homepage = data.getVar("HOMEPAGE") if homepage: try: urllib.request.urlopen(homepage, timeout=5) diff --git a/import-layers/yocto-poky/scripts/contrib/yocto-bsp-kernel-update.sh b/import-layers/yocto-poky/scripts/contrib/yocto-bsp-kernel-update.sh new file mode 100755 index 000000000..b3aa70560 --- /dev/null +++ b/import-layers/yocto-poky/scripts/contrib/yocto-bsp-kernel-update.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Copyright (c) 2017, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Description: creates a new set of kernel templates based on version +# + +set -o nounset +set -o errexit + +if [ $# -ne 4 ]; then + cat << EOF +usage: $0 from_mayor from_minor to_mayor to_minor +EOF + exit 1 +else + fma=$1 # from mayor + fmi=$2 # from minor + tma=$3 # to mayor + tmi=$4 # to minor +fi + +poky=$(readlink -e $(dirname $(dirname $(dirname $0)))) +arch=$poky/scripts/lib/bsp/substrate/target/arch + + +# copy/rename templates +for from in $(ls -1 $arch/*/recipes-kernel/linux/linux-yocto*_$fma\.$fmi.bbappend) +do + to=$(echo $from | sed s/$fma\.$fmi/$tma\.$tmi/) + cp $from $to +done + +# replace versions string inside new templates +for bbappend in $(ls -1 $arch/*/recipes-kernel/linux/linux-yocto*_$tma\.$tmi.bbappend) +do + sed -i 1s/$fma\.$fmi/$tma\.$tmi/ $bbappend + sed -i \$s/$fma\.$fmi/$tma\.$tmi/ $bbappend +done + +# update the noinstall files +for noinstall in $(ls -1 $arch/*/recipes-kernel/linux/kernel-list.noinstall) +do + sed -i s/$fma\.$fmi/$tma\.$tmi/g $noinstall; +done |