Yocto 2.4

Move OpenBMC to Yocto 2.4(rocko)

Tested: Built and verified Witherspoon and Palmetto images
Change-Id: I12057b18610d6fb0e6903c60213690301e9b0c67
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>

6 files changed, 411 insertions, 47 deletions

diff --git a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh
index 3da32532b..19bee1dd0 100755
--- a/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh
+++ b/import-layers/yocto-poky/scripts/contrib/build-perf-test-wrapper.sh
@@ -35,6 +35,7 @@ Optional arguments:
   -C GIT_REPO       commit results into Git
   -E EMAIL_ADDR     send email report
   -P GIT_REMOTE     push results to a remote Git repository
+  -R DEST           rsync reports to a remote destination
   -w WORK_DIR       work dir for this script
                     (default: GIT_TOP_DIR/build-perf-test)
   -x                create xml report (instead of json)
@@ -50,7 +51,7 @@ get_os_release_var () {
 commitish=""
 oe_build_perf_test_extra_opts=()
 oe_git_archive_extra_opts=()
-while getopts "ha:c:C:E:P:w:x" opt; do
+while getopts "ha:c:C:E:P:R:w:x" opt; do
     case $opt in
         h)  usage
             exit 0
@@ -65,6 +66,8 @@ while getopts "ha:c:C:E:P:w:x" opt; do
             ;;
         P)  oe_git_archive_extra_opts+=("--push" "$OPTARG")
             ;;
+        R)  rsync_dst="$OPTARG"
+            ;;
         w)  base_dir=`realpath -s "$OPTARG"`
             ;;
         x)  oe_build_perf_test_extra_opts+=("--xml")
@@ -132,6 +135,11 @@ if [ -n "$commitish" ]; then
     git reset --hard $commit > /dev/null
 fi
 
+# Determine name of the current branch
+branch=`git symbolic-ref HEAD 2> /dev/null`
+# Strip refs/heads/
+branch=${branch:11}
+
 # Setup build environment
 if [ -z "$base_dir" ]; then
     base_dir="$git_topdir/build-perf-test"
@@ -187,13 +195,25 @@ if [ -n "$results_repo" ]; then
         "${oe_git_archive_extra_opts[@]}" \
         "$results_dir"
 
+    # Generate test reports
+    sanitized_branch=`echo $branch | tr / _`
+    report_txt=`hostname`_${sanitized_branch}_${machine}.txt
+    report_html=`hostname`_${sanitized_branch}_${machine}.html
+    echo -e "\nGenerating test report"
+    oe-build-perf-report -r "$results_repo" > $report_txt
+    oe-build-perf-report -r "$results_repo" --html > $report_html
+
     # Send email report
     if [ -n "$email_to" ]; then
-        echo -e "\nEmailing test report"
+        echo "Emailing test report"
         os_name=`get_os_release_var PRETTY_NAME`
-        oe-build-perf-report -r "$results_repo" > report.txt
-        oe-build-perf-report -r "$results_repo" --html > report.html
-        "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text report.txt --html report.html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}"
+        "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt --html $report_html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}"
+    fi
+
+    # Upload report files, unless we're on detached head
+    if [ -n "$rsync_dst" -a -n "$branch" ]; then
+        echo "Uploading test report"
+        rsync $report_txt $report_html $rsync_dst
     fi
 fi
 
diff --git a/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py b/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py
index 261ca514e..913847bbe 100755
--- a/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py
+++ b/import-layers/yocto-poky/scripts/contrib/oe-build-perf-report-email.py
@@ -25,6 +25,7 @@ import socket
 import subprocess
 import sys
 import tempfile
+from email.mime.image import MIMEImage
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
@@ -71,6 +72,10 @@ def parse_args(argv):
                         help="Only print errors")
     parser.add_argument('--to', action='append',
                         help="Recipients of the email")
+    parser.add_argument('--cc', action='append',
+                        help="Carbon copy recipients of the email")
+    parser.add_argument('--bcc', action='append',
+                        help="Blind carbon copy recipients of the email")
     parser.add_argument('--subject', default="Yocto build perf test report",
                         help="Email subject")
     parser.add_argument('--outdir', '-o',
@@ -107,15 +112,6 @@ def decode_png(infile, outfile):
     subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT)
 
 
-def encode_png(pngfile):
-    """Encode png into a <img> html element"""
-    with open(pngfile, 'rb') as f:
-        data = f.read()
-
-    b64_data = base64.b64encode(data)
-    return '<img src="data:image/png;base64,' + b64_data.decode('utf-8') + '">\n'
-
-
 def mangle_html_report(infile, outfile, pngs):
     """Mangle html file into a email compatible format"""
     paste = True
@@ -140,9 +136,7 @@ def mangle_html_report(infile, outfile, pngs):
                     # Replace charts with <img> elements
                     match = re.match('<div id="(?P<id>\w+)"', stripped)
                     if match and match.group('id') in pngs:
-                        #f_out.write('<img src="{}">\n'.format(match.group('id') + '.png'))
-                        png_file = os.path.join(png_dir, match.group('id') + '.png')
-                        f_out.write(encode_png(png_file))
+                        f_out.write('<img src="cid:{}"\n'.format(match.group('id')))
                     else:
                         f_out.write(line)
 
@@ -166,7 +160,7 @@ def scrape_html_report(report, outdir, phantomjs_extra_args=None):
                                 stderr=subprocess.STDOUT)
 
         pngs = []
-        attachments = []
+        images = []
         for fname in os.listdir(tmpdir):
             base, ext = os.path.splitext(fname)
             if ext == '.png':
@@ -174,7 +168,7 @@ def scrape_html_report(report, outdir, phantomjs_extra_args=None):
                 decode_png(os.path.join(tmpdir, fname),
                            os.path.join(outdir, fname))
                 pngs.append(base)
-                attachments.append(fname)
+                images.append(fname)
             elif ext in ('.html', '.htm'):
                 report_file = fname
             else:
@@ -184,11 +178,13 @@ def scrape_html_report(report, outdir, phantomjs_extra_args=None):
         log.debug("Mangling html report file %s", report_file)
         mangle_html_report(os.path.join(tmpdir, report_file),
                            os.path.join(outdir, report_file), pngs)
-        return report_file, attachments
+        return (os.path.join(outdir, report_file),
+                [os.path.join(outdir, i) for i in images])
     finally:
         shutil.rmtree(tmpdir)
 
-def send_email(text_fn, html_fn, subject, recipients):
+def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[],
+               blind_copy=[]):
     """Send email"""
     # Generate email message
     text_msg = html_msg = None
@@ -197,8 +193,16 @@ def send_email(text_fn, html_fn, subject, recipients):
             text_msg = MIMEText("Yocto build performance test report.\n" +
                                 f.read(), 'plain')
     if html_fn:
+        html_msg = msg = MIMEMultipart('related')
         with open(html_fn) as f:
-            html_msg = MIMEText(f.read(), 'html')
+            html_msg.attach(MIMEText(f.read(), 'html'))
+        for img_fn in image_fns:
+            # Expect that content id is same as the filename
+            cid = os.path.splitext(os.path.basename(img_fn))[0]
+            with open(img_fn, 'rb') as f:
+                image_msg = MIMEImage(f.read())
+            image_msg['Content-ID'] = '<{}>'.format(cid)
+            html_msg.attach(image_msg)
 
     if text_msg and html_msg:
         msg = MIMEMultipart('alternative')
@@ -217,6 +221,10 @@ def send_email(text_fn, html_fn, subject, recipients):
                            '{}@{}'.format(pw_data.pw_name, socket.getfqdn()))
     msg['From'] = "{} <{}>".format(full_name, email)
     msg['To'] = ', '.join(recipients)
+    if copy:
+        msg['Cc'] = ', '.join(copy)
+    if blind_copy:
+        msg['Bcc'] = ', '.join(blind_copy)
     msg['Subject'] = subject
 
     # Send email
@@ -243,14 +251,19 @@ def main(argv=None):
 
     try:
         log.debug("Storing email parts in %s", outdir)
-        html_report = None
+        html_report = images = None
         if args.html:
-            scrape_html_report(args.html, outdir, args.phantomjs_args)
-            html_report = os.path.join(outdir, os.path.basename(args.html))
+            html_report, images = scrape_html_report(args.html, outdir,
+                                                     args.phantomjs_args)
 
         if args.to:
             log.info("Sending email to %s", ', '.join(args.to))
-            send_email(args.text, html_report, args.subject, args.to)
+            if args.cc:
+                log.info("Copying to %s", ', '.join(args.cc))
+            if args.bcc:
+                log.info("Blind copying to %s", ', '.join(args.bcc))
+            send_email(args.text, html_report, images, args.subject,
+                       args.to, args.cc, args.bcc)
     except subprocess.CalledProcessError as err:
         log.error("%s, with output:\n%s", str(err), err.output.decode())
         return 1
diff --git a/import-layers/yocto-poky/scripts/contrib/patchreview.py b/import-layers/yocto-poky/scripts/contrib/patchreview.py
new file mode 100755
index 000000000..4e3e73c7a
--- /dev/null
+++ b/import-layers/yocto-poky/scripts/contrib/patchreview.py
@@ -0,0 +1,211 @@
+#! /usr/bin/env python3
+
+# TODO
+# - option to just list all broken files
+# - test suite
+# - validate signed-off-by
+
+
+class Result:
+    # Whether the patch has an Upstream-Status or not
+    missing_upstream_status = False
+    # If the Upstream-Status tag is malformed in some way (string for bad bit)
+    malformed_upstream_status = None
+    # If the Upstream-Status value is unknown (boolean)
+    unknown_upstream_status = False
+    # The upstream status value (Pending, etc)
+    upstream_status = None
+    # Whether the patch has a Signed-off-by or not
+    missing_sob = False
+    # Whether the Signed-off-by tag is malformed in some way
+    malformed_sob = False
+    # The Signed-off-by tag value
+    sob = None
+    # Whether a patch looks like a CVE but doesn't have a CVE tag
+    missing_cve = False
+
+def blame_patch(patch):
+    """
+    From a patch filename, return a list of "commit summary (author name <author
+    email>)" strings representing the history.
+    """
+    import subprocess
+    return subprocess.check_output(("git", "log",
+                                    "--follow", "--find-renames", "--diff-filter=A",
+                                    "--format=%s (%aN <%aE>)",
+                                    "--", patch)).decode("utf-8").splitlines()
+
+def patchreview(patches):
+    import re
+
+    # General pattern: start of line, optional whitespace, tag with optional
+    # hyphen or spaces, maybe a colon, some whitespace, then the value, all case
+    # insensitive.
+    sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE)
+    status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*(\w*)", re.IGNORECASE | re.MULTILINE)
+    status_values = ("accepted", "pending", "inappropriate", "backport", "submitted", "denied")
+    cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE)
+    cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE)
+
+    results = {}
+
+    for patch in patches:
+        result = Result()
+        results[patch] = result
+
+        content = open(patch, encoding='ascii', errors='ignore').read()
+
+        # Find the Signed-off-by tag
+        match = sob_re.search(content)
+        if match:
+            value = match.group(1)
+            if value != "Signed-off-by:":
+                result.malformed_sob = value
+            result.sob = match.group(2)
+        else:
+            result.missing_sob = True
+
+
+        # Find the Upstream-Status tag
+        match = status_re.search(content)
+        if match:
+            value = match.group(1)
+            if value != "Upstream-Status:":
+                result.malformed_upstream_status = value
+
+            value = match.group(2).lower()
+            # TODO: check case
+            if value not in status_values:
+                result.unknown_upstream_status = True
+            result.upstream_status = value
+        else:
+            result.missing_upstream_status = True
+
+        # Check that patches which looks like CVEs have CVE tags
+        if cve_re.search(patch) or cve_re.search(content):
+            if not cve_tag_re.search(content):
+                result.missing_cve = True
+        # TODO: extract CVE list
+
+    return results
+
+
+def analyse(results, want_blame=False, verbose=True):
+    """
+    want_blame: display blame data for each malformed patch
+    verbose: display per-file results instead of just summary
+    """
+
+    # want_blame requires verbose, so disable blame if we're not verbose
+    if want_blame and not verbose:
+        want_blame = False
+
+    total_patches = 0
+    missing_sob = 0
+    malformed_sob = 0
+    missing_status = 0
+    malformed_status = 0
+    missing_cve = 0
+    pending_patches = 0
+
+    for patch in sorted(results):
+        r = results[patch]
+        total_patches += 1
+        need_blame = False
+
+        # Build statistics
+        if r.missing_sob:
+            missing_sob += 1
+        if r.malformed_sob:
+            malformed_sob += 1
+        if r.missing_upstream_status:
+            missing_status += 1
+        if r.malformed_upstream_status or r.unknown_upstream_status:
+            malformed_status += 1
+        if r.missing_cve:
+            missing_cve += 1
+        if r.upstream_status == "pending":
+            pending_patches += 1
+
+        # Output warnings
+        if r.missing_sob:
+            need_blame = True
+            if verbose:
+                print("Missing Signed-off-by tag (%s)" % patch)
+        # TODO: disable this for now as too much fails
+        if False and r.malformed_sob:
+            need_blame = True
+            if verbose:
+                print("Malformed Signed-off-by '%s' (%s)" % (r.malformed_sob, patch))
+        if r.missing_cve:
+            need_blame = True
+            if verbose:
+                print("Missing CVE tag (%s)" % patch)
+        if r.missing_upstream_status:
+            need_blame = True
+            if verbose:
+                print("Missing Upstream-Status tag (%s)" % patch)
+        if r.malformed_upstream_status:
+            need_blame = True
+            if verbose:
+                print("Malformed Upstream-Status '%s' (%s)" % (r.malformed_upstream_status, patch))
+        if r.unknown_upstream_status:
+            need_blame = True
+            if verbose:
+                print("Unknown Upstream-Status value '%s' (%s)" % (r.upstream_status, patch))
+
+        if want_blame and need_blame:
+            print("\n".join(blame_patch(patch)) + "\n")
+
+    def percent(num):
+        try:
+            return "%d (%d%%)" % (num, round(num * 100.0 / total_patches))
+        except ZeroDivisionError:
+            return "N/A"
+
+    if verbose:
+        print()
+
+    print("""Total patches found: %d
+Patches missing Signed-off-by: %s
+Patches with malformed Signed-off-by: %s
+Patches missing CVE: %s
+Patches missing Upstream-Status: %s
+Patches with malformed Upstream-Status: %s
+Patches in Pending state: %s""" % (total_patches,
+                                   percent(missing_sob),
+                                   percent(malformed_sob),
+                                   percent(missing_cve),
+                                   percent(missing_status),
+                                   percent(malformed_status),
+                                   percent(pending_patches)))
+
+
+
+def histogram(results):
+    from toolz import recipes, dicttoolz
+    import math
+    counts = recipes.countby(lambda r: r.upstream_status, results.values())
+    bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts)
+    for k in bars:
+        print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k]))
+
+
+if __name__ == "__main__":
+    import argparse, subprocess, os
+
+    args = argparse.ArgumentParser(description="Patch Review Tool")
+    args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches")
+    args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results")
+    args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram")
+    args.add_argument("directory", nargs="?", help="directory to scan")
+    args = args.parse_args()
+
+    if args.directory:
+        os.chdir(args.directory)
+    patches = subprocess.check_output(("git", "ls-files", "*.patch", "*.diff")).decode("utf-8").split()
+    results = patchreview(patches)
+    analyse(results, want_blame=args.blame, verbose=args.verbose)
+    if args.histogram:
+        print()
+        histogram(results)
diff --git a/import-layers/yocto-poky/scripts/contrib/patchtest.sh b/import-layers/yocto-poky/scripts/contrib/patchtest.sh
new file mode 100755
index 000000000..7fe566666
--- /dev/null
+++ b/import-layers/yocto-poky/scripts/contrib/patchtest.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# patchtest: Run patchtest on commits starting at master
+#
+# Copyright (c) 2017, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+set -o errexit
+
+# Default values
+pokydir=''
+
+usage() {
+CMD=$(basename $0)
+cat <<EOM
+Usage: $CMD [-h] [-p pokydir]
+  -p pokydir  Defaults to current directory
+EOM
+>&2
+    exit 1
+}
+
+function clone() {
+    local REPOREMOTE=$1
+    local REPODIR=$2
+    if [ ! -d $REPODIR ]; then
+	git clone $REPOREMOTE $REPODIR --quiet
+    else
+	( cd $REPODIR; git pull --quiet )
+    fi
+}
+
+while getopts ":p:h" opt; do
+    case $opt in
+	p)
+	    pokydir=$OPTARG
+	    ;;
+	h)
+	    usage
+	    ;;
+	\?)
+	    echo "Invalid option: -$OPTARG" >&2
+	    usage
+	    ;;
+	:)
+	    echo "Option -$OPTARG requires an argument." >&2
+	    usage
+	    ;;
+    esac
+done
+shift $((OPTIND-1))
+
+CDIR="$PWD"
+
+# default pokydir to current directory if user did not specify one
+if [ -z "$pokydir" ]; then
+    pokydir="$CDIR"
+fi
+
+PTENV="$PWD/patchtest"
+PT="$PTENV/patchtest"
+PTOE="$PTENV/patchtest-oe"
+
+if ! which virtualenv > /dev/null; then
+    echo "Install virtualenv before proceeding"
+    exit 1;
+fi
+
+# activate the virtual env
+virtualenv $PTENV --quiet
+source $PTENV/bin/activate
+
+cd $PTENV
+
+# clone or pull
+clone git://git.yoctoproject.org/patchtest $PT
+clone git://git.yoctoproject.org/patchtest-oe $PTOE
+
+# install requirements
+pip install -r $PT/requirements.txt --quiet
+pip install -r $PTOE/requirements.txt --quiet
+
+PATH="$PT:$PT/scripts:$PATH"
+
+# loop through parent to HEAD and execute patchtest on each commit
+for commit in $(git rev-list master..HEAD --reverse)
+do
+    shortlog="$(git log "$commit^1..$commit" --pretty='%h: %aN: %cd: %s')"
+    log="$(git format-patch "$commit^1..$commit" --stdout | patchtest - -r $pokydir -s $PTOE/tests --base-commit $commit^1 --json 2>/dev/null | create-summary --fail --only-results)"
+    if [ -z "$log" ]; then
+	shortlog="$shortlog: OK"
+    else
+	shortlog="$shortlog: FAIL"
+    fi
+    echo "$shortlog"
+    echo "$log" | sed -n -e '/Issue/p' -e '/Suggested fix/p'
+    echo ""
+done
+
+deactivate
+
+cd $CDIR
diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py
index 8c3655d39..586b329c1 100755
--- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py
+++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-2.7.py
@@ -28,6 +28,7 @@ class MakefileMaker:
     def __init__( self, outfile, isNative ):
         """initialize"""
         self.packages = {}
+        self.excluded_pkgs = []
         self.targetPrefix = "${libdir}/python%s/" % VERSION[:3]
         self.isNative = isNative
         self.output = outfile
@@ -52,7 +53,7 @@ class MakefileMaker:
         self.out( """ """ )
         self.out( "" )
 
-    def addPackage( self, name, description, dependencies, filenames ):
+    def addPackage( self, name, description, dependencies, filenames, mod_exclude = False ):
         """add a package to the Makefile"""
         if type( filenames ) == type( "" ):
             filenames = filenames.split()
@@ -62,6 +63,8 @@ class MakefileMaker:
                 fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) )
             else:
                 fullFilenames.append( filename )
+        if mod_exclude:
+            self.excluded_pkgs.append( name )
         self.packages[name] = description, dependencies, fullFilenames
 
     def doBody( self ):
@@ -74,13 +77,11 @@ class MakefileMaker:
         #
 
         if self.isNative:
-            rprovideLine = 'RPROVIDES+="'
-            for name in sorted(self.packages):
-                rprovideLine += "%s-native " % name.replace( '${PN}', 'python' )
-            rprovideLine += '"'
+            pkglist = []
+            for name in ['${PN}-modules'] + sorted(self.packages):
+                pkglist.append('%s-native' % name.replace('${PN}', 'python'))
 
-            self.out( rprovideLine )
-            self.out( "" )
+            self.out('RPROVIDES += "%s"' % " ".join(pkglist))
             return
 
         #
@@ -149,7 +150,7 @@ class MakefileMaker:
         line = 'RDEPENDS_${PN}-modules="'
 
         for name, data in sorted(self.packages.items()):
-            if name not in ['${PN}-dev', '${PN}-distutils-staticdev']:
+            if name not in ['${PN}-dev', '${PN}-distutils-staticdev'] and name not in self.excluded_pkgs:
                 line += "%s " % name
 
         self.out( "%s \"" % line )
@@ -384,7 +385,7 @@ if __name__ == "__main__":
     "pty.* tty.*" )
 
     m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-modules",
-    "test" ) # package
+    "test", True ) # package
 
     m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang",
     "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" )
diff --git a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py
index 075860c41..6352f8f12 100755
--- a/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py
+++ b/import-layers/yocto-poky/scripts/contrib/python/generate-manifest-3.5.py
@@ -31,6 +31,7 @@ class MakefileMaker:
     def __init__( self, outfile, isNative ):
         """initialize"""
         self.packages = {}
+        self.excluded_pkgs = []
         self.targetPrefix = "${libdir}/python%s/" % VERSION[:3]
         self.isNative = isNative
         self.output = outfile
@@ -55,7 +56,7 @@ class MakefileMaker:
         self.out( """ """ )
         self.out( "" )
 
-    def addPackage( self, name, description, dependencies, filenames ):
+    def addPackage( self, name, description, dependencies, filenames, mod_exclude = False ):
         """add a package to the Makefile"""
         if type( filenames ) == type( "" ):
             filenames = filenames.split()
@@ -67,6 +68,8 @@ class MakefileMaker:
                                                  self.pycachePath( filename ) ) )
             else:
                 fullFilenames.append( filename )
+        if mod_exclude:
+            self.excluded_pkgs.append( name )
         self.packages[name] = description, dependencies, fullFilenames
 
     def pycachePath( self, filename ):
@@ -87,13 +90,11 @@ class MakefileMaker:
         #
 
         if self.isNative:
-            rprovideLine = 'RPROVIDES+="'
-            for name in sorted(self.packages):
-                rprovideLine += "%s-native " % name.replace( '${PN}', 'python3' )
-            rprovideLine += '"'
+            pkglist = []
+            for name in ['${PN}-modules'] + sorted(self.packages):
+                pkglist.append('%s-native' % name.replace('${PN}', 'python3'))
 
-            self.out( rprovideLine )
-            self.out( "" )
+            self.out('RPROVIDES += "%s"' % " ".join(pkglist))
             return
 
         #
@@ -162,7 +163,7 @@ class MakefileMaker:
         line = 'RDEPENDS_${PN}-modules="'
 
         for name, data in sorted(self.packages.items()):
-            if name not in ['${PN}-dev', '${PN}-distutils-staticdev']:
+            if name not in ['${PN}-dev', '${PN}-distutils-staticdev'] and name not in self.excluded_pkgs:
                 line += "%s " % name
 
         self.out( "%s \"" % line )
@@ -224,7 +225,7 @@ if __name__ == "__main__":
     "${base_libdir}/*.o " +
     "${datadir}/aclocal " +
     "${datadir}/pkgconfig " +
-    "config/Makefile ")
+    "config*/Makefile ")
 
     m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core",
     "lib2to3" ) # package
@@ -254,7 +255,7 @@ if __name__ == "__main__":
     "py_compile.* compileall.*" )
 
     m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs ${PN}-importlib ${PN}-threading ${PN}-shell",
-    "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so lib-dynload/zlib.*.so" )
+    "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so lib-dynload/zlib.*.so bz2.py lzma.py _compression.py" )
 
     m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core",
     "hashlib.* md5.* sha.* lib-dynload/crypt.*.so lib-dynload/_hashlib.*.so lib-dynload/_sha256.*.so lib-dynload/_sha512.*.so" )
@@ -402,8 +403,8 @@ if __name__ == "__main__":
     m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io",
     "pty.* tty.*" )
 
-    m.addPackage( "${PN}-tests", "Python tests", "${PN}-core",
-    "test" ) # package
+    m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-compression",
+    "test", True ) # package
 
     m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang",
     "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" )