meta-openembedded and poky: subtree updates

Squash of the following due to dependencies among them
and OpenBMC changes:

meta-openembedded: subtree update:d0748372d2..9201611135
meta-openembedded: subtree update:9201611135..17fd382f34
poky: subtree update:9052e5b32a..2e11d97b6c
poky: subtree update:2e11d97b6c..a8544811d7

The change log was too large for the jenkins plugin
to handle therefore it has been removed. Here is
the first and last commit of each subtree:

meta-openembedded:d0748372d2
      cppzmq: bump to version 4.6.0
meta-openembedded:17fd382f34
      mpv: Remove X11 dependency
poky:9052e5b32a
      package_ipk: Remove pointless comment to trigger rebuild
poky:a8544811d7
      pbzip2: Fix license warning

Change-Id: If0fc6c37629642ee207a4ca2f7aa501a2c673cd6
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>

17 files changed, 508 insertions, 396 deletions

diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py
index 1c0cd8ab5..bab644b83 100644
--- a/poky/scripts/lib/devtool/standard.py
+++ b/poky/scripts/lib/devtool/standard.py
@@ -145,8 +145,8 @@ def add(args, config, basepath, workspace):
         extracmdopts += ' --src-subdir "%s"' % args.src_subdir
     if args.autorev:
         extracmdopts += ' -a'
-    if args.fetch_dev:
-        extracmdopts += ' --fetch-dev'
+    if args.npm_dev:
+        extracmdopts += ' --npm-dev'
     if args.mirrors:
         extracmdopts += ' --mirrors'
     if args.srcrev:
@@ -260,14 +260,10 @@ def add(args, config, basepath, workspace):
                 f.write('}\n')
 
             if bb.data.inherits_class('npm', rd):
-                f.write('do_install_append() {\n')
-                f.write('    # Remove files added to source dir by devtool/externalsrc\n')
-                f.write('    rm -f ${NPM_INSTALLDIR}/singletask.lock\n')
-                f.write('    rm -rf ${NPM_INSTALLDIR}/.git\n')
-                f.write('    rm -rf ${NPM_INSTALLDIR}/oe-local-files\n')
-                f.write('    for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n')
-                f.write('        rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n')
-                f.write('    done\n')
+                f.write('python do_configure_append() {\n')
+                f.write('    pkgdir = d.getVar("NPM_PACKAGE")\n')
+                f.write('    lockfile = os.path.join(pkgdir, "singletask.lock")\n')
+                f.write('    bb.utils.remove(lockfile)\n')
                 f.write('}\n')
 
         # Check if the new layer provides recipes whose priorities have been
@@ -940,8 +936,10 @@ def modify(args, config, basepath, workspace):
                         '}\n')
             if rd.getVarFlag('do_menuconfig','task'):
                 f.write('\ndo_configure_append() {\n'
-                '    cp ${B}/.config ${S}/.config.baseline\n'
-                '    ln -sfT ${B}/.config ${S}/.config.new\n'
+                '    if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n'
+                '        cp ${B}/.config ${S}/.config.baseline\n'
+                '        ln -sfT ${B}/.config ${S}/.config.new\n'
+                '    fi\n'
                 '}\n')
             if initial_rev:
                 f.write('\n# initial_rev: %s\n' % initial_rev)
@@ -2197,7 +2195,7 @@ def register_commands(subparsers, context):
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
     parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
-    parser_add.add_argument('--fetch-dev', help='For npm, also fetch devDependencies', action="store_true")
+    parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
     parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
     parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
     group = parser_add.add_mutually_exclusive_group()
diff --git a/poky/scripts/lib/recipetool/create.py b/poky/scripts/lib/recipetool/create.py
index 4c4bbadb4..6cbf4de67 100644
--- a/poky/scripts/lib/recipetool/create.py
+++ b/poky/scripts/lib/recipetool/create.py
@@ -477,8 +477,6 @@ def create_recipe(args):
             storeTagName = params['tag']
             params['nobranch'] = '1'
             del params['tag']
-        if scheme == 'npm':
-            params['noverify'] = '1'
         fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
 
         tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -714,10 +712,8 @@ def create_recipe(args):
         lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
         lines_after.append('')
 
-    if args.fetch_dev:
-        extravalues['fetchdev'] = True
-    else:
-        extravalues['fetchdev'] = None
+    if args.npm_dev:
+        extravalues['NPM_INSTALL_DEV'] = 1
 
     # Find all plugins that want to register handlers
     logger.debug('Loading recipe handlers')
@@ -1313,7 +1309,7 @@ def register_commands(subparsers):
     group.add_argument('-S', '--srcrev', help='Source revision to fetch if fetching from an SCM such as git (default latest)')
     parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)')
     parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
-    parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
+    parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies')
     parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
     parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
     parser_create.set_defaults(func=create_recipe)
diff --git a/poky/scripts/lib/recipetool/create_buildsys.py b/poky/scripts/lib/recipetool/create_buildsys.py
index 3cb02766c..35a97c934 100644
--- a/poky/scripts/lib/recipetool/create_buildsys.py
+++ b/poky/scripts/lib/recipetool/create_buildsys.py
@@ -226,9 +226,9 @@ class CmakeRecipeHandler(RecipeHandler):
                         elif pkg == 'PkgConfig':
                             inherits.append('pkgconfig')
                         elif pkg == 'PythonInterp':
-                            inherits.append('pythonnative')
+                            inherits.append('python3native')
                         elif pkg == 'PythonLibs':
-                            inherits.append('python-dir')
+                            inherits.append('python3-dir')
                         else:
                             # Try to map via looking at installed CMake packages in pkgdata
                             dep = find_cmake_package(pkg)
@@ -417,7 +417,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 }
         progclassmap = {'gconftool-2': 'gconf',
                 'pkg-config': 'pkgconfig',
-                'python': 'pythonnative',
+                'python': 'python3native',
                 'python3': 'python3native',
                 'perl': 'perlnative',
                 'makeinfo': 'texinfo',
@@ -566,16 +566,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
             elif keyword == 'AX_PROG_XSLTPROC':
                 deps.append('libxslt-native')
             elif keyword in ['AC_PYTHON_DEVEL', 'AX_PYTHON_DEVEL', 'AM_PATH_PYTHON']:
-                pythonclass = 'pythonnative'
-                res = version_re.search(value)
-                if res:
-                    if res.group(1).startswith('3'):
-                        pythonclass = 'python3native'
-                # Avoid replacing python3native with pythonnative
-                if not pythonclass in inherits and not 'python3native' in inherits:
-                    if 'pythonnative' in inherits:
-                        inherits.remove('pythonnative')
-                    inherits.append(pythonclass)
+                pythonclass = 'python3native'
             elif keyword == 'AX_WITH_CURSES':
                 deps.append('ncurses')
             elif keyword == 'AX_PATH_BDB':
diff --git a/poky/scripts/lib/recipetool/create_npm.py b/poky/scripts/lib/recipetool/create_npm.py
index 39429ebad..579b7ae48 100644
--- a/poky/scripts/lib/recipetool/create_npm.py
+++ b/poky/scripts/lib/recipetool/create_npm.py
@@ -1,321 +1,255 @@
-# Recipe creation tool - node.js NPM module support plugin
-#
 # Copyright (C) 2016 Intel Corporation
+# Copyright (C) 2020 Savoir-Faire Linux
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
+"""Recipe creation tool - npm module support plugin"""
 
+import json
 import os
+import re
 import sys
-import logging
-import subprocess
 import tempfile
-import shutil
-import json
-from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
+import bb
+from bb.fetch2.npm import NpmEnvironment
+from bb.fetch2.npmsw import foreach_dependencies
+from recipetool.create import RecipeHandler
+from recipetool.create import guess_license
+from recipetool.create import split_pkg_licenses
 
-logger = logging.getLogger('recipetool')
+TINFOIL = None
 
+def tinfoil_init(instance):
+    """Initialize tinfoil"""
+    global TINFOIL
+    TINFOIL = instance
 
-tinfoil = None
+class NpmRecipeHandler(RecipeHandler):
+    """Class to handle the npm recipe creation"""
+
+    @staticmethod
+    def _npm_name(name):
+        """Generate a Yocto friendly npm name"""
+        name = re.sub("/", "-", name)
+        name = name.lower()
+        name = re.sub(r"[^\-a-z0-9]", "", name)
+        name = name.strip("-")
+        return name
+
+    @staticmethod
+    def _get_registry(lines):
+        """Get the registry value from the 'npm://registry' url"""
+        registry = None
+
+        def _handle_registry(varname, origvalue, op, newlines):
+            nonlocal registry
+            if origvalue.startswith("npm://"):
+                registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
+            return origvalue, None, 0, True
 
-def tinfoil_init(instance):
-    global tinfoil
-    tinfoil = instance
+        bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
 
+        return registry
 
-class NpmRecipeHandler(RecipeHandler):
-    lockdownpath = None
+    @staticmethod
+    def _ensure_npm():
+        """Check if the 'npm' command is available in the recipes"""
+        if not TINFOIL.recipes_parsed:
+            TINFOIL.parse_recipes()
 
-    def _ensure_npm(self, fixed_setup=False):
-        if not tinfoil.recipes_parsed:
-            tinfoil.parse_recipes()
         try:
-            rd = tinfoil.parse_recipe('nodejs-native')
+            d = TINFOIL.parse_recipe("nodejs-native")
         except bb.providers.NoProvider:
-            if fixed_setup:
-                msg = 'nodejs-native is required for npm but is not available within this SDK'
-            else:
-                msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
-            logger.error(msg)
-            return None
-        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
-        npmpath = os.path.join(bindir, 'npm')
+            bb.error("Nothing provides 'nodejs-native' which is required for the build")
+            bb.note("You will likely need to add a layer that provides nodejs")
+            sys.exit(14)
+
+        bindir = d.getVar("STAGING_BINDIR_NATIVE")
+        npmpath = os.path.join(bindir, "npm")
+
         if not os.path.exists(npmpath):
-            tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
+            TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
+
             if not os.path.exists(npmpath):
-                logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
-                return None
+                bb.error("Failed to add 'npm' to sysroot")
+                sys.exit(14)
+
         return bindir
 
-    def _handle_license(self, data):
-        '''
-        Handle the license value from an npm package.json file
-        '''
-        license = None
-        if 'license' in data:
-            license = data['license']
-            if isinstance(license, dict):
-                license = license.get('type', None)
-            if license:
-                if 'OR' in license:
-                    license = license.replace('OR', '|')
-                    license = license.replace('AND', '&')
-                    license = license.replace(' ', '_')
-                    if not license[0] == '(':
-                        license = '(' + license + ')'
-                else:
-                    license = license.replace('AND', '&')
-                    if license[0] == '(':
-                        license = license[1:]
-                    if license[-1] == ')':
-                        license = license[:-1]
-                license = license.replace('MIT/X11', 'MIT')
-                license = license.replace('Public Domain', 'PD')
-                license = license.replace('SEE LICENSE IN EULA',
-                                          'SEE-LICENSE-IN-EULA')
-        return license
-
-    def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
-        try:
-            runenv = dict(os.environ, PATH=d.getVar('PATH'))
-            bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
-        except bb.process.ExecutionError as e:
-            logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
-            return
-
-        tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
-        shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
-        extravalues.setdefault('extrafiles', {})
-        extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
-        lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
-
-    def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
-        runenv = dict(os.environ, PATH=d.getVar('PATH'))
-        if not NpmRecipeHandler.lockdownpath:
-            NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
-            bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
-                           cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
-        relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
-        if not os.path.exists(relockbin):
-            logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
-            return
-        try:
-            bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
-        except bb.process.ExecutionError as e:
-            logger.warning('lockdown-relock failed:\n%s' % e.stdout)
-            return
-
-        tmpfile = os.path.join(localfilesdir, 'lockdown.json')
-        shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
-        extravalues.setdefault('extrafiles', {})
-        extravalues['extrafiles']['lockdown.json'] = tmpfile
-        lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
-
-    def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
-        import scriptutils
-        # If this isn't a single module we need to get the dependencies
-        # and add them to SRC_URI
-        def varfunc(varname, origvalue, op, newlines):
-            if varname == 'SRC_URI':
-                if not origvalue.startswith('npm://'):
-                    src_uri = origvalue.split()
-                    deplist = {}
-                    for dep, depver in optdeps.items():
-                        depdata = self.get_npm_data(dep, depver, d)
-                        if self.check_npm_optional_dependency(depdata):
-                            deplist[dep] = depdata
-                    for dep, depver in devdeps.items():
-                        depdata = self.get_npm_data(dep, depver, d)
-                        if self.check_npm_optional_dependency(depdata):
-                            deplist[dep] = depdata
-                    for dep, depver in deps.items():
-                        depdata = self.get_npm_data(dep, depver, d)
-                        deplist[dep] = depdata
-
-                    extra_urls = []
-                    for dep, depdata in deplist.items():
-                        version = depdata.get('version', None)
-                        if version:
-                            url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
-                            extra_urls.append(url)
-                    if extra_urls:
-                        scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
-                        src_uri.extend(extra_urls)
-                        return src_uri, None, -1, True
-            return origvalue, None, 0, True
-        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
-        if updated:
-            del lines_before[:]
-            for line in newlines:
-                # Hack to avoid newlines that edit_metadata inserts
-                if line.endswith('\n'):
-                    line = line[:-1]
-                lines_before.append(line)
-        return updated
+    @staticmethod
+    def _npm_global_configs(dev):
+        """Get the npm global configuration"""
+        configs = []
+
+        if dev:
+            configs.append(("also", "development"))
+        else:
+            configs.append(("only", "production"))
+
+        configs.append(("save", "false"))
+        configs.append(("package-lock", "false"))
+        configs.append(("shrinkwrap", "false"))
+        return configs
+
+    def _run_npm_install(self, d, srctree, registry, dev):
+        """Run the 'npm install' command without building the addons"""
+        configs = self._npm_global_configs(dev)
+        configs.append(("ignore-scripts", "true"))
+
+        if registry:
+            configs.append(("registry", registry))
+
+        bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
+
+        env = NpmEnvironment(d, configs=configs)
+        env.run("npm install", workdir=srctree)
+
+    def _generate_shrinkwrap(self, d, srctree, dev):
+        """Check and generate the 'npm-shrinkwrap.json' file if needed"""
+        configs = self._npm_global_configs(dev)
+
+        env = NpmEnvironment(d, configs=configs)
+        env.run("npm shrinkwrap", workdir=srctree)
+
+        return os.path.join(srctree, "npm-shrinkwrap.json")
+
+    def _handle_licenses(self, srctree, shrinkwrap_file, dev):
+        """Return the extra license files and the list of packages"""
+        licfiles = []
+        packages = {}
+
+        def _licfiles_append(licfile):
+            """Append 'licfile' to the license files list"""
+            licfilepath = os.path.join(srctree, licfile)
+            licmd5 = bb.utils.md5_file(licfilepath)
+            licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
+
+        # Handle the parent package
+        _licfiles_append("package.json")
+        packages["${PN}"] = ""
+
+        # Handle the dependencies
+        def _handle_dependency(name, params, deptree):
+            suffix = "-".join([self._npm_name(dep) for dep in deptree])
+            destdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            destdir = os.path.join(*destdirs)
+            _licfiles_append(os.path.join(destdir, "package.json"))
+            packages["${PN}-" + suffix] = destdir
+
+        with open(shrinkwrap_file, "r") as f:
+            shrinkwrap = json.load(f)
+
+        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
+
+        return licfiles, packages
 
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
-        import bb.utils
-        import oe.package
-        from collections import OrderedDict
+        """Handle the npm recipe creation"""
 
-        if 'buildsystem' in handled:
+        if "buildsystem" in handled:
             return False
 
-        def read_package_json(fn):
-            with open(fn, 'r', errors='surrogateescape') as f:
-                return json.loads(f.read())
+        files = RecipeHandler.checkfiles(srctree, ["package.json"])
 
-        files = RecipeHandler.checkfiles(srctree, ['package.json'])
-        if files:
-            d = bb.data.createCopy(tinfoil.config_data)
-            npm_bindir = self._ensure_npm()
-            if not npm_bindir:
-                sys.exit(14)
-            d.prependVar('PATH', '%s:' % npm_bindir)
-
-            data = read_package_json(files[0])
-            if 'name' in data and 'version' in data:
-                extravalues['PN'] = data['name']
-                extravalues['PV'] = data['version']
-                classes.append('npm')
-                handled.append('buildsystem')
-                if 'description' in data:
-                    extravalues['SUMMARY'] = data['description']
-                if 'homepage' in data:
-                    extravalues['HOMEPAGE'] = data['homepage']
-
-                fetchdev = extravalues['fetchdev'] or None
-                deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
-                self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
-
-                # Shrinkwrap
-                localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
-                self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
-
-                # Lockdown
-                self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
-
-                # Split each npm module out to is own package
-                npmpackages = oe.package.npm_split_package_dirs(srctree)
-                licvalues = None
-                for item in handled:
-                    if isinstance(item, tuple):
-                        if item[0] == 'license':
-                            licvalues = item[1]
-                            break
-                if not licvalues:
-                    licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
-                if licvalues:
-                    # Augment the license list with information we have in the packages
-                    licenses = {}
-                    license = self._handle_license(data)
-                    if license:
-                        licenses['${PN}'] = license
-                    for pkgname, pkgitem in npmpackages.items():
-                        _, pdata = pkgitem
-                        license = self._handle_license(pdata)
-                        if license:
-                            licenses[pkgname] = license
-                    # Now write out the package-specific license values
-                    # We need to strip out the json data dicts for this since split_pkg_licenses
-                    # isn't expecting it
-                    packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
-                    packages['${PN}'] = ''
-                    pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
-                    all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
-                    if '&' in all_licenses:
-                        all_licenses.remove('&')
-                    extravalues['LICENSE'] = ' & '.join(all_licenses)
-
-                # Need to move S setting after inherit npm
-                for i, line in enumerate(lines_before):
-                    if line.startswith('S ='):
-                        lines_before.pop(i)
-                        lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
-                        lines_after.insert(1, line)
-                        break
-
-                return True
-
-        return False
-
-    # FIXME this is duplicated from lib/bb/fetch2/npm.py
-    def _parse_view(self, output):
-        '''
-        Parse the output of npm view --json; the last JSON result
-        is assumed to be the one that we're interested in.
-        '''
-        pdata = None
-        outdeps = {}
-        datalines = []
-        bracelevel = 0
-        for line in output.splitlines():
-            if bracelevel:
-                datalines.append(line)
-            elif '{' in line:
-                datalines = []
-                datalines.append(line)
-            bracelevel = bracelevel + line.count('{') - line.count('}')
-        if datalines:
-            pdata = json.loads('\n'.join(datalines))
-        return pdata
-
-    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
-    # (split out from _getdependencies())
-    def get_npm_data(self, pkg, version, d):
-        import bb.fetch2
-        pkgfullname = pkg
-        if version != '*' and not '/' in version:
-            pkgfullname += "@'%s'" % version
-        logger.debug(2, "Calling getdeps on %s" % pkg)
-        runenv = dict(os.environ, PATH=d.getVar('PATH'))
-        fetchcmd = "npm view %s --json" % pkgfullname
-        output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
-        data = self._parse_view(output)
-        return data
-
-    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
-    # (split out from _getdependencies())
-    def get_npm_package_dependencies(self, pdata, fetchdev):
-        dependencies = pdata.get('dependencies', {})
-        optionalDependencies = pdata.get('optionalDependencies', {})
-        dependencies.update(optionalDependencies)
-        if fetchdev:
-            devDependencies = pdata.get('devDependencies', {})
-            dependencies.update(devDependencies)
-        else:
-            devDependencies = {}
-        depsfound = {}
-        optdepsfound = {}
-        devdepsfound = {}
-        for dep in dependencies:
-            if dep in optionalDependencies:
-                optdepsfound[dep] = dependencies[dep]
-            elif dep in devDependencies:
-                devdepsfound[dep] = dependencies[dep]
-            else:
-                depsfound[dep] = dependencies[dep]
-        return depsfound, optdepsfound, devdepsfound
-
-    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
-    # (split out from _getdependencies())
-    def check_npm_optional_dependency(self, pdata):
-        pkg_os = pdata.get('os', None)
-        if pkg_os:
-            if not isinstance(pkg_os, list):
-                pkg_os = [pkg_os]
-            blacklist = False
-            for item in pkg_os:
-                if item.startswith('!'):
-                    blacklist = True
-                    break
-            if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
-                pkg = pdata.get('name', 'Unnamed package')
-                logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
-                return False
-        return True
+        if not files:
+            return False
 
+        with open(files[0], "r") as f:
+            data = json.load(f)
+
+        if "name" not in data or "version" not in data:
+            return False
+
+        extravalues["PN"] = self._npm_name(data["name"])
+        extravalues["PV"] = data["version"]
+
+        if "description" in data:
+            extravalues["SUMMARY"] = data["description"]
+
+        if "homepage" in data:
+            extravalues["HOMEPAGE"] = data["homepage"]
+
+        dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
+        registry = self._get_registry(lines_before)
+
+        bb.note("Checking if npm is available ...")
+        # The native npm is used here (and not the host one) to ensure that the
+        # npm version is high enough to ensure an efficient dependency tree
+        # resolution and avoid issue with the shrinkwrap file format.
+        # Moreover the native npm is mandatory for the build.
+        bindir = self._ensure_npm()
+
+        d = bb.data.createCopy(TINFOIL.config_data)
+        d.prependVar("PATH", bindir + ":")
+        d.setVar("S", srctree)
+
+        bb.note("Generating shrinkwrap file ...")
+        # To generate the shrinkwrap file the dependencies have to be installed
+        # first. During the generation process some files may be updated /
+        # deleted. By default devtool tracks the diffs in the srctree and raises
+        # errors when finishing the recipe if some diffs are found.
+        git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
+        if os.path.exists(git_exclude_file):
+            with open(git_exclude_file, "r+") as f:
+                lines = f.readlines()
+                for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
+                    if line not in lines:
+                        f.write(line + "\n")
+
+        lock_file = os.path.join(srctree, "package-lock.json")
+        lock_copy = lock_file + ".copy"
+        if os.path.exists(lock_file):
+            bb.utils.copyfile(lock_file, lock_copy)
+
+        self._run_npm_install(d, srctree, registry, dev)
+        shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
+
+        if os.path.exists(lock_copy):
+            bb.utils.movefile(lock_copy, lock_file)
+
+        # Add the shrinkwrap file as 'extrafiles'
+        shrinkwrap_copy = shrinkwrap_file + ".copy"
+        bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
+        extravalues.setdefault("extrafiles", {})
+        extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
+
+        url_local = "npmsw://%s" % shrinkwrap_file
+        url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
+
+        if dev:
+            url_local += ";dev=1"
+            url_recipe += ";dev=1"
+
+        # Add the npmsw url in the SRC_URI of the generated recipe
+        def _handle_srcuri(varname, origvalue, op, newlines):
+            """Update the version value and add the 'npmsw://' url"""
+            value = origvalue.replace("version=" + data["version"], "version=${PV}")
+            value = value.replace("version=latest", "version=${PV}")
+            values = [line.strip() for line in value.strip('\n').splitlines()]
+            values.append(url_recipe)
+            return values, None, 4, False
+
+        (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
+        lines_before[:] = [line.rstrip('\n') for line in newlines]
+
+        # In order to generate correct licence checksums in the recipe the
+        # dependencies have to be fetched again using the npmsw url
+        bb.note("Fetching npm dependencies ...")
+        bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
+        fetcher = bb.fetch2.Fetch([url_local], d)
+        fetcher.download()
+        fetcher.unpack(srctree)
+
+        bb.note("Handling licences ...")
+        (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
+        extravalues["LIC_FILES_CHKSUM"] = licfiles
+        split_pkg_licenses(guess_license(srctree, d), packages, lines_after, [])
+
+        classes.append("npm")
+        handled.append("buildsystem")
+
+        return True
 
 def register_recipe_handlers(handlers):
+    """Register the npm handler"""
     handlers.append((NpmRecipeHandler(), 60))
diff --git a/poky/scripts/lib/resulttool/report.py b/poky/scripts/lib/resulttool/report.py
index 692dd7a85..7ceceac80 100644
--- a/poky/scripts/lib/resulttool/report.py
+++ b/poky/scripts/lib/resulttool/report.py
@@ -212,7 +212,21 @@ class ResultsTextReport(object):
                                  maxlen=maxlen)
         print(output)
 
-    def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map, raw_test):
+    def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map, raw_test, selected_test_case_only):
+        def print_selected_testcase_result(testresults, selected_test_case_only):
+            for testsuite in testresults:
+                for resultid in testresults[testsuite]:
+                    result = testresults[testsuite][resultid]['result']
+                    test_case_result = result.get(selected_test_case_only, {})
+                    if test_case_result.get('status'):
+                        print('Found selected test case result for %s from %s' % (selected_test_case_only,
+                                                                                           resultid))
+                        print(test_case_result['status'])
+                    else:
+                        print('Could not find selected test case result for %s from %s' % (selected_test_case_only,
+                                                                                           resultid))
+                    if test_case_result.get('log'):
+                        print(test_case_result['log'])
         test_count_reports = []
         configmap = resultutils.store_map
         if use_regression_map:
@@ -235,12 +249,18 @@ class ResultsTextReport(object):
             for testsuite in testresults:
                 result = testresults[testsuite].get(raw_test, {})
                 if result:
-                    raw_results[testsuite] = result
+                    raw_results[testsuite] = {raw_test: result}
             if raw_results:
-                print(json.dumps(raw_results, sort_keys=True, indent=4))
+                if selected_test_case_only:
+                    print_selected_testcase_result(raw_results, selected_test_case_only)
+                else:
+                    print(json.dumps(raw_results, sort_keys=True, indent=4))
             else:
                 print('Could not find raw test result for %s' % raw_test)
             return 0
+        if selected_test_case_only:
+            print_selected_testcase_result(testresults, selected_test_case_only)
+            return 0
         for testsuite in testresults:
             for resultid in testresults[testsuite]:
                 skip = False
@@ -268,7 +288,7 @@ class ResultsTextReport(object):
 def report(args, logger):
     report = ResultsTextReport()
     report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map,
-                            args.raw_test_only)
+                            args.raw_test_only, args.selected_test_case_only)
     return 0
 
 def register_commands(subparsers):
@@ -287,4 +307,7 @@ def register_commands(subparsers):
                               help='instead of the default "store_map", use the "regression_map" for report')
     parser_build.add_argument('-r', '--raw_test_only', default='',
                               help='output raw test result only for the user provided test result id')
-
+    parser_build.add_argument('-s', '--selected_test_case_only', default='',
+                              help='output selected test case result for the user provided test case id, if both test '
+                                   'result id and test case id are provided then output the selected test case result '
+                                   'from the provided test result id')
diff --git a/poky/scripts/lib/resulttool/resultutils.py b/poky/scripts/lib/resulttool/resultutils.py
index f0ae8ec1c..5fec01f6f 100644
--- a/poky/scripts/lib/resulttool/resultutils.py
+++ b/poky/scripts/lib/resulttool/resultutils.py
@@ -127,10 +127,7 @@ def decode_log(logdata):
             data = logdata.get("compressed")
             data = base64.b64decode(data.encode("utf-8"))
             data = zlib.decompress(data)
-            try:
-                return data.decode("utf-8")
-            except UnicodeDecodeError:
-                return data
+            return data.decode("utf-8", errors='ignore')
     return None
 
 def ptestresult_get_log(results, section):
diff --git a/poky/scripts/lib/scriptutils.py b/poky/scripts/lib/scriptutils.py
index 45bdaf5f4..f92255d8d 100644
--- a/poky/scripts/lib/scriptutils.py
+++ b/poky/scripts/lib/scriptutils.py
@@ -77,7 +77,6 @@ def logger_setup_color(logger, color='auto'):
 
 
 def load_plugins(logger, plugins, pluginpath):
-    import imp
 
     def load_plugin(name):
         logger.debug('Loading plugin %s' % name)
diff --git a/poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
index c8d9f121b..22b45217f 100644
--- a/poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
+++ b/poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
@@ -4,5 +4,5 @@
 
 include common.wks.inc
 
-bootloader  --timeout=0  --append="vga=0 rw oprofile.timer=1 rootfstype=ext4 "
+bootloader  --timeout=0  --append="rw oprofile.timer=1 rootfstype=ext4 "
 
diff --git a/poky/scripts/lib/wic/engine.py b/poky/scripts/lib/wic/engine.py
index 7e6620747..018815b96 100644
--- a/poky/scripts/lib/wic/engine.py
+++ b/poky/scripts/lib/wic/engine.py
@@ -280,7 +280,7 @@ class Disk:
     def __getattr__(self, name):
         """Get path to the executable in a lazy way."""
         if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
-                    "resize2fs", "mkswap", "mkdosfs", "debugfs"):
+                    "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
             aname = "_%s" % name
             if aname not in self.__dict__:
                 setattr(self, aname, find_executable(name, self.paths))
@@ -291,7 +291,7 @@ class Disk:
 
     def _get_part_image(self, pnum):
         if pnum not in self.partitions:
-            raise WicError("Partition %s is not in the image")
+            raise WicError("Partition %s is not in the image" % pnum)
         part = self.partitions[pnum]
         # check if fstype is supported
         for fstype in self.fstypes:
@@ -314,6 +314,9 @@ class Disk:
                     seek=self.partitions[pnum].start)
 
     def dir(self, pnum, path):
+        if pnum not in self.partitions:
+            raise WicError("Partition %s is not in the image" % pnum)
+
         if self.partitions[pnum].fstype.startswith('ext'):
             return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
                                                          self._get_part_image(pnum),
@@ -323,16 +326,31 @@ class Disk:
                                                    self._get_part_image(pnum),
                                                    path))
 
-    def copy(self, src, pnum, path):
+    def copy(self, src, dest):
         """Copy partition image into wic image."""
+        pnum =  dest.part if isinstance(src, str) else src.part
+
         if self.partitions[pnum].fstype.startswith('ext'):
-            cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
-                      format(path, src, os.path.basename(src),
+            if isinstance(src, str):
+                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
+                      format(os.path.dirname(dest.path), src, os.path.basename(src),
                              self.debugfs, self._get_part_image(pnum))
+            else: # copy from wic
+                # run both dump and rdump to support both files and directory
+                cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
+                      format(os.path.dirname(src.path), src.path,
+                             dest, src.path, dest, self.debugfs,
+                             self._get_part_image(pnum))
         else: # fat
-            cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
+            if isinstance(src, str):
+                cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
+                                                  self._get_part_image(pnum),
+                                                  src, dest.path)
+            else:
+                cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
                                                   self._get_part_image(pnum),
-                                                  src, path)
+                                                  src.path, dest)
+
         exec_cmd(cmd, as_shell=True)
         self._put_part_image(pnum)
 
@@ -424,7 +442,7 @@ class Disk:
             outf.flush()
 
         def read_ptable(path):
-            out = exec_cmd("{} -dJ {}".format(self.sfdisk, path))
+            out = exec_cmd("{} -J {}".format(self.sfdisk, path))
             return json.loads(out)
 
         def write_ptable(parts, target):
@@ -525,7 +543,8 @@ class Disk:
                         logger.info("creating swap partition {}".format(pnum))
                         label = part.get("name")
                         label_str = "-L {}".format(label) if label else ''
-                        uuid = part.get("uuid")
+                        out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum)))
+                        uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42]
                         uuid_str = "-U {}".format(uuid) if uuid else ''
                         with open(partfname, 'w') as sparse:
                             os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
@@ -551,11 +570,15 @@ def wic_ls(args, native_sysroot):
 
 def wic_cp(args, native_sysroot):
     """
-    Copy local file or directory to the vfat partition of
+    Copy file or directory to/from the vfat/ext partition of
     partitioned image.
     """
-    disk = Disk(args.dest.image, native_sysroot)
-    disk.copy(args.src, args.dest.part, args.dest.path)
+    if isinstance(args.dest, str):
+        disk = Disk(args.src.image, native_sysroot)
+    else:
+        disk = Disk(args.dest.image, native_sysroot)
+    disk.copy(args.src, args.dest)
+
 
 def wic_rm(args, native_sysroot):
     """
diff --git a/poky/scripts/lib/wic/help.py b/poky/scripts/lib/wic/help.py
index 812ebe3ec..bd3a2b97d 100644
--- a/poky/scripts/lib/wic/help.py
+++ b/poky/scripts/lib/wic/help.py
@@ -341,12 +341,15 @@ DESCRIPTION
 
 wic_cp_usage = """
 
- Copy files and directories to the vfat or ext* partition
+ Copy files and directories to/from the vfat or ext* partition
 
- usage: wic cp <src> <image>:<partition>[<path>] [--native-sysroot <path>]
+ usage: wic cp <src> <dest> [--native-sysroot <path>]
 
- This command  copies local files or directories to the vfat or ext* partitions
-of partitioned  image.
+ source/destination image in format <image>:<partition>[<path>]
+
+ This command copies files or directories either
+  - from local to vfat or ext* partitions of partitioned image
+  - from vfat or ext* partitions of partitioned image to local
 
  See 'wic help cp' for more detailed instructions.
 
@@ -355,16 +358,18 @@ of partitioned  image.
 wic_cp_help = """
 
 NAME
-    wic cp - copy files and directories to the vfat or ext* partitions
+    wic cp - copy files and directories to/from the vfat or ext* partitions
 
 SYNOPSIS
-    wic cp <src> <image>:<partition>
-    wic cp <src> <image>:<partition><path>
-    wic cp <src> <image>:<partition><path> --native-sysroot <path>
+    wic cp <src> <dest>:<partition>
+    wic cp <src>:<partition> <dest>
+    wic cp <src> <dest-image>:<partition><path>
+    wic cp <src> <dest-image>:<partition><path> --native-sysroot <path>
 
 DESCRIPTION
-    This command copies files and directories to the vfat or ext* partition of
-    the partitioned image.
+    This command copies files or directories either
+      - from local to vfat or ext* partitions of partitioned image
+      - from vfat or ext* partitions of partitioned image to local
 
     The first form of it copies file or directory to the root directory of
     the partition:
@@ -397,6 +402,10 @@ DESCRIPTION
                4 files                   0 bytes
                                 15 675 392 bytes free
 
+    The third form of the command copies file or directory from the specified directory
+    on the partition to local:
+       $ wic cp tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/vmlinuz test
+
     The -n option is used to specify the path to the native sysroot
     containing the tools(parted and mtools) to use.
 """
@@ -527,7 +536,8 @@ DESCRIPTION
 
     Source plugins can also be implemented and added by external
     layers - any plugins found in a scripts/lib/wic/plugins/source/
-    directory in an external layer will also be made available.
+    or lib/wic/plugins/source/ directory in an external layer will
+    also be made available.
 
     When the wic implementation needs to invoke a partition-specific
     implementation, it looks for the plugin that has the same name as
@@ -960,6 +970,26 @@ DESCRIPTION
                          is omitted, not the directory itself. This option only
                          has an effect with the rootfs source plugin.
 
+         --include-path: This option is specific to wic. It adds the contents
+                         of the given path or a rootfs to the resulting image.
+                         The option contains two fields, the origin and the
+                         destination. When the origin is a rootfs, it follows
+                         the same logic as the rootfs-dir argument and the
+                         permissions and owners are kept. When the origin is a
+                         path, it is relative to the directory in which wic is
+                         running not the rootfs itself so use of an absolute
+                         path is recommended, and the owner and group is set to
+                         root:root. If no destination is given it is
+                         automatically set to the root of the rootfs. This
+                         option only has an effect with the rootfs source
+                         plugin.
+
+         --change-directory: This option is specific to wic. It changes to the
+                             given directory before copying the files. This
+                             option is useful when we want to split a rootfs in
+                             multiple partitions and we want to keep the right
+                             permissions and usernames in all the partitions.
+
          --extra-space: This option is specific to wic. It adds extra
                         space after the space filled by the content
                         of the partition. The final size can go
diff --git a/poky/scripts/lib/wic/ksparser.py b/poky/scripts/lib/wic/ksparser.py
index 6a643ba3a..b8befe78e 100644
--- a/poky/scripts/lib/wic/ksparser.py
+++ b/poky/scripts/lib/wic/ksparser.py
@@ -137,6 +137,8 @@ class KickStart():
         part.add_argument('--active', action='store_true')
         part.add_argument('--align', type=int)
         part.add_argument('--exclude-path', nargs='+')
+        part.add_argument('--include-path', nargs='+', action='append')
+        part.add_argument('--change-directory')
         part.add_argument("--extra-space", type=sizetype)
         part.add_argument('--fsoptions', dest='fsopts')
         part.add_argument('--fstype', default='vfat',
@@ -245,6 +247,11 @@ class KickStart():
                     elif line.startswith('bootloader'):
                         if not self.bootloader:
                             self.bootloader = parsed
+                            # Concatenate the strings set in APPEND
+                            append_var = get_bitbake_var("APPEND")
+                            if append_var:
+                                self.bootloader.append = ' '.join(filter(None, \
+                                                         (self.bootloader.append, append_var)))
                         else:
                             err = "%s:%d: more than one bootloader specified" \
                                       % (confpath, lineno)
diff --git a/poky/scripts/lib/wic/misc.py b/poky/scripts/lib/wic/misc.py
index 1f199b9f2..91975ba15 100644
--- a/poky/scripts/lib/wic/misc.py
+++ b/poky/scripts/lib/wic/misc.py
@@ -45,7 +45,8 @@ NATIVE_RECIPES = {"bmaptool": "bmap-tools",
                   "parted": "parted",
                   "sfdisk": "util-linux",
                   "sgdisk": "gptfdisk",
-                  "syslinux": "syslinux"
+                  "syslinux": "syslinux",
+                  "tar": "tar"
                  }
 
 def runtool(cmdln_or_args):
@@ -112,6 +113,15 @@ def exec_cmd(cmd_and_args, as_shell=False):
     """
     return _exec_cmd(cmd_and_args, as_shell)[1]
 
+def find_executable(cmd, paths):
+    recipe = cmd
+    if recipe in NATIVE_RECIPES:
+        recipe =  NATIVE_RECIPES[recipe]
+    provided = get_bitbake_var("ASSUME_PROVIDED")
+    if provided and "%s-native" % recipe in provided:
+        return True
+
+    return spawn.find_executable(cmd, paths)
 
 def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
     """
@@ -136,7 +146,7 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
     logger.debug("exec_native_cmd: %s", native_cmd_and_args)
 
     # If the command isn't in the native sysroot say we failed.
-    if spawn.find_executable(args[0], native_paths):
+    if find_executable(args[0], native_paths):
         ret, out = _exec_cmd(native_cmd_and_args, True)
     else:
         ret = 127
diff --git a/poky/scripts/lib/wic/partition.py b/poky/scripts/lib/wic/partition.py
index d809408e1..7d9dd616a 100644
--- a/poky/scripts/lib/wic/partition.py
+++ b/poky/scripts/lib/wic/partition.py
@@ -30,6 +30,8 @@ class Partition():
         self.device = None
         self.extra_space = args.extra_space
         self.exclude_path = args.exclude_path
+        self.include_path = args.include_path
+        self.change_directory = args.change_directory
         self.fsopts = args.fsopts
         self.fstype = args.fstype
         self.label = args.label
@@ -189,7 +191,7 @@ class Partition():
                            (self.mountpoint, self.size, self.fixed_size))
 
     def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir,
-                       native_sysroot, real_rootfs = True):
+                       native_sysroot, real_rootfs = True, pseudo_dir = None):
         """
         Prepare content for a rootfs partition i.e. create a partition
         and fill it from a /rootfs dir.
@@ -197,15 +199,14 @@ class Partition():
         Currently handles ext2/3/4, btrfs, vfat and squashfs.
         """
         p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
-        p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR",
-                                         "%s/../pseudo" %  rootfs_dir)
-        p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir)
-        p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1")
-        pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
-        pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir
-        pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd
-        pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp
-        pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+        if (pseudo_dir):
+            pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
+            pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
+            pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
+            pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+            pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+        else:
+            pseudo = None
 
         rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label,
                                          self.lineno, self.fstype)
diff --git a/poky/scripts/lib/wic/pluginbase.py b/poky/scripts/lib/wic/pluginbase.py
index f74d6430f..d9b4e5774 100644
--- a/poky/scripts/lib/wic/pluginbase.py
+++ b/poky/scripts/lib/wic/pluginbase.py
@@ -18,7 +18,7 @@ from wic.misc import get_bitbake_var
 
 PLUGIN_TYPES = ["imager", "source"]
 
-SCRIPTS_PLUGIN_DIR = "scripts/lib/wic/plugins"
+SCRIPTS_PLUGIN_DIR = ["scripts/lib/wic/plugins", "lib/wic/plugins"]
 
 logger = logging.getLogger('wic')
 
@@ -38,10 +38,11 @@ class PluginMgr:
             cls._plugin_dirs = [os.path.join(os.path.dirname(__file__), 'plugins')]
             layers = get_bitbake_var("BBLAYERS") or ''
             for layer_path in layers.split():
-                path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR)
-                path = os.path.abspath(os.path.expanduser(path))
-                if path not in cls._plugin_dirs and os.path.isdir(path):
-                    cls._plugin_dirs.insert(0, path)
+                for script_plugin_dir in SCRIPTS_PLUGIN_DIR:
+                    path = os.path.join(layer_path, script_plugin_dir)
+                    path = os.path.abspath(os.path.expanduser(path))
+                    if path not in cls._plugin_dirs and os.path.isdir(path):
+                        cls._plugin_dirs.insert(0, path)
 
         if ptype not in PLUGINS:
             # load all ptype plugins
diff --git a/poky/scripts/lib/wic/plugins/imager/direct.py b/poky/scripts/lib/wic/plugins/imager/direct.py
index 2441cc33a..2d06c242b 100644
--- a/poky/scripts/lib/wic/plugins/imager/direct.py
+++ b/poky/scripts/lib/wic/plugins/imager/direct.py
@@ -403,7 +403,7 @@ class PartitionedImage():
                 # Reserve a sector for EBR for every logical partition
                 # before alignment is performed.
                 if part.type == 'logical':
-                    self.offset += 1
+                    self.offset += 2
 
             align_sectors = 0
             if part.align:
@@ -446,7 +446,7 @@ class PartitionedImage():
                         self.extendedpart = part.num
                     else:
                         self.extended_size_sec += align_sectors
-                    self.extended_size_sec += part.size_sec + 1
+                    self.extended_size_sec += part.size_sec + 2
                 else:
                     self.primary_part_num += 1
                     part.num = self.primary_part_num
@@ -512,7 +512,7 @@ class PartitionedImage():
                 # add a sector at the back, so that there is enough
                 # room for all logical partitions.
                 self._create_partition(self.path, "extended",
-                                       None, part.start - 1,
+                                       None, part.start - 2,
                                        self.extended_size_sec)
 
             if part.fstype == "swap":
@@ -580,9 +580,7 @@ class PartitionedImage():
                                 self.native_sysroot)
 
     def cleanup(self):
-        # remove partition images
-        for image in set(self.partimages):
-            os.remove(image)
+        pass
 
     def assemble(self):
         logger.debug("Installing partitions")
diff --git a/poky/scripts/lib/wic/plugins/source/rawcopy.py b/poky/scripts/lib/wic/plugins/source/rawcopy.py
index 82970ce51..3c4997d8b 100644
--- a/poky/scripts/lib/wic/plugins/source/rawcopy.py
+++ b/poky/scripts/lib/wic/plugins/source/rawcopy.py
@@ -57,7 +57,7 @@ class RawCopyPlugin(SourcePlugin):
             raise WicError("No file specified")
 
         src = os.path.join(kernel_dir, source_params['file'])
-        dst = os.path.join(cr_workdir, "%s.%s" % (source_params['file'], part.lineno))
+        dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
 
         if not os.path.exists(os.path.dirname(dst)):
             os.makedirs(os.path.dirname(dst))
diff --git a/poky/scripts/lib/wic/plugins/source/rootfs.py b/poky/scripts/lib/wic/plugins/source/rootfs.py
index e26e95b99..f1db83f8a 100644
--- a/poky/scripts/lib/wic/plugins/source/rootfs.py
+++ b/poky/scripts/lib/wic/plugins/source/rootfs.py
@@ -17,10 +17,11 @@ import shutil
 import sys
 
 from oe.path import copyhardlinktree
+from pathlib import Path
 
 from wic import WicError
 from wic.pluginbase import SourcePlugin
-from wic.misc import get_bitbake_var
+from wic.misc import get_bitbake_var, exec_native_cmd
 
 logger = logging.getLogger('wic')
 
@@ -32,6 +33,22 @@ class RootfsPlugin(SourcePlugin):
     name = 'rootfs'
 
     @staticmethod
+    def __validate_path(cmd, rootfs_dir, path):
+        if os.path.isabs(path):
+            logger.error("%s: Must be relative: %s" % (cmd, orig_path))
+            sys.exit(1)
+
+        # Disallow climbing outside of parent directory using '..',
+        # because doing so could be quite disastrous (we will delete the
+        # directory, or modify a directory outside OpenEmbedded).
+        full_path = os.path.realpath(os.path.join(rootfs_dir, path))
+        if not full_path.startswith(os.path.realpath(rootfs_dir)):
+            logger.error("%s: Must point inside the rootfs:" % (cmd, path))
+            sys.exit(1)
+
+        return full_path
+
+    @staticmethod
     def __get_rootfs_dir(rootfs_dir):
         if os.path.isdir(rootfs_dir):
             return os.path.realpath(rootfs_dir)
@@ -44,6 +61,15 @@ class RootfsPlugin(SourcePlugin):
 
         return os.path.realpath(image_rootfs_dir)
 
+    @staticmethod
+    def __get_pseudo(native_sysroot, rootfs, pseudo_dir):
+        pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
+        pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
+        pseudo += "export PSEUDO_PASSWD=%s;" % rootfs
+        pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+        pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+        return pseudo
+
     @classmethod
     def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
                              oe_builddir, bootimg_dir, kernel_dir,
@@ -68,10 +94,16 @@ class RootfsPlugin(SourcePlugin):
                                "it is not a valid path, exiting" % part.rootfs_dir)
 
         part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
+        pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo")
+        if not os.path.lexists(pseudo_dir):
+            logger.warn("%s folder does not exist. "
+                        "Usernames and permissions will be invalid " % pseudo_dir)
+            pseudo_dir = None
 
         new_rootfs = None
+        new_pseudo = None
         # Handle excluded paths.
-        if part.exclude_path is not None:
+        if part.exclude_path or part.include_path or part.change_directory:
             # We need a new rootfs directory we can delete files from. Copy to
             # workdir.
             new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno))
@@ -79,22 +111,93 @@ class RootfsPlugin(SourcePlugin):
             if os.path.lexists(new_rootfs):
                 shutil.rmtree(os.path.join(new_rootfs))
 
-            copyhardlinktree(part.rootfs_dir, new_rootfs)
+            if part.change_directory:
+                cd = part.change_directory
+                if cd[-1] == '/':
+                    cd = cd[:-1]
+                orig_dir = cls.__validate_path("--change-directory", part.rootfs_dir, cd)
+            else:
+                orig_dir = part.rootfs_dir
+            copyhardlinktree(orig_dir, new_rootfs)
+
+            # Convert the pseudo directory to its new location
+            if (pseudo_dir):
+                new_pseudo = os.path.realpath(
+                             os.path.join(cr_workdir, "pseudo%d" % part.lineno))
+                if os.path.lexists(new_pseudo):
+                    shutil.rmtree(new_pseudo)
+                os.mkdir(new_pseudo)
+                shutil.copy(os.path.join(pseudo_dir, "files.db"),
+                            os.path.join(new_pseudo, "files.db"))
+
+                pseudo_cmd = "%s -B -m %s -M %s" % (cls.__get_pseudo(native_sysroot,
+                                                                     new_rootfs,
+                                                                     new_pseudo),
+                                                    orig_dir, new_rootfs)
+                exec_native_cmd(pseudo_cmd, native_sysroot)
+
+            for in_path in part.include_path or []:
+                #parse arguments
+                include_path = in_path[0]
+                if len(in_path) > 2:
+                    logger.error("'Invalid number of arguments for include-path")
+                    sys.exit(1)
+                if len(in_path) == 2:
+                    path = in_path[1]
+                else:
+                    path = None
+
+                # Pack files to be included into a tar file.
+                # We need to create a tar file, because that way we can keep the
+                # permissions from the files even when they belong to different
+                # pseudo enviroments.
+                # If we simply copy files using copyhardlinktree/copytree... the
+                # copied files will belong to the user running wic.
+                tar_file = os.path.realpath(
+                           os.path.join(cr_workdir, "include-path%d.tar" % part.lineno))
+                if os.path.isfile(include_path):
+                    parent = os.path.dirname(os.path.realpath(include_path))
+                    tar_cmd = "tar c --owner=root --group=root -f %s -C %s %s" % (
+                                tar_file, parent, os.path.relpath(include_path, parent))
+                    exec_native_cmd(tar_cmd, native_sysroot)
+                else:
+                    if include_path in krootfs_dir:
+                        include_path = krootfs_dir[include_path]
+                    include_path = cls.__get_rootfs_dir(include_path)
+                    include_pseudo = os.path.join(include_path, "../pseudo")
+                    if os.path.lexists(include_pseudo):
+                        pseudo = cls.__get_pseudo(native_sysroot, include_path,
+                                                  include_pseudo)
+                        tar_cmd = "tar cf %s -C %s ." % (tar_file, include_path)
+                    else:
+                        pseudo = None
+                        tar_cmd = "tar c --owner=root --group=root -f %s -C %s ." % (
+                                tar_file, include_path)
+                    exec_native_cmd(tar_cmd, native_sysroot, pseudo)
+
+                #create destination
+                if path:
+                    destination = cls.__validate_path("--include-path", new_rootfs, path)
+                    Path(destination).mkdir(parents=True, exist_ok=True)
+                else:
+                    destination = new_rootfs
 
-            for orig_path in part.exclude_path:
+                #extract destination
+                untar_cmd = "tar xf %s -C %s" % (tar_file, destination)
+                if new_pseudo:
+                    pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
+                else:
+                    pseudo = None
+                exec_native_cmd(untar_cmd, native_sysroot, pseudo)
+                os.remove(tar_file)
+
+            for orig_path in part.exclude_path or []:
                 path = orig_path
-                if os.path.isabs(path):
-                    logger.error("Must be relative: --exclude-path=%s" % orig_path)
-                    sys.exit(1)
 
-                full_path = os.path.realpath(os.path.join(new_rootfs, path))
+                full_path = cls.__validate_path("--exclude-path", new_rootfs, path)
 
-                # Disallow climbing outside of parent directory using '..',
-                # because doing so could be quite disastrous (we will delete the
-                # directory).
-                if not full_path.startswith(new_rootfs):
-                    logger.error("'%s' points to a path outside the rootfs" % orig_path)
-                    sys.exit(1)
+                if not os.path.lexists(full_path):
+                    continue
 
                 if path.endswith(os.sep):
                     # Delete content only.
@@ -109,4 +212,5 @@ class RootfsPlugin(SourcePlugin):
                     shutil.rmtree(full_path)
 
         part.prepare_rootfs(cr_workdir, oe_builddir,
-                            new_rootfs or part.rootfs_dir, native_sysroot)
+                            new_rootfs or part.rootfs_dir, native_sysroot,
+                            pseudo_dir = new_pseudo or pseudo_dir)