From f8caae304a2fa94cf2770b72a313ee843b2f177b Mon Sep 17 00:00:00 2001 From: Brad Bishop Date: Mon, 25 Mar 2019 13:13:56 -0400 Subject: poky: refresh thud: 506ec088e5..e4c0a8a7cb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update poky to thud HEAD. Alexander Kanavin (1): ca-certificates: upgrade 20180409 -> 20190110 André Draszik (1): systemd: RDEPENDS on util-linux-umount Changqing Li (1): libsndfile1: Security fix CVE-2018-19432 Chen Qi (1): target-sdk-provides-dummy: add more perl modules to avoid populate_sdk failure Douglas Royds (1): libpam: libpamc is licensed under its own BSD-style licence George McCollister (1): systemd: fix CVE-2019-6454 Jonathan Rajotte-Julien (3): lttng-ust: update to 2.10.3 lttng-modules: update to 2.10.9 lttng-tools: update to 2.9.11 Mark Hatle (10): bitbake: gitsm.py: Fix when a submodule is defined, but not initialized bitbake: gitsm.py: Add support for alternative URL formats from submodule files bitbake: tests/fetch.py: Add alternative gitsm test case bitbake: gitsm.py: Optimize code and attempt to resolve locking issue bitbake: gitsm.py: revise unpack bitbake: gitsm.py: Rework the shallow fetcher and test case bitbake: gitsm.py: Refactor the functions and simplify the class bitbake: gitsm.py: Fix relative URLs bitbake: gitsmy.py: Fix unpack of submodules of submodules bitbake: gitsm: The fetcher did not process some recursive submodules properly. Ming Liu (1): rm_work: sort the value of do_build dependencies Oleksandr Kravchuk (1): target-sdk-provides-dummy: add perl-module-overload Richard Purdie (3): target-sdk-provides-dummy: Extend to -dev and -src packages systemd: Update recent CVE patches kernel: Ensure an initramfs is added if configured Robert Yang (1): send-error-report: Add --no-ssl to use http protocol Ross Burton (1): libpng: fix CVE-2019-7317 Change-Id: I3e03c837688d49703b4989a561f3728d616abbec Signed-off-by: Brad Bishop --- poky/bitbake/lib/bb/fetch2/gitsm.py | 253 +++++++++++++++--------------------- poky/bitbake/lib/bb/tests/fetch.py | 70 +++++++++- 2 files changed, 172 insertions(+), 151 deletions(-) (limited to 'poky/bitbake') diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index 35729dbc0f..b21fed2669 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -45,60 +45,97 @@ class GitSM(Git): """ return ud.type in ['gitsm'] - @staticmethod - def parse_gitmodules(gitmodules): - modules = {} - module = "" - for line in gitmodules.splitlines(): - if line.startswith('[submodule'): - module = line.split('"')[1] - modules[module] = {} - elif module and line.strip().startswith('path'): - path = line.split('=')[1].strip() - modules[module]['path'] = path - elif module and line.strip().startswith('url'): - url = line.split('=')[1].strip() - modules[module]['url'] = url - return modules - - def update_submodules(self, ud, d): + def process_submodules(self, ud, workdir, function, d): + """ + Iterate over all of the submodules in this repository and execute + the 'function' for each of them. + """ + submodules = [] paths = {} + revision = {} uris = {} - local_paths = {} - + subrevision = {} + + def parse_gitmodules(gitmodules): + modules = {} + module = "" + for line in gitmodules.splitlines(): + if line.startswith('[submodule'): + module = line.split('"')[1] + modules[module] = {} + elif module and line.strip().startswith('path'): + path = line.split('=')[1].strip() + modules[module]['path'] = path + elif module and line.strip().startswith('url'): + url = line.split('=')[1].strip() + modules[module]['url'] = url + return modules + + # Collect the defined submodules, and their attributes for name in ud.names: try: - gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) + gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) except: # No submodules to update continue - for m, md in self.parse_gitmodules(gitmodules).items(): + for m, md in parse_gitmodules(gitmodules).items(): + try: + module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) + except: + # If the command fails, we don't have a valid file to check. If it doesn't + # fail -- it still might be a failure, see next check... + module_hash = "" + + if not module_hash: + logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) + continue + submodules.append(m) paths[m] = md['path'] + revision[m] = ud.revisions[name] uris[m] = md['url'] + subrevision[m] = module_hash.split()[2] + + # Convert relative to absolute uri based on parent uri if uris[m].startswith('..'): newud = copy.copy(ud) - newud.path = os.path.realpath(os.path.join(newud.path, md['url'])) + newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) uris[m] = Git._get_repo_url(self, newud) for module in submodules: - module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir) - module_hash = module_hash.split()[2] + # Translate the module url into a SRC_URI + + if "://" in uris[module]: + # Properly formated URL already + proto = uris[module].split(':', 1)[0] + url = uris[module].replace('%s:' % proto, 'gitsm:', 1) + else: + if ":" in uris[module]: + # Most likely an SSH style reference + proto = "ssh" + if ":/" in uris[module]: + # Absolute reference, easy to convert.. + url = "gitsm://" + uris[module].replace(':/', '/', 1) + else: + # Relative reference, no way to know if this is right! + logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) + url = "gitsm://" + uris[module].replace(':', '/', 1) + else: + # This has to be a file reference + proto = "file" + url = "gitsm://" + uris[module] - # Build new SRC_URI - proto = uris[module].split(':', 1)[0] - url = uris[module].replace('%s:' % proto, 'gitsm:', 1) url += ';protocol=%s' % proto url += ";name=%s" % module - url += ";bareclone=1;nocheckout=1;nobranch=1" + url += ";subpath=%s" % paths[module] ld = d.createCopy() # Not necessary to set SRC_URI, since we're passing the URI to # Fetch. #ld.setVar('SRC_URI', url) - ld.setVar('SRCREV_%s' % module, module_hash) + ld.setVar('SRCREV_%s' % module, subrevision[module]) # Workaround for issues with SRCPV/SRCREV_FORMAT errors # error refer to 'multiple' repositories. Only the repository @@ -106,145 +143,63 @@ class GitSM(Git): ld.setVar('SRCPV', d.getVar('SRCPV')) ld.setVar('SRCREV_FORMAT', module) - newfetch = Fetch([url], ld, cache=False) - newfetch.download() - local_paths[module] = newfetch.localpath(url) - - # Correct the submodule references to the local download version... - runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir) - - symlink_path = os.path.join(ud.clonedir, 'modules', paths[module]) - if not os.path.exists(symlink_path): - try: - os.makedirs(os.path.dirname(symlink_path), exist_ok=True) - except OSError: - pass - os.symlink(local_paths[module], symlink_path) - - return True + function(ud, url, module, paths[module], ld) - def need_update(self, ud, d): - main_repo_needs_update = Git.need_update(self, ud, d) - - # First check that the main repository has enough history fetched. If it doesn't, then we don't - # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the - # submodules' histories are recent enough. - if main_repo_needs_update: - return True - - # Now check that the submodule histories are new enough. The git-submodule command doesn't have - # any clean interface for doing this aside from just attempting the checkout (with network - # fetched disabled). - return not self.update_submodules(ud, d) + return submodules != [] def download(self, ud, d): - Git.download(self, ud, d) + def download_submodule(ud, url, module, modpath, d): + url += ";bareclone=1;nobranch=1" - if not ud.shallow or ud.localpath != ud.fullshallow: - self.update_submodules(ud, d) + # Is the following still needed? + #url += ";nocheckout=1" - def copy_submodules(self, submodules, ud, destdir, d): - if ud.bareclone: - repo_conf = destdir - else: - repo_conf = os.path.join(destdir, '.git') - - if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')): - os.mkdir(os.path.join(repo_conf, 'modules')) - - for module, md in submodules.items(): - srcpath = os.path.join(ud.clonedir, 'modules', md['path']) - modpath = os.path.join(repo_conf, 'modules', md['path']) + try: + newfetch = Fetch([url], d, cache=False) + newfetch.download() + except Exception as e: + logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) + raise - if os.path.exists(srcpath): - if os.path.exists(os.path.join(srcpath, '.git')): - srcpath = os.path.join(srcpath, '.git') + Git.download(self, ud, d) + self.process_submodules(ud, ud.clonedir, download_submodule, d) - target = modpath - if os.path.exists(modpath): - target = os.path.dirname(modpath) + def unpack(self, ud, destdir, d): + def unpack_submodules(ud, url, module, modpath, d): + url += ";bareclone=1;nobranch=1" - os.makedirs(os.path.dirname(target), exist_ok=True) - runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d) - elif os.path.exists(modpath): - # Module already exists, likely unpacked from a shallow mirror clone - pass + # Figure out where we clone over the bare submodules... + if ud.bareclone: + repo_conf = ud.destdir else: - # This is fatal, as we do NOT want git-submodule to hit the network - raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath)) - - def clone_shallow_local(self, ud, dest, d): - super(GitSM, self).clone_shallow_local(ud, dest, d) + repo_conf = os.path.join(ud.destdir, '.git') - # Copy over the submodules' fetched histories too. - repo_conf = os.path.join(dest, '.git') - - submodules = [] - for name in ud.names: try: - gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest) - except: - # No submodules to update - continue + newfetch = Fetch([url], d, cache=False) + newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath))) + except Exception as e: + logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) + raise - submodules = self.parse_gitmodules(gitmodules) - self.copy_submodules(submodules, ud, dest, d) + local_path = newfetch.localpath(url) - def unpack(self, ud, destdir, d): - Git.unpack(self, ud, destdir, d) + # Correct the submodule references to the local download version... + runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) - # Copy over the submodules' fetched histories too. - if ud.bareclone: - repo_conf = ud.destdir - else: - repo_conf = os.path.join(ud.destdir, '.git') + if ud.shallow: + runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) - update_submodules = False - paths = {} - uris = {} - local_paths = {} - for name in ud.names: + # Ensure the submodule repository is NOT set to bare, since we're checking it out... try: - gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath)) except: - # No submodules to update - continue - - submodules = self.parse_gitmodules(gitmodules) - self.copy_submodules(submodules, ud, ud.destdir, d) - - submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()] - while len(submodules_queue) != 0: - module, modpath = submodules_queue.pop() - - # add submodule children recursively - try: - gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath) - for m, md in self.parse_gitmodules(gitmodules).items(): - submodules_queue.append([m, os.path.join(modpath, 'modules', md['path'])]) - except: - # no children - pass - + logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath)) + raise - # There are submodules to update - update_submodules = True - - # Determine (from the submodule) the correct url to reference - try: - output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath) - except bb.fetch2.FetchError as e: - # No remote url defined in this submodule - continue - - local_paths[module] = output - - # Setup the local URL properly (like git submodule init or sync would do...) - runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) + Git.unpack(self, ud, destdir, d) - # Ensure the submodule repository is NOT set to bare, since we're checking it out... - runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath) + ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) - if update_submodules: + if not ud.bareclone and ret: # Run submodule update, this sets up the directories -- without touching the config runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 6848095cfd..522d2024fa 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -893,12 +893,70 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule(self): - fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d) + # URL with ssh submodules + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7" + # Original URL (comment this if you have ssh access to git.yoctoproject.org) + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee" + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') + self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') + self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') + + # Only when we're running the extended test with a submodule's submodule, can we check this. + if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): + self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') + + def test_git_submodule_dbus_broker(self): + # The following external repositories have show failures in fetch and unpack operations + # We want to avoid regressions! + url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2" + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') + + def test_git_submodule_CLI11(self): + url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" + fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted os.chdir(os.path.dirname(self.unpackdir)) fetcher.unpack(self.unpackdir) + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + + def test_git_submodule_aktualizr(self): + url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Previous cwd has been deleted + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + repo_path = os.path.join(self.tempdir, 'unpacked', 'git') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') + self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') class TrustedNetworksTest(FetcherTest): def test_trusted_network(self): @@ -1312,6 +1370,7 @@ class GitShallowTest(FetcherTest): # fetch and unpack, from the shallow tarball bb.utils.remove(self.gitdir, recurse=True) bb.utils.remove(ud.clonedir, recurse=True) + bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) # confirm that the unpacked repo is used when no git clone or git # mirror tarball is available @@ -1466,6 +1525,7 @@ class GitShallowTest(FetcherTest): self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) self.add_empty_file('asub', cwd=smdir) + self.add_empty_file('bsub', cwd=smdir) self.git('submodule init', cwd=self.srcdir) self.git('submodule add file://%s' % smdir, cwd=self.srcdir) @@ -1475,10 +1535,16 @@ class GitShallowTest(FetcherTest): uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir fetcher, ud = self.fetch_shallow(uri) + # Verify the main repository is shallow self.assertRevCount(1) - assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] + + # Verify the gitsubmodule directory is present assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) + # Verify the submodule is also shallow + self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) + + if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): def test_shallow_annex(self): self.add_empty_file('a') -- cgit v1.2.3