diff options
Diffstat (limited to 'poky/bitbake/lib/bb/fetch2')
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 12 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/gitsm.py | 18 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/npm.py | 44 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/npmsw.py | 27 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/perforce.py | 2 |
5 files changed, 63 insertions, 40 deletions
diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index ee29d89b18..666cc1306a 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -884,7 +884,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) success = True except bb.process.NotFoundError as e: - error_message = "Fetch command %s" % (e.command) + error_message = "Fetch command %s not found" % (e.command) except bb.process.ExecutionError as e: if e.stdout: output = "output:\n%s\n%s" % (e.stdout, e.stderr) @@ -1721,7 +1721,9 @@ class Fetch(object): self.d.setVar("BB_NO_NETWORK", "1") firsterr = None - verified_stamp = m.verify_donestamp(ud, self.d) + verified_stamp = False + if done: + verified_stamp = m.verify_donestamp(ud, self.d) if not done and (not verified_stamp or m.need_update(ud, self.d)): try: if not trusted_network(self.d, ud.url): @@ -1780,7 +1782,11 @@ class Fetch(object): def checkstatus(self, urls=None): """ - Check all urls exist upstream + Check all URLs exist upstream. + + Returns None if the URLs exist, raises FetchError if the check wasn't + successful but there wasn't an error (such as file not found), and + raises other exceptions in error cases. """ if not urls: diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index a4527bf364..a7110a988d 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -140,16 +140,6 @@ class GitSM(Git): if Git.need_update(self, ud, d): return True - try: - # Check for the nugget dropped by the download operation - known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ - (ud.basecmd), d, workdir=ud.clonedir) - - if ud.revisions[ud.names[0]] in known_srcrevs.split(): - return False - except bb.fetch2.FetchError: - pass - need_update_list = [] def need_update_submodule(ud, url, module, modpath, workdir, d): url += ";bareclone=1;nobranch=1" @@ -172,11 +162,6 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, need_update_submodule, d) - if len(need_update_list) == 0: - # We already have the required commits of all submodules. Drop - # a nugget so we don't need to check again. - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) if len(need_update_list) > 0: logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) @@ -209,9 +194,6 @@ class GitSM(Git): shutil.rmtree(tmpdir) else: self.process_submodules(ud, ud.clonedir, download_submodule, d) - # Drop a nugget for the srcrev we've fetched (used by need_update) - runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) def unpack(self, ud, destdir, d): def unpack_submodules(ud, url, module, modpath, workdir, d): diff --git a/poky/bitbake/lib/bb/fetch2/npm.py b/poky/bitbake/lib/bb/fetch2/npm.py index 47898509ff..e497c38dc7 100644 --- a/poky/bitbake/lib/bb/fetch2/npm.py +++ b/poky/bitbake/lib/bb/fetch2/npm.py @@ -69,17 +69,35 @@ def npm_unpack(tarball, destdir, d): bb.utils.mkdirhier(destdir) cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) cmd += " --no-same-owner" + cmd += " --delay-directory-restore" cmd += " --strip-components=1" runfetchcmd(cmd, d, workdir=destdir) + runfetchcmd("chmod -R +X %s" % (destdir), d, quiet=True, workdir=destdir) class NpmEnvironment(object): """ Using a npm config file seems more reliable than using cli arguments. This class allows to create a controlled environment for npm commands. """ - def __init__(self, d, configs=None): + def __init__(self, d, configs=None, npmrc=None): self.d = d - self.configs = configs + + if configs: + self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) + self.user_config_name = self.user_config.name + for key, value in configs: + self.user_config.write("%s=%s\n" % (key, value)) + else: + self.user_config_name = "/dev/null" + + if npmrc: + self.global_config_name = npmrc + else: + self.global_config_name = "/dev/null" + + def __del__(self): + if self.user_config: + self.user_config.close() def run(self, cmd, args=None, configs=None, workdir=None): """Run npm command in a controlled environment""" @@ -87,23 +105,19 @@ class NpmEnvironment(object): d = bb.data.createCopy(self.d) d.setVar("HOME", tmpdir) - cfgfile = os.path.join(tmpdir, "npmrc") - if not workdir: workdir = tmpdir def _run(cmd): - cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd - cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd + cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config_name) + cmd + cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd return runfetchcmd(cmd, d, workdir=workdir) - if self.configs: - for key, value in self.configs: - _run("npm config set %s %s" % (key, shlex.quote(value))) - if configs: + bb.warn("Use of configs argument of NpmEnvironment.run() function" + " is deprecated. Please use args argument instead.") for key, value in configs: - _run("npm config set %s %s" % (key, shlex.quote(value))) + cmd += " --%s=%s" % (key, shlex.quote(value)) if args: for key, value in args: @@ -165,14 +179,14 @@ class Npm(FetchMethod): def _resolve_proxy_url(self, ud, d): def _npm_view(): - configs = [] - configs.append(("json", "true")) - configs.append(("registry", ud.registry)) + args = [] + args.append(("json", "true")) + args.append(("registry", ud.registry)) pkgver = shlex.quote(ud.package + "@" + ud.version) cmd = ud.basecmd + " view %s" % pkgver env = NpmEnvironment(d) check_network_access(d, cmd, ud.registry) - view_string = env.run(cmd, configs=configs) + view_string = env.run(cmd, args=args) if not view_string: raise FetchError("Unavailable package %s" % pkgver, ud.url) diff --git a/poky/bitbake/lib/bb/fetch2/npmsw.py b/poky/bitbake/lib/bb/fetch2/npmsw.py index 0c3511d8ab..426a139653 100644 --- a/poky/bitbake/lib/bb/fetch2/npmsw.py +++ b/poky/bitbake/lib/bb/fetch2/npmsw.py @@ -24,11 +24,14 @@ import bb from bb.fetch2 import Fetch from bb.fetch2 import FetchMethod from bb.fetch2 import ParameterError +from bb.fetch2 import runfetchcmd from bb.fetch2 import URI from bb.fetch2.npm import npm_integrity from bb.fetch2.npm import npm_localfile from bb.fetch2.npm import npm_unpack from bb.utils import is_semver +from bb.utils import lockfile +from bb.utils import unlockfile def foreach_dependencies(shrinkwrap, callback=None, dev=False): """ @@ -78,6 +81,7 @@ class NpmShrinkWrap(FetchMethod): extrapaths = [] destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] destsuffix = os.path.join(*destsubdirs) + unpack = True integrity = params.get("integrity", None) resolved = params.get("resolved", None) @@ -148,7 +152,12 @@ class NpmShrinkWrap(FetchMethod): url = str(uri) - # local tarball sources and local link sources are unsupported + # Handle local tarball and link sources + elif version.startswith("file"): + localpath = version[5:] + if not version.endswith(".tgz"): + unpack = False + else: raise ParameterError("Unsupported dependency: %s" % name, ud.url) @@ -157,6 +166,7 @@ class NpmShrinkWrap(FetchMethod): "localpath": localpath, "extrapaths": extrapaths, "destsuffix": destsuffix, + "unpack": unpack, }) try: @@ -177,7 +187,7 @@ class NpmShrinkWrap(FetchMethod): # This fetcher resolves multiple URIs from a shrinkwrap file and then # forwards it to a proxy fetcher. The management of the donestamp file, # the lockfile and the checksums are forwarded to the proxy fetcher. - ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) + ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data) ud.needdonestamp = False @staticmethod @@ -187,7 +197,9 @@ class NpmShrinkWrap(FetchMethod): proxy_ud = ud.proxy.ud[proxy_url] proxy_d = ud.proxy.d proxy_ud.setup_localpath(proxy_d) + lf = lockfile(proxy_ud.lockfile) returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) + unlockfile(lf) return returns def verify_donestamp(self, ud, d): @@ -237,7 +249,16 @@ class NpmShrinkWrap(FetchMethod): for dep in manual: depdestdir = os.path.join(destdir, dep["destsuffix"]) - npm_unpack(dep["localpath"], depdestdir, d) + if dep["url"]: + npm_unpack(dep["localpath"], depdestdir, d) + else: + depsrcdir= os.path.join(destdir, dep["localpath"]) + if dep["unpack"]: + npm_unpack(depsrcdir, depdestdir, d) + else: + bb.utils.mkdirhier(depdestdir) + cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) + runfetchcmd(cmd, d, workdir=depdestdir) def clean(self, ud, d): """Clean any existing full or partial download""" diff --git a/poky/bitbake/lib/bb/fetch2/perforce.py b/poky/bitbake/lib/bb/fetch2/perforce.py index e2a41a4a12..3b6fa4b1ec 100644 --- a/poky/bitbake/lib/bb/fetch2/perforce.py +++ b/poky/bitbake/lib/bb/fetch2/perforce.py @@ -134,7 +134,7 @@ class Perforce(FetchMethod): ud.setup_revisions(d) - ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision)) + ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision)) def _buildp4command(self, ud, d, command, depot_filename=None): """ |