diff options
Diffstat (limited to 'poky/bitbake/lib/bb/tests/fetch.py')
| -rw-r--r-- | poky/bitbake/lib/bb/tests/fetch.py | 113 |
1 files changed, 90 insertions, 23 deletions
diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py index 8ad1c85990..ec7d83c959 100644 --- a/poky/bitbake/lib/bb/tests/fetch.py +++ b/poky/bitbake/lib/bb/tests/fetch.py @@ -430,6 +430,12 @@ class MirrorUriTest(FetcherTest): : "http://somewhere2.org/somefile_1.2.3.tar.gz", ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", + ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") + : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", + ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") + : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", + + ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", #Renaming files doesn't work #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" @@ -491,7 +497,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest): super(GitDownloadDirectoryNamingTest, self).setUp() self.recipe_url = "git://git.openembedded.org/bitbake" self.recipe_dir = "git.openembedded.org.bitbake" - self.mirror_url = "git://github.com/openembedded/bitbake.git" + self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https" self.mirror_dir = "github.com.openembedded.bitbake.git" self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') @@ -539,7 +545,7 @@ class TarballNamingTest(FetcherTest): super(TarballNamingTest, self).setUp() self.recipe_url = "git://git.openembedded.org/bitbake" self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" - self.mirror_url = "git://github.com/openembedded/bitbake.git" + self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https" self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') @@ -573,7 +579,7 @@ class GitShallowTarballNamingTest(FetcherTest): super(GitShallowTarballNamingTest, self).setUp() self.recipe_url = "git://git.openembedded.org/bitbake" self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" - self.mirror_url = "git://github.com/openembedded/bitbake.git" + self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https" self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" self.d.setVar('BB_GIT_SHALLOW', '1') @@ -620,6 +626,9 @@ class FetcherLocalTest(FetcherTest): os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) + bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) + bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) + bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) self.d.setVar("FILESPATH", self.localsrcdir) def fetchUnpack(self, uris): @@ -674,6 +683,18 @@ class FetcherLocalTest(FetcherTest): with self.assertRaises(bb.fetch2.UnpackError): self.fetchUnpack(['file://a;subdir=/bin/sh']) + def test_local_striplevel(self): + tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) + self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) + + def test_local_striplevel_gzip(self): + tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) + self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) + + def test_local_striplevel_bzip2(self): + tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) + self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) + def dummyGitTest(self, suffix): # Create dummy local Git repo src_dir = tempfile.mkdtemp(dir=self.tempdir, @@ -875,17 +896,25 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_fetch_premirror_specify_downloadfilename_regex_uri(self): self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") - fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) fetcher.download() - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() # BZ13039 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") - fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) fetcher.download() - self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) + + @skipIfNoNetwork() + def test_fetch_premirror_use_downloadfilename_to_fetch(self): + # Ensure downloadfilename is used when fetching from premirror. + self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") + fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) + fetcher.download() + self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) @skipIfNoNetwork() def gitfetcher(self, url1, url2): @@ -996,7 +1025,7 @@ class FetcherNetworkTest(FetcherTest): def test_git_submodule_dbus_broker(self): # The following external repositories have show failures in fetch and unpack operations # We want to avoid regressions! - url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" + url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1012,7 +1041,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule_CLI11(self): - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1027,12 +1056,12 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule_update_CLI11(self): """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # CLI11 that pulls in a newer nlohmann-json - url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" + url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1046,7 +1075,7 @@ class FetcherNetworkTest(FetcherTest): @skipIfNoNetwork() def test_git_submodule_aktualizr(self): - url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" + url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1066,7 +1095,7 @@ class FetcherNetworkTest(FetcherTest): """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ # This repository also has submodules where the module (name), path and url do not align - url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699" + url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1124,7 +1153,7 @@ class SVNTest(FetcherTest): bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) # Github will emulate SVN. Use this to check if we're downloding... - bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .", + bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) bb.process.run("svn commit --non-interactive -m 'Add external'", cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) @@ -1242,7 +1271,7 @@ class FetchLatestVersionTest(FetcherTest): test_git_uris = { # version pattern "X.Y.Z" - ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") + ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") : "1.99.4", # version pattern "vX.Y" # mirror of git.infradead.org since network issues interfered with testing @@ -1269,9 +1298,9 @@ class FetchLatestVersionTest(FetcherTest): : "0.4.3", ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))") : "11.0.0", - ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") + ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") : "1.3.59", - ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") + ("remake", "git://github.com/rocky/remake.git;protocol=https", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") : "3.82+dbg0.9", } @@ -1317,6 +1346,12 @@ class FetchLatestVersionTest(FetcherTest): # http://ftp.debian.org/debian/pool/main/d/db5.3/ ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") : "5.3.10", + # + # packages where the tarball compression changed in the new version + # + # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz + ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") + : "2.8", } @skipIfNoNetwork() @@ -1365,9 +1400,6 @@ class FetchCheckStatusTest(FetcherTest): "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", - "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz", - "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz", - "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz", # GitHub releases are hosted on Amazon S3, which doesn't support HEAD "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" ] @@ -2058,7 +2090,7 @@ class GitShallowTest(FetcherTest): @skipIfNoNetwork() def test_bitbake(self): - self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir) + self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) self.git('config core.bare true', cwd=self.srcdir) self.git('fetch', cwd=self.srcdir) @@ -2149,7 +2181,7 @@ class GitLfsTest(FetcherTest): def test_lfs_enabled(self): import shutil - uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir + uri = 'git://%s;protocol=file;lfs=1' % self.srcdir self.d.setVar('SRC_URI', uri) # Careful: suppress initial attempt at downloading until @@ -2174,7 +2206,7 @@ class GitLfsTest(FetcherTest): def test_lfs_disabled(self): import shutil - uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir + uri = 'git://%s;protocol=file;lfs=0' % self.srcdir self.d.setVar('SRC_URI', uri) # In contrast to test_lfs_enabled(), allow the implicit download @@ -2228,6 +2260,41 @@ class GitURLWithSpacesTest(FetcherTest): self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) +class CrateTest(FetcherTest): + def test_crate_url(self): + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + + def test_crate_url_multi(self): + + uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) + self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) + class NPMTest(FetcherTest): def skipIfNoNpm(): import shutil |