diff options
Diffstat (limited to 'poky/scripts/lib')
-rw-r--r-- | poky/scripts/lib/devtool/standard.py | 36 | ||||
-rw-r--r-- | poky/scripts/lib/devtool/upgrade.py | 4 | ||||
-rw-r--r-- | poky/scripts/lib/recipetool/create_npm.py | 6 | ||||
-rw-r--r-- | poky/scripts/lib/wic/ksparser.py | 17 | ||||
-rw-r--r-- | poky/scripts/lib/wic/misc.py | 7 | ||||
-rw-r--r-- | poky/scripts/lib/wic/partition.py | 44 | ||||
-rw-r--r-- | poky/scripts/lib/wic/plugins/imager/direct.py | 61 | ||||
-rw-r--r-- | poky/scripts/lib/wic/plugins/source/empty.py | 32 | ||||
-rw-r--r-- | poky/scripts/lib/wic/plugins/source/rootfs.py | 33 |
9 files changed, 187 insertions, 53 deletions
diff --git a/poky/scripts/lib/devtool/standard.py b/poky/scripts/lib/devtool/standard.py index 261d642d4..7b62b7e7b 100644 --- a/poky/scripts/lib/devtool/standard.py +++ b/poky/scripts/lib/devtool/standard.py @@ -474,7 +474,11 @@ def symlink_oelocal_files_srctree(rd,srctree): destpth = os.path.join(srctree, relpth, fn) if os.path.exists(destpth): os.unlink(destpth) - os.symlink('oe-local-files/%s' % fn, destpth) + if relpth != '.': + back_relpth = os.path.relpath(local_files_dir, root) + os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) + else: + os.symlink('oe-local-files/%s' % fn, destpth) addfiles.append(os.path.join(relpth, fn)) if addfiles: bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) @@ -589,6 +593,16 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works else: task = 'do_patch' + if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): + logger.info('The %s recipe has %s disabled. Running only ' + 'do_configure task dependencies' % (pn, task)) + + if 'depends' in d.getVarFlags('do_configure', False): + pn = d.getVarFlags('do_configure', False)['depends'] + pn = pn.replace('${PV}', d.getVar('PV')) + pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) + task = None + # Run the fetch + unpack tasks res = tinfoil.build_targets(pn, task, @@ -600,6 +614,17 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works if not res: raise DevtoolError('Extracting source for %s failed' % pn) + if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): + workshareddir = d.getVar('S') + if os.path.islink(srctree): + os.unlink(srctree) + + os.symlink(workshareddir, srctree) + + # The initial_rev file is created in devtool_post_unpack function that will not be executed if + # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful + return True, True + try: with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: initial_rev = f.read() @@ -847,10 +872,11 @@ def modify(args, config, basepath, workspace): if not initial_rev: return 1 logger.info('Source tree extracted to %s' % srctree) - # Get list of commits since this revision - (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) - commits = stdout.split() - check_commits = True + if os.path.exists(os.path.join(srctree, '.git')): + # Get list of commits since this revision + (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) + commits = stdout.split() + check_commits = True else: if os.path.exists(os.path.join(srctree, '.git')): # Check if it's a tree previously extracted by us. This is done diff --git a/poky/scripts/lib/devtool/upgrade.py b/poky/scripts/lib/devtool/upgrade.py index 0c1de8cdc..5a057e95f 100644 --- a/poky/scripts/lib/devtool/upgrade.py +++ b/poky/scripts/lib/devtool/upgrade.py @@ -178,7 +178,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee uri, rev = _get_uri(crd) if srcrev: rev = srcrev - if uri.startswith('git://'): + if uri.startswith('git://') or uri.startswith('gitsm://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') @@ -270,7 +270,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee else: logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: - if uri.startswith('git://'): + if uri.startswith('git://') or uri.startswith('gitsm://'): suffix = 'new' else: suffix = newpv diff --git a/poky/scripts/lib/recipetool/create_npm.py b/poky/scripts/lib/recipetool/create_npm.py index 579b7ae48..2bcae91df 100644 --- a/poky/scripts/lib/recipetool/create_npm.py +++ b/poky/scripts/lib/recipetool/create_npm.py @@ -204,6 +204,9 @@ class NpmRecipeHandler(RecipeHandler): self._run_npm_install(d, srctree, registry, dev) shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev) + with open(shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + if os.path.exists(lock_copy): bb.utils.movefile(lock_copy, lock_file) @@ -226,7 +229,8 @@ class NpmRecipeHandler(RecipeHandler): value = origvalue.replace("version=" + data["version"], "version=${PV}") value = value.replace("version=latest", "version=${PV}") values = [line.strip() for line in value.strip('\n').splitlines()] - values.append(url_recipe) + if "dependencies" in shrinkwrap: + values.append(url_recipe) return values, None, 4, False (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) diff --git a/poky/scripts/lib/wic/ksparser.py b/poky/scripts/lib/wic/ksparser.py index 913e3283d..3eb669da3 100644 --- a/poky/scripts/lib/wic/ksparser.py +++ b/poky/scripts/lib/wic/ksparser.py @@ -229,6 +229,23 @@ class KickStart(): err = "%s:%d: SquashFS does not support LABEL" \ % (confpath, lineno) raise KickStartError(err) + if parsed.fstype == 'msdos' or parsed.fstype == 'vfat': + if parsed.fsuuid: + if parsed.fsuuid.upper().startswith('0X'): + if len(parsed.fsuuid) > 10: + err = "%s:%d: fsuuid %s given in wks kickstart file " \ + "exceeds the length limit for %s filesystem. " \ + "It should be in the form of a 32 bit hexadecimal" \ + "number (for example, 0xABCD1234)." \ + % (confpath, lineno, parsed.fsuuid, parsed.fstype) + raise KickStartError(err) + elif len(parsed.fsuuid) > 8: + err = "%s:%d: fsuuid %s given in wks kickstart file " \ + "exceeds the length limit for %s filesystem. " \ + "It should be in the form of a 32 bit hexadecimal" \ + "number (for example, 0xABCD1234)." \ + % (confpath, lineno, parsed.fsuuid, parsed.fstype) + raise KickStartError(err) if parsed.use_label and not parsed.label: err = "%s:%d: Must set the label with --label" \ % (confpath, lineno) diff --git a/poky/scripts/lib/wic/misc.py b/poky/scripts/lib/wic/misc.py index 4b08d649c..75b219cd3 100644 --- a/poky/scripts/lib/wic/misc.py +++ b/poky/scripts/lib/wic/misc.py @@ -138,9 +138,12 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): if pseudo: cmd_and_args = pseudo + cmd_and_args - native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin" % \ + hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") + + native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin:%s" % \ (native_sysroot, native_sysroot, - native_sysroot, native_sysroot) + native_sysroot, native_sysroot, + hosttools_dir) native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ (native_paths, cmd_and_args) diff --git a/poky/scripts/lib/wic/partition.py b/poky/scripts/lib/wic/partition.py index ebe250b00..e574f40c4 100644 --- a/poky/scripts/lib/wic/partition.py +++ b/poky/scripts/lib/wic/partition.py @@ -54,6 +54,9 @@ class Partition(): self.uuid = args.uuid self.fsuuid = args.fsuuid self.type = args.type + self.updated_fstab_path = None + self.has_fstab = False + self.update_fstab_in_rootfs = False self.lineno = lineno self.source_file = "" @@ -118,11 +121,15 @@ class Partition(): return self.fixed_size if self.fixed_size else self.size def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir, - bootimg_dir, kernel_dir, native_sysroot): + bootimg_dir, kernel_dir, native_sysroot, updated_fstab_path): """ Prepare content for individual partitions, depending on partition command parameters. """ + self.updated_fstab_path = updated_fstab_path + if self.updated_fstab_path and not (self.fstype.startswith("ext") or self.fstype == "msdos"): + self.update_fstab_in_rootfs = True + if not self.source: if not self.size and not self.fixed_size: raise WicError("The %s partition has a size of zero. Please " @@ -207,11 +214,21 @@ class Partition(): p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) if (pseudo_dir): + # Canonicalize the ignore paths. This corresponds to + # calling oe.path.canonicalize(), which is used in bitbake.conf. + ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",") + canonical_paths = [] + for path in ignore_paths: + if "$" not in path: + trailing_slash = path.endswith("/") and "/" or "" + canonical_paths.append(os.path.realpath(path) + trailing_slash) + ignore_paths = ",".join(canonical_paths) + pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir pseudo += "export PSEUDO_NOSYMLINKEXP=1;" - pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % (rootfs + "," + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "")) + pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") else: pseudo = None @@ -237,7 +254,7 @@ class Partition(): prefix = "ext" if self.fstype.startswith("ext") else self.fstype method = getattr(self, "prepare_rootfs_" + prefix) - method(rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo) + method(rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo) self.source_file = rootfs # get the rootfs size in the right units for kickstart (kB) @@ -245,7 +262,7 @@ class Partition(): out = exec_cmd(du_cmd) self.size = int(out.split()[0]) - def prepare_rootfs_ext(self, rootfs, oe_builddir, rootfs_dir, + def prepare_rootfs_ext(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ Prepare content for an ext2/3/4 rootfs partition. @@ -269,10 +286,19 @@ class Partition(): (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) + if self.updated_fstab_path and self.has_fstab: + debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") + with open(debugfs_script_path, "w") as f: + f.write("cd etc\n") + f.write("rm fstab\n") + f.write("write %s fstab\n" % (self.updated_fstab_path)) + debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs) + exec_native_cmd(debugfs_cmd, native_sysroot) + mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - def prepare_rootfs_btrfs(self, rootfs, oe_builddir, rootfs_dir, + def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ Prepare content for a btrfs rootfs partition. @@ -295,7 +321,7 @@ class Partition(): self.mkfs_extraopts, self.fsuuid, rootfs) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - def prepare_rootfs_msdos(self, rootfs, oe_builddir, rootfs_dir, + def prepare_rootfs_msdos(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ Prepare content for a msdos/vfat rootfs partition. @@ -324,12 +350,16 @@ class Partition(): mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) exec_native_cmd(mcopy_cmd, native_sysroot) + if self.updated_fstab_path and self.has_fstab: + mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) + exec_native_cmd(mcopy_cmd, native_sysroot) + chmod_cmd = "chmod 644 %s" % rootfs exec_cmd(chmod_cmd) prepare_rootfs_vfat = prepare_rootfs_msdos - def prepare_rootfs_squashfs(self, rootfs, oe_builddir, rootfs_dir, + def prepare_rootfs_squashfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo): """ Prepare content for a squashfs rootfs partition. diff --git a/poky/scripts/lib/wic/plugins/imager/direct.py b/poky/scripts/lib/wic/plugins/imager/direct.py index 55db826e9..ea709e8c5 100644 --- a/poky/scripts/lib/wic/plugins/imager/direct.py +++ b/poky/scripts/lib/wic/plugins/imager/direct.py @@ -54,15 +54,16 @@ class DirectPlugin(ImagerPlugin): self.native_sysroot = native_sysroot self.oe_builddir = oe_builddir + self.debug = options.debug self.outdir = options.outdir self.compressor = options.compressor self.bmap = options.bmap self.no_fstab_update = options.no_fstab_update - self.original_fstab = None + self.updated_fstab_path = None self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], strftime("%Y%m%d%H%M")) - self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') + self.workdir = self.setup_workdir(options.workdir) self._image = None self.ptable_format = self.ks.bootloader.ptable self.parts = self.ks.partitions @@ -78,6 +79,16 @@ class DirectPlugin(ImagerPlugin): self._image = PartitionedImage(image_path, self.ptable_format, self.parts, self.native_sysroot) + def setup_workdir(self, workdir): + if workdir: + if os.path.exists(workdir): + raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir)) + + os.makedirs(workdir) + return workdir + else: + return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') + def do_create(self): """ Plugin entry point. @@ -90,11 +101,8 @@ class DirectPlugin(ImagerPlugin): finally: self.cleanup() - def _write_fstab(self, image_rootfs): - """overriden to generate fstab (temporarily) in rootfs. This is called - from _create, make sure it doesn't get called from - BaseImage.create() - """ + def update_fstab(self, image_rootfs): + """Assume partition order same as in wks""" if not image_rootfs: return @@ -104,18 +112,9 @@ class DirectPlugin(ImagerPlugin): with open(fstab_path) as fstab: fstab_lines = fstab.readlines() - self.original_fstab = fstab_lines.copy() - - if self._update_fstab(fstab_lines, self.parts): - with open(fstab_path, "w") as fstab: - fstab.writelines(fstab_lines) - else: - self.original_fstab = None - def _update_fstab(self, fstab_lines, parts): - """Assume partition order same as in wks""" updated = False - for part in parts: + for part in self.parts: if not part.realnum or not part.mountpoint \ or part.mountpoint == "/": continue @@ -144,7 +143,10 @@ class DirectPlugin(ImagerPlugin): fstab_lines.append(line) updated = True - return updated + if updated: + self.updated_fstab_path = os.path.join(self.workdir, "fstab") + with open(self.updated_fstab_path, "w") as f: + f.writelines(fstab_lines) def _full_path(self, path, name, extention): """ Construct full file path to a file we generate. """ @@ -160,7 +162,7 @@ class DirectPlugin(ImagerPlugin): a partitioned image. """ if not self.no_fstab_update: - self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) + self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR")) for part in self.parts: # get rootfs size from bitbake variable if it's not set in .ks file @@ -273,14 +275,9 @@ class DirectPlugin(ImagerPlugin): if os.path.isfile(path): shutil.move(path, os.path.join(self.outdir, fname)) - #Restore original fstab - if self.original_fstab: - fstab_path = self.rootfs_dir.get("ROOTFS_DIR") + "/etc/fstab" - with open(fstab_path, "w") as fstab: - fstab.writelines(self.original_fstab) - - # remove work directory - shutil.rmtree(self.workdir, ignore_errors=True) + # remove work directory when it is not in debugging mode + if not self.debug: + shutil.rmtree(self.workdir, ignore_errors=True) # Overhead of the MBR partitioning scheme (just one sector) MBR_OVERHEAD = 1 @@ -343,6 +340,13 @@ class PartitionedImage(): part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() else: part.fsuuid = str(uuid.uuid4()) + else: + #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY + if part.fstype == 'vfat' or part.fstype == 'msdos': + if part.fsuuid.upper().startswith("0X"): + part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0") + else: + part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0") def prepare(self, imager): """Prepare an image. Call prepare method of all image partitions.""" @@ -351,7 +355,8 @@ class PartitionedImage(): # sizes before we can add them and do the layout. part.prepare(imager, imager.workdir, imager.oe_builddir, imager.rootfs_dir, imager.bootimg_dir, - imager.kernel_dir, imager.native_sysroot) + imager.kernel_dir, imager.native_sysroot, + imager.updated_fstab_path) # Converting kB to sectors for parted part.size_sec = part.disk_size * 1024 // self.sector_size diff --git a/poky/scripts/lib/wic/plugins/source/empty.py b/poky/scripts/lib/wic/plugins/source/empty.py new file mode 100644 index 000000000..041617d64 --- /dev/null +++ b/poky/scripts/lib/wic/plugins/source/empty.py @@ -0,0 +1,32 @@ +# +# SPDX-License-Identifier: MIT +# + +# The empty wic plugin is used to create unformatted empty partitions for wic +# images. +# To use it you must pass "empty" as argument for the "--source" parameter in +# the wks file. For example: +# part foo --source empty --ondisk sda --size="1024" --align 1024 + +import logging + +from wic.pluginbase import SourcePlugin + +logger = logging.getLogger('wic') + +class EmptyPartitionPlugin(SourcePlugin): + """ + Populate unformatted empty partition. + """ + + name = 'empty' + + @classmethod + def do_prepare_partition(cls, part, source_params, cr, cr_workdir, + oe_builddir, bootimg_dir, kernel_dir, + rootfs_dir, native_sysroot): + """ + Called to do the actual content population for a partition i.e. it + 'prepares' the partition to be incorporated into the image. + """ + return diff --git a/poky/scripts/lib/wic/plugins/source/rootfs.py b/poky/scripts/lib/wic/plugins/source/rootfs.py index f1db83f8a..96d940a91 100644 --- a/poky/scripts/lib/wic/plugins/source/rootfs.py +++ b/poky/scripts/lib/wic/plugins/source/rootfs.py @@ -94,6 +94,7 @@ class RootfsPlugin(SourcePlugin): "it is not a valid path, exiting" % part.rootfs_dir) part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir) + part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab")) pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo") if not os.path.lexists(pseudo_dir): logger.warn("%s folder does not exist. " @@ -103,9 +104,9 @@ class RootfsPlugin(SourcePlugin): new_rootfs = None new_pseudo = None # Handle excluded paths. - if part.exclude_path or part.include_path or part.change_directory: - # We need a new rootfs directory we can delete files from. Copy to - # workdir. + if part.exclude_path or part.include_path or part.change_directory or part.update_fstab_in_rootfs: + # We need a new rootfs directory we can safely modify without + # interfering with other tasks. Copy to workdir. new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno)) if os.path.lexists(new_rootfs): @@ -199,17 +200,33 @@ class RootfsPlugin(SourcePlugin): if not os.path.lexists(full_path): continue + if new_pseudo: + pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) + else: + pseudo = None if path.endswith(os.sep): # Delete content only. for entry in os.listdir(full_path): full_entry = os.path.join(full_path, entry) - if os.path.isdir(full_entry) and not os.path.islink(full_entry): - shutil.rmtree(full_entry) - else: - os.remove(full_entry) + rm_cmd = "rm -rf %s" % (full_entry) + exec_native_cmd(rm_cmd, native_sysroot, pseudo) else: # Delete whole directory. - shutil.rmtree(full_path) + rm_cmd = "rm -rf %s" % (full_path) + exec_native_cmd(rm_cmd, native_sysroot, pseudo) + + # Update part.has_fstab here as fstab may have been added or + # removed by the above modifications. + part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab")) + if part.update_fstab_in_rootfs and part.has_fstab: + fstab_path = os.path.join(new_rootfs, "etc/fstab") + # Assume that fstab should always be owned by root with fixed permissions + install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path) + if new_pseudo: + pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) + else: + pseudo = None + exec_native_cmd(install_cmd, native_sysroot, pseudo) part.prepare_rootfs(cr_workdir, oe_builddir, new_rootfs or part.rootfs_dir, native_sysroot, |