From 99467dab23c4af816958fdd98218ca613308b402 Mon Sep 17 00:00:00 2001 From: Andrew Geissler Date: Mon, 25 Feb 2019 18:54:23 -0600 Subject: poky: refresh thud: b904775c2b..7c76c5d78b Update poky to thud HEAD. Adam Trhon (1): icecc-env: don't raise error when icecc not installed Alexander Kanavin (1): openssl10: update to 1.0.2q Armin Kuster (1): perl: add testdepends for ssh Bruce Ashfield (2): linux-yocto/4.18: update to v4.18.26 linux-yocto/4.18: update to v4.18.27 Changqing Li (1): checklayer: generate locked-sigs.inc under builddir Dan Dedrick (2): devtool: remove duplicate overrides devtool: improve git repo checks before check_commits logic Daniel Ammann (1): ref-manual: Typo found and fixed. Douglas Royds (2): openssl ptest: Strip build host paths from configdata.pm openssl: Strip perl version from installed ptest configdata.pm file Dustin Bain (1): busybox: update to 1.29.3 Jan Kiszka (1): oe-git-proxy: Avoid resolving NO_PROXY against local files Jens Rehsack (1): avahi: avoid depending on skipped package Jonas Bonn (1): keymaps: tighten package write dependency Kai Kang (1): selftest/wic: update test case test_qemu Khem Raj (3): openssl10: Fix mutliple include assumptions for bn.h in opensslconf.h send-error-report: Use https instead of http protocol multilib_header_wrapper.h: Use #pragma once Leonardo Augusto (1): scripts/lib/wic/engine: Fix cp's target path for ext* filesystems Liu Haitao (1): iw: fix parsing of WEP keys Mingli Yu (1): logrotate.py: restore /etc/logrotate.d/wtmp Otavio Salvador (1): linux-firmware: Bump to 710963f revision Ovidiu Panait (1): ghostscript: Fix CVE-2019-6116 Peter Kjellerstedt (1): libaio: Extend to native Richard Purdie (23): package: Add pkg_postinst_ontarget to PACKAGEVARS oeqa/runtime/ptest: Avoid traceback for tests with no section oeqa/utils/logparser: Simplify ptest log parsing code oeqa/logparser: Further simplification/clarification oeqa/logparser: Reform the ptest results parser oeqa/utils/logparser: Add in support for duration, exitcode and logs by section oeqa/logparser: Improve results handling oeqa/logparser: Various misc cleanups oeqa/runtime/ptest: Ensure OOM errors are logged scripts/contrib/build-perf-test-wrapper.sh: Improve interaction with autobuilder automation scripts/contrib/build-perf-test.sh: Remove it oe-build-perf-report: Allow branch without hostname oe-build-perf-report: Allow commits from different branches oe-build-perf-report: Improve branch comparision handling oe-build-perf-report: Fix missing buildstats comparisions wic/engine: Fix missing parted autobuilder failures lib/buildstats: Improve error message scripts/oe-git-archive: Separate out functionality to library function oe-build-perf-report/gitarchive: Move common useful functions to library bitbake: runqueue: Fix dependency loop analysis 'hangs' bitbake: runqueue: Filter out multiconfig dependencies from BB_TASKDEPDATA bitbake: siggen: Fix multiconfig corner case bitbake: cooker: Tweak multiconfig dependency resolution Robert Yang (5): bluez5: Fix a race issue for tools yocto-check-layer-wrapper: Fix path for oe-init-build-env checklayer: Avoid adding the layer if it is already present runqemu: Let qemuparams override default settings runqemu: Make QB_MEM easier to set Ross Burton (3): e2fsprogs: fix file system generation with large files linux-firmware: recommend split up packages linux-firmware: split out liquidio firmware Scott Rifenbark (2): poky.ent: Updated "meta-intel" version to "10.1" overview-manual, mega-manual: Updated Package Feeds diagram Serhey Popovych (1): openssl: Skip assembler optimized code for powerpc64 with musl William Bourque (1): wic/engine.py: Load paths from PATH environment variable Xulin Sun (1): openssl: fix multilib file install conflicts Zheng Ruoqin (1): mdadm: add init and service scripts Change-Id: Ib14c2fb69d25d84aa3d4bf0a6715bba57d1eb900 Signed-off-by: Andrew Geissler --- poky/meta/lib/oeqa/utils/gitarchive.py | 244 +++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 poky/meta/lib/oeqa/utils/gitarchive.py (limited to 'poky/meta/lib/oeqa/utils/gitarchive.py') diff --git a/poky/meta/lib/oeqa/utils/gitarchive.py b/poky/meta/lib/oeqa/utils/gitarchive.py new file mode 100644 index 000000000..ff614d06b --- /dev/null +++ b/poky/meta/lib/oeqa/utils/gitarchive.py @@ -0,0 +1,244 @@ +# +# Helper functions for committing data to git and pushing upstream +# +# Copyright (c) 2017, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# + +import os +import re +import sys +from operator import attrgetter +from collections import namedtuple +from oeqa.utils.git import GitRepo, GitError + +class ArchiveError(Exception): + """Internal error handling of this script""" + +def format_str(string, fields): + """Format string using the given fields (dict)""" + try: + return string.format(**fields) + except KeyError as err: + raise ArchiveError("Unable to expand string '{}': unknown field {} " + "(valid fields are: {})".format( + string, err, ', '.join(sorted(fields.keys())))) + + +def init_git_repo(path, no_create, bare, log): + """Initialize local Git repository""" + path = os.path.abspath(path) + if os.path.isfile(path): + raise ArchiveError("Invalid Git repo at {}: path exists but is not a " + "directory".format(path)) + if not os.path.isdir(path) or not os.listdir(path): + if no_create: + raise ArchiveError("No git repo at {}, refusing to create " + "one".format(path)) + if not os.path.isdir(path): + try: + os.mkdir(path) + except (FileNotFoundError, PermissionError) as err: + raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) + if not os.listdir(path): + log.info("Initializing a new Git repo at %s", path) + repo = GitRepo.init(path, bare) + try: + repo = GitRepo(path, is_topdir=True) + except GitError: + raise ArchiveError("Non-empty directory that is not a Git repository " + "at {}\nPlease specify an existing Git repository, " + "an empty directory or a non-existing directory " + "path.".format(path)) + return repo + + +def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): + """Commit data into a Git repository""" + log.info("Committing data into to branch %s", branch) + tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') + try: + # Create new tree object from the data + env_update = {'GIT_INDEX_FILE': tmp_index, + 'GIT_WORK_TREE': os.path.abspath(data_dir)} + repo.run_cmd('add .', env_update) + + # Remove files that are excluded + if exclude: + repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) + + tree = repo.run_cmd('write-tree', env_update) + + # Create new commit object from the tree + parent = repo.rev_parse(branch) + git_cmd = ['commit-tree', tree, '-m', message] + if parent: + git_cmd += ['-p', parent] + commit = repo.run_cmd(git_cmd, env_update) + + # Create git notes + for ref, filename in notes: + ref = ref.format(branch_name=branch) + repo.run_cmd(['notes', '--ref', ref, 'add', + '-F', os.path.abspath(filename), commit]) + + # Update branch head + git_cmd = ['update-ref', 'refs/heads/' + branch, commit] + if parent: + git_cmd.append(parent) + repo.run_cmd(git_cmd) + + # Update current HEAD, if we're on branch 'branch' + if not repo.bare and repo.get_current_branch() == branch: + log.info("Updating %s HEAD to latest commit", repo.top_dir) + repo.run_cmd('reset --hard') + + return commit + finally: + if os.path.exists(tmp_index): + os.unlink(tmp_index) + + +def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, + keywords): + """Generate tag name and message, with support for running id number""" + keyws = keywords.copy() + # Tag number is handled specially: if not defined, we autoincrement it + if 'tag_number' not in keyws: + # Fill in all other fields than 'tag_number' + keyws['tag_number'] = '{tag_number}' + tag_re = format_str(name_pattern, keyws) + # Replace parentheses for proper regex matching + tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' + # Inject regex group pattern for 'tag_number' + tag_re = tag_re.format(tag_number='(?P[0-9]{1,5})') + + keyws['tag_number'] = 0 + for existing_tag in repo.run_cmd('tag').splitlines(): + match = re.match(tag_re, existing_tag) + + if match and int(match.group('tag_number')) >= keyws['tag_number']: + keyws['tag_number'] = int(match.group('tag_number')) + 1 + + tag_name = format_str(name_pattern, keyws) + msg_subj= format_str(msg_subj_pattern.strip(), keyws) + msg_body = format_str(msg_body_pattern, keyws) + return tag_name, msg_subj + '\n\n' + msg_body + +def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): + + if not os.path.isdir(data_dir): + raise ArchiveError("Not a directory: {}".format(data_dir)) + + data_repo = init_git_repo(git_dir, no_create, bare, log) + + # Expand strings early in order to avoid getting into inconsistent + # state (e.g. no tag even if data was committed) + commit_msg = format_str(commit_msg_subject.strip(), keywords) + commit_msg += '\n\n' + format_str(commit_msg_body, keywords) + branch_name = format_str(branch_name, keywords) + tag_name = None + if not no_tag and tagname: + tag_name, tag_msg = expand_tag_strings(data_repo, tagname, + tag_msg_subject, + tag_msg_body, keywords) + + # Commit data + commit = git_commit_data(data_repo, data_dir, branch_name, + commit_msg, exclude, notes, log) + + # Create tag + if tag_name: + log.info("Creating tag %s", tag_name) + data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) + + # Push data to remote + if push: + cmd = ['push', '--tags'] + # If no remote is given we push with the default settings from + # gitconfig + if push is not True: + notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) + for ref, _ in notes] + cmd.extend([push, branch_name] + notes_refs) + log.info("Pushing data to remote") + data_repo.run_cmd(cmd) + +# Container class for tester revisions +TestedRev = namedtuple('TestedRev', 'commit commit_number tags') + +def get_test_runs(log, repo, tag_name, **kwargs): + """Get a sorted list of test runs, matching given pattern""" + # First, get field names from the tag name pattern + field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] + undef_fields = [f for f in field_names if f not in kwargs.keys()] + + # Fields for formatting tag name pattern + str_fields = dict([(f, '*') for f in field_names]) + str_fields.update(kwargs) + + # Get a list of all matching tags + tag_pattern = tag_name.format(**str_fields) + tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() + log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) + + # Parse undefined fields from tag names + str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) + str_fields['branch'] = r'(?P[\w\-.()/]+)' + str_fields['commit'] = '(?P[0-9a-f]{7,40})' + str_fields['commit_number'] = '(?P[0-9]{1,7})' + str_fields['tag_number'] = '(?P[0-9]{1,5})' + # escape parenthesis in fields in order to not messa up the regexp + fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) + str_fields.update(fixed_fields) + tag_re = re.compile(tag_name.format(**str_fields)) + + # Parse fields from tags + revs = [] + for tag in tags: + m = tag_re.match(tag) + groups = m.groupdict() + revs.append([groups[f] for f in undef_fields] + [tag]) + + # Return field names and a sorted list of revs + return undef_fields, sorted(revs) + +def get_test_revs(log, repo, tag_name, **kwargs): + """Get list of all tested revisions""" + fields, runs = get_test_runs(log, repo, tag_name, **kwargs) + + revs = {} + commit_i = fields.index('commit') + commit_num_i = fields.index('commit_number') + for run in runs: + commit = run[commit_i] + commit_num = run[commit_num_i] + tag = run[-1] + if not commit in revs: + revs[commit] = TestedRev(commit, commit_num, [tag]) + else: + assert commit_num == revs[commit].commit_number, "Commit numbers do not match" + revs[commit].tags.append(tag) + + # Return in sorted table + revs = sorted(revs.values(), key=attrgetter('commit_number')) + log.debug("Found %d tested revisions:\n %s", len(revs), + "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) + return revs + +def rev_find(revs, attr, val): + """Search from a list of TestedRev""" + for i, rev in enumerate(revs): + if getattr(rev, attr) == val: + return i + raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) + -- cgit v1.2.3