From 8962b6b475bddc011c414f40ffd02f0ed4e02771 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:32 +0900 Subject: kbuild: print short log in addition to the whole command with V=1 "make V=1" prints the whole command instead of the short log, but I think it is nicer to print both so that you can easily spot the build rule of your interest. This commit changes V=1 to print the short log (the line starts with '#'), followed by the full log. In parallel builds, the short/full logs from the same build rule may be interspersed. If you want to avoid it, please add -Otarget option. Kbuild will never set it by default because Make would buffer the logs and lose the escape sequences. (Modern compilers print warnings and errors in color, but only when they write to a terminal.) This is also a preparation for supporting V=12 because V=2 appends the reason for rebuilding to the short log. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 2f7356b2990b..faae3c7986b0 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -125,15 +125,15 @@ dtbinst := -f $(srctree)/scripts/Makefile.dtbinst obj # $(Q)$(MAKE) $(clean)=dir clean := -f $(srctree)/scripts/Makefile.clean obj -# echo command. -# Short version is used, if $(quiet) equals `quiet_', otherwise full one. -echo-cmd = $(if $($(quiet)cmd_$(1)),\ - echo ' $(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) - -# sink stdout for 'make -s' - redirect := - quiet_redirect := -silent_redirect := exec >/dev/null; +# pring log +# +# If quiet is "silent_", print nothing and sink stdout +# If quiet is "quiet_", print short log +# If quiet is empty, print short log and whole command +silent_log_print = exec >/dev/null; + quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption # @@ -156,8 +156,8 @@ delete-on-interrupt = \ $(foreach sig, HUP INT QUIT TERM PIPE, \ trap 'rm -f $@; trap - $(sig); kill -s $(sig) $$$$' $(sig);)) -# printing commands -cmd = @set -e; $(echo-cmd) $($(quiet)redirect) $(delete-on-interrupt) $(cmd_$(1)) +# print and execute commands +cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cmd_$(1)),:) ### # if_changed - execute command if any prerequisite is newer than @@ -234,7 +234,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file ifeq ($(KBUILD_VERBOSE),2) -why = \ +_why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ $(if $(newer-prereqs),- due to: $(newer-prereqs), \ @@ -251,7 +251,7 @@ why = \ ) \ ) -echo-why = $(call escsq, $(strip $(why))) +why = $(space)$(strip $(_why)) endif ############################################################################### -- cgit v1.2.3 From c0d3b83100c896e1b0909023df58a0ebdd428d61 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:33 +0900 Subject: kbuild: do not print extra logs for V=2 Some scripts increase the verbose level when V=1, but others when not V=0. I think the former is correct because V=2 is not a log level but a switch to print the reason for rebuilding. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Documentation/Makefile | 2 +- arch/powerpc/kernel/prom_init_check.sh | 9 ++++----- certs/extract-cert.c | 9 ++++++--- scripts/asn1_compiler.c | 4 ++-- scripts/kernel-doc | 4 ++-- 5 files changed, 15 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/Documentation/Makefile b/Documentation/Makefile index bb73dcb5ed05..023fa658a0a8 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -28,7 +28,7 @@ BUILDDIR = $(obj)/output PDFLATEX = xelatex LATEXOPTS = -interaction=batchmode -no-shell-escape -ifeq ($(KBUILD_VERBOSE),0) +ifeq ($(findstring 1, $(KBUILD_VERBOSE)),) SPHINXOPTS += "-q" endif diff --git a/arch/powerpc/kernel/prom_init_check.sh b/arch/powerpc/kernel/prom_init_check.sh index 311890d71c4c..5a319863f289 100644 --- a/arch/powerpc/kernel/prom_init_check.sh +++ b/arch/powerpc/kernel/prom_init_check.sh @@ -51,11 +51,10 @@ do # a leading . on the name, so strip it off here. UNDEF="${UNDEF#.}" - if [ $KBUILD_VERBOSE ]; then - if [ $KBUILD_VERBOSE -ne 0 ]; then - echo "Checking prom_init.o symbol '$UNDEF'" - fi - fi + case "$KBUILD_VERBOSE" in + *1*) + echo "Checking prom_init.o symbol '$UNDEF'" ;; + esac OK=0 for WHITE in $WHITELIST diff --git a/certs/extract-cert.c b/certs/extract-cert.c index 8c1fb9a70d66..70e9ec89d87d 100644 --- a/certs/extract-cert.c +++ b/certs/extract-cert.c @@ -78,7 +78,7 @@ static void drain_openssl_errors(void) static const char *key_pass; static BIO *wb; static char *cert_dst; -static int kbuild_verbose; +static bool verbose; static void write_cert(X509 *x509) { @@ -90,19 +90,22 @@ static void write_cert(X509 *x509) } X509_NAME_oneline(X509_get_subject_name(x509), buf, sizeof(buf)); ERR(!i2d_X509_bio(wb, x509), "%s", cert_dst); - if (kbuild_verbose) + if (verbose) fprintf(stderr, "Extracted cert: %s\n", buf); } int main(int argc, char **argv) { char *cert_src; + char *verbose_env; OpenSSL_add_all_algorithms(); ERR_load_crypto_strings(); ERR_clear_error(); - kbuild_verbose = atoi(getenv("KBUILD_VERBOSE")?:"0"); + verbose_env = getenv("KBUILD_VERBOSE"); + if (verbose_env && strchr(verbose_env, '1')) + verbose = true; key_pass = getenv("KBUILD_SIGN_PIN"); diff --git a/scripts/asn1_compiler.c b/scripts/asn1_compiler.c index 71d4a7c87900..7b6756a8c15d 100644 --- a/scripts/asn1_compiler.c +++ b/scripts/asn1_compiler.c @@ -567,8 +567,8 @@ int main(int argc, char **argv) int fd; kbuild_verbose = getenv("KBUILD_VERBOSE"); - if (kbuild_verbose) - verbose_opt = atoi(kbuild_verbose); + if (kbuild_verbose && strchr(kbuild_verbose, '1')) + verbose_opt = true; while (argc > 4) { if (strcmp(argv[1], "-v") == 0) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 54b0893cae66..8ad0a7d68d9a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -175,8 +175,8 @@ my $declaration_start_line; my ($type, $declaration_name, $return_type); my ($newsection, $newcontents, $prototype, $brcount, %source_map); -if (defined($ENV{'KBUILD_VERBOSE'})) { - $verbose = "$ENV{'KBUILD_VERBOSE'}"; +if (defined($ENV{'KBUILD_VERBOSE'}) && $ENV{'KBUILD_VERBOSE'} =~ '1') { + $verbose = 1; } if (defined($ENV{'KCFLAGS'})) { -- cgit v1.2.3 From 6ae4b9868a8f723cae2600722eea033fafadd399 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:34 +0900 Subject: kbuild: allow to combine multiple V= levels Commit a6de553da01c ("kbuild: Allow to combine multiple W= levels") supported W=123 to enable all the extra warning groups. I think a similar idea is applicable to the V= option. V=1 echos the whole command V=2 prints the reason for rebuilding These are orthogonal, and can be enabled at the same time. This commit supports V=12 to enable both of them. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- Makefile | 18 +++++++++--------- arch/x86/tools/Makefile | 2 +- scripts/Kbuild.include | 4 ++-- scripts/tags.sh | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/Makefile b/Makefile index 9a919428a726..ed5ce58196be 100644 --- a/Makefile +++ b/Makefile @@ -69,9 +69,8 @@ unexport GREP_OPTIONS # # $(Q)$(MAKE) $(build)=scripts/basic # -# If KBUILD_VERBOSE equals 0 then the above command will be hidden. -# If KBUILD_VERBOSE equals 1 then the above command is displayed. -# If KBUILD_VERBOSE equals 2 then give the reason why each target is rebuilt. +# If KBUILD_VERBOSE contains 1, the whole command is echoed. +# If KBUILD_VERBOSE contains 2, the reason for rebuilding is printed. # # To put more focus on warnings, be less verbose as default # Use 'make V=1' to see the full commands @@ -83,12 +82,12 @@ ifndef KBUILD_VERBOSE KBUILD_VERBOSE = 0 endif -ifeq ($(KBUILD_VERBOSE),1) +quiet = quiet_ +Q = @ + +ifneq ($(findstring 1, $(KBUILD_VERBOSE)),) quiet = Q = -else - quiet=quiet_ - Q = @ endif # If the user is running make -s (silent mode), suppress echoing of @@ -1775,8 +1774,9 @@ help: printf " %-16s - Show all of the above\\n" help-boards; \ echo '') - @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' - @echo ' make V=2 [targets] 2 => give reason for rebuild of target' + @echo ' make V=n [targets] 0: quiet build (default), 1: verbose build' + @echo ' 2: give reason for rebuild of target' + @echo ' V=1 and V=2 can be combined with V=12' @echo ' make O=dir [targets] Locate all output files in "dir", including .config' @echo ' make C=1 [targets] Check re-compiled c source with $$CHECK' @echo ' (sparse by default)' diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index bddfc9a46645..90e820ac9771 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 PHONY += posttest -ifeq ($(KBUILD_VERBOSE),1) +ifneq ($(findstring 1, $(KBUILD_VERBOSE)),) posttest_verbose = -v else posttest_verbose = diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index faae3c7986b0..10cf8d2d82ef 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -132,7 +132,7 @@ clean := -f $(srctree)/scripts/Makefile.clean obj # If quiet is empty, print short log and whole command silent_log_print = exec >/dev/null; quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) - log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@)$(why))'; \ echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption @@ -233,7 +233,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (5) No dir/.target.cmd file (used to store command line) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file -ifeq ($(KBUILD_VERBOSE),2) +ifneq ($(findstring 2, $(KBUILD_VERBOSE)),) _why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ diff --git a/scripts/tags.sh b/scripts/tags.sh index e137cf15aae9..2e756bee1fa9 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -8,7 +8,7 @@ # Uses the following environment variables: # SUBARCH, SRCARCH, srctree -if [ "$KBUILD_VERBOSE" = "1" ]; then +if [[ "$KBUILD_VERBOSE" =~ 1 ]]; then set -x fi -- cgit v1.2.3 From 8f99eb857a96d474bfbb715ff9073b276b87ad2d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 16:06:33 +0900 Subject: kbuild: clean up stale file removal More than one year has passed since the copied *.[cS] files were removed from arch/*/boot/compressed/. Signed-off-by: Masahiro Yamada --- arch/sh/boot/compressed/Makefile | 7 ------- scripts/remove-stale-files | 24 +----------------------- 2 files changed, 1 insertion(+), 30 deletions(-) (limited to 'scripts') diff --git a/arch/sh/boot/compressed/Makefile b/arch/sh/boot/compressed/Makefile index 591125c42d49..b5e29f99c02c 100644 --- a/arch/sh/boot/compressed/Makefile +++ b/arch/sh/boot/compressed/Makefile @@ -8,13 +8,6 @@ OBJECTS := head_32.o misc.o cache.o piggy.o \ ashiftrt.o ashldi3.o ashrsi3.o ashlsi3.o lshrsi3.o -# These were previously generated files. When you are building the kernel -# with O=, make sure to remove the stale files in the output tree. Otherwise, -# the build system wrongly compiles the stale ones. -ifdef building_out_of_srctree -$(shell rm -f $(addprefix $(obj)/, ashiftrt.S ashldi3.c ashrsi3.S ashlsi3.S lshrsi3.S)) -endif - targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 \ vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo $(OBJECTS) diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index 64b14aa5aebf..cdbdde89a271 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -21,31 +21,9 @@ set -e # then will be really dead and removed from the code base entirely. rm -f arch/powerpc/purgatory/kexec-purgatory.c - -# These were previously generated source files. When you are building the kernel -# with O=, make sure to remove the stale files in the output tree. Otherwise, -# the build system wrongly compiles the stale ones. -if [ -n "${building_out_of_srctree}" ]; then - for f in fdt_rw.c fdt_ro.c fdt_wip.c fdt.c ashldi3.S bswapsdi2.S font.c lib1funcs.S hyp-stub.S - do - rm -f arch/arm/boot/compressed/${f} - done - - for f in uart-ath79.c ashldi3.c bswapdi.c bswapsi.c - do - rm -f arch/mips/boot/compressed/${f} - done - - for f in firmware.c real2.S - do - rm -f arch/parisc/boot/compressed/${f} - done -fi - rm -f arch/riscv/purgatory/kexec-purgatory.c +rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert -rm -f arch/x86/purgatory/kexec-purgatory.c - rm -f scripts/kconfig/[gmnq]conf-cfg -- cgit v1.2.3 From 91ecf7ff1b036f3fe1183809661119b1ee109b19 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 16:43:10 +0900 Subject: kbuild: make W=1 warn files that are tracked but ignored by git The top .gitignore comments about how to detect files breaking .gitignore rules, but people rarely care about it. Add a new W=1 warning to detect files that are tracked but ignored by git. If git is not installed or the source tree is not tracked by git at all, this script does not print anything. Running it on v6.2-rc1 detected the following: $ make W=1 misc-check Documentation/devicetree/bindings/.yamllint: warning: ignored by one of the .gitignore files drivers/clk/.kunitconfig: warning: ignored by one of the .gitignore files drivers/gpu/drm/tests/.kunitconfig: warning: ignored by one of the .gitignore files drivers/hid/.kunitconfig: warning: ignored by one of the .gitignore files fs/ext4/.kunitconfig: warning: ignored by one of the .gitignore files fs/fat/.kunitconfig: warning: ignored by one of the .gitignore files kernel/kcsan/.kunitconfig: warning: ignored by one of the .gitignore files lib/kunit/.kunitconfig: warning: ignored by one of the .gitignore files mm/kfence/.kunitconfig: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/.gitignore: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/Makefile: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/run_tags_test.sh: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/tags_test.c: warning: ignored by one of the .gitignore files These are ignored by the '.*' or 'tags' in the top .gitignore, but there is no rule to negate it. You might be tempted to do 'git add -f' but I want to have the real issue fixed (by fixing a .gitignore, or by renaming files, etc.). Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Reviewed-by: Nicolas Schier --- Makefile | 6 ++++++ scripts/misc-check | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 scripts/misc-check (limited to 'scripts') diff --git a/Makefile b/Makefile index cb5f433cf200..b9d1411ef25b 100644 --- a/Makefile +++ b/Makefile @@ -1861,6 +1861,12 @@ rust-analyzer: # Misc # --------------------------------------------------------------------------- +PHONY += misc-check +misc-check: + $(Q)$(srctree)/scripts/misc-check + +all: misc-check + PHONY += scripts_gdb scripts_gdb: prepare0 $(Q)$(MAKE) $(build)=scripts/gdb diff --git a/scripts/misc-check b/scripts/misc-check new file mode 100755 index 000000000000..d40d5484e0c5 --- /dev/null +++ b/scripts/misc-check @@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +set -e + +# Detect files that are tracked but ignored by git. This is checked only when +# ${KBUILD_EXTRA_WARN} contains 1, git is installed, and the source tree is +# tracked by git. +check_tracked_ignored_files () { + case "${KBUILD_EXTRA_WARN}" in + *1*) ;; + *) return;; + esac + + git -C ${srctree:-.} ls-files -i -c --exclude-per-directory=.gitignore 2>/dev/null | + sed 's/$/: warning: ignored by one of the .gitignore files/' >&2 +} + +check_tracked_ignored_files -- cgit v1.2.3 From 92215e7a801da7b89037a185c98f5ebb86a415b9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 18:15:00 +0900 Subject: kbuild: rename cmd_$@ to savedcmd_$@ in *.cmd files The cmd-check macro compares $(cmd_$@) and $(cmd_$1), but a pitfall is that you cannot use cmd_ as the variable name for the command. For example, the following code will not work in the top Makefile or ./Kbuild. quiet_cmd_foo = GEN $@ cmd_foo = touch $@ targets += foo foo: FORCE $(call if_changed,foo) In this case, both $@ and $1 are expanded to 'foo', so $(cmd_check) is always empty. We do not need to use the same prefix for cmd_$@ and cmd_$1. Rename the former to savedcmd_$@. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 8 ++++---- scripts/Makefile.modfinal | 2 +- scripts/basic/fixdep.c | 4 ++-- scripts/clang-tools/gen_compile_commands.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 10cf8d2d82ef..1a7514f49089 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -170,10 +170,10 @@ cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cm ifneq ($(KBUILD_NOCMDDEP),1) # Check if both commands are the same including their order. Result is empty # string if equal. User may override this check using make KBUILD_NOCMDDEP=1 -cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(cmd_$@))), \ +cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(savedcmd_$@))), \ $(subst $(space),$(space_escape),$(strip $(cmd_$1)))) else -cmd-check = $(if $(strip $(cmd_$@)),,1) +cmd-check = $(if $(strip $(savedcmd_$@)),,1) endif # Replace >$< with >$$< to preserve $ when reloading the .cmd file @@ -199,7 +199,7 @@ if_changed = $(if $(if-changed-cond),$(cmd_and_savecmd),@:) cmd_and_savecmd = \ $(cmd); \ - printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd + printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd # Execute the command and also postprocess generated .d dependencies file. if_changed_dep = $(if $(if-changed-cond),$(cmd_and_fixdep),@:) @@ -239,7 +239,7 @@ _why = \ $(if $(wildcard $@), \ $(if $(newer-prereqs),- due to: $(newer-prereqs), \ $(if $(cmd-check), \ - $(if $(cmd_$@),- due to command line change, \ + $(if $(savedcmd_$@),- due to command line change, \ $(if $(filter $@, $(targets)), \ - due to missing .cmd file, \ - due to $(notdir $@) not in $$(targets) \ diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index a30d5b08eee9..4703f652c009 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -54,7 +54,7 @@ newer_prereqs_except = $(filter-out $(PHONY) $(1),$?) # Same as if_changed, but allows to exclude specified extra dependencies if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \ $(cmd); \ - printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) + printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) # Re-generate module BTFs if either module's .ko or vmlinux changed %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),vmlinux) FORCE diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index f932aeaba71a..37782a632494 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -70,7 +70,7 @@ * * It first generates a line * - * cmd_ = + * savedcmd_ = * * and then basically copies the ..d file to stdout, in the * process filtering out the dependency on autoconf.h and adding @@ -343,7 +343,7 @@ int main(int argc, char *argv[]) target = argv[2]; cmdline = argv[3]; - printf("cmd_%s := %s\n\n", target, cmdline); + printf("savedcmd_%s := %s\n\n", target, cmdline); buf = read_file(depfile); parse_dep_file(buf, target); diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index 0227522959a4..15ba56527acd 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- cgit v1.2.3 From ee2162bd4fe85a62dcd3b9133fd0c537ae2eb3c5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 18:15:01 +0900 Subject: kbuild: add more comments for KBUILD_NOCMDDEP=1 The cmd-check for KBUILD_NOCMDDEP=1 may not be clear until you see commit c4d5ee13984f ("kbuild: make KBUILD_NOCMDDEP=1 handle empty built-in.o"). When a phony target (i.e. FORCE) is the only prerequisite, Kbuild uses a tricky way to detect that the target does not exist. Add more comments. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 1a7514f49089..4648ab8f11d4 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -170,9 +170,13 @@ cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cm ifneq ($(KBUILD_NOCMDDEP),1) # Check if both commands are the same including their order. Result is empty # string if equal. User may override this check using make KBUILD_NOCMDDEP=1 +# If the target does not exist, the *.cmd file should not be included so +# $(savedcmd_$@) gets empty. Then, target will be built even if $(newer-prereqs) +# happens to become empty. cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(savedcmd_$@))), \ $(subst $(space),$(space_escape),$(strip $(cmd_$1)))) else +# We still need to detect missing targets. cmd-check = $(if $(strip $(savedcmd_$@)),,1) endif @@ -186,6 +190,8 @@ make-cmd = $(call escsq,$(subst $(pound),$$(pound),$(subst $$,$$$$,$(cmd_$(1)))) # Find any prerequisites that are newer than target or that do not exist. # PHONY targets skipped in both cases. +# If there is no prerequisite other than phony targets, $(newer-prereqs) becomes +# empty even if the target does not exist. cmd-check saves this corner case. newer-prereqs = $(filter-out $(PHONY),$?) # It is a typical mistake to forget the FORCE prerequisite. Check it here so -- cgit v1.2.3 From ecd42fba5765bba5955e3e1f80265295c0f5c32d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Dec 2022 03:46:50 +0900 Subject: kbuild: unify cmd_dt_S_dtb and cmd_dt_S_dtbo cmd_dt_S_dtb and cmd_dt_S_dtbo are almost the same; the only difference is the prefix of the begin/end symbols. (__dtb vs __dtbo) Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.lib | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 4a4a5f67c1a6..100a386fcd71 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -368,40 +368,25 @@ DTC_FLAGS += $(DTC_FLAGS_$(basetarget)) DTC_FLAGS += $(if $(filter $(patsubst $(obj)/%,%,$@), $(base-dtb-y)), -@) # Generate an assembly file to wrap the output of the device tree compiler -quiet_cmd_dt_S_dtb= DTBS $@ -cmd_dt_S_dtb= \ -{ \ - echo '\#include '; \ - echo '.section .dtb.init.rodata,"a"'; \ - echo '.balign STRUCT_ALIGNMENT'; \ - echo '.global __dtb_$(subst -,_,$(*F))_begin'; \ - echo '__dtb_$(subst -,_,$(*F))_begin:'; \ - echo '.incbin "$<" '; \ - echo '__dtb_$(subst -,_,$(*F))_end:'; \ - echo '.global __dtb_$(subst -,_,$(*F))_end'; \ - echo '.balign STRUCT_ALIGNMENT'; \ -} > $@ +quiet_cmd_wrap_S_dtb = WRAP $@ + cmd_wrap_S_dtb = { \ + symbase=__$(patsubst .%,%,$(suffix $<))_$(subst -,_,$(notdir $*)); \ + echo '\#include '; \ + echo '.section .dtb.init.rodata,"a"'; \ + echo '.balign STRUCT_ALIGNMENT'; \ + echo ".global $${symbase}_begin"; \ + echo "$${symbase}_begin:"; \ + echo '.incbin "$<" '; \ + echo ".global $${symbase}_end"; \ + echo "$${symbase}_end:"; \ + echo '.balign STRUCT_ALIGNMENT'; \ + } > $@ $(obj)/%.dtb.S: $(obj)/%.dtb FORCE - $(call if_changed,dt_S_dtb) - -# Generate an assembly file to wrap the output of the device tree compiler -quiet_cmd_dt_S_dtbo= DTBOS $@ -cmd_dt_S_dtbo= \ -{ \ - echo '\#include '; \ - echo '.section .dtb.init.rodata,"a"'; \ - echo '.balign STRUCT_ALIGNMENT'; \ - echo '.global __dtbo_$(subst -,_,$(*F))_begin'; \ - echo '__dtbo_$(subst -,_,$(*F))_begin:'; \ - echo '.incbin "$<" '; \ - echo '__dtbo_$(subst -,_,$(*F))_end:'; \ - echo '.global __dtbo_$(subst -,_,$(*F))_end'; \ - echo '.balign STRUCT_ALIGNMENT'; \ -} > $@ + $(call if_changed,wrap_S_dtb) $(obj)/%.dtbo.S: $(obj)/%.dtbo FORCE - $(call if_changed,dt_S_dtbo) + $(call if_changed,wrap_S_dtb) quiet_cmd_dtc = DTC $@ cmd_dtc = $(HOSTCC) -E $(dtc_cpp_flags) -x assembler-with-cpp -o $(dtc-tmp) $< ; \ -- cgit v1.2.3 From 16169a47d5c36046041527faafb5a3f5c86701c6 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:14 +0900 Subject: kbuild: refactor host*_flags Remove _host*_flags. No functional change is intended. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/Makefile.host | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.host b/scripts/Makefile.host index da133780b751..4a02b31cd102 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -80,25 +80,23 @@ host-rust := $(addprefix $(obj)/,$(host-rust)) ##### # Handle options to gcc. Support building with separate output directory -_hostc_flags = $(KBUILD_HOSTCFLAGS) $(HOST_EXTRACFLAGS) \ +hostc_flags = -Wp,-MMD,$(depfile) \ + $(KBUILD_HOSTCFLAGS) $(HOST_EXTRACFLAGS) \ $(HOSTCFLAGS_$(target-stem).o) -_hostcxx_flags = $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ +hostcxx_flags = -Wp,-MMD,$(depfile) \ + $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) -_hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ - $(HOSTRUSTFLAGS_$(target-stem)) +hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ + $(HOSTRUSTFLAGS_$(target-stem)) # $(objtree)/$(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) ifdef building_out_of_srctree -_hostc_flags += -I $(objtree)/$(obj) -_hostcxx_flags += -I $(objtree)/$(obj) +hostc_flags += -I $(objtree)/$(obj) +hostcxx_flags += -I $(objtree)/$(obj) endif endif -hostc_flags = -Wp,-MMD,$(depfile) $(_hostc_flags) -hostcxx_flags = -Wp,-MMD,$(depfile) $(_hostcxx_flags) -hostrust_flags = $(_hostrust_flags) - ##### # Compile programs on the host -- cgit v1.2.3 From 295d8398c67e314d99bb070f38883f83fe94a97a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:15 +0900 Subject: kbuild: specify output names separately for each emission type from rustc In Kbuild, two different rules must not write to the same file, but it happens when compiling rust source files. For example, set CONFIG_SAMPLE_RUST_MINIMAL=m and run the following: $ make -j$(nproc) samples/rust/rust_minimal.o samples/rust/rust_minimal.rsi \ samples/rust/rust_minimal.s samples/rust/rust_minimal.ll [snip] RUSTC [M] samples/rust/rust_minimal.o RUSTC [M] samples/rust/rust_minimal.rsi RUSTC [M] samples/rust/rust_minimal.s RUSTC [M] samples/rust/rust_minimal.ll mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:334: samples/rust/rust_minimal.ll] Error 1 make[3]: *** Waiting for unfinished jobs.... mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:309: samples/rust/rust_minimal.o] Error 1 mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:326: samples/rust/rust_minimal.s] Error 1 make[2]: *** [scripts/Makefile.build:504: samples/rust] Error 2 make[1]: *** [scripts/Makefile.build:504: samples] Error 2 make: *** [Makefile:2008: .] Error 2 The reason for the error is that 4 threads running in parallel renames the same file, samples/rust/rust_minimal.d. This does not happen when compiling C or assembly files because -Wp,-MMD,$(depfile) explicitly specifies the dependency filepath. $(depfile) is a unique path for each target. Currently, rustc is only given --out-dir and --emit= So, all the rust build rules output the dep-info into the default .d, which causes the path conflict. Fortunately, the --emit option is able to specify the output path individually, with the form --emit==. Add --emit=dep-info=$(depfile) to the common part. Also, remove the redundant --out-dir because the output path is specified for each type. The code gets much cleaner because we do not need to rename *.d files. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- rust/Makefile | 11 +++++------ scripts/Makefile.build | 14 +++++++------- scripts/Makefile.host | 7 +++---- 3 files changed, 15 insertions(+), 17 deletions(-) (limited to 'scripts') diff --git a/rust/Makefile b/rust/Makefile index ff70c4c916f8..865afb87bc9b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -331,10 +331,9 @@ $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ cmd_rustc_procmacro = \ $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ - --emit=dep-info,link --extern proc_macro \ - --crate-type proc-macro --out-dir $(objtree)/$(obj) \ + --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ + --crate-type proc-macro \ --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \ - mv $(objtree)/$(obj)/$(patsubst lib%.so,%,$(notdir $@)).d $(depfile); \ sed -i '/^\#/d' $(depfile) # Procedural macros can only be used with the `rustc` that compiled it. @@ -348,10 +347,10 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \ - --emit=dep-info,obj,metadata --crate-type rlib \ - --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ + --emit=dep-info=$(depfile) --emit=obj=$@ \ + --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ + --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ - mv $(objtree)/$(obj)/$(patsubst %.o,%,$(notdir $@)).d $(depfile); \ sed -i '/^\#/d' $(depfile) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index a0d5c6cca76d..40de20246e50 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -285,11 +285,11 @@ rust_common_cmd = \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ --extern alloc --extern kernel \ - --crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \ - --crate-name $(basename $(notdir $@)) + --crate-type rlib -L $(objtree)/rust/ \ + --crate-name $(basename $(notdir $@)) \ + --emit=dep-info=$(depfile) rust_handle_depfile = \ - mv $(obj)/$(basename $(notdir $@)).d $(depfile); \ sed -i '/^\#/d' $(depfile) # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit @@ -302,7 +302,7 @@ rust_handle_depfile = \ quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_o_rs = \ - $(rust_common_cmd) --emit=dep-info,obj $<; \ + $(rust_common_cmd) --emit=obj=$@ $<; \ $(rust_handle_depfile) $(obj)/%.o: $(src)/%.rs FORCE @@ -310,7 +310,7 @@ $(obj)/%.o: $(src)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ - $(rust_common_cmd) --emit=dep-info -Zunpretty=expanded $< >$@; \ + $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ $(rust_handle_depfile) @@ -319,7 +319,7 @@ $(obj)/%.rsi: $(src)/%.rs FORCE quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_s_rs = \ - $(rust_common_cmd) --emit=dep-info,asm $<; \ + $(rust_common_cmd) --emit=asm=$@ $<; \ $(rust_handle_depfile) $(obj)/%.s: $(src)/%.rs FORCE @@ -327,7 +327,7 @@ $(obj)/%.s: $(src)/%.rs FORCE quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_ll_rs = \ - $(rust_common_cmd) --emit=dep-info,llvm-ir $<; \ + $(rust_common_cmd) --emit=llvm-ir=$@ $<; \ $(rust_handle_depfile) $(obj)/%.ll: $(src)/%.rs FORCE diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 4a02b31cd102..d812241144d4 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -86,7 +86,8 @@ hostc_flags = -Wp,-MMD,$(depfile) \ hostcxx_flags = -Wp,-MMD,$(depfile) \ $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) -hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ +hostrust_flags = --emit=dep-info=$(depfile) \ + $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ $(HOSTRUSTFLAGS_$(target-stem)) # $(objtree)/$(obj) for including generated headers from checkin source files @@ -147,9 +148,7 @@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE # host-rust -> Executable quiet_cmd_host-rust = HOSTRUSTC $@ cmd_host-rust = \ - $(HOSTRUSTC) $(hostrust_flags) --emit=dep-info,link \ - --out-dir=$(obj)/ $<; \ - mv $(obj)/$(target-stem).d $(depfile); \ + $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $<; \ sed -i '/^\#/d' $(depfile) $(host-rust): $(obj)/%: $(src)/%.rs FORCE $(call if_changed_dep,host-rust) -- cgit v1.2.3 From bc6df812a1529ab0cbac7f17ac6c4f4aff2a8bbf Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:16 +0900 Subject: fixdep: parse Makefile more correctly to handle comments etc. fixdep parses dependency files (*.d) emitted by the compiler. *.d files are Makefiles describing the dependencies of the main source file. fixdep understands minimal Makefile syntax. It works well enough for GCC and Clang, but not for rustc. This commit improves the parser a little more for better processing comments, escape sequences, etc. My main motivation is to drop comments. rustc may output comments (e.g. env-dep). Currentyly, rustc build rules invoke sed to remove comments, but it is more efficient to do it in fixdep. Signed-off-by: Masahiro Yamada Acked-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 173 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 119 insertions(+), 54 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 37782a632494..f5a51770eb74 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -94,6 +94,7 @@ #include #include #include +#include #include #include #include @@ -251,75 +252,139 @@ static int is_ignored_file(const char *s, int len) * assignments are parsed not only by make, but also by the rather simple * parser in scripts/mod/sumversion.c. */ -static void parse_dep_file(char *m, const char *target) +static void parse_dep_file(char *p, const char *target) { - char *p; - int is_last, is_target; - int saw_any_target = 0; - int is_first_dep = 0; - void *buf; - - while (1) { - /* Skip any "white space" */ - while (*m == ' ' || *m == '\\' || *m == '\n') - m++; - - if (!*m) + bool saw_any_target = false; + bool is_target = true; + bool is_source = false; + bool need_parse; + char *q, saved_c; + + while (*p) { + /* handle some special characters first. */ + switch (*p) { + case '#': + /* + * skip comments. + * rustc may emit comments to dep-info. + */ + p++; + while (*p != '\0' && *p != '\n') { + /* + * escaped newlines continue the comment across + * multiple lines. + */ + if (*p == '\\') + p++; + p++; + } + continue; + case ' ': + case '\t': + /* skip whitespaces */ + p++; + continue; + case '\\': + /* + * backslash/newline combinations continue the + * statement. Skip it just like a whitespace. + */ + if (*(p + 1) == '\n') { + p += 2; + continue; + } break; - - /* Find next "white space" */ - p = m; - while (*p && *p != ' ' && *p != '\\' && *p != '\n') + case '\n': + /* + * Makefiles use a line-based syntax, where the newline + * is the end of a statement. After seeing a newline, + * we expect the next token is a target. + */ p++; - is_last = (*p == '\0'); - /* Is the token we found a target name? */ - is_target = (*(p-1) == ':'); - /* Don't write any target names into the dependency file */ - if (is_target) { - /* The /next/ file is the first dependency */ - is_first_dep = 1; - } else if (!is_ignored_file(m, p - m)) { - *p = '\0'; - + is_target = true; + continue; + case ':': /* - * Do not list the source file as dependency, so that - * kbuild is not confused if a .c file is rewritten - * into .S or vice versa. Storing it in source_* is - * needed for modpost to compute srcversions. + * assume the first dependency after a colon as the + * source file. */ - if (is_first_dep) { + p++; + is_target = false; + is_source = true; + continue; + } + + /* find the end of the token */ + q = p; + while (*q != ' ' && *q != '\t' && *q != '\n' && *q != '#' && *q != ':') { + if (*q == '\\') { /* - * If processing the concatenation of multiple - * dependency files, only process the first - * target name, which will be the original - * source name, and ignore any other target - * names, which will be intermediate temporary - * files. + * backslash/newline combinations work like as + * a whitespace, so this is the end of token. */ - if (!saw_any_target) { - saw_any_target = 1; - printf("source_%s := %s\n\n", - target, m); - printf("deps_%s := \\\n", target); + if (*(q + 1) == '\n') + break; + + /* escaped special characters */ + if (*(q + 1) == '#' || *(q + 1) == ':') { + memmove(p + 1, p, q - p); + p++; } - is_first_dep = 0; - } else { - printf(" %s \\\n", m); + + q++; } - buf = read_file(m); - parse_config_file(buf); - free(buf); + if (*q == '\0') + break; + q++; } - if (is_last) - break; + /* Just discard the target */ + if (is_target) { + p = q; + continue; + } + + saved_c = *q; + *q = '\0'; + need_parse = false; /* - * Start searching for next token immediately after the first - * "whitespace" character that follows this token. + * Do not list the source file as dependency, so that kbuild is + * not confused if a .c file is rewritten into .S or vice versa. + * Storing it in source_* is needed for modpost to compute + * srcversions. */ - m = p + 1; + if (is_source) { + /* + * The DT build rule concatenates multiple dep files. + * When processing them, only process the first source + * name, which will be the original one, and ignore any + * other source names, which will be intermediate + * temporary files. + */ + if (!saw_any_target) { + saw_any_target = true; + printf("source_%s := %s\n\n", target, p); + printf("deps_%s := \\\n", target); + need_parse = true; + } + } else if (!is_ignored_file(p, q - p)) { + printf(" %s \\\n", p); + need_parse = true; + } + + if (need_parse) { + void *buf; + + buf = read_file(p); + parse_config_file(buf); + free(buf); + } + + is_source = false; + *q = saved_c; + p = q; } if (!saw_any_target) { -- cgit v1.2.3 From 2185242faddd12a1ba1060be5caf584fe5aba93a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:17 +0900 Subject: kbuild: remove sed commands after rustc rules rustc may put comments in dep-info, so sed is used to drop them before passing it to fixdep. Now that fixdep can remove comments, Makefiles do not need to run sed. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- rust/Makefile | 6 ++---- scripts/Makefile.build | 18 ++++-------------- scripts/Makefile.host | 3 +-- 3 files changed, 7 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/rust/Makefile b/rust/Makefile index 865afb87bc9b..f403b79cae5a 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -333,8 +333,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ --crate-type proc-macro \ - --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \ - sed -i '/^\#/d' $(depfile) + --crate-name $(patsubst lib%.so,%,$(notdir $@)) $< # Procedural macros can only be used with the `rustc` that compiled it. # Therefore, to get `libmacros.so` automatically recompiled when the compiler @@ -350,8 +349,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --emit=dep-info=$(depfile) --emit=obj=$@ \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ - --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ - sed -i '/^\#/d' $(depfile) \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) rust-analyzer: diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 40de20246e50..76323201232a 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -289,9 +289,6 @@ rust_common_cmd = \ --crate-name $(basename $(notdir $@)) \ --emit=dep-info=$(depfile) -rust_handle_depfile = \ - sed -i '/^\#/d' $(depfile) - # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit # will be used. We explicitly request `-Ccodegen-units=1` in any case, and # the compiler shows a warning if it is not 1. However, if we ever stop @@ -301,9 +298,7 @@ rust_handle_depfile = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = \ - $(rust_common_cmd) --emit=obj=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(obj)/%.o: $(src)/%.rs FORCE $(call if_changed_dep,rustc_o_rs) @@ -311,24 +306,19 @@ $(obj)/%.o: $(src)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ - command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ - $(rust_handle_depfile) + command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ $(obj)/%.rsi: $(src)/%.rs FORCE $(call if_changed_dep,rustc_rsi_rs) quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_s_rs = \ - $(rust_common_cmd) --emit=asm=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $< $(obj)/%.s: $(src)/%.rs FORCE $(call if_changed_dep,rustc_s_rs) quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_ll_rs = \ - $(rust_common_cmd) --emit=llvm-ir=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $< $(obj)/%.ll: $(src)/%.rs FORCE $(call if_changed_dep,rustc_ll_rs) diff --git a/scripts/Makefile.host b/scripts/Makefile.host index d812241144d4..7aea9005e497 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -148,8 +148,7 @@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE # host-rust -> Executable quiet_cmd_host-rust = HOSTRUSTC $@ cmd_host-rust = \ - $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $<; \ - sed -i '/^\#/d' $(depfile) + $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $< $(host-rust): $(obj)/%: $(src)/%.rs FORCE $(call if_changed_dep,host-rust) -- cgit v1.2.3 From 871d657385466d380133e457fc86672bedf14cd9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:18 +0900 Subject: fixdep: refactor hash table lookup Change the hash table code so it will be easier to add the second table. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index f5a51770eb74..74f90a0deeb9 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -125,25 +125,11 @@ static unsigned int strhash(const char *str, unsigned int sz) return hash; } -/* - * Lookup a value in the configuration string. - */ -static int is_defined_config(const char *name, int len, unsigned int hash) -{ - struct item *aux; - - for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { - if (aux->hash == hash && aux->len == len && - memcmp(aux->name, name, len) == 0) - return 1; - } - return 0; -} - /* * Add a new value to the configuration string. */ -static void define_config(const char *name, int len, unsigned int hash) +static void add_to_hashtable(const char *name, int len, unsigned int hash, + struct item *hashtab[]) { struct item *aux = malloc(sizeof(*aux) + len); @@ -158,17 +144,34 @@ static void define_config(const char *name, int len, unsigned int hash) hashtab[hash % HASHSZ] = aux; } +/* + * Lookup a string in the hash table. If found, just return true. + * If not, add it to the hashtable and return false. + */ +static bool in_hashtable(const char *name, int len, struct item *hashtab[]) +{ + struct item *aux; + unsigned int hash = strhash(name, len); + + for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { + if (aux->hash == hash && aux->len == len && + memcmp(aux->name, name, len) == 0) + return true; + } + + add_to_hashtable(name, len, hash, hashtab); + + return false; +} + /* * Record the use of a CONFIG_* word. */ static void use_config(const char *m, int slen) { - unsigned int hash = strhash(m, slen); - - if (is_defined_config(m, slen, hash)) - return; + if (in_hashtable(m, slen, config_hashtab)) + return; - define_config(m, slen, hash); /* Print out a dependency path from a symbol name. */ printf(" $(wildcard include/config/%.*s) \\\n", slen, m); } -- cgit v1.2.3 From faa91c472be8ffa3121c9db803d3e529d48e246a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:19 +0900 Subject: fixdep: avoid parsing the same file over again The dep files (*.d files) emitted by C compilers usually contain the deduplicated list of included files. One exceptional case is when a header is included by the -include command line option, and also by #include directive. For example, the top Makefile adds the command line option, "-include $(srctree)/include/linux/kconfig.h". You do not need to include in every source file. In fact, include/linux/kconfig.h is listed twice in many .*.cmd files due to include/linux/xarray.h having "#include ". I did not fix that since it is a small redundancy. However, this is more annoying for rustc. rustc emits the dependency for each emission type. For example, cmd_rustc_library emits dep-info, obj, and metadata. So, the emitted *.d file contains the dependency for those 3 targets, which makes fixdep parse the same file 3 times. $ grep rust/alloc/raw_vec.rs rust/.alloc.o.cmd rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ To skip the second parsing, this commit adds a hash table for parsed files, just like we did for CONFIG options. Signed-off-by: Masahiro Yamada Acked-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- scripts/basic/fixdep.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 74f90a0deeb9..e22e689de61e 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *config_hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ], *file_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -365,6 +365,10 @@ static void parse_dep_file(char *p, const char *target) * name, which will be the original one, and ignore any * other source names, which will be intermediate * temporary files. + * + * rustc emits the same dependency list for each + * emission type. It is enough to list the source name + * just once. */ if (!saw_any_target) { saw_any_target = true; @@ -372,7 +376,8 @@ static void parse_dep_file(char *p, const char *target) printf("deps_%s := \\\n", target); need_parse = true; } - } else if (!is_ignored_file(p, q - p)) { + } else if (!is_ignored_file(p, q - p) && + !in_hashtable(p, q - p, file_hashtab)) { printf(" %s \\\n", p); need_parse = true; } -- cgit v1.2.3 From 93c656de8da8b53317bd88ecb7cce592310995d8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:20 +0900 Subject: fixdep: do not parse *.rlib, *.rmeta, *.so fixdep is designed only for parsing text files. read_file() appends a terminating null byte ('\0') and parse_config_file() calls strstr() to search for CONFIG options. rustc outputs *.rlib, *.rmeta, *.so to dep-info. fixdep needs them in the dependency, but there is no point in parsing such binary files. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index e22e689de61e..fa562806c2be 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -250,6 +250,15 @@ static int is_ignored_file(const char *s, int len) str_ends_with(s, len, "include/generated/autoksyms.h"); } +/* Do not parse these files */ +static int is_no_parse_file(const char *s, int len) +{ + /* rustc may list binary files in dep-info */ + return str_ends_with(s, len, ".rlib") || + str_ends_with(s, len, ".rmeta") || + str_ends_with(s, len, ".so"); +} + /* * Important: The below generated source_foo.o and deps_foo.o variable * assignments are parsed not only by make, but also by the rather simple @@ -382,7 +391,7 @@ static void parse_dep_file(char *p, const char *target) need_parse = true; } - if (need_parse) { + if (need_parse && !is_no_parse_file(p, q - p)) { void *buf; buf = read_file(p); -- cgit v1.2.3 From c83b16cefd78f55071840e1159ead9fe62747769 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:45:45 +0900 Subject: kbuild: rust: move rust/target.json to scripts/ scripts/ is a better place to generate files used treewide. With target.json moved to scripts/, you do not need to add target.json to no-clean-files or MRPROPER_FILES. 'make clean' does not visit scripts/, but 'make mrproper' does. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- Makefile | 4 ++-- rust/.gitignore | 1 - rust/Makefile | 10 +--------- scripts/.gitignore | 1 + scripts/Makefile | 8 +++++++- scripts/remove-stale-files | 2 ++ 6 files changed, 13 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/Makefile b/Makefile index b9d1411ef25b..ef6d593634cf 100644 --- a/Makefile +++ b/Makefile @@ -569,7 +569,7 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -std=gnu11 KBUILD_CPPFLAGS := -D__KERNEL__ KBUILD_RUSTFLAGS := $(rust_common_flags) \ - --target=$(objtree)/rust/target.json \ + --target=$(objtree)/scripts/target.json \ -Cpanic=abort -Cembed-bitcode=n -Clto=n \ -Cforce-unwind-tables=n -Ccodegen-units=1 \ -Csymbol-mangling-version=v0 \ @@ -1606,7 +1606,7 @@ MRPROPER_FILES += include/config include/generated \ certs/x509.genkey \ vmlinux-gdb.py \ *.spec \ - rust/target.json rust/libmacros.so + rust/libmacros.so # clean - Delete most, but leave enough to build external modules # diff --git a/rust/.gitignore b/rust/.gitignore index 9bd1af8e05a1..168cb26a31b9 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -target.json bindings_generated.rs bindings_helpers_generated.rs exports_*_generated.h diff --git a/rust/Makefile b/rust/Makefile index f403b79cae5a..829f4adfd85c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -1,8 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -always-$(CONFIG_RUST) += target.json -no-clean-files += target.json - obj-$(CONFIG_RUST) += core.o compiler_builtins.o always-$(CONFIG_RUST) += exports_core_generated.h @@ -231,11 +228,6 @@ rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ $(call if_changed,rustc_test) $(call if_changed,rustc_test_library) -filechk_rust_target = $(objtree)/scripts/generate_rust_target < $< - -$(obj)/target.json: $(objtree)/include/config/auto.conf FORCE - $(call filechk,rust_target) - ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) else @@ -359,7 +351,7 @@ rust-analyzer: $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs $(obj)/target.json FORCE +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE $(call if_changed_dep,rustc_library) $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' diff --git a/scripts/.gitignore b/scripts/.gitignore index b7aec8eb1bd4..11bf3c075fb6 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -8,4 +8,5 @@ /recordmcount /sign-file /sorttable +/target.json /unifdef diff --git a/scripts/Makefile b/scripts/Makefile index 1575af84d557..0e0ae3c06ed7 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -10,8 +10,14 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert -hostprogs-always-$(CONFIG_RUST) += generate_rust_target +always-$(CONFIG_RUST) += target.json +filechk_rust_target = $< < include/config/auto.conf + +$(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE + $(call filechk,rust_target) + +hostprogs += generate_rust_target generate_rust_target-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index cdbdde89a271..c71bf2f68360 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -27,3 +27,5 @@ rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert rm -f scripts/kconfig/[gmnq]conf-cfg + +rm -f rust/target.json -- cgit v1.2.3 From 12fec3d60118237e4674a463c2c2923b8279ad67 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 9 Jan 2023 22:40:40 +0900 Subject: kbuild: replace $(dot-target).tmp in filechk with $(tmp-target) $(tmp-target) is a better fit for local use like this. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 4648ab8f11d4..7778cc97a4e0 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -99,11 +99,11 @@ define filechk $(check-FORCE) $(Q)set -e; \ mkdir -p $(dir $@); \ - trap "rm -f $(dot-target).tmp" EXIT; \ - { $(filechk_$(1)); } > $(dot-target).tmp; \ - if [ ! -r $@ ] || ! cmp -s $@ $(dot-target).tmp; then \ + trap "rm -f $(tmp-target)" EXIT; \ + { $(filechk_$(1)); } > $(tmp-target); \ + if [ ! -r $@ ] || ! cmp -s $@ $(tmp-target); then \ $(kecho) ' UPD $@'; \ - mv -f $(dot-target).tmp $@; \ + mv -f $(tmp-target) $@; \ fi endef -- cgit v1.2.3 From d5c8d6e0fa61401a729e9eb6a9c7077b2d3aebb0 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 11 Jan 2023 20:05:01 -0700 Subject: kbuild: Update assembler calls to use proper flags and language target as-instr uses KBUILD_AFLAGS, but as-option uses KBUILD_CFLAGS. This can cause as-option to fail unexpectedly when CONFIG_WERROR is set, because clang will emit -Werror,-Wunused-command-line-argument for various -m and -f flags in KBUILD_CFLAGS for assembler sources. Callers of as-option and as-instr should be adding flags to KBUILD_AFLAGS / aflags-y, not KBUILD_CFLAGS / cflags-y. Use KBUILD_AFLAGS in all macros to clear up the initial problem. Unfortunately, -Wunused-command-line-argument can still be triggered with clang by the presence of warning flags or macro definitions because '-x assembler' is used, instead of '-x assembler-with-cpp', which will consume these flags. Switch to '-x assembler-with-cpp' in places where '-x assembler' is used, as the compiler is always used as the driver for out of line assembler sources in the kernel. Finally, add -Werror to these macros so that they behave consistently whether or not CONFIG_WERROR is set. [nathan: Reworded and expanded on problems in commit message Use '-x assembler-with-cpp' in a couple more places] Link: https://github.com/ClangBuiltLinux/linux/issues/1699 Suggested-by: Masahiro Yamada Signed-off-by: Nick Desaulniers Signed-off-by: Nathan Chancellor Tested-by: Linux Kernel Functional Testing Tested-by: Anders Roxell Signed-off-by: Masahiro Yamada --- scripts/Kconfig.include | 2 +- scripts/Makefile.compiler | 8 ++++---- scripts/as-version.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include index 274125307ebd..5a84b6443875 100644 --- a/scripts/Kconfig.include +++ b/scripts/Kconfig.include @@ -33,7 +33,7 @@ ld-option = $(success,$(LD) -v $(1)) # $(as-instr,) # Return y if the assembler supports , n otherwise -as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler -o /dev/null -) +as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler-with-cpp -o /dev/null -) # check if $(CC) and $(LD) exist $(error-if,$(failure,command -v $(CC)),C compiler '$(CC)' not found) diff --git a/scripts/Makefile.compiler b/scripts/Makefile.compiler index 3d8adfd34af1..7aa1fbc4aafe 100644 --- a/scripts/Makefile.compiler +++ b/scripts/Makefile.compiler @@ -29,16 +29,16 @@ try-run = $(shell set -e; \ fi) # as-option -# Usage: cflags-y += $(call as-option,-Wa$(comma)-isa=foo,) +# Usage: aflags-y += $(call as-option,-Wa$(comma)-isa=foo,) as-option = $(call try-run,\ - $(CC) $(KBUILD_CFLAGS) $(1) -c -x assembler /dev/null -o "$$TMP",$(1),$(2)) + $(CC) -Werror $(KBUILD_AFLAGS) $(1) -c -x assembler-with-cpp /dev/null -o "$$TMP",$(1),$(2)) # as-instr -# Usage: cflags-y += $(call as-instr,instr,option1,option2) +# Usage: aflags-y += $(call as-instr,instr,option1,option2) as-instr = $(call try-run,\ - printf "%b\n" "$(1)" | $(CC) $(KBUILD_AFLAGS) -c -x assembler -o "$$TMP" -,$(2),$(3)) + printf "%b\n" "$(1)" | $(CC) -Werror $(KBUILD_AFLAGS) -c -x assembler-with-cpp -o "$$TMP" -,$(2),$(3)) # __cc-option # Usage: MY_CFLAGS += $(call __cc-option,$(CC),$(MY_CFLAGS),-march=winchip-c6,-march=i586) diff --git a/scripts/as-version.sh b/scripts/as-version.sh index 1a21495e9ff0..af717476152d 100755 --- a/scripts/as-version.sh +++ b/scripts/as-version.sh @@ -45,7 +45,7 @@ orig_args="$@" # Get the first line of the --version output. IFS=' ' -set -- $(LC_ALL=C "$@" -Wa,--version -c -x assembler /dev/null -o /dev/null 2>/dev/null) +set -- $(LC_ALL=C "$@" -Wa,--version -c -x assembler-with-cpp /dev/null -o /dev/null 2>/dev/null) # Split the line on spaces. IFS=' ' -- cgit v1.2.3 From db1547c56886742283d7566c872f89cbad76a14c Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 11 Jan 2023 20:05:10 -0700 Subject: kbuild: Turn a couple more of clang's unused option warnings into errors Currently, these warnings are hidden with -Qunused-arguments in KBUILD_CPPFLAGS. Once that option is removed, these warnings should be turned into hard errors to make unconditionally added but unsupported flags for the current compilation mode or target obvious due to a failed build; otherwise, the warnings might just be ignored if the build log is not checked. Link: https://github.com/ClangBuiltLinux/linux/issues/1587 Signed-off-by: Nathan Chancellor Reviewed-by: Nick Desaulniers Tested-by: Linux Kernel Functional Testing Tested-by: Anders Roxell Signed-off-by: Masahiro Yamada --- scripts/Makefile.clang | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.clang b/scripts/Makefile.clang index 87285b76adb2..70b354fa1cb4 100644 --- a/scripts/Makefile.clang +++ b/scripts/Makefile.clang @@ -36,6 +36,8 @@ endif # so they can be implemented or wrapped in cc-option. CLANG_FLAGS += -Werror=unknown-warning-option CLANG_FLAGS += -Werror=ignored-optimization-argument +CLANG_FLAGS += -Werror=option-ignored +CLANG_FLAGS += -Werror=unused-command-line-argument KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_AFLAGS += $(CLANG_FLAGS) export CLANG_FLAGS -- cgit v1.2.3 From 87c7ee67deb7fce9951a5f9d80641138694aad17 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 12 Jan 2023 11:30:06 +0900 Subject: scripts: handle BrokenPipeError for python scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the follow-up of commit fb3041d61f68 ("kbuild: fix SIGPIPE error message for AR=gcc-ar and AR=llvm-ar"), Kees Cook pointed out that tools should _not_ catch their own SIGPIPEs [1] [2]. Based on his feedback, LLVM was fixed [3]. However, Python's default behavior is to show noisy bracktrace when SIGPIPE is sent. So, scripts written in Python are basically in the same situation as the buggy llvm tools. Example: $ make -s allnoconfig $ make -s allmodconfig $ scripts/diffconfig .config.old .config | head -n1 -ALIX n Traceback (most recent call last): File "/home/masahiro/linux/scripts/diffconfig", line 132, in main() File "/home/masahiro/linux/scripts/diffconfig", line 130, in main print_config("+", config, None, b[config]) File "/home/masahiro/linux/scripts/diffconfig", line 64, in print_config print("+%s %s" % (config, new_value)) BrokenPipeError: [Errno 32] Broken pipe Python documentation [4] notes how to make scripts die immediately and silently: """ Piping output of your program to tools like head(1) will cause a SIGPIPE signal to be sent to your process when the receiver of its standard output closes early. This results in an exception like BrokenPipeError: [Errno 32] Broken pipe. To handle this case, wrap your entry point to catch this exception as follows: import os import sys def main(): try: # simulate large output (your code replaces this loop) for x in range(10000): print("y") # flush output here to force SIGPIPE to be triggered # while inside this try block. sys.stdout.flush() except BrokenPipeError: # Python flushes standard streams on exit; redirect remaining output # to devnull to avoid another BrokenPipeError at shutdown devnull = os.open(os.devnull, os.O_WRONLY) os.dup2(devnull, sys.stdout.fileno()) sys.exit(1) # Python exits with error code 1 on EPIPE if __name__ == '__main__': main() Do not set SIGPIPE’s disposition to SIG_DFL in order to avoid BrokenPipeError. Doing that would cause your program to exit unexpectedly whenever any socket connection is interrupted while your program is still writing to it. """ Currently, tools/perf/scripts/python/intel-pt-events.py seems to be the only script that fixes the issue that way. tools/perf/scripts/python/compaction-times.py uses another approach signal.signal(signal.SIGPIPE, signal.SIG_DFL) but the Python documentation clearly says "Don't do it". I cannot fix all Python scripts since there are so many. I fixed some in the scripts/ directory. [1]: https://lore.kernel.org/all/202211161056.1B9611A@keescook/ [2]: https://github.com/llvm/llvm-project/issues/59037 [3]: https://github.com/llvm/llvm-project/commit/4787efa38066adb51e2c049499d25b3610c0877b [4]: https://docs.python.org/3/library/signal.html#note-on-sigpipe Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers Reviewed-by: Nicolas Schier --- scripts/checkkconfigsymbols.py | 13 ++++++++++++- scripts/clang-tools/run-clang-tools.py | 21 ++++++++++++++------- scripts/diffconfig | 16 ++++++++++++++-- 3 files changed, 40 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/checkkconfigsymbols.py b/scripts/checkkconfigsymbols.py index 217d21abc86e..36c920e71313 100755 --- a/scripts/checkkconfigsymbols.py +++ b/scripts/checkkconfigsymbols.py @@ -115,7 +115,7 @@ def parse_options(): return args -def main(): +def print_undefined_symbols(): """Main function of this module.""" args = parse_options() @@ -467,5 +467,16 @@ def parse_kconfig_file(kfile): return defined, references +def main(): + try: + print_undefined_symbols() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE + + if __name__ == "__main__": main() diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py index 56f2ec8f0f40..3266708a8658 100755 --- a/scripts/clang-tools/run-clang-tools.py +++ b/scripts/clang-tools/run-clang-tools.py @@ -61,14 +61,21 @@ def run_analysis(entry): def main(): - args = parse_arguments() + try: + args = parse_arguments() - lock = multiprocessing.Lock() - pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) - # Read JSON data into the datastore variable - with open(args.path, "r") as f: - datastore = json.load(f) - pool.map(run_analysis, datastore) + lock = multiprocessing.Lock() + pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) + # Read JSON data into the datastore variable + with open(args.path, "r") as f: + datastore = json.load(f) + pool.map(run_analysis, datastore) + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE if __name__ == "__main__": diff --git a/scripts/diffconfig b/scripts/diffconfig index d5da5fa05d1d..43f0f3d273ae 100755 --- a/scripts/diffconfig +++ b/scripts/diffconfig @@ -65,7 +65,7 @@ def print_config(op, config, value, new_value): else: print(" %s %s -> %s" % (config, value, new_value)) -def main(): +def show_diff(): global merge_style # parse command line args @@ -129,4 +129,16 @@ def main(): for config in new: print_config("+", config, None, b[config]) -main() +def main(): + try: + show_diff() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From ec61452aaad6bb0d4f3f4d4b78d27bed90eb2494 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 Jan 2023 16:12:15 +0900 Subject: scripts: remove bin2c Commit 80f8be7af03f ("tomoyo: Omit use of bin2c") removed the last use of bin2c. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier Reviewed-by: Sedat Dilek --- Documentation/dontdiff | 1 - init/Kconfig | 4 ---- scripts/.gitignore | 1 - scripts/Makefile | 1 - scripts/bin2c.c | 36 ------------------------------------ scripts/remove-stale-files | 2 ++ 6 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 scripts/bin2c.c (limited to 'scripts') diff --git a/Documentation/dontdiff b/Documentation/dontdiff index 352ff53a2306..3c399f132e2d 100644 --- a/Documentation/dontdiff +++ b/Documentation/dontdiff @@ -91,7 +91,6 @@ asm_offsets.h autoconf.h* av_permissions.h bbootsect -bin2c binkernel.spec bootsect bounds.h diff --git a/init/Kconfig b/init/Kconfig index 44e90b28a30f..fded4396f25f 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -682,10 +682,6 @@ config CPU_ISOLATION source "kernel/rcu/Kconfig" -config BUILD_BIN2C - bool - default n - config IKCONFIG tristate "Kernel .config support" help diff --git a/scripts/.gitignore b/scripts/.gitignore index 11bf3c075fb6..6e9ce6720a05 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only /asn1_compiler -/bin2c /generate_rust_target /insert-sys-cert /kallsyms diff --git a/scripts/Makefile b/scripts/Makefile index 0e0ae3c06ed7..32b6ba722728 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -3,7 +3,6 @@ # scripts contains sources for various helper programs used throughout # the kernel for the build process. -hostprogs-always-$(CONFIG_BUILD_BIN2C) += bin2c hostprogs-always-$(CONFIG_KALLSYMS) += kallsyms hostprogs-always-$(BUILD_C_RECORDMCOUNT) += recordmcount hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable diff --git a/scripts/bin2c.c b/scripts/bin2c.c deleted file mode 100644 index c3d7eef3ad06..000000000000 --- a/scripts/bin2c.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Unloved program to convert a binary on stdin to a C include on stdout - * - * Jan 1999 Matt Mackall - * - * This software may be used and distributed according to the terms - * of the GNU General Public License, incorporated herein by reference. - */ - -#include - -int main(int argc, char *argv[]) -{ - int ch, total = 0; - - if (argc > 1) - printf("const char %s[] %s=\n", - argv[1], argc > 2 ? argv[2] : ""); - - do { - printf("\t\""); - while ((ch = getchar()) != EOF) { - total++; - printf("\\x%02x", ch); - if (total % 16 == 0) - break; - } - printf("\"\n"); - } while (ch != EOF); - - if (argc > 1) - printf("\t;\n\n#include \n\nconst size_t %s_size = %d;\n", - argv[1], total); - - return 0; -} diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index c71bf2f68360..04fcdf739638 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -29,3 +29,5 @@ rm -f scripts/extract-cert rm -f scripts/kconfig/[gmnq]conf-cfg rm -f rust/target.json + +rm -f scripts/bin2c -- cgit v1.2.3 From a5e40d86317774ebc94307da395817902ef78252 Mon Sep 17 00:00:00 2001 From: Sven Joachim Date: Thu, 29 Oct 2020 16:39:10 +0100 Subject: builddeb: Consolidate consecutive chmod calls into one No need to call chmod three times when it can do everything at once. Signed-off-by: Sven Joachim Signed-off-by: Masahiro Yamada --- scripts/package/builddeb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 67cd420dcf89..ff5e7d8e380b 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -42,11 +42,9 @@ create_package() { else chown -R root:root "$pdir" fi - chmod -R go-w "$pdir" - # in case we are in a restrictive umask environment like 0077 - chmod -R a+rX "$pdir" - # in case we build in a setuid/setgid directory - chmod -R ug-s "$pdir" + # a+rX in case we are in a restrictive umask environment like 0077 + # ug-s in case we build in a setuid/setgid directory + chmod -R go-w,a+rX,ug-s "$pdir" # Create the package dpkg-gencontrol -p$pname -P"$pdir" -- cgit v1.2.3 From c9f9cf2560e40b62015c6c4a04be60f55ce5240e Mon Sep 17 00:00:00 2001 From: Bastian Germann Date: Wed, 26 May 2021 01:01:37 +0200 Subject: builddeb: clean generated package content For each binary Debian package, a directory with the package name is created in the debian directory. Correct the generated file matches in the package's clean target, which were renamed without adjusting the target. Fixes: 1694e94e4f46 ("builddeb: match temporary directory name to the package name") Signed-off-by: Bastian Germann Signed-off-by: Masahiro Yamada --- scripts/package/mkdebian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index 6cf383225b8b..c3bbef7a6754 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -236,7 +236,7 @@ binary-arch: build-arch KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile intdeb-pkg clean: - rm -rf debian/*tmp debian/files + rm -rf debian/files debian/linux-* \$(MAKE) clean binary: binary-arch -- cgit v1.2.3 From f6e09b07cc12a4d104bb19fe7566b0636f60c413 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Jan 2023 23:14:21 +0900 Subject: kbuild: do not put .scmversion into the source tarball .scmversion is used by (src)rpm-pkg and deb-pkg to carry KERNELRELEASE. In fact, deb-pkg does not rely on it any more because the generated debian/rules specifies KERNELRELEASE from the command line. Do likwise for (src)rpm-pkg, and remove this feature. For the same reason, you do not need to save LOCALVERSION in the spec file. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 6 ++---- scripts/package/mkspec | 8 +++----- scripts/remove-stale-files | 2 ++ scripts/setlocalversion | 19 +------------------ 4 files changed, 8 insertions(+), 27 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 525a2820976f..e84c4e8ceb8e 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -32,7 +32,7 @@ export KDEB_SOURCENAME TAR_CONTENT := Documentation LICENSES arch block certs crypto drivers fs \ include init io_uring ipc kernel lib mm net rust \ samples scripts security sound tools usr virt \ - .config .scmversion Makefile \ + .config Makefile \ Kbuild Kconfig COPYING $(wildcard localversion*) MKSPEC := $(srctree)/scripts/package/mkspec @@ -47,10 +47,8 @@ if test "$(objtree)" != "$(srctree)"; then \ echo >&2; \ false; \ fi ; \ -$(srctree)/scripts/setlocalversion --save-scmversion; \ tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \ - --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \ -rm -f $(objtree)/.scmversion + --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3) # rpm-pkg # --------------------------------------------------------------------------- diff --git a/scripts/package/mkspec b/scripts/package/mkspec index 094e52c979a8..108c0cb95436 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -33,8 +33,6 @@ EXCLUDES="$RCS_TAR_IGNORE --exclude=*vmlinux* --exclude=*.mod \ --exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation \ --exclude=.config.old --exclude=.missing-syscalls.d --exclude=*.s" -test -n "$LOCALVERSION" && MAKE="$MAKE LOCALVERSION=$LOCALVERSION" - # We can label the here-doc lines for conditional output to the spec file # # Labels: @@ -90,7 +88,7 @@ $S rm -f scripts/basic/fixdep scripts/kconfig/conf $S rm -f tools/objtool/{fixdep,objtool} $S $S %build -$S $MAKE %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release} +$S $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE KBUILD_BUILD_VERSION=%{release} $S %install mkdir -p %{buildroot}/boot @@ -101,8 +99,8 @@ $S %else cp \$($MAKE -s image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE %endif -$M $MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install - $MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install +$M $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE INSTALL_MOD_PATH=%{buildroot} modules_install + $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE cp .config %{buildroot}/boot/config-$KERNELRELEASE $S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/build diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index 04fcdf739638..7f432900671a 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -31,3 +31,5 @@ rm -f scripts/kconfig/[gmnq]conf-cfg rm -f rust/target.json rm -f scripts/bin2c + +rm -f .scmversion diff --git a/scripts/setlocalversion b/scripts/setlocalversion index af4754a35e66..3b31702b4a4a 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -11,16 +11,11 @@ # usage() { - echo "Usage: $0 [--save-scmversion] [srctree]" >&2 + echo "Usage: $0 [srctree]" >&2 exit 1 } -scm_only=false srctree=. -if test "$1" = "--save-scmversion"; then - scm_only=true - shift -fi if test $# -gt 0; then srctree=$1 shift @@ -35,10 +30,6 @@ scm_version() short=false cd "$srctree" - if test -e .scmversion; then - cat .scmversion - return - fi if test "$1" = "--short"; then short=true fi @@ -103,14 +94,6 @@ collect_files() echo "$res" } -if $scm_only; then - if test ! -e .scmversion; then - res=$(scm_version) - echo "$res" >.scmversion - fi - exit -fi - if ! test -e include/config/auto.conf; then echo "Error: kernelrelease not valid - run 'make prepare' to update it" >&2 exit 1 -- cgit v1.2.3 From 992ebfab2a75c276fe27f7fd7a3fb326ccd7225b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Jan 2023 23:14:22 +0900 Subject: setlocalversion: simplify the construction of the short version With the --short option given, scm_version() prints "+". Just append it. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/setlocalversion | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/setlocalversion b/scripts/setlocalversion index 3b31702b4a4a..5cdf409204aa 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -121,8 +121,7 @@ elif [ "${LOCALVERSION+set}" != "set" ]; then # # If the variable LOCALVERSION is set (including being set # to an empty string), we don't want to append a plus sign. - scm=$(scm_version --short) - res="$res${scm:++}" + res="$res$(scm_version --short)" fi echo "$res" -- cgit v1.2.3 From 75280bdf49b2f563fb5404df7c7b735c118695fe Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Jan 2023 23:14:23 +0900 Subject: setlocalversion: make indentation shallower Return earlier if we are not in the correct git repository. This makes the code more readable. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/setlocalversion | 79 +++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 38 deletions(-) (limited to 'scripts') diff --git a/scripts/setlocalversion b/scripts/setlocalversion index 5cdf409204aa..b8e1018d611e 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -34,46 +34,49 @@ scm_version() short=true fi - # Check for git and a git repo. - if test -z "$(git rev-parse --show-cdup 2>/dev/null)" && - head=$(git rev-parse --verify HEAD 2>/dev/null); then - - # If we are at a tagged commit (like "v2.6.30-rc6"), we ignore - # it, because this version is defined in the top level Makefile. - if [ -z "$(git describe --exact-match 2>/dev/null)" ]; then - - # If only the short version is requested, don't bother - # running further git commands - if $short; then - echo "+" - return - fi - # If we are past a tagged commit (like - # "v2.6.30-rc5-302-g72357d5"), we pretty print it. - if atag="$(git describe 2>/dev/null)"; then - echo "$atag" | awk -F- '{printf("-%05d", $(NF-1))}' - fi - - # Add -g and exactly 12 hex chars. - printf '%s%s' -g "$(echo $head | cut -c1-12)" - fi + if test -n "$(git rev-parse --show-cdup 2>/dev/null)"; then + return + fi + + if ! head=$(git rev-parse --verify HEAD 2>/dev/null); then + return + fi + + # If we are at a tagged commit (like "v2.6.30-rc6"), we ignore it + # because this version is defined in the top level Makefile. + if [ -z "$(git describe --exact-match 2>/dev/null)" ]; then - # Check for uncommitted changes. - # This script must avoid any write attempt to the source tree, - # which might be read-only. - # You cannot use 'git describe --dirty' because it tries to - # create .git/index.lock . - # First, with git-status, but --no-optional-locks is only - # supported in git >= 2.14, so fall back to git-diff-index if - # it fails. Note that git-diff-index does not refresh the - # index, so it may give misleading results. See - # git-update-index(1), git-diff-index(1), and git-status(1). - if { - git --no-optional-locks status -uno --porcelain 2>/dev/null || - git diff-index --name-only HEAD - } | read dummy; then - printf '%s' -dirty + # If only the short version is requested, don't bother + # running further git commands + if $short; then + echo "+" + return fi + # If we are past a tagged commit (like + # "v2.6.30-rc5-302-g72357d5"), we pretty print it. + if atag="$(git describe 2>/dev/null)"; then + echo "$atag" | awk -F- '{printf("-%05d", $(NF-1))}' + fi + + # Add -g and exactly 12 hex chars. + printf '%s%s' -g "$(echo $head | cut -c1-12)" + fi + + # Check for uncommitted changes. + # This script must avoid any write attempt to the source tree, which + # might be read-only. + # You cannot use 'git describe --dirty' because it tries to create + # .git/index.lock . + # First, with git-status, but --no-optional-locks is only supported in + # git >= 2.14, so fall back to git-diff-index if it fails. Note that + # git-diff-index does not refresh the index, so it may give misleading + # results. + # See git-update-index(1), git-diff-index(1), and git-status(1). + if { + git --no-optional-locks status -uno --porcelain 2>/dev/null || + git diff-index --name-only HEAD + } | read dummy; then + printf '%s' -dirty fi } -- cgit v1.2.3 From ec31f868ec674edfcf653cc7c82b365c6f570cd9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Jan 2023 23:14:25 +0900 Subject: setlocalversion: absorb $(KERNELVERSION) Print $(KERNELVERSION) in setlocalversion so that the callers get simpler. Signed-off-by: Masahiro Yamada --- Makefile | 5 ++--- scripts/setlocalversion | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/Makefile b/Makefile index cf30a8b6463e..203b86e95197 100644 --- a/Makefile +++ b/Makefile @@ -1258,8 +1258,7 @@ vmlinux: vmlinux.o $(KBUILD_LDS) modpost # make sure no implicit rule kicks in $(sort $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)): . ; -filechk_kernel.release = \ - echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))" +filechk_kernel.release = $(srctree)/scripts/setlocalversion $(srctree) # Store (new) KERNELRELEASE string in include/config/kernel.release include/config/kernel.release: FORCE @@ -2124,7 +2123,7 @@ checkstack: $(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH) kernelrelease: - @echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))" + @$(srctree)/scripts/setlocalversion $(srctree) kernelversion: @echo $(KERNELVERSION) diff --git a/scripts/setlocalversion b/scripts/setlocalversion index b8e1018d611e..ad72ea2c8228 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -102,6 +102,11 @@ if ! test -e include/config/auto.conf; then exit 1 fi +if [ -z "${KERNELVERSION}" ]; then + echo "KERNELVERSION is not set" >&2 + exit 1 +fi + # localversion* files in the build and source directory res="$(collect_files localversion*)" if test ! "$srctree" -ef .; then @@ -127,4 +132,4 @@ elif [ "${LOCALVERSION+set}" != "set" ]; then res="$res$(scm_version --short)" fi -echo "$res" +echo "${KERNELVERSION}${res}" -- cgit v1.2.3 From 58e0e5c85e31b0b30f16ac835d537ba0b7ff7d37 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 24 Jan 2023 12:04:16 +0900 Subject: kbuild: deb-pkg: add --source-option=-sP If the source package fails to build, ../linux.orig is left over. In the next run of 'make deb-pkg', you will get the following error: dpkg-source: error: orig directory 'linux.orig' already exists, not overwriting, giving up; use -sA, -sK or -sP to override You can manually remove ../linux.orig, but it is annoying. Pass -sP down to dpkg-source. Signed-off-by: Masahiro Yamada --- scripts/Makefile.package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index e84c4e8ceb8e..dfbf40454a99 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -86,7 +86,7 @@ deb-pkg: $(call cmd,src_tar,$(KDEB_SOURCENAME)) origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\ mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz - +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -i.git -us -uc + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) --source-option=-sP -i.git -us -uc PHONY += bindeb-pkg bindeb-pkg: -- cgit v1.2.3 From 5573b4daa26a0cf15aa0fecd7f1be16e0b6157bc Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 26 Jan 2023 11:26:43 +0900 Subject: kbuild: do not automatically add -w option to modpost When there is a missing input file (vmlinux.o or Module.symvers), you are likely to get a ton of unresolved symbols. Currently, Kbuild automatically adds the -w option to allow module builds to continue with warnings instead of errors. This may not be what the user expects because it is generally more useful to catch all possible issues at build time instead of at run time. Let's not do what the user did not ask. If you still want to build modules anyway, you can proceed by explicitly setting KBUILD_MODPOST_WARN=1. Since you may miss a real issue, you need to be aware of what you are doing. Suggested-by: William McVicker Signed-off-by: Masahiro Yamada Tested-by: Will McVicker --- scripts/Makefile.modpost | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index 43343e13c542..0980c58d8afc 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -121,16 +121,14 @@ modpost-args += -e $(addprefix -i , $(KBUILD_EXTRA_SYMBOLS)) endif # ($(KBUILD_EXTMOD),) -ifneq ($(missing-input),) -modpost-args += -w -endif - quiet_cmd_modpost = MODPOST $@ cmd_modpost = \ $(if $(missing-input), \ echo >&2 "WARNING: $(missing-input) is missing."; \ echo >&2 " Modules may not have dependencies or modversions."; \ - echo >&2 " You may get many unresolved symbol warnings.";) \ + echo >&2 " You may get many unresolved symbol errors."; \ + echo >&2 " You can set KBUILD_MODPOST_WARN=1 to turn errors into warning"; \ + echo >&2 " if you want to proceed at your own risk.";) \ $(MODPOST) $(modpost-args) targets += $(output-symdump) -- cgit v1.2.3 From eed36d77517786e4b3a9f17c6a66c6df2fc99442 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 Feb 2023 21:09:56 +0900 Subject: setlocalversion: clean up the construction of version output Concatenate all components in the last line instead of accumulating them into the 'res' variable. No functional change is intended. A preparation for the next change. Signed-off-by: Masahiro Yamada --- scripts/setlocalversion | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/setlocalversion b/scripts/setlocalversion index ad72ea2c8228..7e2c83f5c50d 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -108,19 +108,18 @@ if [ -z "${KERNELVERSION}" ]; then fi # localversion* files in the build and source directory -res="$(collect_files localversion*)" +file_localversion="$(collect_files localversion*)" if test ! "$srctree" -ef .; then - res="$res$(collect_files "$srctree"/localversion*)" + file_localversion="${file_localversion}$(collect_files "$srctree"/localversion*)" fi -# CONFIG_LOCALVERSION and LOCALVERSION (if set) +# version string from CONFIG_LOCALVERSION config_localversion=$(sed -n 's/^CONFIG_LOCALVERSION=\(.*\)$/\1/p' include/config/auto.conf) -res="${res}${config_localversion}${LOCALVERSION}" # scm version string if not at a tagged commit if grep -q "^CONFIG_LOCALVERSION_AUTO=y$" include/config/auto.conf; then # full scm version string - res="$res$(scm_version)" + scm_version="$(scm_version)" elif [ "${LOCALVERSION+set}" != "set" ]; then # If the variable LOCALVERSION is not set, append a plus # sign if the repository is not in a clean annotated or @@ -129,7 +128,7 @@ elif [ "${LOCALVERSION+set}" != "set" ]; then # # If the variable LOCALVERSION is set (including being set # to an empty string), we don't want to append a plus sign. - res="$res$(scm_version --short)" + scm_version="$(scm_version --short)" fi -echo "${KERNELVERSION}${res}" +echo "${KERNELVERSION}${file_localversion}${config_localversion}${LOCALVERSION}${scm_version}" -- cgit v1.2.3 From 6ab7e1f95e96f0c688ae132b0e9a16c0f206689d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 Feb 2023 21:09:57 +0900 Subject: setlocalversion: use only the correct release tag for git-describe Currently, setlocalversion uses any annotated tag for git-describe. If we are at a tagged commit, it will not append the commit hash. $ git checkout v6.2-rc1^ $ make -s defconfig kernelrelease 6.1.0-14595-g292a089d78d3 $ git tag -a foo -m foo $ make -s kernelrelease 6.1.0 If a local tag 'foo' exists, it pretends to be a released version '6.1.0', while there are many commits on top of it. The output should be consistent irrespective of such a local tag. Pass the correct release tag to --match option of git-describe. In the mainline kernel, the SUBLEVEL is always '0', which is omitted from the tag. KERNELVERSION annotated tag 6.1.0 -> v6.1 (mainline) 6.2.0-rc5 -> v6.2-rc5 (mainline, release candidate) 6.1.7 -> v6.1.7 (stable) To preserve the behavior in linux-next, use the tag derived from localversion* files if exists. In linux-next, the local version is specified by the localversion-next file. Signed-off-by: Masahiro Yamada --- scripts/setlocalversion | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/setlocalversion b/scripts/setlocalversion index 7e2c83f5c50d..e54839a42d4b 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -27,6 +27,7 @@ fi scm_version() { local short + local tag short=false cd "$srctree" @@ -42,9 +43,21 @@ scm_version() return fi - # If we are at a tagged commit (like "v2.6.30-rc6"), we ignore it - # because this version is defined in the top level Makefile. - if [ -z "$(git describe --exact-match 2>/dev/null)" ]; then + # If a localversion*' file and the corresponding annotated tag exist, + # use it. This is the case in linux-next. + tag=${file_localversion#-} + tag=$(git describe --exact-match --match=$tag $tag 2>/dev/null) + + # Otherwise, default to the annotated tag derived from KERNELVERSION. + # mainline kernel: 6.2.0-rc5 -> v6.2-rc5 + # stable kernel: 6.1.7 -> v6.1.7 + if [ -z "${tag}" ]; then + tag=v$(echo "${KERNELVERSION}" | sed -E 's/^([0-9]+\.[0-9]+)\.0(.*)$/\1\2/') + fi + + # If we are at the tagged commit, we ignore it because the version is + # well-defined. + if [ -z "$(git describe --exact-match --match=$tag 2>/dev/null)" ]; then # If only the short version is requested, don't bother # running further git commands @@ -52,9 +65,9 @@ scm_version() echo "+" return fi - # If we are past a tagged commit (like - # "v2.6.30-rc5-302-g72357d5"), we pretty print it. - if atag="$(git describe 2>/dev/null)"; then + # If we are past the tagged commit, we pretty print it. + # (like 6.1.0-14595-g292a089d78d3) + if atag="$(git describe --match=$tag 2>/dev/null)"; then echo "$atag" | awk -F- '{printf("-%05d", $(NF-1))}' fi @@ -116,7 +129,7 @@ fi # version string from CONFIG_LOCALVERSION config_localversion=$(sed -n 's/^CONFIG_LOCALVERSION=\(.*\)$/\1/p' include/config/auto.conf) -# scm version string if not at a tagged commit +# scm version string if not at the kernel version tag or at the file_localversion if grep -q "^CONFIG_LOCALVERSION_AUTO=y$" include/config/auto.conf; then # full scm version string scm_version="$(scm_version)" -- cgit v1.2.3 From 5c3d1d0abb12a6915d0f43233837053945621a89 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:23 +0900 Subject: kbuild: add a tool to list files ignored by git In short, the motivation of this commit is to build a source package without cleaning the source tree. The deb-pkg and (src)rpm-pkg targets first run 'make clean' before creating a source tarball. Otherwise build artifacts such as *.o, *.a, etc. would be included in the tarball. Yet, the tarball ends up containing several garbage files since 'make clean' does not clean everything. Cleaning the tree every time is annoying since it makes the incremental build impossible. It is desirable to create a source tarball without cleaning the tree. In fact, there are some ways to achieve this. The easiest solution is 'git archive'. 'make perf-tar*-src-pkg' uses it, but I do not like it because it works only when the source tree is managed by git, and all files you want in the tarball must be committed in advance. I want to make it work without relying on git. We can do this. Files that are ignored by git are generated files, so should be excluded from the source tarball. We can list them out by parsing the .gitignore files. Of course, .gitignore does not cover all the cases, but it works well enough. tar(1) claims to support it: --exclude-vcs-ignores Exclude files that match patterns read from VCS-specific ignore files. Supported files are: .cvsignore, .gitignore, .bzrignore, and .hgignore. The best scenario would be to use 'tar --exclude-vcs-ignores', but this option does not work. --exclude-vcs-ignore does not understand any of the negation (!), preceding slash, following slash, etc.. So, this option is just useless. Hence, I wrote this gitignore parser. The previous version [1], written in Python, was so slow. This version is implemented in C, so it works much faster. I imported the code from git (commit: 23c56f7bd5f1), so we get the same result. This tool traverses the source tree, parsing all .gitignore files, and prints file paths that are ignored by git. The output is similar to 'git ls-files --ignored --directory --others --exclude-per-directory=.gitignore', except [1] Not sorted [2] No trailing slash for directories [2] is intentional because tar's --exclude-from option cannot handle trailing slashes. [How to test this tool] $ git clean -dfx $ make -s -j$(nproc) defconfig all # or allmodconifg or whatever $ git archive -o ../linux1.tar --prefix=./ HEAD $ tar tf ../linux1.tar | LANG=C sort > ../file-list1 # files emitted by 'git archive' $ make scripts_package HOSTCC scripts/list-gitignored $ scripts/list-gitignored --prefix=./ -o ../exclude-list $ tar cf ../linux2.tar --exclude-from=../exclude-list . $ tar tf ../linux2.tar | LANG=C sort > ../file-list2 # files emitted by 'tar' $ diff ../file-list1 ../file-list2 | grep -E '^(<|>)' < ./Documentation/devicetree/bindings/.yamllint < ./drivers/clk/.kunitconfig < ./drivers/gpu/drm/tests/.kunitconfig < ./drivers/hid/.kunitconfig < ./fs/ext4/.kunitconfig < ./fs/fat/.kunitconfig < ./kernel/kcsan/.kunitconfig < ./lib/kunit/.kunitconfig < ./mm/kfence/.kunitconfig < ./tools/testing/selftests/arm64/tags/ < ./tools/testing/selftests/arm64/tags/.gitignore < ./tools/testing/selftests/arm64/tags/Makefile < ./tools/testing/selftests/arm64/tags/run_tags_test.sh < ./tools/testing/selftests/arm64/tags/tags_test.c < ./tools/testing/selftests/kvm/.gitignore < ./tools/testing/selftests/kvm/Makefile < ./tools/testing/selftests/kvm/config < ./tools/testing/selftests/kvm/settings The source tarball contains most of files that are tracked by git. You see some diffs, but it is just because some .gitignore files are wrong. $ git ls-files -i -c --exclude-per-directory=.gitignore Documentation/devicetree/bindings/.yamllint drivers/clk/.kunitconfig drivers/gpu/drm/tests/.kunitconfig drivers/hid/.kunitconfig fs/ext4/.kunitconfig fs/fat/.kunitconfig kernel/kcsan/.kunitconfig lib/kunit/.kunitconfig mm/kfence/.kunitconfig tools/testing/selftests/arm64/tags/.gitignore tools/testing/selftests/arm64/tags/Makefile tools/testing/selftests/arm64/tags/run_tags_test.sh tools/testing/selftests/arm64/tags/tags_test.c tools/testing/selftests/kvm/.gitignore tools/testing/selftests/kvm/Makefile tools/testing/selftests/kvm/config tools/testing/selftests/kvm/settings [1]: https://lore.kernel.org/all/20230128173843.765212-1-masahiroy@kernel.org/ Signed-off-by: Masahiro Yamada --- Makefile | 7 +- scripts/.gitignore | 1 + scripts/Makefile | 2 +- scripts/list-gitignored.c | 1057 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1065 insertions(+), 2 deletions(-) create mode 100644 scripts/list-gitignored.c (limited to 'scripts') diff --git a/Makefile b/Makefile index 2faf872b6808..7c726fd26540 100644 --- a/Makefile +++ b/Makefile @@ -274,7 +274,8 @@ no-dot-config-targets := $(clean-targets) \ cscope gtags TAGS tags help% %docs check% coccicheck \ $(version_h) headers headers_% archheaders archscripts \ %asm-generic kernelversion %src-pkg dt_binding_check \ - outputmakefile rustavailable rustfmt rustfmtcheck + outputmakefile rustavailable rustfmt rustfmtcheck \ + scripts_package # Installation targets should not require compiler. Unfortunately, vdso_install # is an exception where build artifacts may be updated. This must be fixed. no-compiler-targets := $(no-dot-config-targets) install dtbs_install \ @@ -1652,6 +1653,10 @@ distclean: mrproper %pkg: include/config/kernel.release FORCE $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.package $@ +PHONY += scripts_package +scripts_package: scripts_basic + $(Q)$(MAKE) $(build)=scripts scripts/list-gitignored + # Brief documentation of the typical targets used # --------------------------------------------------------------------------- diff --git a/scripts/.gitignore b/scripts/.gitignore index 6e9ce6720a05..feb43045d1b1 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -3,6 +3,7 @@ /generate_rust_target /insert-sys-cert /kallsyms +/list-gitignored /module.lds /recordmcount /sign-file diff --git a/scripts/Makefile b/scripts/Makefile index 32b6ba722728..e8917975905c 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -38,7 +38,7 @@ HOSTCFLAGS_sorttable.o += -DMCOUNT_SORT_ENABLED endif # The following programs are only built on demand -hostprogs += unifdef +hostprogs += list-gitignored unifdef # The module linker script is preprocessed on demand targets += module.lds diff --git a/scripts/list-gitignored.c b/scripts/list-gitignored.c new file mode 100644 index 000000000000..f9941f8dcd2b --- /dev/null +++ b/scripts/list-gitignored.c @@ -0,0 +1,1057 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Traverse the source tree, parsing all .gitignore files, and print file paths +// that are ignored by git. +// The output is suitable to the --exclude-from option of tar. +// This is useful until the --exclude-vcs-ignores option gets working correctly. +// +// Copyright (C) 2023 Masahiro Yamada +// (a lot of code imported from GIT) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Imported from commit 23c56f7bd5f1667f8b793d796bf30e39545920f6 in GIT +// +//---------------------------(IMPORT FROM GIT BEGIN)--------------------------- + +// Copied from environment.c + +static bool ignore_case; + +// Copied from git-compat-util.h + +/* Sane ctype - no locale, and works with signed chars */ +#undef isascii +#undef isspace +#undef isdigit +#undef isalpha +#undef isalnum +#undef isprint +#undef islower +#undef isupper +#undef tolower +#undef toupper +#undef iscntrl +#undef ispunct +#undef isxdigit + +static const unsigned char sane_ctype[256]; +#define GIT_SPACE 0x01 +#define GIT_DIGIT 0x02 +#define GIT_ALPHA 0x04 +#define GIT_GLOB_SPECIAL 0x08 +#define GIT_REGEX_SPECIAL 0x10 +#define GIT_PATHSPEC_MAGIC 0x20 +#define GIT_CNTRL 0x40 +#define GIT_PUNCT 0x80 +#define sane_istest(x,mask) ((sane_ctype[(unsigned char)(x)] & (mask)) != 0) +#define isascii(x) (((x) & ~0x7f) == 0) +#define isspace(x) sane_istest(x,GIT_SPACE) +#define isdigit(x) sane_istest(x,GIT_DIGIT) +#define isalpha(x) sane_istest(x,GIT_ALPHA) +#define isalnum(x) sane_istest(x,GIT_ALPHA | GIT_DIGIT) +#define isprint(x) ((x) >= 0x20 && (x) <= 0x7e) +#define islower(x) sane_iscase(x, 1) +#define isupper(x) sane_iscase(x, 0) +#define is_glob_special(x) sane_istest(x,GIT_GLOB_SPECIAL) +#define iscntrl(x) (sane_istest(x,GIT_CNTRL)) +#define ispunct(x) sane_istest(x, GIT_PUNCT | GIT_REGEX_SPECIAL | \ + GIT_GLOB_SPECIAL | GIT_PATHSPEC_MAGIC) +#define isxdigit(x) (hexval_table[(unsigned char)(x)] != -1) +#define tolower(x) sane_case((unsigned char)(x), 0x20) +#define toupper(x) sane_case((unsigned char)(x), 0) + +static inline int sane_case(int x, int high) +{ + if (sane_istest(x, GIT_ALPHA)) + x = (x & ~0x20) | high; + return x; +} + +static inline int sane_iscase(int x, int is_lower) +{ + if (!sane_istest(x, GIT_ALPHA)) + return 0; + + if (is_lower) + return (x & 0x20) != 0; + else + return (x & 0x20) == 0; +} + +// Copied from ctype.c + +enum { + S = GIT_SPACE, + A = GIT_ALPHA, + D = GIT_DIGIT, + G = GIT_GLOB_SPECIAL, /* *, ?, [, \\ */ + R = GIT_REGEX_SPECIAL, /* $, (, ), +, ., ^, {, | */ + P = GIT_PATHSPEC_MAGIC, /* other non-alnum, except for ] and } */ + X = GIT_CNTRL, + U = GIT_PUNCT, + Z = GIT_CNTRL | GIT_SPACE +}; + +static const unsigned char sane_ctype[256] = { + X, X, X, X, X, X, X, X, X, Z, Z, X, X, Z, X, X, /* 0.. 15 */ + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 16.. 31 */ + S, P, P, P, R, P, P, P, R, R, G, R, P, P, R, P, /* 32.. 47 */ + D, D, D, D, D, D, D, D, D, D, P, P, P, P, P, G, /* 48.. 63 */ + P, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, /* 64.. 79 */ + A, A, A, A, A, A, A, A, A, A, A, G, G, U, R, P, /* 80.. 95 */ + P, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, /* 96..111 */ + A, A, A, A, A, A, A, A, A, A, A, R, R, U, P, X, /* 112..127 */ + /* Nothing in the 128.. range */ +}; + +// Copied from hex.c + +static const signed char hexval_table[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ +}; + +// Copied from wildmatch.h + +#define WM_CASEFOLD 1 +#define WM_PATHNAME 2 + +#define WM_NOMATCH 1 +#define WM_MATCH 0 +#define WM_ABORT_ALL -1 +#define WM_ABORT_TO_STARSTAR -2 + +// Copied from wildmatch.c + +typedef unsigned char uchar; + +// local modification: remove NEGATE_CLASS(2) + +#define CC_EQ(class, len, litmatch) ((len) == sizeof (litmatch)-1 \ + && *(class) == *(litmatch) \ + && strncmp((char*)class, litmatch, len) == 0) + +// local modification: simpilify macros +#define ISBLANK(c) ((c) == ' ' || (c) == '\t') +#define ISGRAPH(c) (isprint(c) && !isspace(c)) +#define ISPRINT(c) isprint(c) +#define ISDIGIT(c) isdigit(c) +#define ISALNUM(c) isalnum(c) +#define ISALPHA(c) isalpha(c) +#define ISCNTRL(c) iscntrl(c) +#define ISLOWER(c) islower(c) +#define ISPUNCT(c) ispunct(c) +#define ISSPACE(c) isspace(c) +#define ISUPPER(c) isupper(c) +#define ISXDIGIT(c) isxdigit(c) + +/* Match pattern "p" against "text" */ +static int dowild(const uchar *p, const uchar *text, unsigned int flags) +{ + uchar p_ch; + const uchar *pattern = p; + + for ( ; (p_ch = *p) != '\0'; text++, p++) { + int matched, match_slash, negated; + uchar t_ch, prev_ch; + if ((t_ch = *text) == '\0' && p_ch != '*') + return WM_ABORT_ALL; + if ((flags & WM_CASEFOLD) && ISUPPER(t_ch)) + t_ch = tolower(t_ch); + if ((flags & WM_CASEFOLD) && ISUPPER(p_ch)) + p_ch = tolower(p_ch); + switch (p_ch) { + case '\\': + /* Literal match with following character. Note that the test + * in "default" handles the p[1] == '\0' failure case. */ + p_ch = *++p; + /* FALLTHROUGH */ + default: + if (t_ch != p_ch) + return WM_NOMATCH; + continue; + case '?': + /* Match anything but '/'. */ + if ((flags & WM_PATHNAME) && t_ch == '/') + return WM_NOMATCH; + continue; + case '*': + if (*++p == '*') { + const uchar *prev_p = p - 2; + while (*++p == '*') {} + if (!(flags & WM_PATHNAME)) + /* without WM_PATHNAME, '*' == '**' */ + match_slash = 1; + else if ((prev_p < pattern || *prev_p == '/') && + (*p == '\0' || *p == '/' || + (p[0] == '\\' && p[1] == '/'))) { + /* + * Assuming we already match 'foo/' and are at + * , just assume it matches + * nothing and go ahead match the rest of the + * pattern with the remaining string. This + * helps make foo/<*><*>/bar (<> because + * otherwise it breaks C comment syntax) match + * both foo/bar and foo/a/bar. + */ + if (p[0] == '/' && + dowild(p + 1, text, flags) == WM_MATCH) + return WM_MATCH; + match_slash = 1; + } else /* WM_PATHNAME is set */ + match_slash = 0; + } else + /* without WM_PATHNAME, '*' == '**' */ + match_slash = flags & WM_PATHNAME ? 0 : 1; + if (*p == '\0') { + /* Trailing "**" matches everything. Trailing "*" matches + * only if there are no more slash characters. */ + if (!match_slash) { + if (strchr((char *)text, '/')) + return WM_NOMATCH; + } + return WM_MATCH; + } else if (!match_slash && *p == '/') { + /* + * _one_ asterisk followed by a slash + * with WM_PATHNAME matches the next + * directory + */ + const char *slash = strchr((char*)text, '/'); + if (!slash) + return WM_NOMATCH; + text = (const uchar*)slash; + /* the slash is consumed by the top-level for loop */ + break; + } + while (1) { + if (t_ch == '\0') + break; + /* + * Try to advance faster when an asterisk is + * followed by a literal. We know in this case + * that the string before the literal + * must belong to "*". + * If match_slash is false, do not look past + * the first slash as it cannot belong to '*'. + */ + if (!is_glob_special(*p)) { + p_ch = *p; + if ((flags & WM_CASEFOLD) && ISUPPER(p_ch)) + p_ch = tolower(p_ch); + while ((t_ch = *text) != '\0' && + (match_slash || t_ch != '/')) { + if ((flags & WM_CASEFOLD) && ISUPPER(t_ch)) + t_ch = tolower(t_ch); + if (t_ch == p_ch) + break; + text++; + } + if (t_ch != p_ch) + return WM_NOMATCH; + } + if ((matched = dowild(p, text, flags)) != WM_NOMATCH) { + if (!match_slash || matched != WM_ABORT_TO_STARSTAR) + return matched; + } else if (!match_slash && t_ch == '/') + return WM_ABORT_TO_STARSTAR; + t_ch = *++text; + } + return WM_ABORT_ALL; + case '[': + p_ch = *++p; + if (p_ch == '^') + p_ch = '!'; + /* Assign literal 1/0 because of "matched" comparison. */ + negated = p_ch == '!' ? 1 : 0; + if (negated) { + /* Inverted character class. */ + p_ch = *++p; + } + prev_ch = 0; + matched = 0; + do { + if (!p_ch) + return WM_ABORT_ALL; + if (p_ch == '\\') { + p_ch = *++p; + if (!p_ch) + return WM_ABORT_ALL; + if (t_ch == p_ch) + matched = 1; + } else if (p_ch == '-' && prev_ch && p[1] && p[1] != ']') { + p_ch = *++p; + if (p_ch == '\\') { + p_ch = *++p; + if (!p_ch) + return WM_ABORT_ALL; + } + if (t_ch <= p_ch && t_ch >= prev_ch) + matched = 1; + else if ((flags & WM_CASEFOLD) && ISLOWER(t_ch)) { + uchar t_ch_upper = toupper(t_ch); + if (t_ch_upper <= p_ch && t_ch_upper >= prev_ch) + matched = 1; + } + p_ch = 0; /* This makes "prev_ch" get set to 0. */ + } else if (p_ch == '[' && p[1] == ':') { + const uchar *s; + int i; + for (s = p += 2; (p_ch = *p) && p_ch != ']'; p++) {} /*SHARED ITERATOR*/ + if (!p_ch) + return WM_ABORT_ALL; + i = p - s - 1; + if (i < 0 || p[-1] != ':') { + /* Didn't find ":]", so treat like a normal set. */ + p = s - 2; + p_ch = '['; + if (t_ch == p_ch) + matched = 1; + continue; + } + if (CC_EQ(s,i, "alnum")) { + if (ISALNUM(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "alpha")) { + if (ISALPHA(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "blank")) { + if (ISBLANK(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "cntrl")) { + if (ISCNTRL(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "digit")) { + if (ISDIGIT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "graph")) { + if (ISGRAPH(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "lower")) { + if (ISLOWER(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "print")) { + if (ISPRINT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "punct")) { + if (ISPUNCT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "space")) { + if (ISSPACE(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "upper")) { + if (ISUPPER(t_ch)) + matched = 1; + else if ((flags & WM_CASEFOLD) && ISLOWER(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "xdigit")) { + if (ISXDIGIT(t_ch)) + matched = 1; + } else /* malformed [:class:] string */ + return WM_ABORT_ALL; + p_ch = 0; /* This makes "prev_ch" get set to 0. */ + } else if (t_ch == p_ch) + matched = 1; + } while (prev_ch = p_ch, (p_ch = *++p) != ']'); + if (matched == negated || + ((flags & WM_PATHNAME) && t_ch == '/')) + return WM_NOMATCH; + continue; + } + } + + return *text ? WM_NOMATCH : WM_MATCH; +} + +/* Match the "pattern" against the "text" string. */ +static int wildmatch(const char *pattern, const char *text, unsigned int flags) +{ + // local modification: move WM_CASEFOLD here + if (ignore_case) + flags |= WM_CASEFOLD; + + return dowild((const uchar*)pattern, (const uchar*)text, flags); +} + +// Copied from dir.h + +#define PATTERN_FLAG_NODIR 1 +#define PATTERN_FLAG_ENDSWITH 4 +#define PATTERN_FLAG_MUSTBEDIR 8 +#define PATTERN_FLAG_NEGATIVE 16 + +// Copied from dir.c + +static int fspathncmp(const char *a, const char *b, size_t count) +{ + return ignore_case ? strncasecmp(a, b, count) : strncmp(a, b, count); +} + +static int simple_length(const char *match) +{ + int len = -1; + + for (;;) { + unsigned char c = *match++; + len++; + if (c == '\0' || is_glob_special(c)) + return len; + } +} + +static int no_wildcard(const char *string) +{ + return string[simple_length(string)] == '\0'; +} + +static void parse_path_pattern(const char **pattern, + int *patternlen, + unsigned *flags, + int *nowildcardlen) +{ + const char *p = *pattern; + size_t i, len; + + *flags = 0; + if (*p == '!') { + *flags |= PATTERN_FLAG_NEGATIVE; + p++; + } + len = strlen(p); + if (len && p[len - 1] == '/') { + len--; + *flags |= PATTERN_FLAG_MUSTBEDIR; + } + for (i = 0; i < len; i++) { + if (p[i] == '/') + break; + } + if (i == len) + *flags |= PATTERN_FLAG_NODIR; + *nowildcardlen = simple_length(p); + /* + * we should have excluded the trailing slash from 'p' too, + * but that's one more allocation. Instead just make sure + * nowildcardlen does not exceed real patternlen + */ + if (*nowildcardlen > len) + *nowildcardlen = len; + if (*p == '*' && no_wildcard(p + 1)) + *flags |= PATTERN_FLAG_ENDSWITH; + *pattern = p; + *patternlen = len; +} + +static void trim_trailing_spaces(char *buf) +{ + char *p, *last_space = NULL; + + for (p = buf; *p; p++) + switch (*p) { + case ' ': + if (!last_space) + last_space = p; + break; + case '\\': + p++; + if (!*p) + return; + /* fallthrough */ + default: + last_space = NULL; + } + + if (last_space) + *last_space = '\0'; +} + +static int match_basename(const char *basename, int basenamelen, + const char *pattern, int prefix, int patternlen, + unsigned flags) +{ + if (prefix == patternlen) { + if (patternlen == basenamelen && + !fspathncmp(pattern, basename, basenamelen)) + return 1; + } else if (flags & PATTERN_FLAG_ENDSWITH) { + /* "*literal" matching against "fooliteral" */ + if (patternlen - 1 <= basenamelen && + !fspathncmp(pattern + 1, + basename + basenamelen - (patternlen - 1), + patternlen - 1)) + return 1; + } else { + // local modification: call wildmatch() directly + if (!wildmatch(pattern, basename, flags)) + return 1; + } + return 0; +} + +static int match_pathname(const char *pathname, int pathlen, + const char *base, int baselen, + const char *pattern, int prefix, int patternlen) +{ + // local modification: remove local variables + + /* + * match with FNM_PATHNAME; the pattern has base implicitly + * in front of it. + */ + if (*pattern == '/') { + pattern++; + patternlen--; + prefix--; + } + + /* + * baselen does not count the trailing slash. base[] may or + * may not end with a trailing slash though. + */ + if (pathlen < baselen + 1 || + (baselen && pathname[baselen] != '/') || + fspathncmp(pathname, base, baselen)) + return 0; + + // local modification: simplified because always baselen > 0 + pathname += baselen + 1; + pathlen -= baselen + 1; + + if (prefix) { + /* + * if the non-wildcard part is longer than the + * remaining pathname, surely it cannot match. + */ + if (prefix > pathlen) + return 0; + + if (fspathncmp(pattern, pathname, prefix)) + return 0; + pattern += prefix; + patternlen -= prefix; + pathname += prefix; + pathlen -= prefix; + + /* + * If the whole pattern did not have a wildcard, + * then our prefix match is all we need; we + * do not need to call fnmatch at all. + */ + if (!patternlen && !pathlen) + return 1; + } + + // local modification: call wildmatch() directly + return !wildmatch(pattern, pathname, WM_PATHNAME); +} + +// Copied from git/utf8.c + +static const char utf8_bom[] = "\357\273\277"; + +//----------------------------(IMPORT FROM GIT END)---------------------------- + +struct pattern { + unsigned int flags; + int nowildcardlen; + int patternlen; + int dirlen; + char pattern[]; +}; + +static struct pattern **pattern_list; +static int nr_patterns, alloced_patterns; + +// Remember the number of patterns at each directory level +static int *nr_patterns_at; +// Track the current/max directory level; +static int depth, max_depth; +static bool debug_on; +static FILE *out_fp, *stat_fp; +static char *prefix = ""; +static char *progname; + +static void __attribute__((noreturn)) perror_exit(const char *s) +{ + perror(s); + + exit(EXIT_FAILURE); +} + +static void __attribute__((noreturn)) error_exit(const char *fmt, ...) +{ + va_list args; + + fprintf(stderr, "%s: error: ", progname); + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + exit(EXIT_FAILURE); +} + +static void debug(const char *fmt, ...) +{ + va_list args; + int i; + + if (!debug_on) + return; + + fprintf(stderr, "[DEBUG] "); + + for (i = 0; i < depth * 2; i++) + fputc(' ', stderr); + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +static void *xrealloc(void *ptr, size_t size) +{ + ptr = realloc(ptr, size); + if (!ptr) + perror_exit(progname); + + return ptr; +} + +static void *xmalloc(size_t size) +{ + return xrealloc(NULL, size); +} + +// similar to last_matching_pattern_from_list() in GIT +static bool is_ignored(const char *path, int pathlen, int dirlen, bool is_dir) +{ + int i; + + // Search in the reverse order because the last matching pattern wins. + for (i = nr_patterns - 1; i >= 0; i--) { + struct pattern *p = pattern_list[i]; + unsigned int flags = p->flags; + const char *gitignore_dir = p->pattern + p->patternlen + 1; + bool ignored; + + if ((flags & PATTERN_FLAG_MUSTBEDIR) && !is_dir) + continue; + + if (flags & PATTERN_FLAG_NODIR) { + if (!match_basename(path + dirlen + 1, + pathlen - dirlen - 1, + p->pattern, + p->nowildcardlen, + p->patternlen, + p->flags)) + continue; + } else { + if (!match_pathname(path, pathlen, + gitignore_dir, p->dirlen, + p->pattern, + p->nowildcardlen, + p->patternlen)) + continue; + } + + debug("%s: matches %s%s%s (%s/.gitignore)\n", path, + flags & PATTERN_FLAG_NEGATIVE ? "!" : "", p->pattern, + flags & PATTERN_FLAG_MUSTBEDIR ? "/" : "", + gitignore_dir); + + ignored = (flags & PATTERN_FLAG_NEGATIVE) == 0; + if (ignored) + debug("Ignore: %s\n", path); + + return ignored; + } + + debug("%s: no match\n", path); + + return false; +} + +static void add_pattern(const char *string, const char *dir, int dirlen) +{ + struct pattern *p; + int patternlen, nowildcardlen; + unsigned int flags; + + parse_path_pattern(&string, &patternlen, &flags, &nowildcardlen); + + if (patternlen == 0) + return; + + p = xmalloc(sizeof(*p) + patternlen + dirlen + 2); + + memcpy(p->pattern, string, patternlen); + p->pattern[patternlen] = 0; + memcpy(p->pattern + patternlen + 1, dir, dirlen); + p->pattern[patternlen + 1 + dirlen] = 0; + + p->patternlen = patternlen; + p->nowildcardlen = nowildcardlen; + p->dirlen = dirlen; + p->flags = flags; + + debug("Add pattern: %s%s%s\n", + flags & PATTERN_FLAG_NEGATIVE ? "!" : "", p->pattern, + flags & PATTERN_FLAG_MUSTBEDIR ? "/" : ""); + + if (nr_patterns >= alloced_patterns) { + alloced_patterns += 128; + pattern_list = xrealloc(pattern_list, + sizeof(*pattern_list) * alloced_patterns); + } + + pattern_list[nr_patterns++] = p; +} + +// similar to add_patterns_from_buffer() in GIT +static void add_patterns_from_gitignore(const char *dir, int dirlen) +{ + struct stat st; + char path[PATH_MAX], *buf, *entry; + size_t size; + int fd, pathlen, i; + + pathlen = snprintf(path, sizeof(path), "%s/.gitignore", dir); + if (pathlen >= sizeof(path)) + error_exit("%s: too long path was truncated\n", path); + + fd = open(path, O_RDONLY | O_NOFOLLOW); + if (fd < 0) { + if (errno != ENOENT) + return perror_exit(path); + return; + } + + if (fstat(fd, &st) < 0) + perror_exit(path); + + size = st.st_size; + + buf = xmalloc(size + 1); + if (read(fd, buf, st.st_size) != st.st_size) + perror_exit(path); + + buf[st.st_size] = '\n'; + if (close(fd)) + perror_exit(path); + + debug("Parse %s\n", path); + + entry = buf; + + // skip utf8 bom + if (!strncmp(entry, utf8_bom, strlen(utf8_bom))) + entry += strlen(utf8_bom); + + for (i = entry - buf; i < size; i++) { + if (buf[i] == '\n') { + if (entry != buf + i && entry[0] != '#') { + buf[i - (i && buf[i-1] == '\r')] = 0; + trim_trailing_spaces(entry); + add_pattern(entry, dir, dirlen); + } + entry = buf + i + 1; + } + } + + free(buf); +} + +// Save the current number of patterns and increment the depth +static void increment_depth(void) +{ + if (depth >= max_depth) { + max_depth += 1; + nr_patterns_at = xrealloc(nr_patterns_at, + sizeof(*nr_patterns_at) * max_depth); + } + + nr_patterns_at[depth] = nr_patterns; + depth++; +} + +// Decrement the depth, and free up the patterns of this directory level. +static void decrement_depth(void) +{ + depth--; + assert(depth >= 0); + + while (nr_patterns > nr_patterns_at[depth]) + free(pattern_list[--nr_patterns]); +} + +static void print_path(const char *path) +{ + // The path always starts with "./" + assert(strlen(path) >= 2); + + // Replace the root directory with a preferred prefix. + // This is useful for the tar command. + fprintf(out_fp, "%s%s\n", prefix, path + 2); +} + +static void print_stat(const char *path, struct stat *st) +{ + if (!stat_fp) + return; + + if (!S_ISREG(st->st_mode) && !S_ISLNK(st->st_mode)) + return; + + assert(strlen(path) >= 2); + + fprintf(stat_fp, "%c %9ld %10ld %s\n", + S_ISLNK(st->st_mode) ? 'l' : '-', + st->st_size, st->st_mtim.tv_sec, path + 2); +} + +// Traverse the entire directory tree, parsing .gitignore files. +// Print file paths that are not tracked by git. +// +// Return true if all files under the directory are ignored, false otherwise. +static bool traverse_directory(const char *dir, int dirlen) +{ + bool all_ignored = true; + DIR *dirp; + + debug("Enter[%d]: %s\n", depth, dir); + increment_depth(); + + add_patterns_from_gitignore(dir, dirlen); + + dirp = opendir(dir); + if (!dirp) + perror_exit(dir); + + while (1) { + struct dirent *d; + struct stat st; + char path[PATH_MAX]; + int pathlen; + bool ignored; + + errno = 0; + d = readdir(dirp); + if (!d) { + if (errno) + perror_exit(dir); + break; + } + + if (!strcmp(d->d_name, "..") || !strcmp(d->d_name, ".")) + continue; + + pathlen = snprintf(path, sizeof(path), "%s/%s", dir, d->d_name); + if (pathlen >= sizeof(path)) + error_exit("%s: too long path was truncated\n", path); + + if (lstat(path, &st) < 0) + perror_exit(path); + + if ((!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) || + is_ignored(path, pathlen, dirlen, S_ISDIR(st.st_mode))) { + ignored = true; + } else { + if (S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + // If all the files in a directory are ignored, + // let's ignore that directory as well. This + // will avoid empty directories in the tarball. + ignored = traverse_directory(path, pathlen); + else + ignored = false; + } + + if (ignored) { + print_path(path); + } else { + print_stat(path, &st); + all_ignored = false; + } + } + + if (closedir(dirp)) + perror_exit(dir); + + decrement_depth(); + debug("Leave[%d]: %s\n", depth, dir); + + return all_ignored; +} + +static void usage(void) +{ + fprintf(stderr, + "usage: %s [options]\n" + "\n" + "Show files that are ignored by git\n" + "\n" + "options:\n" + " -d, --debug print debug messages to stderr\n" + " -e, --exclude PATTERN add the given exclude pattern\n" + " -h, --help show this help message and exit\n" + " -i, --ignore-case Ignore case differences between the patterns and the files\n" + " -o, --output FILE output the ignored files to a file (default: '-', i.e. stdout)\n" + " -p, --prefix PREFIX prefix added to each path (default: empty string)\n" + " -r, --rootdir DIR root of the source tree (default: current working directory)\n" + " -s, --stat FILE output the file stat of non-ignored files to a file\n", + progname); +} + +static void open_output(const char *pathname, FILE **fp) +{ + if (strcmp(pathname, "-")) { + *fp = fopen(pathname, "w"); + if (!*fp) + perror_exit(pathname); + } else { + *fp = stdout; + } +} + +static void close_output(const char *pathname, FILE *fp) +{ + fflush(fp); + + if (ferror(fp)) + error_exit("not all data was written to the output\n"); + + if (fclose(fp)) + perror_exit(pathname); +} + +int main(int argc, char *argv[]) +{ + const char *output = "-"; + const char *rootdir = "."; + const char *stat = NULL; + + progname = strrchr(argv[0], '/'); + if (progname) + progname++; + else + progname = argv[0]; + + while (1) { + static struct option long_options[] = { + {"debug", no_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, + {"ignore-case", no_argument, NULL, 'i'}, + {"output", required_argument, NULL, 'o'}, + {"prefix", required_argument, NULL, 'p'}, + {"rootdir", required_argument, NULL, 'r'}, + {"stat", required_argument, NULL, 's'}, + {"exclude", required_argument, NULL, 'x'}, + {}, + }; + + int c = getopt_long(argc, argv, "dhino:p:r:s:x:", long_options, NULL); + + if (c == -1) + break; + + switch (c) { + case 'd': + debug_on = true; + break; + case 'h': + usage(); + exit(0); + case 'i': + ignore_case = true; + break; + case 'o': + output = optarg; + break; + case 'p': + prefix = optarg; + break; + case 'r': + rootdir = optarg; + break; + case 's': + stat = optarg; + break; + case 'x': + add_pattern(optarg, ".", strlen(".")); + break; + case '?': + usage(); + /* fallthrough */ + default: + exit(EXIT_FAILURE); + } + } + + open_output(output, &out_fp); + if (stat && stat[0]) + open_output(stat, &stat_fp); + + if (chdir(rootdir)) + perror_exit(rootdir); + + add_pattern(".git/", ".", strlen(".")); + + if (traverse_directory(".", strlen("."))) + print_path("./"); + + assert(depth == 0); + + while (nr_patterns > 0) + free(pattern_list[--nr_patterns]); + free(pattern_list); + free(nr_patterns_at); + + close_output(output, out_fp); + if (stat_fp) + close_output(stat, stat_fp); + + return 0; +} -- cgit v1.2.3 From 7bf4582d7aad870ecb4f760743307ecba7a960f4 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:24 +0900 Subject: kbuild: deb-pkg: create source package without cleaning If you run 'make deb-pkg', all objects are lost due to 'make clean', which makes the incremental builds impossible. Instead of cleaning, pass the exclude list to tar's --exclude-from option. Previously, *.diff.gz contained some check-in files such as .clang-format, .cocciconfig. With this commit, *.diff.gz will only contain the .config and debian/. The other source files will go into the .orig tarball. linux.tar.gz is rebuilt only when the source files that would go into the tarball are changed. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 48 ++++++++++++++++++++++++++++++++++++++++++------ scripts/package/mkdebian | 24 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index dfbf40454a99..f0002ace4156 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -43,13 +43,47 @@ if test "$(objtree)" != "$(srctree)"; then \ echo >&2 " ERROR:"; \ echo >&2 " Building source tarball is not possible outside the"; \ echo >&2 " kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \ - echo >&2 " binrpm-pkg or bindeb-pkg target instead."; \ + echo >&2 " binrpm-pkg target instead."; \ echo >&2; \ false; \ fi ; \ tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \ --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3) +# .tmp_filelist .tmp_filelist_exclude +# --------------------------------------------------------------------------- + +scripts/list-gitignored: FORCE + $(Q)$(MAKE) -f $(srctree)/Makefile scripts_package + +# 1f5d3a6b6532e25a5cdf1f311956b2b03d343a48 removed '*.rej' from .gitignore, +# but it is definitely a generated file. +filechk_filelist = \ + $< --exclude='*.rej' --output=$@_exclude --prefix=./ --rootdir=$(srctree) --stat=- + +.tmp_filelist: scripts/list-gitignored FORCE + $(call filechk,filelist) + +# tarball +# --------------------------------------------------------------------------- + +quiet_cmd_tar = TAR $@ + cmd_tar = tar -c -f $@ $(tar-compress-opt) $(tar-exclude-opt) \ + --owner=0 --group=0 --sort=name \ + --transform 's:^\.:$*:S' -C $(tar-rootdir) . + +tar-rootdir := $(srctree) + +%.tar.gz: private tar-compress-opt := -I $(KGZIP) +%.tar.gz: + $(call cmd,tar) + +# Linux source tarball +# --------------------------------------------------------------------------- + +linux.tar.gz: tar-exclude-opt = --exclude=./$@ --exclude-from=$<_exclude +linux.tar.gz: .tmp_filelist + # rpm-pkg # --------------------------------------------------------------------------- PHONY += rpm-pkg @@ -80,13 +114,12 @@ binrpm-pkg: $(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec PHONY += deb-pkg -deb-pkg: - $(MAKE) clean +deb-pkg: linux.tar.gz $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian - $(call cmd,src_tar,$(KDEB_SOURCENAME)) origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\ - mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz - +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) --source-option=-sP -i.git -us -uc + cp linux.tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) \ + --build=source,binary --source-option=-sP -nc -us -uc PHONY += bindeb-pkg bindeb-pkg: @@ -174,4 +207,7 @@ help: @echo ' perf-tarxz-src-pkg - Build $(perf-tar).tar.xz source tarball' @echo ' perf-tarzst-src-pkg - Build $(perf-tar).tar.zst source tarball' +PHONY += FORCE +FORCE: + .PHONY: $(PHONY) diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index c3bbef7a6754..68b13ef590ba 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -84,6 +84,8 @@ set_debarch() { fi } +rm -rf debian + # Some variables and settings used throughout the script version=$KERNELRELEASE if [ -n "$KDEB_PKGVERSION" ]; then @@ -135,6 +137,28 @@ fi mkdir -p debian/source/ echo "1.0" > debian/source/format +# Ugly: ignore anything except .config or debian/ +cat<<'EOF' > debian/source/local-options +diff-ignore + +extend-diff-ignore = ^[^.d] + +extend-diff-ignore = ^\.[^c] +extend-diff-ignore = ^\.c($|[^o]) +extend-diff-ignore = ^\.co($|[^n]) +extend-diff-ignore = ^\.con($|[^f]) +extend-diff-ignore = ^\.conf($|[^i]) +extend-diff-ignore = ^\.confi($|[^g]) +extend-diff-ignore = ^\.config. + +extend-diff-ignore = ^d($|[^e]) +extend-diff-ignore = ^de($|[^b]) +extend-diff-ignore = ^deb($|[^i]) +extend-diff-ignore = ^debi($|[^a]) +extend-diff-ignore = ^debia($|[^n]) +extend-diff-ignore = ^debian[^/] +EOF + echo $debarch > debian/arch extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)" extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)" -- cgit v1.2.3 From 1ec9bb704faf50c48727b5daa33bc42e7334db60 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:25 +0900 Subject: kbuild: rpm-pkg: build binary packages from source rpm The build rules of rpm-pkg and srcrpm-pkg are almost the same. Remove the code duplication. Change rpm-pkg to build binary packages from the source package generated by srcrpm-pkg. This changes the output directory of the srpm generated by 'make rpm-pkg' because srcrpm-pkg overrides _srcrpmdir. Signed-off-by: Masahiro Yamada --- scripts/Makefile.package | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index f0002ace4156..55441d776d70 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -87,11 +87,9 @@ linux.tar.gz: .tmp_filelist # rpm-pkg # --------------------------------------------------------------------------- PHONY += rpm-pkg -rpm-pkg: - $(MAKE) clean - $(CONFIG_SHELL) $(MKSPEC) >$(objtree)/kernel.spec - $(call cmd,src_tar,$(KERNELPATH),kernel.spec) - +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE)-linux -ta $(KERNELPATH).tar.gz \ +rpm-pkg: srpm = $(shell rpmspec --srpm --query --queryformat='%{name}-%{VERSION}-%{RELEASE}.src.rpm' kernel.spec) +rpm-pkg: srcrpm-pkg + +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE)-linux -rb $(srpm) \ --define='_smp_mflags %{nil}' # srcrpm-pkg -- cgit v1.2.3 From 6fc91752d7c4706f1036838544b1ef5d9f41c058 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:26 +0900 Subject: kbuild: srcrpm-pkg: create source package without cleaning If you run 'make (src)rpm-pkg', all objects are lost due to 'make clean', which makes the incremental builds impossible. Instead of cleaning, pass the exclude list to tar's --exclude-from option. Previously, the .config was contained in the source tarball. With this commit, the source rpm consists of separate linux.tar.gz and .config. Remove stale comments. Now, 'make (src)rpm-pkg' works with O= option. Signed-off-by: Masahiro Yamada --- scripts/Makefile.package | 32 ++++---------------------------- scripts/package/mkspec | 8 ++++---- 2 files changed, 8 insertions(+), 32 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 55441d776d70..7266a11855f4 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -3,27 +3,6 @@ include $(srctree)/scripts/Kbuild.include -# RPM target -# --------------------------------------------------------------------------- -# The rpm target generates two rpm files: -# /usr/src/packages/SRPMS/kernel-2.6.7rc2-1.src.rpm -# /usr/src/packages/RPMS/i386/kernel-2.6.7rc2-1..rpm -# The src.rpm files includes all source for the kernel being built -# The .rpm includes kernel configuration, modules etc. -# -# Process to create the rpm files -# a) clean the kernel -# b) Generate .spec file -# c) Build a tar ball, using symlink to make kernel version -# first entry in the path -# d) and pack the result to a tar.gz file -# e) generate the rpm files, based on kernel.spec -# - Use /. to avoid tar packing just the symlink - -# Note that the rpm-pkg target cannot be used with KBUILD_OUTPUT, -# but the binrpm-pkg target can; for some reason O= gets ignored. - -# Remove hyphens since they have special meaning in RPM filenames KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE)) KDEB_SOURCENAME ?= linux-upstream KBUILD_PKG_ROOTCMD ?="fakeroot -u" @@ -42,8 +21,7 @@ if test "$(objtree)" != "$(srctree)"; then \ echo >&2; \ echo >&2 " ERROR:"; \ echo >&2 " Building source tarball is not possible outside the"; \ - echo >&2 " kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \ - echo >&2 " binrpm-pkg target instead."; \ + echo >&2 " kernel source tree. Don't set KBUILD_OUTPUT"; \ echo >&2; \ false; \ fi ; \ @@ -95,12 +73,10 @@ rpm-pkg: srcrpm-pkg # srcrpm-pkg # --------------------------------------------------------------------------- PHONY += srcrpm-pkg -srcrpm-pkg: - $(MAKE) clean +srcrpm-pkg: linux.tar.gz $(CONFIG_SHELL) $(MKSPEC) >$(objtree)/kernel.spec - $(call cmd,src_tar,$(KERNELPATH),kernel.spec) - +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE)-linux -ts $(KERNELPATH).tar.gz \ - --define='_smp_mflags %{nil}' --define='_srcrpmdir $(srctree)' + +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE)-linux -bs kernel.spec \ + --define='_smp_mflags %{nil}' --define='_sourcedir .' --define='_srcrpmdir .' # binrpm-pkg # --------------------------------------------------------------------------- diff --git a/scripts/package/mkspec b/scripts/package/mkspec index 108c0cb95436..83a64d9d7372 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -47,7 +47,8 @@ sed -e '/^DEL/d' -e 's/^\t*//' < Date: Wed, 15 Feb 2023 10:20:27 +0900 Subject: kbuild: deb-pkg: hide KDEB_SOURCENAME from Makefile scripts/Makefile.package does not need to know the value of KDEB_SOURCENAME because the source name can be taken from debian/changelog by using dpkg-parsechangelog. Move the default of KDEB_SOURCENAME (i.e. linux-upstream) to scripts/package/mkdebian. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 23 +++++++++++++++-------- scripts/package/mkdebian | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 7266a11855f4..c207f4ec47ac 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -4,9 +4,7 @@ include $(srctree)/scripts/Kbuild.include KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE)) -KDEB_SOURCENAME ?= linux-upstream KBUILD_PKG_ROOTCMD ?="fakeroot -u" -export KDEB_SOURCENAME # Include only those top-level files that are needed by make, plus the GPL copy TAR_CONTENT := Documentation LICENSES arch block certs crypto drivers fs \ include init io_uring ipc kernel lib mm net rust \ @@ -87,17 +85,26 @@ binrpm-pkg: +rpmbuild $(RPMOPTS) --define "_builddir $(objtree)" --target \ $(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec +quiet_cmd_debianize = GEN $@ + cmd_debianize = $(srctree)/scripts/package/mkdebian + +debian: FORCE + $(call cmd,debianize) + +PHONY += debian-orig +debian-orig: private source = $(shell dpkg-parsechangelog -S Source) +debian-orig: private version = $(shell dpkg-parsechangelog -S Version | sed 's/-[^-]*$$//') +debian-orig: private orig-name = $(source)_$(version).orig.tar.gz +debian-orig: linux.tar.gz debian + $(Q)cp $< ../$(orig-name) + PHONY += deb-pkg -deb-pkg: linux.tar.gz - $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian - origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\ - cp linux.tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz +deb-pkg: debian-orig +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) \ --build=source,binary --source-option=-sP -nc -us -uc PHONY += bindeb-pkg -bindeb-pkg: - $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian +bindeb-pkg: debian +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -b -nc -uc PHONY += intdeb-pkg diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index 68b13ef590ba..534e9713761a 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -95,7 +95,7 @@ else revision=$($srctree/init/build-version) packageversion=$version-$revision fi -sourcename=$KDEB_SOURCENAME +sourcename=${KDEB_SOURCENAME:-linux-upstream} if [ "$ARCH" = "um" ] ; then packagename=user-mode-linux -- cgit v1.2.3 From b44aa8c96e9eea54a42c074c278a8e05e1c231d5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:28 +0900 Subject: kbuild: deb-pkg: make .orig tarball a hard link if possible If '..' belongs to the same filesystem, create a hard link instead of a copy. In most cases, you can save disk space. I do not want to use 'mv' because keeping linux.tar.gz is useful to avoid unneeded rebuilding of the tarball. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index c207f4ec47ac..80a96eb61426 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -96,7 +96,11 @@ debian-orig: private source = $(shell dpkg-parsechangelog -S Source) debian-orig: private version = $(shell dpkg-parsechangelog -S Version | sed 's/-[^-]*$$//') debian-orig: private orig-name = $(source)_$(version).orig.tar.gz debian-orig: linux.tar.gz debian - $(Q)cp $< ../$(orig-name) + $(Q)if [ "$(df --output=target .. 2>/dev/null)" = "$(df --output=target $< 2>/dev/null)" ]; then \ + ln -f $< ../$(orig-name); \ + else \ + cp $< ../$(orig-name); \ + fi PHONY += deb-pkg deb-pkg: debian-orig -- cgit v1.2.3 From e7853995594baa1a1258b20ff077b60a958becdc Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:29 +0900 Subject: kbuild: deb-pkg: switch over to source format 3.0 (quilt) Change the source format from "1.0" to "3.0 (quilt)" because it works more cleanly. All files except .config and debian/ go into the orig tarball. Add a single patch, debian/patches/config, and delete the ugly extend-diff-ignore patterns. The debian tarball will be compressed into *.debian.tar.xz by default. If you like to use a different compression mode, you can pass the command line option, DPKG_FLAGS=-Zgzip, for example. The orig tarball only supports gzip for now. The combination of gzip and xz is somewhat clumsy, but it is not a practical problem. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 2 +- scripts/package/mkdebian | 41 ++++++++++++++++++----------------------- 2 files changed, 19 insertions(+), 24 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 80a96eb61426..5538deacb136 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -105,7 +105,7 @@ debian-orig: linux.tar.gz debian PHONY += deb-pkg deb-pkg: debian-orig +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) \ - --build=source,binary --source-option=-sP -nc -us -uc + --build=source,binary -nc -us -uc PHONY += bindeb-pkg bindeb-pkg: debian diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index 534e9713761a..22fc73a5e4f2 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -135,29 +135,24 @@ else fi mkdir -p debian/source/ -echo "1.0" > debian/source/format - -# Ugly: ignore anything except .config or debian/ -cat<<'EOF' > debian/source/local-options -diff-ignore - -extend-diff-ignore = ^[^.d] - -extend-diff-ignore = ^\.[^c] -extend-diff-ignore = ^\.c($|[^o]) -extend-diff-ignore = ^\.co($|[^n]) -extend-diff-ignore = ^\.con($|[^f]) -extend-diff-ignore = ^\.conf($|[^i]) -extend-diff-ignore = ^\.confi($|[^g]) -extend-diff-ignore = ^\.config. - -extend-diff-ignore = ^d($|[^e]) -extend-diff-ignore = ^de($|[^b]) -extend-diff-ignore = ^deb($|[^i]) -extend-diff-ignore = ^debi($|[^a]) -extend-diff-ignore = ^debia($|[^n]) -extend-diff-ignore = ^debian[^/] -EOF +echo "3.0 (quilt)" > debian/source/format + +{ + echo "diff-ignore" + echo "extend-diff-ignore = .*" +} > debian/source/local-options + +# Add .config as a patch +mkdir -p debian/patches +{ + echo "Subject: Add .config" + echo "Author: ${maintainer}" + echo + echo "--- /dev/null" + echo "+++ linux/.config" + diff -u /dev/null "${KCONFIG_CONFIG}" | tail -n +3 +} > debian/patches/config +echo config > debian/patches/series echo $debarch > debian/arch extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)" -- cgit v1.2.3 From e0ca16749ac3392a5a2ea4e67327fa71ef4aa680 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:30 +0900 Subject: kbuild: make perf-tar*-src-pkg work without relying on git Currently, perf-tar*-src-pkg only uses 'git archive', but it is better to make it work without relying on git. The file, HEAD, which saves the commit hash, will be included in the tarball only when the source tree is managed by git. The git tree is more precisely checked; it has been copied from scripts/setlocalversion. Signed-off-by: Masahiro Yamada --- scripts/Makefile.package | 97 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 33 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 5538deacb136..50b86b325d7a 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -50,10 +50,25 @@ quiet_cmd_tar = TAR $@ tar-rootdir := $(srctree) +%.tar: + $(call cmd,tar) + %.tar.gz: private tar-compress-opt := -I $(KGZIP) %.tar.gz: $(call cmd,tar) +%.tar.bz2: private tar-compress-opt := -I $(KBZIP2) +%.tar.bz2: + $(call cmd,tar) + +%.tar.xz: private tar-compress-opt := -I $(XZ) +%.tar.xz: + $(call cmd,tar) + +%.tar.zst: private tar-compress-opt := -I $(ZSTD) +%.tar.zst: + $(call cmd,tar) + # Linux source tarball # --------------------------------------------------------------------------- @@ -138,36 +153,52 @@ $(tar-pkgs): $(MAKE) -f $(srctree)/Makefile +$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ -# perf-pkg - generate a source tarball with perf source +# perf-tar*-src-pkg - generate a source tarball with perf source # --------------------------------------------------------------------------- -perf-tar=perf-$(KERNELVERSION) - -quiet_cmd_perf_tar = TAR - cmd_perf_tar = \ -git --git-dir=$(srctree)/.git archive --prefix=$(perf-tar)/ \ - HEAD^{tree} $$(cd $(srctree); \ - echo $$(cat tools/perf/MANIFEST)) \ - -o $(perf-tar).tar; \ -mkdir -p $(perf-tar); \ -git --git-dir=$(srctree)/.git rev-parse HEAD > $(perf-tar)/HEAD; \ -(cd $(srctree)/tools/perf; \ -util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \ -tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \ -rm -r $(perf-tar); \ -$(if $(findstring tar-src,$@),, \ -$(if $(findstring bz2,$@),$(KBZIP2), \ -$(if $(findstring gz,$@),$(KGZIP), \ -$(if $(findstring xz,$@),$(XZ), \ -$(if $(findstring zst,$@),$(ZSTD), \ -$(error unknown target $@))))) \ - -f -9 $(perf-tar).tar) - -perf-tar-pkgs := perf-tar-src-pkg perf-targz-src-pkg perf-tarbz2-src-pkg \ - perf-tarxz-src-pkg perf-tarzst-src-pkg -PHONY += $(perf-tar-pkgs) -$(perf-tar-pkgs): - $(call cmd,perf_tar) +perf-tar-src-pkg-tarball = perf-$(KERNELVERSION).$(1) +perf-tar-src-pkg-phony = perf-$(subst .,,$(1))-src-pkg + +quiet_cmd_stage_perf_src = STAGE $@ + cmd_stage_perf_src = \ + rm -rf $@; \ + mkdir -p $@; \ + tar -c -f - --exclude-from=$<_exclude -C $(srctree) --files-from=$(srctree)/tools/perf/MANIFEST | \ + tar -x -f - -C $@ + +.tmp_perf: .tmp_filelist + $(call cmd,stage_perf_src) + +filechk_perf_head = \ + if test -z "$(git -C $(srctree) rev-parse --show-cdup 2>/dev/null)" && \ + head=$$(git -C $(srctree) rev-parse --verify HEAD 2>/dev/null); then \ + echo $$head; \ + else \ + echo "not a git tree"; \ + fi + +.tmp_perf/HEAD: .tmp_perf FORCE + $(call filechk,perf_head) + +quiet_cmd_perf_version_file = GEN $@ + cmd_perf_version_file = cd $(srctree)/tools/perf; util/PERF-VERSION-GEN $(dir $(abspath $@)) + +# PERF-VERSION-FILE and HEAD are independent, but this avoids updating the +# timestamp of PERF-VERSION-FILE. +# The best is to fix tools/perf/util/PERF-VERSION-GEN. +.tmp_perf/PERF-VERSION-FILE: .tmp_perf/HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN + $(call cmd,perf_version_file) + +define perf-tar-src-pkg-rule +PHONY += $(perf-tar-src-pkg-phony) +$(perf-tar-src-pkg-phony): $(perf-tar-src-pkg-tarball) + @: + +$(perf-tar-src-pkg-tarball): private tar-rootdir := .tmp_perf +$(perf-tar-src-pkg-tarball): .tmp_filelist .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE +endef + +$(foreach x, tar tar.gz tar.bz2 tar.xz tar.zst, $(eval $(call perf-tar-src-pkg-rule,$(x)))) # Help text displayed when executing 'make help' # --------------------------------------------------------------------------- @@ -186,11 +217,11 @@ help: @echo ' tarbz2-pkg - Build the kernel as a bzip2 compressed tarball' @echo ' tarxz-pkg - Build the kernel as a xz compressed tarball' @echo ' tarzst-pkg - Build the kernel as a zstd compressed tarball' - @echo ' perf-tar-src-pkg - Build $(perf-tar).tar source tarball' - @echo ' perf-targz-src-pkg - Build $(perf-tar).tar.gz source tarball' - @echo ' perf-tarbz2-src-pkg - Build $(perf-tar).tar.bz2 source tarball' - @echo ' perf-tarxz-src-pkg - Build $(perf-tar).tar.xz source tarball' - @echo ' perf-tarzst-src-pkg - Build $(perf-tar).tar.zst source tarball' + @echo ' perf-tar-src-pkg - Build the perf source tarball with no compression' + @echo ' perf-targz-src-pkg - Build the perf source tarball with gzip compression' + @echo ' perf-tarbz2-src-pkg - Build the perf source tarball with bz2 compression' + @echo ' perf-tarxz-src-pkg - Build the perf source tarball with xz compression' + @echo ' perf-tarzst-src-pkg - Build the perf source tarball with zst compression' PHONY += FORCE FORCE: -- cgit v1.2.3 From 1fc9095846cc0b7a4d027da2d0bd7569914bf129 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:31 +0900 Subject: kbuild: tar-pkg: use tar rules in scripts/Makefile.package Use %.tar, %.tar.gz, %.tar.bz2, %.tar.xz, %.tar.zst rules in scripts/Makefile.package. Signed-off-by: Masahiro Yamada --- scripts/Makefile.package | 29 +++++++++++++++++++++------ scripts/package/buildtar | 52 +----------------------------------------------- 2 files changed, 24 insertions(+), 57 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 50b86b325d7a..b941e6341b36 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -145,13 +145,30 @@ snap-pkg: cd $(objtree)/snap && \ snapcraft --target-arch=$(UTS_MACHINE) -# tarball targets +# dir-pkg tar*-pkg - tarball targets # --------------------------------------------------------------------------- -tar-pkgs := dir-pkg tar-pkg targz-pkg tarbz2-pkg tarxz-pkg tarzst-pkg -PHONY += $(tar-pkgs) -$(tar-pkgs): - $(MAKE) -f $(srctree)/Makefile - +$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ + +tar-pkg-tarball = linux-$(KERNELRELEASE)-$(ARCH).$(1) +tar-pkg-phony = $(subst .,,$(1))-pkg + +tar-install: FORCE + $(Q)$(MAKE) -f $(srctree)/Makefile + +$(Q)$(srctree)/scripts/package/buildtar $@ + +PHONY += dir-pkg +dir-pkg: tar-install + @echo "Kernel tree successfully created in $<" + +define tar-pkg-rule +PHONY += $(tar-pkg-phony) +$(tar-pkg-phony): $(tar-pkg-tarball) + @: + +$(tar-pkg-tarball): private tar-rootdir := tar-install +$(tar-pkg-tarball): tar-install +endef + +$(foreach x, tar tar.gz tar.bz2 tar.xz tar.zst, $(eval $(call tar-pkg-rule,$(x)))) # perf-tar*-src-pkg - generate a source tarball with perf source # --------------------------------------------------------------------------- diff --git a/scripts/package/buildtar b/scripts/package/buildtar index 4d6f0b128efd..65b4ea502962 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -16,39 +16,7 @@ set -e # # Some variables and settings used throughout the script # -tmpdir="${objtree}/tar-install" -tarball="${objtree}/linux-${KERNELRELEASE}-${ARCH}.tar" - - -# -# Figure out how to compress, if requested at all -# -case "${1}" in - dir-pkg|tar-pkg) - opts= - ;; - targz-pkg) - opts="-I ${KGZIP}" - tarball=${tarball}.gz - ;; - tarbz2-pkg) - opts="-I ${KBZIP2}" - tarball=${tarball}.bz2 - ;; - tarxz-pkg) - opts="-I ${XZ}" - tarball=${tarball}.xz - ;; - tarzst-pkg) - opts="-I ${ZSTD}" - tarball=${tarball}.zst - ;; - *) - echo "Unknown tarball target \"${1}\" requested, please add it to ${0}." >&2 - exit 1 - ;; -esac - +tmpdir=$1 # # Clean-up and re-create the temporary directory @@ -148,21 +116,3 @@ case "${ARCH}" in sleep 5 ;; esac - -if [ "${1}" = dir-pkg ]; then - echo "Kernel tree successfully created in $tmpdir" - exit 0 -fi - -# -# Create the tarball -# -if tar --owner=root --group=root --help >/dev/null 2>&1; then - opts="$opts --owner=root --group=root" -fi - -tar cf $tarball -C $tmpdir $opts $dirs - -echo "Tarball successfully created in $tarball" - -exit 0 -- cgit v1.2.3 From c5bf2efb058d841bf8993b9a9b4522abda804d39 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:32 +0900 Subject: kbuild: deb-pkg: fix binary-arch and clean in debian/rules The clean target needs ARCH=${ARCH} to clean up the tree for the correct architecture. 'make (bin)deb-pkg' skips cleaning, but the preclean hook may be executed if dpkg-buildpackage is directly used. The binary-arch target does not need KERNELRELEASE because it is not updated during the installation. KBUILD_BUILD_VERSION is not needed either because binary-arch does not build vmlinux. Signed-off-by: Masahiro Yamada --- scripts/package/mkdebian | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index 22fc73a5e4f2..eef20d5dc288 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -251,12 +251,10 @@ build: build-arch binary-indep: binary-arch: build-arch - \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ - KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile intdeb-pkg - + \$(MAKE) -f \$(srctree)/Makefile ARCH=${ARCH} intdeb-pkg clean: rm -rf debian/files debian/linux-* - \$(MAKE) clean + \$(MAKE) -f \$(srctree)/Makefile ARCH=${ARCH} clean binary: binary-arch EOF -- cgit v1.2.3 From 3ab18a625ce42163da91ee4096460218d11bed36 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:33 +0900 Subject: kbuild: deb-pkg: improve the usability of source package Improve the source package support in case the dpkg-buildpackage is directly used to build binary packages. For cross-compiling, you can set CROSS_COMPILE via the environment variable, but it is better to set it automatically - set it to ${DEB_HOST_GNU_TYPE}- if we are cross-compiling but not from the top Makefile. The generated source package may be carried to a different build environment, which may have a different compiler installed. Run olddefconfig first to set new CONFIG options to their default values without prompting. Take KERNELRELEASE and KBUILD_BUILD_VERSION from the version field of debian/changelog in case it is updated afterwards. Signed-off-by: Masahiro Yamada --- scripts/package/deb-build-option | 16 ++++++++++++++++ scripts/package/mkdebian | 9 ++++----- 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100755 scripts/package/deb-build-option (limited to 'scripts') diff --git a/scripts/package/deb-build-option b/scripts/package/deb-build-option new file mode 100755 index 000000000000..b079b0d121d4 --- /dev/null +++ b/scripts/package/deb-build-option @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +# Set up CROSS_COMPILE if we are cross-compiling, but not called from the +# kernel toplevel Makefile +if [ -z "${CROSS_COMPILE}${cross_compiling}" -a "${DEB_HOST_ARCH}" != "${DEB_BUILD_ARCH}" ]; then + echo CROSS_COMPILE=${DEB_HOST_GNU_TYPE}- +fi + +version=$(dpkg-parsechangelog -S Version) +version_upstream="${version%-*}" +debian_revision="${version#${version_upstream}}" +debian_revision="${debian_revision#*-}" + +echo KERNELRELEASE=${version_upstream} +echo KBUILD_BUILD_VERSION=${debian_revision} diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index eef20d5dc288..f74380036bb5 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -90,10 +90,8 @@ rm -rf debian version=$KERNELRELEASE if [ -n "$KDEB_PKGVERSION" ]; then packageversion=$KDEB_PKGVERSION - revision=${packageversion##*-} else - revision=$($srctree/init/build-version) - packageversion=$version-$revision + packageversion=$version-$($srctree/init/build-version) fi sourcename=${KDEB_SOURCENAME:-linux-upstream} @@ -244,8 +242,9 @@ srctree ?= . build-indep: build-arch: - \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ - KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile + \$(MAKE) -f \$(srctree)/Makefile ARCH=${ARCH} \ + \$(shell \$(srctree)/scripts/package/deb-build-option) \ + olddefconfig all build: build-arch -- cgit v1.2.3 From 7adf14d8aca1ea53bf9ccf8463809c82adb8c23a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 22 Feb 2023 23:47:29 +0900 Subject: kbuild: rpm-pkg: remove unneeded KERNELRELEASE from modules/headers_install This is a temporary workaround added by commit f6e09b07cc12 ("kbuild: do not put .scmversion into the source tarball"). Since commit 1cb86b6c3136 ("kbuild: save overridden KERNELRELEASE in include/config/kernel.release"), the user-supplied KERNELRELEASE is saved in include/config/kernel.release. Remove it again. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor --- scripts/package/mkspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/package/mkspec b/scripts/package/mkspec index 83a64d9d7372..3c550960dd39 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -99,8 +99,8 @@ $S %else cp \$($MAKE -s image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE %endif -$M $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE INSTALL_MOD_PATH=%{buildroot} modules_install - $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE INSTALL_HDR_PATH=%{buildroot}/usr headers_install +$M $MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install + $MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE cp .config %{buildroot}/boot/config-$KERNELRELEASE $S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/build -- cgit v1.2.3