diff options
| -rw-r--r-- | .azure-pipelines.yml | 59 | ||||
| -rw-r--r-- | .gitlab-ci.yml | 90 | ||||
| -rw-r--r-- | .travis.yml | 88 | ||||
| -rw-r--r-- | common/main.c | 5 | ||||
| -rw-r--r-- | doc/arch/sandbox.rst | 10 | ||||
| -rw-r--r-- | test/py/README.md | 13 | ||||
| -rw-r--r-- | test/py/conftest.py | 30 | ||||
| -rw-r--r-- | tools/buildman/README | 35 | ||||
| -rw-r--r-- | tools/buildman/builder.py | 51 | ||||
| -rw-r--r-- | tools/buildman/builderthread.py | 34 | ||||
| -rw-r--r-- | tools/buildman/cmdline.py | 8 | ||||
| -rw-r--r-- | tools/buildman/control.py | 52 | ||||
| -rw-r--r-- | tools/buildman/func_test.py | 46 | ||||
| -rw-r--r-- | tools/buildman/test.py | 20 |
14 files changed, 280 insertions, 261 deletions
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 65e07bf20a..d3e7b4dd02 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -52,7 +52,7 @@ jobs: image: $(ci_runner_image) options: $(container_option) steps: - - script: cppcheck --force --quiet --inline-suppr . + - script: cppcheck -j$(nproc) --force --quiet --inline-suppr . - job: htmldocs displayName: 'Build HTML documentation' @@ -136,10 +136,10 @@ jobs: virtualenv -p /usr/bin/python3 /tmp/venv . /tmp/venv/bin/activate pip install pyelftools pytest - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/sandbox_spl + export UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox_spl export PYTHONPATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt export PATH=${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH} - ./tools/buildman/buildman -o /tmp -P sandbox_spl + ./tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w sandbox_spl ./tools/binman/binman --toolpath ${UBOOT_TRAVIS_BUILD_DIR}/tools test ./tools/buildman/buildman -t ./tools/dtoc/dtoc -t @@ -159,102 +159,78 @@ jobs: matrix: sandbox: TEST_PY_BD: "sandbox" - BUILDMAN: "^sandbox$" sandbox_clang: TEST_PY_BD: "sandbox" - BUILDMAN: "^sandbox$" OVERRIDE: "-O clang-7" sandbox_spl: TEST_PY_BD: "sandbox_spl" - TEST_PY_TEST_SPEC: "test_ofplatdata" - BUILDMAN: "^sandbox_spl$" + TEST_PY_TEST_SPEC: "test_ofplatdata or test_handoff" sandbox_flattree: TEST_PY_BD: "sandbox_flattree" - BUILDMAN: "^sandbox_flattree$" evb_ast2500: TEST_PY_BD: "evb-ast2500" TEST_PY_ID: "--id qemu" - BUILDMAN: "^evb-ast2500$" vexpress_ca15_tc2: TEST_PY_BD: "vexpress_ca15_tc2" TEST_PY_ID: "--id qemu" - BUILDMAN: "^vexpress_ca15_tc2$" vexpress_ca9x4: TEST_PY_BD: "vexpress_ca9x4" TEST_PY_ID: "--id qemu" - BUILDMAN: "^vexpress_ca9x4$" integratorcp_cm926ejs: TEST_PY_BD: "integratorcp_cm926ejs" TEST_PY_ID: "--id qemu" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^integratorcp_cm926ejs$" qemu_arm: TEST_PY_BD: "qemu_arm" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_arm$" qemu_arm64: TEST_PY_BD: "qemu_arm64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_arm64$" qemu_mips: TEST_PY_BD: "qemu_mips" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips$" qemu_mipsel: TEST_PY_BD: "qemu_mipsel" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mipsel$" qemu_mips64: TEST_PY_BD: "qemu_mips64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips64$" qemu_mips64el: TEST_PY_BD: "qemu_mips64el" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips64el$" qemu_ppce500: TEST_PY_BD: "qemu-ppce500" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-ppce500$" qemu_riscv32: TEST_PY_BD: "qemu-riscv32" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv32$" qemu_riscv64: TEST_PY_BD: "qemu-riscv64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv64$" qemu_riscv32_spl: TEST_PY_BD: "qemu-riscv32_spl" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv32_spl$" qemu_riscv64_spl: TEST_PY_BD: "qemu-riscv64_spl" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv64_spl$" qemu_x86: TEST_PY_BD: "qemu-x86" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-x86$" qemu_x86_64: TEST_PY_BD: "qemu-x86_64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-x86_64$" xilinx_zynq_virt: TEST_PY_BD: "xilinx_zynq_virt" TEST_PY_ID: "--id qemu" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^xilinx_zynq_virt$" xilinx_versal_virt: TEST_PY_BD: "xilinx_versal_virt" TEST_PY_ID: "--id qemu" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^xilinx_versal_virt$" xtfpga: TEST_PY_BD: "xtfpga" TEST_PY_ID: "--id qemu" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^xtfpga$" steps: - script: | cat << EOF > test.sh @@ -264,7 +240,6 @@ jobs: export TEST_PY_BD="${TEST_PY_BD}" export TEST_PY_ID="${TEST_PY_ID}" export TEST_PY_TEST_SPEC="${TEST_PY_TEST_SPEC}" - export BUILDMAN="${BUILDMAN}" export OVERRIDE="${OVERRIDE}" EOF cat << "EOF" >> test.sh @@ -289,27 +264,15 @@ jobs: fi # the below corresponds to .gitlab-ci.yml "script" cd ${WORK_DIR} - if [[ "${BUILDMAN}" != "" ]]; then - ret=0; - tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE} || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN}; - exit $ret; - fi; - fi + export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD}; + tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e --board ${TEST_PY_BD} ${OVERRIDE} virtualenv -p /usr/bin/python3 /tmp/venv . /tmp/venv/bin/activate pip install -r test/py/requirements.txt - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/${TEST_PY_BD}; export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH}; export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci; - if [[ "${TEST_PY_BD}" != "" ]]; then - ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}" --build-dir "$UBOOT_TRAVIS_BUILD_DIR"; - ret=$?; - if [[ $ret -ne 0 ]]; then - exit $ret; - fi; - fi + # "${var:+"-k $var"}" expands to "" if $var is empty, "-k $var" if not + ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} ${TEST_PY_TEST_SPEC:+"-k ${TEST_PY_TEST_SPEC}"} --build-dir "$UBOOT_TRAVIS_BUILD_DIR"; # the below corresponds to .gitlab-ci.yml "after_script" rm -rf /tmp/uboot-test-hooks /tmp/venv EOF @@ -443,9 +406,9 @@ jobs: cat << "EOF" >> build.sh if [[ "${BUILDMAN}" != "" ]]; then ret=0; - tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE} || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN}; + tools/buildman/buildman -o /tmp -P -W ${BUILDMAN} ${OVERRIDE} || ret=$?; + if [[ $ret -ne 0 ]]; then + tools/buildman/buildman -o /tmp -seP ${BUILDMAN}; exit $ret; fi; fi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf39435631..08bdf81e74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,44 +36,28 @@ stages: after_script: - rm -rf /tmp/uboot-test-hooks /tmp/venv script: - # From buildman, exit code 129 means warnings only. If we've been asked to - # use clang only do one configuration. - - if [[ "${BUILDMAN}" != "" ]]; then - ret=0; - tools/buildman/buildman -o /tmp -P -E ${BUILDMAN} ${OVERRIDE}|| ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - tools/buildman/buildman -o /tmp -sdeP ${BUILDMAN}; - exit $ret; - fi; - fi - # "not a_test_which_does_not_exist" is a dummy -k parameter which will - # never prevent any test from running. That way, we can always pass - # "-k something" even when $TEST_PY_TEST_SPEC doesnt need a custom - # value. + # If we've been asked to use clang only do one configuration. + - export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD} + - tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e + --board ${TEST_PY_BD} ${OVERRIDE} - virtualenv -p /usr/bin/python3 /tmp/venv - . /tmp/venv/bin/activate - pip install -r test/py/requirements.txt - - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/${TEST_PY_BD}; - export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH}; + # "${var:+"-k $var"}" expands to "" if $var is empty, "-k $var" if not + - export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH}; export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci; - if [[ "${TEST_PY_BD}" != "" ]]; then - ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} - -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}" - --build-dir "$UBOOT_TRAVIS_BUILD_DIR"; - ret=$?; - if [[ $ret -ne 0 ]]; then - exit $ret; - fi; - fi; + ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} + ${TEST_PY_TEST_SPEC:+"-k ${TEST_PY_TEST_SPEC}"} + --build-dir "$UBOOT_TRAVIS_BUILD_DIR" build all 32bit ARM platforms: tags: [ 'all' ] stage: world build script: - ret=0; - ./tools/buildman/buildman -o /tmp -P -E arm -x aarch64 || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - ./tools/buildman/buildman -o /tmp -sdeP; + ./tools/buildman/buildman -o /tmp -P -E -W arm -x aarch64 || ret=$?; + if [[ $ret -ne 0 ]]; then + ./tools/buildman/buildman -o /tmp -seP; exit $ret; fi; @@ -85,9 +69,9 @@ build all 64bit ARM platforms: - . /tmp/venv/bin/activate - pip install pyelftools - ret=0; - ./tools/buildman/buildman -o /tmp -P -E aarch64 || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - ./tools/buildman/buildman -o /tmp -sdeP; + ./tools/buildman/buildman -o /tmp -P -E -W aarch64 || ret=$?; + if [[ $ret -ne 0 ]]; then + ./tools/buildman/buildman -o /tmp -seP; exit $ret; fi; @@ -96,9 +80,9 @@ build all PowerPC platforms: stage: world build script: - ret=0; - ./tools/buildman/buildman -o /tmp -P -E powerpc || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - ./tools/buildman/buildman -o /tmp -sdeP; + ./tools/buildman/buildman -o /tmp -P -E -W powerpc || ret=$?; + if [[ $ret -ne 0 ]]; then + ./tools/buildman/buildman -o /tmp -seP; exit $ret; fi; @@ -107,9 +91,9 @@ build all other platforms: stage: world build script: - ret=0; - ./tools/buildman/buildman -o /tmp -P -E -x arm,powerpc || ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - ./tools/buildman/buildman -o /tmp -sdeP; + ./tools/buildman/buildman -o /tmp -P -E -W -x arm,powerpc || ret=$?; + if [[ $ret -ne 0 ]]; then + ./tools/buildman/buildman -o /tmp -seP; exit $ret; fi; @@ -119,7 +103,7 @@ cppcheck: tags: [ 'all' ] stage: testsuites script: - - cppcheck --force --quiet --inline-suppr . + - cppcheck -j$(nproc) --force --quiet --inline-suppr . # search for TODO within source tree grep TODO/FIXME/HACK: @@ -176,10 +160,10 @@ Run binman, buildman, dtoc, Kconfig and patman testsuites: virtualenv -p /usr/bin/python3 /tmp/venv; . /tmp/venv/bin/activate; pip install pyelftools pytest; - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/sandbox_spl; + export UBOOT_TRAVIS_BUILD_DIR=/tmp/sandbox_spl; export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt"; export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}"; - ./tools/buildman/buildman -o /tmp -P sandbox_spl; + ./tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w sandbox_spl; ./tools/binman/binman --toolpath ${UBOOT_TRAVIS_BUILD_DIR}/tools test; ./tools/buildman/buildman -t; ./tools/dtoc/dtoc -t; @@ -191,14 +175,12 @@ sandbox test.py: tags: [ 'all' ] variables: TEST_PY_BD: "sandbox" - BUILDMAN: "^sandbox$" <<: *buildman_and_testpy_dfn sandbox with clang test.py: tags: [ 'all' ] variables: TEST_PY_BD: "sandbox" - BUILDMAN: "^sandbox$" OVERRIDE: "-O clang-7" <<: *buildman_and_testpy_dfn @@ -206,8 +188,7 @@ sandbox_spl test.py: tags: [ 'all' ] variables: TEST_PY_BD: "sandbox_spl" - BUILDMAN: "^sandbox_spl$" - TEST_PY_TEST_SPEC: "test_ofplatdata" + TEST_PY_TEST_SPEC: "test_ofplatdata or test_handoff" <<: *buildman_and_testpy_dfn evb-ast2500 test.py: @@ -215,14 +196,12 @@ evb-ast2500 test.py: variables: TEST_PY_BD: "evb-ast2500" TEST_PY_ID: "--id qemu" - BUILDMAN: "^evb-ast2500$" <<: *buildman_and_testpy_dfn sandbox_flattree test.py: tags: [ 'all' ] variables: TEST_PY_BD: "sandbox_flattree" - BUILDMAN: "^sandbox_flattree$" <<: *buildman_and_testpy_dfn vexpress_ca15_tc2 test.py: @@ -230,7 +209,6 @@ vexpress_ca15_tc2 test.py: variables: TEST_PY_BD: "vexpress_ca15_tc2" TEST_PY_ID: "--id qemu" - BUILDMAN: "^vexpress_ca15_tc2$" <<: *buildman_and_testpy_dfn vexpress_ca9x4 test.py: @@ -238,7 +216,6 @@ vexpress_ca9x4 test.py: variables: TEST_PY_BD: "vexpress_ca9x4" TEST_PY_ID: "--id qemu" - BUILDMAN: "^vexpress_ca9x4$" <<: *buildman_and_testpy_dfn integratorcp_cm926ejs test.py: @@ -247,7 +224,6 @@ integratorcp_cm926ejs test.py: TEST_PY_BD: "integratorcp_cm926ejs" TEST_PY_TEST_SPEC: "not sleep" TEST_PY_ID: "--id qemu" - BUILDMAN: "^integratorcp_cm926ejs$" <<: *buildman_and_testpy_dfn qemu_arm test.py: @@ -255,7 +231,6 @@ qemu_arm test.py: variables: TEST_PY_BD: "qemu_arm" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_arm$" <<: *buildman_and_testpy_dfn qemu_arm64 test.py: @@ -263,7 +238,6 @@ qemu_arm64 test.py: variables: TEST_PY_BD: "qemu_arm64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_arm64$" <<: *buildman_and_testpy_dfn qemu_mips test.py: @@ -271,7 +245,6 @@ qemu_mips test.py: variables: TEST_PY_BD: "qemu_mips" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips$" <<: *buildman_and_testpy_dfn qemu_mipsel test.py: @@ -279,7 +252,6 @@ qemu_mipsel test.py: variables: TEST_PY_BD: "qemu_mipsel" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mipsel$" <<: *buildman_and_testpy_dfn qemu_mips64 test.py: @@ -287,7 +259,6 @@ qemu_mips64 test.py: variables: TEST_PY_BD: "qemu_mips64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips64$" <<: *buildman_and_testpy_dfn qemu_mips64el test.py: @@ -295,7 +266,6 @@ qemu_mips64el test.py: variables: TEST_PY_BD: "qemu_mips64el" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu_mips64el$" <<: *buildman_and_testpy_dfn qemu-ppce500 test.py: @@ -303,7 +273,6 @@ qemu-ppce500 test.py: variables: TEST_PY_BD: "qemu-ppce500" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-ppce500$" <<: *buildman_and_testpy_dfn qemu-riscv32 test.py: @@ -311,7 +280,6 @@ qemu-riscv32 test.py: variables: TEST_PY_BD: "qemu-riscv32" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv32$" <<: *buildman_and_testpy_dfn qemu-riscv64 test.py: @@ -319,7 +287,6 @@ qemu-riscv64 test.py: variables: TEST_PY_BD: "qemu-riscv64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv64$" <<: *buildman_and_testpy_dfn qemu-riscv32_spl test.py: @@ -327,7 +294,6 @@ qemu-riscv32_spl test.py: variables: TEST_PY_BD: "qemu-riscv32_spl" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv32_spl$" <<: *buildman_and_testpy_dfn qemu-riscv64_spl test.py: @@ -335,7 +301,6 @@ qemu-riscv64_spl test.py: variables: TEST_PY_BD: "qemu-riscv64_spl" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-riscv64_spl$" <<: *buildman_and_testpy_dfn qemu-x86 test.py: @@ -343,7 +308,6 @@ qemu-x86 test.py: variables: TEST_PY_BD: "qemu-x86" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-x86$" <<: *buildman_and_testpy_dfn qemu-x86_64 test.py: @@ -351,7 +315,6 @@ qemu-x86_64 test.py: variables: TEST_PY_BD: "qemu-x86_64" TEST_PY_TEST_SPEC: "not sleep" - BUILDMAN: "^qemu-x86_64$" <<: *buildman_and_testpy_dfn xilinx_zynq_virt test.py: @@ -360,7 +323,6 @@ xilinx_zynq_virt test.py: TEST_PY_BD: "xilinx_zynq_virt" TEST_PY_TEST_SPEC: "not sleep" TEST_PY_ID: "--id qemu" - BUILDMAN: "^xilinx_zynq_virt$" <<: *buildman_and_testpy_dfn xilinx_versal_virt test.py: @@ -369,7 +331,6 @@ xilinx_versal_virt test.py: TEST_PY_BD: "xilinx_versal_virt" TEST_PY_TEST_SPEC: "not sleep" TEST_PY_ID: "--id qemu" - BUILDMAN: "^xilinx_versal_virt$" <<: *buildman_and_testpy_dfn xtfpga test.py: @@ -378,5 +339,4 @@ xtfpga test.py: TEST_PY_BD: "xtfpga" TEST_PY_TEST_SPEC: "not sleep" TEST_PY_ID: "--id qemu" - BUILDMAN: "^xtfpga$" <<: *buildman_and_testpy_dfn diff --git a/.travis.yml b/.travis.yml index 5309a0bf4e..b3253da13c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -204,46 +204,42 @@ script: # Comments must be outside the command strings below, or the Travis parser # will get confused. # - # From buildman, exit code 129 means warnings only. If we've been asked to - # use clang only do one configuration. + # If we've been asked to use clang only do one configuration. + # + # Build a selection of boards if TEST_PY_BD is empty - if [[ "${BUILDMAN}" != "" ]]; then - ret=0; - tools/buildman/buildman -P -E ${BUILDMAN} ${OVERRIDE}|| ret=$?; - if [[ $ret -ne 0 && $ret -ne 129 ]]; then - tools/buildman/buildman -sdeP ${BUILDMAN}; + tools/buildman/buildman -P -E -W ${BUILDMAN} ${OVERRIDE}; + if [[ $ret -ne 0 ]]; then + tools/buildman/buildman -seP ${BUILDMAN}; exit $ret; fi; fi - # "not a_test_which_does_not_exist" is a dummy -k parameter which will - # never prevent any test from running. That way, we can always pass - # "-k something" even when $TEST_PY_TEST_SPEC doesnt need a custom - # value. - - export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/.bm-work/${TEST_PY_BD}; - cp ~/grub_x86.efi $UBOOT_TRAVIS_BUILD_DIR/; - cp ~/grub_x64.efi $UBOOT_TRAVIS_BUILD_DIR/; - if [[ -e ~/grub_arm.efi ]]; then - cp ~/grub_arm.efi $UBOOT_TRAVIS_BUILD_DIR/; - fi; - if [[ -e ~/grub_arm64.efi ]]; then - cp ~/grub_arm64.efi $UBOOT_TRAVIS_BUILD_DIR/; - fi; - if [[ -e ~/grub_riscv32.efi ]]; then - cp ~/grub_riscv32.efi $UBOOT_TRAVIS_BUILD_DIR/; - fi; - if [[ -e ~/grub_riscv64.efi ]]; then - cp ~/grub_riscv64.efi $UBOOT_TRAVIS_BUILD_DIR/; - fi; - if [[ "${TEST_PY_BD}" != "" ]]; then + # Build just the one board needed for testing, if TEST_PY_BD is non-empty + # Note: "${var:+"-k $var"}" expands to "" if $var is empty, "-k $var" if not + - if [[ "${TEST_PY_BD}" != "" ]]; then + export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/${TEST_PY_BD}; + cp ~/grub_x86.efi $UBOOT_TRAVIS_BUILD_DIR/; + cp ~/grub_x64.efi $UBOOT_TRAVIS_BUILD_DIR/; + if [[ -e ~/grub_arm.efi ]]; then + cp ~/grub_arm.efi $UBOOT_TRAVIS_BUILD_DIR/; + fi; + if [[ -e ~/grub_arm64.efi ]]; then + cp ~/grub_arm64.efi $UBOOT_TRAVIS_BUILD_DIR/; + fi; + if [[ -e ~/grub_riscv32.efi ]]; then + cp ~/grub_riscv32.efi $UBOOT_TRAVIS_BUILD_DIR/; + fi; + if [[ -e ~/grub_riscv64.efi ]]; then + cp ~/grub_riscv64.efi $UBOOT_TRAVIS_BUILD_DIR/; + fi; + tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e + --board ${TEST_PY_BD} ${OVERRIDE} || exit; virtualenv -p /usr/bin/python3 /tmp/venv; . /tmp/venv/bin/activate; pip install -r test/py/requirements.txt; ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID} - -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}" - --build-dir "$UBOOT_TRAVIS_BUILD_DIR"; - ret=$?; - if [[ $ret -ne 0 ]]; then - exit $ret; - fi; + ${TEST_PY_TEST_SPEC:+"-k ${TEST_PY_TEST_SPEC}"} + --build-dir "$UBOOT_TRAVIS_BUILD_DIR" || exit; if [[ -n "${TEST_PY_TOOLS}" ]]; then export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt"; export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}"; @@ -452,7 +448,7 @@ matrix: # static code analysis with cppcheck (we can add --enable=all later) - name: "cppcheck" script: - - cppcheck --force --quiet --inline-suppr . + - cppcheck -j$(nproc) --force --quiet --inline-suppr . # build HTML documentation - name: "htmldocs" script: @@ -489,131 +485,111 @@ matrix: - name: "test/py sandbox" env: - TEST_PY_BD="sandbox" - BUILDMAN="^sandbox$" TOOLCHAIN="i386" - name: "test/py sandbox with clang" env: - TEST_PY_BD="sandbox" - BUILDMAN="^sandbox$" OVERRIDE="-O clang-7" - name: "test/py sandbox_spl" env: - TEST_PY_BD="sandbox_spl" - TEST_PY_TEST_SPEC="test_ofplatdata" - BUILDMAN="^sandbox$" + TEST_PY_TEST_SPEC="test_ofplatdata or test_handoff" TOOLCHAIN="i386" TEST_PY_TOOLS="yes" - name: "test/py sandbox_flattree" env: - TEST_PY_BD="sandbox_flattree" - BUILDMAN="^sandbox_flattree$" TOOLCHAIN="i386" - name: "test/py evb-ast2500" env: - TEST_PY_BD="evb-ast2500" TEST_PY_ID="--id qemu" QEMU_TARGET="arm-softmmu" - BUILDMAN="^evb-ast2500$" - name: "test/py vexpress_ca15_tc2" env: - TEST_PY_BD="vexpress_ca15_tc2" TEST_PY_ID="--id qemu" QEMU_TARGET="arm-softmmu" - BUILDMAN="^vexpress_ca15_tc2$" - name: "test/py vexpress_ca9x4" env: - TEST_PY_BD="vexpress_ca9x4" TEST_PY_ID="--id qemu" QEMU_TARGET="arm-softmmu" - BUILDMAN="^vexpress_ca9x4$" - name: "test/py integratorcp_cm926ejs" env: - TEST_PY_BD="integratorcp_cm926ejs" TEST_PY_TEST_SPEC="not sleep" TEST_PY_ID="--id qemu" QEMU_TARGET="arm-softmmu" - BUILDMAN="^integratorcp_cm926ejs$" - name: "test/py qemu_arm" env: - TEST_PY_BD="qemu_arm" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="arm-softmmu" - BUILDMAN="^qemu_arm$" - name: "test/py qemu_arm64" env: - TEST_PY_BD="qemu_arm64" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="aarch64-softmmu" - BUILDMAN="^qemu_arm64$" - name: "test/py qemu_mips" env: - TEST_PY_BD="qemu_mips" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="mips-softmmu" - BUILDMAN="^qemu_mips$" TOOLCHAIN="mips" - name: "test/py qemu_mipsel" env: - TEST_PY_BD="qemu_mipsel" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="mipsel-softmmu" - BUILDMAN="^qemu_mipsel$" TOOLCHAIN="mips" - name: "test/py qemu_mips64" env: - TEST_PY_BD="qemu_mips64" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="mips64-softmmu" - BUILDMAN="^qemu_mips64$" TOOLCHAIN="mips" - name: "test/py qemu_mips64el" env: - TEST_PY_BD="qemu_mips64el" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="mips64el-softmmu" - BUILDMAN="^qemu_mips64el$" TOOLCHAIN="mips" - name: "test/py qemu-ppce500" env: - TEST_PY_BD="qemu-ppce500" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="ppc-softmmu" - BUILDMAN="^qemu-ppce500$" TOOLCHAIN="powerpc" - name: "test/py qemu-riscv32" env: - TEST_PY_BD="qemu-riscv32" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="riscv32-softmmu" - BUILDMAN="^qemu-riscv32$" TOOLCHAIN="riscv" - name: "test/py qemu-riscv64" env: - TEST_PY_BD="qemu-riscv64" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="riscv64-softmmu" - BUILDMAN="^qemu-riscv64$" TOOLCHAIN="riscv" - name: "test/py qemu-riscv32_spl" env: - TEST_PY_BD="qemu-riscv32_spl" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="riscv32-softmmu" - BUILDMAN="^qemu-riscv32_spl$" TOOLCHAIN="riscv" - name: "test/py qemu-riscv64_spl" env: - TEST_PY_BD="qemu-riscv64_spl" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="riscv64-softmmu" - BUILDMAN="^qemu-riscv64_spl$" TOOLCHAIN="riscv" - name: "test/py qemu-x86" env: - TEST_PY_BD="qemu-x86" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="i386-softmmu" - BUILDMAN="^qemu-x86$" TOOLCHAIN="i386" BUILD_ROM="yes" - name: "test/py qemu-x86_64" @@ -621,7 +597,6 @@ matrix: - TEST_PY_BD="qemu-x86_64" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="x86_64-softmmu" - BUILDMAN="^qemu-x86_64$" TOOLCHAIN="i386" BUILD_ROM="yes" - name: "test/py xilinx_zynq_virt" @@ -630,21 +605,18 @@ matrix: TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="arm-softmmu" TEST_PY_ID="--id qemu" - BUILDMAN="^xilinx_zynq_virt$" - name: "test/py xilinx_versal_virt" env: - TEST_PY_BD="xilinx_versal_virt" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="aarch64-softmmu" TEST_PY_ID="--id qemu" - BUILDMAN="^xilinx_versal_virt$" - name: "test/py xtfpga" env: - TEST_PY_BD="xtfpga" TEST_PY_TEST_SPEC="not sleep" QEMU_TARGET="xtensa-softmmu" TEST_PY_ID="--id qemu" - BUILDMAN="^xtfpga$" TOOLCHAIN="xtensa-dc233c-elf" # TODO make it perfect ;-r diff --git a/common/main.c b/common/main.c index ec8994ad45..06d7ff56d6 100644 --- a/common/main.c +++ b/common/main.c @@ -15,11 +15,6 @@ #include <init.h> #include <version.h> -/* - * Board-specific Platform code can reimplement show_boot_progress () if needed - */ -__weak void show_boot_progress(int val) {} - static void run_preboot_environment_command(void) { char *p; diff --git a/doc/arch/sandbox.rst b/doc/arch/sandbox.rst index e577a95716..6a1c6fc552 100644 --- a/doc/arch/sandbox.rst +++ b/doc/arch/sandbox.rst @@ -34,6 +34,16 @@ integers can only be built on 64-bit hosts. Note that standalone/API support is not available at present. +Prerequisites +------------- + +Here are some packages that are worth installing if you are doing sandbox or +tools development in U-Boot: + + python3-pytest lzma lzma-alone lz4 python3 python3-virtualenv + libssl1.0-dev + + Basic Operation --------------- diff --git a/test/py/README.md b/test/py/README.md index 3cbe01b73e..2e5025258d 100644 --- a/test/py/README.md +++ b/test/py/README.md @@ -138,6 +138,9 @@ command-line option; see the next section. before running the tests. If using this option, make sure that any environment variables required by the build process are already set, such as `$CROSS_COMPILE`. +- `--buildman` indicates that `--build` should use buildman to build U-Boot. + There is no need to set $CROSS_COMPILE` in this case since buildman handles + it. - `--build-dir` sets the directory containing the compiled U-Boot binaries. If omitted, this is `${source_dir}/build-${board_type}`. - `--result-dir` sets the directory to write results, such as log files, @@ -333,7 +336,7 @@ PATH=$HOME/ubtest/bin:$PATH \ If you want the test script to compile U-Boot for you too, then you likely need to set `$CROSS_COMPILE` to allow this, and invoke the test script as -follow: +follows: ```bash CROSS_COMPILE=arm-none-eabi- \ @@ -342,6 +345,14 @@ CROSS_COMPILE=arm-none-eabi- \ ./test/py/test.py --bd seaboard --build ``` +or, using buildman to handle it: + +```bash + PATH=$HOME/ubtest/bin:$PATH \ + PYTHONPATH=${HOME}/ubtest/py/${HOSTNAME}:${PYTHONPATH} \ + ./test/py/test.py --bd seaboard --build --buildman +``` + ## Writing tests Please refer to the pytest documentation for details of writing pytest tests. diff --git a/test/py/conftest.py b/test/py/conftest.py index 34ac4fb062..e3392ff6bc 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -70,6 +70,8 @@ def pytest_addoption(parser): help='U-Boot board identity/instance') parser.addoption('--build', default=False, action='store_true', help='Compile U-Boot before running tests') + parser.addoption('--buildman', default=False, action='store_true', + help='Use buildman to build U-Boot (assuming --build is given)') parser.addoption('--gdbserver', default=None, help='Run sandbox under gdbserver. The argument is the channel '+ 'over which gdbserver should communicate, e.g. localhost:1234') @@ -140,16 +142,26 @@ def pytest_configure(config): log = multiplexed_log.Logfile(result_dir + '/test-log.html') if config.getoption('build'): - if build_dir != source_dir: - o_opt = 'O=%s' % build_dir + if config.getoption('buildman'): + if build_dir != source_dir: + dest_args = ['-o', build_dir, '-w'] + else: + dest_args = ['-i'] + cmds = (['buildman', '--board', board_type] + dest_args,) + name = 'buildman' else: - o_opt = '' - cmds = ( - ['make', o_opt, '-s', board_type + '_defconfig'], - ['make', o_opt, '-s', '-j8'], - ) - with log.section('make'): - runner = log.get_runner('make', sys.stdout) + if build_dir != source_dir: + o_opt = 'O=%s' % build_dir + else: + o_opt = '' + cmds = ( + ['make', o_opt, '-s', board_type + '_defconfig'], + ['make', o_opt, '-s', '-j8'], + ) + name = 'make' + + with log.section(name): + runner = log.get_runner(name, sys.stdout) for cmd in cmds: runner.run(cmd, cwd=source_dir) runner.close() diff --git a/tools/buildman/README b/tools/buildman/README index c1ac0d0f58..4cf0114157 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -1056,19 +1056,46 @@ toolchain. For example: buildman -O clang-7 --board sandbox +Doing a simple build +==================== + +In some cases you just want to build a single board and get the full output, use +the -w option, for example: + + buildman -o /tmp/build --board sandbox -w + +This will write the full build into /tmp/build including object files. + + Other options ============= -Buildman has various other command line options. Try --help to see them. +Buildman has various other command-line options. Try --help to see them. + +To find out what toolchain prefix buildman will use for a build, use the -A +option. + +To request that compiler warnings be promoted to errors, use -E. This passes the +-Werror flag to the compiler. Note that the build can still produce warnings +with -E, e.g. the migration warnings: -To find out what architecture or toolchain prefix buildman will use for a build, -see the -a and -A options. + ===================== WARNING ====================== + This board does not use CONFIG_DM_MMC. Please update + ... + ==================================================== When doing builds, Buildman's return code will reflect the overall result: 0 (success) No errors or warnings found 128 Errors found - 129 Warnings found + 129 Warnings found (only if no -W) + +You can use -W to tell Buildman to return 0 (success) instead of 129 when +warnings are found. Note that it can be useful to combine -E and -W. This means +that all compiler warnings will produce failures (code 128) and all other +warnings will produce success (since 129 is changed to 0). + +If there are both warnings and errors, errors win, so buildman returns 128. How to change from MAKEALL diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 3fd4fac695..70c55c588a 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -174,6 +174,8 @@ class Builder: in_tree: Build U-Boot in-tree instead of specifying an output directory separate from the source code. This option is really only useful for testing in-tree builds. + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. Private members: _base_board_dict: Last-summarised Dict of boards @@ -224,7 +226,7 @@ class Builder: no_subdirs=False, full_path=False, verbose_build=False, incremental=False, per_board_out_dir=False, config_only=False, squash_config_y=False, - warnings_as_errors=False): + warnings_as_errors=False, work_in_output=False): """Create a new Builder object Args: @@ -250,10 +252,15 @@ class Builder: config_only: Only configure each build, don't build it squash_config_y: Convert CONFIG options with the value 'y' to '1' warnings_as_errors: Treat all compiler warnings as errors + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ self.toolchains = toolchains self.base_dir = base_dir - self._working_dir = os.path.join(base_dir, '.bm-work') + if work_in_output: + self._working_dir = base_dir + else: + self._working_dir = os.path.join(base_dir, '.bm-work') self.threads = [] self.do_make = self.Make self.gnu_make = gnu_make @@ -280,6 +287,7 @@ class Builder: self.config_only = config_only self.squash_config_y = squash_config_y self.config_filenames = BASE_CONFIG_FILENAMES + self.work_in_output = work_in_output if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES @@ -329,7 +337,7 @@ class Builder: show_errors: True to show summarised error/warning info show_sizes: Show size deltas - show_detail: Show detail for each board + show_detail: Show size delta detail for each board if show_sizes show_bloat: Show detail for each function list_error_boards: Show the boards which caused each error/warning show_config: Show config deltas @@ -477,6 +485,7 @@ class Builder: if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) + # See _GetOutputSpaceRemovals() which parses this name commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) elif not self.no_subdirs: @@ -992,7 +1001,7 @@ class Builder: board.target board_dict: Dict containing boards for which we built this commit, keyed by board.target. The value is an Outcome object. - show_detail: Show detail for each board + show_detail: Show size delta detail for each board show_bloat: Show detail for each function """ arch_list = {} @@ -1109,7 +1118,7 @@ class Builder: environment: Dictionary keyed by environment variable, Each value is the value of environment variable. show_sizes: Show image size deltas - show_detail: Show detail for each board + show_detail: Show size delta detail for each board if show_sizes show_bloat: Show detail for each function show_config: Show config changes show_environment: Show environment changes @@ -1474,6 +1483,8 @@ class Builder: Args: thread_num: Number of thread to check. """ + if self.work_in_output: + return self._working_dir return os.path.join(self._working_dir, '%02d' % thread_num) def _PrepareThread(self, thread_num, setup_git): @@ -1515,12 +1526,15 @@ class Builder: for thread in range(max_threads): self._PrepareThread(thread, setup_git) - def _PrepareOutputSpace(self): + def _GetOutputSpaceRemovals(self): """Get the output directories ready to receive files. - We delete any output directories which look like ones we need to - create. Having left over directories is confusing when the user wants - to check the output manually. + Figure out what needs to be deleted in the output directory before it + can be used. We only delete old buildman directories which have the + expected name pattern. See _GetOutputDir(). + + Returns: + List of full paths of directories to remove """ if not self.commits: return @@ -1531,12 +1545,26 @@ class Builder: to_remove = [] for dirname in glob.glob(os.path.join(self.base_dir, '*')): if dirname not in dir_list: - to_remove.append(dirname) + leaf = dirname[len(self.base_dir) + 1:] + m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf) + if m: + to_remove.append(dirname) + return to_remove + + def _PrepareOutputSpace(self): + """Get the output directories ready to receive files. + + We delete any output directories which look like ones we need to + create. Having left over directories is confusing when the user wants + to check the output manually. + """ + to_remove = self._GetOutputSpaceRemovals() if to_remove: - Print('Removing %d old build directories' % len(to_remove), + Print('Removing %d old build directories...' % len(to_remove), newline=False) for dirname in to_remove: shutil.rmtree(dirname) + Print('done') def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards @@ -1571,6 +1599,7 @@ class Builder: job.board = brd job.commits = commits job.keep_outputs = keep_outputs + job.work_in_output = self.work_in_output job.step = self._step self.queue.put(job) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 570c1f6595..7561f39942 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -39,11 +39,18 @@ class BuilderJob: Members: board: Board object to build - commits: List of commit options to build. + commits: List of Commit objects to build + keep_outputs: True to save build output files + step: 1 to process every commit, n to process every nth commit + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ def __init__(self): self.board = None self.commits = [] + self.keep_outputs = False + self.step = 1 + self.work_in_output = False class ResultThread(threading.Thread): @@ -114,7 +121,7 @@ class BuilderThread(threading.Thread): **kwargs) def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only, - force_build, force_build_failures): + force_build, force_build_failures, work_in_output): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -129,6 +136,8 @@ class BuilderThread(threading.Thread): force_build: Force a build even if one was previously done force_build_failures: Force a bulid if the previous result showed failure + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. Returns: tuple containing: @@ -139,7 +148,7 @@ class BuilderThread(threading.Thread): # self.Make() below, in the event that we do a build. result = command.CommandResult() result.return_code = 0 - if self.builder.in_tree: + if work_in_output or self.builder.in_tree: out_dir = work_dir else: if self.per_board_out_dir: @@ -261,14 +270,18 @@ class BuilderThread(threading.Thread): result.out_dir = out_dir return result, do_config - def _WriteResult(self, result, keep_outputs): + def _WriteResult(self, result, keep_outputs, work_in_output): """Write a built result to the output directory. Args: result: CommandResult object containing result to write keep_outputs: True to store the output binaries, False to delete them + work_in_output: Use the output directory as the work directory and + don't write to a separate output directory. """ + if work_in_output: + return # Fatal error if result.return_code < 0: return @@ -430,7 +443,8 @@ class BuilderThread(threading.Thread): result, request_config = self.RunCommit(commit_upto, brd, work_dir, do_config, self.builder.config_only, force_build or self.builder.force_build, - self.builder.force_build_failures) + self.builder.force_build_failures, + work_in_output=job.work_in_output) failed = result.return_code or result.stderr did_config = do_config if failed and not do_config: @@ -438,7 +452,8 @@ class BuilderThread(threading.Thread): # with a reconfig. if self.builder.force_config_on_failure: result, request_config = self.RunCommit(commit_upto, - brd, work_dir, True, False, True, False) + brd, work_dir, True, False, True, False, + work_in_output=job.work_in_output) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -477,15 +492,16 @@ class BuilderThread(threading.Thread): raise ValueError('Interrupt') # We have the build results, so output the result - self._WriteResult(result, job.keep_outputs) + self._WriteResult(result, job.keep_outputs, job.work_in_output) self.builder.out_queue.put(result) else: # Just build the currently checked-out build result, request_config = self.RunCommit(None, brd, work_dir, True, self.builder.config_only, True, - self.builder.force_build_failures) + self.builder.force_build_failures, + work_in_output=job.work_in_output) result.commit_upto = 0 - self._WriteResult(result, job.keep_outputs) + self._WriteResult(result, job.keep_outputs, job.work_in_output) self.builder.out_queue.put(result) def run(self): diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index b41209373d..17ea015a95 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -13,8 +13,6 @@ def ParseArgs(): args: command lin arguments """ parser = OptionParser() - parser.add_option('-a', '--print-arch', action='store_true', - help='Print the architecture for a board (ARCH=)') parser.add_option('-A', '--print-prefix', action='store_true', help='Print the tool-chain prefix for a board (CROSS_COMPILE=)') parser.add_option('-b', '--branch', type='string', @@ -31,7 +29,7 @@ def ParseArgs(): help='Reconfigure for every commit (disable incremental build)') parser.add_option('-d', '--detail', dest='show_detail', action='store_true', default=False, - help='Show detailed information for each board in summary') + help='Show detailed size delta for each board in the -S summary') parser.add_option('-D', '--config-only', action='store_true', default=False, help="Don't build, just configure each commit") parser.add_option('-e', '--show_errors', action='store_true', @@ -106,6 +104,10 @@ def ParseArgs(): default=False, help='Show build results while the build progresses') parser.add_option('-V', '--verbose-build', action='store_true', default=False, help='Run make with V=1, logging all output') + parser.add_option('-w', '--work-in-output', action='store_true', + default=False, help='Use the output directory as the work directory') + parser.add_option('-W', '--ignore-warnings', action='store_true', + default=False, help='Return success even if there are warnings') parser.add_option('-x', '--exclude', dest='exclude', type='string', action='append', help='Specify a list of boards to exclude, separated by comma') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 969d866547..5ddc598c95 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -85,38 +85,15 @@ def ShowActions(series, why_selected, boards_selected, builder, options, for warning in board_warnings: print(col.Color(col.YELLOW, warning)) -def CheckOutputDir(output_dir): - """Make sure that the output directory is not within the current directory - - If we try to use an output directory which is within the current directory - (which is assumed to hold the U-Boot source) we may end up deleting the - U-Boot source code. Detect this and print an error in this case. - - Args: - output_dir: Output directory path to check - """ - path = os.path.realpath(output_dir) - cwd_path = os.path.realpath('.') - while True: - if os.path.realpath(path) == cwd_path: - Print("Cannot use output directory '%s' since it is within the current directory '%s'" % - (path, cwd_path)) - sys.exit(1) - parent = os.path.dirname(path) - if parent == path: - break - path = parent - -def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix): +def ShowToolchainPrefix(boards, toolchains): """Show information about a the tool chain used by one or more boards - The function checks that all boards use the same toolchain. + The function checks that all boards use the same toolchain, then prints + the correct value for CROSS_COMPILE. Args: boards: Boards object containing selected boards toolchains: Toolchains object containing available toolchains - print_arch: True to print ARCH value - print_prefix: True to print CROSS_COMPILE value Return: None on success, string error message otherwise @@ -129,10 +106,7 @@ def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix): return 'Supplied boards must share one toolchain' return False tc = tc_set.pop() - if print_arch: - print(tc.GetEnvArgs(toolchain.VAR_ARCH)) - if print_prefix: - print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) + print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) return None def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, @@ -228,9 +202,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, if not len(selected): sys.exit(col.Color(col.RED, 'No matching boards found')) - if options.print_arch or options.print_prefix: - err = ShowToolchainInfo(boards, toolchains, options.print_arch, - options.print_prefix) + if options.print_prefix: + err = ShowToolchainInfo(boards, toolchains) if err: sys.exit(col.Color(col.RED, err)) return 0 @@ -263,6 +236,13 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, str = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) sys.exit(col.Color(col.RED, str)) + if options.work_in_output: + if len(selected) != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single board')) + if count != 1: + sys.exit(col.Color(col.RED, + '-w can only be used with a single commit')) # Read the metadata from the commits. First look at the upstream commit, # then the ones in the branch. We would like to do something like @@ -324,7 +304,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, output_dir = os.path.join(options.output_dir, dirname) if clean_dir and os.path.exists(output_dir): shutil.rmtree(output_dir) - CheckOutputDir(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -334,7 +313,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, per_board_out_dir=options.per_board_out_dir, config_only=options.config_only, squash_config_y=not options.preserve_config_y, - warnings_as_errors=options.warnings_as_errors) + warnings_as_errors=options.warnings_as_errors, + work_in_output=options.work_in_output) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func @@ -378,6 +358,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, options.keep_outputs, options.verbose) if fail: return 128 - elif warned: + elif warned and not options.ignore_warnings: return 129 return 0 diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 4c3d497294..2a256a9263 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -16,6 +16,7 @@ import control import gitutil import terminal import toolchain +import tools settings_data = ''' # Buildman settings file @@ -208,7 +209,7 @@ class TestFunctional(unittest.TestCase): def tearDown(self): shutil.rmtree(self._base_dir) - shutil.rmtree(self._output_dir) + #shutil.rmtree(self._output_dir) def setupToolchains(self): self._toolchains = toolchain.Toolchains() @@ -218,12 +219,12 @@ class TestFunctional(unittest.TestCase): return command.RunPipe([[self._buildman_pathname] + list(args)], capture=True, capture_stderr=True) - def _RunControl(self, *args, **kwargs): + def _RunControl(self, *args, clean_dir=False, boards=None): sys.argv = [sys.argv[0]] + list(args) options, args = cmdline.ParseArgs() result = control.DoBuildman(options, args, toolchains=self._toolchains, - make_func=self._HandleMake, boards=self._boards, - clean_dir=kwargs.get('clean_dir', True)) + make_func=self._HandleMake, boards=boards or self._boards, + clean_dir=clean_dir) self._builder = control.builder return result @@ -397,6 +398,12 @@ class TestFunctional(unittest.TestCase): combined='Test configuration complete') elif stage == 'build': stderr = '' + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] + fname = os.path.join(cwd or '', out_dir, 'u-boot') + tools.WriteFile(fname, b'U-Boot') if type(commit) is not str: stderr = self._error.get((brd.target, commit.sequence)) if stderr: @@ -527,11 +534,26 @@ class TestFunctional(unittest.TestCase): self.assertEqual(self._builder.count, self._total_builds) self.assertEqual(self._builder.fail, 0) - def testBadOutputDir(self): - """Test building with an output dir the same as out current dir""" - self._test_branch = '/__dev/__testbranch' - with self.assertRaises(SystemExit): - self._RunControl('-b', self._test_branch, '-o', os.getcwd()) - with self.assertRaises(SystemExit): - self._RunControl('-b', self._test_branch, '-o', - os.path.join(os.getcwd(), 'test')) + def testWorkInOutput(self): + """Test the -w option which should write directly to the output dir""" + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + self._RunControl('-o', self._output_dir, '-w', clean_dir=False, + boards=board_list) + self.assertTrue( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + + def testWorkInOutputFail(self): + """Test the -w option failures""" + with self.assertRaises(SystemExit) as e: + self._RunControl('-o', self._output_dir, '-w', clean_dir=False) + self.assertIn("single board", str(e.exception)) + self.assertFalse( + os.path.exists(os.path.join(self._output_dir, 'u-boot'))) + + board_list = board.Boards() + board_list.AddBoard(board.Board(*boards[0])) + with self.assertRaises(SystemExit) as e: + self._RunControl('-b', self._test_branch, '-o', self._output_dir, + '-w', clean_dir=False, boards=board_list) + self.assertIn("single commit", str(e.exception)) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index acd862b3b0..2aaedf44ac 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -22,6 +22,7 @@ import commit import terminal import test_util import toolchain +import tools use_network = True @@ -469,6 +470,25 @@ class TestBuild(unittest.TestCase): self.assertEqual('HOSTCC=clang CC=clang', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS)) + def testPrepareOutputSpace(self): + def _Touch(fname): + tools.WriteFile(os.path.join(base_dir, fname), b'') + + base_dir = tempfile.mkdtemp() + + # Add various files that we want removed and left alone + to_remove = ['01_of_22_g0982734987_title', '102_of_222_g92bf_title', + '01_of_22_g2938abd8_title'] + to_leave = ['something_else', '01-something.patch', '01_of_22_another'] + for name in to_remove + to_leave: + _Touch(name) + + build = builder.Builder(self.toolchains, base_dir, None, 1, 2) + build.commits = self.commits + build.commit_count = len(commits) + result = set(build._GetOutputSpaceRemovals()) + expected = set([os.path.join(base_dir, f) for f in to_remove]) + self.assertEqual(expected, result) if __name__ == "__main__": unittest.main() |