Linux Auto test with Linaro Test-definition

Prepare Test Environment

Ubuntu 22.04

git clone https://github.com/Linaro/test-definitions
cd ./test-definitions
. ./automated/bin/setenv.sh

pip install -r ${REPO_PATH}/automated/utils/requirements.txt

test-runner -h

/home/baifudong/workdir/bsp/test-definitions/automated/bin/test-runner:19: DeprecationWarning: The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives
  from distutils.spawn import find_executable
usage: test-runner [-h] [-o OUTPUT] [-p TEST_PLAN] [-d TEST_DEF] [-r KEY=VALUE [KEY=VALUE ...]] [-k {automated,manual}]
                   [-t TIMEOUT] [-g TARGET] [-s] [-e] [-l] [-O OVERLAY] [-v] [--qa-reports-server QA_REPORTS_SERVER]
                   [--qa-reports-token QA_REPORTS_TOKEN] [--qa-reports-project QA_REPORTS_PROJECT]
                   [--qa-reports-group QA_REPORTS_GROUP] [--qa-reports-env QA_REPORTS_ENV]
                   [--qa-reports-build-version QA_REPORTS_BUILD_VERSION] [--qa-reports-disable-metadata]
                   [--qa-reports-metadata KEY=VALUE [KEY=VALUE ...]] [--qa-reports-metadata-file QA_REPORTS_METADATA_FILE]
                   [--cleanup]

options:
  -h, --help            show this help message and exit
  -o OUTPUT, --output OUTPUT
                        specify a directory to store test and result files.
                        Default: $HOME/output
  -p TEST_PLAN, --test_plan TEST_PLAN
                        specify an test plan file which has tests and related
                        params listed in yaml format.
  -d TEST_DEF, --test_def TEST_DEF
                        base on test definition repo location, specify relative
                        path to the test definition to run.
                        Format example: "ubuntu/smoke-tests-basic.yaml"
  -r KEY=VALUE [KEY=VALUE ...], --test_def_params KEY=VALUE [KEY=VALUE ...]
                        Set additional parameters when using test definition without
                        a test plan. The name values are set similarily to environment
                        variables:
                        --test_def_params KEY1=VALUE1 KEY2=VALUE2 ...
  -k {automated,manual}, --kind {automated,manual}
                        Selects type of tests to be executed from the test plan.
                        Possible options: automated, manual
  -t TIMEOUT, --timeout TIMEOUT
                        Specify test timeout
  -g TARGET, --target TARGET
                        Specify SSH target to execute tests.
                        Format: user@host
                        Note: ssh authentication must be paswordless
  -s, --skip_install    skip install section defined in test definition.
  -e, --skip_environment
                        skip environmental data collection (board name, distro, etc)
  -l, --lava_run        send test result to LAVA with lava-test-case.
  -O OVERLAY, --overlay OVERLAY
                        Specify test plan ovelay file to:
                        * skip tests
                        * amend test parameters
                        * add new tests
  -v, --verbose         Set log level to DEBUG.
  --qa-reports-server QA_REPORTS_SERVER
                        qa reports server where the results have to be sent
  --qa-reports-token QA_REPORTS_TOKEN
                        qa reports token to upload the results to qa_reports_server
  --qa-reports-project QA_REPORTS_PROJECT
                        qa reports projects to which the results have to be uploaded
  --qa-reports-group QA_REPORTS_GROUP
                        qa reports group in which the results have to be stored
  --qa-reports-env QA_REPORTS_ENV
                        qa reports environment for the results that have to be stored
  --qa-reports-build-version QA_REPORTS_BUILD_VERSION
                        qa reports build id for the result set
  --qa-reports-disable-metadata
                        Disable sending metadata to SQUAD. Default: false
  --qa-reports-metadata KEY=VALUE [KEY=VALUE ...]
                        List of metadata key=value pairs to be sent to SQUAD
  --qa-reports-metadata-file QA_REPORTS_METADATA_FILE
                        YAML file that defines metadata to be reported to SQUAD
  --cleanup             If set to true, test-runner will remove all temporary files               after running the test. It includes all collected logs and               test results. This option should only be used if uploading               results to SQUAD or LAVA.               Default: false

Example

test-runner --target root@192.168.1.81 --test_def automated/linux/smoke/smoke.yaml

# Skip install
test-runner -s --target root@192.168.1.81 --test_def automated/linux/smoke/smoke.yaml

Test Plan

test-runner --target root@192.168.1.81 -p ./plans/linux-yocto-test-plan.yaml
test-runner --target root@192.168.1.81 -p ./plans/linux-yocto-test-plan.yaml -O test-plan-overlay-example.yaml

metadata:
  # name key is needed by test-runner
  name: Yocto-linux-test-plan
  description: Yocto Linux test plan
  os: poky
  devices:
    - roc-rk3328-cc
  maintainer:
    - first.last@linaro.org
  approver:
    - first.last@linaro.org
  owner:
    - first.last@linaro.org
  # format key is needed by test-runner
  format: Linaro Test Plan v2

tests:
  automated:
    - path: automated/linux/smoke/smoke.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/blogbench/blogbench.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/mmc/mmc-test.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/lmbench/lmbench-memory.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/sigwaittest/sigwaittest.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/fio-test/fio-test.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      timeout: 1800
    - path: automated/linux/sysbench/sysbench.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      parameters:
        SKIP_INSTALL: True
        STATIC: True
      timeout: 1800
    - path: automated/linux/hackbench/hackbench.yaml
      repository: https://git.linaro.org/qa/test-definitions.git
      parameters:
        ITERATION: "10"
      timeout: 1800