From ead4604d74c96bc1d2625bcec4a28f0c335a9ae4 Mon Sep 17 00:00:00 2001 From: Willem Jan Withagen Date: Sun, 9 Feb 2020 14:49:34 +0100 Subject: [PATCH 1/3] doc: Add manual for LABELS and COST to developer guide Signed-off-by: Willem Jan Withagen --- doc/dev/developer_guide/tests-unit-tests.rst | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/dev/developer_guide/tests-unit-tests.rst b/doc/dev/developer_guide/tests-unit-tests.rst index 315811e696765..ac17a529cc0d2 100644 --- a/doc/dev/developer_guide/tests-unit-tests.rst +++ b/doc/dev/developer_guide/tests-unit-tests.rst @@ -48,6 +48,38 @@ functions, which are themselves defined in others are binaries that are compiled during the build process. The ``add_ceph_test`` function is used to declare unit test scripts, while ``add_ceph_unittest`` is used for unit test binaries. +``add_tox_test`` declare tests that require ``tox`` for testing and sets up +the full environment for this during testing. +:: + + add_ceph_test(test_name test_path ) + add_ceph_unittest(test_name ) + add_tox_test(test_name ) + args: any parameter, not being an options keyword + options: + [COST ] + Order of testing is set by COST. + Highest COST are tested first. + [TIMEOUT sec] + Restrict the max runtime of this test to + [LABELS "labels"] + Append "labels" to a test. + Labels can be selected with ctest -L or -LE + +Since COST is used to determine the testing order, values can be chosen rather +arbitrarily. Currently long running programs are given a value equal to the +execution time in a OpenStack VM. The idea is to first run the long standing +tests, and then fill up free cores with programs with shorter execution time. +Turning this into a bin-packing challenge, So the exact value is not important, +it is about the relative COST value. + +Currently some LABELS are introduced:: + + unittest as declared above (DEF COST = 5) + tox the test used tox to do the testing (DEF COST = 25) + long when the test executes for more than 60 secs (DEF COST = 50) + vstart when the test uses vsstart.sh to setup a cluster + Unit testing of CLI tools ------------------------- From cfd7e86926c2ccd57478a282a25418b9dab606ea Mon Sep 17 00:00:00 2001 From: Willem Jan Withagen Date: Sun, 9 Feb 2020 15:19:00 +0100 Subject: [PATCH 2/3] cmake: Adding COST, LABELS and TIMEOUT to CephTest's Signed-off-by: Willem Jan Withagen --- cmake/modules/AddCephTest.cmake | 39 ++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake index c9ee8c1b8f646..eb6b324b1bf7f 100644 --- a/cmake/modules/AddCephTest.cmake +++ b/cmake/modules/AddCephTest.cmake @@ -1,8 +1,25 @@ #AddCephTest is a module for adding tests to the "make check" target which runs CTest +macro(set_test_props labels cost cost_default timeout) + if("${timeout}" GREATER 0) + set_property(TEST ${test_name} PROPERTY TIMEOUT ${timeout}) + else() + set_property(TEST ${test_name} PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT}) + endif() + if("${cost}" GREATER 0) + set_property(TEST ${test_name} PROPERTY COST ${cost}) + else() + set_property(TEST ${test_name} PROPERTY COST ${cost_default}) + endif() + if(NOT "${labels}" EQUAL "") + set_property(TEST ${test_name} APPEND PROPERTY LABELS "${labels}") + endif() +endmacro() + #adds makes target/script into a test, test to check target, sets necessary environment variables function(add_ceph_test test_name test_path) - add_test(NAME ${test_name} COMMAND ${test_path} ${ARGN}) + cmake_parse_arguments(PROPS "" "COST;TIMEOUT" "LABELS" ${ARGN}) + add_test(NAME ${test_name} COMMAND ${test_path} ${PROPS_UNPARSED_ARGUMENTS}) if(TARGET ${test_name}) add_dependencies(tests ${test_name}) endif() @@ -17,10 +34,7 @@ function(add_ceph_test test_name test_path) PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH} PYTHONPATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cython_modules/lib.3:${CMAKE_SOURCE_DIR}/src/pybind CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV}) - # none of the tests should take more than 1 hour to complete - set_property(TEST - ${test_name} - PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT}) + set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 5.0 "${PROPS_TIMEOUT}") endfunction() option(WITH_GTEST_PARALLEL "Enable running gtest based tests in parallel" OFF) @@ -43,20 +57,22 @@ if(WITH_GTEST_PARALLEL) endif() #sets uniform compiler flags and link libraries -function(add_ceph_unittest unittest_name) - set(UNITTEST "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${unittest_name}") +function(add_ceph_unittest test_name) + cmake_parse_arguments(PROPS "" "COST;TIMEOUT" "LABELS" ${ARGN}) + set(UNITTEST "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_name}") # If the second argument is "parallel", it means we want a parallel run if(WITH_GTEST_PARALLEL AND "${ARGV1}" STREQUAL "parallel") set(UNITTEST ${GTEST_PARALLEL_COMMAND} ${UNITTEST}) endif() - add_ceph_test(${unittest_name} "${UNITTEST}") - target_link_libraries(${unittest_name} ${UNITTEST_LIBS}) + add_ceph_test(${test_name} "${UNITTEST}" "unittest" LABELS "unittest") + target_link_libraries(${test_name} ${UNITTEST_LIBS}) + set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 1.0 "${PROPS_TIMEOUT}") endfunction() function(add_tox_test name) set(test_name run-tox-${name}) set(venv_path ${CEPH_BUILD_VIRTUALENV}/${name}-virtualenv) - cmake_parse_arguments(TOXTEST "" "TOX_PATH" "TOX_ENVS" ${ARGN}) + cmake_parse_arguments(TOXTEST "" "TOX_PATH;COST;TIMEOUT" "TOX_ENVS;LABELS" ${ARGN}) if(DEFINED TOXTEST_TOX_PATH) set(tox_path ${TOXTEST_TOX_PATH}) else() @@ -95,4 +111,7 @@ function(add_tox_test name) PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH} PYTHONPATH=${CMAKE_SOURCE_DIR}/src/pybind) list(APPEND tox_test run-tox-${name}) + set_tests_properties(${test_name} PROPERTIES LABELS "tox") + set_test_props("${TOXTEST_LABELS}" "${TOXTEST_COST}" 25.0 "${TOXTEST_TIMEOUT}") endfunction() + From 9f6518e5ea58c6b62c9dc3ce058b0f798c621b43 Mon Sep 17 00:00:00 2001 From: Willem Jan Withagen Date: Sun, 9 Feb 2020 15:45:56 +0100 Subject: [PATCH 3/3] cmake: Add LABELS and COST to some long running tests Signed-off-by: Willem Jan Withagen --- qa/CMakeLists.txt | 2 +- src/pybind/mgr/dashboard/CMakeLists.txt | 4 ++-- src/test/CMakeLists.txt | 20 ++++++++++---------- src/test/encoding/CMakeLists.txt | 6 ++++-- src/test/erasure-code/CMakeLists.txt | 2 +- src/test/mgr/CMakeLists.txt | 2 +- src/test/objectstore/CMakeLists.txt | 6 +++--- src/test/osd/CMakeLists.txt | 7 ++++--- src/test/pybind/CMakeLists.txt | 3 ++- src/test/rbd_mirror/CMakeLists.txt | 2 +- 10 files changed, 29 insertions(+), 25 deletions(-) diff --git a/qa/CMakeLists.txt b/qa/CMakeLists.txt index 6d06fe9ea3df6..f734822e4d56a 100644 --- a/qa/CMakeLists.txt +++ b/qa/CMakeLists.txt @@ -5,5 +5,5 @@ endif() if(WITH_TESTS) include(AddCephTest) - add_tox_test(qa TOX_ENVS flake8 import-tasks) + add_tox_test(qa TOX_ENVS flake8 import-tasks COST 215.0) endif() diff --git a/src/pybind/mgr/dashboard/CMakeLists.txt b/src/pybind/mgr/dashboard/CMakeLists.txt index e94817d4fa138..7edd0798b45b6 100644 --- a/src/pybind/mgr/dashboard/CMakeLists.txt +++ b/src/pybind/mgr/dashboard/CMakeLists.txt @@ -106,6 +106,6 @@ add_custom_target(mgr-dashboard-frontend-build add_dependencies(tests mgr-dashboard-frontend-build) if(WITH_TESTS) - include(AddCephTest) - add_tox_test(mgr-dashboard TOX_ENVS lint check) + include(AddCephTest) + add_tox_test(mgr-dashboard TOX_ENVS lint check LABELS "long" COST 300.0) endif() diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 0d82c929f4c5a..5f5be8e5024a2 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -515,17 +515,17 @@ endif() if(WITH_RBD) # Run rbd-unit-tests separate so they an run in parallel # For values see: src/include/rbd/features.h - add_ceph_test(run-rbd-unit-tests-N.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh N) - add_ceph_test(run-rbd-unit-tests-0.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 0) - add_ceph_test(run-rbd-unit-tests-1.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 1) - add_ceph_test(run-rbd-unit-tests-61.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 61) - add_ceph_test(run-rbd-unit-tests-109.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 109) - add_ceph_test(run-rbd-unit-tests-127.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 127) + add_ceph_test(run-rbd-unit-tests-N.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh N COST 500.0) + add_ceph_test(run-rbd-unit-tests-0.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 0 COST 181.0) + add_ceph_test(run-rbd-unit-tests-1.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 1 COST 255.0) + add_ceph_test(run-rbd-unit-tests-61.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 61 COST 267.0) + add_ceph_test(run-rbd-unit-tests-109.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 109 COST 346.0) + add_ceph_test(run-rbd-unit-tests-127.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 127 COST 350.0) if(FREEBSD) - add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh) + add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh COST 160.0) endif(FREEBSD) endif(WITH_RBD) -add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests) +add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests COST 264.0) # flaky, see https://tracker.ceph.com/issues/44243 #add_ceph_test(test_objectstore_memstore.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_objectstore_memstore.sh) @@ -533,7 +533,7 @@ add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests) # flaky #add_ceph_test(test_pidfile.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_pidfile.sh) -add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh) +add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh COST 230.0) set_property( TEST ${tox_tests} @@ -657,7 +657,7 @@ add_executable(unittest_bufferlist bufferlist.cc $ ) -add_ceph_unittest(unittest_bufferlist) +add_ceph_unittest(unittest_bufferlist LABELS "long" COST 400.0) target_link_libraries(unittest_bufferlist global) # compiletest_cxx11_client diff --git a/src/test/encoding/CMakeLists.txt b/src/test/encoding/CMakeLists.txt index e8c57a042c00d..4282feb316ba6 100644 --- a/src/test/encoding/CMakeLists.txt +++ b/src/test/encoding/CMakeLists.txt @@ -1,3 +1,5 @@ # scripts -add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh) -add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh) +add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh + LABELS "long" COST 619.0) +add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh + LABELS "long" COST 813.0) diff --git a/src/test/erasure-code/CMakeLists.txt b/src/test/erasure-code/CMakeLists.txt index 640de24bd5fc3..81cabc88e8ff6 100644 --- a/src/test/erasure-code/CMakeLists.txt +++ b/src/test/erasure-code/CMakeLists.txt @@ -196,7 +196,7 @@ target_link_libraries(unittest_erasure_code_shec add_executable(unittest_erasure_code_shec_all TestErasureCodeShec_all.cc ) -add_ceph_unittest(unittest_erasure_code_shec_all parallel) +add_ceph_unittest(unittest_erasure_code_shec_all parallel LABELS "long" COST 560.0) target_link_libraries(unittest_erasure_code_shec_all global ${CMAKE_DL_LIBS} diff --git a/src/test/mgr/CMakeLists.txt b/src/test/mgr/CMakeLists.txt index 9e6950d799ee8..fc35ff30d8b37 100644 --- a/src/test/mgr/CMakeLists.txt +++ b/src/test/mgr/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(unittest_mgr_mgrcap global) #scripts if(WITH_MGR_DASHBOARD_FRONTEND) if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm|ARM") - add_ceph_test(mgr-dashboard-frontend-unittests ${CMAKE_SOURCE_DIR}/src/pybind/mgr/dashboard/run-frontend-unittests.sh) + add_ceph_test(mgr-dashboard-frontend-unittests ${CMAKE_SOURCE_DIR}/src/pybind/mgr/dashboard/run-frontend-unittests.sh COST 764.0) endif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm|ARM") add_ceph_test(mgr-dashboard-smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/mgr-dashboard-smoke.sh) diff --git a/src/test/objectstore/CMakeLists.txt b/src/test/objectstore/CMakeLists.txt index 233baa46ebf74..df45ba1f62f42 100644 --- a/src/test/objectstore/CMakeLists.txt +++ b/src/test/objectstore/CMakeLists.txt @@ -106,7 +106,7 @@ if(WITH_BLUESTORE) fastbmap_allocator_test.cc $ ) - add_ceph_unittest(unittest_fastbmap_allocator) + add_ceph_unittest(unittest_fastbmap_allocator LABELS "long" COST 102.0) target_link_libraries(unittest_fastbmap_allocator os global) set_target_properties(unittest_fastbmap_allocator PROPERTIES COMPILE_FLAGS @@ -130,7 +130,7 @@ if(WITH_BLUESTORE) add_executable(unittest_bluefs test_bluefs.cc ) - add_ceph_unittest(unittest_bluefs) + add_ceph_unittest(unittest_bluefsi COST 702.0) target_link_libraries(unittest_bluefs os global) # unittest_bluestore_types @@ -159,7 +159,7 @@ target_link_libraries(unittest_transaction os ceph-common) add_executable(unittest_memstore_clone test_memstore_clone.cc $) -add_ceph_unittest(unittest_memstore_clone) +add_ceph_unittest(unittest_memstore_clone 195.0) target_link_libraries(unittest_memstore_clone os global) if(WITH_BLUESTORE) diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt index fc343d969b213..381c6b4abff65 100644 --- a/src/test/osd/CMakeLists.txt +++ b/src/test/osd/CMakeLists.txt @@ -34,13 +34,14 @@ install(TARGETS DESTINATION ${CMAKE_INSTALL_BINDIR}) # scripts -add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh) +add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh + LABELS "long" COST 441.0) # unittest_osdmap add_executable(unittest_osdmap TestOSDMap.cc ) -add_ceph_unittest(unittest_osdmap) +add_ceph_unittest(unittest_osdmap LABELS "long" COST 91.0) target_link_libraries(unittest_osdmap global ${BLKID_LIBRARIES}) # unittest_osd_types @@ -71,7 +72,7 @@ add_executable(unittest_pglog $ $ ) -add_ceph_unittest(unittest_pglog) +add_ceph_unittest(unittest_pglog COST 214.0) target_link_libraries(unittest_pglog osd os global ${CMAKE_DL_LIBS} ${BLKID_LIBRARIES}) # unittest_hitset diff --git a/src/test/pybind/CMakeLists.txt b/src/test/pybind/CMakeLists.txt index bc91bce305dad..79adb7aa225ae 100644 --- a/src/test/pybind/CMakeLists.txt +++ b/src/test/pybind/CMakeLists.txt @@ -1,2 +1,3 @@ add_ceph_test(test_ceph_daemon.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_daemon.py) -add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py) +add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py + LABELS "long" COST 104.0) diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 1226735d54659..8ed0040be8419 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -49,7 +49,7 @@ add_executable(unittest_rbd_mirror image_sync/test_mock_SyncPointPruneRequest.cc pool_watcher/test_mock_RefreshImagesRequest.cc ) -add_ceph_unittest(unittest_rbd_mirror) +add_ceph_unittest(unittest_rbd_mirror LABELS "long" COST 196.0) add_dependencies(unittest_rbd_mirror cls_journal