From 97bc0f44865b1d7bf3a7e1a56d959dfd402c2241 Mon Sep 17 00:00:00 2001
From: Daniel Agar <daniel@agar.ca>
Date: Sat, 31 Dec 2016 16:34:07 -0500
Subject: [PATCH] gather test results
---
CMakeLists.txt | 1 +
CTestConfig.cmake | 15 ++++
Makefile | 12 +--
Tools/CTest2JUnit.xsl | 120 ++++++++++++++++++++++++++++++
circle.yml | 14 ++--
src/firmware/posix/CMakeLists.txt | 18 +++++
unittests/CMakeLists.txt | 4 +-
7 files changed, 172 insertions(+), 12 deletions(-)
create mode 100644 CTestConfig.cmake
create mode 100644 Tools/CTest2JUnit.xsl
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 101ea12a4c..bcf3e3741f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -271,6 +271,7 @@ find_package(PythonInterp REQUIRED)
# cmake modules
#
enable_testing()
+include(CTest)
#=============================================================================
# generate compile command database
diff --git a/CTestConfig.cmake b/CTestConfig.cmake
new file mode 100644
index 0000000000..5762625f32
--- /dev/null
+++ b/CTestConfig.cmake
@@ -0,0 +1,15 @@
+## This file should be placed in the root directory of your project.
+## Then modify the CMakeLists.txt file in the root directory of your
+## project to incorporate the testing dashboard.
+##
+## # The following are required to submit to the CDash dashboard:
+## ENABLE_TESTING()
+## INCLUDE(CTest)
+
+set(CTEST_PROJECT_NAME "PX4 Firmware")
+set(CTEST_NIGHTLY_START_TIME "00:00:00 EST")
+
+set(CTEST_DROP_METHOD "http")
+set(CTEST_DROP_SITE "my.cdash.org")
+set(CTEST_DROP_LOCATION "/submit.php?project=PX4+Firmware")
+set(CTEST_DROP_SITE_CDASH TRUE)
diff --git a/Makefile b/Makefile
index 5aea66e4ed..4477a1c8f4 100755
--- a/Makefile
+++ b/Makefile
@@ -278,17 +278,17 @@ unittest: posix_sitl_default
$(call cmake-build,unittest,$(SRC_DIR)/unittests)
@(cd build_unittest && ctest -j2 --output-on-failure)
-run_tests_posix: posix_sitl_default
- @(cd build_posix_sitl_default/ && ctest --output-on-failure)
+run_tests_posix:
+ @$(call PX4_RUN,$(MAKE) --no-print-directory posix_sitl_default test_results_junit)
tests: unittest run_tests_posix
tests_coverage:
- @(PX4_CODE_COVERAGE=1 CCACHE_DISABLE=1 ${MAKE} tests)
- @(lcov --directory . --capture --quiet --output-file coverage.info)
- @(lcov --remove coverage.info '/usr/*' --quiet --output-file coverage.info)
+ @$(call PX4_RUN,$(MAKE) --no-print-directory tests PX4_CODE_COVERAGE=1 CCACHE_DISABLE=1)
+ @$(call PX4_RUN,lcov --directory . --capture --quiet --output-file coverage.info)
+ @$(call PX4_RUN,lcov --remove coverage.info '/usr/*' --quiet --output-file coverage.info)
#@(lcov --list coverage.info)
- @(genhtml coverage.info --quiet --output-directory coverage-html)
+ @$(call PX4_RUN,genhtml coverage.info --quiet --output-directory coverage-html)
package_firmware:
@zip --junk-paths Firmware.zip `find Binaries/. -name \*.px4`
diff --git a/Tools/CTest2JUnit.xsl b/Tools/CTest2JUnit.xsl
new file mode 100644
index 0000000000..8ba21f4e60
--- /dev/null
+++ b/Tools/CTest2JUnit.xsl
@@ -0,0 +1,120 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="xml" indent="yes"/>
+ <xsl:template match="/Site">
+ <xsl:variable name="Name"><xsl:value-of select="@Name"/></xsl:variable>
+ <xsl:variable name="Hostname"><xsl:value-of select="@Hostname"/></xsl:variable>
+ <xsl:variable name="TestCount"><xsl:value-of select="count(//TestList/Test)"/> </xsl:variable>
+ <xsl:variable name="ErrorCount"><xsl:value-of select="count(//TestList/Test[@Status='error'])"/> </xsl:variable>
+ <xsl:variable name="FailureCount"><xsl:value-of select="count(//Testing/Test[@Status='failed'])"/> </xsl:variable>
+ <testsuite name="{$Name}" hostname="{$Hostname}" errors="0" failures="{$FailureCount}" tests="{$TestCount}">
+ <xsl:variable name="BuildName"><xsl:value-of select="@BuildName"/></xsl:variable>
+ <xsl:variable name="BuildStamp"><xsl:value-of select="@BuildStamp"/></xsl:variable>
+ <xsl:variable name="Generator"><xsl:value-of select="@Generator"/></xsl:variable>
+ <xsl:variable name="CompilerName"><xsl:value-of select="@CompilerName"/></xsl:variable>
+ <xsl:variable name="OSName"><xsl:value-of select="@OSName"/></xsl:variable>
+ <xsl:variable name="OSRelease"><xsl:value-of select="@OSRelease"/></xsl:variable>
+ <xsl:variable name="OSVersion"><xsl:value-of select="@OSVersion"/></xsl:variable>
+ <xsl:variable name="OSPlatform"><xsl:value-of select="@OSPlatform"/></xsl:variable>
+ <xsl:variable name="Is64Bits"><xsl:value-of select="@Is64Bits"/></xsl:variable>
+ <xsl:variable name="VendorString"><xsl:value-of select="@VendorString"/></xsl:variable>
+ <xsl:variable name="VendorID"><xsl:value-of select="@VendorID"/></xsl:variable>
+ <xsl:variable name="FamilyID"><xsl:value-of select="@FamilyID"/></xsl:variable>
+ <xsl:variable name="ModelID"><xsl:value-of select="@ModelID"/></xsl:variable>
+ <xsl:variable name="ProcessorCacheSize"><xsl:value-of select="@ProcessorCacheSize"/></xsl:variable>
+ <xsl:variable name="NumberOfLogicalCPU"><xsl:value-of select="@NumberOfLogicalCPU"/></xsl:variable>
+ <xsl:variable name="NumberOfPhysicalCPU"><xsl:value-of select="@NumberOfPhysicalCPU"/></xsl:variable>
+ <xsl:variable name="TotalVirtualMemory"><xsl:value-of select="@TotalVirtualMemory"/></xsl:variable>
+ <xsl:variable name="TotalPhysicalMemory"><xsl:value-of select="@TotalPhysicalMemory"/></xsl:variable>
+ <xsl:variable name="LogicalProcessorsPerPhysical"><xsl:value-of select="@LogicalProcessorsPerPhysical"/></xsl:variable>
+ <xsl:variable name="ProcessorClockFrequency"><xsl:value-of select="@ProcessorClockFrequency"/></xsl:variable>
+ <properties>
+ <property name="BuildName" value="{$BuildName}" />
+ <property name="BuildStamp" value="{$BuildStamp}" />
+ <property name="Name" value="{$Name}" />
+ <property name="Generator" value="{$Generator}" />
+ <property name="CompilerName" value="{$CompilerName}" />
+ <property name="OSName" value="{$OSName}" />
+ <property name="Hostname" value="{$Hostname}" />
+ <property name="OSRelease" value="{$OSRelease}" />
+ <property name="OSVersion" value="{$OSVersion}" />
+ <property name="OSPlatform" value="{$OSPlatform}" />
+ <property name="Is64Bits" value="{$Is64Bits}" />
+ <property name="VendorString" value="{$VendorString}" />
+ <property name="VendorID" value="{$VendorID}" />
+ <property name="FamilyID" value="{$FamilyID}" />
+ <property name="ModelID" value="{$ModelID}" />
+ <property name="ProcessorCacheSize" value="{$ProcessorCacheSize}" />
+ <property name="NumberOfLogicalCPU" value="{$NumberOfLogicalCPU}" />
+ <property name="NumberOfPhysicalCPU" value="{$NumberOfPhysicalCPU}" />
+ <property name="TotalVirtualMemory" value="{$TotalVirtualMemory}" />
+ <property name="TotalPhysicalMemory" value="{$TotalPhysicalMemory}" />
+ <property name="LogicalProcessorsPerPhysical" value="{$LogicalProcessorsPerPhysical}" />
+ <property name="ProcessorClockFrequency" value="{$ProcessorClockFrequency}" />
+ </properties>
+ <xsl:apply-templates select="Testing/Test"/>
+
+ <system-out>
+ BuildName: <xsl:value-of select="$BuildName" />
+ BuildStamp: <xsl:value-of select="$BuildStamp" />
+ Name: <xsl:value-of select="$Name" />
+ Generator: <xsl:value-of select="$Generator" />
+ CompilerName: <xsl:value-of select="$CompilerName" />
+ OSName: <xsl:value-of select="$OSName" />
+ Hostname: <xsl:value-of select="$Hostname" />
+ OSRelease: <xsl:value-of select="$OSRelease" />
+ OSVersion: <xsl:value-of select="$OSVersion" />
+ OSPlatform: <xsl:value-of select="$OSPlatform" />
+ Is64Bits: <xsl:value-of select="$Is64Bits" />
+ VendorString: <xsl:value-of select="$VendorString" />
+ VendorID: <xsl:value-of select="$VendorID" />
+ FamilyID: <xsl:value-of select="$FamilyID" />
+ ModelID: <xsl:value-of select="$ModelID" />
+ ProcessorCacheSize: <xsl:value-of select="$ProcessorCacheSize" />
+ NumberOfLogicalCPU: <xsl:value-of select="$NumberOfLogicalCPU" />
+ NumberOfPhysicalCPU: <xsl:value-of select="$NumberOfPhysicalCPU" />
+ TotalVirtualMemory: <xsl:value-of select="$TotalVirtualMemory" />
+ TotalPhysicalMemory: <xsl:value-of select="$TotalPhysicalMemory" />
+ LogicalProcessorsPerPhysical: <xsl:value-of select="$LogicalProcessorsPerPhysical" />
+ ProcessorClockFrequency: <xsl:value-of select="$ProcessorClockFrequency" />
+ </system-out>
+ </testsuite>
+ </xsl:template>
+
+ <xsl:template match="Testing/Test">
+ <xsl:variable name="testcasename"><xsl:value-of select= "Name"/></xsl:variable>
+ <xsl:variable name="testclassname"><xsl:value-of select= " concat('this', substring(Path,2))"/></xsl:variable>
+ <xsl:variable name="exectime">
+ <xsl:for-each select="Results/NamedMeasurement">
+ <xsl:if test="@name = 'Execution Time'">
+ <xsl:value-of select="."/>
+ </xsl:if>
+ </xsl:for-each>
+ </xsl:variable>
+
+ <testcase name="{$testcasename}" classname="{$testclassname}" time="{$exectime}">
+ <xsl:if test="@Status = 'passed'">
+ </xsl:if>
+ <xsl:if test="@Status = 'failed'">
+ <xsl:variable name="failtype">
+ <xsl:for-each select="Results/NamedMeasurement">
+ <xsl:if test="@name = 'Exit Code'">
+ <xsl:value-of select="."/>
+ </xsl:if>
+ </xsl:for-each>
+ </xsl:variable>
+ <xsl:variable name="failcode">
+ <xsl:for-each select="Results/NamedMeasurement">
+ <xsl:if test="@name = 'Exit Value'">
+ <xsl:value-of select="."/>
+ </xsl:if>
+ </xsl:for-each>
+ </xsl:variable>
+ <failure message="{$failtype} ({$failcode})"><xsl:value-of select="Results/Measurement/Value/text()" /></failure>
+ </xsl:if>
+ <xsl:if test="@Status = 'notrun'">
+ <skipped><xsl:value-of select="Results/Measurement/Value/text()" /></skipped>
+ </xsl:if>
+ </testcase>
+ </xsl:template>
+
+</xsl:stylesheet>
diff --git a/circle.yml b/circle.yml
index b92e992381..e01f15b864 100644
--- a/circle.yml
+++ b/circle.yml
@@ -12,15 +12,19 @@ checkout:
dependencies:
pre:
- - docker pull px4io/px4-dev-nuttx:2016-12-26
+ - docker pull px4io/px4-dev-base
test:
override:
- - docker run --rm -v `pwd`:`pwd`:rw -e CI=true -w=`pwd` -e LOCAL_USER_ID=$UID -it px4io/px4-dev-nuttx:2016-12-26 /bin/bash -c "make quick_check"
+ - PX4_DOCKER=1 make tests
+ # copy test results for circleci
+ - mkdir -p $CIRCLE_TEST_REPORTS/unittests/ && cp build_unittest/test_detail.xml $CIRCLE_TEST_REPORTS/unittests/
+ - mkdir -p $CIRCLE_TEST_REPORTS/sitlunittests/ && cp ./build_posix_sitl_default/JUnitTestResults.xml $CIRCLE_TEST_REPORTS/sitlunittests/
+ - make distclean
+ - PX4_DOCKER=1 make tests_coverage
general:
artifacts:
- - "build_px4fmu-v2_default/parameters.xml"
- - "build_px4fmu-v2_default/airframes.xml"
- - "build_.*/src/firmware/nuttx/.*.px4"
+ - "coverage-html"
+ - "coverage.info"
diff --git a/src/firmware/posix/CMakeLists.txt b/src/firmware/posix/CMakeLists.txt
index 2530c160fa..56faefcfd5 100644
--- a/src/firmware/posix/CMakeLists.txt
+++ b/src/firmware/posix/CMakeLists.txt
@@ -187,4 +187,22 @@ foreach(test_name ${test_src_files})
endif()
endforeach()
+add_custom_target(test_results
+ COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -T Test
+ DEPENDS px4
+ USES_TERMINAL
+ COMMENT "Running tests in sitl"
+ WORKING_DIRECTORY ${PX4_BINARY_DIR})
+
+add_custom_target(test_results_junit
+ COMMAND xsltproc ${PX4_SOURCE_DIR}/Tools/CTest2JUnit.xsl Testing/`head -n 1 < Testing/TAG`/Test.xml > JUnitTestResults.xml
+ DEPENDS test_results
+ COMMENT "Converting ctest output to junit xml"
+ WORKING_DIRECTORY ${PX4_BINARY_DIR})
+
+add_custom_target(test_cdash_submit
+ COMMAND ${CMAKE_CTEST_COMMAND} -D Experimental
+ USES_TERMINAL
+ WORKING_DIRECTORY ${PX4_BINARY_DIR})
+
# vim: set noet ft=cmake fenc=utf-8 ff=unix :
diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt
index 0ed8ec9781..9f534c933c 100644
--- a/unittests/CMakeLists.txt
+++ b/unittests/CMakeLists.txt
@@ -10,6 +10,8 @@ endif()
project(px4_unittests)
enable_testing()
+message(STATUS ${CONFIG})
+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -std=gnu99 -g")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -std=gnu++0x -g -fno-exceptions -fno-rtti -fno-threadsafe-statics -DCONFIG_WCHAR_BUILTIN -D__CUSTOM_FILE_IO__")
@@ -90,7 +92,7 @@ function(add_gtest)
else()
target_link_libraries(${test_name} gtest_main pthread rt ${PX4_PLATFORM})
endif()
- add_test(NAME ${test_name} COMMAND ${test_name} WORKING_DIRECTORY ${PX4_SOURCE_DIR})
+ add_test(NAME ${test_name} COMMAND ${test_name} --gtest_output=xml WORKING_DIRECTORY ${PX4_BINARY_DIR})
add_dependencies(check ${test_name})
endforeach()
endfunction()
--
GitLab