Jenkins add SITL tests coverage pipeline

.ci/Jenkinsfile-SITL_tests

@@ -3,81 +3,32 @@
 pipeline {
   agent none
 
-  parameters {
-    choice(
-      name: 'PX4_CMAKE_BUILD_TYPE',
-      choices: ['RelWithDebInfo', 'Coverage', 'AddressSanitizer', 'UndefinedBehaviorSanitizer'],
-      description: "CMake build type"
-      )
-  }
-
   stages {
 
     stage('Build') {
-
       agent {
         docker {
           image 'px4io/px4-dev-ros-kinetic:2018-09-11'
           args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE --cap-add SYS_PTRACE --entrypoint=""'
         }
       }
-
-      stages {
-
-        stage('build px4') {
-          steps {
-            sh('export')
-            sh('make distclean')
-            sh "ccache -z"
-            sh "git fetch --tags"
-            sh('make px4_sitl_default')
-            sh "ccache -s"
-          }
-        }
-
-        stage('stash build for coverage') {
-          steps {
-            stash includes: 'build/**/*', name: 'build_sitl_coverage', useDefaultExcludes: false
-          }
-          when {
-            environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage'
-          }
-        }
-
-        stage('unit tests') {
-          steps {
-            sh 'export'
-            sh 'make px4_sitl_test test_results_junit'
-            junit 'build/px4_sitl_test/JUnitTestResults.xml'
-          }
-        }
-
-        stage('build sitl_gazebo') {
-          steps {
-            sh 'export'
-            sh "ccache -z"
-            sh('make px4_sitl_default sitl_gazebo')
-            sh "ccache -s"
-          }
-        }
-
-        stage('package') {
-          steps {
-            sh 'export'
-            sh('make px4_sitl_default package')
-            stash(name: "px4_sitl_package", includes: "build/px4_sitl_default/*.bz2")
-            archiveArtifacts(artifacts: "build/px4_sitl_default/*.bz2", fingerprint: true, onlyIfSuccessful: true)
-          }
-        }
-
+      steps {
+        sh 'export'
+        sh 'make distclean'
+        sh 'ccache -z'
+        sh 'git fetch --tags'
+        sh 'make px4_sitl_default'
+        sh 'make px4_sitl_default sitl_gazebo'
+        sh 'make px4_sitl_default package'
+        sh 'ccache -s'
+        stash(name: "px4_sitl_package", includes: "build/px4_sitl_default/*.bz2")
+        archiveArtifacts(artifacts: "build/px4_sitl_default/*.bz2", fingerprint: true, onlyIfSuccessful: true)
       }
-
       post {
         always {
             sh 'make distclean'
         }
       }
-
     } // stage Build
 
     stage('ROS Tests') {
@@ -141,64 +92,11 @@ pipeline {
       } // steps
     } // stage ROS Tests
 
-    stage('Coverage') {
-      parallel {
-
-        stage('code coverage (python)') {
-          agent {
-            docker {
-              image 'px4io/px4-dev-base:2019-01-26'
-              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw --cap-add SYS_PTRACE'
-            }
-          }
-          steps {
-            sh 'export'
-            sh 'make distclean'
-            sh 'make python_coverage'
-            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
-              sh 'curl -s https://codecov.io/bash | bash -s - -F python'
-            }
-
-            sh 'make distclean'
-          }
-
-          when {
-            environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage'
-          }
-        }
-
-        stage('unit tests') {
-          agent {
-            docker {
-              image 'px4io/px4-dev-base:2019-01-26'
-              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw --cap-add SYS_PTRACE'
-            }
-          }
-          steps {
-            sh 'export'
-            sh 'make distclean'
-            sh 'make px4_sitl_test test_results_junit'
-            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
-              sh 'curl -s https://codecov.io/bash | bash -s - -F unittest'
-            }
-            sh 'make distclean'
-          }
-          when {
-            environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage'
-          }
-        }
-
-      } // parallel
-    } // stage Coverage
-
   } //stages
 
   environment {
-    ASAN_OPTIONS = 'detect_stack_use_after_return=1:check_initialization_order=1'
-    UBSAN_OPTIONS = 'print_stacktrace=1'
     CCACHE_DIR = '/tmp/ccache'
     CI = true
-    CTEST_OUTPUT_ON_FAILURE = 1
   }
 
   options {
@@ -216,11 +114,6 @@ def createTestNode(Map test_def) {
           def test_ok = true
           sh('export')
 
-          if (env.PX4_CMAKE_BUILD_TYPE == 'Coverage') {
-              checkout(scm)
-              unstash 'build_sitl_coverage'
-          }
-
           unstash('px4_sitl_package')
           sh('tar -xjpvf build/px4_sitl_default/px4-px4_sitl_default*.bz2')
 
@@ -235,41 +128,19 @@ def createTestNode(Map test_def) {
           }
 
           // log analysis
-          if (env.PX4_CMAKE_BUILD_TYPE == 'Coverage') {
-            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
-              sh 'curl -s https://codecov.io/bash | bash -s - -F sitl_mission_${STAGE_NAME}'
-
-              // process log data (with python code coverage)
-              try {
-                sh('coverage run -p px4-px4_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg')
-              } catch (exc) {
-                // save log analysis artifacts for debugging
-                archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv')
-                // FIXME: don't let the script to fail the build
-                // test_ok = false
-              }
-
-              // upload log to flight review (https://logs.px4.io/) with python code coverage
-              sh('coverage run -p px4-px4_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg')
-
-              // upload python code coverage to codecov.io
-              sh 'curl -s https://codecov.io/bash | bash -s - -X gcov -F sitl_python_${STAGE_NAME}'
-            }
-          } else { // non code coverage
-            // process ekf log data
-            try {
-              sh('px4-px4_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg')
-            } catch (exc) {
-              // save log analysis artifacts for debugging
-              archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv')
-              // FIXME: don't let the script to fail the build
-              // test_ok = false
-            }
-
-            // upload log to flight review (https://logs.px4.io/)
-            sh('px4-px4_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg')
+          // process ekf log data
+          try {
+            sh('px4-px4_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg')
+          } catch (exc) {
+            // save log analysis artifacts for debugging
+            archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv')
+            // FIXME: don't let the script to fail the build
+            // test_ok = false
           }
 
+          // upload log to flight review (https://logs.px4.io/)
+          sh('px4-px4_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg')
+
           if (!test_ok) {
             error('ROS Test failed')
           }

.ci/Jenkinsfile-SITL_tests_coverage

+#!/usr/bin/env groovy
+
+pipeline {
+  agent none
+
+  stages {
+
+    stage('Build') {
+      agent {
+        docker {
+          image 'px4io/px4-dev-ros-kinetic:2018-09-11'
+          args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE --cap-add SYS_PTRACE --entrypoint=""'
+        }
+      }
+      steps {
+        sh 'export'
+        sh 'make distclean'
+        sh 'ccache -z'
+        sh 'git fetch --tags'
+        sh 'make px4_sitl_default'
+        stash includes: 'build/**/*', name: 'build_sitl_coverage', useDefaultExcludes: false
+        sh 'make px4_sitl_default sitl_gazebo'
+        sh 'make px4_sitl_default package'
+        sh 'ccache -s'
+        stash(name: "px4_sitl_package", includes: "build/px4_sitl_default/*.bz2")
+        archiveArtifacts(artifacts: "build/px4_sitl_default/*.bz2", fingerprint: true, onlyIfSuccessful: true)
+      }
+      post {
+        always {
+            sh 'make distclean'
+        }
+      }
+    } // stage Build
+
+    stage('ROS Tests') {
+      steps {
+        script {
+          def missions = [
+            [
+              name: "FW",
+              test: "mavros_posix_test_mission.test",
+              mission: "FW_mission_1",
+              vehicle: "plane"
+            ],
+
+            [
+              name: "MC_box",
+              test: "mavros_posix_test_mission.test",
+              mission: "MC_mission_box",
+              vehicle: "iris"
+            ],
+            [
+              name: "MC_offboard_att",
+              test: "mavros_posix_tests_offboard_attctl.test",
+              mission: "",
+              vehicle: "iris"
+            ],
+            [
+              name: "MC_offboard_pos",
+              test: "mavros_posix_tests_offboard_posctl.test",
+              mission: "",
+              vehicle: "iris"
+            ],
+
+            [
+              name: "VTOL_standard",
+              test: "mavros_posix_test_mission.test",
+              mission: "VTOL_mission_1",
+              vehicle: "standard_vtol"
+            ],
+            [
+              name: "VTOL_tailsitter",
+              test: "mavros_posix_test_mission.test",
+              mission: "VTOL_mission_1",
+              vehicle: "tailsitter"
+            ],
+            [
+              name: "VTOL_tiltrotor",
+              test: "mavros_posix_test_mission.test",
+              mission: "VTOL_mission_1",
+              vehicle: "tiltrotor"
+            ],
+
+          ]
+
+          def test_nodes = [:]
+          for (def i = 0; i < missions.size(); i++) {
+            test_nodes.put(missions[i].name, createTestNode(missions[i]))
+          }
+
+          parallel test_nodes
+        } // script
+      } // steps
+    } // stage ROS Tests
+
+    stage('Coverage') {
+      parallel {
+
+        stage('code coverage (python)') {
+          agent {
+            docker {
+              image 'px4io/px4-dev-base:2019-01-26'
+              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw --cap-add SYS_PTRACE'
+            }
+          }
+          steps {
+            sh 'export'
+            sh 'make distclean'
+            sh 'make python_coverage'
+            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+              sh 'curl -s https://codecov.io/bash | bash -s - -F python'
+            }
+
+            sh 'make distclean'
+          }
+        }
+
+        stage('unit tests') {
+          agent {
+            docker {
+              image 'px4io/px4-dev-base:2019-01-26'
+              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw --cap-add SYS_PTRACE'
+            }
+          }
+          steps {
+            sh 'export'
+            sh 'make distclean'
+            sh 'make px4_sitl_test test_results_junit'
+            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+              sh 'curl -s https://codecov.io/bash | bash -s - -F unittest'
+            }
+            sh 'make distclean'
+          }
+        }
+
+      } // parallel
+    } // stage Coverage
+
+  } //stages
+
+  environment {
+    CCACHE_DIR = '/tmp/ccache'
+    CI = true
+    CTEST_OUTPUT_ON_FAILURE = 1
+    PX4_CMAKE_BUILD_TYPE = 'Coverage'
+  }
+
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '2', artifactDaysToKeepStr: '14'))
+    timeout(time: 60, unit: 'MINUTES')
+  }
+} // pipeline
+
+def createTestNode(Map test_def) {
+  return {
+    node {
+      cleanWs()
+      docker.image("px4io/px4-dev-ros-kinetic:2018-09-11").inside('-e HOME=${WORKSPACE} --cap-add SYS_PTRACE --entrypoint=""') {
+        stage(test_def.name) {
+          def test_ok = true
+          sh('export')
+
+          checkout(scm)
+          unstash 'build_sitl_coverage'
+
+          unstash('px4_sitl_package')
+          sh('tar -xjpvf build/px4_sitl_default/px4-px4_sitl_default*.bz2')
+
+          // run test
+          try {
+            sh('px4-px4_sitl_default*/px4/test/rostest_px4_run.sh ' + test_def.test + ' mission:=' + test_def.mission + ' vehicle:=' + test_def.vehicle)
+
+          } catch (exc) {
+            // save all test artifacts for debugging
+            archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.ulg, .ros/**/rosunit-*.xml, .ros/**/rostest-*.log')
+            test_ok = false
+          }
+
+          // log analysis
+          withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+            sh 'curl -s https://codecov.io/bash | bash -s - -F sitl_mission_${STAGE_NAME}'
+
+            // process log data (with python code coverage)
+            try {
+              sh('coverage run -p px4-px4_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg')
+            } catch (exc) {
+              // save log analysis artifacts for debugging
+              archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv')
+              // FIXME: don't let the script to fail the build
+              // test_ok = false
+            }
+
+            // upload log to flight review (https://logs.px4.io/) with python code coverage
+            sh('coverage run -p px4-px4_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg')
+
+            // upload python code coverage to codecov.io
+            sh 'curl -s https://codecov.io/bash | bash -s - -X gcov -F sitl_python_${STAGE_NAME}'
+          }
+
+          if (!test_ok) {
+            error('ROS Test failed')
+          }
+        } // stage
+        cleanWs()
+      } // docker.image
+    } // node
+  } // return
+} // createTestNode

Jenkinsfile

@@ -141,6 +141,28 @@ pipeline {
           }
         }
 
+        stage('SITL unit tests') {
+          agent {
+            docker {
+              image 'px4io/px4-dev-base:2019-02-03'
+              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw'
+            }
+          }
+          steps {
+            sh 'export'
+            sh 'make distclean'
+            sh 'ccache -z'
+            sh 'git fetch --tags'
+            sh 'make tests'
+            sh 'ccache -s'
+          }
+          post {
+            always {
+              sh 'make distclean'
+            }
+          }
+        }
+
         stage('Clang analyzer') {
           agent {
             docker {