by José Antonio Parejo, Ana B. Sanchez, Sergio Segura, Antonio Ruiz Cortes, Roberto E. Lopez-Herrejon, Alexander Egyed
Abstract:
Test case prioritization schedules test cases for execution in an order that attempts to accelerate the detection of faults. The order of test cases is determined by prioritization objectives such as covering code or critical components as rapidly as possible. The importance of this technique has been recognized in the context of Highly-Configurable Systems (HCSs), where the potentially huge number of configurations makes testing extremely challenging. However, current approaches for test case prioritization in HCSs suffer from two main limitations. First, the prioritization is usually driven by a single objective which neglects the potential benefits of combining multiple criteria to guide the detection of faults. Second, instead of using industry-strength case studies, evaluations are conducted using synthetic data, which provides no information about the effectiveness of different prioritization objectives. In this paper, we address both limitations by studying 63 combinations of up to three prioritization objectives in accelerating the detection of faults in the Drupal framework. Results show that non-functional properties such as the number of changes in the features are more effective than functional metrics extracted from the configuration model. Results also suggest that multi-objective prioritization typically results in faster fault detection than mono-objective prioritization.
Reference:
Multi-objective Test Case Prioritization in Highly Configurable Systems: A Case Study (José Antonio Parejo, Ana B. Sanchez, Sergio Segura, Antonio Ruiz Cortes, Roberto E. Lopez-Herrejon, Alexander Egyed), In Journal of Systems and Software, volume 122, 2016.
Bibtex Entry:
@Article{DBLP:journals/jss/ParejoSSCLE16,
author = {José Antonio Parejo and Ana B. Sanchez and Sergio Segura and Antonio Ruiz Cortes and Roberto E. Lopez-Herrejon and Alexander Egyed},
title = {Multi-objective Test Case Prioritization in Highly Configurable Systems: A Case Study},
journal = {Journal of Systems and Software},
year = {2016},
volume = {122},
pages = {287--310},
abstract = {Test case prioritization schedules test cases for execution in an
order that attempts to accelerate the detection of faults. The order
of test cases is determined by prioritization objectives such as
covering code or critical components as rapidly as possible. The
importance of this technique has been recognized in the context of
Highly-Configurable Systems (HCSs), where the potentially huge number
of configurations makes testing extremely challenging. However, current
approaches for test case prioritization in HCSs suffer from two main
limitations. First, the prioritization is usually driven by a single
objective which neglects the potential benefits of combining multiple
criteria to guide the detection of faults. Second, instead of using
industry-strength case studies, evaluations are conducted using synthetic
data, which provides no information about the effectiveness of different
prioritization objectives. In this paper, we address both limitations
by studying 63 combinations of up to three prioritization objectives
in accelerating the detection of faults in the Drupal framework.
Results show that non-functional properties such as the number of
changes in the features are more effective than functional metrics
extracted from the configuration model. Results also suggest that
multi-objective prioritization typically results in faster fault
detection than mono-objective prioritization.},
bibsource = {dblp computer science bibliography, http://dblp.org},
biburl = {http://dblp.uni-trier.de/rec/bib/journals/jss/ParejoSSCLE16},
doi = {10.1016/j.jss.2016.09.045},
file = {:Journals\\JSS 2016 - Multi-Objective Test Case Prioritization in Highly Configurable Systems\\Multi-Objective Test Case Prioritization in Highly Configurable Systems-preprint.pdf:PDF},
keywords = {FWF P25513},
timestamp = {Thu, 10 Nov 2016 17:02:36 +0100},
url = {http://dx.doi.org/10.1016/j.jss.2016.09.045},
}