Isolating and repairing buggy software behavior requires finding where the bug is happening and understanding the root cause of the buggy behavior. While there exist tools that can help developers identify where a bug is located, existing work has paid little, if any, attention to helping developers understand the root cause of buggy behavior. We present Causal Testing, a new method of root-cause analysis that relies on the theory of counterfactual causality to identify a set of executions that likely hold key causal information necessary to understand and repair buggy behavior. Evaluating Causal Testing on the Defects4J benchmark, we find that Causal Testing could be applied to 71% of real-world defects, and for 77% of those, it can help developers identify the root cause of the defect. A controlled experiment with 37 developers showed that Causal Testing improved participants' ability to identify the cause of the defect: Users with standard testing tools correctly identified the cause 88% of the time in comparison to 96% of the time with Causal Testing. Overall, participants agreed Causal Testing provided useful information they could not get using tools like JUnit alone.
@inproceedings{Johnson20icse,
author = {Brittany Johnson and Yuriy Brun and Alexandra Meliou},
title =
{\href{http://people.cs.umass.edu/brun/pubs/pubs/Johnson20icse.pdf}{Causal Testing: Understanding Defects' Root Causes}},
booktitle = {Proceedings of the 42nd International Conference on Software Engineering (ICSE)},
venue = {ICSE},
address = {Seoul, Republic of Korea},
month = {May},
date = {27--29},
year = {2020},
doi = {10.1145/3377811.3380377},
note = {ACM artifact badges granted:
\href{https://www.acm.org/publications/policies/artifact-review-badging}{Artifact
Available \raisebox{-.75ex}{\includegraphics[height=2.5ex]{ACMArtifactAvailable}},
Artifact Reusable \raisebox{-.75ex}{\includegraphics[height=2.5ex]{ACMArtifactReusable}}}.
\href{https://doi.org/10.1145/3377811.3380377}{DOI: 10.1145/3377811.3380377}},
accept = {$\frac{129}{617} \approx 21\%$},
abstract = {<p>Isolating and repairing buggy software behavior requires finding where the
bug is happening and understanding the root cause of the buggy behavior.
While there exist tools that can help developers identify where a bug is
located, existing work has paid little, if any, attention to helping
developers understand the root cause of buggy behavior. We present Causal
Testing, a new method of root-cause analysis that relies on the theory of
counterfactual causality to identify a set of executions that likely hold key
causal information necessary to understand and repair buggy behavior.
Evaluating Causal Testing on the Defects4J benchmark, we find that Causal
Testing could be applied to 71% of real-world defects, and for 77% of those,
it can help developers identify the root cause of the defect. A controlled
experiment with 37 developers showed that Causal Testing improved
participants' ability to identify the cause of the defect: Users with
standard testing tools correctly identified the cause 88% of the time in
comparison to 96% of the time with Causal Testing. Overall, participants
agreed Causal Testing provided useful information they could not get using
tools like JUnit alone.</p>},
fundedBy = {NSF CCF-1453474, NSF IIS-1453543, NSF CNS-1744471, Google, Oracle Labs},
comment = {<span class="emphasis">[ACM SIGSOFT Distinguished Artifact Award]</span>},
}