Questions of fairness, robustness, and transparency are paramount to address before deploying NLP systems. Central to these concerns is the question of reliability: Can NLP systems reliably treat different demographics fairly \emphand function correctly in diverse and noisy environments? To address this, we argue for the need for reliability testing and contextualize it among existing work on improving accountability. We show how adversarial attacks can be reframed for this goal, via a framework for developing reliability tests. We argue that reliability testing -- with an emphasis on interdisciplinary collaboration -- will enable rigorous and targeted testing, and aid in the enactment and enforcement of industry standards.

@inproceedings{Tan-et-al-acl-21, abstract = {Questions of fairness, robustness, and transparency are paramount to address before deploying NLP systems. Central to these concerns is the question of reliability: Can NLP systems reliably treat different demographics fairly \emph{and} function correctly in diverse and noisy environments? To address this, we argue for the need for reliability testing and contextualize it among existing work on improving accountability. We show how adversarial attacks can be reframed for this goal, via a framework for developing reliability tests. We argue that reliability testing --- with an emphasis on interdisciplinary collaboration --- will enable rigorous and targeted testing, and aid in the enactment and enforcement of industry standards.}, address = {Bangkok, Thailand}, author = {Samson Tan and Shafiq Joty and Kathy Baxter and Araz Taeihagh and Gregory A. Bennett and Min-Yen Kan}, booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics}, numpages = {9}, pages = {4153–4169}, publisher = {ACL}, series = {ACL'21}, title = {Reliability Testing for Natural Language Processing Systems}, url = {https://aclanthology.org/2021.acl-long.321/}, year = {2021} }