@inproceedings{41833a240ece487498193092e56ed979,
title = "Strategies for deploying unreliable AI graders in high-transparency high-stakes exams",
abstract = "We describe the deployment of an imperfect NLP-based automatic short answer grading system on an exam in a large-enrollment introductory college course. We characterize this deployment as both high stakes (the questions were on an mid-term exam worth 10% of students{\textquoteright} final grade) and high transparency (the question was graded interactively during the computer-based exam and correct solutions were shown to students that could be compared to their answer). We study two techniques designed to mitigate the potential student dissatisfaction resulting from students incorrectly not granted credit by the imperfect AI grader. We find (1) that providing multiple attempts can eliminate first-attempt false negatives at the cost of additional false positives, and (2) that students not granted credit from the algorithm cannot reliably determine if their answer was mis-scored.",
keywords = "Automatic short answer grading, CS1, Code reading, Computer-based exams, EiPE, Transparency",
author = "Sushmita Azad and Binglin Chen and Maxwell Fowler and Matthew West and Craig Zilles",
note = "Publisher Copyright: {\textcopyright} Springer Nature Switzerland AG 2020.; 21st International Conference on Artificial Intelligence in Education, AIED 2020 ; Conference date: 06-07-2020 Through 10-07-2020",
year = "2020",
doi = "10.1007/978-3-030-52237-7_2",
language = "English (US)",
isbn = "9783030522360",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer",
pages = "16--28",
editor = "Bittencourt, {Ig Ibert} and Mutlu Cukurova and Rose Luckin and Kasia Muldner and Eva Mill{\'a}n",
booktitle = "Artificial Intelligence in Education- 21st International Conference, AIED 2020, Proceedings, Part I",
address = "Germany",
}