@inproceedings{91bbec3bea8c4e3380f5a6b7235ba34b,
title = "Code Generation Based Grading: Evaluating an Auto-grading Mechanism for {"}Explain-in-Plain-English{"} Questions",
abstract = "Comprehending and conveying the purpose of code is often cited as being a key learning objective within introductory programming courses. To address this objective, {"}Explain in Plain English'' questions, where students are shown a segment of code and asked to provide an abstract description of the code's purpose, have been adopted. However, given EiPE questions require a natural language response, they often require manual grading which is time-consuming for course staff and delays feedback for students. With the advent of large language models (LLMs) capable of generating code, responses to EiPE questions can be used to generate code segments, the correctness of which can then be easily verified using test cases. We refer to this approach as {"}Code Generation Based Grading'' (CGBG) and in this paper we explore its agreement with human graders using EiPE responses from past exams in an introductory programming course taught in Python. Overall, we find that all CGBG approaches achieve moderate agreement with human graders with the primary area of disagreement being its leniency with respect to low-level and line-by-line descriptions of code.",
keywords = "auto-grading, eipe, gpt-4, large language models",
author = "Smith, {David H.} and Craig Zilles",
note = "Publisher Copyright: {\textcopyright} 2024 ACM.; 29th Conference Innovation and Technology in Computer Science Education, ITiCSE 2024 ; Conference date: 08-07-2024 Through 10-07-2024",
year = "2024",
month = jul,
day = "3",
doi = "10.1145/3649217.3653582",
language = "English (US)",
series = "Annual Conference on Innovation and Technology in Computer Science Education, ITiCSE",
publisher = "Association for Computing Machinery",
pages = "171--177",
booktitle = "ITiCSE 2024 - Proceedings of the 2024 Conference Innovation and Technology in Computer Science Education",
address = "United States",
}