@inproceedings{bf73e01cd5ab44b79ab4fb141f6094fe,
title = "Improving affect detection in game-based learning with multimodal data fusion",
abstract = "Accurately recognizing learner affect is critically important for enabling affect-responsive learning environments to support student learning and engagement. Multimodal affect detection combining sensor-based and sensor-free approaches has shown significant promise in both laboratory and classroom settings. However, important questions remain regarding which data channels are most predictive and how they should be combined. In this paper, we investigate a multimodal affect detection framework that integrates motion tracking-based posture data and interaction-based trace data to recognize the affective states of students engaged with a game-based learning environment for emergency medical training. We compare several machine learning-based affective models using competing feature-level and decision-level multimodal data fusion approaches. Results indicate that multimodal affect detectors induced using joint feature representations from posture-based and interaction-based data channels yield improved accuracy relative to unimodal models across several learner-centered affective states. These findings point toward implications for the design of multimodal affect-responsive learning environments that support learning and engagement.",
keywords = "Affect detection, Game-based learning, Multimodal data fusion",
author = "Nathan Henderson and Jonathan Rowe and Luc Paquette and Baker, {Ryan S.} and James Lester",
note = "Funding Information: Acknowledgements. We wish to thank Dr. Jeanine DeFalco, Dr. Benjamin Goldberg, and Dr. Keith Brawner at the U.S. Army Combat Capabilities Development Command, Dr. Mike Matthews and COL James Ness at the U.S. Military Academy, and Dr. Robert Sottilare at SoarTech for their assistance in facilitating this research. The research was supported by the U.S. Army Research Laboratory under cooperative agreement #W911NF-13-2-0008. Any opinions, findings, and conclusions expressed in this paper are those of the authors and do not necessarily reflect the views of the U.S. Army. Funding Information: We wish to thank Dr. Jeanine DeFalco, Dr. Benjamin Goldberg, and Dr. Keith Brawner at the U.S. Army Combat Capabilities Development Command, Dr. Mike Matthews and COL James Ness at the U.S. Military Academy, and Dr. Robert Sottilare at SoarTech for their assistance in facilitating this research. The research was supported by the U.S. Army Research Laboratory under cooperative agreement #W911NF-13-2-0008. Any opinions, findings, and conclusions expressed in this paper are those of the authors and do not necessarily reflect the views of the U.S. Army. Publisher Copyright: {\textcopyright} Springer Nature Switzerland AG 2020.; 21st International Conference on Artificial Intelligence in Education, AIED 2020 ; Conference date: 06-07-2020 Through 10-07-2020",
year = "2020",
doi = "10.1007/978-3-030-52237-7_19",
language = "English (US)",
isbn = "9783030522360",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer",
pages = "228--239",
editor = "Bittencourt, {Ig Ibert} and Mutlu Cukurova and Rose Luckin and Kasia Muldner and Eva Mill{\'a}n",
booktitle = "Artificial Intelligence in Education- 21st International Conference, AIED 2020, Proceedings, Part I",
address = "Germany",
}