@inproceedings{4e36d24216e740438bbb6d1aee1c08be,
title = "Humans vs. ChatGPT: Evaluating Annotation Methods for Financial Corpora",
abstract = "Given the vast amount of unstructured financial text data available today, there is a high demand for reliable, quality annotations to facilitate robust model development. However, traditional methods can often be expensive and time-inefficient. In this study, we investigate annotations for emotion, sentiment, and cognitive dissonance generated by the large language models (LLMs), GPT-3.5 and GPT-4, for quarterly earnings conference calls and compare them against human annotations obtained via traditional methods. We also investigate different prompt engineering choices on LLM annotation quality, experimenting with 4 styles of prompts centered around varying the amount of contextual information given and how it is presented to the models. Our results show the GPT models are not only more consistent and reliable than human annotators, but also provide annotations in a more cost- and time-efficient manner.",
keywords = "earnings calls, emotion recognition, large language models, sentiment analysis",
author = "Jamshed Kaikaus and Haoen Li and Brunner, {Robert J.}",
note = "Publisher Copyright: {\textcopyright} 2023 IEEE.; 2023 IEEE International Conference on Big Data, BigData 2023 ; Conference date: 15-12-2023 Through 18-12-2023",
year = "2023",
doi = "10.1109/BigData59044.2023.10386425",
language = "English (US)",
series = "Proceedings - 2023 IEEE International Conference on Big Data, BigData 2023",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "2831--2838",
editor = "Jingrui He and Themis Palpanas and Xiaohua Hu and Alfredo Cuzzocrea and Dejing Dou and Dominik Slezak and Wei Wang and Aleksandra Gruca and Lin, {Jerry Chun-Wei} and Rakesh Agrawal",
booktitle = "Proceedings - 2023 IEEE International Conference on Big Data, BigData 2023",
address = "United States",
}