@inproceedings{0f253d62e89b438e87c7777678d5d75d,
title = "MIRACLE: An Online, Explainable Multimodal Interactive Concept Learning System",
abstract = "We present MIRACLE, a system for online, interpretable visual concept and video action recognition. Through a chat interface, users query the recognition system with an uploaded image or video. For images, MIRACLE returns concept predictions from its structured knowledge base, justifying its predictions with heatmaps and natural language-based attribute detections. For videos, MIRACLE predicts an action and justifies its prediction with time varying entity-entity relations. With its ability to learn new concepts in an online, few-shot manner and its support of dynamic changes to its knowledge base, MIRACLE represents a step forward in interpretable multimodal learning systems.",
keywords = "few-shot learning, interpretability, multimodal interaction, object recognition, online learning, video action detection",
author = "Ansel Blume and Nguyen, {Khanh Duy} and Zhenhailong Wang and Yangyi Chen and Michal Shlapentokh-Rothman and Xiaomeng Jin and Jeonghwan Kim and Zhen Zhu and Jiateng Liu and Huang, {Kuan Hao} and Mankeerat Sidhu and Xuanming Zhang and Vivian Liu and Raunak Sinha and Wu, {Te Lin} and Abhay Zala and Elias Stengel-Eskin and Da Yin and Yao Xiao and Utkarsh Mall and Zhou Yu and Chang, {Kai Wei} and Camille Cobb and Karrie Karahalios and Lydia Chilton and Mohit Bansal and Nanyun Peng and Carl Vondrick and Derek Hoiem and Heng Ji",
note = "This research is based upon work supported by U.S. DARPA ECOLE Program No. HR00112390060. The views and conclusions contained herein are those of the authors and should not be interpreted as necessarily representing the official policies, either expressed or implied, of DARPA, or the U.S. Government. The U.S. Government is authorized to reproduce and distribute reprints for governmental purposes notwithstanding any copyright annotation therein.; 32nd ACM International Conference on Multimedia, MM 2024 ; Conference date: 28-10-2024 Through 01-11-2024",
year = "2024",
month = oct,
day = "28",
doi = "10.1145/3664647.3684993",
language = "English (US)",
series = "MM 2024 - Proceedings of the 32nd ACM International Conference on Multimedia",
publisher = "Association for Computing Machinery",
pages = "11252--11254",
booktitle = "MM 2024 - Proceedings of the 32nd ACM International Conference on Multimedia",
address = "United States",
}