@inproceedings{8d14fd0b19964efabc03b3a531cd2c48,
title = "Learning design semantics for mobile apps",
abstract = "Recently, researchers have developed black-box approaches to mine design and interaction data from mobile apps. Although the data captured during this interaction mining is descriptive, it does not expose the design semantics of UIs: what elements on the screen mean and how they are used. This paper introduces an automatic approach for generating semantic annotations for mobile app UIs. Through an iterative open coding of 73k UI elements and 720 screens, we contribute a lexical database of 25 types of UI components, 197 text button concepts, and 135 icon classes shared across apps. We use this labeled data to learn code-based patterns to detect UI components and to train a convolutional neural network that distinguishes between icon classes with 94% accuracy. To demonstrate the efficacy of our approach at scale, we compute semantic annotations for the 72k unique UIs in the Rico dataset, assigning labels for 78% of the total visible, non-redundant elements.",
keywords = "Design semantics, Machine learning, Mobile app design",
author = "Liu, {Thomas F.} and Mark Craft and Jason Situ and Ersin Yumer and Radomir Mech and Ranjitha Kumar",
note = "Publisher Copyright: {\textcopyright} 2018 Association for Computing Machinery.; 31st Annual ACM Symposium on User Interface Software and Technology, UIST 2018 ; Conference date: 14-10-2018 Through 17-10-2018",
year = "2018",
month = oct,
day = "11",
doi = "10.1145/3242587.3242650",
language = "English (US)",
series = "UIST 2018 - Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology",
publisher = "Association for Computing Machinery",
pages = "569--579",
booktitle = "UIST 2018 - Proceedings of the 31st Annual ACM Symposium on User Interface Software and Technology",
address = "United States",
}