@inproceedings{f971c435a9a94960b3d90030b5227330,
title = "Critique style guide: Improving crowdsourced design feedback with a natural language model",
abstract = "Designers are increasingly leveraging online crowds; yet, online contributors may lack the expertise, context, and sensitivity to provide effective critique. Rubrics help feedback providers but require domain experts to write them and may not generalize across design domains. This paper introduces and tests a novel semi-automated method to support feedback providers by analysing feedback language. In our first study, 52 students from two design courses created design solutions and received feedback from 176 online providers. Instructors, students, and crowd contributors rated the helpfulness of each feedback response. From this data, an algorithm extracted a set of natural language features (e.g., specificity, sentiment etc.) that correlated with the ratings. The features accurately predicted the ratings and remained stable across different raters and design solutions. Based on these features, we produced a critique style guide with feedback examples - automatically selected for each feature - to help providers revise their feedback through self-assessment. In a second study, we tested the validity of the guide through a between-subjects experiment (n=50). Providers wrote feedback on design solutions with or without the guide. Providers generated feedback with higher perceived helpfulness when using our style-based guidance.",
keywords = "Critique, Crowdsourcing, Design, Expertise, Feedback, Rubrics",
author = "Markus Krause and Tom Garncarz and Song, {Jiao Jiao} and Gerber, {Elizabeth M.} and Bailey, {Brian P.} and Dow, {Steven P.}",
note = "Publisher Copyright: {\textcopyright} 2017 ACM.; 2017 ACM SIGCHI Conference on Human Factors in Computing Systems, CHI 2017 ; Conference date: 06-05-2017 Through 11-05-2017",
year = "2017",
month = may,
day = "2",
doi = "10.1145/3025453.3025883",
language = "English (US)",
series = "Conference on Human Factors in Computing Systems - Proceedings",
publisher = "Association for Computing Machinery",
pages = "4627--4639",
booktitle = "CHI 2017 - Proceedings of the 2017 ACM SIGCHI Conference on Human Factors in Computing Systems",
address = "United States",
}