{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T20:47:52Z","timestamp":1776113272188,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,5,2]],"date-time":"2017-05-02T00:00:00Z","timestamp":1493683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001655","name":"Deutscher Akademischer Austauschdienst","doi-asserted-by":"publisher","award":["FitWorldWide"],"award-info":[{"award-number":["FitWorldWide"]}],"id":[{"id":"10.13039\/501100001655","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1122206, 1122320"],"award-info":[{"award-number":["1122206, 1122320"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,5,2]]},"DOI":"10.1145\/3025453.3025883","type":"proceedings-article","created":{"date-parts":[[2017,5,3]],"date-time":"2017-05-03T12:34:28Z","timestamp":1493814868000},"page":"4627-4639","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":47,"title":["Critique Style Guide"],"prefix":"10.1145","author":[{"given":"Markus","family":"Krause","sequence":"first","affiliation":[{"name":"University of California, Berkeley, Berkeley, CA, USA"}]},{"given":"Tom","family":"Garncarz","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, Pittsburgh, PA, USA"}]},{"given":"JiaoJiao","family":"Song","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology, Huazhong, China"}]},{"given":"Elizabeth M.","family":"Gerber","sequence":"additional","affiliation":[{"name":"Northwestern University, Evanston, IL, USA"}]},{"given":"Brian P.","family":"Bailey","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}]},{"given":"Steven P.","family":"Dow","sequence":"additional","affiliation":[{"name":"University of California, San Diego, La Jolla, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2017,5,2]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"https:\/\/www.mturk.com","author":"Turk Mechanical","year":"2016","unstructured":"Amazon. Mechanical Turk . ( 2016 ). https:\/\/www.mturk.com Amazon. Mechanical Turk. (2016). https:\/\/www.mturk.com"},{"key":"e_1_3_2_1_2_1","volume-title":"https:\/\/www.behance.net\/","year":"2016","unstructured":"Behance. Behance. ( 2016 ). https:\/\/www.behance.net\/ Behance. Behance. (2016). https:\/\/www.behance.net\/"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_1_4_1","unstructured":"Steven Bird Ewan Klein and Edward Loper. 2009. Natural Language Processing with Python.   Steven Bird Ewan Klein and Edward Loper. 2009. Natural Language Processing with Python."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1010933404324"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1067445.1067468"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1177\/0741088306289261"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1531674.1531681"},{"key":"e_1_3_2_1_9_1","first-page":"604","article-title":"Multivariate Analysis","volume":"36","author":"Cramer E. M.","year":"1966","unstructured":"E. M. Cramer and R. D. Bock . 1966 . Multivariate Analysis . Review of Educational Research 36 (1966), 604 -- 617 . http:\/\/library.wur.nl\/WebQuery\/clc\/1809603 E. M. Cramer and R. D. Bock. 1966. Multivariate Analysis. Review of Educational Research 36 (1966), 604--617. http:\/\/library.wur.nl\/WebQuery\/clc\/1809603","journal-title":"Review of Educational Research"},{"key":"e_1_3_2_1_10_1","volume-title":"https:\/\/crowdflower.com","year":"2016","unstructured":"Crowdflower. Crowdflower. ( 2016 ). https:\/\/crowdflower.com Crowdflower. Crowdflower. (2016). https:\/\/crowdflower.com"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1476-8070.2009.01591.x"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979359"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470686"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1640233.1640260"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753688"},{"key":"e_1_3_2_1_17_1","unstructured":"Edmund Burke Feldman. 1994. Practical Art Criticism. Pearson Englewood Cliffs N.J.  Edmund Burke Feldman. 1994. Practical Art Criticism. Pearson Englewood Cliffs N.J."},{"key":"e_1_3_2_1_18_1","volume-title":"WordNet: An Electronic Lexical Database","author":"Fellbaum Christiane","unstructured":"Christiane Fellbaum . 1998. WordNet: An Electronic Lexical Database . Bradford Books . Christiane Fellbaum. 1998. WordNet: An Electronic Lexical Database. Bradford Books."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.5555\/164632.164891"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2757226.2757249"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/5254.889104"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1080\/00401706.1993.10485053"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858195"},{"key":"e_1_3_2_1_24_1","volume-title":"Predicting Crowd-based Translation Quality with Language-independent Feature Vectors. In HComp'12 Proceedings of the AAAI Workshop on Human Computation. AAAI Press","author":"Kilian Niklas","year":"2012","unstructured":"Niklas Kilian , Markus Krause , Nina Runge , and Jan Smeddinck . 2012 . Predicting Crowd-based Translation Quality with Language-independent Feature Vectors. In HComp'12 Proceedings of the AAAI Workshop on Human Computation. AAAI Press , Toronto, ON, Canada, 114--115. http:\/\/www.aaai.org\/ocs\/index.php\/WS\/AAAIW12\/ paper\/viewPDFInterstitial\/5237\/5611 Niklas Kilian, Markus Krause, Nina Runge, and Jan Smeddinck. 2012. Predicting Crowd-based Translation Quality with Language-independent Feature Vectors. In HComp'12 Proceedings of the AAAI Workshop on Human Computation. AAAI Press, Toronto, ON, Canada, 114--115. http:\/\/www.aaai.org\/ocs\/index.php\/WS\/AAAIW12\/ paper\/viewPDFInterstitial\/5237\/5611"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556325.2567881"},{"key":"e_1_3_2_1_27_1","volume-title":"HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts","author":"Krause Markus","year":"2015","unstructured":"Markus Krause . 2015 a. A Method to automatically choose Suggestions to Improve Perceived Quality of Peer Reviews based on Linguistic Features . In HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts . San Diego, CA, USA. Markus Krause. 2015a. A Method to automatically choose Suggestions to Improve Perceived Quality of Peer Reviews based on Linguistic Features. In HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts. San Diego, CA, USA."},{"key":"e_1_3_2_1_28_1","volume-title":"HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts","author":"Krause Markus","year":"2015","unstructured":"Markus Krause . 2015 b. Bull-O-Meter : Predicting the Quality of Natural Language Responses . In HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts . San Diego, CA, USA. Markus Krause. 2015b. Bull-O-Meter : Predicting the Quality of Natural Language Responses. In HComp'15 Proceedings of the AAAI Conference on Human Computation: Works in Progress and Demonstration Abstracts. San Diego, CA, USA."},{"key":"e_1_3_2_1_29_1","volume-title":"Proceeding of the KSS Workshop'15","author":"Krause Markus","year":"2015","unstructured":"Markus Krause . 2015 c. Stylometry-based Fraud and Plagiarism Detection for Learning at Scale . In Proceeding of the KSS Workshop'15 . Karlsruhe, Germany. Markus Krause. 2015c. Stylometry-based Fraud and Plagiarism Detection for Learning at Scale. In Proceeding of the KSS Workshop'15. Karlsruhe, Germany."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316447003000105"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505057"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2724660.2724670"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675283"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531651"},{"key":"e_1_3_2_1_35_1","volume-title":"Applied multivariate research: Design and interpretation","author":"Meyers LS","unstructured":"LS Meyers , G Gamst , and AJ Guarino . 2006. Applied multivariate research: Design and interpretation . Sage Publishers , Thousand Oaks, CA, USA . http:\/\/scholar.google.com\/scholar?hl=en LS Meyers, G Gamst, and AJ Guarino. 2006. Applied multivariate research: Design and interpretation. Sage Publishers, Thousand Oaks, CA, USA. http:\/\/scholar.google.com\/scholar?hl=en"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10459-010-9222-y"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11251-012-9236-3"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11251-015-9353-x"},{"key":"e_1_3_2_1_39_1","first-page":"3","article-title":"Assessing Student Learning Outcomes: An Investigation of the Relationship among Feedback Measures","volume":"37","author":"Rucker Mary L.","year":"2003","unstructured":"Mary L. Rucker and Stephanie Thomson . 2003 . Assessing Student Learning Outcomes: An Investigation of the Relationship among Feedback Measures . College Student Journal 37 , 3 (Sept. 2003), 400. Mary L. Rucker and Stephanie Thomson. 2003. Assessing Student Learning Outcomes: An Investigation of the Relationship among Feedback Measures. College Student Journal 37, 3 (Sept. 2003), 400.","journal-title":"College Student Journal"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00117714"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1002\/jaal.525"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958865"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.4324\/9780203122761"},{"key":"e_1_3_2_1_44_1","first-page":"29","article-title":"CritViz: Web-based software supporting peer critique in large creative classrooms","volume":"15","author":"Tinapple David","year":"2013","unstructured":"David Tinapple , Loren Olson , and John Sadauskas . 2013 . CritViz: Web-based software supporting peer critique in large creative classrooms . Bulletin of the IEEE Technical Committee on Learning Technology 15 , 1 (2013), 29 . http:\/\/www.ieeetclt.org\/issues\/ january2013\/Tinapple.pdf David Tinapple, Loren Olson, and John Sadauskas. 2013. CritViz: Web-based software supporting peer critique in large creative classrooms. Bulletin of the IEEE Technical Committee on Learning Technology 15, 1 (2013), 29. http:\/\/www.ieeetclt.org\/issues\/ january2013\/Tinapple.pdf","journal-title":"Bulletin of the IEEE Technical Committee on Learning Technology"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124960"},{"key":"e_1_3_2_1_46_1","volume-title":"https:\/\/www.upwork.com\/","year":"2016","unstructured":"UpWork. UpWork. ( 2016 ). https:\/\/www.upwork.com\/ UpWork. UpWork. (2016). https:\/\/www.upwork.com\/"},{"key":"e_1_3_2_1_47_1","volume-title":"An overview of current research on automated essay grading. Journal of Information Technology Education 2","author":"Valenti Salvatore","year":"2003","unstructured":"Salvatore Valenti , Francesca Neri , and Alessandro Cucchiarelli . 2003. An overview of current research on automated essay grading. Journal of Information Technology Education 2 ( 2003 ). http:\/\/citeseerx.ist.psu.edu\/viewdoc\/download?doi=10.1.1.101.5757 Salvatore Valenti, Francesca Neri, and Alessandro Cucchiarelli. 2003. An overview of current research on automated essay grading. Journal of Information Technology Education 2 (2003). http:\/\/citeseerx.ist.psu.edu\/viewdoc\/download?doi=10.1.1.101.5757"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1080\/1470329032000103816"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.17239\/jowr-2012.04.02.3"},{"key":"e_1_3_2_1_50_1","volume-title":"IUNLPBEA '11 Proceedings of the 6th Workshop on Innovative Use of NLP for Building Educational Applications. Association for Computational Linguistics","author":"Xiong Wenting","year":"2043","unstructured":"Wenting Xiong and Diane J. Litman . 2011. Understanding Differences in Perceived Peer-Review Helpfulness using Natural Language Processing . In IUNLPBEA '11 Proceedings of the 6th Workshop on Innovative Use of NLP for Building Educational Applications. Association for Computational Linguistics , Stroudsburg, PA, USA, 10--19. http:\/\/dl. acm.org\/citation.cfm?id= 2043 132&picked=prox Wenting Xiong and Diane J. Litman. 2011. Understanding Differences in Perceived Peer-Review Helpfulness using Natural Language Processing. In IUNLPBEA '11 Proceedings of the 6th Workshop on Innovative Use of NLP for Building Educational Applications. Association for Computational Linguistics, Stroudsburg, PA, USA, 10--19. http:\/\/dl. acm.org\/citation.cfm?id=2043132&picked=prox"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145252"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531604"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675140"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2819953"},{"key":"e_1_3_2_1_55_1","unstructured":"ZURB. Forrst. (2015). http:\/\/zurb.com\/forrst  ZURB. Forrst. (2015). http:\/\/zurb.com\/forrst"}],"event":{"name":"CHI '17: CHI Conference on Human Factors in Computing Systems","location":"Denver Colorado USA","acronym":"CHI '17","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3025453.3025883","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3025453.3025883","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3025453.3025883","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:03:39Z","timestamp":1750215819000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3025453.3025883"}},"subtitle":["Improving Crowdsourced Design Feedback with a Natural Language Model"],"short-title":[],"issued":{"date-parts":[[2017,5,2]]},"references-count":54,"alternative-id":["10.1145\/3025453.3025883","10.1145\/3025453"],"URL":"https:\/\/doi.org\/10.1145\/3025453.3025883","relation":{},"subject":[],"published":{"date-parts":[[2017,5,2]]},"assertion":[{"value":"2017-05-02","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}