{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T18:59:13Z","timestamp":1776106753236,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,2,27]],"date-time":"2016-02-27T00:00:00Z","timestamp":1456531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001655","name":"German Academic Exchange Service","doi-asserted-by":"publisher","award":["FIT Worldwide Program"],"award-info":[{"award-number":["FIT Worldwide Program"]}],"id":[{"id":"10.13039\/501100001655","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NSF","award":["1210836, 1208382, 1217096, and 1122206"],"award-info":[{"award-number":["1210836, 1208382, 1217096, and 1122206"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,2,27]]},"DOI":"10.1145\/2818048.2819953","type":"proceedings-article","created":{"date-parts":[[2016,2,29]],"date-time":"2016-02-29T11:40:27Z","timestamp":1456746027000},"page":"1005-1017","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":89,"title":["Almost an Expert"],"prefix":"10.1145","author":[{"given":"Alvin","family":"Yuan","sequence":"first","affiliation":[{"name":"University of California, Berkeley"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kurt","family":"Luther","sequence":"additional","affiliation":[{"name":"Virginia Tech"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markus","family":"Krause","sequence":"additional","affiliation":[{"name":"University of California, Berkeley"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sophie Isabel","family":"Vennix","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Steven P","family":"Dow","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bjorn","family":"Hartmann","sequence":"additional","affiliation":[{"name":"University of California, Berkeley"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2016,2,27]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","unstructured":"Steven Bird Ewan Klein and Edward Loper. 2009. Natural Language Processing with Python. DOI:http: \/\/dx.doi.org\/10.1097\/00004770-200204000-00018","DOI":"10.1097\/00004770-200204000-00018"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2923.2008.03172.x"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1067445.1067468"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1177\/0741088306289261"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1476-8070.2009.01591.x"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979359"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470686"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1640233.1640260"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753688"},{"key":"e_1_3_2_1_12_1","volume-title":"The role of deliberate practice in the acquisition of expert performance. Psychological Review","author":"Ericsson K. Anders","year":"1993","unstructured":"K. Anders Ericsson, Ralf Th. Krampe, and Clemens Tesch-R\u00f6mer. 1993. The role of deliberate practice in the acquisition of expert performance. Psychological Review (1993), 363\u2013406."},{"key":"e_1_3_2_1_13_1","unstructured":"Feedback Army. Website Usability Testing Service Feedback Army. (2015). http:\/\/www.feedbackarmy.com\/"},{"key":"e_1_3_2_1_14_1","unstructured":"Edmund Burke Feldman. 1994. Practical Art Criticism. Pearson Englewood Cliffs N.J."},{"key":"e_1_3_2_1_15_1","volume-title":"WordNet: An Electronic Lexical Database","author":"Fellbaum Christiane","unstructured":"Christiane Fellbaum. 1998. WordNet: An Electronic Lexical Database. Bradford Books."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.5555\/164632.164891"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.58680\/rte197220151"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2757226.2757249"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"M. R. Harwell E. N. Rubinstein W. S. Hayes and C. C. Olds. Summarizing Monte Carlo Results in Methodological Research: The One- and Two-Factor Fixed Effects ANOVA Cases. (1992). DOI: http:\/\/dx.doi.org\/10.3102\/10769986017004315","DOI":"10.3102\/10769986017004315"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.3102\/003465430298487"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/5254.889104"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.86.6.1232"},{"key":"e_1_3_2_1_23_1","volume-title":"Predicting Crowd-based Translation Quality with Language-independent Feature Vectors. In HComp'12 Proceedings of the AAAI Workshop on Human Computation. AAAI Press","author":"Kilian Niklas","year":"2012","unstructured":"Niklas Kilian, Markus Krause, Nina Runge, and Jan Smeddinck. 2012. Predicting Crowd-based Translation Quality with Language-independent Feature Vectors. In HComp'12 Proceedings of the AAAI Workshop on Human Computation. AAAI Press, Toronto, ON, Canada, 114\u2013115. http:\/\/www.aaai.org\/ocs\/index.php\/WS\/AAAIW12\/ paper\/viewPDFInterstitial\/5237\/5611"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702507"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1142405.1142429"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556325.2567881"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505057"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation. 17\u201320","author":"Le John","year":"2010","unstructured":"John Le, Andy Edmonds, Vaughn Hester, and Lukas Biewald. 2010. Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution. In Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation. 17\u201320. http:\/\/ir.ischool.utexas.edu\/cse2010\/ materials\/leetal.pdf"},{"key":"e_1_3_2_1_29_1","first-page":"1","article-title":"Four aspects of strategic change: contributions to children's learning of multiplication. Journal of Experimental Psychology","volume":"124","author":"Lemaire P.","year":"1995","unstructured":"P. Lemaire and R. S. Siegler. 1995. Four aspects of strategic change: contributions to children's learning of multiplication. Journal of Experimental Psychology. General 124, 1 (March 1995), 83\u201397.","journal-title":"General"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675283"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531651"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1809400.1809422"},{"key":"e_1_3_2_1_33_1","volume-title":"Number 8474 in Lecture Notes in Computer Science","author":"Nguyen Huy","unstructured":"Huy Nguyen, Wenting Xiong, and Diane Litman. 2014. Classroom Evaluation of a Scaffolding Intervention for Improving Peer Review Localization. In Intelligent Tutoring Systems, Stefan Trausan-Matu, Kristy Elizabeth Boyer, Martha Crosby, and Kitty Panourgia (Eds.). Number 8474 in Lecture Notes in Computer Science. Springer International Publishing, 272\u2013282. http:\/\/link.springer.com\/chapter\/10. 1007\/978-3-319-07221-0_34"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702470"},{"key":"e_1_3_2_1_35_1","first-page":"3","article-title":"Assessing Student Learning Outcomes: An Investigation of the Relationship among Feedback Measures","volume":"37","author":"Rucker Mary L.","year":"2003","unstructured":"Mary L. Rucker and Stephanie Thomson. 2003. Assessing Student Learning Outcomes: An Investigation of the Relationship among Feedback Measures. College Student Journal 37, 3 (Sept. 2003), 400.","journal-title":"College Student Journal"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00117714"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1027\/1614-2241\/a000016"},{"key":"e_1_3_2_1_38_1","unstructured":"Donald A. Sch\u00f6n. 1985. The Design Studio: An Exploration of Its Traditions and Potentials. Riba-Publ."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.3758\/BF03193370"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958865"},{"key":"e_1_3_2_1_41_1","first-page":"29","article-title":"CritViz: Web-based software supporting peer critique in large creative classrooms","volume":"15","author":"Tinapple David","year":"2013","unstructured":"David Tinapple, Loren Olson, and John Sadauskas. 2013. CritViz: Web-based software supporting peer critique in large creative classrooms. Bulletin of the IEEE Technical Committee on Learning Technology 15, 1 (2013), 29. http:\/\/www.ieeetclt.org\/issues\/ january2013\/Tinapple.pdf","journal-title":"Bulletin of the IEEE Technical Committee on Learning Technology"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124960"},{"key":"e_1_3_2_1_43_1","unstructured":"Upwork. Upwork the world's largest online workplace. (2015). https:\/\/www.upwork.com"},{"key":"e_1_3_2_1_44_1","unstructured":"UsabilityHub. Five Second Test. (2015). http:\/\/fivesecondtest.com\/"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1080\/1470329032000103816"},{"key":"e_1_3_2_1_46_1","unstructured":"P.H. Winne and D. L. Butler. 1994. Student cognition in learning from teaching. In International encyclopaedia of education (2 ed.) T. Husen and T. Postlewaite (Eds.). Pergamon Oxford UK 5738\u20135745."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.5555\/2043132.2043134"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145252"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531604"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675140"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145269"},{"key":"e_1_3_2_1_52_1","unstructured":"ZURB. Forrst. (2015). http:\/\/zurb.com\/forrst"}],"event":{"name":"CSCW '16: Computer Supported Cooperative Work and Social Computing","location":"San Francisco California USA","acronym":"CSCW '16","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 19th ACM Conference on Computer-Supported Cooperative Work & Social Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2818048.2819953","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2818048.2819953","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2818048.2819953","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:24:38Z","timestamp":1763457878000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2818048.2819953"}},"subtitle":["The Effects of Rubrics and Expertise on Perceived Value of Crowdsourced Design Critiques"],"short-title":[],"issued":{"date-parts":[[2016,2,27]]},"references-count":52,"alternative-id":["10.1145\/2818048.2819953","10.1145\/2818048"],"URL":"https:\/\/doi.org\/10.1145\/2818048.2819953","relation":{},"subject":[],"published":{"date-parts":[[2016,2,27]]},"assertion":[{"value":"2016-02-27","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}