{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,3]],"date-time":"2026-01-03T15:13:42Z","timestamp":1767453222587,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,4,12]],"date-time":"2021-04-12T00:00:00Z","timestamp":1618185600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,4,12]]},"DOI":"10.1145\/3448139.3448146","type":"proceedings-article","created":{"date-parts":[[2021,4,5]],"date-time":"2021-04-05T22:29:58Z","timestamp":1617661798000},"page":"66-76","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Transforming Everyday Information into Practical Analytics with Crowdsourced Assessment Tasks"],"prefix":"10.1145","author":[{"given":"June","family":"Ahn","sequence":"first","affiliation":[{"name":"University of California Irvine, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ha","family":"Nguyen","sequence":"additional","affiliation":[{"name":"University of California-Irvine, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fabio","family":"Campos","sequence":"additional","affiliation":[{"name":"New York University, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"William","family":"Young","sequence":"additional","affiliation":[{"name":"Competitive Intelligence Solutions, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2021,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Computer-Based Writing Instruction.Grantee Submission","author":"Allen K","year":"2016","unstructured":"Laura\u00a0 K Allen , Matthew\u00a0 E Jacovina , and Danielle\u00a0 S McNamara . 2016. Computer-Based Writing Instruction.Grantee Submission ( 2016 ). Laura\u00a0K Allen, Matthew\u00a0E Jacovina, and Danielle\u00a0S McNamara. 2016. Computer-Based Writing Instruction.Grantee Submission (2016)."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.2200\/S00904ED1V01Y201903ICR066"},{"volume-title":"Towards a Comprehensive Online Peer Assessment System","author":"Babik Dmytro","key":"e_1_3_2_1_3_1","unstructured":"Dmytro Babik , Lakshmi\u00a0 S. Iyer , and Eric\u00a0 W. Ford . 2012. Towards a Comprehensive Online Peer Assessment System . Springer , Berlin, Heidelberg , 1\u20138. https:\/\/doi.org\/10.1007\/978-3-642-29863-9_1 Dmytro Babik, Lakshmi\u00a0S. Iyer, and Eric\u00a0W. Ford. 2012. Towards a Comprehensive Online Peer Assessment System. Springer, Berlin, Heidelberg, 1\u20138. https:\/\/doi.org\/10.1007\/978-3-642-29863-9_1"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300773"},{"volume-title":"Learning to improve: how America\u2019s schools can get better at getting better","author":"Bryk S.","key":"e_1_3_2_1_5_1","unstructured":"Anthony\u00a0 S. Bryk , Louis\u00a0 M. Gomez , Alicia Grunow , and Paul\u00a0 G. LeMahieu . 2015. Learning to improve: how America\u2019s schools can get better at getting better . Harvard Education Press , Cambridge, MA . Anthony\u00a0S. Bryk, Louis\u00a0M. Gomez, Alicia Grunow, and Paul\u00a0G. LeMahieu. 2015. Learning to improve: how America\u2019s schools can get better at getting better. Harvard Education Press, Cambridge, MA."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173868"},{"key":"e_1_3_2_1_7_1","unstructured":"Daniel Cer Yinfei Yang Sheng-yi Kong Nan Hua Nicole Limtiaco Rhomni\u00a0St John Noah Constant Mario Guajardo-Cespedes Steve Yuan and Chris Tar. 2018. Universal sentence encoder. arXiv preprint arXiv:1803.11175(2018).  Daniel Cer Yinfei Yang Sheng-yi Kong Nan Hua Nicole Limtiaco Rhomni\u00a0St John Noah Constant Mario Guajardo-Cespedes Steve Yuan and Chris Tar. 2018. Universal sentence encoder. arXiv preprint arXiv:1803.11175(2018)."},{"key":"e_1_3_2_1_8_1","unstructured":"J. Chung M.\u00a0A. Cannady C. Schunn R. Dorph and P. Vincent-Ruz. 2016. Measures Technical Brief: Competency Beliefs in Science. Technical Report. Activation Lab.  J. Chung M.\u00a0A. Cannady C. Schunn R. Dorph and P. Vincent-Ruz. 2016. Measures Technical Brief: Competency Beliefs in Science. Technical Report. Activation Lab."},{"key":"e_1_3_2_1_9_1","volume-title":"Quality Control in Crowdsourcing. Comput. Surveys 51, 1 (1","author":"Daniel Florian","year":"2018","unstructured":"Florian Daniel , Pavel Kucherbaev , Cinzia Cappiello , Boualem Benatallah , and Mohammad Allahbakhsh . 2018. Quality Control in Crowdsourcing. Comput. Surveys 51, 1 (1 2018 ), 1\u201340. https:\/\/doi.org\/10.1145\/3148148 Florian Daniel, Pavel Kucherbaev, Cinzia Cappiello, Boualem Benatallah, and Mohammad Allahbakhsh. 2018. Quality Control in Crowdsourcing. Comput. Surveys 51, 1 (1 2018), 1\u201340. https:\/\/doi.org\/10.1145\/3148148"},{"key":"e_1_3_2_1_10_1","volume-title":"CrowdGrader. In Proceedings of the 45th ACM technical symposium on Computer science education - SIGCSE \u201914","author":"de Alfaro Luca","year":"2014","unstructured":"Luca de Alfaro and Michael Shavlovsky . 2014 . CrowdGrader. In Proceedings of the 45th ACM technical symposium on Computer science education - SIGCSE \u201914 . ACM Press, New York, New York, USA, 415\u2013420. https:\/\/doi.org\/10.1145\/2538862.2538900 Luca de Alfaro and Michael Shavlovsky. 2014. CrowdGrader. In Proceedings of the 45th ACM technical symposium on Computer science education - SIGCSE \u201914. ACM Press, New York, New York, USA, 415\u2013420. https:\/\/doi.org\/10.1145\/2538862.2538900"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858268"},{"key":"e_1_3_2_1_12_1","volume-title":"Proceedings of International Conference of the Learning Sciences, ICLS.","author":"Doroudi Shayan","year":"2018","unstructured":"Shayan Doroudi , Joseph\u00a0Jay Williams , Juho Kim , Thanaporn Patikorn , Korinn\u00a0 S. Ostrow , Douglas Selent , Neil\u00a0 T. Heffernan , Thomas Hills , and Carolyn\u00a0 P. Ros\u00e9 . 2018 . Crowdsourcing and education: Towards a theory and praxis of learnersourcing . In Proceedings of International Conference of the Learning Sciences, ICLS. Shayan Doroudi, Joseph\u00a0Jay Williams, Juho Kim, Thanaporn Patikorn, Korinn\u00a0S. Ostrow, Douglas Selent, Neil\u00a0T. Heffernan, Thomas Hills, and Carolyn\u00a0P. Ros\u00e9. 2018. Crowdsourcing and education: Towards a theory and praxis of learnersourcing. In Proceedings of International Conference of the Learning Sciences, ICLS."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2470654.2470686"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13270"},{"key":"e_1_3_2_1_16_1","volume-title":"Contributing conditions: A qualitative comparative analysis of teachers","author":"Farrell A.","year":"2016","unstructured":"Julie\u00a0 A. Farrell , Caitlin C.;\u00a0 Marsh . 2016. Contributing conditions: A qualitative comparative analysis of teachers \u2019 instructional responses to data. Teaching and Teacher Education 60 (11 2016 ), 398\u2013412. https:\/\/doi.org\/10.1016\/J.TATE.2016.07.010 Julie\u00a0A. Farrell, Caitlin C.;\u00a0Marsh. 2016. Contributing conditions: A qualitative comparative analysis of teachers\u2019 instructional responses to data. Teaching and Teacher Education 60 (11 2016), 398\u2013412. https:\/\/doi.org\/10.1016\/J.TATE.2016.07.010"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.3102\/0013189X11413260"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/s40593-014-0024-x"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.3102\/0002831215573773"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2014.6883049"},{"key":"e_1_3_2_1_21_1","volume-title":"Pacific Asia Conference on Information Systems. 180","author":"Jiang Yuchao","year":"2018","unstructured":"Yuchao Jiang , Daniel Schlagwein , and Boualem Benatallah . 2018 . A Review on Crowdsourcing for Education: State of the Art of Literature and Practice . In Pacific Asia Conference on Information Systems. 180 . https:\/\/aisel.aisnet.org\/pacis2018\/180 Yuchao Jiang, Daniel Schlagwein, and Boualem Benatallah. 2018. A Review on Crowdsourcing for Education: State of the Art of Literature and Practice. In Pacific Asia Conference on Information Systems. 180. https:\/\/aisel.aisnet.org\/pacis2018\/180"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3231644.3231666"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2556986"},{"volume-title":"Proceeding of the twenty-sixth annual CHI conference on Human factors in computing systems - CHI \u201908","author":"Kittur Aniket","key":"e_1_3_2_1_24_1","unstructured":"Aniket Kittur , Ed\u00a0 H. Chi , and Bongwon Suh . 2008. Crowdsourcing user studies with Mechanical Turk . In Proceeding of the twenty-sixth annual CHI conference on Human factors in computing systems - CHI \u201908 . ACM Press , New York , New York, USA, 453. https:\/\/doi.org\/10.1145\/1357054.1357127 Aniket Kittur, Ed\u00a0H. Chi, and Bongwon Suh. 2008. Crowdsourcing user studies with Mechanical Turk. In Proceeding of the twenty-sixth annual CHI conference on Human factors in computing systems - CHI \u201908. ACM Press, New York, New York, USA, 453. https:\/\/doi.org\/10.1145\/1357054.1357127"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1999747.1999793"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2876034.2893424"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18608\/jla.2016.32.6"},{"key":"e_1_3_2_1_28_1","volume-title":"Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning 5, 3 (6","author":"Law Edith","year":"2011","unstructured":"Edith Law and Luis\u00a0von Ahn . 2011. Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning 5, 3 (6 2011 ), 1\u2013121. https:\/\/doi.org\/10.2200\/S00371ED1V01Y201107AIM013 Edith Law and Luis\u00a0von Ahn. 2011. Human Computation. Synthesis Lectures on Artificial Intelligence and Machine Learning 5, 3 (6 2011), 1\u2013121. https:\/\/doi.org\/10.2200\/S00371ED1V01Y201107AIM013"},{"key":"e_1_3_2_1_29_1","volume-title":"What Is Improvement Science? Do We Need It in Education?Educational Researcher 44, 1 (1","author":"Lewis Catherine","year":"2015","unstructured":"Catherine Lewis . 2015. What Is Improvement Science? Do We Need It in Education?Educational Researcher 44, 1 (1 2015 ), 54\u201361. https:\/\/doi.org\/10.3102\/0013189X15570388 Catherine Lewis. 2015. What Is Improvement Science? Do We Need It in Education?Educational Researcher 44, 1 (1 2015), 54\u201361. https:\/\/doi.org\/10.3102\/0013189X15570388"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1002\/tl.47"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.bea-1.15"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702553"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3330430.3333626"},{"key":"e_1_3_2_1_34_1","volume-title":"Impact of computer modeling on learning and teaching systems thinking. Journal of Research in Science Teaching (10","author":"Nguyen Ha","year":"2020","unstructured":"Ha Nguyen and Rossella Santagata . 2020. Impact of computer modeling on learning and teaching systems thinking. Journal of Research in Science Teaching (10 2020 ), tea.21674. https:\/\/doi.org\/10.1002\/tea.21674 Ha Nguyen and Rossella Santagata. 2020. Impact of computer modeling on learning and teaching systems thinking. Journal of Research in Science Teaching (10 2020), tea.21674. https:\/\/doi.org\/10.1002\/tea.21674"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the 2011 annual conference on Human factors in computing systems - CHI \u201911","author":"J.","year":"1978","unstructured":"Alexander\u00a0 J. Quinn and Benjamin\u00a0B. Bederson. 2011. Human computation . In Proceedings of the 2011 annual conference on Human factors in computing systems - CHI \u201911 . ACM Press, New York, New York, USA, 1403. https:\/\/doi.org\/10.1145\/ 1978 942.1979148 Alexander\u00a0J. Quinn and Benjamin\u00a0B. Bederson. 2011. Human computation. In Proceedings of the 2011 annual conference on Human factors in computing systems - CHI \u201911. ACM Press, New York, New York, USA, 1403. https:\/\/doi.org\/10.1145\/1978942.1979148"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1037\/edu0000190"},{"key":"e_1_3_2_1_37_1","volume-title":"The Critical Thinking Analytic Rubric (CTAR): Investigating intra-rater and inter-rater reliability of a scoring mechanism for critical thinking performance assessments. Assessing Writing 17, 4 (10","author":"Saxton Emily","year":"2012","unstructured":"Emily Saxton , Secret Belanger , and William Becker . 2012. The Critical Thinking Analytic Rubric (CTAR): Investigating intra-rater and inter-rater reliability of a scoring mechanism for critical thinking performance assessments. Assessing Writing 17, 4 (10 2012 ), 251\u2013270. https:\/\/doi.org\/10.1016\/J.ASW.2012.07.002 Emily Saxton, Secret Belanger, and William Becker. 2012. The Critical Thinking Analytic Rubric (CTAR): Investigating intra-rater and inter-rater reliability of a scoring mechanism for critical thinking performance assessments. Assessing Writing 17, 4 (10 2012), 251\u2013270. https:\/\/doi.org\/10.1016\/J.ASW.2012.07.002"},{"key":"e_1_3_2_1_38_1","volume-title":"Data-based decision-making for school improvement: Research insights and gaps. Educational research 61, 3","author":"Schildkamp Kim","year":"2019","unstructured":"Kim Schildkamp . 2019. Data-based decision-making for school improvement: Research insights and gaps. Educational research 61, 3 ( 2019 ), 257\u2013273. Kim Schildkamp. 2019. Data-based decision-making for school improvement: Research insights and gaps. Educational research 61, 3 (2019), 257\u2013273."},{"key":"e_1_3_2_1_39_1","volume-title":"Data-informed curriculum reform: Which data, what purposes, and promoting and hindering factors. Teaching and Teacher Education 26, 3 (4","author":"Schildkamp Kim","year":"2010","unstructured":"Kim Schildkamp and Wilmad Kuiper . 2010. Data-informed curriculum reform: Which data, what purposes, and promoting and hindering factors. Teaching and Teacher Education 26, 3 (4 2010 ), 482\u2013496. https:\/\/doi.org\/10.1016\/J.TATE.2009.06.007 Kim Schildkamp and Wilmad Kuiper. 2010. Data-informed curriculum reform: Which data, what purposes, and promoting and hindering factors. Teaching and Teacher Education 26, 3 (4 2010), 482\u2013496. https:\/\/doi.org\/10.1016\/J.TATE.2009.06.007"},{"key":"e_1_3_2_1_40_1","volume-title":"Skaalvik and Sidsel Skaalvik","author":"M.","year":"2011","unstructured":"Einar\u00a0 M. Skaalvik and Sidsel Skaalvik . 2011 . Teacher job satisfaction and motivation to leave the teaching profession: Relations with school context, feeling of belonging, and emotional exhaustion. Teaching and Teacher Education 27, 6 (8 2011), 1029\u20131038. https:\/\/doi.org\/10.1016\/J.TATE.2011.04.001 Einar\u00a0M. Skaalvik and Sidsel Skaalvik. 2011. Teacher job satisfaction and motivation to leave the teaching profession: Relations with school context, feeling of belonging, and emotional exhaustion. Teaching and Teacher Education 27, 6 (8 2011), 1029\u20131038. https:\/\/doi.org\/10.1016\/J.TATE.2011.04.001"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1177\/003172170208301010"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1080\/13664530.2017.1308428"},{"volume-title":"Visual Thinking Strategies: Using Art to Deepen Learning Across School","author":"Yenawine Phillip","key":"e_1_3_2_1_44_1","unstructured":"Phillip Yenawine . 2013. Visual Thinking Strategies: Using Art to Deepen Learning Across School . Harvard Education Press , Cambridge . Phillip Yenawine. 2013. Visual Thinking Strategies: Using Art to Deepen Learning Across School. Harvard Education Press, Cambridge."}],"event":{"name":"LAK21: 11th International Learning Analytics and Knowledge Conference","acronym":"LAK21","location":"Irvine CA USA"},"container-title":["LAK21: 11th International Learning Analytics and Knowledge Conference"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3448139.3448146","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3448139.3448146","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:25:00Z","timestamp":1750195500000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3448139.3448146"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,12]]},"references-count":43,"alternative-id":["10.1145\/3448139.3448146","10.1145\/3448139"],"URL":"https:\/\/doi.org\/10.1145\/3448139.3448146","relation":{},"subject":[],"published":{"date-parts":[[2021,4,12]]},"assertion":[{"value":"2021-04-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}