{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:00:52Z","timestamp":1784178052704,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,3,14]],"date-time":"2022-03-14T00:00:00Z","timestamp":1647216000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,3,14]]},"DOI":"10.1145\/3498366.3505815","type":"proceedings-article","created":{"date-parts":[[2022,3,12]],"date-time":"2022-03-12T00:21:06Z","timestamp":1647044466000},"page":"25-35","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["The Crowd is Made of People"],"prefix":"10.1145","author":[{"given":"Paul","family":"Thomas","sequence":"first","affiliation":[{"name":"Microsoft, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gabriella","family":"Kazai","sequence":"additional","affiliation":[{"name":"Microsoft, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ryen","family":"White","sequence":"additional","affiliation":[{"name":"Microsoft, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nick","family":"Craswell","sequence":"additional","affiliation":[{"name":"Microsoft, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,3,14]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-20161-5_16"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Omar Alonso Daniel\u00a0E Rose and Benjamin Stewart. 2008. Crowdsourcing for relevance evaluation. In ACM SIGIR Forum Vol.\u00a042. 9\u201315.","DOI":"10.1145\/1480506.1480508"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052637"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3406522.3446023"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1840784.1840801"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v067.i01"},{"key":"e_1_3_2_1_8_1","first-page":"18","article-title":"Cognitive biases and their effects on information behaviour of graduate students in their research projects","volume":"8","author":"Behimehr Sara","year":"2020","unstructured":"Sara Behimehr and Hamid\u00a0R. Jamali. 2020. Cognitive biases and their effects on information behaviour of graduate students in their research projects. Journal of Information Science Theory and Practice 8, 2 (2020), 18\u201331.","journal-title":"Journal of Information Science Theory and Practice"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.appet.2014.02.014"},{"key":"e_1_3_2_1_10_1","unstructured":"Buster Benson. 2016. Cognitive bias cheat sheet. https:\/\/betterhumans.pub\/cognitive-bias-cheat-sheet-55a472476b18. Downloaded: 2022-01-08."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.106.2.265"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858237"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835540"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2012.95"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159661"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v9i1.18939"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159654"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348400"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1002\/bdm.495"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1257\/app.20160390"},{"key":"e_1_3_2_1_21_1","volume-title":"Proceedings of the European Conference on Information Systems.","author":"Fleischmann Marvin","year":"2014","unstructured":"Marvin Fleischmann, Miglena Amirpur, Alexander Benlian, and Thomas Hess. 2014. Cognitive biases in information systems research: A scientometric analysis. In Proceedings of the European Conference on Information Systems."},{"key":"e_1_3_2_1_22_1","first-page":"114","article-title":"Rating the rating scales","volume":"9","author":"Friedman H","year":"1999","unstructured":"Hershey\u00a0H Friedman and Taiwo Amoo. 1999. Rating the rating scales. Journal of Marketing Management 9, 3 (1999), 114\u2013123.","journal-title":"Journal of Marketing Management"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3027385.3027402"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1037\/a0021378"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.1202775"},{"key":"e_1_3_2_1_26_1","volume-title":"Proceedings of the ACM SIGCHI Conference on Human Factors in Computing Systems. 113\u2013120","author":"M.","unstructured":"Victor\u00a0M. Gonz\u00e1lez and Gloria Mark. 2004. Constant, constant, multi-tasking craziness. In Proceedings of the ACM SIGCHI Conference on Human Factors in Computing Systems. 113\u2013120."},{"key":"e_1_3_2_1_27_1","unstructured":"Google. 2020. Search quality evaluator guidelines. https:\/\/static.googleusercontent.com\/media\/guidelines.raterhub.com\/en\/\/searchqualityevaluatorguidelines.pdf. Downloaded 2021-10-11."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2928269"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2370216.2370264"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-29381-9_28"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1986.10478354"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300637"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v6i1.13327"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-20161-5_17"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2396779"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v082.i13"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300649"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-013-9222-7"},{"key":"e_1_3_2_1_40_1","unstructured":"Jim Lewis and Jeff Sauro. 2020. Revisiting the evidence for the left-side bias in rating scales. https:\/\/measuringu.com\/revisiting-the-left-side-bias\/. Downloaded 2021-09-09."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.5555\/3542805.3542809"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858202"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10606-015-9241-6"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858490"},{"key":"e_1_3_2_1_47_1","volume-title":"Proceedings of the International Conference on Web Engineering. 381\u2013396","author":"Qiu Sihang","year":"2020","unstructured":"Sihang Qiu, Ujwal Gadiraju, and Alessandro Bozzon. 2020. Just the right mood for HIT! Analyzing the role of worker moods in conversational microtask crowdsourcing. In Proceedings of the International Conference on Web Engineering. 381\u2013396."},{"key":"e_1_3_2_1_48_1","volume-title":"R: A Language and Environment for Statistical Computing","author":"Team R Core","year":"2020","unstructured":"R Core Team. 2020. R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria. https:\/\/www.R-project.org\/"},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the ACM SIGCHI Conference on Human Factors in Computing Systems. 2863\u20132872","author":"Ross Joel","year":"2010","unstructured":"Joel Ross, Lilly Irani, M.\u00a0Six Silberman, Andrew Zaldivar, and Bill Tomlinson. 2010. Who are the crowdworkers? Shifting demographics in Mechanical Turk. In Proceedings of the ACM SIGCHI Conference on Human Factors in Computing Systems. 2863\u20132872."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20682"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the ASIS&T Pre-conference on the History of ASIS&T and Information Science and Technology. 49\u201360","author":"Saracevic Tefko","year":"2012","unstructured":"Tefko Saracevic. 2012. Research on relevance in information science: A historical perspective. In Proceedings of the ASIS&T Pre-conference on the History of ASIS&T and Information Science and Technology. 49\u201360."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484090"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010057"},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767841"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183632"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10902-018-0035-7"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.185.4157.1124"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1037\/0096-3445.125.4.387"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSCWD.2018.8465305"}],"event":{"name":"CHIIR '22: ACM SIGIR Conference on Human Information Interaction and Retrieval","location":"Regensburg Germany","acronym":"CHIIR '22","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["ACM SIGIR Conference on Human Information Interaction and Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3498366.3505815","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3498366.3505815","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:10:05Z","timestamp":1750183805000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3498366.3505815"}},"subtitle":["Observations from Large-Scale Crowd Labelling"],"short-title":[],"issued":{"date-parts":[[2022,3,14]]},"references-count":60,"alternative-id":["10.1145\/3498366.3505815","10.1145\/3498366"],"URL":"https:\/\/doi.org\/10.1145\/3498366.3505815","relation":{},"subject":[],"published":{"date-parts":[[2022,3,14]]},"assertion":[{"value":"2022-03-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}