{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T15:54:52Z","timestamp":1776095692765,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":59,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,2,2]],"date-time":"2018-02-02T00:00:00Z","timestamp":1517529600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,2,2]]},"DOI":"10.1145\/3159652.3159654","type":"proceedings-article","created":{"date-parts":[[2018,2,6]],"date-time":"2018-02-06T18:12:23Z","timestamp":1517940743000},"page":"162-170","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":100,"title":["Cognitive Biases in Crowdsourcing"],"prefix":"10.1145","author":[{"given":"Carsten","family":"Eickhoff","sequence":"first","affiliation":[{"name":"ETH Zurich, Zurich, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2018,2,2]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983716"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1086\/261849"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010039"},{"key":"e_1_3_2_1_5_1","volume-title":"Social cognition: How individuals construct social reality","author":"Bless Herbert"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","unstructured":"Ben Carterette Paul Bennett David Chickering and Susan Dumais . 2008. Here or there. Advances in Information Retrieval(2008) 16--27.   Ben Carterette Paul Bennett David Chickering and Susan Dumais . 2008. Here or there. Advances in Information Retrieval(2008) 16--27.","DOI":"10.1007\/978-3-540-78646-7_5"},{"key":"e_1_3_2_1_7_1","first-page":"11","article-title":"Labor allocation in paid crowdsourcing: Experimental evidence on positioning, nudges and prices","volume":"11","author":"Chandler Dana","year":"2011","journal-title":"Human Computation"},{"key":"e_1_3_2_1_8_1","volume-title":"Readings in Information Retrieval. Readings in Information Retrieval, bibfieldeditor","author":"Cyril Cleverdon"},{"key":"e_1_3_2_1_9_1","volume-title":"TREC 2013 web track overview. Technical Report. MICHIGAN UNIV ANN ARBOR.","author":"Collins-Thompson Kevyn","year":"2014"},{"key":"e_1_3_2_1_10_1","volume-title":"TREC 2014 web track overview. Technical Report. MICHIGAN UNIV ANN ARBOR.","author":"Collins-Thompson Kevyn","year":"2015"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806460"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"crossref","unstructured":"Alexander Philip Dawid and Allan M Skene . 1979. Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied statistics(1979) 20--28.  Alexander Philip Dawid and Allan M Skene . 1979. Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied statistics(1979) 20--28.","DOI":"10.2307\/2346806"},{"key":"e_1_3_2_1_13_1","volume-title":"Scaling-up the Crowd: Micro-Task Pricing Schemes for Worker Retention and Latency Improvement Second AAAI Conference on Human Computation and Crowdsourcing.","author":"Difallah Djellel Eddine","year":"2014"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488421"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2594776.2594788"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the workshop on crowdsourcing for search and data mining(CSDM) at the fourth ACM international conference on web search and data mining(WSDM). 11--14","author":"Eickhoff Carsten","year":"2011"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-011-9181-9"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348400"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Daniel Ellsberg . 1961. Risk ambiguity and the Savage axioms. The quarterly journal of economics(1961) 643--669.  Daniel Ellsberg . 1961. Risk ambiguity and the Savage axioms. The quarterly journal of economics(1961) 643--669.","DOI":"10.2307\/1884324"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v2i1.13145"},{"key":"e_1_3_2_1_21_1","unstructured":"Marvin Fleischmann Miglena Amirpur Alexander Benlian and Thomas Hess . 2014. Cognitive biases in information systems research: a scientometric analysis. (2014).  Marvin Fleischmann Miglena Amirpur Alexander Benlian and Thomas Hess . 2014. Cognitive biases in information systems research: a scientometric analysis. (2014)."},{"key":"e_1_3_2_1_22_1","volume-title":"Crowdsourcing document relevance assessment with mechanical turk Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon's mechanical turk","author":"Grady Catherine"},{"key":"e_1_3_2_1_23_1","volume-title":"ETH Zurich at TREC Clinical Decision Support","author":"Greuter Simon","year":"2016"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2983323.2983769"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2945386"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/146565.146567"},{"key":"e_1_3_2_1_27_1","unstructured":"Martie G Haselton Daniel Nettle and Damian R Murray . 2005. The evolution of cognitive bias. The handbook of evolutionary psychology(2005).  Martie G Haselton Daniel Nettle and Damian R Murray . 2005. The evolution of cognitive bias. The handbook of evolutionary psychology(2005)."},{"key":"e_1_3_2_1_28_1","volume-title":"Cheat-detection mechanisms for crowdsourcing","author":"Hirth Matthias","year":"2010"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-28997-2_16"},{"key":"e_1_3_2_1_30_1","volume-title":"Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business(bibinfoedition1 ed.)","author":"Jeff Howe","year":"2008"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1086\/208899"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1982624.1982625"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1963192.1963314"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1287\/opre.2013.1235"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009947"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063860"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9205-0"},{"key":"e_1_3_2_1_38_1","unstructured":"Gabriella Kazai and Natasa Milic-Frayling . 2009. On the evaluation of the quality of relevance assessments collected through crowdsourcing SIGIR 2009 Workshop on the Future of IR Evaluation. 21.  Gabriella Kazai and Natasa Milic-Frayling . 2009. On the evaluation of the quality of relevance assessments collected through crowdsourcing SIGIR 2009 Workshop on the Future of IR Evaluation. 21."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_2_1_40_1","first-page":"11","article-title":"On Quality Control and Machine Learning in Crowdsourcing","volume":"11","author":"Matthew Lease","year":"2011","journal-title":"Human Computation"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/1998076.1998108"},{"key":"e_1_3_2_1_42_1","unstructured":"Rohan Ramanath Monojit Choudhury Kalika Bali and Rishiraj Saha Roy . 2013. Crowd Prefers the Middle Path: A New IAA Metric for Crowdsourcing Reveals Turker Biases in Query Segmentation.. In ACL(1). 1713--1722.  Rohan Ramanath Monojit Choudhury Kalika Bali and Rishiraj Saha Roy . 2013. Crowd Prefers the Middle Path: A New IAA Metric for Crowdsourcing Reveals Turker Biases in Query Segmentation.. In ACL(1). 1713--1722."},{"key":"e_1_3_2_1_43_1","volume-title":"Overview of the TREC 2016 Clinical Decision Support Track. TREC.","author":"Roberts Kirk","year":"2016"},{"key":"e_1_3_2_1_44_1","volume-title":"Overview of the TREC 2015 Clinical Decision Support Track. TREC.","author":"Roberts Kirk","year":"2015"},{"key":"e_1_3_2_1_45_1","first-page":"109","article-title":"Okapi at TREC-3","volume":"109","author":"Robertson Stephen E","year":"1995","journal-title":"Nist Special Publication Sp"},{"key":"e_1_3_2_1_46_1","unstructured":"Marta Sabou Kalina Bontcheva Leon Derczynski and Arno Scharl . 2014. Corpus Annotation through Crowdsourcing: Towards Best Practice Guidelines. LREC. 859--866.  Marta Sabou Kalina Bontcheva Leon Derczynski and Arno Scharl . 2014. Corpus Annotation through Crowdsourcing: Towards Best Practice Guidelines. LREC. 859--866."},{"key":"e_1_3_2_1_48_1","unstructured":"Wei Tang and Matthew Lease . 2011. Semi-supervised consensus labeling for crowdsourcing SIGIR 2011 workshop on crowdsourcing for information retrieval(CIR).  Wei Tang and Matthew Lease . 2011. Semi-supervised consensus labeling for crowdsourcing SIGIR 2011 workshop on crowdsourcing for information retrieval(CIR)."},{"key":"e_1_3_2_1_49_1","volume-title":"Judgment under uncertainty: Heuristics and biases. Utility, probability, and human decision making","author":"Tversky Amos"},{"key":"e_1_3_2_1_50_1","volume-title":"TREC: Experiment and evaluation in information retrieval.","author":"Voorhees Ellen M","year":"2005"},{"key":"e_1_3_2_1_51_1","volume-title":"ACM SIGIR Workshop on Crowdsourcing for Information Retrieval(CIR'11)","author":"Vuurens Jeroen","year":"2011"},{"key":"e_1_3_2_1_52_1","unstructured":"Jing Wang Siamak Faridani and Panagiotis G Ipeirotis . 2011. Estimating the Completion Time of Crowdsourced Tasks Using Survival Analysis Models. Crowdsourcing for Search and Data Mining(CSDM 2011)(2011) 31.  Jing Wang Siamak Faridani and Panagiotis G Ipeirotis . 2011. Estimating the Completion Time of Crowdsourced Tasks Using Survival Analysis Models. Crowdsourcing for Search and Data Mining(CSDM 2011)(2011) 31."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2534169.2491719"},{"key":"e_1_3_2_1_54_1","unstructured":"Fabian L Wauthier and Michael I Jordan . 2011. Bayesian bias mitigation for crowdsourcing. In Advances in neural information processing systems. 1800--1808.   Fabian L Wauthier and Michael I Jordan . 2011. Bayesian bias mitigation for crowdsourcing. In Advances in neural information processing systems. 1800--1808."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484053"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-009-9112-1"},{"key":"e_1_3_2_1_57_1","volume-title":"Proceedings of the 28th international conference on machine learning(ICML-11)","author":"Yan Yan","year":"2011"},{"key":"e_1_3_2_1_58_1","volume-title":"Conf. on Machine Learning. 1--7.","author":"Ye Peng","year":"2013"},{"key":"e_1_3_2_1_59_1","unstructured":"Dongqing Zhu and Ben Carterette . 2010. An analysis of assessor behavior in crowdsourced preference judgments SIGIR 2010 workshop on crowdsourcing for search evaluation. 17--20.  Dongqing Zhu and Ben Carterette . 2010. An analysis of assessor behavior in crowdsourced preference judgments SIGIR 2010 workshop on crowdsourcing for search evaluation. 17--20."},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685301"}],"event":{"name":"WSDM 2018: The Eleventh ACM International Conference on Web Search and Data Mining","location":"Marina Del Rey CA USA","acronym":"WSDM 2018","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3159652.3159654","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3159652.3159654","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T02:26:11Z","timestamp":1750213571000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3159652.3159654"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,2,2]]},"references-count":59,"alternative-id":["10.1145\/3159652.3159654","10.1145\/3159652"],"URL":"https:\/\/doi.org\/10.1145\/3159652.3159654","relation":{},"subject":[],"published":{"date-parts":[[2018,2,2]]},"assertion":[{"value":"2018-02-02","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}