{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,1]],"date-time":"2025-12-01T02:54:03Z","timestamp":1764557643684,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":69,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T00:00:00Z","timestamp":1650844800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,4,25]]},"DOI":"10.1145\/3485447.3512239","type":"proceedings-article","created":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T05:11:23Z","timestamp":1650863483000},"page":"1685-1696","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["The Influences of Task Design on Crowdsourced Judgement: A Case Study of Recidivism Risk Evaluation"],"prefix":"10.1145","author":[{"given":"Xiaoni","family":"Duan","sequence":"first","affiliation":[{"name":"Department of Computer Science, Purdue Univeristy, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chien-Ju","family":"Ho","sequence":"additional","affiliation":[{"name":"Department of Computer Science & Engineering, Washington University in St. Louis, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ming","family":"Yin","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Purdue University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,4,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557155"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1126\/sciadv.abf4393"},{"key":"e_1_3_2_1_3_1","volume-title":"Costs and benefits of judgment errors: Implications for debiasing.Psychological bulletin 110, 3","author":"Arkes R","year":"1991","unstructured":"Hal\u00a0R Arkes. 1991. Costs and benefits of judgment errors: Implications for debiasing.Psychological bulletin 110, 3 (1991), 486."},{"key":"e_1_3_2_1_4_1","volume-title":"Fairness in machine learning. Nips tutorial 1(2017)","author":"Barocas Solon","year":"2017","unstructured":"Solon Barocas, Moritz Hardt, and Arvind Narayanan. 2017. Fairness in machine learning. Nips tutorial 1(2017), 2017."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1995.tb02031.x"},{"key":"e_1_3_2_1_6_1","unstructured":"Sarah Bird Miro Dud\u00edk Richard Edgar Brandon Horn Roman Lutz Vanessa Milan Mehrnoosh Sameki Hanna Wallach and Kathleen Walker. 2020. Fairlearn: A toolkit for assessing and improving fairness in AI. Technical Report MSR-TR-2020-32. Microsoft. https:\/\/www.microsoft.com\/en-us\/research\/publication\/fairlearn-a-toolkit-for-assessing-and-improving-fairness-in-ai\/"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3378393.3402507"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDMW.2009.83"},{"key":"e_1_3_2_1_9_1","volume-title":"Semantics derived automatically from language corpora contain human-like biases. Science 356, 6334","author":"Caliskan Aylin","year":"2017","unstructured":"Aylin Caliskan, Joanna\u00a0J Bryson, and Arvind Narayanan. 2017. Semantics derived automatically from language corpora contain human-like biases. Science 356, 6334 (2017), 183\u2013186."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274306"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICTAI.2008.141"},{"key":"e_1_3_2_1_13_1","volume-title":"Suppl 2","author":"Croskerry Pat","year":"2013","unstructured":"Pat Croskerry, Geeta Singhal, and S\u00edlvia Mamede. 2013. Cognitive debiasing 1: origins of bias and theory of debiasing. BMJ quality & safety 22, Suppl 2 (2013), ii58\u2013ii64."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.2307\/2346806"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2538862.2538900"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.2517-6161.1977.tb01600.x"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13270"},{"key":"e_1_3_2_1_19_1","volume-title":"The accuracy, fairness, and limits of predicting recidivism. Science advances 4, 1","author":"Dressel Julia","year":"2018","unstructured":"Julia Dressel and Hany Farid. 2018. The accuracy, fairness, and limits of predicting recidivism. Science advances 4, 1 (2018), eaao5580."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v8i1.7474"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2090236.2090255"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159654"},{"key":"e_1_3_2_1_23_1","unstructured":"Jonathan St\u00a0BT Evans and Keith\u00a0Ed Frankish. 2009. In two minds: Dual processes and beyond.Oxford University Press."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2499149.2499168"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1257\/089533005775196732"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2021\/238"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_3_2_1_28_1","unstructured":"Ben Green and Yiling Chen. 2020. Algorithmic risk assessments can alter human decision-making processes in high-stakes government contexts. arXiv preprint arXiv:2012.05370(2020)."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3465622"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2998181.2998327"},{"key":"e_1_3_2_1_31_1","volume-title":"Equality of opportunity in supervised learning. Advances in neural information processing systems 29","author":"Hardt Moritz","year":"2016","unstructured":"Moritz Hardt, Eric Price, and Nati Srebro. 2016. Equality of opportunity in supervised learning. Advances in neural information processing systems 29 (2016), 3315\u20133323."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v15i1.18133"},{"key":"e_1_3_2_1_33_1","volume-title":"Proceedings of the 30th International Conference on Neural Information Processing Systems. 2450\u20132458","author":"Ho Chien-Ju","year":"2016","unstructured":"Chien-Ju Ho, Rafael Frongillo, and Yiling Chen. 2016. Eliciting categorical data for optimal aggregation. In Proceedings of the 30th International Conference on Neural Information Processing Systems. 2450\u20132458."},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 30th International Conference on Machine Learning. 534\u2013542","author":"Ho Chien-Ju","year":"2013","unstructured":"Chien-Ju Ho, Shahin Jabbari, and Jennifer\u00a0Wortman Vaughan. 2013. Adaptive task assignment for crowdsourced classification. In Proceedings of the 30th International Conference on Machine Learning. 534\u2013542."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741102"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.5555\/3013558.3013568"},{"key":"e_1_3_2_1_37_1","volume-title":"Proceedings of the 4th Human Computation Workshop.","author":"Ho Chien-Ju","year":"2012","unstructured":"Chien-Ju Ho, Yu Zhang, Jennifer\u00a0Wortman Vaughan, and Mihaela Van Der\u00a0Schaar. 2012. Towards social norm design for crowdsourcing markets. In Proceedings of the 4th Human Computation Workshop."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/1807342.1807376"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445433"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300637"},{"key":"e_1_3_2_1_41_1","volume-title":"The limits of human predictions of recidivism. Science advances 6, 7","author":"Jung Jongbin","year":"2020","unstructured":"Jongbin Jung, Sharad Goel, Jennifer Skeem, 2020. The limits of human predictions of recidivism. Science advances 6, 7 (2020)."},{"volume-title":"fast and slow","author":"Kahneman Daniel","key":"e_1_3_2_1_42_1","unstructured":"Daniel Kahneman. 2011. Thinking, fast and slow. Macmillan."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1952.10483441"},{"key":"e_1_3_2_1_44_1","first-page":"207","article-title":"Crowdsourcing Truthfulness: The Impact of Judgment Scale and Assessor Bias","volume":"12036","author":"La\u00a0Barbera David","year":"2020","unstructured":"David La\u00a0Barbera, Kevin Roitero, Gianluca Demartini, Stefano Mizzaro, and Damiano Spina. 2020. Crowdsourcing Truthfulness: The Impact of Judgment Scale and Assessor Bias. Advances in Information Retrieval 12036 (2020), 207.","journal-title":"Advances in Information Retrieval"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"e_1_3_2_1_46_1","volume-title":"How we analyzed the COMPAS recidivism algorithm. ProPublica (5 2016) 9, 1","author":"Larson Jeff","year":"2016","unstructured":"Jeff Larson, Surya Mattu, Lauren Kirchner, and Julia Angwin. 2016. How we analyzed the COMPAS recidivism algorithm. ProPublica (5 2016) 9, 1 (2016)."},{"key":"e_1_3_2_1_47_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction 3, CSCW(2019)","author":"Li Tianyi","year":"2019","unstructured":"Tianyi Li, Chandler\u00a0J Manns, Chris North, and Kurt Luther. 2019. Dropping the baton? Understanding errors and bottlenecks in a crowdsourced sensemaking pipeline. Proceedings of the ACM on Human-Computer Interaction 3, CSCW(2019), 1\u201326."},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376257"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858560"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600175"},{"key":"e_1_3_2_1_51_1","first-page":"1195","article-title":"Does unconscious racial bias affect trial judges","volume":"84","author":"Rachlinski J","year":"2008","unstructured":"Jeffrey\u00a0J Rachlinski, Sheri\u00a0Lynn Johnson, Andrew\u00a0J Wistrich, and Chris Guthrie. 2008. Does unconscious racial bias affect trial judges. Notre Dame L. Rev. 84(2008), 1195.","journal-title":"Notre Dame L. Rev."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.5555\/1756006.1859894"},{"key":"e_1_3_2_1_53_1","volume-title":"Data Cascades in High-Stakes AI. In proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201315","author":"Sambasivan Nithya","year":"2021","unstructured":"Nithya Sambasivan, Shivani Kapania, Hannah Highfill, Diana Akrong, Praveen Paritosh, and Lora\u00a0M Aroyo. 2021. \u201cEveryone wants to do the model work, not the data work\u201d: Data Cascades in High-Stakes AI. In proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201315."},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274423"},{"key":"e_1_3_2_1_55_1","volume-title":"Cognitive reflection vs. calculation in decision making. Frontiers in psychology 6","author":"Sinayev Aleksandr","year":"2015","unstructured":"Aleksandr Sinayev and Ellen Peters. 2015. Cognitive reflection vs. calculation in decision making. Frontiers in psychology 6 (2015), 532."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102710"},{"key":"e_1_3_2_1_57_1","volume-title":"Individual differences in reasoning: Implications for the rationality debate?Behavioral and brain sciences 23, 5","author":"Stanovich E","year":"2000","unstructured":"Keith\u00a0E Stanovich and Richard\u00a0F West. 2000. Individual differences in reasoning: Implications for the rationality debate?Behavioral and brain sciences 23, 5 (2000), 645\u2013665."},{"key":"e_1_3_2_1_58_1","volume-title":"Proceedings of the 18th International Conference on Autonomous Agents and Multiagent Systems. 1324\u20131332","author":"Tang Wei","year":"2019","unstructured":"Wei Tang and Chien-Ju Ho. 2019. Bandit Learning with Biased Human Feedback. In Proceedings of the 18th International Conference on Autonomous Agents and Multiagent Systems. 1324\u20131332."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313554"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380199"},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450650"},{"volume-title":"All of statistics: a concise course in statistical inference. Vol.\u00a026","author":"Wasserman Larry","key":"e_1_3_2_1_62_1","unstructured":"Larry Wasserman. 2004. All of statistics: a concise course in statistical inference. Vol.\u00a026. Springer."},{"key":"e_1_3_2_1_63_1","volume-title":"Jacob Bergsma, Javier\u00a0R. Movellan, and Paul\u00a0L. Ruvolo.","author":"Whitehill Jacob","year":"2009","unstructured":"Jacob Whitehill, Ting fan Wu, Jacob Bergsma, Javier\u00a0R. Movellan, and Paul\u00a0L. Ruvolo. 2009. Whose vote should count more: Optimal integration of labels from labelers of unknown expertise. In Advances in Neural Information Processing Systems (NIPS)."},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3375709"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v4i1.13282"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052660"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"crossref","unstructured":"Jieyu Zhao Tianlu Wang Mark Yatskar Vicente Ordonez and Kai-Wei Chang. 2017. Men also like shopping: Reducing gender bias amplification using corpus-level constraints. arXiv preprint arXiv:1707.09457(2017).","DOI":"10.18653\/v1\/D17-1323"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.14778\/3055540.3055547"},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531718"}],"event":{"name":"WWW '22: The ACM Web Conference 2022","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Virtual Event, Lyon France","acronym":"WWW '22"},"container-title":["Proceedings of the ACM Web Conference 2022"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3512239","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3485447.3512239","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:30:13Z","timestamp":1750188613000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3512239"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,25]]},"references-count":69,"alternative-id":["10.1145\/3485447.3512239","10.1145\/3485447"],"URL":"https:\/\/doi.org\/10.1145\/3485447.3512239","relation":{},"subject":[],"published":{"date-parts":[[2022,4,25]]},"assertion":[{"value":"2022-04-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}