{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T15:45:36Z","timestamp":1776095136557,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T00:00:00Z","timestamp":1655683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Credibility Coalition"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,21]]},"DOI":"10.1145\/3531146.3534629","type":"proceedings-article","created":{"date-parts":[[2022,6,20]],"date-time":"2022-06-20T14:27:10Z","timestamp":1655735230000},"page":"2114-2124","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":15,"title":["The Effects of Crowd Worker Biases in Fact-Checking Tasks"],"prefix":"10.1145","author":[{"given":"Tim","family":"Draws","sequence":"first","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"given":"David","family":"La Barbera","sequence":"additional","affiliation":[{"name":"University of Udine, Italy"}]},{"given":"Michael","family":"Soprano","sequence":"additional","affiliation":[{"name":"University of Udine, Italy"}]},{"given":"Kevin","family":"Roitero","sequence":"additional","affiliation":[{"name":"University of Udine, Italy"}]},{"given":"Davide","family":"Ceolin","sequence":"additional","affiliation":[{"name":"CWI, Netherlands"}]},{"given":"Alessandro","family":"Checco","sequence":"additional","affiliation":[{"name":"University of Rome La Sapienza, Italy"}]},{"given":"Stefano","family":"Mizzaro","sequence":"additional","affiliation":[{"name":"University of Udine, Italy"}]}],"member":"320","published-online":{"date-parts":[[2022,6,20]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Scaling up fact-checking using the wisdom of crowds. Science advances 7, 36","author":"Allen Jennifer","year":"2021","unstructured":"Jennifer Allen, Antonio\u00a0A Arechar, Gordon Pennycook, and David\u00a0G Rand. 2021. Scaling up fact-checking using the wisdom of crowds. Science advances 7, 36 (2021), eabf4393."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209581"},{"key":"e_1_3_2_1_3_1","volume-title":"Crowdsourcing of News Credibility. In Computational Journalism Symposium.","author":"Bhuiyan Momen","year":"2020","unstructured":"Momen Bhuiyan, Amy Zhang, Connie Sehat, and Tanushree Mitra. 2020. Investigating \u201cWho\u201d in the Crowdsourcing of News Credibility. In Computational Journalism Symposium."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3269315"},{"key":"e_1_3_2_1_5_1","unstructured":"Jiawei Chen Hande Dong Xiang Wang Fuli Feng Meng Wang and Xiangnan He. 2020. Bias and Debias in Recommender System: A Survey and Future Directions. arXiv preprint arXiv:2010.03240(2020)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.3389\/fpsyg.2019.00861"},{"key":"e_1_3_2_1_7_1","first-page":"65","article-title":"Human-in-the-loop Artificial Intelligence for Fighting Online Misinformation: Challenges and Opportunities","volume":"43","author":"Demartini Gianluca","year":"2020","unstructured":"Gianluca Demartini, Stefano Mizzaro, and Damiano Spina. 2020. Human-in-the-loop Artificial Intelligence for Fighting Online Misinformation: Challenges and Opportunities. Bulletin of IEEE Computer Society 43, 3 (2020), 65\u201374.","journal-title":"Bulletin of IEEE Computer Society"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3498366.3505812"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v9i1.18939"},{"key":"e_1_3_2_1_10_1","volume-title":"This Is Not What We Ordered: Exploring Why Biased Search Result Rankings Affect User Attitudes on Debated Topics","author":"Draws Tim","unstructured":"Tim Draws, Nava Tintarev, Ujwal Gadiraju, Alessandro Bozzon, and Benjamin Timmermans. 2021. This Is Not What We Ordered: Exploring Why Biased Search Result Rankings Affect User Attitudes on Debated Topics. ACM, New York, NY, USA, 295\u2013305."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159654"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1419828112"},{"key":"e_1_3_2_1_13_1","volume-title":"Will the Crowd Game the Algorithm? Using Layperson Judgments to Combat Misinformation on Social Media by Downranking Distrusted Sources","author":"Epstein Ziv","unstructured":"Ziv Epstein, Gordon Pennycook, and David Rand. 2020. Will the Crowd Game the Algorithm? Using Layperson Judgments to Combat Misinformation on Social Media by Downranking Distrusted Sources. ACM, New York, NY, USA, 1\u201311."},{"key":"e_1_3_2_1_14_1","volume-title":"GPOWER: A general power analysis program. Behavior Research Methods, Instruments, & Computers 28, 1 (01","author":"Erdfelder Edgar","year":"1996","unstructured":"Edgar Erdfelder, Franz Faul, and Axel Buchner. 1996. GPOWER: A general power analysis program. Behavior Research Methods, Instruments, & Computers 28, 1 (01 Mar 1996), 1\u201311."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1257\/089533005775196732"},{"key":"e_1_3_2_1_16_1","volume-title":"Catching Zika Fever: Application of Crowdsourcing and Machine Learning for Tracking Health Misinformation on Twitter. In 2017 IEEE International Conference on Healthcare Informatics (ICHI). 518\u2013518","author":"Ghenai Amira","year":"2017","unstructured":"Amira Ghenai and Yelena Mejova. 2017. Catching Zika Fever: Application of Crowdsourcing and Machine Learning for Tracking Health Misinformation on Twitter. In 2017 IEEE International Conference on Healthcare Informatics (ICHI). 518\u2013518."},{"key":"e_1_3_2_1_17_1","volume-title":"The Battle Against Online Harmful Information: The Cases of Fake News and Hate Speech","author":"Giachanou Anastasia","unstructured":"Anastasia Giachanou and Paolo Rosso. 2020. The Battle Against Online Harmful Information: The Cases of Fake News and Hate Speech. Association for Computing Machinery, New York, NY, USA, 3503\u20133504."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1177\/0020852315585950"},{"key":"e_1_3_2_1_19_1","volume-title":"Has the Standard Cognitive Reflection Test Become a Victim of Its Own Success?Advances in cognitive psychology 12, 3 (30","author":"Haigh Matthew","year":"2016","unstructured":"Matthew Haigh. 2016. Has the Standard Cognitive Reflection Test Become a Victim of Its Own Success?Advances in cognitive psychology 12, 3 (30 Sep 2016), 145\u2013149."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2019.2948168"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300637"},{"key":"e_1_3_2_1_22_1","unstructured":"Klaus Krippendorff. 2011. Computing Krippendorff\u2019s Alpha-Reliability. (2011)."},{"key":"e_1_3_2_1_23_1","volume-title":"Crowdsourcing Truthfulness: The Impact of Judgment Scale and Assessor Bias","author":"Barbera David La","year":"2020","unstructured":"David La Barbera, Kevin Roitero, Damiano Spina, Stefano Mizzaro, and Gianluca Demartini. 2020. Crowdsourcing Truthfulness: The Impact of Judgment Scale and Assessor Bias. In Proceedings of ECIR. Springer, New York, NY, USA, 207\u2013214."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1083-6101.2012.01597.x"},{"key":"e_1_3_2_1_25_1","volume-title":"Comparison of peer assessment methods: Reliability, validity, friendship bias, and user reaction.Journal of Applied Psychology 66, 4","author":"Love G.","year":"1981","unstructured":"Kevin\u00a0G. Love. 1981. Comparison of peer assessment methods: Reliability, validity, friendship bias, and user reaction.Journal of Applied Psychology 66, 4 (1981), 451."},{"key":"e_1_3_2_1_26_1","volume-title":"CIKM Workshops. ACM","author":"Maddalena Eddy","year":"2018","unstructured":"Eddy Maddalena, Davide Ceolin, and Stefano Mizzaro. 2018. Multidimensional News Quality: A Comparison of Crowdsourcing and Nichesourcing.. In CIKM Workshops. ACM, New York, NY, USA."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/S16-1003"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1080\/23738871.2018.1462395"},{"key":"e_1_3_2_1_29_1","volume-title":"Social influence bias: A randomized experiment. Science 341, 6146","author":"Muchnik Lev","year":"2013","unstructured":"Lev Muchnik, Sinan Aral, and Sean\u00a0J Taylor. 2013. Social influence bias: A randomized experiment. Science 341, 6146 (2013), 647\u2013651."},{"key":"e_1_3_2_1_30_1","volume-title":"Crowdsourcing judgments of news source quality. SSRN. com","author":"Pennycook Gordon","year":"2018","unstructured":"Gordon Pennycook and David\u00a0G Rand. 2018. Crowdsourcing judgments of news source quality. SSRN. com (2018)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1806781116"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/CSCWD.2019.8791903"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121074"},{"key":"e_1_3_2_1_34_1","unstructured":"Nicolas Pr\u00f6llochs. 2021. Community-Based Fact-Checking on Twitter\u2019s Birdwatch Platform. arXiv preprint arXiv:2104.07175(2021)."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3465336.3475101"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the CIKM 2018 Workshops co-located with 27th ACM International Conference on Information and Knowledge Management (CIKM 2018","author":"Roitero Kevin","year":"2018","unstructured":"Kevin Roitero, Gianluca Demartini, Stefano Mizzaro, and Damiano Spina. 2018. How Many Truth Levels? Six? One Hundred? Even More? Validating Truthfulness of Statements via Crowdsourcing. In Proceedings of the CIKM 2018 Workshops co-located with 27th ACM International Conference on Information and Knowledge Management (CIKM 2018), Torino, Italy, October 22, 2018. ACM, New York, NY, USA, 6\u00a0pages."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401112"},{"key":"e_1_3_2_1_38_1","first-page":"1","article-title":"Can the crowd judge truthfulness? A longitudinal study on recent misinformation about COVID-19","volume":"1","author":"Roitero Kevin","year":"2021","unstructured":"Kevin Roitero, Michael Soprano, Beatrice Portelli, Massimiliano De\u00a0Luise, Damiano Spina, Vincenzo\u00a0Della Mea, Giuseppe Serra, Stefano Mizzaro, and Gianluca Demartini. 2021. Can the crowd judge truthfulness? A longitudinal study on recent misinformation about COVID-19. Personal and Ubiquitous Computing 1, 1 (2021), 1\u201331.","journal-title":"Personal and Ubiquitous Computing"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412048"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3078714.3078746"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/CIC.2018.00048"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488560.3502182"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102710"},{"key":"e_1_3_2_1_44_1","unstructured":"Sebastian Tschiatschek Adish Singla Manuel Gomez\u00a0Rodriguez Arpit Merchant and Andreas Krause. 2017. Detecting fake news in social networks via crowdsourcing. arXiv preprint arXiv:1711.09025(2017)."},{"key":"e_1_3_2_1_45_1","first-page":"11","article-title":"Reason-Checking Fake","volume":"63","author":"Visser Jacky","year":"2020","unstructured":"Jacky Visser, John Lawrence, and Chris Reed. 2020. Reason-Checking Fake News. Commun. ACM 63, 11 (oct 2020), 38\u201340.","journal-title":"News. Commun. ACM"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1080\/1461670X.2018.1492881"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P17-2067"},{"key":"e_1_3_2_1_48_1","first-page":"422","author":"Wang William\u00a0Yang","year":"2017","unstructured":"William\u00a0Yang Wang. 2017. \u201dLiar, Liar Pants on Fire\u201d: A New Benchmark Dataset for Fake News Detection. In Proceedings of ACL. 422\u2013426.","journal-title":"A New Benchmark Dataset for Fake News Detection. In Proceedings of ACL."},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23128"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.5555\/2871950.2871954"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1772690.1772793"}],"event":{"name":"FAccT '22: 2022 ACM Conference on Fairness, Accountability, and Transparency","location":"Seoul Republic of Korea","acronym":"FAccT '22","sponsor":["ACM Association for Computing Machinery"]},"container-title":["2022 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3534629","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3531146.3534629","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:31:30Z","timestamp":1750188690000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3531146.3534629"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,20]]},"references-count":51,"alternative-id":["10.1145\/3531146.3534629","10.1145\/3531146"],"URL":"https:\/\/doi.org\/10.1145\/3531146.3534629","relation":{},"subject":[],"published":{"date-parts":[[2022,6,20]]},"assertion":[{"value":"2022-06-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}