{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,8]],"date-time":"2026-06-08T12:08:48Z","timestamp":1780920528760,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":75,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,30]],"date-time":"2023-04-30T00:00:00Z","timestamp":1682812800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,30]]},"DOI":"10.1145\/3543507.3583214","type":"proceedings-article","created":{"date-parts":[[2023,4,26]],"date-time":"2023-04-26T23:30:25Z","timestamp":1682551825000},"page":"3660-3669","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":24,"title":["Simplistic Collection and Labeling Practices Limit the Utility of Benchmark Datasets for Twitter Bot Detection"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3524-5564","authenticated-orcid":false,"given":"Chris","family":"Hays","sequence":"first","affiliation":[{"name":"Massachusetts Institute of Technology, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3448-5654","authenticated-orcid":false,"given":"Zachary","family":"Schutzman","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4155-8145","authenticated-orcid":false,"given":"Manish","family":"Raghavan","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8305-3161","authenticated-orcid":false,"given":"Erin","family":"Walk","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5249-5151","authenticated-orcid":false,"given":"Philipp","family":"Zimmer","sequence":"additional","affiliation":[{"name":"Massachusetts Institute of Technology, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,4,30]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.3390\/app12168117"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","unstructured":"Adam Badawy Emilio Ferrara and Kristina Lerman. 2018. Analyzing the Digital Traces of Political Manipulation: The 2016 Russian Interference Twitter Campaign. In 2018 IEEE\/ACM International Conference on Advances in Social Networks Analysis and Mining (ASONAM). 258\u2013265. https:\/\/doi.org\/10.1109\/ASONAM.2018.8508646","DOI":"10.1109\/ASONAM.2018.8508646"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1177\/0894439317734157"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v21i11.7090"},{"key":"e_1_3_2_2_5_1","unstructured":"Christopher Bouzy. 2018. Bot sentinel Platform developed to detect and track political bots trollbots and untrustworthy accounts.https:\/\/botsentinel.com\/"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.2105\/AJPH.2018.304567"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-019-57272-3"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2012.75"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409116"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3041021.3055135"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"crossref","unstructured":"Stefano Cresci Fabrizio Lillo Daniele Regoli Serena Tardelli and Maurizio Tesconi. 2019. $ FAKE: Evidence of Spam and Bot Activity in Stock Microblogs on Twitter. (2019).","DOI":"10.1609\/icwsm.v12i1.15073"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2015.09.003"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIS.2016.29"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2017.57"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/TDSC.2017.2681672"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2740908.2741722"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11219857"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274694.3274738"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v16i1.19282"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i4.20314"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3481949"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"crossref","unstructured":"Shangbin Feng Herun Wan Ningnan Wang Jundong Li and Minnan Luo. 2021. TwiBot-20: A Comprehensive Twitter Bot Detection Benchmark. (2021) 4485\u20134494.","DOI":"10.1145\/3459637.3482019"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3487351.3488336"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v22i18.8005"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.5210\/fm.v25i6.10633"},{"key":"e_1_3_2_2_26_1","unstructured":"Florian Gallwitz and Michael Kreil. 2021. The Rise and Fall of \u2019Social Bot\u2019 Research. https:\/\/ssrn.com\/abstract=3814191"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"crossref","unstructured":"Florian Gallwitz and Michael Kreil. 2022. Investigating the Validity of Botometer-based Social Bot Studies. arxiv:2207.11474\u00a0[cs.SI]","DOI":"10.1007\/978-3-031-18253-2_5"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3458723"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3298789"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3110025.3110090"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3110025.3110091"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3110025.3110091"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.findings-acl.457"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1093\/pnasnexus"},{"key":"e_1_3_2_2_36_1","volume-title":"Social Media, Sentiment and Public Opinions: Evidence from #Brexit and #USElection. NBER Working Papers 24631","author":"Gorodnichenko Yuriy","unstructured":"Yuriy Gorodnichenko, Tho Pham, and Oleksandr Talavera. 2018. Social Media, Sentiment and Public Opinions: Evidence from #Brexit and #USElection. NBER Working Papers 24631. National Bureau of Economic Research, Inc. https:\/\/ideas.repec.org\/p\/nbr\/nberwo\/24631.html"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.3390\/sym14010030"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/UEMCON51285.2020.9298158"},{"key":"e_1_3_2_2_39_1","volume-title":"Detecting malicious activity in Twitter using deep learning techniques. Applied Soft Computing 107","author":"Ilias Loukas","year":"2021","unstructured":"Loukas Ilias and Ioanna Roussaki. 2021. Detecting malicious activity in Twitter using deep learning techniques. Applied Soft Computing 107 (2021)."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2018.02.032"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1080\/10584609.2018.1526238"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415958.3433075"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2018.08.019"},{"key":"e_1_3_2_2_44_1","volume-title":"Fifth International AAAI Conference on Weblogs and Social Media. Association for the Advancement of Artificial Intelligence, 185\u2013192","author":"Lee Kyumin","year":"2011","unstructured":"Kyumin Lee, Brian Eoff, and James Caverlee. 2011. A Long-Term Study of Content Polluters on Twitter. In Fifth International AAAI Conference on Weblogs and Social Media. Association for the Advancement of Artificial Intelligence, 185\u2013192."},{"key":"e_1_3_2_2_45_1","first-page":"I","article-title":"A Unified Approach to Interpreting Model Predictions","volume":"30","author":"Lundberg M","year":"2017","unstructured":"Scott\u00a0M Lundberg and Su-In Lee. 2017. A Unified Approach to Interpreting Model Predictions. In Advances in Neural Information Processing Systems 30, I.\u00a0Guyon, U.\u00a0V. Luxburg, S.\u00a0Bengio, H.\u00a0Wallach, R.\u00a0Fergus, S.\u00a0Vishwanathan, and R.\u00a0Garnett (Eds.). Curran Associates, Inc., 4765\u20134774. http:\/\/papers.nips.cc\/paper\/7062-a-unified-approach-to-interpreting-model-predictions.pdf","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1088\/1757-899x"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1177\/20539517211033566"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2021.3068659"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3292522.3326015"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-60975-7_17"},{"key":"e_1_3_2_2_51_1","unstructured":"Elon Musk. 2022. Bot Percentage Thread. https:\/\/twitter.com\/elonmusk\/status\/1555950698252181507"},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.3003370"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-021-03344-2"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0227821"},{"key":"e_1_3_2_2_55_1","volume-title":"CLEF Evaluation Labs and Workshop Working Notes Papers.","author":"Rangel Francisco","year":"2015","unstructured":"Francisco Rangel and Paolo Rosso. 2015. Overview of the 7th Author Profiling Task at PAN 2019: Bots and Gender Profiling in Twitter. In CLEF Evaluation Labs and Workshop Working Notes Papers."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0241045"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.3390\/app12125915"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-019-0048-x"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412698"},{"key":"e_1_3_2_2_61_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-018-06930-7"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0196087"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1089\/big.2020.0062"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1803470115"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1109\/UBMK50275.2020.9219433"},{"key":"e_1_3_2_2_66_1","unstructured":"Twitter. 2021. FORM 10-K. https:\/\/www.sec.gov\/Archives\/edgar\/data\/1418091\/000141809121000031\/twtr-20201231.htm"},{"key":"e_1_3_2_2_67_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.aap9559"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.3390\/app11125482"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445109"},{"key":"e_1_3_2_2_70_1","volume-title":"Adaptive Spammer Detection with Sparse Group Modeling. In Eleventh International AAAI Conference on Web and Social Media (ICWSM). Association for the Advancement of Artificial Intelligence, 319\u2013326","author":"Wu Liang","year":"2017","unstructured":"Liang Wu, Xia Hu, Fred Morstatter, and Huan Liu. 2017. Adaptive Spammer Detection with Sparse Group Modeling. In Eleventh International AAAI Conference on Web and Social Media (ICWSM). Association for the Advancement of Artificial Intelligence, 319\u2013326."},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1177\/1461444820942744"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/TIFS.2013.2267732"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1002\/hbe2.115"},{"key":"e_1_3_2_2_74_1","volume-title":"Botometer 101: Social bot practicum for computational social scientists. arXiv preprint arXiv:2201.01608","author":"Yang Kai-Cheng","year":"2022","unstructured":"Kai-Cheng Yang, Emilio Ferrara, and Filippo Menczer. 2022. Botometer 101: Social bot practicum for computational social scientists. arXiv preprint arXiv:2201.01608 (2022)."},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i01.5460"}],"event":{"name":"WWW '23: The ACM Web Conference 2023","location":"Austin TX USA","acronym":"WWW '23","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM Web Conference 2023"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583214","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3543507.3583214","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:18Z","timestamp":1750183758000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583214"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,30]]},"references-count":75,"alternative-id":["10.1145\/3543507.3583214","10.1145\/3543507"],"URL":"https:\/\/doi.org\/10.1145\/3543507.3583214","relation":{},"subject":[],"published":{"date-parts":[[2023,4,30]]},"assertion":[{"value":"2023-04-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}