{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,10]],"date-time":"2025-12-10T12:26:23Z","timestamp":1765369583354},"reference-count":69,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2020,3,1]],"date-time":"2020-03-01T00:00:00Z","timestamp":1583020800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,3,1]],"date-time":"2020-03-01T00:00:00Z","timestamp":1583020800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["SN COMPUT. SCI."],"published-print":{"date-parts":[[2020,3]]},"DOI":"10.1007\/s42979-020-0082-0","type":"journal-article","created":{"date-parts":[[2020,3,28]],"date-time":"2020-03-28T18:02:40Z","timestamp":1585418560000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Tracking Hate in Social Media: Evaluation, Challenges and Approaches"],"prefix":"10.1007","volume":"1","author":[{"given":"Sandip","family":"Modha","sequence":"first","affiliation":[]},{"given":"Thomas","family":"Mandl","sequence":"additional","affiliation":[]},{"given":"Prasenjit","family":"Majumder","sequence":"additional","affiliation":[]},{"given":"Daksh","family":"Patel","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,3,28]]},"reference":[{"issue":"2","key":"82_CR1","first-page":"83","volume":"9","author":"A Al-Hassan","year":"2019","unstructured":"Al-Hassan A, Al-Dossari H. Detection of hate speech in social networks: a survey on multilingual corpus. Comput Sci Inf Technol (CS & IT). 2019;9(2):83.","journal-title":"Comput Sci Inf Technol (CS & IT)"},{"key":"82_CR2","unstructured":"Saroj A, Mundotiya RK, Pal S. IRLab@ IITBHU at hasoc 2019: traditional machine learning for hate speech and offensive content identification. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR3","unstructured":"Baruah A, Barbhuiya F, Dey K. IIITG-ADBU at HASOC 2019: automated hate speech and offensive content detection in english and code-mixed hindi text. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR4","unstructured":"Bashar MA, Nayak R. QutNocturnal@HASOC\u201919: CNN for hate speech and offensive content identification in hindi language. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR5","doi-asserted-by":"crossref","unstructured":"Basile V, Bosco C, Fersini E, Nozza D, Patti V, Pardo FMR, Rosso P, Sanguinetti M. Semeval-2019 task 5: Multilingual detection of hate speech against immigrants and women in twitter. In: Proceedings of the 13th international workshop on semantic evaluation. 2019. p. 54\u201363.","DOI":"10.18653\/v1\/S19-2007"},{"key":"82_CR6","unstructured":"Wang B, Yunxia Ding SL, Zhou X. YNU Wb at HASOC 2019: ordered neurons LSTM with attention for identifying hate speech and offensive language. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"issue":"2","key":"82_CR7","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1002\/poi3.85","volume":"7","author":"P Burnap","year":"2015","unstructured":"Burnap P, Williams ML. Cyber hate speech on twitter: an application of machine classification and statistical modeling for policy and decision making. Policy Internet. 2015;7(2):223\u201342.","journal-title":"Policy Internet"},{"key":"82_CR8","unstructured":"Casavantes M, L\u00f3pez R, Gonz\u00e1lez LC, Montes-y G\u00f3mez M. UACh-INAOE at HASOC 2019: detecting aggressive tweets by incorporating authors\u2019 traits as descriptors. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR9","unstructured":"Conover MD, Ratkiewicz J, Francisco M, Gon\u00e7alves B, Menczer F, Flammini A. Political polarization on twitter. In: Fifth international AAAI conference on weblogs and social media. 2011."},{"key":"82_CR10","unstructured":"Dana\u00a0Ruiter MAR, Klakow D. LSV-UdS at HASOC 2019: the problem of defining hate? In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR11","doi-asserted-by":"crossref","unstructured":"Davidson T, Warmsley D, Macy M, Weber I. Automated hate speech detection and the problem of offensive language. In: Proceedings of ICWSM. 2017.","DOI":"10.1609\/icwsm.v11i1.14955"},{"key":"82_CR12","unstructured":"De\u00a0Smedt T, Jaki S, Kotz\u00e9 E, Saoud L, Gw\u00f3\u017ad\u017a M, De\u00a0Pauw G, Daelemans W. Multilingual cross-domain perspectives on online hate speech. 2018. https:\/\/repository.uantwerpen.be\/docman\/irua\/e092ae\/156589.pdf. Accessed 7 Mar 2020."},{"key":"82_CR13","unstructured":"Devlin J, Chang MW, Lee K, Toutanova K. Bert: pre-training of deep bidirectional transformers for language understanding. 2018. arXiv:1810.04805."},{"key":"82_CR14","doi-asserted-by":"crossref","unstructured":"Djuric N, Zhou J, Morris R, Grbovic M, Radosavljevic V, Bhamidipati N. Hate speech detection with comment embeddings. In: Proceedings of the 24th international conference on world wide web companion. International World Wide Web conferences steering committee. 2015. p. 29\u201330.","DOI":"10.1145\/2740908.2742760"},{"key":"82_CR15","doi-asserted-by":"crossref","unstructured":"Fersini E, Rosso P, Anzovino M. Overview of the task on automatic misogyny identification at ibereval 2018. 2018.","DOI":"10.4000\/books.aaccademia.4497"},{"key":"82_CR16","doi-asserted-by":"crossref","unstructured":"Fi\u0161er D, Erjavec T, Ljube\u0161i\u0107 N. Legal framework, dataset and annotation schema for socially unacceptable on-line discourse practices in Slovene. In: Proceedings of the workshop on abusive language online (ALW). Canada: Vancouver; 2017.","DOI":"10.18653\/v1\/W17-3007"},{"issue":"4","key":"82_CR17","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1145\/3232676","volume":"51","author":"P Fortuna","year":"2018","unstructured":"Fortuna P, Nunes S. A survey on automatic detection of hate speech in text. ACM Comput Surv (CSUR). 2018;51(4):85.","journal-title":"ACM Comput Surv (CSUR)"},{"key":"82_CR18","volume-title":"Vorstudien und Erg\u00e4nzungen zur Theorie des kommunikativen Handelns","author":"J Habermas","year":"1984","unstructured":"Habermas J. Vorstudien und Erg\u00e4nzungen zur Theorie des kommunikativen Handelns. Frankfurt: Suhrkamp; 1984."},{"key":"82_CR19","unstructured":"Mensonides J-C, Jean, P-A, Tchechmedjiev A, Harispe S. IMT mines ales at HASOC 2019: automatic hate speech detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"issue":"2","key":"82_CR20","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1145\/3185593","volume":"25","author":"S Jhaver","year":"2018","unstructured":"Jhaver S, Ghoshal S, Bruckman A, Gilbert E. Online harassment and content moderation: the case of blocklists. ACM Trans Comput Hum Interact (TOCHI). 2018;25(2):12.","journal-title":"ACM Trans Comput Hum Interact (TOCHI)"},{"key":"82_CR21","unstructured":"Jiang A. QMUL-NLP at HASOC 2019: offensive content detection and classification in social media. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR22","unstructured":"Das KA, Barbhuiya FA. FalsePostive at HASOC 2019: transfer-learning for detection and classification of hate speech. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR23","unstructured":"Kumari K, Singh JP. AI ML NIT Patna at HASOC 2019: deep learning approach for identification of abusive content. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR24","doi-asserted-by":"crossref","unstructured":"Kwok I, Wang Y. Locate the hate: detecting tweets against blacks. In: Twenty-seventh AAAI conference on artificial intelligence. 2013.","DOI":"10.1609\/aaai.v27i1.8539"},{"key":"82_CR25","unstructured":"Lu Z, Nie JY. RALIGRAPH at HASOC 2019: VGCN-BERT: augmenting BERT with graph embedding for offensive language detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR26","unstructured":"Mishra A, Pal S. IIT Varanasi at HASOC 2019: hate speech and offensive content identification in Indo-European languages. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR27","unstructured":"Mishra S, Mishra S. 3Idiots at HASOC 2019: Fine-tuning transformer neural networks for hate speech identification in Indo-European languages. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR28","doi-asserted-by":"crossref","unstructured":"Mondal M, Silva LA, Benevenuto F. A measurement study of hate speech in social media. In: Proceedings of the 28th ACM conference on hypertext and social media. ACM; 2017. p. 85\u201394.","DOI":"10.1145\/3078714.3078723"},{"key":"82_CR29","doi-asserted-by":"crossref","unstructured":"Mubarak H, Darwish K, Magdy W. Abusive language detection on Arabic social media. In: Proceedings of the first workshop on abusive language online. 2017. p. 52\u20136.","DOI":"10.18653\/v1\/W17-3008"},{"key":"82_CR30","doi-asserted-by":"crossref","unstructured":"Mubarak H, Kareem D, Walid M. Abusive language detection on Arabic social media. In: Proceedings of the workshop on abusive language online (ALW). Vancouver, Canada; 2017.","DOI":"10.18653\/v1\/W17-3008"},{"key":"82_CR31","unstructured":"Mujadia V, Mishra P, Sharma DM. IIIT-Hyderabad at HASOC 2019: hate speech detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR32","volume-title":"Kill all normies: online culture wars from 4chan and Tumblr to Trump and the alt-right","author":"A Nagle","year":"2017","unstructured":"Nagle A. Kill all normies: online culture wars from 4chan and Tumblr to Trump and the alt-right. London: John Hunt Publishing; 2017."},{"key":"82_CR33","unstructured":"Nayel HA, Shashirekha HL. DEEP at HASOC2019: a machine learning framework for hate speech and offensive language detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR34","unstructured":"Nina-Alcocer V. Vito at HASOC 2019: detecting hate speech and offensive content through ensembles. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR35","doi-asserted-by":"crossref","unstructured":"Nobata C, Tetreault J, Thomas A, Mehdad Y, Chang Y. Abusive language detection in online user content. In: Proceedings of the 25th international conference on world wide web. International world wide web conferences steering committee. 2016. p. 145\u201353.","DOI":"10.1145\/2872427.2883062"},{"issue":"2","key":"82_CR36","first-page":"1277","volume":"3","author":"JT Nockleby","year":"2000","unstructured":"Nockleby JT. Hate speech. Encycl Am Const. 2000;3(2):1277\u20139.","journal-title":"Encycl Am Const"},{"key":"82_CR37","unstructured":"Parikh A, Desai H, Bisht AS. DA Master at HASOC 2019: identification of hate speech using machine learning and deep learning approaches for social media post. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR38","unstructured":"Alonso P, Saini R, Kov\u00e1cs G. TheNorth at HASOC 2019: hate speech detection in social media data. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"issue":"21","key":"82_CR39","doi-asserted-by":"publisher","first-page":"4654","DOI":"10.3390\/s19214654","volume":"19","author":"JC Pereira-Kohatsu","year":"2019","unstructured":"Pereira-Kohatsu JC, Quijano-S\u00e1nchez L, Liberatore F, Camacho-Collados M. Detecting and monitoring hate speech in twitter. Sensors. 2019;19(21):4654.","journal-title":"Sensors"},{"key":"82_CR40","doi-asserted-by":"crossref","unstructured":"Peters ME, Neumann M, Iyyer M, Gardner M, Clark C, Lee K, Zettlemoyer L. Deep contextualized word representations. 2018. arXiv:1802.05365","DOI":"10.18653\/v1\/N18-1202"},{"key":"82_CR41","unstructured":"Rajalakshmi, R, Reddy BY. YR: DLRG@HASOC 2019\u2014an enhanced ensemble classifier for hate and offensive content identification. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR42","unstructured":"Ranasinghe T, Zampieri M, Hettiarachchi H. BRUMS at HASOC 2019: deep learning models for multilingual hate speech and offensive language identification. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR43","unstructured":"Kumar R, Reganti AN, Bhatia A, Maheshwari T. Aggression-annotated corpus of hindi\u2013english code-mixed data. In: Proceedings of the 11th language resources and evaluation conference (LREC). Miyazaki: Japan; 2018. p. 1\u201311."},{"key":"82_CR44","unstructured":"Kumar R, Ojha AK. KMI-Panlingua at HASOC 2019: SVM vs BERT for hate speech and offensive content detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR45","unstructured":"Ross B, Rist M, Carbonell G, Cabrera B, Kurowsky N, Wojatzki M. Measuring the reliability of hate speech annotations: the case of the european refugee crisis. In: Proceedings of the workshop on natural language processing for computer-mediated communication (NLP4CMC). Germany: Bochum; 2016."},{"key":"82_CR46","unstructured":"Ross B, Rist M, Carbonell G, Cabrera B, Kurowsky N, Wojatzki M. Measuring the reliability of hate speech annotations: the case of the european refugee crisis. 2017. arXiv:1701.08118"},{"key":"82_CR47","unstructured":"Saha BN, Senapati A. CIT Kokrajhar team: LSTM based deep RNN architecture for hate speech and offensive content (HASOC) identification in Indo-European languages. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR48","unstructured":"Saha P, Mathew B, Goyal P, Mukherjee A. HateMonitors at HASOC 2019: language agnostic online abuse detection. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR49","doi-asserted-by":"crossref","unstructured":"Salminen J, Almerekhi H, Kamel AM, Jung SQ, Jansen BJ. Online hate ratings vary by extremes: a statistical analysis. In: Proceedings of the 2019 conference on human information interaction and retrieval. ACM; 2019. p. 213\u20137.","DOI":"10.1145\/3295750.3298954"},{"key":"82_CR50","doi-asserted-by":"crossref","unstructured":"Sap M, Card D, Gabriel S, Choi Y, Smith NA. The risk of racial bias in hate speech detection. In: Proceedings of the 57th annual meeting of the association for computational linguistics. 2019. p. 1668\u201378.","DOI":"10.18653\/v1\/P19-1163"},{"key":"82_CR51","doi-asserted-by":"crossref","unstructured":"Schmidt A, Wiegand M. A survey on hate speech detection using natural language processing. In: Proceedings of the fifth international workshop on natural language processing for social media. Association for computational linguistics. Valencia, Spain; 2017. p. 1\u201310.","DOI":"10.18653\/v1\/W17-1101"},{"issue":"11","key":"82_CR52","doi-asserted-by":"publisher","first-page":"4177","DOI":"10.1177\/1461444818768547","volume":"20","author":"T Sharon","year":"2018","unstructured":"Sharon T, John NA. Unpacking (the) secret: anonymous social media and the impossibility of networked anonymity. New Media Soc. 2018;20(11):4177\u201394.","journal-title":"New Media Soc"},{"key":"82_CR53","unstructured":"Silva L, Mondal M, Correa D, Benevenuto F, Weber I. Analyzing the targets of hate in online social media. In: Tenth international AAAI conference on web and social media. 2016."},{"key":"82_CR54","unstructured":"Sreelakshmi KP. AmritaCEN at HASOC 2019: hate speech detection in Roman and Devanagiri scripted text. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR55","unstructured":"Stru\u00df JM, Siegel M, Ruppenhofer J, Wiegand M, Klenner M. Overview of germeval task 2, 2019 shared task on the identification of offensive language. 2019."},{"key":"82_CR56","doi-asserted-by":"crossref","unstructured":"Su HP, Huang CJ, Chang HT, Lin CJ. Rephrasing profanity in Chinese Text. In: Proceedings of the workshop on abusive language online (ALW). Vancouver, Canada; 2017.","DOI":"10.18653\/v1\/W17-3003"},{"key":"82_CR57","doi-asserted-by":"crossref","unstructured":"Swinger N, De-Arteaga M, Heffernan\u00a0IV NT, Leiserson MD, Kalai AT. What are the biases in my word embedding? In: Proceedings of the 2019 AAAI\/ACM conference on AI, ethics, and society. ACM; 2019. p. 305\u201311.","DOI":"10.1145\/3306618.3314270"},{"key":"82_CR58","unstructured":"Tulkens S, Hilte L, Lodewyckx E, Verhoeven B, Daelemans W. A dictionary-based approach to racism detection in dutch social media. 2016. arXiv:1608.08738."},{"key":"82_CR59","unstructured":"Urmi\u00a0Saha AD, Bhattacharyya P. IIT Bombay at HASOC 2019: supervised hate speech and offensive content detection in Indo-European languages. In: Proceedings of the 11th annual meeting of the Forum for Information Retrieval Evaluation. 2019."},{"key":"82_CR60","unstructured":"Wagner K, Bumann C. Challenges in annotating a corpus for automatic hate speech detection. In: BOBCATSSS Paris. January 2020."},{"key":"82_CR61","doi-asserted-by":"crossref","unstructured":"Wallach HM. Topic modeling: beyond bag-of-words. In: Proceedings of the 23rd international conference on machine learning. ACM; 2006. p. 977\u201384.","DOI":"10.1145\/1143844.1143967"},{"key":"82_CR62","unstructured":"Warner W, Hirschberg J. Detecting hate speech on the world wide web. In: Proceedings of the second workshop on language in social media. Association for Computational Linguistics. 2012. p. 19\u201326."},{"key":"82_CR63","doi-asserted-by":"crossref","unstructured":"Waseem Z, Hovy D. Hateful symbols or hateful people? Predictive features for hate speech detection on twitter. In: Proceedings of the NAACL student research workshop. 2016. p. 88\u201393.","DOI":"10.18653\/v1\/N16-2013"},{"key":"82_CR64","unstructured":"Wiegand M, Ruppenhofer J, Kleinbauer T. Detection of abusive language: the problem of biased datasets. In: Proceedings of the 2019 conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Vol. 1 (long and short papers). 2019. p. 602\u20138."},{"key":"82_CR65","unstructured":"Wiegand M, Siegel M, Ruppenhofer J. Overview of the germeval 2018 shared task on the identification of offensive language. 2018."},{"key":"82_CR66","doi-asserted-by":"crossref","unstructured":"Zafar MB, Valera I, Gomez\u00a0Rodriguez M, Gummadi KP. Fairness beyond disparate treatment & disparate impact: learning classification without disparate mistreatment. In: Proceedings of the 26th international conference on world wide web. International world wide web conferences steering committee. 2017. p. 1171\u201380.","DOI":"10.1145\/3038912.3052660"},{"key":"82_CR67","doi-asserted-by":"crossref","unstructured":"Zampieri M, Malmasi S, Nakov P, Rosenthal S, Farra N, Kumar R. Predicting the type and target of offensive posts in social media. In: Proceedings of NAACL. 2019.","DOI":"10.18653\/v1\/N19-1144"},{"key":"82_CR68","doi-asserted-by":"crossref","unstructured":"Zampieri M, Malmasi S, Nakov P, Rosenthal S, Farra N, Kumar R. Predicting the type and target of offensive posts in social media. 2019. arXiv:1902.09666.","DOI":"10.18653\/v1\/N19-1144"},{"key":"82_CR69","doi-asserted-by":"crossref","unstructured":"Zampieri M, Malmasi S, Nakov P, Rosenthal S, Farra N, Kumar R. Semeval-2019 task 6: identifying and categorizing offensive language in social media (offenseval). 2019. arXiv:1903.08983.","DOI":"10.18653\/v1\/S19-2010"}],"container-title":["SN Computer Science"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-020-0082-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s42979-020-0082-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-020-0082-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,29]],"date-time":"2023-09-29T07:03:21Z","timestamp":1695971001000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s42979-020-0082-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,3]]},"references-count":69,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2020,3]]}},"alternative-id":["82"],"URL":"https:\/\/doi.org\/10.1007\/s42979-020-0082-0","relation":{},"ISSN":["2662-995X","2661-8907"],"issn-type":[{"type":"print","value":"2662-995X"},{"type":"electronic","value":"2661-8907"}],"subject":[],"published":{"date-parts":[[2020,3]]},"assertion":[{"value":"27 November 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"24 February 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 March 2020","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with Ethical Standards"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"105"}}