{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T17:32:52Z","timestamp":1773768772139,"version":"3.50.1"},"reference-count":47,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,8,1]],"date-time":"2026-08-01T00:00:00Z","timestamp":1785542400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Computers in Human Behavior"],"published-print":{"date-parts":[[2026,8]]},"DOI":"10.1016\/j.chb.2026.108988","type":"journal-article","created":{"date-parts":[[2026,3,9]],"date-time":"2026-03-09T16:35:52Z","timestamp":1773074152000},"page":"108988","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["Rubric-conditioned large language model labeling: Agreement, uncertainty, and label consistency in subjective text annotation"],"prefix":"10.1016","volume":"181","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-5922-6643","authenticated-orcid":false,"given":"Jin","family":"Liu","sequence":"first","affiliation":[]}],"member":"78","reference":[{"issue":"S1","key":"10.1016\/j.chb.2026.108988_b1","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1093\/poq\/nfw088","article-title":"Effects of mobile versus PC web on survey response quality: A crossover experiment in a probability web panel","volume":"81","author":"Antoun","year":"2017","journal-title":"Public Opinion Quarterly"},{"issue":"3","key":"10.1016\/j.chb.2026.108988_b2","doi-asserted-by":"crossref","first-page":"337","DOI":"10.1017\/pan.2023.2","article-title":"Out of one, many: Using language models to simulate human samples","volume":"31","author":"Argyle","year":"2023","journal-title":"Political Analysis"},{"issue":"4","key":"10.1016\/j.chb.2026.108988_b3","doi-asserted-by":"crossref","first-page":"343","DOI":"10.1016\/S0895-4356(00)00314-0","article-title":"Adjusting for multiple testing\u2014When and how?","volume":"54","author":"Bender","year":"2001","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/j.chb.2026.108988_b4","series-title":"Natural language processing with python","author":"Bird","year":"2009"},{"key":"10.1016\/j.chb.2026.108988_b5","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests","volume":"45","author":"Breiman","year":"2001","journal-title":"Machine Learning"},{"issue":"1","key":"10.1016\/j.chb.2026.108988_b6","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1175\/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2","article-title":"Verification of forecasts expressed in terms of probability","volume":"78","author":"Brier","year":"1950","journal-title":"Monthly Weather Review"},{"key":"10.1016\/j.chb.2026.108988_b7","series-title":"Proceedings of the 2014 conference on empirical methods in natural language processing","first-page":"1724","article-title":"Learning phrase representations using RNN encoder\u2013decoder for statistical machine translation","author":"Cho","year":"2014"},{"issue":"3","key":"10.1016\/j.chb.2026.108988_b8","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1023\/A:1022627411411","article-title":"Support-vector networks","volume":"20","author":"Cortes","year":"1995","journal-title":"Machine Learning"},{"key":"10.1016\/j.chb.2026.108988_b9","first-page":"512","article-title":"Automated hate speech detection and the problem of offensive language","volume":"Vol. 11","author":"Davidson","year":"2017"},{"key":"10.1016\/j.chb.2026.108988_b10","series-title":"Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies","first-page":"4171","article-title":"BERT: Pre-training of deep bidirectional transformers for language understanding","author":"Devlin","year":"2019"},{"key":"10.1016\/j.chb.2026.108988_b11","doi-asserted-by":"crossref","first-page":"14497","DOI":"10.1038\/s41598-025-99167-6","article-title":"Trade-offs between machine learning and deep learning for mental illness detection on social media","volume":"15","author":"Ding","year":"2025","journal-title":"Scientific Reports"},{"issue":"3","key":"10.1016\/j.chb.2026.108988_b12","first-page":"403","article-title":"Empirical methodology for crowdsourcing ground truth","volume":"12","author":"Dumitrache","year":"2021","journal-title":"Semantic Web: \u2013 Interoperability, Usability, Applicability"},{"key":"10.1016\/j.chb.2026.108988_b13","series-title":"An introduction to the bootstrap","author":"Efron","year":"1994"},{"issue":"44","key":"10.1016\/j.chb.2026.108988_b14","doi-asserted-by":"crossref","first-page":"11203","DOI":"10.1073\/pnas.1802331115","article-title":"Facebook language predicts depression in medical records","volume":"115","author":"Eichstaedt","year":"2018","journal-title":"Proceedings of the National Academy of Sciences of the United States of America"},{"issue":"5","key":"10.1016\/j.chb.2026.108988_b15","doi-asserted-by":"crossref","first-page":"845","DOI":"10.1109\/TNNLS.2013.2292894","article-title":"Classification in the presence of label noise: A survey","volume":"25","author":"Frenay","year":"2014","journal-title":"IEEE Transactions on Neural Networks and Learning Systems"},{"issue":"5","key":"10.1016\/j.chb.2026.108988_b16","doi-asserted-by":"crossref","first-page":"1189","DOI":"10.1214\/aos\/1013203451","article-title":"Greedy function approximation: A gradient boosting machine","volume":"29","author":"Friedman","year":"2001","journal-title":"Annals of Statistics"},{"issue":"30","key":"10.1016\/j.chb.2026.108988_b17","article-title":"ChatGPT outperforms crowd-workers for text-annotation tasks","volume":"120","author":"Gilardi","year":"2023","journal-title":"Proceedings of the National Academy of Sciences of the United States of America"},{"key":"10.1016\/j.chb.2026.108988_b18","series-title":"Applied logistic regression","author":"Hosmer","year":"2000"},{"key":"10.1016\/j.chb.2026.108988_b19","series-title":"Language models (mostly) know what they know","author":"Kadavath","year":"2022"},{"key":"10.1016\/j.chb.2026.108988_b20","first-page":"3149","article-title":"LightGBM: A highly efficient gradient boosting decision tree","volume":"Vol. 30","author":"Ke","year":"2017"},{"key":"10.1016\/j.chb.2026.108988_b21","series-title":"Proceedings of the 2014 conference on empirical methods in natural language processing","first-page":"1746","article-title":"Convolutional neural networks for sentence classification","author":"Kim","year":"2014"},{"issue":"6","key":"10.1016\/j.chb.2026.108988_b22","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1037\/a0039210","article-title":"Facebook as a research tool for the social sciences: Opportunities, challenges, ethical considerations, and practical guidelines","volume":"70","author":"Kosinski","year":"2015","journal-title":"American Psychologist"},{"key":"10.1016\/j.chb.2026.108988_b23","series-title":"Proceedings of the 8th international conference on learning representations","article-title":"ALBERT: A lite BERT for self-supervised learning of language representations","author":"Lan","year":"2020"},{"key":"10.1016\/j.chb.2026.108988_b24","series-title":"The thirty-ninth annual conference on neural information processing systems","article-title":"ACT as human: Multimodal large language model data annotation with critical thinking","author":"Lin","year":"2025"},{"key":"10.1016\/j.chb.2026.108988_b25","series-title":"Findings of the association for computational linguistics","first-page":"5609","article-title":"Is LLM an overconfident judge? Unveiling the capabilities of LLMs in detecting offensive language with annotation disagreement","author":"Lu","year":"2025"},{"key":"10.1016\/j.chb.2026.108988_b26","series-title":"Introduction to information retrieval","author":"Manning","year":"2008"},{"key":"10.1016\/j.chb.2026.108988_b27","series-title":"Proceedings of the 10th ACM conference on web science","first-page":"173","article-title":"Spread of hate speech in online social media","author":"Mathew","year":"2019"},{"key":"10.1016\/j.chb.2026.108988_b28","series-title":"Proceedings of the 7th ACM IKDD CoDS and 25th COMAD","first-page":"116","article-title":"Interaction dynamics between hate and counter users on Twitter","author":"Mathew","year":"2020"},{"key":"10.1016\/j.chb.2026.108988_b29","first-page":"14867","article-title":"HateXplain: A benchmark dataset for explainable hate speech detection","volume":"Vol. 35","author":"Mathew","year":"2021"},{"issue":"2","key":"10.1016\/j.chb.2026.108988_b30","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1007\/BF02295996","article-title":"Note on the sampling error of the difference between correlated proportions or percentages","volume":"12","author":"McNemar","year":"1947","journal-title":"Psychometrika"},{"key":"10.1016\/j.chb.2026.108988_b31","series-title":"Proceedings of the 2025 conference on empirical methods in natural language processing","first-page":"26782","article-title":"Are LLMs better than reported? Detecting label errors and mitigating their effect on model performance","author":"Nahum","year":"2025"},{"key":"10.1016\/j.chb.2026.108988_b32","doi-asserted-by":"crossref","first-page":"1373","DOI":"10.1613\/jair.1.12125","article-title":"Confident learning: Estimating uncertainty in dataset labels","volume":"70","author":"Northcutt","year":"2021","journal-title":"Journal of Artificial Intelligence Research"},{"key":"10.1016\/j.chb.2026.108988_b33","series-title":"Openai API documentation: gpt-4o-mini","author":"OpenAI","year":"2024"},{"key":"10.1016\/j.chb.2026.108988_b34","series-title":"ChatGPT","author":"OpenAI","year":"2025"},{"key":"10.1016\/j.chb.2026.108988_b35","series-title":"Proceedings of the 2019 conference on empirical methods in natural language processing and the 9th international joint conference on natural language processing","first-page":"4675","article-title":"Multilingual and multi-aspect hate speech analysis","author":"Ousidhoum","year":"2019"},{"key":"10.1016\/j.chb.2026.108988_b36","unstructured":"Paszke, A., Gross, S., Massa, F., Lerer, A., Bradbury, J., Chanan, G., Killeen, T., Lin, Z., Gimelshein, N., Antiga, L., Desmaison, A., Kopf, A., Yang, E., DeVito, Z., Raison, M., Tejani, A., Chilamkurthy, S., Steiner, B., Fang, L., .... Chintala, S. (2019). PyTorch: An Imperative Style, High-Performance Deep Learning Library. In Advances in neural information processing systems 32 (pp. 8026\u20138037). URL: https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2019\/file\/bdbca288fee7f92f2bfa9f7012727740-Paper.pdf."},{"key":"10.1016\/j.chb.2026.108988_b37","first-page":"2825","article-title":"Scikit-learn: Machine learning in python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"10.1016\/j.chb.2026.108988_b38","series-title":"Bit by bit: Social research in the digital age","author":"Salganik","year":"2017"},{"issue":"3","key":"10.1016\/j.chb.2026.108988_b39","doi-asserted-by":"crossref","first-page":"379","DOI":"10.1002\/j.1538-7305.1948.tb01338.x","article-title":"A mathematical theory of communication","volume":"27","author":"Shannon","year":"1948","journal-title":"Bell System Technical Journal"},{"key":"10.1016\/j.chb.2026.108988_b40","series-title":"Towards consistent detection of cognitive distortions: LLM-based annotation and dataset-agnostic evaluation","author":"Sharma","year":"2025"},{"key":"10.1016\/j.chb.2026.108988_b41","series-title":"Proceedings of the 2008 conference on empirical methods in natural language processing","first-page":"254","article-title":"Cheap and fast \u2013 but is it good? Evaluating non-expert annotations for natural language tasks","author":"Snow","year":"2008"},{"issue":"4","key":"10.1016\/j.chb.2026.108988_b42","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1016\/j.ipm.2009.03.002","article-title":"A systematic analysis of performance measures for classification tasks","volume":"45","author":"Sokolova","year":"2009","journal-title":"Information Processing & Management"},{"key":"10.1016\/j.chb.2026.108988_b43","doi-asserted-by":"crossref","first-page":"1385","DOI":"10.1613\/jair.1.12752","article-title":"Learning from disagreement: A survey","volume":"72","author":"Uma","year":"2021","journal-title":"Journal of Artificial Intelligence Research"},{"key":"10.1016\/j.chb.2026.108988_b44","series-title":"Proceedings of the 2020 conference on empirical methods in natural language processing: system demonstrations","first-page":"38","article-title":"Transformers: State-of-the-art natural language processing","author":"Wolf","year":"2020"},{"key":"10.1016\/j.chb.2026.108988_b45","series-title":"Proceedings of the 26th international conference on world wide web","first-page":"1391","article-title":"Ex Machina: Personal attacks seen at scale","author":"Wulczyn","year":"2017"},{"issue":"2","key":"10.1016\/j.chb.2026.108988_b46","doi-asserted-by":"crossref","first-page":"43","DOI":"10.35566\/jbds\/xu2025","article-title":"More than a model: The compounding impact of behavioral ambiguity and task complexity on hate speech detection","volume":"5","author":"Xu","year":"2025","journal-title":"Journal of Behavioral Data Science"},{"issue":"3","key":"10.1016\/j.chb.2026.108988_b47","doi-asserted-by":"crossref","first-page":"107","DOI":"10.1145\/3446776","article-title":"Understanding deep learning (still) requires rethinking generalization","volume":"64","author":"Zhang","year":"2021","journal-title":"Communications of the ACM"}],"container-title":["Computers in Human Behavior"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0747563226000853?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0747563226000853?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T14:32:39Z","timestamp":1773757959000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0747563226000853"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,8]]},"references-count":47,"alternative-id":["S0747563226000853"],"URL":"https:\/\/doi.org\/10.1016\/j.chb.2026.108988","relation":{},"ISSN":["0747-5632"],"issn-type":[{"value":"0747-5632","type":"print"}],"subject":[],"published":{"date-parts":[[2026,8]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Rubric-conditioned large language model labeling: Agreement, uncertainty, and label consistency in subjective text annotation","name":"articletitle","label":"Article Title"},{"value":"Computers in Human Behavior","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.chb.2026.108988","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 Elsevier Ltd. All rights are reserved, including those for text and data mining, AI training, and similar technologies.","name":"copyright","label":"Copyright"}],"article-number":"108988"}}