{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,11,10]],"date-time":"2026-11-10T04:32:25Z","timestamp":1794285145240,"version":"4.9.0"},"reference-count":55,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T00:00:00Z","timestamp":1725148800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2024,9,1]],"date-time":"2024-09-01T00:00:00Z","timestamp":1725148800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2024,7,4]],"date-time":"2024-07-04T00:00:00Z","timestamp":1720051200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000026","name":"National Institute on Drug Abuse","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000026","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["clinicalkey.fr","clinicalkey.jp","clinicalkey.es","clinicalkey.com.au","clinicalkey.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Psychiatry Research"],"published-print":{"date-parts":[[2024,9]]},"DOI":"10.1016\/j.psychres.2024.116058","type":"journal-article","created":{"date-parts":[[2024,6,26]],"date-time":"2024-06-26T12:48:45Z","timestamp":1719406125000},"page":"116058","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":27,"special_numbering":"C","title":["Evaluating generative AI responses to real-world drug-related questions"],"prefix":"10.1016","volume":"339","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7381-6295","authenticated-orcid":false,"given":"Salvatore","family":"Giorgi","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-3414-9337","authenticated-orcid":false,"given":"Kelsey","family":"Isman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tingting","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1577-8136","authenticated-orcid":false,"given":"Zachary","family":"Fried","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6369-3711","authenticated-orcid":false,"given":"Jo\u00e3o","family":"Sedoc","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2511-3322","authenticated-orcid":false,"given":"Brenda","family":"Curtis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.psychres.2024.116058_bib0001","series-title":"Proceedings of the 2023 Conference on Empirical Meth-ods in Natural Language Processing, Association for Computational Lin-guistics","first-page":"4776","article-title":"Mirages. on anthropomorphism in dialogue systems","author":"Abercrombie","year":"2023"},{"key":"10.1016\/j.psychres.2024.116058_bib0002","doi-asserted-by":"crossref","DOI":"10.1136\/tc-2023-058009","article-title":"Exploring the chatgpt platform with scenario-specific prompts for vaping cessation","author":"Amin","year":"2023","journal-title":"Tob. Control"},{"key":"10.1016\/j.psychres.2024.116058_bib0003","series-title":"Designing Guiding Principles For Nlp For healthcare: A case Study of Maternal Health","author":"Antoniak","year":"2023"},{"key":"10.1016\/j.psychres.2024.116058_bib0004","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1016\/j.drugalcdep.2018.05.005","article-title":"Substance use, recovery, and linguistics: the impact of word choice on explicit and implicit bias","volume":"189","author":"Ashford","year":"2018","journal-title":"Drug Alcohol. Depend."},{"issue":"2","key":"10.1016\/j.psychres.2024.116058_bib0005","doi-asserted-by":"crossref","first-page":"257","DOI":"10.1080\/07347324.2018.1513777","article-title":"abusing addiction\u201d: our lan-guage still isn't good enough","volume":"37","author":"Ashford","year":"2019","journal-title":"Alcoholism Treatm. Quarterly"},{"issue":"6","key":"10.1016\/j.psychres.2024.116058_bib0006","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2023.17517","article-title":"Evaluating artificial intelligence responses to public health questions","volume":"6","author":"Ayers","year":"2023","journal-title":"JAMA Netw. Open"},{"key":"10.1016\/j.psychres.2024.116058_bib0007","series-title":"Proceedings of the international AAAI conference on web and social media","first-page":"830","article-title":"The pushshift reddit dataset","volume":"14","author":"Baumgartner","year":"2020"},{"key":"10.1016\/j.psychres.2024.116058_bib0008","series-title":"Proceedings of the 2021 ACM conference on fairness, accountability, and transparency","first-page":"610","article-title":"On the dangers of stochastic parrots: can language models be too big?","author":"Bender","year":"2021"},{"key":"10.1016\/j.psychres.2024.116058_bib0009","unstructured":"Berglund, L., Tong, M., Kaufmann, M., Balesni, M., Stickland, A.C., Korbak, T., Evans, O., The Reversal curse: Llms trained on\u201d a is b\u201d Fail to learn\u201d b is a\u201d, arXiv preprint arXiv:2309.12288 (2023)."},{"key":"10.1016\/j.psychres.2024.116058_bib0010","series-title":"Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024), ELRA and ICCL","first-page":"3098","article-title":"ChatGPT is a knowledgeable but inexperienced solver: an investigation of commonsense problem in large language models","author":"Bian","year":"2024"},{"issue":"11","key":"10.1016\/j.psychres.2024.116058_bib0011","doi-asserted-by":"crossref","first-page":"e29487","DOI":"10.2196\/29487","article-title":"Studies of depression and anxiety using reddit as a data source: scoping review","volume":"8","author":"Boettcher","year":"2021","journal-title":"JMIR Mental Health"},{"key":"10.1016\/j.psychres.2024.116058_bib0012","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"10.1016\/j.psychres.2024.116058_bib0013","article-title":"Examining stigma relating to sub-stance use and contextual factors in social media discussions","volume":"3","author":"Chen","year":"2022","journal-title":"Drug and Alcohol Depend. Report."},{"key":"10.1016\/j.psychres.2024.116058_bib0014","series-title":"Proceedings of the in-ternational AAAI conference on web and social media","first-page":"71","article-title":"Mental health discourse on reddit: self-disclosure, social support, and anonymity","volume":"8","author":"De Choudhury","year":"2014"},{"key":"10.1016\/j.psychres.2024.116058_bib0015","first-page":"1","article-title":"Using large language models in psychology","author":"Demszky","year":"2023","journal-title":"Nature Reviews Psychology"},{"key":"10.1016\/j.psychres.2024.116058_bib0016","article-title":"A framework for language technologies in behavioral research and clinical applications: ethical challenges, implications, and solutions","author":"Diaz-Asper","year":"2023","journal-title":"Am. Psychol."},{"issue":"3-4","key":"10.1016\/j.psychres.2024.116058_bib0017","doi-asserted-by":"crossref","first-page":"313","DOI":"10.1080\/19371918.2013.774663","article-title":"12-step inter-ventions and mutual support programs for substance use disorders: an overview","volume":"28","author":"Donovan","year":"2013","journal-title":"Social work in public health"},{"issue":"11","key":"10.1016\/j.psychres.2024.116058_bib0018","doi-asserted-by":"crossref","first-page":"494","DOI":"10.3928\/00485713-20201008-01","article-title":"Systemic racism and sub-stance use disorders","volume":"50","author":"Farahmand","year":"2020","journal-title":"Psychiatric Annals"},{"key":"10.1016\/j.psychres.2024.116058_bib0019","series-title":"Proceedings of the International AAAI Conference on Web and Social Media","article-title":"Lived experience matters: automatic detection of stigma on social media toward people who use substances","author":"Giorgi","year":"2024"},{"key":"10.1016\/j.psychres.2024.116058_bib0020","doi-asserted-by":"crossref","DOI":"10.3389\/fpubh.2023.1275975","article-title":"A linguistic analysis of dehumanization toward substance use across three decades of news articles","volume":"11","author":"Giorgi","year":"2023","journal-title":"Frontiers in Public Health"},{"key":"10.1016\/j.psychres.2024.116058_bib0021","doi-asserted-by":"crossref","DOI":"10.1177\/1357633X231181922","article-title":"Chat gpt-4 significantly surpasses gpt-3.5 in drug information queries","author":"He","year":"2023","journal-title":"J. Telemed. Telecare"},{"key":"10.1016\/j.psychres.2024.116058_bib0022","series-title":"Evaluating Risk Progression in Mental Health Chatbots Using Escalating Prompts","author":"Heston","year":"2023"},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0023","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2021.44967","article-title":"Examination of stigmatizing lan-guage in the electronic health record","volume":"5","author":"Himmelstein","year":"2022","journal-title":"JAMA Netw. Open"},{"key":"10.1016\/j.psychres.2024.116058_bib0024","series-title":"Web, Artificial Intelligence and Network Applications: Proceedings of the Workshops of the 33rd International Conference on Advanced Information Networking and Applications (WAINA-2019) 33","first-page":"946","article-title":"A survey on conversational agents\/chatbots classification and design techniques","author":"Hussain","year":"2019"},{"key":"10.1016\/j.psychres.2024.116058_bib0025","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1016\/j.drugalcdep.2016.05.010","article-title":"Primary care physicians\u2019 perspectives on the prescription opioid epidemic","volume":"165","author":"Kennedy-Hendricks","year":"2016","journal-title":"Drug Alcohol Depend."},{"key":"10.1016\/j.psychres.2024.116058_bib0026","doi-asserted-by":"crossref","DOI":"10.1016\/j.drugalcdep.2021.109081","article-title":"Robustness of estimated access to opioid use disorder treatment providers in rural vs. urban areas of the united states","volume":"228","author":"Kiang","year":"2021","journal-title":"Drug Alcohol Depend."},{"key":"10.1016\/j.psychres.2024.116058_bib0027","article-title":"Beyond rating scales: with tar-geted evaluation, language models are poised for psychological assess-ment","author":"Kjell","year":"2023","journal-title":"Psychiatry Res."},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0028","doi-asserted-by":"crossref","first-page":"93","DOI":"10.1038\/s41746-021-00464-x","article-title":"Considering the possibilities and pit-falls of generative pre-trained transformer 3 (gpt-3) in healthcare deliv-ery","volume":"4","author":"Korngiebel","year":"2021","journal-title":"NPJ Digital Med."},{"key":"10.1016\/j.psychres.2024.116058_bib0029","series-title":"Social Media and News Fact Sheet","author":"Liedke","year":"2022"},{"issue":"3","key":"10.1016\/j.psychres.2024.116058_bib0030","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1016\/j.beth.2014.11.002","article-title":"More than re-flections: empathy in motivational interviewing includes language style synchrony between therapist and client","volume":"46","author":"Lord","year":"2015","journal-title":"Behav. Therapy"},{"issue":"6","key":"10.1016\/j.psychres.2024.116058_bib0031","doi-asserted-by":"crossref","first-page":"419","DOI":"10.1016\/j.evolhumbehav.2013.08.001","article-title":"Convergence of speech rate in conversation predicts cooperation","volume":"34","author":"Manson","year":"2013","journal-title":"Evol. Hum. Behav."},{"issue":"3","key":"10.1016\/j.psychres.2024.116058_bib0032","doi-asserted-by":"crossref","first-page":"437","DOI":"10.1162\/COLI_a_00227","article-title":"The unified and holistic method gamma (\u03b3) for inter-annotator agreement measure and align-ment","volume":"41","author":"Mathet","year":"2015","journal-title":"Computat. Linguist."},{"key":"10.1016\/j.psychres.2024.116058_bib0033","series-title":"Self-Stigma and Addiction","first-page":"5","author":"Matthews","year":"2019"},{"issue":"7","key":"10.1016\/j.psychres.2024.116058_bib0034","doi-asserted-by":"crossref","DOI":"10.1001\/jamanetworkopen.2023.23741","article-title":"Provision of digital health technologies for opioid use disorder treatment by us health care organizations","volume":"6","author":"Miller-Rosales","year":"2023","journal-title":"JAMA Netw. Open"},{"key":"10.1016\/j.psychres.2024.116058_bib0035","unstructured":"National survey on drug use and health 2020, Center for Behavioral Health Statistics and Quality (2021). URL https:\/\/www.samhsa.gov\/data\/."},{"key":"10.1016\/j.psychres.2024.116058_bib55","year":"2016"},{"issue":"6","key":"10.1016\/j.psychres.2024.116058_bib0036","doi-asserted-by":"crossref","first-page":"405","DOI":"10.1159\/000525959","article-title":"The use of chatbots as supportive agents for people seeking help with substance use disorder: a systematic review","volume":"28","author":"Ogilvie","year":"2022","journal-title":"Eur. Addict. Res."},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0037","doi-asserted-by":"crossref","first-page":"195","DOI":"10.1038\/s41746-023-00939-z","article-title":"Large language models propagate race-based medicine","volume":"6","author":"Omiye","year":"2023","journal-title":"NPJ Digital Med."},{"key":"10.1016\/j.psychres.2024.116058_bib0038","unstructured":"OpenAI, Introducing chatgpt. URL https:\/\/openai.com\/blog\/chatgpt."},{"key":"10.1016\/j.psychres.2024.116058_bib0039","series-title":"Gpt-4 Technical Report","year":"2023"},{"issue":"2","key":"10.1016\/j.psychres.2024.116058_bib0040","article-title":"Studying red-dit: a systematic overview of disciplines, approaches, methods, and ethics","volume":"7","author":"Proferes","year":"2021","journal-title":"Social Media+ Society"},{"key":"10.1016\/j.psychres.2024.116058_bib0041","series-title":"Proceedings of the 2008 ACM conference on Computer supported cooperative work","first-page":"277","article-title":"Linguistic mimicry and trust in text-based cmc","author":"Scissors","year":"2008"},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0042","doi-asserted-by":"crossref","first-page":"46","DOI":"10.1038\/s42256-022-00593-2","article-title":"Human\u2013ai collaboration enables more empathic conversations in text-based peer-to-peer mental health support","volume":"5","author":"Sharma","year":"2023","journal-title":"Nature Machine Intelligence"},{"issue":"7972","key":"10.1016\/j.psychres.2024.116058_bib0043","doi-asserted-by":"crossref","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"Singhal","year":"2023","journal-title":"Nature"},{"issue":"8","key":"10.1016\/j.psychres.2024.116058_bib0044","doi-asserted-by":"crossref","first-page":"935","DOI":"10.1176\/appi.ps.202000312","article-title":"Barriers to integrated medication-assisted treatment for rural patients with co-occurring disor-ders: the gap in managing addiction","volume":"72","author":"Snell-Rood","year":"2021","journal-title":"Psychiatr. Serv."},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0045","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1038\/s44184-024-00056-z","article-title":"Large language models could change the future of behavioral healthcare: a proposal for responsible development and evaluation","volume":"3","author":"Stade","year":"2024","journal-title":"NPJ Ment. Health Res."},{"issue":"2","key":"10.1016\/j.psychres.2024.116058_bib0046","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1097\/ADM.0000000000000867","article-title":"Po-tential value of the insights and lived experiences of addiction researchers with addiction","volume":"16","author":"Stull","year":"2022","journal-title":"J. Addict. Med."},{"key":"10.1016\/j.psychres.2024.116058_bib0047","series-title":"Llama: Open and Efficient Foundation Language Models","author":"Touvron","year":"2023"},{"issue":"11","key":"10.1016\/j.psychres.2024.116058_bib0048","doi-asserted-by":"crossref","DOI":"10.1371\/journal.pdig.0000143","article-title":"Computational analyses identify addiction help-seeking behaviors on the social networking website reddit: insights into online social interactions and addiction support communities","volume":"1","author":"Valdez","year":"2022","journal-title":"PLOS Digital Health"},{"key":"10.1016\/j.psychres.2024.116058_bib0049","article-title":"Chatgpt: the transformative influence of gen-erative ai on science and healthcare","author":"Varghese","year":"2023","journal-title":"J. Hepatol."},{"issue":"14","key":"10.1016\/j.psychres.2024.116058_bib0050","doi-asserted-by":"crossref","first-page":"1289","DOI":"10.1056\/NEJMp1917360","article-title":"Stigma and the toll of addiction","volume":"382","author":"Volkow","year":"2020","journal-title":"N. Engl. J. Med."},{"issue":"2","key":"10.1016\/j.psychres.2024.116058_bib0051","doi-asserted-by":"crossref","first-page":"330","DOI":"10.1080\/10826084.2017.1363238","article-title":"Barriers to medications for addiction treat-ment: how stigma kills","volume":"53","author":"Wakeman","year":"2018","journal-title":"Subst. Use Misuse"},{"key":"10.1016\/j.psychres.2024.116058_bib0052","doi-asserted-by":"crossref","first-page":"60","DOI":"10.3389\/fdgth.2023.1161098","article-title":"Ai chatbots not yet ready for clinical use","volume":"5","author":"Yeung","year":"2023","journal-title":"Frontiers in Digital Health"},{"issue":"1","key":"10.1016\/j.psychres.2024.116058_bib0053","doi-asserted-by":"crossref","first-page":"e12","DOI":"10.1016\/S2589-7500(23)00225-X","article-title":"As-sessing the potential of gpt-4 to perpetuate racial and gender biases in health care: a model evaluation study","volume":"6","author":"Zack","year":"2024","journal-title":"The Lancet Digital Health"},{"issue":"9","key":"10.1016\/j.psychres.2024.116058_bib0054","doi-asserted-by":"crossref","first-page":"286","DOI":"10.3390\/fi15090286","article-title":"Generative ai in medicine and health-care: promises, opportunities and challenges","volume":"15","author":"Zhang","year":"2023","journal-title":"Future Internet"}],"container-title":["Psychiatry Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0165178124003433?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0165178124003433?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2024,12,2]],"date-time":"2024-12-02T15:56:11Z","timestamp":1733154971000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0165178124003433"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,9]]},"references-count":55,"alternative-id":["S0165178124003433"],"URL":"https:\/\/doi.org\/10.1016\/j.psychres.2024.116058","relation":{},"ISSN":["0165-1781"],"issn-type":[{"value":"0165-1781","type":"print"}],"subject":[],"published":{"date-parts":[[2024,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Evaluating generative AI responses to real-world drug-related questions","name":"articletitle","label":"Article Title"},{"value":"Psychiatry Research","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.psychres.2024.116058","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2024 The Authors. Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"116058"}}