{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,21]],"date-time":"2026-05-21T04:27:29Z","timestamp":1779337649407,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,6,3]],"date-time":"2024-06-03T00:00:00Z","timestamp":1717372800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/https:\/\/doi.org\/10.13039\/100006754","name":"Army Research Laboratory","doi-asserted-by":"publisher","award":["W911NF2120076"],"award-info":[{"award-number":["W911NF2120076"]}],"id":[{"id":"10.13039\/https:\/\/doi.org\/10.13039\/100006754","id-type":"DOI","asserted-by":"publisher"}]},{"name":"ARO","award":["310902-00001"],"award-info":[{"award-number":["310902-00001"]}]},{"name":"DARPA\/GARD","award":["HR001119S0026"],"award-info":[{"award-number":["HR001119S0026"]}]},{"DOI":"10.13039\/https:\/\/doi.org\/10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CCF2212458"],"award-info":[{"award-number":["CCF2212458"]}],"id":[{"id":"10.13039\/https:\/\/doi.org\/10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"name":"ONR","award":["N00014-22-1-2271"],"award-info":[{"award-number":["N00014-22-1-2271"]}]},{"name":"NSF Career","award":["1942230"],"award-info":[{"award-number":["1942230"]}]},{"name":"DARPA\/RED","award":["HR00112090132"],"award-info":[{"award-number":["HR00112090132"]}]},{"name":"NSF Trails","award":["2229885"],"award-info":[{"award-number":["2229885"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,6,3]]},"DOI":"10.1145\/3630106.3658967","type":"proceedings-article","created":{"date-parts":[[2024,6,5]],"date-time":"2024-06-05T13:14:21Z","timestamp":1717593261000},"page":"1211-1228","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":24,"title":["WorldBench: Quantifying Geographic Disparities in LLM Factual Recall"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0006-2842-5352","authenticated-orcid":false,"given":"Mazda","family":"Moayeri","sequence":"first","affiliation":[{"name":"University of Maryland, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9536-2722","authenticated-orcid":false,"given":"Elham","family":"Tabassi","sequence":"additional","affiliation":[{"name":"Michigan State University, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0944-8242","authenticated-orcid":false,"given":"Soheil","family":"Feizi","sequence":"additional","affiliation":[{"name":"University of Maryland, United States of America"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,6,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Ayush Agrawal Mirac Suzgun Lester Mackey and Adam\u00a0Tauman Kalai. 2023. Do Language Models Know When They\u2019re Hallucinating References?arxiv:2305.18248\u00a0[cs.CL]"},{"key":"e_1_3_2_1_2_1","unstructured":"Cohere AI. 2023. Cohere API. https:\/\/github.com\/cohere-ai\/cohere-python"},{"key":"e_1_3_2_1_3_1","unstructured":"Rohan Anil et al. 2023. PaLM 2 Technical Report. arxiv:2305.10403\u00a0[cs.CL]"},{"key":"e_1_3_2_1_5_1","volume-title":"Retrieved","author":"Bank World","year":"2024","unstructured":"World Bank. 2024. World Bank Country and Lending Groups. Retrieved January 16, 2024 from https:\/\/datahelpdesk.worldbank.org\/knowledgebase\/articles\/906519-world-bank-country-and-lending-groups#:\u00a0:text=For%20the%20current%202024%20fiscal, those%20with%20a%20GNI%20per"},{"key":"e_1_3_2_1_6_1","volume-title":"Retrieved","author":"Bank World","year":"2024","unstructured":"World Bank. 2024. World Bank Open Data. Retrieved January 16, 2024 from https:\/\/data.worldbank.org\/"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of the 1st Conference on Fairness, Accountability and Transparency(Proceedings of Machine Learning Research, Vol.\u00a081)","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification. In Proceedings of the 1st Conference on Fairness, Accountability and Transparency(Proceedings of Machine Learning Research, Vol.\u00a081), Sorelle\u00a0A. Friedler and Christo Wilson (Eds.). PMLR, 77\u201391. https:\/\/proceedings.mlr.press\/v81\/buolamwini18a.html"},{"key":"e_1_3_2_1_8_1","volume-title":"Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification. In Conference on Fairness, Accountability and Transparency, FAT 2018","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification. In Conference on Fairness, Accountability and Transparency, FAT 2018, 23-24 February 2018, New York, NY, USA(Proceedings of Machine Learning Research, Vol.\u00a081), Sorelle\u00a0A. Friedler and Christo Wilson (Eds.). PMLR, 77\u201391. http:\/\/proceedings.mlr.press\/v81\/buolamwini18a.html"},{"key":"e_1_3_2_1_9_1","unstructured":"Terrance DeVries Ishan Misra Changhan Wang and Laurens van\u00a0der Maaten. 2019. Does Object Recognition Work for Everyone?arxiv:1906.02659\u00a0[cs.CV]"},{"key":"e_1_3_2_1_10_1","unstructured":"Laura Gustafson Megan Richards Melissa Hall Caner Hazirbas Diane Bouchacourt and Mark Ibrahim. 2023. Pinpointing Why Object Recognition Performance Degrades Across Income Levels and Geographies. arxiv:2304.05391\u00a0[cs.CV]"},{"key":"e_1_3_2_1_11_1","volume-title":"DIG In: Evaluating Disparities in Image Generations with Indicators for Geographic Diversity. Transactions on Machine Learning Research","author":"Hall Melissa","year":"2024","unstructured":"Melissa Hall, Candace Ross, Adina Williams, Nicolas Carion, Michal Drozdzal, and Adriana Romero-Soriano. 2024. DIG In: Evaluating Disparities in Image Generations with Indicators for Geographic Diversity. Transactions on Machine Learning Research (2024). https:\/\/openreview.net\/forum?id=FDt2UGM1Nz Featured Certification."},{"key":"e_1_3_2_1_12_1","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin and Ting Liu. 2023. A Survey on Hallucination in Large Language Models: Principles Taxonomy Challenges and Open Questions. arxiv:2311.05232\u00a0[cs.CL]"},{"key":"e_1_3_2_1_13_1","unstructured":"Albert\u00a0Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra\u00a0Singh Chaplot Diego de\u00a0las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio\u00a0Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven\u00a0Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William\u00a0El Sayed. 2023. Mistral 7B. arxiv:2310.06825\u00a0[cs.CL]"},{"key":"e_1_3_2_1_14_1","unstructured":"Nikhil Kandpal Haikang Deng Adam Roberts Eric Wallace and Colin Raffel. 2023. Large Language Models Struggle to Learn Long-Tail Knowledge. arxiv:2211.08411\u00a0[cs.CL]"},{"key":"e_1_3_2_1_15_1","unstructured":"Jungo Kasai Yuhei Kasai Keisuke Sakaguchi Yutaro Yamada and Dragomir Radev. 2023. Evaluating GPT-4 and ChatGPT on Japanese Medical Licensing Examinations. arxiv:2303.18027\u00a0[cs.CL]"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4389233"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"crossref","unstructured":"Alex Mallen Akari Asai Victor Zhong Rajarshi Das Daniel Khashabi and Hannaneh Hajishirzi. 2023. When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories. arxiv:2212.10511\u00a0[cs.CL]","DOI":"10.18653\/v1\/2023.acl-long.546"},{"key":"e_1_3_2_1_18_1","unstructured":"Arindam Mitra Luciano\u00a0Del Corro Shweti Mahajan Andres Codas Clarisse Simoes Sahaj Agrawal Xuxi Chen Anastasia Razdaibiedina Erik Jones Kriti Aggarwal Hamid Palangi Guoqing Zheng Corby Rosset Hamed Khanpour and Ahmed Awadallah. 2023. Orca 2: Teaching Small Language Models How to Reason. arxiv:2311.11045\u00a0[cs.AI]"},{"key":"e_1_3_2_1_19_1","volume-title":"Capabilities of GPT-4 on Medical Challenge Problems. ArXiv abs\/2303.13375","author":"Nori Harsha","year":"2023","unstructured":"Harsha Nori, Nicholas King, Scott\u00a0Mayer McKinney, Dean Carignan, and Eric Horvitz. 2023. Capabilities of GPT-4 on Medical Challenge Problems. ArXiv abs\/2303.13375 (2023). https:\/\/api.semanticscholar.org\/CorpusID:257687695"},{"key":"e_1_3_2_1_20_1","unstructured":"Jessica Ojo Kelechi Ogueji Pontus Stenetorp and David\u00a0I. Adelani. 2023. How good are Large Language Models on African Languages?arxiv:2311.07978\u00a0[cs.CL]"},{"key":"e_1_3_2_1_21_1","unstructured":"OpenAI. 2023. OpenAI API. https:\/\/openai.com\/product"},{"key":"e_1_3_2_1_22_1","volume-title":"Code Llama: Open Foundation Models for Code. arxiv:2308.12950\u00a0[cs.CL]","author":"Rozi\u00e8re Baptiste","year":"2023","unstructured":"Baptiste Rozi\u00e8re, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing\u00a0Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, J\u00e9r\u00e9my Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian\u00a0Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre D\u00e9fossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, and Gabriel Synnaeve. 2023. Code Llama: Open Foundation Models for Code. arxiv:2308.12950\u00a0[cs.CL]"},{"key":"e_1_3_2_1_23_1","unstructured":"Kai Sun Yifan\u00a0Ethan Xu Hanwen Zha Yue Liu and Xin\u00a0Luna Dong. 2023. Head-to-Tail: How Knowledgeable are Large Language Models (LLM)? A.K.A. Will LLMs Replace Knowledge Graphs?arxiv:2308.10168\u00a0[cs.CL]"},{"key":"e_1_3_2_1_24_1","volume-title":"Recitation-Augmented Language Models. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=-cqvvvb-NkI","author":"Sun Zhiqing","year":"2023","unstructured":"Zhiqing Sun, Xuezhi Wang, Yi Tay, Yiming Yang, and Denny Zhou. 2023. Recitation-Augmented Language Models. In The Eleventh International Conference on Learning Representations. https:\/\/openreview.net\/forum?id=-cqvvvb-NkI"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Liyan Tang Tanya Goyal Alexander\u00a0R. Fabbri Philippe Laban Jiacheng Xu Semih Yavuz Wojciech Kry\u015bci\u0144ski Justin\u00a0F. Rousseau and Greg Durrett. 2023. Understanding Factual Errors in Summarization: Errors Summarizers Datasets Error Detectors. arxiv:2205.12854\u00a0[cs.CL]","DOI":"10.18653\/v1\/2023.acl-long.650"},{"key":"e_1_3_2_1_26_1","volume-title":"Gemini: A Family of Highly Capable Multimodal Models. arxiv:2312.11805\u00a0[cs.CL]","author":"Team Gemini","year":"2023","unstructured":"Gemini Team. 2023. Gemini: A Family of Highly Capable Multimodal Models. arxiv:2312.11805\u00a0[cs.CL]"},{"key":"e_1_3_2_1_27_1","volume-title":"Llama 2: Open Foundation and Fine-Tuned Chat Models. ArXiv abs\/2307.09288","author":"Touvron Hugo","year":"2023","unstructured":"Hugo Touvron, Louis Martin, Kevin\u00a0R. Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Daniel\u00a0M. Bikel, Lukas Blecher, Cristian\u00a0Cant\u00f3n Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony\u00a0S. Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez, Madian Khabsa, Isabel\u00a0M. Kloumann, A.\u00a0V. Korenev, Punit\u00a0Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushkar Mishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric\u00a0Michael Smith, R. Subramanian, Xia Tan, Binh Tang, Ross Taylor, Adina Williams, Jian\u00a0Xiang Kuan, Puxin Xu, Zhengxu Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, and Thomas Scialom. 2023. Llama 2: Open Foundation and Fine-Tuned Chat Models. ArXiv abs\/2307.09288 (2023). https:\/\/api.semanticscholar.org\/CorpusID:259950998"},{"key":"e_1_3_2_1_28_1","volume-title":"Zephyr: Direct Distillation of LM Alignment. arxiv:2310.16944\u00a0[cs.LG]","author":"Tunstall Lewis","year":"2023","unstructured":"Lewis Tunstall, Edward Beeching, Nathan Lambert, Nazneen Rajani, Kashif Rasul, Younes Belkada, Shengyi Huang, Leandro von Werra, Cl\u00e9mentine Fourrier, Nathan Habib, Nathan Sarrazin, Omar Sanseviero, Alexander\u00a0M. Rush, and Thomas Wolf. 2023. Zephyr: Direct Distillation of LM Alignment. arxiv:2310.16944\u00a0[cs.LG]"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Thomas Wolf Lysandre Debut Victor Sanh Julien Chaumond Clement Delangue Anthony Moi Pierric Cistac Tim Rault R\u00e9mi Louf Morgan Funtowicz Joe Davison Sam Shleifer Patrick von Platen Clara Ma Yacine Jernite Julien Plu Canwen Xu Teven\u00a0Le Scao Sylvain Gugger Mariama Drame Quentin Lhoest and Alexander\u00a0M. Rush. 2020. HuggingFace\u2019s Transformers: State-of-the-art Natural Language Processing. arxiv:1910.03771\u00a0[cs.CL]","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_1_30_1","unstructured":"Tianyi Zhang Faisal Ladhak Esin Durmus Percy Liang Kathleen McKeown and Tatsunori\u00a0B. Hashimoto. 2023. Benchmarking Large Language Models for News Summarization. arxiv:2301.13848\u00a0[cs.CL]"},{"key":"e_1_3_2_1_31_1","volume-title":"GeoGPT: Understanding and Processing Geospatial Tasks through An Autonomous GPT. ArXiv abs\/2307.07930","author":"Zhang Yifan","year":"2023","unstructured":"Yifan Zhang, Cheng Wei, Shangyou Wu, Zhengting He, and Wenhao Yu. 2023. GeoGPT: Understanding and Processing Geospatial Tasks through An Autonomous GPT. ArXiv abs\/2307.07930 (2023). https:\/\/api.semanticscholar.org\/CorpusID:259937048"},{"key":"e_1_3_2_1_32_1","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric\u00a0P. Xing Hao Zhang Joseph\u00a0E. Gonzalez and Ion Stoica. 2023. Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena. arxiv:2306.05685\u00a0[cs.CL]"}],"event":{"name":"FAccT '24: The 2024 ACM Conference on Fairness, Accountability, and Transparency","location":"Rio de Janeiro Brazil","acronym":"FAccT '24"},"container-title":["The 2024 ACM Conference on Fairness, Accountability, and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3658967","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3630106.3658967","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3630106.3658967","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:50:58Z","timestamp":1750287058000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3630106.3658967"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,3]]},"references-count":31,"alternative-id":["10.1145\/3630106.3658967","10.1145\/3630106"],"URL":"https:\/\/doi.org\/10.1145\/3630106.3658967","relation":{},"subject":[],"published":{"date-parts":[[2024,6,3]]},"assertion":[{"value":"2024-06-05","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}