{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,13]],"date-time":"2026-02-13T11:45:25Z","timestamp":1770983125644,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T00:00:00Z","timestamp":1730160000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,29]]},"DOI":"10.1145\/3687123.3698286","type":"proceedings-article","created":{"date-parts":[[2024,11,18]],"date-time":"2024-11-18T06:23:21Z","timestamp":1731911001000},"page":"54-62","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["Evaluation of Code LLMs on Geospatial Code Generation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4587-5586","authenticated-orcid":false,"given":"Piotr","family":"Gramacki","sequence":"first","affiliation":[{"name":"Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3856-2936","authenticated-orcid":false,"given":"Bruno","family":"Martins","sequence":"additional","affiliation":[{"name":"INESC-ID & LUMLIS (Lisbon ELLIS Unit), Instituto Superior T\u00e9cnico, University of Lisbon, Lisbon, Portugal"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7733-3239","authenticated-orcid":false,"given":"Piotr","family":"Szyma\u0144ski","sequence":"additional","affiliation":[{"name":"Wroc\u0142aw University of Science and Technology, Department of Artificial Intelligence Kraina.AI, Wroc\u0142aw, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,11,18]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"AI@Meta. 2024. Llama 3 Model Card. https:\/\/github.com\/meta-llama\/llama3\/blob\/main\/MODEL_CARD.md"},{"key":"e_1_3_2_1_2_1","unstructured":"Jacob Austin Augustus Odena Maxwell Nye Maarten Bosma Henryk Michalewski David Dohan Ellen Jiang Carrie Cai Michael Terry Quoc Le and Charles Sutton. 2021. Program Synthesis with Large Language Models. arXiv:2108.07732 [cs.PL] https:\/\/arxiv.org\/abs\/2108.07732"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3589132.3625625"},{"key":"e_1_3_2_1_4_1","unstructured":"Mark Chen Jerry Tworek Heewoo Jun Qiming Yuan Henrique Ponde de Oliveira Pinto Jared Kaplan Harri Edwards Yuri Burda Nicholas Joseph Greg Brockman Alex Ray Raul Puri Gretchen Krueger Michael Petrov Heidy Khlaaf Girish Sastry Pamela Mishkin Brooke Chan Scott Gray Nick Ryder Mikhail Pavlov Alethea Power Lukasz Kaiser Mohammad Bavarian Clemens Winter Philippe Tillet Felipe Petroski Such Dave Cummings Matthias Plappert Fotios Chantzis Elizabeth Barnes Ariel Herbert-Voss William Hebgen Guss Alex Nichol Alex Paino Nikolas Tezak Jie Tang Igor Babuschkin Suchir Balaji Shantanu Jain William Saunders Christopher Hesse Andrew N. Carr Jan Leike Josh Achiam Vedant Misra Evan Morikawa Alec Radford Matthew Knight Miles Brundage Mira Murati Katie Mayer Peter Welinder Bob McGrew Dario Amodei Sam McCandlish Ilya Sutskever and Wojciech Zaremba. 2021. Evaluating Large Language Models Trained on Code. arXiv:2107.03374 [cs.LG] https:\/\/arxiv.org\/abs\/2107.03374"},{"key":"e_1_3_2_1_5_1","unstructured":"CodeGemma Team. 2024. CodeGemma: Open Code Models Based on Gemma. https:\/\/storage.googleapis.com\/deepmind-media\/gemma\/codegemma_report.pdf"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"Joris Van den Bossche Kelsey Jordahl Martin Fleischmann Matt Richards James McBride Jacob Wasserman Adrian Garcia Badaracco Alan D. Snow Brendan Ward Jeff Tratner Jeffrey Gerard Matthew Perry cjqf Geir Arne Hjelle Mike Taves Ewout ter Hoeven Micah Cochran Ray Bell rraymondgh Matt Bartos Pieter Roggemans Lucas Culbertson Giacomo Caria Nicholas YS Tan Nick Eubank sangarshanan John Flavin Sergio Rey and James Gardiner. 2024. geopandas\/geopandas: vl.0.1. https:\/\/doi.org\/10.5281\/zenodo.12625316","DOI":"10.5281\/zenodo.12625316"},{"key":"e_1_3_2_1_7_1","volume-title":"Gemma: Open Models Based on Gemini Research and Technology. https:\/\/storage.googleapis.com\/deepmind-media\/gemma\/gemma-report.pdf","author":"Team Gemma","year":"2024","unstructured":"Gemma Team. 2024. Gemma: Open Models Based on Gemini Research and Technology. https:\/\/storage.googleapis.com\/deepmind-media\/gemma\/gemma-report.pdf"},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks, J. Vanschoren and S. Yeung (Eds.)","volume":"1","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Steven Basart, Saurav Kadavath, Mantas Mazeika, Akul Arora, Ethan Guo, Collin Burns, Samir Puranik, Horace He, Dawn Song, and Jacob Steinhardt. 2021. Measuring Coding Challenge Competence With APPS. In Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks, J. Vanschoren and S. Yeung (Eds.), Vol. 1. Neural Information Processing Systems Foundation, Inc. (NeurIPS), San Diego, California, USA. https:\/\/datasets-benchmarks-proceedings.neurips.cc\/paper_files\/paper\/2021\/file\/c24cd76e1ce41366a4bbe8a49b02a028-Paper-round2.pdf"},{"key":"e_1_3_2_1_9_1","unstructured":"Albert Q. Jiang Alexandre Sablayrolles Arthur Mensch Chris Bamford Devendra Singh Chaplot Diego de las Casas Florian Bressand Gianna Lengyel Guillaume Lample Lucile Saulnier L\u00e9lio Renard Lavaud Marie-Anne Lachaux Pierre Stock Teven Le Scao Thibaut Lavril Thomas Wang Timoth\u00e9e Lacroix and William El Sayed. 2023. Mistral 7B. arXiv:2310.06825 [cs.CL] https:\/\/arxiv.org\/abs\/2310.06825"},{"key":"e_1_3_2_1_10_1","volume-title":"Jia Li, Chenghao Mou, Carlos Mu\u00f1oz Ferrandis, Yacine Jernite, Margaret Mitchell, Sean Hughes, Thomas Wolf, Dzmitry Bahdanau, Leandro von Werra, and Harm de Vries.","author":"Kocetkov Denis","year":"2022","unstructured":"Denis Kocetkov, Raymond Li, Loubna Ben Allal, Jia Li, Chenghao Mou, Carlos Mu\u00f1oz Ferrandis, Yacine Jernite, Margaret Mitchell, Sean Hughes, Thomas Wolf, Dzmitry Bahdanau, Leandro von Werra, and Harm de Vries. 2022. The Stack: 3 TB of permissively licensed source code. arXiv:2211.15533 [cs.CL] https:\/\/arxiv.org\/abs\/2211.15533"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 40th International Conference on Machine Learning (ICML'23)","author":"Lai Yuhang","year":"2023","unstructured":"Yuhang Lai, Chengxi Li, Yiming Wang, Tianyi Zhang, Ruiqi Zhong, Luke Zettlemoyer, Wen-tau Yih, Daniel Fried, Sida Wang, and Tao Yu. 2023. DS-1000: a natural and reliable benchmark for data science code generation. In Proceedings of the 40th International Conference on Machine Learning (ICML'23). JMLR.org, Honolulu, Hawaii, USA, Article 756, 27 pages."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1080\/17538947.2023.2278895"},{"key":"e_1_3_2_1_13_1","volume-title":"ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out","author":"Lin Chin-Yew","year":"2004","unstructured":"Chin-Yew Lin. 2004. ROUGE: A Package for Automatic Evaluation of Summaries. In Text Summarization Branches Out. Association for Computational Linguistics, Barcelona, Spain, 74--81. https:\/\/aclanthology.org\/W04-1013"},{"key":"e_1_3_2_1_14_1","volume-title":"Yuyao Wang, and LINGMING ZHANG.","author":"Liu Jiawei","year":"2023","unstructured":"Jiawei Liu, Chunqiu Steven Xia, Yuyao Wang, and LINGMING ZHANG. 2023. Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation. In Advances in Neural Information Processing Systems, A. Oh, T. Naumann, A. Globerson, K. Saenko, M. Hardt, and S. Levine (Eds.), Vol. 36. Neural Information Processing Systems Foundation, Inc. (NeurIPS), San Diego, California, USA, 21558--21572. https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2023\/file\/43e9d647ccd3e4b7b5baab53f0368686-Paper-Conference.pdf"},{"key":"e_1_3_2_1_15_1","unstructured":"Anton Lozhkov Raymond Li Loubna Ben Allal Federico Cassano Joel Lamy-Poirier Nouamane Tazi Ao Tang Dmytro Pykhtar Jiawei Liu Yuxiang Wei Tianyang Liu Max Tian Denis Kocetkov Arthur Zucker Younes Belkada Zijian Wang Qian Liu Dmitry Abulkhanov Indraneil Paul Zhuang Li Wen-Ding Li Megan Risdal Jia Li Jian Zhu Terry Yue Zhuo Evgenii Zheltonozhskii Nii Osae Osae Dade Wenhao Yu Lucas Krau\u00df Naman Jain Yixuan Su Xuanli He Manan Dey Edoardo Abati Yekun Chai Niklas Muennighoff Xiangru Tang Muhtasham Oblokulov Christopher Akiki Marc Marone Chenghao Mou Mayank Mishra Alex Gu Binyuan Hui Tri Dao Armel Zebaze Olivier Dehaene Nicolas Patry Canwen Xu Julian McAuley Han Hu Torsten Scholak Sebastien Paquet Jennifer Robinson Carolyn Jane Anderson Nicolas Chapados Mostofa Patwary Nima Tajbakhsh Yacine Jernite Carlos Mu\u00f1oz Ferrandis Lingming Zhang Sean Hughes Thomas Wolf Arjun Guha Leandro von Werra and Harm de Vries. 2024. StarCoder 2 and The Stack v2: The Next Generation. arXiv:2402.19173 [cs.SE] https:\/\/arxiv.org\/abs\/2402.19173"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3653070"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.5194\/agilegiss-2-8-2021"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceedings of the 41st International Conference on Machine Learning (Proceedings of Machine Learning Research","volume":"34669","author":"Manvi Rohin","year":"2024","unstructured":"Rohin Manvi, Samar Khanna, Marshall Burke, David B. Lobell, and Stefano Ermon. 2024. Large Language Models are Geographically Biased. In Proceedings of the 41st International Conference on Machine Learning (Proceedings of Machine Learning Research, Vol. 235), Ruslan Salakhutdinov, Zico Kolter, Katherine Heller, Adrian Weller, Nuria Oliver, Jonathan Scarlett, and Felix Berkenkamp (Eds.). PMLR, Vienna, Austria, 34654--34669. https:\/\/proceedings.mlr.press\/v235\/manvi24a.html"},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the 12th International Conference on Learning Representations. ICLR","author":"Manvi Rohin","year":"2024","unstructured":"Rohin Manvi, Samar Khanna, Gengchen Mai, Marshall Burke, David B. Lobell, and Stefano Ermon. 2024. GeoLLM: Extracting Geospatial Knowledge from Large Language Models. In Proceedings of the 12th International Conference on Learning Representations. ICLR, Vienna, Austria. https:\/\/openreview.net\/forum?id=TqL2xBwXP3"},{"key":"e_1_3_2_1_20_1","unstructured":"Mayank Mishra Matt Stallone Gaoyuan Zhang Yikang Shen Aditya Prasad Adriana Meza Soria Michele Merler Parameswaran Selvam Saptha Surendran Shivdeep Singh Manish Sethi Xuan-Hong Dang Pengyuan Li Kun-Lung Wu Syed Zawad Andrew Coleman Matthew White Mark Lewis Raju Pavuluri Yan Koyfman Boris Lublinsky Maximilien de Bayser Ibrahim Abdelaziz Kinjal Basu Mayank Agarwal Yi Zhou Chris Johnson Aanchal Goyal Hima Patel Yousaf Shah Petros Zerfos Heiko Ludwig Asim Munawar Maxwell Crouse Pavan Kapanipathi Shweta Salaria Bob Calio Sophia Wen Seetharami Seelam Brian Belgodere Carlos Fonseca Amith Singhee Nirmit Desai David D. Cox Ruchir Puri and Rameswar Panda. 2024. Granite Code Models: A Family of Open Foundation Models for Code Intelligence. arXiv:2405.04324 [cs.AI] https:\/\/arxiv.org\/abs\/2405.04324"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3615886.3627745"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_1_23_1","unstructured":"Jonathan Roberts Timo L\u00fcddecke Sowmen Das Kai Han and Samuel Albanie. 2023. GPT4GEO: How a Language Model Sees the World's Geography. arXiv:2306.00020 [cs.CL] https:\/\/arxiv.org\/abs\/2306.00020"},{"key":"e_1_3_2_1_24_1","unstructured":"Baptiste Rozi\u00e8re Jonas Gehring Fabian Gloeckle Sten Sootla Itai Gat Xiaoqing Ellen Tan Yossi Adi Jingyu Liu Romain Sauvestre Tal Remez J\u00e9r\u00e9my Rapin Artyom Kozhevnikov Ivan Evtimov Joanna Bitton Manish Bhatt Cristian Canton Ferrer Aaron Grattafiori Wenhan Xiong Alexandre D\u00e9fossez Jade Copet Faisal Azhar Hugo Touvron Louis Martin Nicolas Usunier Thomas Scialom and Gabriel Synnaeve. 2024. Code Llama: Open Foundation Models for Code. arXiv:2308.12950 [cs.CL] https:\/\/arxiv.org\/abs\/2308.12950"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1080\/17538947.2020.1738568"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1080\/17538947.2024.2353122"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-demos.6"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2022\/329"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2023.acl-long.411"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jag.2024.103976"}],"event":{"name":"SIGSPATIAL '24: The 32nd ACM International Conference on Advances in Geographic Information Systems","location":"Atlanta GA USA","acronym":"SIGSPATIAL '24","sponsor":["SIGSPATIAL ACM Special Interest Group on Spatial Information"]},"container-title":["Proceedings of the 7th ACM SIGSPATIAL International Workshop on AI for Geographic Knowledge Discovery"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3687123.3698286","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3687123.3698286","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T02:28:12Z","timestamp":1755916092000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3687123.3698286"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,29]]},"references-count":30,"alternative-id":["10.1145\/3687123.3698286","10.1145\/3687123"],"URL":"https:\/\/doi.org\/10.1145\/3687123.3698286","relation":{},"subject":[],"published":{"date-parts":[[2024,10,29]]},"assertion":[{"value":"2024-11-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}