{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T10:11:40Z","timestamp":1783764700394,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,4,12]],"date-time":"2026-04-12T00:00:00Z","timestamp":1775952000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,4,12]]},"DOI":"10.1145\/3786583.3786851","type":"proceedings-article","created":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T09:12:14Z","timestamp":1783761134000},"page":"34-45","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["RovoDev Code Reviewer: A Large-Scale Online Evaluation of LLM-based Code Review Automation at Atlassian"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5516-9984","authenticated-orcid":false,"given":"Kla","family":"Tantithamthavorn","sequence":"first","affiliation":[{"name":"Monash University, Melbourne, Australia and Atlassian, Melbourne, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-9699-0429","authenticated-orcid":false,"given":"Yaotian","family":"Zou","sequence":"additional","affiliation":[{"name":"Atlassian, Seattle, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-7426-204X","authenticated-orcid":false,"given":"Andy","family":"Wong","sequence":"additional","affiliation":[{"name":"Atlassian, Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-4406-2556","authenticated-orcid":false,"given":"Michael","family":"Gupta","sequence":"additional","affiliation":[{"name":"Atlassian, Seattle, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-8179-3347","authenticated-orcid":false,"given":"Zhe","family":"Wang","sequence":"additional","affiliation":[{"name":"Atlassian, Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-7856-7882","authenticated-orcid":false,"given":"Mike","family":"Buller","sequence":"additional","affiliation":[{"name":"Atlassian, Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-0262-0507","authenticated-orcid":false,"given":"Ryan","family":"Jiang","sequence":"additional","affiliation":[{"name":"Atlassian, Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6525-7778","authenticated-orcid":false,"given":"Matthew","family":"Watson","sequence":"additional","affiliation":[{"name":"Atlassian, Sydney, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-9960-1741","authenticated-orcid":false,"given":"Minwoo","family":"Jeong","sequence":"additional","affiliation":[{"name":"Atlassian, Seattle, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-1419-1788","authenticated-orcid":false,"given":"Kun","family":"Chen","sequence":"additional","affiliation":[{"name":"Atlassian, Seattle, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6993-1431","authenticated-orcid":false,"given":"Ming","family":"Wu","sequence":"additional","affiliation":[{"name":"Atlassian, Seattle, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,11]]},"reference":[{"key":"e_1_3_3_1_2_2","unstructured":"2025. Bitbucket | Git solution for teams using Jira. https:\/\/bitbucket.org\/product\/. Bitbucket Cloud is a Git-based code and CI\/CD tool optimized for teams using Jira.."},{"key":"e_1_3_3_1_3_2","unstructured":"2025. Collaboration software for software IT and business teams. https:\/\/www.atlassian.com\/. Atlassian\u2019s team collaboration software like Jira Confluence and Trello help teams organize discuss and complete shared work.."},{"key":"e_1_3_3_1_4_2","unstructured":"2025. Gerrit Code Review. https:\/\/www.gerritcodereview.com\/. Gerrit Code Review | Gerrit Code Review.."},{"key":"e_1_3_3_1_5_2","unstructured":"2025. GitHub \u00b7 Build and ship software on a single collaborative platform. https:\/\/github.com\/. Join the world\u2019s most widely adopted AI-powered developer platform where millions of developers businesses and the largest open source community build software that advances humanity.."},{"key":"e_1_3_3_1_6_2","unstructured":"2025. The most-comprehensive AI-powered DevSecOps platform. https:\/\/about.gitlab.com\/. From planning to production bring teams together in one application. Ship secure code more efficiently to deliver value faster.."},{"key":"e_1_3_3_1_7_2","unstructured":"Atlassian. 2025. Privacy Policy | Atlassian. https:\/\/www.atlassian.com\/legal\/privacy-policy#what-this-policy-covers. Atlassian\u2019s privacy policy covers the information collected from customers and explains choices surrounding how information is used.."},{"key":"e_1_3_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606617"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606642"},{"key":"e_1_3_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1109\/MSR.2015.21"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","unstructured":"David Byrne. 2022. A worked example of Braun and Clarke\u2019s approach to reflexive thematic analysis. Quality & Quantity 56 (2022) 1391\u20131412. 10.1007\/s11135-021-01182-y","DOI":"10.1007\/s11135-021-01182-y"},{"key":"e_1_3_3_1_12_2","unstructured":"Wachiraphan Charoenwet Kla Tantithamthavorn Patanamon Thongtanunam Hong\u00a0Yi Lin Minwoo Jeong and Ming Wu. 2026. AgenticSCR: An Autonomous Agentic Secure Code Review for Immature Vulnerabilities Detection. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2601.19138 (2026)."},{"key":"e_1_3_3_1_13_2","unstructured":"I Chern Steffi Chern Shiqi Chen Weizhe Yuan Kehua Feng Chunting Zhou Junxian He Graham Neubig Pengfei Liu et\u00a0al. 2023. FacTool: Factuality Detection in Generative AI\u2013A Tool Augmented Framework for Multi-Task and Multi-Domain Scenarios. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2307.13528 (2023)."},{"key":"e_1_3_3_1_14_2","first-page":"20","volume-title":"2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering Education and Training (ICSE-SEET)","author":"Chong Chun\u00a0Yong","year":"2021","unstructured":"Chun\u00a0Yong Chong, Patanamon Thongtanunam, and Chakkrit Tantithamthavorn. 2021. Assessing the students\u2019 understanding and their mistakes in code review checklists: an experience report of 1,791 code review checklist questions from 394 students. In 2021 IEEE\/ACM 43rd International Conference on Software Engineering: Software Engineering Education and Training (ICSE-SEET). IEEE, 20\u201329."},{"key":"e_1_3_3_1_15_2","unstructured":"Umut Cihan Vahid Haratian Arda \u0130\u00e7\u00f6z Mert\u00a0Kaan G\u00fcl \u00d6mercan Devran Emircan\u00a0Furkan Bayendur Baykal\u00a0Mehmet U\u00e7ar and Eray T\u00fcz\u00fcn. 2024. Automated Code Review In Practice. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2412.18531 (2024)."},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/3639477.3639746"},{"key":"e_1_3_3_1_17_2","doi-asserted-by":"publisher","DOI":"10.1145\/3524842.3528452"},{"key":"e_1_3_3_1_18_2","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549098"},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1109\/ASE63991.2025.00312"},{"key":"e_1_3_3_1_20_2","unstructured":"Jiawei Gu Xuhui Jiang Zhichao Shi Hexiang Tan Xuehao Zhai Chengjin Xu Wei Li Yinghan Shen Shengjie Ma Honghao Liu et\u00a0al. 2024. A survey on llm-as-a-judge. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.15594 (2024)."},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.1145\/2970276.2970306"},{"key":"e_1_3_3_1_22_2","unstructured":"Hyunsun Hong and Jongmoon Baik. 2025. Retrieval-Augmented Code Review Comment Generation. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2506.11591 (2025)."},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.1109\/SANER53432.2022.00121"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549119"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"crossref","unstructured":"Lei Huang Weijiang Yu Weitao Ma Weihong Zhong Zhangyin Feng Haotian Wang Qianglong Chen Weihua Peng Xiaocheng Feng Bing Qin et\u00a0al. 2025. A survey on hallucination in large language models: Principles taxonomy challenges and open questions. ACM Transactions on Information Systems 43 2 (2025) 1\u201355.","DOI":"10.1145\/3703155"},{"key":"e_1_3_3_1_26_2","doi-asserted-by":"crossref","unstructured":"Ziwei Ji Nayeon Lee Rita Frieske Tiezheng Yu Dan Su Yan Xu Etsuko Ishii Ye\u00a0Jin Bang Andrea Madotto and Pascale Fung. 2023. Survey of hallucination in natural language generation. ACM computing surveys 55 12 (2023) 1\u201338.","DOI":"10.1145\/3571730"},{"key":"e_1_3_3_1_27_2","unstructured":"Dawei Li Bohan Jiang Liangjie Huang Alimohammad Beigi Chengshuai Zhao Zhen Tan Amrita Bhattacharjee Yuxuan Jiang Canyu Chen Tianhao Wu et\u00a0al. 2024. From generation to judgment: Opportunities and challenges of llm-as-a-judge. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.16594 (2024)."},{"key":"e_1_3_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549081"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.1109\/MSR66628.2025.00043"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE59848.2023.00026"},{"key":"e_1_3_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE59848.2023.00026"},{"key":"e_1_3_3_1_32_2","doi-asserted-by":"crossref","unstructured":"Laura MacLeod Michaela Greiler Margaret-Anne Storey Christian Bird and Jacek Czerwonka. 2017. Code Reviewing in the Trenches: Challenges and Best Practices. IEEE Software (2017) 34\u201342.","DOI":"10.1109\/MS.2017.265100500"},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"crossref","unstructured":"Potsawee Manakul Adian Liusie and Mark\u00a0JF Gales. 2023. Selfcheckgpt: Zero-resource black-box hallucination detection for generative large language models. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2303.08896 (2023).","DOI":"10.18653\/v1\/2023.emnlp-main.557"},{"key":"e_1_3_3_1_34_2","unstructured":"Doriane Olewicki Leuson Da\u00a0Silva Suhaib Mujahid Arezou Amini Benjamin Mah Marco Castelluccio Sarra Habchi Foutse Khomh and Bram Adams. 2024. Impact of LLM-based Review Comment Generation in Practice: A Mixed Open-\/Closed-source User Study. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2411.07091 (2024)."},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1109\/IWESEP.2014.11"},{"key":"e_1_3_3_1_36_2","unstructured":"Chanathip Pornprasit. 2025. Context-Aware AI-Powered Code Review Assistant. Ph.\u00a0D. Dissertation. Monash University."},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"crossref","unstructured":"Chanathip Pornprasit and Chakkrit Tantithamthavorn. 2024. Fine-tuning and prompt engineering for large language models-based code review automation. Information and Software Technology 175 (2024) 107523.","DOI":"10.1016\/j.infsof.2024.107523"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1109\/MSR52588.2021.00049"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/1985793.1985867"},{"key":"e_1_3_3_1_40_2","doi-asserted-by":"publisher","unstructured":"Peter\u00a0C. Rigby and Margaret-Anne Storey. 2014. A decade of modern code review: A systematic literature review. Information and Software Technology 56 11 (2014) 1189\u20131203. 10.1016\/j.infsof.2014.06.004","DOI":"10.1016\/j.infsof.2014.06.004"},{"key":"e_1_3_3_1_41_2","unstructured":"Xinyu She Yue Liu Yanjie Zhao Yiling He Li Li Chakkrit Tantithamthavorn Zhan Qin and Haoyu Wang. 2023. Pitfalls in language models for code intelligence: A taxonomy and survey. ACM Transactions on Software Engineering and Methodology (2023)."},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP66354.2025.00036"},{"key":"e_1_3_3_1_43_2","doi-asserted-by":"publisher","DOI":"10.1145\/2593702.2593705"},{"key":"e_1_3_3_1_44_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510067"},{"key":"e_1_3_3_1_45_2","doi-asserted-by":"publisher","DOI":"10.1109\/SANER.2015.7081824"},{"key":"e_1_3_3_1_46_2","unstructured":"David Torgerson. 2024. iTWire - Data security the biggest concern as AI adoption grows in Australia. https:\/\/itwire.com\/business-it-news\/data\/data-security-the-biggest-concern-as-ai-adoption-grows-in-australia.html. Over 80% of Australian knowledge workers note data security is extremely or very important when using AI-powered visual collaboration tools.."},{"key":"e_1_3_3_1_47_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00021"},{"key":"e_1_3_3_1_48_2","unstructured":"Rosalia Tufano and Gabriele Bavota. 2025. Automating Code Review: A Systematic Literature Review. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2503.09510 (2025)."},{"key":"e_1_3_3_1_49_2","doi-asserted-by":"crossref","unstructured":"Rosalia Tufano Ozren Dabi\u0107 Antonio Mastropaolo Matteo Ciniselli and Gabriele Bavota. 2024. Code review automation: strengths and weaknesses of the state of the art. IEEE Transactions on Software Engineering 50 2 (2024) 338\u2013353.","DOI":"10.1109\/TSE.2023.3348172"},{"key":"e_1_3_3_1_50_2","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510621"},{"key":"e_1_3_3_1_51_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00027"},{"key":"e_1_3_3_1_52_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00027"},{"key":"e_1_3_3_1_53_2","doi-asserted-by":"publisher","DOI":"10.1145\/3664646.3665664"},{"key":"e_1_3_3_1_54_2","unstructured":"Benjamin Warner Antoine Chaffin Benjamin Clavi\u00e9 Orion Weller Oskar Hallstr\u00f6m Said Taghadouini Alexis Gallagher Raja Biswas Faisal Ladhak Tom Aarsen et\u00a0al. 2024. Smarter better faster longer: A modern bidirectional encoder for fast memory efficient and long context finetuning and inference. arXiv preprint arXiv:https:\/\/arXiv.org\/abs\/2412.13663 (2024)."},{"key":"e_1_3_3_1_55_2","unstructured":"Supatsara Wattanakriengkrai Patanamon Thongtanunam Chakkrit Tantithamthavorn Hideaki Hata and Kenichi Matsumoto. 2020. Predicting defective lines using a model-agnostic technique. IEEE Transactions on Software Engineering (2020)."},{"key":"e_1_3_3_1_56_2","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180233"},{"key":"e_1_3_3_1_57_2","first-page":"3938","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Xu X","year":"2020","unstructured":"X Xu, Quzhe Huang, Zheng Wang, Yansong Feng, and Dongyan Zhao. 2020. Towards context-aware code comment generation. In Findings of the Association for Computational Linguistics: EMNLP 2020. Association for Computational Linguistics, 3938\u20133947."},{"key":"e_1_3_3_1_58_2","doi-asserted-by":"crossref","unstructured":"Ziyao Zhang Chong Wang Yanlin Wang Ensheng Shi Yuchi Ma Wanjun Zhong Jiachi Chen Mingzhi Mao and Zibin Zheng. 2025. LLM hallucinations in practical code generation: Phenomena mechanism and mitigation. Proceedings of the ACM on Software Engineering 2 ISSTA (2025) 481\u2013503.","DOI":"10.1145\/3728894"},{"key":"e_1_3_3_1_59_2","doi-asserted-by":"crossref","unstructured":"Lianmin Zheng Wei-Lin Chiang Ying Sheng Siyuan Zhuang Zhanghao Wu Yonghao Zhuang Zi Lin Zhuohan Li Dacheng Li Eric Xing et\u00a0al. 2023. Judging llm-as-a-judge with mt-bench and chatbot arena. Advances in Neural Information Processing Systems 36 (2023) 46595\u201346623.","DOI":"10.52202\/075280-2020"}],"event":{"name":"ICSE-SEIP '26: 2026 IEEE\/ACM 48th International Conference on Software Engineering","location":"Rio de Janeiro Brazil","acronym":"ICSE-SEIP '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 48th International Conference on Software Engineering: Software Engineering in Practice"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3786583.3786851","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T09:14:09Z","timestamp":1783761249000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3786583.3786851"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,12]]},"references-count":58,"alternative-id":["10.1145\/3786583.3786851","10.1145\/3786583"],"URL":"https:\/\/doi.org\/10.1145\/3786583.3786851","relation":{},"subject":[],"published":{"date-parts":[[2026,4,12]]},"assertion":[{"value":"2026-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}