{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,18]],"date-time":"2026-03-18T13:40:28Z","timestamp":1773841228239,"version":"3.50.1"},"reference-count":87,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"4","license":[{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,4,1]],"date-time":"2025-04-01T00:00:00Z","timestamp":1743465600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IIEEE Trans. Software Eng."],"published-print":{"date-parts":[[2025,4]]},"DOI":"10.1109\/tse.2025.3533972","type":"journal-article","created":{"date-parts":[[2025,1,27]],"date-time":"2025-01-27T18:47:03Z","timestamp":1738003623000},"page":"929-946","source":"Crossref","is-referenced-by-count":3,"title":["On the Workflows and Smells of Leaderboard Operations (LBOps): An Exploratory Study of Foundation Model Leaderboards"],"prefix":"10.1109","volume":"51","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0337-808X","authenticated-orcid":false,"given":"Zhimin","family":"Zhao","sequence":"first","affiliation":[{"name":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen’s University, Kingston, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5311-6061","authenticated-orcid":false,"given":"Abdul Ali","family":"Bangash","sequence":"additional","affiliation":[{"name":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen’s University, Kingston, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5494-685X","authenticated-orcid":false,"given":"Filipe Roseiro","family":"C\u00f4go","sequence":"additional","affiliation":[{"name":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen’s University, Kingston, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7213-4006","authenticated-orcid":false,"given":"Bram","family":"Adams","sequence":"additional","affiliation":[{"name":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen’s University, Kingston, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7749-5513","authenticated-orcid":false,"given":"Ahmed E.","family":"Hassan","sequence":"additional","affiliation":[{"name":"Software Analysis and Intelligence Laboratory (SAIL), School of Computing, Queen’s University, Kingston, ON, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3613904.3642016"},{"key":"ref2","article-title":"MS MARCO: A human generated machine reading comprehension dataset","author":"Bajaj","year":"2016"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.21236\/ada307888"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3586030"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/1453101.1453146"},{"key":"ref6","article-title":"Executive order on the safe, secure, and trustworthy development and use of artificial intelligence","author":"Biden","year":"2023"},{"key":"ref7","article-title":"On the opportunities and risks of foundation models","author":"Bommasani","year":"2021"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1609\/aies.v7i1.31628"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1111\/nyas.15007"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2018.09.016"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.2307\/2334029"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1177\/0049124113500475"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"654","DOI":"10.1145\/3377811.3380330","article-title":"A tale from the trenches: Cognitive biases and software development","volume-title":"Proc. ACM\/IEEE 42nd Int. Conf. Softw. Eng.","author":"Chattopadhyay","year":"2020"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00051"},{"key":"ref15","article-title":"MLLM-as-a-judge: Assessing multimodal LLM-as-a-judge with vision-language benchmark","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","author":"Chen","year":"2024"},{"key":"ref16","article-title":"Evaluating large language models trained on code","author":"Chen","year":"2021"},{"key":"ref17","article-title":"Chatbot arena: An open platform for evaluating llms by human preference","volume-title":"Proc. 41st Int. Conf. Mach. Learn.","author":"Chiang","year":"2024"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1017\/S1351324921000322"},{"key":"ref19","article-title":"Github copilot AI pair programmer: Asset or liability?","volume":"203","author":"Dakhel","year":"2023","journal-title":"J. Syst. Softw."},{"key":"ref20","article-title":"Benchmark probing: Investigating data leakage in large language models","volume-title":"Proc. NeurIPS Workshop Backdoors Deep Learn.","author":"Deng","year":"2023"},{"key":"ref21","article-title":"Bringing the people back in: Contesting benchmark machine learning datasets","author":"Denton","year":"2020"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1175\/AIES-D-21-0002.1"},{"key":"ref23","article-title":"On the impossible safety of large AI models","author":"El-Mhamdi","year":"2022"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.113"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00128"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/3524613.3527814"},{"key":"ref27","first-page":"1","article-title":"Prompting is all you need: Automated android bug replay with large language models","volume-title":"Proc. 46th IEEE\/ACM Int. Conf. Softw. Eng.","author":"Feng","year":"2024"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CSMR.2009.59"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1201\/9781003162810-13"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-021-01453-z"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbusres.2018.10.046"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijforecast.2009.10.002"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3106247"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/2372251.2372257"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1249"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/SANER53432.2022.00048"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-91669-5_35"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1007\/s00799-023-00366-1"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.692"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1145\/3377812.3382176","article-title":"An empirical study on the evolution of test smell","volume-title":"Proc. ACM\/IEEE 42nd Int. Conf. Softw. Eng.: Companion Proc.","author":"Kim","year":"2020"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/access.2023.3262138"},{"key":"ref42","first-page":"9459","article-title":"Retrieval-augmented generation for knowledge-intensive NLP tasks","volume":"33","author":"Lewis","year":"2020","journal-title":"Adv. Neur. Inf. Process. Syst."},{"issue":"05470","key":"ref43","volume":"2310","author":"Li","year":"2023","journal-title":"Generative judge for evaluating alignment"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1370\/afm.1195"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3571884.3604316"},{"key":"ref46","article-title":"Visual instruction tuning","volume":"36","author":"Liu","year":"2024","journal-title":"Adv. Neur. Information Process. Syst."},{"key":"ref47","article-title":"Alignbench: Benchmarking chinese alignment of large language models","volume-title":"Proc. 62nd Annu. Meet. Assoc. Comput. Linguist.","author":"Liu","year":"2023"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2876857"},{"issue":"03715","key":"ref49","volume":"2310","author":"Maslej","year":"2023","journal-title":"Artificial Intelligence index report"},{"key":"ref50","volume":"2024","author":"Maslej","year":"2024","journal-title":"Artificial Intelligence index report"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1002\/inst.12278"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/s10742-022-00293-9"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1108\/INTR-12-2021-0897"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639187"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/52.991329"},{"key":"ref56","article-title":"Proving test set contamination in black box language models","volume-title":"Proc. 12th Int. Conf. Learn. Represent.","author":"Oren","year":"2023"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-022-34591-0"},{"key":"ref58","article-title":"AI2APPS: A visual ide for building llm-based AI agent applications","author":"Pang","year":"2024"},{"key":"ref59","article-title":"Tinybenchmarks: Evaluating LLMS with fewer examples","volume-title":"Proc. ICLR Workshop Math. Emp. Understanding Found. Models","author":"Polo","year":"2024"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2014.31"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.810"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00033"},{"key":"ref63","article-title":"Model evaluation, model selection, and algorithm selection in machine learning","author":"Raschka","year":"2018"},{"key":"ref64","first-page":"5389","article-title":"Do imagenet classifiers generalize to imagenet?","volume-title":"Proc. Int. Conf. Mach. Learn.","author":"Recht","year":"2019"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.14778\/3685800.3685835"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2017.12.034"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2020.2988241"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1109\/AIIoT61789.2024.10578990"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-emnlp.855"},{"key":"ref70","volume-title":"Refactoring for Software Design Smells: Managing Technical Debt","author":"Suryanarayana","year":"2014"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1038\/s42254-022-00441-7"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187902"},{"key":"ref73","article-title":"HelpSteer2-preference: Complementing ratings with preferences","author":"Wang","year":"2024"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1109\/tse.2022.3194188"},{"issue":"1","key":"ref75","first-page":"1","article-title":"Card sorting: Current practices and beyond","volume":"4","author":"Wood","year":"2008","journal-title":"J. Usability Stud."},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111677"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1145\/3643662.3643957"},{"key":"ref78","article-title":"Llava-critic: Learning to evaluate multimodal models","author":"Xiong","year":"2024"},{"key":"ref79","article-title":"Benchmarking benchmark leakage in large language models","author":"Xu","year":"2024"},{"key":"ref80","article-title":"Rethinking benchmark and contamination for language models with rephrased samples","author":"Yang","year":"2023"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2022.wiesp-1.3"},{"key":"ref82","article-title":"Foundation model leaderboard survey","author":"Zhao","year":"2024"},{"key":"ref83","article-title":"Judging LLM-as-a-judge with MT-bench and chatbot arena","volume":"36","author":"Zheng","year":"2024","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-15-1967-3_2"},{"key":"ref85","article-title":"Agent-as-a-judge: Evaluate agents with agents","author":"Zhuge","year":"2024"},{"key":"ref86","article-title":"A developer\u2019s guide to prompt engineering and LLMS","author":"Ziegler","year":"2023"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.63"}],"container-title":["IEEE Transactions on Software Engineering"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/32\/10969500\/10855627.pdf?arnumber=10855627","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,18]],"date-time":"2025-04-18T17:37:49Z","timestamp":1744997869000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10855627\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4]]},"references-count":87,"journal-issue":{"issue":"4"},"URL":"https:\/\/doi.org\/10.1109\/tse.2025.3533972","relation":{},"ISSN":["0098-5589","1939-3520","2326-3881"],"issn-type":[{"value":"0098-5589","type":"print"},{"value":"1939-3520","type":"electronic"},{"value":"2326-3881","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,4]]}}}