{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,28]],"date-time":"2026-01-28T20:05:54Z","timestamp":1769630754293,"version":"3.49.0"},"reference-count":81,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100009227","name":"Iowa State University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009227","id-type":"DOI","asserted-by":"publisher"}]},{"name":"U.S. Department of Commerce, National Institute of Standards","award":["70NANB25H125"],"award-info":[{"award-number":["70NANB25H125"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Robot."],"published-print":{"date-parts":[[2026]]},"DOI":"10.1109\/tro.2025.3645934","type":"journal-article","created":{"date-parts":[[2025,12,18]],"date-time":"2025-12-18T18:36:10Z","timestamp":1766082970000},"page":"561-578","source":"Crossref","is-referenced-by-count":0,"title":["Rethink Repeatable Measures of Robot Performance With Statistical Query"],"prefix":"10.1109","volume":"42","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7868-9747","authenticated-orcid":false,"given":"Bowen","family":"Weng","sequence":"first","affiliation":[{"name":"Department of Computer Science, Iowa State University, Ames, IA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9871-1243","authenticated-orcid":false,"given":"Linda","family":"Capito","sequence":"additional","affiliation":[{"name":"Transportation Research Center Inc., East Liberty, OH, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1326-5836","authenticated-orcid":false,"given":"Guillermo A.","family":"Castillo","sequence":"additional","affiliation":[{"name":"is an independent researcher, resides in Round Rock, Texas, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1958-1863","authenticated-orcid":false,"given":"Dylan","family":"Khor","sequence":"additional","affiliation":[{"name":"Department of Computer Science, Iowa State University, Ames, IA, USA"}]}],"member":"263","reference":[{"key":"ref1","volume-title":"The Kinematics of Manipulators Under Computer Control","author":"Pieper","year":"1969"},{"key":"ref2","first-page":"39","article-title":"Performance evaluation of manipulators from a kinematic viewpoint","volume":"459","author":"Roth","year":"1976","journal-title":"NBS Special Publ."},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.6028\/NBS.SP.459"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ROBOT.1986.1087508"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1017\/S0263574700015873"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/978-94-011-6768-0"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1002\/9780470172506.ch24"},{"key":"ref8","volume-title":"Manipulating Industrial RobotsPerformance Criteria and Related Test Methods","year":"1998"},{"key":"ref9","volume-title":"Industrial Robots and Robot SystemsPerformance Characteristics","year":"1992"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1163\/156855396X00192"},{"key":"ref11","volume-title":"Robots and Robotic DevicesPerformance Criteria and Related Test Methods for Service Robots","year":"2016"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-658-18107-9_2"},{"key":"ref13","article-title":"Euro NCAP AEB car-to-car test protocol","volume-title":"Eur. New Car Assessment Programme (Euro NCAP)","year":"2023"},{"key":"ref14","article-title":"Traffic jam assist test development considerations","volume-title":"U.S. Dept. Transp., Nat. Highway Traffic Saf. Admin.","author":"Rao","year":"2019"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.6028\/nist.ams.100-45r1"},{"key":"ref16","volume-title":"Accuracy (Trueness and Precision) of Measurement Methods and ResultsPart 1: General Principles and Definitions","year":"1994"},{"key":"ref17","article-title":"NIST engineering statistics handbook: Glossary of statistical terms","year":"2023"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-021-21007-8"},{"key":"ref19","article-title":"Naturalistic and adversarial driving environment","year":"2023"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA55743.2025.11128524"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1080\/02286203.2002.11442246"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/ROBOT.2010.5509444"},{"key":"ref23","first-page":"93","article-title":"Risk ellipsoids and granularity ration for industrial robot","volume":"2","author":"Breth","year":"2007","journal-title":"Int. J. Factory Autom. Robot. Soft Comput."},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1007\/BF00710855"},{"key":"ref25","volume-title":"Applied Statistics and Probability for Engineers","author":"Montgomery","year":"2017"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2016.2582208"},{"key":"ref27","volume-title":"Simulation Modeling and Analysis","author":"Law","year":"2014"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-05732-2"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2022.3190086"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.15607\/RSS.2024.XX.061"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1126\/scirobotics.aay6004"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-024-49194-0"},{"key":"ref33","article-title":"NHTSA\u2019s 2014 automatic emergency braking test track evaluations","author":"Forkenbrock","year":"2015","journal-title":"National Highway Traffic Safety Admin."},{"issue":"164","key":"ref34","first-page":"1","article-title":"Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program)","volume":"22","author":"Pineau","year":"2021","journal-title":"J. Mach. Learn. Res."},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/3519935.3519973"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.70002"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1126\/science.1250475"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1161\/CIRCRESAHA.114.303819"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.17226\/25303"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.7554\/eLife.04333"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1038\/s41583-020-0313-3"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1016\/j.anbehav.2008.12.022"},{"issue":"136","key":"ref43","first-page":"1","article-title":"Probabilistic search algorithms with unique answers and their cryptographic applications","volume":"18","author":"Gat","year":"2011","journal-title":"Electron. Colloq. Comput. Complexity"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-88946-2_2"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/FOCS46700.2020.00044"},{"key":"ref46","first-page":"30784","article-title":"List and certificate complexities in replicable learning","volume":"36","author":"Dixon","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref47","first-page":"78996","article-title":"Replicability in learning: Geometric partitions and KKM-sperner lemma","volume":"37","author":"Woude","year":"2024","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref48","article-title":"Replicable bandits","volume-title":"Proc. 11th Int. Conf. Learn. Representations","author":"Esfandiari","year":"2023"},{"key":"ref49","first-page":"74702","article-title":"Replicability in reinforcement learning","volume":"36","author":"Karbasi","year":"2023","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/FOCS57990.2023.00148"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/3618260.3649632"},{"key":"ref52","article-title":"Robotic calibration issues: Accuracy, repeatability and calibration","volume-title":"Proc. 8th Mediterranean Conf. Control Autom.","volume":"1719","author":"Conrad","year":"2000"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/ICRA.2019.8794390"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1109\/MRA.2015.2452073"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1007\/s43154-022-00094-5"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/MRA.2015.2460891"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.3389\/frobt.2023.1127972"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/MRA.2015.2448311"},{"key":"ref59","first-page":"110","article-title":"Challenges and opportunities for replication science in HRI: A case study in human-robot trust","volume-title":"Proc. ACM\/IEEE Int. Conf. Human-Robot Interaction","author":"Ullman","year":"2021"},{"key":"ref60","article-title":"Empirical bernstein bounds and sample-variance penalization","volume-title":"Proc. Annu. Conf. Comput. Learn. Theory","author":"Maurer","year":"2009"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1109\/TRO.2023.3267020"},{"key":"ref62","first-page":"785","article-title":"A general characterization of the statistical query complexity","volume-title":"Conf. Learn. theory","author":"Feldman","year":"2017"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1214\/17-AAP1326"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1080\/00949659208810398"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2017.2699226"},{"key":"ref66","article-title":"Gradient-based sampling: An adaptive importance sampling for least-squares","volume":"29","author":"Zhu","year":"2016","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.23919\/ACC50511.2021.9482763"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1198\/106186008X320456"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1002\/wics.56"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/TITS.2020.2972211"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-0865-5_26"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-12788-9_6"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/ISSRE5003.2020.00012"},{"key":"ref74","article-title":"Adastress: Adaptive stress testing and interpretable categorization for safety-critical systems","author":"Lee","year":"2019"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/IVS.2018.8500400"},{"key":"ref76","article-title":"Unitree RL gym","author":"Robotics","year":"2024"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1177\/0278364919894385"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1109\/IROS55552.2023.10342061"},{"key":"ref79","article-title":"Post-convergence sim-to-real policy transfer: A principled alternative to cherry-picking","author":"Khor","year":"2025"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1201\/9781482276596"},{"key":"ref81","volume-title":"Stochastic Approximation and Recursive Algorithms and Applications","author":"Kushner","year":"2003"}],"container-title":["IEEE Transactions on Robotics"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/8860\/11297026\/11304172-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/8860\/11297026\/11304172.pdf?arnumber=11304172","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,27]],"date-time":"2026-01-27T06:09:53Z","timestamp":1769494193000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/11304172\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"references-count":81,"URL":"https:\/\/doi.org\/10.1109\/tro.2025.3645934","relation":{},"ISSN":["1552-3098","1941-0468"],"issn-type":[{"value":"1552-3098","type":"print"},{"value":"1941-0468","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]}}}