{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T13:35:27Z","timestamp":1768570527620,"version":"3.49.0"},"reference-count":20,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2015,9,1]],"date-time":"2015-09-01T00:00:00Z","timestamp":1441065600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Robot. Automat. Mag."],"published-print":{"date-parts":[[2015,9]]},"DOI":"10.1109\/mra.2015.2448871","type":"journal-article","created":{"date-parts":[[2015,9,11]],"date-time":"2015-09-11T03:19:29Z","timestamp":1441941569000},"page":"53-61","source":"Crossref","is-referenced-by-count":56,"title":["Competitions for Benchmarking: Task and Functionality Scoring Complete Performance Assessment"],"prefix":"10.1109","volume":"22","author":[{"given":"Francesco","family":"Amigoni","sequence":"first","affiliation":[]},{"given":"Emanuele","family":"Bastianelli","sequence":"additional","affiliation":[]},{"given":"Jakob","family":"Berghofer","sequence":"additional","affiliation":[]},{"given":"Andrea","family":"Bonarini","sequence":"additional","affiliation":[]},{"given":"Giulio","family":"Fontana","sequence":"additional","affiliation":[]},{"given":"Nico","family":"Hochgeschwender","sequence":"additional","affiliation":[]},{"given":"Luca","family":"Iocchi","sequence":"additional","affiliation":[]},{"given":"Gerhard","family":"Kraetzschmar","sequence":"additional","affiliation":[]},{"given":"Pedro","family":"Lima","sequence":"additional","affiliation":[]},{"given":"Matteo","family":"Matteucci","sequence":"additional","affiliation":[]},{"given":"Pedro","family":"Miraldo","sequence":"additional","affiliation":[]},{"given":"Daniele","family":"Nardi","sequence":"additional","affiliation":[]},{"given":"Viola","family":"Schiaffonati","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref10","first-page":"328","article-title":"The RoCKIn@Work challenge","author":"dwiputra","year":"2014","journal-title":"Proc ISR\/ROBOTIK"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511814563"},{"key":"ref12","article-title":"AI evaluation: Past, present and future","author":"hern\u00e1ndez-orallo","year":"2014"},{"key":"ref13","first-page":"27","article-title":"Benchmarking intelligent service robots through scientific competitions: The RoboCup@Home approach","author":"holz","year":"2013","journal-title":"Proc AAAI Spring Symposium on Designing Intelligent Robots Reintegrating AI"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1002\/rob.21438"},{"key":"ref15","article-title":"RoboCup@Work: Competing for the factory of the future","author":"kraetzchmar","year":"2014","journal-title":"Proc RoboCup Symp"},{"key":"ref16","article-title":"Issues with methods for scoring competitors in RoboCup rescue","author":"parker","year":"2014","journal-title":"Proc AAMAS Workshop Autonomous Robots Multirobot Systems"},{"key":"ref17","article-title":"RoboCup rescue robot league","author":"pellenz","year":"2014","journal-title":"Proc RoboCup Symp"},{"key":"ref18","first-page":"321","article-title":"The RoCKIn@Home challenge","author":"schneider","year":"2014","journal-title":"Proc ISR\/ROBOTIK"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/MRA.2012.2186709"},{"key":"ref4","article-title":"Robot competitions—Ideal benchmarks for robotics research","author":"behnke","year":"2006","journal-title":"Proc IROS'06 Workshop on Benchmarks in Robotics Research"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/MRA.2011.941627"},{"key":"ref6","article-title":"Special interest group on good experimental methodology—GEM guidelines","author":"bonsignorio","year":"2007"},{"key":"ref5","first-page":"11","article-title":"A retrospective of the AAAI robot competitions","volume":"18","author":"bonasso","year":"1997","journal-title":"AI Mag"},{"key":"ref8","author":"collberg","year":"2014","journal-title":"Measuring Reproducibility in Computer Systems Research"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/100.813825"},{"key":"ref2","article-title":"RoCKIn deliverable D1.2 General evaluation criteria, modules and metrics for benchmarking through competitions","author":"ahmad","year":"2014"},{"key":"ref1","year":"2015","journal-title":"Darpa Robotics Challenge"},{"key":"ref9","article-title":"Replicability is not reproducibility: Nor is it good science","author":"drummond","year":"2009","journal-title":"Proceedings of the Evaluation Methods for Machine Learning Workshop at the 26th ICML"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1075\/is.10.3.06wis"}],"container-title":["IEEE Robotics & Automation Magazine"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/100\/7254280\/07254311.pdf?arnumber=7254311","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:46:38Z","timestamp":1642005998000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/7254311\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,9]]},"references-count":20,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/mra.2015.2448871","relation":{},"ISSN":["1070-9932"],"issn-type":[{"value":"1070-9932","type":"print"}],"subject":[],"published":{"date-parts":[[2015,9]]}}}