{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T13:40:56Z","timestamp":1730209256690,"version":"3.28.0"},"reference-count":55,"publisher":"IEEE","license":[{"start":{"date-parts":[[2022,8,21]],"date-time":"2022-08-21T00:00:00Z","timestamp":1661040000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2022,8,21]],"date-time":"2022-08-21T00:00:00Z","timestamp":1661040000000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,8,21]]},"DOI":"10.1109\/cog51982.2022.9893715","type":"proceedings-article","created":{"date-parts":[[2022,9,20]],"date-time":"2022-09-20T19:33:31Z","timestamp":1663702411000},"page":"159-166","source":"Crossref","is-referenced-by-count":1,"title":["Towards verifiable Benchmarks for Reinforcement Learning"],"prefix":"10.1109","author":[{"given":"Matthias","family":"Muller-Brockhausen","sequence":"first","affiliation":[{"name":"Leiden Institute of Advanced Computer Science (LIACS), Leiden University,The Netherlands"}]},{"given":"Aske","family":"Plaat","sequence":"additional","affiliation":[{"name":"Leiden Institute of Advanced Computer Science (LIACS), Leiden University,The Netherlands"}]},{"given":"Mike","family":"Preuss","sequence":"additional","affiliation":[{"name":"Leiden Institute of Advanced Computer Science (LIACS), Leiden University,The Netherlands"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v37i2.2657"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TG.2018.2834566"},{"journal-title":"Extracting inputs from replays","year":"2021","author":"donadigo","key":"ref33"},{"journal-title":"Reproducibility - pytorch documentation","year":"2021","key":"ref32"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1016\/j.simpa.2021.100061"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/SII46433.2020.9025951"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1109\/BigData50022.2020.9378154"},{"key":"ref36","first-page":"1","article-title":"Using an artificial neural network to detect aim assistance in counter-strike: Global offensive","volume":"10","author":"maberry","year":"0","journal-title":"DOI"},{"journal-title":"Replay system - unreal engine documentation","year":"2021","author":"games","key":"ref35"},{"journal-title":"Tmx replay investigation","year":"2021","author":"donadigo","key":"ref34"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2018.8490422"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2017.2771241"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/TCIAIG.2012.2186810"},{"key":"ref2","article-title":"The minerl competition on sample efficient reinforcement learning using human priors","author":"guss","year":"2019","journal-title":"Thirty-third Conference on Neural Information Processing Systems (NeurIPS) Competition track"},{"journal-title":"Reproducibility and Replicability in Science","article-title":"National Academies of Sciences, Engineering, and Medicine and others","year":"2019","key":"ref1"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1007\/s13164-020-00471-z"},{"journal-title":"Garage A toolkit for reproducible reinforcement learning research","year":"2019","author":"garage contributors","key":"ref22"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1287\/moor.2014.0699"},{"journal-title":"Reevaluate Reproducibility in evaluating reinforcement learning algorithms","year":"2018","author":"khetarpal","key":"ref24"},{"journal-title":"Reproducibility of benchmarked deep reinforcement learning tasks for continuous control","year":"2017","author":"islam","key":"ref23"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2016.7860430"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/j.yjmcc.2017.10.009"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2016.2599030"},{"journal-title":"Zenodo","year":"2013","key":"ref51"},{"journal-title":"Chia-Network\/Chia-Blockchain","year":"2019","author":"cohen","key":"ref55"},{"journal-title":"Carbon emissions and large neural network training","year":"2021","author":"patterson","key":"ref54"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.17487\/rfc1950"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.17487\/rfc7049"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1355"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/3381831"},{"journal-title":"Ai space invaders 2021 competition","year":"2021","author":"brown","key":"ref40"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"ref13","article-title":"Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program)","volume":"22","author":"pineau","year":"2021","journal-title":"Journal of Machine Learning Research"},{"journal-title":"Code and trace repository","year":"2022","author":"paper authors","key":"ref14"},{"article-title":"Openai gym","year":"2016","author":"brockman","key":"ref15"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1512\/iumj.1957.6.56038"},{"journal-title":"Deep reinforcement learning","year":"2021","author":"plaat","key":"ref17"},{"key":"ref18","article-title":"An optimistic perspective on offline reinforcement learning","author":"agarwal","year":"2020","journal-title":"International Conference on Machine Learning"},{"journal-title":"Reinforcement learning An introduction Second edition","year":"2018","author":"sutton","key":"ref19"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723882"},{"journal-title":"The minerl 2020 competition on sample efficient reinforcement learning using human priors","year":"2021","author":"guss","key":"ref3"},{"key":"ref6","article-title":"On the replicability and reproducibility of deep learning in software engineering","author":"liu","year":"2020","journal-title":"arXiv preprint arXiv 2006 14410"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3466624"},{"key":"ref8","doi-asserted-by":"crossref","first-page":"350","DOI":"10.1038\/s41586-019-1724-z","article-title":"Grandmaster level in starcraft II using multi-agent reinforcement learning","volume":"575","author":"vinyals","year":"2019","journal-title":"Nat"},{"key":"ref7","first-page":"1373","article-title":"Revisiting rainbow: Promoting more insightful and inclusive deep reinforcement learning research","author":"ceron","year":"2021","journal-title":"International Conference on Machine Learning"},{"key":"ref49","first-page":"1","article-title":"Stable-baselines3: Reliable reinforcement learning implementations","volume":"22","author":"raffin","year":"2021","journal-title":"Journal of Machine Learning Research"},{"key":"ref9","article-title":"Dota 2 with large scale deep reinforcement learning","volume":"abs 1912 6680","author":"berner","year":"2019","journal-title":"CoRR"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/CoG52621.2021.9619000"},{"article-title":"Evolutionarily-curated curriculum learning for deep reinforcement learning agents","year":"2019","author":"green","key":"ref45"},{"key":"ref48","article-title":"Proximal policy optimization algorithms","author":"schulman","year":"2017","journal-title":"arXiv preprint arXiv 1707 06347"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/ICCE.2019.8662002"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/CIG.2019.8848063"},{"journal-title":"Snakes ai competition 2020 and 2021 report","year":"2021","author":"brown","key":"ref41"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-020-0208-z"},{"key":"ref43","article-title":"Leveraging procedural generation to benchmark reinforcement learning","author":"cobbe","year":"2019","journal-title":"arXiv preprint arXiv 1912 01588"}],"event":{"name":"2022 IEEE Conference on Games (CoG)","start":{"date-parts":[[2022,8,21]]},"location":"Beijing, China","end":{"date-parts":[[2022,8,24]]}},"container-title":["2022 IEEE Conference on Games (CoG)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/9893561\/9893544\/09893715.pdf?arnumber=9893715","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,10]],"date-time":"2022-10-10T20:25:38Z","timestamp":1665433538000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9893715\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,21]]},"references-count":55,"URL":"https:\/\/doi.org\/10.1109\/cog51982.2022.9893715","relation":{},"subject":[],"published":{"date-parts":[[2022,8,21]]}}}