{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T16:03:57Z","timestamp":1787069037717,"version":"build-2736575974"},"reference-count":71,"publisher":"Informa UK Limited","issue":"1","content-domain":{"domain":["www.tandfonline.com"],"crossmark-restriction":true},"short-container-title":["Journal of Computational and Graphical Statistics"],"published-print":{"date-parts":[[2022,1,2]]},"DOI":"10.1080\/10618600.2021.1950004","type":"journal-article","created":{"date-parts":[[2021,7,6]],"date-time":"2021-07-06T12:36:53Z","timestamp":1625575013000},"page":"84-99","update-policy":"https:\/\/doi.org\/10.1080\/tandf_crossmark_01","source":"Crossref","is-referenced-by-count":14,"title":["Reproducible Hyperparameter Optimization"],"prefix":"10.1080","volume":"31","author":[{"given":"Lars","family":"Hertel","sequence":"first","affiliation":[{"name":"Department of Statistics, University of California, Irvine, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pierre","family":"Baldi","sequence":"additional","affiliation":[{"name":"Department of Computer Science, University of California, Irvine, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniel L.","family":"Gillen","sequence":"additional","affiliation":[{"name":"Department of Statistics, University of California, Irvine, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"301","published-online":{"date-parts":[[2021,8,20]]},"reference":[{"key":"CIT0001","doi-asserted-by":"publisher","DOI":"10.2307\/2343787"},{"key":"CIT0002","unstructured":"Balandat, M., Karrer, B., Jiang, D. R., Daulton, S., Letham, B., Wilson, A. G., and Bakshy, E. (2019), \u201cBotorch: Programmable Bayesian Optimization in Pytorch.\u201d arXiv:1910.06403."},{"key":"CIT0003","first-page":"2814","volume-title":"Advances in Neural Information Processing Systems","author":"Baldi P.","year":"2013"},{"key":"CIT0004","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.1983.6313077"},{"key":"CIT0005","first-page":"281","volume":"13","author":"Bergstra J.","year":"2012","journal-title":"Journal of Machine Learning Research"},{"key":"CIT0006","first-page":"2546","volume-title":"Advances in Neural Information Processing Systems","author":"Bergstra J. S.","year":"2011"},{"key":"CIT0007","unstructured":"Brockman, G., Cheung, V., Pettersson, L., Schneider, J., Schulman, J., Tang, J., and Zaremba, W. (2016), \u201cOpenai Gym.\u201d arXiv: 1606.01540."},{"key":"CIT0008","unstructured":"Chollet, F. et\u00a0al. (2018),\n                      Keras: The Python Deep Learning Library\n                      ,\n                      Astrophysics Source Code Library\n                      ."},{"key":"CIT0009","first-page":"25","volume-title":"Twenty-Fourth International Joint Conference on Artificial Intelligence","author":"Domhan T.","year":"2015"},{"key":"CIT0010","first-page":"1329","volume-title":"International Conference on Machine Learning","author":"Duan Y.","year":"2016"},{"key":"CIT0011","doi-asserted-by":"publisher","DOI":"10.1002\/sim.2901"},{"key":"CIT0012","unstructured":"Falkner, S., Klein, A., and Hutter, F. (2018), \u201cBohb: Robust and Efficient Hyperparameter Optimization at Scale. arXiv:1807.01774."},{"key":"CIT0013","first-page":"3","volume-title":"AutoML: Methods, Sytems, Challenges","author":"Feurer M.","year":"2018"},{"key":"CIT0014","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098043"},{"key":"CIT0015","first-page":"790","volume-title":"Volume 51 of Proceedings of Machine Learning Research","author":"Gonzalez J.","year":"2016"},{"key":"CIT0016","unstructured":"Goodfellow, I., Bengio, Y., and Courville, A. (2016),\n                      Deep Learning\n                      . MIT Press. http:\/\/www.deeplearningbook.org."},{"key":"CIT0017","doi-asserted-by":"publisher","DOI":"10.1145\/332148.332154"},{"key":"CIT0018","doi-asserted-by":"publisher","DOI":"10.1109\/TNNLS.2016.2582924"},{"key":"CIT0019","doi-asserted-by":"publisher","DOI":"10.1162\/106365603321828970"},{"key":"CIT0020","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11694"},{"key":"CIT0021","first-page":"1809","volume":"13","author":"Hennig P.","year":"2012","journal-title":"Journal of Machine Learning Research"},{"key":"CIT0022","volume-title":"The Art of Multiprocessor Programming","author":"Herlihy M.","year":"2012"},{"key":"CIT0023","first-page":"918","volume-title":"Advances in Neural Information Processing Systems","author":"Hern\u00e1ndez-Lobato J. M.","year":"2014"},{"key":"CIT0024","doi-asserted-by":"publisher","DOI":"10.1016\/j.softx.2020.100591"},{"key":"CIT0025","unstructured":"Hill, A., Raffin, A., Ernestus, M., Gleave, A., Kanervisto, A., Traore, R., Dhariwal, P., Hesse, C., Klimov, O., Nichol, A., Plappert, M., Radford, A., Schulman, J., Sidor, S., and Wu, Y. (2018), \u201cStable Baselines.\u201d Available at: https:\/\/github.com\/hill-a\/stable-baselines."},{"key":"CIT0026","doi-asserted-by":"publisher","DOI":"10.1126\/science.359.6377.725"},{"key":"CIT0027","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-25566-3_40"},{"key":"CIT0028","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-05318-5"},{"key":"CIT0029","unstructured":"Islam, R., Henderson, P., Gomrokchi, M., and Precup, D. (2017), \u201cReproducibility of Benchmarked Deep Reinforcement Learning Tasks for Continuous Control,\u201d arXiv: 1708.04133."},{"key":"CIT0030","first-page":"240","volume-title":"Proceedings of the 19th International Conference on Artificial Intelligence and Statistics, Proceedings of Machine Learning Research","volume":"51","author":"Jamieson K.","year":"2016"},{"key":"CIT0031","doi-asserted-by":"publisher","DOI":"10.1126\/science.aaf7894"},{"key":"CIT0032","doi-asserted-by":"publisher","DOI":"10.1201\/9780367805326"},{"key":"CIT0033","unstructured":"Jiang, S., Chai, H., Gonzalez, J., and Garnett, R. (2019), \u201cBinoculars for Efficient, Nonmyopic Sequential Experimental Design,\u201d arXiv-1909."},{"key":"CIT0034","unstructured":"Jiang, S., Jiang, D. R., Balandat, M., Karrer, B., Gardner, J. R., and Garnett, R. (2020), \u201cEfficient Nonmyopic Bayesian Optimization Via One-Shot Multi-Step Trees,\u201d arXiv: 2006.15779."},{"key":"CIT0035","doi-asserted-by":"publisher","DOI":"10.1023\/A:1008306431147"},{"key":"CIT0036","first-page":"133","volume-title":"International Conference on Artificial Intelligence and Statistics","author":"Kandasamy K.","year":"2018"},{"key":"CIT0037","unstructured":"Karpathy, A. (2017), \u201cCs231n Convolutional Neural Networks for Visual Recognition. 2016,\u201d Available at: http:\/\/cs231n.github.io."},{"key":"CIT0038","unstructured":"Keras (2019), \u201cKeras Optimizers,\u201d Available at: https:\/\/keras.io\/optimizers\/."},{"key":"CIT0039","doi-asserted-by":"publisher","DOI":"10.1111\/j.0006-341x.1999.00874.x"},{"key":"CIT0040","unstructured":"Klein, A., Falkner, S., Bartels, S., Hennig, P., and Hutter, F. (2016a), \u201cFast Bayesian Optimization of Machine Learning Hyperparameters on Large Datasets. arXiv:1605.07079."},{"key":"CIT0041","unstructured":"Klein, A., Falkner, S., Springenberg, J. T., and Hutter, F. (2016b), \u201cLearning Curve Prediction With Bayesian Neural Networks.\u201d"},{"key":"CIT0042","first-page":"87","volume-title":"ELPUB","author":"Kluyver T.","year":"2016"},{"key":"CIT0043","first-page":"1103","volume":"16","author":"Krueger T.","year":"2015","journal-title":"Journal of Machine Learning Research"},{"key":"CIT0044","doi-asserted-by":"publisher","DOI":"10.1007\/s10278-018-0079-6"},{"key":"CIT0045","first-page":"6765","volume":"18","author":"Li L.","year":"2016","journal-title":"\u201cHyperband: A Novel Bandit-Based Approach to Hyperparameter Optimization,\u201d"},{"key":"CIT0046","unstructured":"Li, L., Jamieson, K., Rostamizadeh, A., Gonina, E., Hardt, M., Recht, B., and Talwalkar, A. (2018), \u201cMassively Parallel Hyperparameter Tuning,\u201d arXiv: 1810.05934."},{"key":"CIT0047","unstructured":"Lipton, Z. C., and Steinhardt, J. (2018), \u201cTroubling Trends in Machine Learning Scholarship,\u201d arXiv: 1807.03341."},{"key":"CIT0048","unstructured":"Loshchilov, I., and Hutter, F. (2016), \u201cCma-es for Hyperparameter Optimization of Deep Neural Networks,\u201d arXiv:1604.07269."},{"key":"CIT0049","first-page":"698","volume-title":"Advances in Neural Information Processing Systems","volume":"31","author":"Lucic M.","year":"2018"},{"key":"CIT0050","unstructured":"Melis, G., Dyer, C., and Blunsom, P. (2017), \u201cOn the State of the Art of Evaluation in Neural Language Models,\u201d arXiv:1707.05589."},{"key":"CIT0051","unstructured":"Nagarajan, P., Warnell, G., and Stone, P. (2018), \u201cThe Impact of Nondeterminism on Reproducibility in Deep Reinforcement Learning.\u201d"},{"key":"CIT0052","doi-asserted-by":"publisher","DOI":"10.2307\/2530245"},{"key":"CIT0053","first-page":"2825","volume":"12","author":"Pedregosa F.","year":"2011","journal-title":"Journal of Machine Learning Research"},{"key":"CIT0054","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/64.2.191"},{"key":"CIT0055","volume-title":"R: A Language and Environment for Statistical Computing","author":"Core Team R","year":"2020"},{"key":"CIT0056","first-page":"2902","volume":"70","author":"Real E.","year":"2017","journal-title":"Proceedings of the 34th International Conference on Machine Learning-Volume"},{"key":"CIT0057","unstructured":"Schulman, J., Wolski, F., Dhariwal, P., Radford, A., and Klimov, O. (2017), \u201cProximal Policy Optimization Algorithms,\u201d arXiv: 1707.06347."},{"key":"CIT0058","doi-asserted-by":"publisher","DOI":"10.1137\/100801275"},{"key":"CIT0059","unstructured":"Sculley, D., Snoek, J., Wiltschko, A., and Rahimi, A. (2018), \u201cWinner\u2019s Curse? On Pace, Progress, and Empirical Rigor.\u201d https:\/\/openreview.net\/forum?id=rJWF0Fywf"},{"key":"CIT0060","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2015.2494218"},{"key":"CIT0061","first-page":"2951","volume-title":"Advances in Neural Information Processing Systems","volume":"25","author":"Snoek J.","year":"2012"},{"key":"CIT0062","first-page":"2171","volume-title":"in International Conference on Machine Learning","author":"Snoek J.","year":"2015"},{"key":"CIT0063","first-page":"1929","volume":"15","author":"Srivastava N.","year":"2014","journal-title":"The Journal of Machine Learning Research"},{"key":"CIT0064","unstructured":"Swersky, K., Snoek, J., and Adams, R. P. (2014), \u201cFreeze-Thaw Bayesian Optimization.\u201d arXiv: 1406.3896."},{"key":"CIT0065","first-page":"3627","volume-title":"Proceedings of the 34th International Conference on Machine Learning\u2013Volume","volume":"70","author":"Wang Z.","year":"2017"},{"key":"CIT0066","first-page":"3126","volume-title":"Advances in Neural Information Processing Systems","volume":"29","author":"Wu J.","year":"2016"},{"key":"CIT0067","first-page":"9813","volume-title":"Advances in Neural Information Processing Systems","volume":"32","author":"Wu J.","year":"2019"},{"key":"CIT0068","first-page":"5267","volume-title":"Advances in Neural Information Processing Systems","volume":"30","author":"Wu J.","year":"2017"},{"key":"CIT0069","first-page":"788","volume-title":"Volume 115 of Proceedings of Machine Learning Research","author":"Wu J.","year":"2020"},{"key":"CIT0070","first-page":"2808","volume":"108","author":"Yue X.","year":"2020","journal-title":"Proceedings of Machine Learning Research"},{"key":"CIT0071","unstructured":"Zoph, B., and Le, Q. V. (2016), \u201cNeural Architecture Search With Reinforcement Learning,\u201d arXiv: 1611.01578."}],"container-title":["Journal of Computational and Graphical Statistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.tandfonline.com\/doi\/pdf\/10.1080\/10618600.2021.1950004","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,12]],"date-time":"2022-10-12T13:31:49Z","timestamp":1665581509000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.tandfonline.com\/doi\/full\/10.1080\/10618600.2021.1950004"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8,20]]},"references-count":71,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2022,1,2]]}},"alternative-id":["10.1080\/10618600.2021.1950004"],"URL":"https:\/\/doi.org\/10.1080\/10618600.2021.1950004","relation":{},"ISSN":["1061-8600","1537-2715"],"issn-type":[{"value":"1061-8600","type":"print"},{"value":"1537-2715","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,8,20]]},"assertion":[{"value":"The publishing and review policy for this title is described in its Aims & Scope.","order":1,"name":"peerreview_statement","label":"Peer Review Statement"},{"value":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=ucgs20","URL":"http:\/\/www.tandfonline.com\/action\/journalInformation?show=aimsScope&journalCode=ucgs20","order":2,"name":"aims_and_scope_url","label":"Aim & Scope"},{"value":"2019-07-07","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2021-06-06","order":1,"name":"revised","label":"Revised","group":{"name":"publication_history","label":"Publication History"}},{"value":"2021-06-18","order":2,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2021-08-20","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}