{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,13]],"date-time":"2026-01-13T21:55:04Z","timestamp":1768341304606,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":29,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,7,9]],"date-time":"2022-07-09T00:00:00Z","timestamp":1657324800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"Dutch Research Council (NWO)"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,7,9]]},"DOI":"10.1145\/3520304.3533995","type":"proceedings-article","created":{"date-parts":[[2022,7,19]],"date-time":"2022-07-19T15:29:44Z","timestamp":1658244584000},"page":"2104-2112","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Heed the noise in performance evaluations in neural architecture search"],"prefix":"10.1145","author":[{"given":"Arkadiy","family":"Dushatskiy","sequence":"first","affiliation":[{"name":"Centrum Wiskunde & Informatica, Amsterdam, the Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Tanja","family":"Alderliesten","sequence":"additional","affiliation":[{"name":"Leiden University Medical Center, Leiden, the Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Peter A. N.","family":"Bosman","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,7,19]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Michela Antonelli Annika Reinke Spyridon Bakas Keyvan Farahani Bennett A Landman Geert Litjens Bjoern Menze Olaf Ronneberger Ronald M Summers Bram van Ginneken et al. 2021. The Medical Segmentation Decathlon. arXiv preprint arXiv:2106.05735 (2021)."},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of the International Conference on Machine Learning. PMLR, 115--123","author":"Bergstra James","year":"2013","unstructured":"James Bergstra, Daniel Yamins, and David Cox. 2013. Making a science of model search: Hyperparameter optimization in hundreds of dimensions for vision architectures. In Proceedings of the International Conference on Machine Learning. PMLR, 115--123."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMI.2018.2837502"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.195"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV48922.2021.00659"},{"key":"e_1_3_2_2_6_1","volume-title":"Bosman","author":"Ottelander Tom Den","year":"2021","unstructured":"Tom Den Ottelander, Arkadiy Dushatskiy, Marco Virgolin, and Peter A.N. Bosman. 2021. Local search is a remarkably strong baseline for neural architecture search. In International Conference on Evolutionary Multi-Criterion Optimization. Springer, 465--479."},{"key":"e_1_3_2_2_7_1","volume-title":"Nas-bench-201: Extending the scope of reproducible neural architecture search. arXiv preprint arXiv:2001.00326","author":"Dong Xuanyi","year":"2020","unstructured":"Xuanyi Dong and Yi Yang. 2020. Nas-bench-201: Extending the scope of reproducible neural architecture search. arXiv preprint arXiv:2001.00326 (2020)."},{"key":"e_1_3_2_2_8_1","volume-title":"Proceedings of the Genetic and Evolutionary Computation Conference. 583--591","author":"Dushatskiy Arkadiy","unstructured":"Arkadiy Dushatskiy, Tanja Alderliesten, and Peter A.N. Bosman. 2021. A novel surrogate-assisted evolutionary algorithm applied to partition-based ensemble learning. In Proceedings of the Genetic and Evolutionary Computation Conference. 583--591."},{"key":"e_1_3_2_2_9_1","volume-title":"Bosman","author":"Dushatskiy Arkadiy","year":"2021","unstructured":"Arkadiy Dushatskiy, Marco Virgolin, Anton Bouter, Dirk Thierens, and Peter A.N. Bosman. 2021. Parameterless Gene-pool Optimal Mixing Evolutionary Algorithms. arXiv e-prints (2021), arXiv-2109."},{"key":"e_1_3_2_2_10_1","volume-title":"GraphNAS: Graph neural architecture search with reinforcement learning. arXiv preprint arXiv:1904.09981","author":"Gao Yang","year":"2019","unstructured":"Yang Gao, Hong Yang, Peng Zhang, Chuan Zhou, and Yue Hu. 2019. GraphNAS: Graph neural architecture search with reinforcement learning. arXiv preprint arXiv:1904.09981 (2019)."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.90"},{"key":"e_1_3_2_2_12_1","volume-title":"Jens Petersen, and Klaus H Maier-Hein.","author":"Isensee Fabian","year":"2021","unstructured":"Fabian Isensee, Paul F Jaeger, Simon AA Kohl, Jens Petersen, and Klaus H Maier-Hein. 2021. nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods 18, 2 (2021), 203--211."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2019.00017"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.2196\/26151"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33014780"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24574-4_28"},{"key":"e_1_3_2_2_17_1","first-page":"1808","article-title":"Bridging the gap between sample-based and one-shot neural architecture search with BONAS","volume":"33","author":"Shi Han","year":"2020","unstructured":"Han Shi, Renjie Pi, Hang Xu, Zhenguo Li, James Kwok, and Tong Zhang. 2020. Bridging the gap between sample-based and one-shot neural architecture search with BONAS. Proceedings of the Advances in Neural Information Processing Systems 33 (2020), 1808--1819.","journal-title":"Proceedings of the Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_2_18_1","volume-title":"Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556","author":"Simonyan Karen","year":"2014","unstructured":"Karen Simonyan and Andrew Zisserman. 2014. Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)."},{"key":"e_1_3_2_2_19_1","volume-title":"Proceedings of the International Conference on Machine Learning. PMLR, 5877--5886","author":"So David","year":"2019","unstructured":"David So, Quoc Le, and Chen Liang. 2019. The evolved transformer. In Proceedings of the International Conference on Machine Learning. PMLR, 5877--5886."},{"key":"e_1_3_2_2_20_1","volume-title":"Proceedings of the International Conference on Machine Learning. PMLR, 6105--6114","author":"Tan Mingxing","year":"2019","unstructured":"Mingxing Tan and Quoc Le. 2019. Efficientnet: Rethinking model scaling for convolutional neural networks. In Proceedings of the International Conference on Machine Learning. PMLR, 6105--6114."},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2908991"},{"key":"e_1_3_2_2_22_1","volume-title":"Local search is state of the art for NAS benchmarks. arXiv preprint arXiv:2005.02960","author":"White Colin","year":"2020","unstructured":"Colin White, Sam Nolen, and Yash Savani. 2020. Local search is state of the art for NAS benchmarks. arXiv preprint arXiv:2005.02960 (2020)."},{"key":"e_1_3_2_2_23_1","volume-title":"How Powerful are Performance Predictors in Neural Architecture Search? arXiv preprint arXiv:2104.01177","author":"White Colin","year":"2021","unstructured":"Colin White, Arber Zela, Binxin Ru, Yang Liu, and Frank Hutter. 2021. How Powerful are Performance Predictors in Neural Architecture Search? arXiv preprint arXiv:2104.01177 (2021)."},{"key":"e_1_3_2_2_24_1","volume-title":"ResNet strikes back: An improved training procedure in timm. arXiv preprint arXiv:2110.00476","author":"Wightman Ross","year":"2021","unstructured":"Ross Wightman, Hugo Touvron, and Herv\u00e9 J\u00e9gou. 2021. ResNet strikes back: An improved training procedure in timm. arXiv preprint arXiv:2110.00476 (2021)."},{"key":"e_1_3_2_2_25_1","volume-title":"Segmentation Models Pytorch. GitHub repository https:\/\/github.com\/qubvel\/segmentation_models.pytorch","author":"Yakubovskiy Pavel","year":"2020","unstructured":"Pavel Yakubovskiy. 2020. Segmentation Models Pytorch. GitHub repository https:\/\/github.com\/qubvel\/segmentation_models.pytorch (2020)."},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-59710-8_38"},{"key":"e_1_3_2_2_27_1","volume-title":"NAS evaluation is frustratingly hard. arXiv preprint arXiv:1912.12522","author":"Yang Antoine","year":"2019","unstructured":"Antoine Yang, Pedro M Esperan\u00e7a, and Fabio M Carlucci. 2019. NAS evaluation is frustratingly hard. arXiv preprint arXiv:1912.12522 (2019)."},{"key":"e_1_3_2_2_28_1","volume-title":"International Conference on Machine Learning. PMLR, 7105--7114","author":"Ying Chris","year":"2019","unstructured":"Chris Ying, Aaron Klein, Eric Christiansen, Esteban Real, Kevin Murphy, and Frank Hutter. 2019. Nas-bench-101: Towards reproducible neural architecture search. In International Conference on Machine Learning. PMLR, 7105--7114."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR42600.2020.00418"}],"event":{"name":"GECCO '22: Genetic and Evolutionary Computation Conference","location":"Boston Massachusetts","acronym":"GECCO '22","sponsor":["SIGEVO ACM Special Interest Group on Genetic and Evolutionary Computation"]},"container-title":["Proceedings of the Genetic and Evolutionary Computation Conference Companion"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3520304.3533995","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3520304.3533995","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:02:27Z","timestamp":1750186947000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3520304.3533995"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,7,9]]},"references-count":29,"alternative-id":["10.1145\/3520304.3533995","10.1145\/3520304"],"URL":"https:\/\/doi.org\/10.1145\/3520304.3533995","relation":{},"subject":[],"published":{"date-parts":[[2022,7,9]]},"assertion":[{"value":"2022-07-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}