{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,12]],"date-time":"2026-01-12T12:01:08Z","timestamp":1768219268805,"version":"3.49.0"},"publisher-location":"Cham","reference-count":54,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030328122","type":"print"},{"value":"9783030328139","type":"electronic"}],"license":[{"start":{"date-parts":[[2019,1,1]],"date-time":"2019-01-01T00:00:00Z","timestamp":1546300800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019]]},"DOI":"10.1007\/978-3-030-32813-9_5","type":"book-chapter","created":{"date-parts":[[2019,10,15]],"date-time":"2019-10-15T15:01:33Z","timestamp":1571151693000},"page":"36-49","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["A Survey on Deep Learning Benchmarks: Do We Still Need New Ones?"],"prefix":"10.1007","author":[{"given":"Qin","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Li","family":"Zha","sequence":"additional","affiliation":[]},{"given":"Jian","family":"Lin","sequence":"additional","affiliation":[]},{"given":"Dandan","family":"Tu","sequence":"additional","affiliation":[]},{"given":"Mingzhe","family":"Li","sequence":"additional","affiliation":[]},{"given":"Fan","family":"Liang","sequence":"additional","affiliation":[]},{"given":"Ren","family":"Wu","sequence":"additional","affiliation":[]},{"given":"Xiaoyi","family":"Lu","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2019,10,8]]},"reference":[{"key":"5_CR1","unstructured":"Abadi, M., et al.: TensorFlow: large-scale machine learning on heterogeneous distributed systems. arXiv preprint arXiv:1603.04467 (2016)"},{"key":"5_CR2","unstructured":"Abadi, M., et al.: TensorFlow: a system for large-scale machine learning. In: 12th $$\\{$$USENIX$$\\}$$ Symposium on Operating Systems Design and Implementation ($$\\{$$OSDI$$\\}$$ 2016), pp. 265\u2013283 (2016)"},{"key":"5_CR3","doi-asserted-by":"crossref","unstructured":"Adolf, R., Rama, S., Reagen, B., Wei, G.-Y., Brooks, D.M.: Fathom: reference workloads for modern deep learning methods. CoRR, abs\/1608.06581 (2016)","DOI":"10.1109\/IISWC.2016.7581275"},{"key":"5_CR4","doi-asserted-by":"crossref","unstructured":"Akioka, S., Muraoka, Y.: HPC benchmarks on Amazon EC2. In: 2010 IEEE 24th International Conference on Advanced Information Networking and Applications Workshops, pp. 1029\u20131034. IEEE (2010)","DOI":"10.1109\/WAINA.2010.166"},{"key":"5_CR5","unstructured":"Amodei, D., et al.: Deep speech 2: end-to-end speech recognition in English and Mandarin. In: International Conference on Machine Learning, pp. 173\u2013182 (2016)"},{"key":"5_CR6","doi-asserted-by":"crossref","unstructured":"Ben-Nun, T., Besta, M., Huber, S., Ziogas, A.N., Peter, D., Hoefler, T.: A modular benchmarking infrastructure for high-performance and reproducible deep learning. arXiv:1901.10183 (2019)","DOI":"10.1109\/IPDPS.2019.00018"},{"issue":"Feb","key":"5_CR7","first-page":"1137","volume":"3","author":"Y Bengio","year":"2003","unstructured":"Bengio, Y., Ducharme, R., Vincent, P., Jauvin, C.: A neural probabilistic language model. J. Mach. Learn. Res. 3(Feb), 1137\u20131155 (2003)","journal-title":"J. Mach. Learn. Res."},{"key":"5_CR8","unstructured":"BigDataBench: A Big Data and AI Benchmark Suite (2018). http:\/\/prof.ict.ac.cn\/"},{"key":"5_CR9","unstructured":"Huang, C., et al.: AIoT bench: towards comprehensive benchmarking mobile and embedded device intelligence. In: BenchCouncil International Symposium on Benchmarking, Measuring and Optimizing (Bench), Seattle, WA, USA (2018)"},{"key":"5_CR10","doi-asserted-by":"crossref","unstructured":"Chen, T., et al.: BenchNN: on the broad potential application scope of hardware neural network accelerators. In: Proceedings - 2012 IEEE International Symposium on Workload Characterization, IISWC 2012, pp. 36\u201345, November 2012","DOI":"10.1109\/IISWC.2012.6402898"},{"key":"5_CR11","doi-asserted-by":"crossref","unstructured":"Cho, K., et al.: Learning phrase representations using RNN encoder-decoder for statistical machine translation. arXiv preprint arXiv:1406.1078 (2014)","DOI":"10.3115\/v1\/D14-1179"},{"key":"5_CR12","unstructured":"Coleman, C., et al.: DAWNBench: an end-to-end deep learning benchmark and competition. In: Proceedings of ML Systems Workshop, Co-Located with 31st Conference on Neural Information Processing Systems (NIPS) (2017)"},{"key":"5_CR13","unstructured":"Stanford DAWNBench: An End-to-End Deep Learning Benchmark and Competition (2018). https:\/\/dawn.cs.stanford.edu\/benchmark\/"},{"key":"5_CR14","unstructured":"Baidu DeepBench: Benchmarking Deep Learning Operations on Different Hardware (2018). https:\/\/github.com\/baidu-research\/DeepBench"},{"key":"5_CR15","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.-J., Li, K., Fei-Fei, L.: ImageNet: a large-scale hierarchical image database. In: 2009 IEEE Conference on Computer Vision and Pattern Recognition, pp. 248\u2013255. IEEE (2009)","DOI":"10.1109\/CVPR.2009.5206848"},{"issue":"6","key":"5_CR16","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1109\/MSP.2012.2211477","volume":"29","author":"L Deng","year":"2012","unstructured":"Deng, L.: The MNIST database of handwritten digit images for machine learning research. IEEE Sign. Process. Mag. 29(6), 141\u2013142 (2012)","journal-title":"IEEE Sign. Process. Mag."},{"key":"5_CR17","unstructured":"Dongarra, J., Heroux, M.A., Luszczek, P.: HPCG benchmark: a new metric for ranking high performance computing systems. Technical report UT-EECS-15-736, Electrical Engineering and Computer Science Department, University of Tennessee (2015)"},{"issue":"9","key":"5_CR18","doi-asserted-by":"publisher","first-page":"803","DOI":"10.1002\/cpe.728","volume":"15","author":"JJ Dongarra","year":"2003","unstructured":"Dongarra, J.J., Luszczek, P., Petitet, A.: The LINPACK benchmark: past, present and future. Concurrency Comput.: Pract. Exp. 15(9), 803\u2013820 (2003)","journal-title":"Concurrency Comput.: Pract. Exp."},{"issue":"1","key":"5_CR19","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1134\/S1054661816010065","volume":"26","author":"PN Druzhkov","year":"2016","unstructured":"Druzhkov, P.N., Kustikova, V.D.: A survey of deep learning methods and software tools for image classification and object detection. Pattern Recogn. Image Anal. 26(1), 9\u201315 (2016)","journal-title":"Pattern Recogn. Image Anal."},{"issue":"4","key":"5_CR20","doi-asserted-by":"publisher","first-page":"400","DOI":"10.1007\/s10278-017-9965-6","volume":"30","author":"BJ Erickson","year":"2017","unstructured":"Erickson, B.J., Korfiatis, P., Akkus, Z., Kline, T., Philbrick, K.: Toolkits and libraries for deep learning. J. Dig. Imaging 30(4), 400\u2013405 (2017)","journal-title":"J. Dig. Imaging"},{"key":"5_CR21","unstructured":"Facebook AI Performance Evaluation Platform (2018). https:\/\/github.com\/facebook\/FAI-PEP"},{"key":"5_CR22","unstructured":"Gao, W., et al.: BigDataBench: a dwarf-based big data and AI benchmark suite, pp. 1\u201323 (2018)"},{"key":"5_CR23","unstructured":"Hannun, A., et al.: Deep speech: scaling up end-to-end speech recognition. arXiv preprint arXiv:1412.5567 (2014)"},{"key":"5_CR24","doi-asserted-by":"publisher","first-page":"24411","DOI":"10.1109\/ACCESS.2018.2830661","volume":"6","author":"WG Hatcher","year":"2018","unstructured":"Hatcher, W.G., Yu, W.: A survey of deep learning: platforms, applications and emerging research trends. IEEE Access 6, 24411\u201324432 (2018)","journal-title":"IEEE Access"},{"key":"5_CR25","doi-asserted-by":"crossref","unstructured":"Hauswald, J., et al.: DjiNN and Tonic: DNN as a service and its implications for future warehouse scale computers. In: Proceedings of the 42nd Annual International Symposium on Computer Architecture (ISCA), ISCA 2015. ACM, New York (2015)","DOI":"10.1145\/2749469.2749472"},{"key":"5_CR26","doi-asserted-by":"crossref","unstructured":"He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition (2015)","DOI":"10.1109\/CVPR.2016.90"},{"key":"5_CR27","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"599","DOI":"10.1007\/978-3-642-35289-8_32","volume-title":"Neural Networks: Tricks of the Trade","author":"GE Hinton","year":"2012","unstructured":"Hinton, G.E.: A practical guide to training restricted boltzmann machines. In: Montavon, G., Orr, G.B., M\u00fcller, K.-R. (eds.) Neural Networks: Tricks of the Trade. LNCS, vol. 7700, pp. 599\u2013619. Springer, Heidelberg (2012). https:\/\/doi.org\/10.1007\/978-3-642-35289-8_32"},{"issue":"8","key":"5_CR28","doi-asserted-by":"publisher","first-page":"1735","DOI":"10.1162\/neco.1997.9.8.1735","volume":"9","author":"S Hochreiter","year":"1997","unstructured":"Hochreiter, S., Schmidhuber, J.: Long short-term memory. Neural Comput. 9(8), 1735\u20131780 (1997)","journal-title":"Neural Comput."},{"key":"5_CR29","unstructured":"Howard, A.G., et al.: MobileNets: efficient convolutional neural networks for mobile vision applications. arXiv preprint arXiv:1704.04861 (2017)"},{"key":"5_CR30","unstructured":"Iandola, F.N., Han, S., Moskewicz, M.W., Ashraf, K., Dally, W.J., Keutzer, K.: SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and $$<$$0.5 MB model size. arXiv preprint arXiv:1602.07360 (2016)"},{"key":"5_CR31","unstructured":"Facebook Inc.: Caffe2. https:\/\/caffe2.ai\/"},{"key":"5_CR32","unstructured":"Ioffe, S., Szegedy, C.: Batch normalization: accelerating deep network training by reducing internal covariate shift. arXiv preprint arXiv:1502.03167 (2015)"},{"key":"5_CR33","doi-asserted-by":"crossref","unstructured":"Jia, Y., et al.: Caffe: convolutional architecture for fast feature embedding. In: Proceedings of the 22nd ACM International Conference on Multimedia, pp. 675\u2013678. ACM (2014)","DOI":"10.1145\/2647868.2654889"},{"key":"5_CR34","doi-asserted-by":"publisher","first-page":"195","DOI":"10.1007\/978-1-4842-2766-4_12","volume-title":"Deep Learning with Python","author":"N Ketkar","year":"2017","unstructured":"Ketkar, N.: Introduction to PyTorch. Deep Learning with Python, pp. 195\u2013208. Apress, Berkeley (2017). https:\/\/doi.org\/10.1007\/978-1-4842-2766-4_12"},{"key":"5_CR35","unstructured":"Krizhevsky, A., Hinton, G., et al.: Learning multiple layers of features from tiny images. Technical report, Citeseer (2009)"},{"key":"5_CR36","unstructured":"Krizhevsky, A., Sutskever, I., Hinton, G.E.: ImageNet classification with deep convolutional neural networks. In: Advances in Neural Information Processing Systems, pp. 1097\u20131105 (2012)"},{"issue":"1","key":"5_CR37","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1109\/72.554195","volume":"8","author":"S Lawrence","year":"1997","unstructured":"Lawrence, S., Giles, C.L., Tsoi, A.C., Back, A.D.: Face recognition: a convolutional neural-network approach. IEEE Trans. Neural Netw. 8(1), 98\u2013113 (1997)","journal-title":"IEEE Trans. Neural Netw."},{"key":"5_CR38","unstructured":"Lee, J., et al.: On-device augmented reality with mobile GPUs (2019). https:\/\/mixedreality.cs.cornell.edu\/s\/10_CV4ARVR2019-jet-camera-ready.pdf"},{"key":"5_CR39","doi-asserted-by":"crossref","unstructured":"Luszczek, P.R., et al.: The HPC challenge (HPCC) benchmark suite. In: Proceedings of the 2006 ACM\/IEEE Conference on Supercomputing, vol. 213. Citeseer (2006)","DOI":"10.1145\/1188455.1188677"},{"key":"5_CR40","unstructured":"MLPerf: A Broad ML Benchmark Suite for Measuring Performance of ML Software Frameworks, ML Hardware Accelerators, and ML Cloud Platforms (2018). https:\/\/mlperf.org\/"},{"key":"5_CR41","unstructured":"Nelson, M.T., et al.: NAMD: a parallel, object-oriented molecular dynamics program. Int. J. Supercomput. Appl. High Perform. Comput. 10(4), 251\u2013268 (1996)"},{"key":"5_CR42","doi-asserted-by":"crossref","unstructured":"Ota, K., Dao, M.S., Mezaris, V., De Natale, F.G.B.: Deep learning for mobile multimedia: a survey. ACM Trans. Multimed. Comput. Commun. Appl. 13(3s), 34:1\u201334:22 (2017)","DOI":"10.1145\/3092831"},{"key":"5_CR43","doi-asserted-by":"crossref","unstructured":"Rajpurkar, P., Zhang, J., Lopyrev, K., Liang, P.: SQuAD: 100,000+ questions for machine comprehension of text. arXiv preprint arXiv:1606.05250 (2016)","DOI":"10.18653\/v1\/D16-1264"},{"issue":"1","key":"5_CR44","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1016\/j.cels.2016.01.009","volume":"2","author":"L Rampasek","year":"2016","unstructured":"Rampasek, L., Goldenberg, A.: TensorFlow: biology\u2019s gateway to deep learning? Cell Syst. 2(1), 12\u201314 (2016)","journal-title":"Cell Syst."},{"issue":"1","key":"5_CR45","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1109\/89.365379","volume":"3","author":"DA Reynolds","year":"1995","unstructured":"Reynolds, D.A., Rose, R.C.: Robust text-independent speaker identification using Gaussian mixture speaker models. IEEE Trans. Speech Audio Process. 3(1), 72\u201383 (1995)","journal-title":"IEEE Trans. Speech Audio Process."},{"key":"5_CR46","doi-asserted-by":"crossref","unstructured":"Sandler, M., Howard, A., Zhu, M., Zhmoginov, A., Chen, L.-C.: MobileNetV2: inverted residuals and linear bottlenecks. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 4510\u20134520 (2018)","DOI":"10.1109\/CVPR.2018.00474"},{"key":"5_CR47","unstructured":"Simonyan, K., Zisserman, A.: Very deep convolutional networks for large-scale image recognition. arXiv preprint arXiv:1409.1556 (2014)"},{"key":"5_CR48","doi-asserted-by":"crossref","unstructured":"Szegedy, C., Ioffe, S., Vanhoucke, V., Alemi, A.A.: Inception-v4, inception-ResNet and the impact of residual connections on learning. In: Thirty-First AAAI Conference on Artificial Intelligence (2017)","DOI":"10.1609\/aaai.v31i1.11231"},{"key":"5_CR49","doi-asserted-by":"crossref","unstructured":"Szegedy, C., et al.: Going deeper with convolutions. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 1\u20139 (2015)","DOI":"10.1109\/CVPR.2015.7298594"},{"key":"5_CR50","unstructured":"Wen, X., et al.: EdgeAI bench: towards comprehensive end-to-end edge computing benchmarking. In: BenchCouncil International Symposium on Benchmarking, Measuring and Optimizing (Bench), Seattle, WA, USA (2018)"},{"key":"5_CR51","unstructured":"TensorFlow Benchmarks (2018). https:\/\/www.tensorflow.org\/performance\/benchmarks"},{"key":"5_CR52","doi-asserted-by":"crossref","unstructured":"Thomas, S., et al.: CortexSuite: a synthetic brain benchmark suite. In: 2014 IEEE International Symposium on Workload Characterization (IISWC), pp. 76\u201379, October 2014","DOI":"10.1109\/IISWC.2014.6983043"},{"key":"5_CR53","unstructured":"Wang, L., et al.: HPC AI500: a benchmark suite for HPC AI systems. In: BenchCouncil International Symposium on Benchmarking, Measuring and Optimizing (Bench), Seattle, WA, USA (2018)"},{"key":"5_CR54","doi-asserted-by":"crossref","unstructured":"Zhang, X., Zhou, X., Lin, M., Sun, J.: ShuffleNet: an extremely efficient convolutional neural network for mobile devices. In: Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 6848\u20136856 (2018)","DOI":"10.1109\/CVPR.2018.00716"}],"container-title":["Lecture Notes in Computer Science","Benchmarking, Measuring, and Optimizing"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-32813-9_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,1]],"date-time":"2022-10-01T20:52:23Z","timestamp":1664657543000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-32813-9_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019]]},"ISBN":["9783030328122","9783030328139"],"references-count":54,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-32813-9_5","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2019]]},"assertion":[{"value":"8 October 2019","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"Bench","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Symposium on Benchmarking, Measuring and Optimization","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Seattle, WA","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"USA","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2018","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 December 2018","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 December 2018","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"bench2018","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"http:\/\/prof.ict.ac.cn\/Bench18\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CyberDhair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"51","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"20","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"39% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}