{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,10]],"date-time":"2026-07-10T20:33:03Z","timestamp":1783715583754,"version":"3.55.0"},"reference-count":60,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"1","license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"name":"Exascale Computing Project","award":["17-SC-20-SC"],"award-info":[{"award-number":["17-SC-20-SC"]}]},{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006168","name":"National Nuclear Security Administration","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006168","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","award":["66150"],"award-info":[{"award-number":["66150"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100011661","name":"Pacific Northwest National Laboratory","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100011661","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","award":["DE-AC05-00OR22725"],"award-info":[{"award-number":["DE-AC05-00OR22725"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000015","name":"U.S. Department of Energy","doi-asserted-by":"publisher","award":["DE-AC05-76RL01830"],"award-info":[{"award-number":["DE-AC05-76RL01830"]}],"id":[{"id":"10.13039\/100000015","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Parallel Distrib. Syst."],"published-print":{"date-parts":[[2020,1,1]]},"DOI":"10.1109\/tpds.2019.2928289","type":"journal-article","created":{"date-parts":[[2019,7,15]],"date-time":"2019-07-15T23:39:13Z","timestamp":1563233953000},"page":"94-110","source":"Crossref","is-referenced-by-count":231,"title":["Evaluating Modern GPU Interconnect: PCIe, NVLink, NV-SLI, NVSwitch and GPUDirect"],"prefix":"10.1109","volume":"31","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3734-9137","authenticated-orcid":false,"given":"Ang","family":"Li","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shuaiwen Leon","family":"Song","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jieyang","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiajia","family":"Li","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8984-5485","authenticated-orcid":false,"given":"Xu","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nathan R.","family":"Tallent","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kevin J.","family":"Barker","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS.2016.89"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/2749246.2749265"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/HiPC.2017.00037"},{"key":"ref32","first-page":"3","article-title":"Evaluating on-node GPU interconnects for deep learning workloads","author":"tallent","year":"2017","journal-title":"Proc 1st Int Workshop Perform Model Benchmark Simul High Perform Comput Syst"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1109\/ICDCS.2017.259"},{"key":"ref30","article-title":"Developing a Linux kernel module using GPUDirect RDMA.","year":"2018"},{"key":"ref37","article-title":"Perlmutter: NERSC's next supercomputer.","year":"2018"},{"key":"ref36","article-title":"Early experiences with machine learning and deep learning on Summit\/SummitDev","author":"yin","year":"2018"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/3126908.3126950"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1016\/j.jpdc.2017.12.007"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3236367.3236381"},{"key":"ref28","article-title":"ROCm communication collectives library (RCCL).","year":"2018"},{"key":"ref27","article-title":"Cirrascale SR3514: Unexpected performance inequality. Technical brief M901A-092014.","year":"2018"},{"key":"ref29","article-title":"Summit early access development platform.","year":"2017"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.5194\/gmd-11-1665-2018"},{"key":"ref1","article-title":"Accurate, large minibatch SGD: Training ImageNet in 1 Hour","volume":"abs 1706 2677","author":"goyal","year":"2017","journal-title":"CoRR"},{"key":"ref20","article-title":"CUDA SDK Code Samples","year":"2015"},{"key":"ref22","article-title":"NVIDIA Collective Communications Library (NCCL-V1).","year":"2018"},{"key":"ref21","article-title":"NCCL Tests.","year":"0"},{"key":"ref24","year":"2017"},{"key":"ref23","article-title":"NVIDIA Collective Communications Library (NCCL-V2).","year":"2018"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1287\/opre.9.3.383"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/IISWC.2018.8573483"},{"key":"ref50","first-page":"214","article-title":"CUDAAdvisor: LLVM-based runtime profiling for modern GPUs","author":"shen","year":"2018","journal-title":"Proc Int Symp Code Generation Optimization"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/3200691.3178491"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPS.2017.94"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/SC.2016.51"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1007\/s00450-011-0171-3"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS.2018.00034"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1145\/2751205.2751218"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807611"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1109\/TPDS.2013.222"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/1964179.1964194"},{"key":"ref10","year":"2017"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/HOTI.2010.24"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3126908.3126931"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2017.37"},{"key":"ref13","article-title":"SLI best practices","year":"2011"},{"key":"ref14","article-title":"ATI CrossFire Pro User Guide","year":"2009"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO.2014.55"},{"key":"ref16","article-title":"NVIDIA NVSWITCH – The World's highest-bandwidth on-node switch.","year":"2018"},{"key":"ref17","article-title":"Introduction to infiniband for end users","author":"grun","year":"2010"},{"key":"ref18","article-title":"Developing a Linux Kernel module using GPUDirect RDMA.","year":"2018"},{"key":"ref19","article-title":"ROCm driver RDMA peer to peer support.","year":"2016"},{"key":"ref4","article-title":"NVIDIA DGX-1 System Architecture White Paper","year":"2017"},{"key":"ref3","article-title":"ImageNet\/ResNet-50 training in 224 Seconds","volume":"abs 1811 5233","author":"mikami","year":"2018","journal-title":"CoRR"},{"key":"ref6","article-title":"Summit: The next leap in leadership-class computing systems for open science.","year":"2018"},{"key":"ref5","article-title":"NVIDIA DGX-2: The world's most powerful system for the most complex AI challenges","year":"2018"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-8659.2010.01769.x"},{"key":"ref7","article-title":"Sierra: Supporting NNSA's stockpile stewardship mission through simulation in lieu of underground testing.","year":"2018"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3205289.3205294"},{"key":"ref9","first-page":"140","article-title":"Graph processing on GPU: Where are the bottlenecks?","author":"xu","year":"2014","journal-title":"Proc Int Symp Workload Characterization"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/IA3.2016.010"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-43659-3_45"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/3123939.3123944"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/3093315.3037709"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1145\/2751205.2751232"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807606"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.3850\/9783981537079_0394"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/2925426.2926255"}],"container-title":["IEEE Transactions on Parallel and Distributed Systems"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielam\/71\/8935555\/8763922-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/71\/8935555\/08763922.pdf?arnumber=8763922","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,4,27]],"date-time":"2022-04-27T14:40:33Z","timestamp":1651070433000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8763922\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,1,1]]},"references-count":60,"journal-issue":{"issue":"1"},"URL":"https:\/\/doi.org\/10.1109\/tpds.2019.2928289","relation":{},"ISSN":["1045-9219","1558-2183","2161-9883"],"issn-type":[{"value":"1045-9219","type":"print"},{"value":"1558-2183","type":"electronic"},{"value":"2161-9883","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,1,1]]}}}