{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T11:59:16Z","timestamp":1792756756521,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":33,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,27]],"date-time":"2024-10-27T00:00:00Z","timestamp":1729987200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,27]]},"DOI":"10.1145\/3676536.3676766","type":"proceedings-article","created":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T08:53:56Z","timestamp":1744188836000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":16,"title":["ConSmax: Hardware-Friendly Alternative Softmax with Learnable Parameters"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0564-4900","authenticated-orcid":false,"given":"Shiwei","family":"Liu","sequence":"first","affiliation":[{"name":"Google Research, mountain view, USA"},{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6225-0423","authenticated-orcid":false,"given":"Guanchen","family":"Tao","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-7012-2560","authenticated-orcid":false,"given":"Yifei","family":"Zou","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-7931-7765","authenticated-orcid":false,"given":"Derek","family":"Chow","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8181-2996","authenticated-orcid":false,"given":"Zichen","family":"Fan","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-3760-6441","authenticated-orcid":false,"given":"Kauna","family":"Lei","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-6393-8175","authenticated-orcid":false,"given":"Bangfei","family":"Pan","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2598-0458","authenticated-orcid":false,"given":"Dennis","family":"Sylvester","sequence":"additional","affiliation":[{"name":"University of Michigan, Ann Arbor, MI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8527-792X","authenticated-orcid":false,"given":"Gregory","family":"Kielian","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0237-4690","authenticated-orcid":false,"given":"Mehdi","family":"Saligane","sequence":"additional","affiliation":[{"name":"Electrical Engineering and Computer Science, University of Michigan, Ann Arbor, Michigan, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,4,9]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"1877","volume-title":"Advances in Neural Information Processing Systems (NeurIPS)","volume":"33","author":"Brown T.","year":"2020","unstructured":"T. Brown et al. Language models are few-shot learners. In Advances in Neural Information Processing Systems (NeurIPS), volume 33, pages 1877--1901, 2020."},{"key":"e_1_3_2_1_2_1","first-page":"148","volume-title":"Proceedings of the Workshop on Neural Generation and Translation","year":"2019","unstructured":"Gong et al. Enhanced transformer model for data-to-text generation. In Proceedings of the Workshop on Neural Generation and Translation, pages 148--156, 2019."},{"key":"e_1_3_2_1_3_1","first-page":"6836","volume-title":"Proceedings of the international conference on computer vision (ICCV)","author":"Arnab A.","year":"2021","unstructured":"A. Arnab et al. Vivit: A video vision transformer. In Proceedings of the international conference on computer vision (ICCV), pages 6836--6846, 2021."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA47549.2020.00035"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISCA52012.2021.00060"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA51647.2021.00018"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/DAC18074.2021.9586134"},{"key":"e_1_3_2_1_8_1","volume-title":"Mistral 7b. In arXiv preprint arXiv:2310.06825","author":"Jiang A.","year":"2023","unstructured":"A. Jiang et al. Mistral 7b. In arXiv preprint arXiv:2310.06825, 2023."},{"key":"e_1_3_2_1_9_1","volume-title":"Llama: Open and efficient foundation language models","author":"Touvron H.","year":"2023","unstructured":"H. Touvron et al. Llama: Open and efficient foundation language models, 2023."},{"key":"e_1_3_2_1_10_1","first-page":"28092","volume-title":"Advances in Neural Information Processing Systems (NeurIPS)","volume":"34","author":"Liu Z.","year":"2021","unstructured":"Z. Liu et al. Post-training quantization for vision transformer. In Advances in Neural Information Processing Systems (NeurIPS), volume 34, pages 28092--28103, 2021."},{"key":"e_1_3_2_1_11_1","volume-title":"Gptq: Accurate post-training quantization for generative pre-trained transformers. In arXiv preprint arXiv:2210.17323","author":"Frantar F.","year":"2022","unstructured":"F. Frantar et al. Gptq: Accurate post-training quantization for generative pre-trained transformers. In arXiv preprint arXiv:2210.17323, 2022."},{"key":"e_1_3_2_1_12_1","volume-title":"Quip: 2-bit quantization of large language models with guarantees. In arXiv preprint arXiv:2307.13304","author":"Chee J.","year":"2023","unstructured":"J. Chee et al. Quip: 2-bit quantization of large language models with guarantees. In arXiv preprint arXiv:2307.13304, 2023."},{"key":"e_1_3_2_1_13_1","first-page":"10323","volume-title":"International Conference on Machine Learning (ICML)","author":"Frantar F.","year":"2023","unstructured":"F. Frantar et al. Sparsegpt: Massive language models can be accurately pruned in one-shot. In International Conference on Machine Learning (ICML), pages 10323--10337, 2023."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISSCC42615.2023.10067360"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/MICRO56248.2022.00059"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3579371.3589057"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3503222.3507738"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/HPCA53966.2022.00082"},{"key":"e_1_3_2_1_19_1","first-page":"616","volume-title":"International Symposium on Microarchitecture (MICRO)","author":"Hong S.","year":"2022","unstructured":"S. Hong et al. Dfx: A low-latency multi-fpga appliance for accelerating transformer-based text generation. In International Symposium on Microarchitecture (MICRO), pages 616--630, 2022."},{"key":"e_1_3_2_1_20_1","first-page":"16344","volume-title":"Advances in Neural Information Processing Systems (NeurIPS)","volume":"35","author":"Dao T.","year":"2022","unstructured":"T. Dao et al. Flashattention: Fast and memory-efficient exact attention with io-awareness. In Advances in Neural Information Processing Systems (NeurIPS), volume 35, pages 16344--16359, 2022."},{"key":"e_1_3_2_1_21_1","volume-title":"Flashdecoding++: Faster large language model inference on gpus. In arXiv preprint arXiv:2307.08691","author":"Hong K.","year":"2023","unstructured":"K. Hong et al. Flashdecoding++: Faster large language model inference on gpus. In arXiv preprint arXiv:2307.08691, 2023."},{"key":"e_1_3_2_1_22_1","first-page":"3605","volume-title":"Transactions on Circuits and Systems I: Regular Papers","volume":"69","author":"Zhang Y.","year":"2022","unstructured":"Y. Zhang et al. Base-2 softmax function: Suitability for training and efficient hardware implementation. In Transactions on Circuits and Systems I: Regular Papers, volume 69, pages 3605--3618, 2022."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVLSI.2022.3224011"},{"key":"e_1_3_2_1_24_1","first-page":"577","volume-title":"Proceedings of the ACM\/IEEE Design Automation Conference (DAC)","author":"Joonsang Y.","year":"2022","unstructured":"Y. Joonsang et al. Nn-lut: neural approximation of non-linear operations for efficient transformer inference. In Proceedings of the ACM\/IEEE Design Automation Conference (DAC), pages 577--582, 2022."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3299874.3317988"},{"key":"e_1_3_2_1_26_1","volume-title":"Exploring alternatives to softmax function. In arXiv preprint arXiv:2011.11538","author":"Banerjee E.","year":"2020","unstructured":"E. Banerjee et al. Exploring alternatives to softmax function. In arXiv preprint arXiv:2011.11538, 2020."},{"key":"e_1_3_2_1_27_1","volume-title":"An exploration of softmax alternatives belonging to the spherical loss family. In arXiv preprint arXiv:1511.05042","author":"Br\u00e9bisson A.","year":"2015","unstructured":"A. Br\u00e9bisson and P. Vincent. An exploration of softmax alternatives belonging to the spherical loss family. In arXiv preprint arXiv:1511.05042, 2015."},{"key":"e_1_3_2_1_28_1","volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. In arXiv preprint arXiv:1810.04805","author":"Devlin J.","year":"2018","unstructured":"J. Devlin et al. Bert: Pre-training of deep bidirectional transformers for language understanding. In arXiv preprint arXiv:1810.04805, 2018."},{"key":"e_1_3_2_1_29_1","volume-title":"Faster attention with better parallelism and work partitioning. In arXiv preprint arXiv:2307.08691","author":"Dao T.","year":"2023","unstructured":"T. Dao. Flashattention-2: Faster attention with better parallelism and work partitioning. In arXiv preprint arXiv:2307.08691, 2023."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP39728.2021.9414076"},{"key":"e_1_3_2_1_31_1","volume-title":"Pointer sentinel mixture models","author":"Merity S.","year":"2016","unstructured":"S. Merity et al. Pointer sentinel mixture models. 2016."},{"key":"e_1_3_2_1_32_1","first-page":"943","volume-title":"Proceedings of the 26th ACM International Conference on Architectural Support for Programming Languages and Operating Systems","author":"Fletcher W","year":"2021","unstructured":"Kartik Hegde, Po-An Tsai, Sitao Huang, Vikas Chandra, Angshuman Parashar, and Christopher W Fletcher. Mind mappings: enabling efficient algorithm-accelerator mapping space search. In Proceedings of the 26th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, pages 943--958, 2021."},{"key":"e_1_3_2_1_33_1","first-page":"1","volume-title":"Proceedings of the 56th Annual Design Automation Conference 2019","author":"Chhabria A","year":"2019","unstructured":"Tutu Ajayi, Vidya A Chhabria, Mateus Foga\u00e7a, Soheil Hashemi, Abdelrahman Hosny, Andrew B Kahng, Minsoo Kim, Jeongsup Lee, Uday Mallappa, Marina Neseem, et al. Toward an open-source digital flow: First learnings from the openroad project. In Proceedings of the 56th Annual Design Automation Conference 2019, pages 1--4, 2019."}],"event":{"name":"ICCAD '24: 43rd IEEE\/ACM International Conference on Computer-Aided Design","location":"Newark Liberty International Airport Marriott New York NY USA","acronym":"ICCAD '24","sponsor":["SIGDA ACM Special Interest Group on Design Automation","IEEE CAS","IEEE CEDA","IEEE EDS"]},"container-title":["Proceedings of the 43rd IEEE\/ACM International Conference on Computer-Aided Design"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676536.3676766","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3676536.3676766","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T19:43:58Z","timestamp":1750275838000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3676536.3676766"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,27]]},"references-count":33,"alternative-id":["10.1145\/3676536.3676766","10.1145\/3676536"],"URL":"https:\/\/doi.org\/10.1145\/3676536.3676766","relation":{},"subject":[],"published":{"date-parts":[[2024,10,27]]},"assertion":[{"value":"2025-04-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}