{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T13:49:31Z","timestamp":1765547371050,"version":"3.33.0"},"reference-count":52,"publisher":"IEEE","license":[{"start":{"date-parts":[[2024,12,15]],"date-time":"2024-12-15T00:00:00Z","timestamp":1734220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,12,15]],"date-time":"2024-12-15T00:00:00Z","timestamp":1734220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,12,15]]},"DOI":"10.1109\/bigdata62323.2024.10825630","type":"proceedings-article","created":{"date-parts":[[2025,1,16]],"date-time":"2025-01-16T18:31:23Z","timestamp":1737052283000},"page":"24-33","source":"Crossref","is-referenced-by-count":2,"title":["Zero-th Order Algorithm for Softmax Attention Optimization"],"prefix":"10.1109","author":[{"given":"Yichuan","family":"Deng","sequence":"first","affiliation":[{"name":"University of Washington,Paul G. Allen School of CSE,Seattle,WA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhihang","family":"Li","sequence":"additional","affiliation":[{"name":"UCSD,Department of CSE,San Diego,CA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sridhar","family":"Mahadevan","sequence":"additional","affiliation":[{"name":"Adobe Research Adobe,San Jose,CA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhao","family":"Song","sequence":"additional","affiliation":[{"name":"UC Berkeley,Simons Institute,Berkeley,CA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"article-title":"Improving language understanding by generative pre-training","year":"2018","author":"Radford","key":"ref2"},{"article-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","year":"2018","author":"Devlin","key":"ref3"},{"issue":"8","key":"ref4","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI blog"},{"key":"ref5","first-page":"1877","article-title":"Language models are few-shot learners","volume":"33","author":"Brown","year":"2020","journal-title":"Advances in neural information processing systems"},{"article-title":"Palm: Scaling language modeling with pathways","year":"2022","author":"Chowdhery","key":"ref6"},{"article-title":"Opt: Open pre-trained transformer language models","year":"2022","author":"Zhang","key":"ref7"},{"article-title":"Optimizing language models for dialogue","volume-title":"OpenAI Blog","year":"2022","key":"ref8"},{"year":"2023","key":"ref9","article-title":"Gpt-4 technical report"},{"article-title":"Sparks of artificial general intelligence: Early experiments with gpt-4","year":"2023","author":"Bubeck","key":"ref10"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/ITNEC52019.2021.9586824"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2020.07.022"},{"article-title":"Camembert: a tasty french language model","year":"2019","author":"Martin","key":"ref13"},{"article-title":"Kdeformer: Accelerating transformers via kernel density estimation","year":"2023","author":"Zandieh","key":"ref14"},{"article-title":"Fast attention requires bounded entries","year":"2023","author":"Alman","key":"ref15"},{"article-title":"Algorithm and hardness for dynamic attention maintenance in large language models","year":"2023","author":"Brand","key":"ref16"},{"article-title":"Solving regularized exp, cosh and sinh regression problems","year":"2023","author":"Li","key":"ref17"},{"article-title":"An over-parameterized exponential regression","year":"2023","author":"Gao","key":"ref18"},{"article-title":"Attention scheme inspired softmax regression","year":"2023","author":"Deng","key":"ref19"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/9.119632"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/7.705889"},{"article-title":"Fine-tuning language models with just forward passes","year":"2023","author":"Malladi","key":"ref22"},{"article-title":"Reformer: The efficient transformer","year":"2020","author":"Kitaev","key":"ref23"},{"article-title":"Approximating how single head attention learns","year":"2021","author":"Snell","key":"ref24"},{"article-title":"Mongoose: A learnable lsh framework for efficient neural network training","volume-title":"International Conference on Learning Representations","author":"Chen","key":"ref25"},{"article-title":"How do transformers learn topic structure: Towards a mechanistic understanding","year":"2023","author":"Li","key":"ref26"},{"key":"ref27","article-title":"Randomized and deterministic attention sparsification algorithms for over-parameterized feature dimension","author":"Deng","year":"2023","journal-title":"arxiv preprint: arxiv 2304.03426"},{"article-title":"In-context learning for attention scheme: from single softmax regression to multiple softmax regression via a tensor trick","year":"2023","author":"Gao","key":"ref28"},{"article-title":"The closeness of in-context learning and weight shifting for softmax regression","year":"2023","author":"Li","key":"ref29"},{"article-title":"An iterative algorithm for rescaled hyperbolic functions regression","year":"2023","author":"Gao","key":"ref30"},{"article-title":"Differentially private attention computation","year":"2023","author":"Gao","key":"ref31"},{"key":"ref32","article-title":"H2o: Heavy-hitter oracle for efficient generative inference of large language models","volume":"abs\/2306.14048","author":"Zhang","year":"2023","journal-title":"CoRR"},{"article-title":"Infoprompt: Information-theoretic soft prompt tuning for natural language understanding","year":"2023","author":"Wu","key":"ref33"},{"article-title":"A mathematical abstraction for balancing the trade-off between creativity and reality in large language models","year":"2023","author":"Sinha","key":"ref34"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177729392"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.23919\/ACC.1987.4789489"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/7.4.308"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TIT.2015.2409256"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/s10208-021-09499-8"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i04.5926"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/3128572.3140448"},{"key":"ref42","article-title":"Practical bayesian optimization of machine learning algorithms","volume":"25","author":"Snoek","year":"2012","journal-title":"Advances in neural information processing systems"},{"article-title":"Evolution strategies as a scalable alternative to reinforcement learning","year":"2017","author":"Salimans","key":"ref43"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/s10208-015-9296-2"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1137\/120880811"},{"article-title":"Just one byte (per gradient): A note on low-bandwidth decentralized language model finetuning using shared randomness","year":"2023","author":"Zelikman","key":"ref46"},{"article-title":"signsgd via zeroth-order oracle","volume-title":"International Conference on Learning Representations","author":"Liu","key":"ref47"},{"key":"ref48","article-title":"Simple random search of static linear policies is competitive for reinforcement learning","volume":"31","author":"Mania","year":"2018","journal-title":"Advances in Neural Information Processing Systems"},{"article-title":"Gradientless descent: High-dimensional zeroth-order optimization","year":"2019","author":"Golovin","key":"ref49"},{"article-title":"The forward-forward algorithm: Some preliminary investigations","year":"2022","author":"Hinton","key":"ref50"},{"article-title":"Optimal approximate matrix product in terms of stable rank","year":"2015","author":"Cohen","key":"ref51"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-46128-1_50"}],"event":{"name":"2024 IEEE International Conference on Big Data (BigData)","start":{"date-parts":[[2024,12,15]]},"location":"Washington, DC, USA","end":{"date-parts":[[2024,12,18]]}},"container-title":["2024 IEEE International Conference on Big Data (BigData)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10824975\/10824942\/10825630.pdf?arnumber=10825630","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,17]],"date-time":"2025-01-17T07:47:39Z","timestamp":1737100059000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10825630\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12,15]]},"references-count":52,"URL":"https:\/\/doi.org\/10.1109\/bigdata62323.2024.10825630","relation":{},"subject":[],"published":{"date-parts":[[2024,12,15]]}}}