{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,22]],"date-time":"2026-08-22T10:22:57Z","timestamp":1787394177683,"version":"3.56.0"},"reference-count":13,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2026,9,1]],"date-time":"2026-09-01T00:00:00Z","timestamp":1788220800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2026,6,20]],"date-time":"2026-06-20T00:00:00Z","timestamp":1781913600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100004063","name":"Knut and Alice Wallenberg Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004063","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Theoretical Computer Science"],"published-print":{"date-parts":[[2026,9]]},"DOI":"10.1016\/j.tcs.2026.116131","type":"journal-article","created":{"date-parts":[[2026,6,23]],"date-time":"2026-06-23T07:21:07Z","timestamp":1782199267000},"page":"116131","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":0,"special_numbering":"C","title":["Constructing a BPE tokenization DFA"],"prefix":"10.1016","volume":"1083","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3692-6994","authenticated-orcid":false,"given":"Martin","family":"Berglund","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Willeke","family":"Martens","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brink","family":"van der Merwe","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.tcs.2026.116131_bib0001","series-title":"Implementation and Application of Automata","first-page":"66","article-title":"Constructing a BPE tokenization DFA","author":"Berglund","year":"2024"},{"key":"10.1016\/j.tcs.2026.116131_bib0002","series-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","first-page":"1715","article-title":"Neural machine translation of rare words with subword units","author":"Sennrich","year":"2016"},{"issue":"8","key":"10.1016\/j.tcs.2026.116131_bib0003","first-page":"9","article-title":"Language models are unsupervised multitask learners","volume":"1","author":"Radford","year":"2019","journal-title":"OpenAI Blog"},{"key":"10.1016\/j.tcs.2026.116131_bib0004","unstructured":"OpenAI, ChatGPT: optimizing language models for dialogue, 2022, https:\/\/openai.com\/blog\/chatgpt\/."},{"key":"10.1016\/j.tcs.2026.116131_bib0005","series-title":"13th International Workshop on Non-Classical Models of Automata and Applications, NCMA 2023, 18-19 September, 2023, Famagusta, Cyprus","first-page":"16","article-title":"Formalizing BPE tokenization","author":"Berglund","year":"2023"},{"key":"10.1016\/j.tcs.2026.116131_bib0006","doi-asserted-by":"crossref","unstructured":"Y. Li, Y. Liu, G. Deng, Y. Zhang, W. Song, L. Shi, K. Wang, Y. Li, Y. Liu, H. Wang, Glitch tokens in large language models: categorization taxonomy and effective detection, (2024). arXiv preprint arXiv: 2404.09894.","DOI":"10.1145\/3660799"},{"key":"10.1016\/j.tcs.2026.116131_bib0007","doi-asserted-by":"crossref","unstructured":"T. Kudo, J. Richardson, SentencePiece: a simple and language independent subword tokenizer and detokenizer for neural text processing, CoRR(2018). arXiv: 1808.06226.","DOI":"10.18653\/v1\/D18-2012"},{"key":"10.1016\/j.tcs.2026.116131_bib0008","unstructured":"Hugging face transformers, 2023, https:\/\/github.com\/huggingface\/transformers\/blob\/v4.28.1\/src\/transformers\/models\/gpt2\/tokenization_gpt2.py."},{"issue":"1-2","key":"10.1016\/j.tcs.2026.116131_bib0009","doi-asserted-by":"crossref","first-page":"361","DOI":"10.1016\/S0304-3975(98)00332-6","article-title":"Families of locally testable languages","volume":"242","author":"Caron","year":"2000","journal-title":"Theor. Comput. Sci."},{"issue":"2-3","key":"10.1016\/j.tcs.2026.116131_bib0010","doi-asserted-by":"crossref","first-page":"283","DOI":"10.1016\/0304-3975(88)90043-6","article-title":"Codes circulaires, automates locaux et entropie","volume":"57","author":"Beal","year":"1988","journal-title":"Theor. Comput. Sci."},{"issue":"2","key":"10.1016\/j.tcs.2026.116131_bib0011","doi-asserted-by":"crossref","first-page":"439","DOI":"10.1016\/0304-3975(95)00104-2","article-title":"Local languages and the Berry-Sethi algorithm","volume":"155","author":"Berstel","year":"1996","journal-title":"Theor. Comput. Sci."},{"issue":"1","key":"10.1016\/j.tcs.2026.116131_bib0012","doi-asserted-by":"crossref","first-page":"225","DOI":"10.1016\/S0304-3975(01)00271-7","article-title":"Determinization of transducers over finite and infinite words","volume":"289","author":"B\u00e9al","year":"2002","journal-title":"Theor. Comput. Sci."},{"key":"10.1016\/j.tcs.2026.116131_bib0013","unstructured":"M. Cognetta, N. Okazaki, Tokenization as finite-state transduction, (2024). arXiv preprint arXiv: 2410.15696."}],"container-title":["Theoretical Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0304397526003609?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0304397526003609?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,8,22]],"date-time":"2026-08-22T09:24:05Z","timestamp":1787390645000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0304397526003609"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,9]]},"references-count":13,"alternative-id":["S0304397526003609"],"URL":"https:\/\/doi.org\/10.1016\/j.tcs.2026.116131","relation":{},"ISSN":["0304-3975"],"issn-type":[{"value":"0304-3975","type":"print"}],"subject":[],"published":{"date-parts":[[2026,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Constructing a BPE tokenization DFA","name":"articletitle","label":"Article Title"},{"value":"Theoretical Computer Science","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.tcs.2026.116131","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2026 The Author(s). Published by Elsevier B.V.","name":"copyright","label":"Copyright"}],"article-number":"116131"}}