{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,9]],"date-time":"2026-06-09T08:39:50Z","timestamp":1780994390580,"version":"3.54.1"},"publisher-location":"Cham","reference-count":7,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031711114","type":"print"},{"value":"9783031711121","type":"electronic"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-71112-1_5","type":"book-chapter","created":{"date-parts":[[2024,9,3]],"date-time":"2024-09-03T02:02:14Z","timestamp":1725328934000},"page":"66-78","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Constructing a\u00a0BPE Tokenization DFA"],"prefix":"10.1007","author":[{"given":"Martin","family":"Berglund","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Willeke","family":"Martens","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brink","family":"van der Merwe","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,9,3]]},"reference":[{"key":"5_CR1","doi-asserted-by":"crossref","unstructured":"Berglund, M., van\u00a0der Merwe, B.: Formalizing BPE tokenization. In: 13th International Workshop on Non-classical Models of Automata and Applications, NCMA 2023, Famagusta, Cyprus, 18\u201319 September 2023, pp. 16\u201327. Open Publishing Association (2023)","DOI":"10.4204\/EPTCS.388.4"},{"key":"5_CR2","unstructured":"Hugging Face: Transformers (2023). https:\/\/github.com\/huggingface\/transformers\/blob\/v4.28.1\/src\/transformers\/models\/gpt2\/tokenization_gpt2.py"},{"key":"5_CR3","doi-asserted-by":"crossref","unstructured":"Kudo, T., Richardson, J.: SentencePiece: a simple and language independent subword tokenizer and detokenizer for neural text processing. CoRR abs\/1808.06226 (2018)","DOI":"10.18653\/v1\/D18-2012"},{"key":"5_CR4","doi-asserted-by":"crossref","unstructured":"Li, Y., et al.: Glitch tokens in large language models: categorization taxonomy and effective detection. arXiv preprint arXiv:2404.09894 (2024)","DOI":"10.1145\/3660799"},{"key":"5_CR5","unstructured":"OpenAI: ChatGPT: optimizing language models for dialogue (2022). https:\/\/openai.com\/blog\/chatgpt\/"},{"issue":"8","key":"5_CR6","first-page":"9","volume":"1","author":"A Radford","year":"2019","unstructured":"Radford, A., Wu, J., Child, R., Luan, D., Amodei, D., Sutskever, I., et al.: Language models are unsupervised multitask learners. OpenAI Blog 1(8), 9 (2019)","journal-title":"OpenAI Blog"},{"key":"5_CR7","doi-asserted-by":"publisher","unstructured":"Sennrich, R., Haddow, B., Birch, A.: Neural machine translation of rare words with subword units. In: Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Berlin, Germany, pp. 1715\u20131725. Association for Computational Linguistics (2016). https:\/\/doi.org\/10.18653\/v1\/P16-1162","DOI":"10.18653\/v1\/P16-1162"}],"container-title":["Lecture Notes in Computer Science","Implementation and Application of Automata"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-71112-1_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,3]],"date-time":"2024-09-03T02:03:03Z","timestamp":1725328983000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-71112-1_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031711114","9783031711121"],"references-count":7,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-71112-1_5","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"3 September 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"CIAA","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Implementation and Application of Automata","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Akita","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Japan","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2024","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 September 2024","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6 September 2024","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"wia2024","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}