{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,7]],"date-time":"2026-05-07T15:20:26Z","timestamp":1778167226362,"version":"3.51.4"},"reference-count":28,"publisher":"IEEE","license":[{"start":{"date-parts":[[2024,12,15]],"date-time":"2024-12-15T00:00:00Z","timestamp":1734220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2024,12,15]],"date-time":"2024-12-15T00:00:00Z","timestamp":1734220800000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024,12,15]]},"DOI":"10.1109\/bigdata62323.2024.10825085","type":"proceedings-article","created":{"date-parts":[[2025,1,16]],"date-time":"2025-01-16T18:31:23Z","timestamp":1737052283000},"page":"2234-2243","source":"Crossref","is-referenced-by-count":3,"title":["Data-Prep-Kit: getting your data ready for LLM application development"],"prefix":"10.1109","author":[{"given":"David","family":"Wood","sequence":"first","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Boris","family":"Lublinsky","sequence":"additional","affiliation":[{"name":"IBM Research,Mulhaddart,Ireland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexy","family":"Roytman","sequence":"additional","affiliation":[{"name":"IBM Research Israel,Haifa,Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shivdeep","family":"Singh","sequence":"additional","affiliation":[{"name":"IBM Research India,Bangalore,India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Constantin","family":"Adam","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Abdulhamid","family":"Adebayo","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sungeun","family":"An","sequence":"additional","affiliation":[{"name":"Almaden Research Center IBM,San Jose,CA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yuan Chi","family":"Chang","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xuan-Hong","family":"Dang","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nirmit","family":"Desai","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michele","family":"Dolfi","sequence":"additional","affiliation":[{"name":"IBM Research,Zurich,Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hajar","family":"Emami-Gohari","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Revital","family":"Eres","sequence":"additional","affiliation":[{"name":"IBM Research Israel,Haifa,Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Takuya","family":"Goto","sequence":"additional","affiliation":[{"name":"IBM Software,Tokyo,Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dhiraj","family":"Joshi","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yan","family":"Koyfman","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mohammad","family":"Nassar","sequence":"additional","affiliation":[{"name":"IBM Research Israel,Haifa,Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hima","family":"Patel","sequence":"additional","affiliation":[{"name":"IBM Research India,Bangalore,India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Paramesvaran","family":"Selvam","sequence":"additional","affiliation":[{"name":"IBM Research India,Bangalore,India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Yousaf","family":"Shah","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Saptha","family":"Surendran","sequence":"additional","affiliation":[{"name":"IBM Research India,Bangalore,India"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Daiki","family":"Tsuzuku","sequence":"additional","affiliation":[{"name":"IBM Software,Tokyo,Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Petros","family":"Zerfos","sequence":"additional","affiliation":[{"name":"IBM Research,New York,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shahrokh","family":"Daijavad","sequence":"additional","affiliation":[{"name":"Almaden Research Center IBM,San Jose,CA,USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","article-title":"Granite Code Models: A Family of Open Foundation Models for Code Intelligence","author":"Mishra","year":"2024"},{"key":"ref2","volume-title":"IBM Granite Models","year":"2024"},{"key":"ref3","article-title":"The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only","author":"Penedo","year":"2023"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/3603709"},{"key":"ref5","article-title":"Ray: A distributed framework for emerging {AI} applications","volume-title":"13th USENIX symposium on operating systems design and implementation","author":"Moritz"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/2934664"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4842-4470-8_46"},{"key":"ref9","article-title":"Datatrove: large scale data processing","author":"Penedo","year":"2024"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.840"},{"key":"ref11","article-title":"NeMo: a toolkit for Conversational AI and Large Language Models","author":"Harper","year":"2024"},{"key":"ref12","article-title":"DataComp-LM: In search of the next generation of training sets for language models","author":"Li","year":"2024"},{"key":"ref13","volume-title":"Get your data RAG ready","year":"2024"},{"key":"ref14","article-title":"Large Language Models as Advanced Data Preprocessors: Transforming Unstructured Text into Fine-Tuning Datasets","author":"Vangeli","year":"2024"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.2307\/j.ctt1ffjh7m.4"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.25080\/Majora-7b98e3ed-013"},{"key":"ref17","article-title":"Python Multiprocessing Pool"},{"key":"ref18","article-title":"Docling Technical Report","author":"Auer","year":"2024"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-demo.15"},{"key":"ref20","article-title":"Library for efficient text classification and representation learning","author":"Mouselimis","year":"2022"},{"key":"ref21","article-title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","author":"Rae","year":"2021"},{"key":"ref22","volume-title":"Quackling","year":"2024"},{"key":"ref23","article-title":"Context aware, pluggable and customizable data protection and de-identification SDK for text and images"},{"key":"ref24","article-title":"Presidio"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N19-4010"},{"key":"ref26","article-title":"Starcoder: may the source be with you!","author":"Li","year":"2023"},{"key":"ref27","article-title":"ClamAV"},{"key":"ref28","article-title":"ScanCode"},{"key":"ref29","article-title":"Scaling Granite Code Models to 128K Context","author":"Stallone","year":"2024"}],"event":{"name":"2024 IEEE International Conference on Big Data (BigData)","location":"Washington, DC, USA","start":{"date-parts":[[2024,12,15]]},"end":{"date-parts":[[2024,12,18]]}},"container-title":["2024 IEEE International Conference on Big Data (BigData)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx8\/10824975\/10824942\/10825085.pdf?arnumber=10825085","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,17]],"date-time":"2025-01-17T08:20:45Z","timestamp":1737102045000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/10825085\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,12,15]]},"references-count":28,"URL":"https:\/\/doi.org\/10.1109\/bigdata62323.2024.10825085","relation":{},"subject":[],"published":{"date-parts":[[2024,12,15]]}}}