{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T13:34:45Z","timestamp":1781876085354,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,15]],"date-time":"2024-04-15T00:00:00Z","timestamp":1713139200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,15]]},"DOI":"10.1145\/3643991.3644898","type":"proceedings-article","created":{"date-parts":[[2024,8,6]],"date-time":"2024-08-06T21:19:25Z","timestamp":1722979165000},"page":"607-618","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":23,"title":["Analyzing the Evolution and Maintenance of ML Models on Hugging Face"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-6385-9255","authenticated-orcid":false,"given":"Joel","family":"Casta\u00f1o","sequence":"first","affiliation":[{"name":"Universitat Polit\u00e8cnica de Catalunya, Barcelona, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9928-133X","authenticated-orcid":false,"given":"Silverio","family":"Mart\u00ednez-Fern\u00e1ndez","sequence":"additional","affiliation":[{"name":"Universitat Polit\u00e8cnica de Catalunya, Barcelona, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9733-8830","authenticated-orcid":false,"given":"Xavier","family":"Franch","sequence":"additional","affiliation":[{"name":"Universitat Polit\u00e8cnica de Catalunya, Barcelona, Spain"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5788-0991","authenticated-orcid":false,"given":"Justus","family":"Bogner","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,2]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Hugging Face Inc. \"Hugging Face Hub Documentation \" https:\/\/huggingface.co\/docs\/hub\/index 2023."},{"key":"e_1_3_2_1_2_1","first-page":"2463","volume-title":"Australia: IEEE","author":"Jiang W.","year":"2023","unstructured":"W. Jiang, N. Synovic, M. Hyatt, T. R. Schorlemmer, R. Sethi, Y.-H. Lu, G. K. Thiruvathukal, and J. C. Davis, \"An Empirical Study of Pre-Trained Model Reuse in the Hugging Face Deep Learning Model Registry,\" in 2023 IEEE\/ACM 45th International Conference on Software Engineering (ICSE). Melbourne, Australia: IEEE, May 2023, pp. 2463--2475. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/10172757\/"},{"key":"e_1_3_2_1_3_1","volume-title":"Exploring the Carbon Footprint of Hugging Face's ML Models: A Repository Mining Study,\" in ACM \/ IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","author":"Casta\u00f1o J.","year":"2023","unstructured":"J. Casta\u00f1o, S. Mart\u00ednez-Fern\u00e1ndez, X. Franch, and J. Bogner, \"Exploring the Carbon Footprint of Hugging Face's ML Models: A Repository Mining Study,\" in ACM \/ IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM). New Orleans, LA, USA: IEEE, 2023."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569934"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/s42979-021-00592-x"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3487043"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2876857"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40537-020-00312-x"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3533378"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2023.111860"},{"key":"e_1_3_2_1_11_1","volume-title":"Std.","author":"IEC","year":"2011","unstructured":"ISO\/IEC 25010, ISO\/IEC 25010:2011, Systems and software engineering --- Systems and software Quality Requirements and Evaluation (SQuaRE) --- System and software quality models, Std., 2011."},{"key":"e_1_3_2_1_12_1","first-page":"541","article-title":"Defining systems evolvability-a taxonomy of change","volume":"94","author":"Rowe D.","year":"1994","unstructured":"D. Rowe, J. Leaney, and D. Lowe, \"Defining systems evolvability-a taxonomy of change,\" Change, vol. 94, pp. 541--545, 1994.","journal-title":"Change"},{"key":"e_1_3_2_1_13_1","first-page":"291","volume-title":"IEEE","author":"Amershi S.","year":"2019","unstructured":"S. Amershi, A. Begel, C. Bird, R. DeLine, H. Gall, E. Kamar, N. Nagappan, B. Nushi, and T. Zimmermann, \"Software Engineering for Machine Learning: A Case Study,\" in 2019 IEEE\/ACM 41st International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP). IEEE, May 2019, pp. 291--300. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/8804457\/"},{"key":"e_1_3_2_1_14_1","first-page":"73","article-title":"Software maintenance and evolution: a roadmap","author":"Bennett K. H.","year":"2000","unstructured":"K. H. Bennett and V. T. Rajlich, \"Software maintenance and evolution: a roadmap,\" in Proceedings of the Conference on the Future of Software Engineering, 2000, pp. 73--87.","journal-title":"Proceedings of the Conference on the Future of Software Engineering"},{"key":"e_1_3_2_1_15_1","first-page":"20","volume-title":"IEEE","author":"Lehman M. M.","year":"1997","unstructured":"M. M. Lehman, J. F. Ramil, P. D. Wernick, D. E. Perry, and W. M. Turski, \"Metrics and laws of software evolution-the nineties view,\" in Proceedings Fourth International Software Metrics Symposium. IEEE, 1997, pp. 20--32."},{"key":"e_1_3_2_1_16_1","first-page":"51","article-title":"Hugging Face,\" in Introduction to Transformers for NLP. Berkeley","author":"Jain S. M.","year":"2022","unstructured":"S. M. Jain, \"Hugging Face,\" in Introduction to Transformers for NLP. Berkeley, CA: Apress, 2022, pp. 51--67. [Online]. Available: https:\/\/link.springer.com\/10.1007\/978-1-4842-8844-3_4","journal-title":"CA: Apress"},{"key":"e_1_3_2_1_17_1","volume-title":"Funtowicz et al., \"Huggingface's transformers: State-of-the-art natural language processing,\" arXiv preprint arXiv:1910.03771","author":"Wolf T.","year":"2019","unstructured":"T. Wolf, L. Debut, V. Sanh, J. Chaumond, C. Delangue, A. Moi, P. Cistac, T. Rault, R. Louf, M. Funtowicz et al., \"Huggingface's transformers: State-of-the-art natural language processing,\" arXiv preprint arXiv:1910.03771, 2019."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_19_1","first-page":"60","volume-title":"Spain: IEEE","author":"Shivashankar K.","year":"2022","unstructured":"K. Shivashankar and A. Martini, \"Maintainability Challenges in ML: A Systematic Literature Review,\" in 2022 48th Euromicro Conference on Software Engineering and Advanced Applications (SEAA). Gran Canaria, Spain: IEEE, Aug. 2022, pp. 60--67. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/10011474\/"},{"key":"e_1_3_2_1_20_1","first-page":"64","volume-title":"IEEE","author":"Bogner J.","year":"2021","unstructured":"J. Bogner, R. Verdecchia, and I. Gerostathopoulos, \"Characterizing Technical Debt and Antipatterns in AI-Based Systems: A Systematic Mapping Study,\" in 2021 IEEE\/ACM International Conference on Technical Debt (TechDebt). IEEE, May 2021, pp. 64--73, arXiv: 2103.09783. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/9463054\/"},{"key":"e_1_3_2_1_21_1","first-page":"238","volume-title":"IEEE","author":"Tang Y.","year":"2021","unstructured":"Y. Tang, R. Khatchadourian, M. Bagherzadeh, R. Singh, A. Stewart, and A. Raja, \"An Empirical Study of Refactorings and Technical Debt in Machine Learning Systems,\" in 2021 IEEE\/ACM 43rd International Conference on Software Engineering (ICSE). IEEE, May 2021, pp. 238--250. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/9401990\/"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3453478"},{"key":"e_1_3_2_1_23_1","first-page":"106","volume-title":"Italy: IEEE","author":"Leest J.","year":"2023","unstructured":"J. Leest, I. Gerostathopoulos, and C. Raibulet, \"Evolvability of Machine Learning-based Systems: An Architectural Design Decision Framework,\" in 2023 IEEE 20th International Conference on Software Architecture Companion (ICSA-C). L'Aquila, Italy: IEEE, Mar. 2023, pp. 106--110. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/10092638\/"},{"key":"e_1_3_2_1_24_1","volume-title":"Assessing the Vulnerabilities of the Open-Source Artificial Intelligence (AI) Landscape: A Large-Scale Analysis of the Hugging Face Platform,\" in IEEE Intelligence and Security Informatics","author":"Kathikar A.","year":"2023","unstructured":"A. Kathikar, A. Nair, B. Lazarine, A. Sachdeva, and S. Samtani, \"Assessing the Vulnerabilities of the Open-Source Artificial Intelligence (AI) Landscape: A Large-Scale Analysis of the Hugging Face Platform,\" in IEEE Intelligence and Security Informatics. Charlotte, NC, USA: IEEE, Oct. 2023."},{"key":"e_1_3_2_1_25_1","first-page":"728","volume-title":"Macao: IEEE","author":"Ait A.","year":"2023","unstructured":"A. Ait, J. L. C. Izquierdo, and J. Cabot, \"HFCommunity: A Tool to Analyze the Hugging Face Hub Community,\" in 2023 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER). Taipa, Macao: IEEE, Mar. 2023, pp. 728--732. [Online]. Available: https:\/\/ieeexplore.ieee.org\/document\/10123660\/"},{"key":"e_1_3_2_1_26_1","first-page":"528","volume-title":"The goal question metric approach,\" Encyclopedia of software engineering","author":"Caldiera V. R. B. G.","year":"1994","unstructured":"V. R. B. G. Caldiera and H. D. Rombach, \"The goal question metric approach,\" Encyclopedia of software engineering, pp. 528--532, 1994."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"A. Anonymous \"Replication Package for 'What is the Evolution and Maintenance of Pre-Trained ML models on Hugging Face?' \" Nov. 2023. [Online]. 10.5281\/zenodo.10153155","DOI":"10.5281\/zenodo.10153155"},{"key":"e_1_3_2_1_28_1","unstructured":"\"HfApi Client \" https:\/\/huggingface.co\/docs\/huggingface_hub\/package_reference\/hf_api Accessed: 01-02-2024."},{"key":"e_1_3_2_1_29_1","unstructured":"E. B. Swanson \"The dimensions of maintenance \" in Proceedings of the 2nd international conference on Software engineering 1976 pp. 492--497."},{"key":"e_1_3_2_1_30_1","first-page":"37","volume-title":"IEEE","author":"Sarwar M. U.","year":"2020","unstructured":"M. U. Sarwar, S. Zafar, M. W. Mkaouer, G. S. Walia, and M. Z. Malik, \"Multi-label classification of commit messages using transfer learning,\" in 2020 IEEE International Symposium on Software Reliability Engineering Workshops (ISSREW). IEEE, 2020, pp. 37--42."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1088\/1742-5468\/2008\/10\/P10008"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/944919.944937"},{"key":"e_1_3_2_1_33_1","first-page":"399","volume-title":"Exploring the space of topic coherence measures,\" in Proceedings of the eighth ACM international conference on Web search and data mining","author":"R\u00f6der M.","year":"2015","unstructured":"M. R\u00f6der, A. Both, and A. Hinneburg, \"Exploring the space of topic coherence measures,\" in Proceedings of the eighth ACM international conference on Web search and data mining, 2015, pp. 399--408."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106274"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3068335"},{"key":"e_1_3_2_1_36_1","first-page":"1","article-title":"Mann-whitney u test","author":"McKnight P. E.","year":"2010","unstructured":"P. E. McKnight and J. Najab, \"Mann-whitney u test,\" The Corsini encyclopedia of psychology, pp. 1--1, 2010.","journal-title":"The Corsini encyclopedia of psychology"},{"key":"e_1_3_2_1_37_1","first-page":"38","volume-title":"Funtowicz et al., \"Transformers: State-of-the-art natural language processing,\" in Proceedings of the 2020 conference on empirical methods in natural language processing: system demonstrations","author":"Wolf T.","year":"2020","unstructured":"T. Wolf, L. Debut, V. Sanh, J. Chaumond, C. Delangue, A. Moi, P. Cistac, T. Rault, R. Louf, M. Funtowicz et al., \"Transformers: State-of-the-art natural language processing,\" in Proceedings of the 2020 conference on empirical methods in natural language processing: system demonstrations, 2020, pp. 38--45."},{"key":"e_1_3_2_1_38_1","volume-title":"The state of documentation practices of third-party machine learning models and datasets,\" arXiv preprint arXiv:2312.15058","author":"Oreamuno E. L.","year":"2023","unstructured":"E. L. Oreamuno, R. F. Khan, A. A. Bangash, C. Stinson, and B. Adams, \"The state of documentation practices of third-party machine learning models and datasets,\" arXiv preprint arXiv:2312.15058, 2023."},{"key":"e_1_3_2_1_39_1","first-page":"1","article-title":"Aspirations and practice of ml model documentation: Moving the needle with nudging and traceability,\" in Proceedings of the 2023 CHI Conference on Human Factors","author":"Bhat A.","year":"2023","unstructured":"A. Bhat, A. Coursey, G. Hu, S. Li, N. Nahar, S. Zhou, C. K\u00e4stner, and J. L. Guo, \"Aspirations and practice of ml model documentation: Moving the needle with nudging and traceability,\" in Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems, 2023, pp. 1--17.","journal-title":"Computing Systems"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1137983.1138001"},{"key":"e_1_3_2_1_41_1","volume-title":"Software engineering: a practitioner's approach. Palgrave macmillan","author":"Pressman R. S.","year":"2005","unstructured":"R. S. Pressman, Software engineering: a practitioner's approach. Palgrave macmillan, 2005."},{"key":"e_1_3_2_1_42_1","unstructured":"\"Data Version Control \u00b7 DVC \" https:\/\/dvc.org\/ Accessed: 01-02-2024."},{"key":"e_1_3_2_1_43_1","unstructured":"\"DagsHub: The Home for Machine Learning Collaboration \" https:\/\/dagshub.com\/ Accessed: 01-02-2024."},{"key":"e_1_3_2_1_44_1","volume-title":"Training future ml engineers: a project-based course on mlops,\" IEEE software","author":"Lanubile F.","year":"2023","unstructured":"F. Lanubile, S. Mart\u00ednez-Fern\u00e1ndez, and L. Quaranta, \"Training future ml engineers: a project-based course on mlops,\" IEEE software, 2023."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3379597.3387448"}],"event":{"name":"MSR '24: 21st International Conference on Mining Software Repositories","location":"Lisbon Portugal","acronym":"MSR '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS"]},"container-title":["Proceedings of the 21st International Conference on Mining Software Repositories"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643991.3644898","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3643991.3644898","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:44Z","timestamp":1750291004000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643991.3644898"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,15]]},"references-count":45,"alternative-id":["10.1145\/3643991.3644898","10.1145\/3643991"],"URL":"https:\/\/doi.org\/10.1145\/3643991.3644898","relation":{},"subject":[],"published":{"date-parts":[[2024,4,15]]},"assertion":[{"value":"2024-07-02","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}