{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,13]],"date-time":"2026-06-13T17:53:26Z","timestamp":1781373206962,"version":"3.54.1"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T00:00:00Z","timestamp":1762560000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T00:00:00Z","timestamp":1762560000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000088","name":"Directorate for Social, Behavioral and Economic Sciences","doi-asserted-by":"publisher","award":["2219575"],"award-info":[{"award-number":["2219575"]}],"id":[{"id":"10.13039\/100000088","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Scientometrics"],"published-print":{"date-parts":[[2026,4]]},"DOI":"10.1007\/s11192-025-05445-5","type":"journal-article","created":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T08:34:56Z","timestamp":1762590896000},"page":"2303-2332","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":3,"title":["Using language models to label clusters of scientific documents"],"prefix":"10.1007","volume":"131","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7119-0169","authenticated-orcid":false,"given":"Dakota","family":"Murray","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chaoqun","family":"Ni","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiye","family":"Gu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Trevor","family":"Hubbard","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2025,11,8]]},"reference":[{"key":"5445_CR1","doi-asserted-by":"crossref","unstructured":"Allahyari, M., & Kochut, K. (2015). Automatic topic labeling using ontology-based topic models. In 2015 IEEE 14th international conference on machine learning and applications (ICMLA) (pp. 259\u2013264). IEEE.","DOI":"10.1109\/ICMLA.2015.88"},{"key":"5445_CR2","doi-asserted-by":"crossref","unstructured":"Beltagy, I., Lo, K., & Cohan, A. (2019). SciBERT: A pretrained nolanguage model for scientific text. In Proceedings of the 2019 conference on empirical methods in natural nolanguage processing and the 9th international joint conference on natural nolanguage processing (EMNLP-IJCNLP) (pp. 3615\u20133620). Hong Kong, China.","DOI":"10.18653\/v1\/D19-1371"},{"issue":"4","key":"5445_CR3","doi-asserted-by":"publisher","first-page":"1570","DOI":"10.1162\/qss_a_00085","volume":"1","author":"KW Boyack","year":"2020","unstructured":"Boyack, K. W., & Klavans, R. (2020). A comparison of large-scale science models based on textual, direct citation and hybrid relatedness. Quantitative Science Studies, 1(4), 1570\u20131585.","journal-title":"Quantitative Science Studies"},{"key":"5445_CR4","doi-asserted-by":"publisher","unstructured":"Carmel, D., Roitman, H., & Zwerdling, N. (2009). Enhancing cluster labeling using Wikipedia. In Proceedings of the 32nd international ACM SIGIR conference on research and development in information retrieval (pp. 139\u2013146). ACM. https:\/\/doi.org\/10.1145\/1571941.1571967","DOI":"10.1145\/1571941.1571967"},{"key":"5445_CR5","unstructured":"Chang, J., Boyd-Graber, J., Gerrish, S., Wang, C., & Blei, D. M. (2009). Reading tea leaves: How humans interpret topic models. In Proceedings of the 23rd international conference on neural information processing systems (pp. 288\u2013296)."},{"issue":"3","key":"5445_CR6","doi-asserted-by":"publisher","first-page":"359","DOI":"10.1002\/asi.20317","volume":"57","author":"C Chen","year":"2005","unstructured":"Chen, C. (2005). CiteSpace II: Detecting and visualizing emerging trends and transient patterns in scientific literature. Journal of the American Society for Information Science and Technology, 57(3), 359\u2013377.","journal-title":"Journal of the American Society for Information Science and Technology"},{"issue":"7","key":"5445_CR7","doi-asserted-by":"publisher","first-page":"1386","DOI":"10.1002\/asi.21309","volume":"61","author":"C Chen","year":"2010","unstructured":"Chen, C., Ibekwe-SanJuan, F., & Hou, J. (2010). The structure and dynamics of cocitation clusters: A multiple-perspective cocitation analysis. Journal of the American Society for Information Science and Technology, 61(7), 1386\u20131409.","journal-title":"Journal of the American Society for Information Science and Technology"},{"key":"5445_CR8","doi-asserted-by":"crossref","unstructured":"Cohan, A., Feldman, S., Beltagy, I., Downey, D., & Weld, D. (2020). SPECTER: Document-level representation learning using citation-informed transformers. In Proceedings of the 58th annual meeting of the association for computational linguistics (pp. 2270\u20132282).","DOI":"10.18653\/v1\/2020.acl-main.207"},{"issue":"30","key":"5445_CR9","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2305016120","volume":"120","author":"F Gilardi","year":"2023","unstructured":"Gilardi, F., Alizadeh, M., & Kubli, M. (2023). Chatgpt outperforms crowd workers for text-annotation tasks. Proceedings of the National Academy of Sciences, 120(30), e2305016120.","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"1","key":"5445_CR10","doi-asserted-by":"publisher","first-page":"387","DOI":"10.1162\/qss_a_00020","volume":"1","author":"C Herzog","year":"2020","unstructured":"Herzog, C., Hook, D., & Konkiel, S. (2020). Dimensions: Bringing down barriers between scientometricians and data. Quantitative Science Studies, 1(1), 387\u2013395.","journal-title":"Quantitative Science Studies"},{"key":"5445_CR11","doi-asserted-by":"publisher","DOI":"10.3389\/frma.2020.595299","volume":"5","author":"DW Hook","year":"2021","unstructured":"Hook, D. W., Porter, S. J., Draux, H., & Herzog, C. T. (2021). Real-time bibliometrics: Dimensions as a resource for analyzing aspects of COVID-19. Frontiers in Research Metrics and Analytics, 5, 595299.","journal-title":"Frontiers in Research Metrics and Analytics"},{"key":"5445_CR12","doi-asserted-by":"crossref","unstructured":"Hook, D. W., Porter, S. J., & Herzog, C. (2018). Dimensions: Building context for search and evaluation. Frontiers in Research Metrics and Analytics, 3, 23.","DOI":"10.3389\/frma.2018.00023"},{"key":"5445_CR13","doi-asserted-by":"crossref","unstructured":"Huang, F., Kwak, H., & An, J. (2023). Is ChatGPT better than human annotators? Potential and limitations of ChatGPT in explaining implicit hate speech. In Companion proceedings of the ACM web conference 2023 (pp. 294\u2013297). Association for Computing Machinery.","DOI":"10.1145\/3543873.3587368"},{"key":"5445_CR14","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3597610","volume":"43","author":"L Huang","year":"2024","unstructured":"Huang, L., Yu, W., Ma, W., Zhong, W., Feng, Z., Wang, H., Chen, Q., Peng, W., Feng, X., Qin, B., & Liu, T. (2024). A survey on hallucination in large nolanguage models: Principles, taxonomy, challenges, and open questions. ACM Transactions on Information Systems, 43, 1\u201355.","journal-title":"ACM Transactions on Information Systems"},{"key":"5445_CR15","unstructured":"Khandelwal, T. (2025). Using LLM-based approaches to enhance and automate topic labeling. ArXiv, https:\/\/arxiv.org\/abs\/2502.18469."},{"issue":"2","key":"5445_CR16","doi-asserted-by":"publisher","first-page":"1157","DOI":"10.1007\/s11192-017-2305-2","volume":"111","author":"R Koopman","year":"2017","unstructured":"Koopman, R., & Wang, S. (2017). Mutual information based labelling and comparing clusters. Scientometrics, 111(2), 1157\u20131167.","journal-title":"Scientometrics"},{"key":"5445_CR17","unstructured":"Kozlowski, D., Pradier, C., & Benz, P. (2024). Generative AI for automatic topic labelling. ArXiv. https:\/\/arxiv.org\/abs\/2408.07003."},{"key":"5445_CR18","unstructured":"Kuzman, T., Mozetic, I., & Ljube\u0161ic, N. (2023). ChatGPT: Beginning of an end of manual linguistic data annotation? Use case of automatic genre identification. ArXiv. https:\/\/arxiv.org\/abs\/2303.03953."},{"key":"5445_CR19","doi-asserted-by":"crossref","unstructured":"Lamers, W.S., Boyack, K., Larivi\u00e8re, V., Sugimoto, C.R., van Eck, N.J., Waltman, L., & Murray, D. (2021). Investigating disagreement in the scientific literature. eLife, 10:e72737.","DOI":"10.7554\/eLife.72737"},{"key":"5445_CR20","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1016\/j.knosys.2014.12.019","volume":"76","author":"Z Li","year":"2015","unstructured":"Li, Z., Li, J., Liao, Y., Wen, S., & Tang, J. (2015). Labeling clusters from both linguistic and statistical perspectives: A hybrid approach. Knowledge-Based Systems, 76, 219\u2013227.","journal-title":"Knowledge-Based Systems"},{"key":"5445_CR21","doi-asserted-by":"crossref","unstructured":"Liu, Y., Fabbri, A., Liu, P., Zhao, Y., Nan, L., Han, R., Joty, S., Wu, C. S., Xiong, C., & Radev, D. (2023). Revisiting the gold standard: Grounding summarization evaluation with robust human evaluation. In Proceedings of the 61st annual meeting of the association for computational linguistics (volume 1: Long papers) (pp. 4140\u20134170). Association for Computational Linguistics.","DOI":"10.18653\/v1\/2023.acl-long.228"},{"key":"5445_CR22","doi-asserted-by":"crossref","unstructured":"Liu, Y., Shi, K., He, K., Ye, L., Fabbri, A., Liu, P., Radev, D., & Cohan, A. (2024). On learning to summarize with large nolanguage models as references. In Proceedings of the 2024 conference of the North American chapter of the association for computational linguistics: Human nolanguage technologies (volume 1: Long papers) (pp. 8647\u20138664). Association for Computational Linguistics.","DOI":"10.18653\/v1\/2024.naacl-long.478"},{"key":"5445_CR23","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3717437","author":"A Marchetti","year":"2020","unstructured":"Marchetti, A., & Puranam, P. (2020). Interpreting topic models using prototypical text: From \u2018telling\u2019 to \u2018showing\u2019. SSRN Electronic Journal. https:\/\/doi.org\/10.2139\/ssrn.3717437","journal-title":"SSRN Electronic Journal"},{"issue":"29","key":"5445_CR24","doi-asserted-by":"publisher","first-page":"861","DOI":"10.21105\/joss.00861","volume":"3","author":"L McInnes","year":"2018","unstructured":"McInnes, L., Healy, J., Saul, N., & Grossberger, L. (2018). UMAP: Uniform manifold approximation and projection. The Journal of Open Source Software, 3(29), 861.","journal-title":"The Journal of Open Source Software"},{"key":"5445_CR25","doi-asserted-by":"publisher","unstructured":"Mei, Q., Shen, X., & Zhai, C. (2007). Automatic labeling of multinomial topic models. In Proceedings of the 13th ACM SIGKDD international conference on knowledge discovery and data mining (pp. 490\u2013499). ACM. https:\/\/doi.org\/10.1145\/1281192.1281246","DOI":"10.1145\/1281192.1281246"},{"key":"5445_CR26","doi-asserted-by":"crossref","unstructured":"Moratanch, N., & Chitrakala, S. (2017). A survey on extractive text summarization. In 2017 international conference on computer, communication and signal processing (ICCCSP) (pp. 1\u20136). IEEE.","DOI":"10.1109\/ICCCSP.2017.7944061"},{"key":"5445_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3697010","volume":"34","author":"S Ouyang","year":"2024","unstructured":"Ouyang, S., Zhang, J. M., Harman, M., & Wang, M. (2024). An empirical study of the non-determinism of ChatGPT in code generation. ACM Transactions on Software Engineering and Methodology, 34, 1\u201328.","journal-title":"ACM Transactions on Software Engineering and Methodology"},{"key":"5445_CR28","doi-asserted-by":"crossref","unstructured":"Petroni, F., Rockt\u00e4schel, T., Riedel, S., Lewis, P. S. H., Bakhtin, A., Wu, Y., & Miller, A.H. (2019). Nolanguage models as knowledge bases? In EMNLP\/IJCNLP (1) (pp. 2463\u20132473). Association for Computational Linguistics.","DOI":"10.18653\/v1\/D19-1250"},{"issue":"1","key":"5445_CR29","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1162\/qss_a_00244","volume":"4","author":"SJ Porter","year":"2023","unstructured":"Porter, S. J., Hawizy, L., & Hook, D. W. (2023). Recategorising research: Mapping from for 2008 to for 2020 in dimensions. Quantitative Science Studies, 4(1), 127\u2013143.","journal-title":"Quantitative Science Studies"},{"key":"5445_CR30","unstructured":"Pu, X., Gao, M., & Wan, X. (2023). Summarization is (almost) dead. arXiv. https:\/\/arxiv.org\/abs\/2309.09558"},{"key":"5445_CR31","doi-asserted-by":"publisher","unstructured":"Reddivari, S. (2019). Enhancing software requirements cluster labeling using Wikipedia. In 2019 IEEE 20th international conference on information reuse and integration for data science (IRI) (pp. 123\u2013126). IEEE. https:\/\/doi.org\/10.1109\/IRI.2019.00031","DOI":"10.1109\/IRI.2019.00031"},{"key":"5445_CR32","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1016\/j.knosys.2013.11.005","volume":"56","author":"F Role","year":"2014","unstructured":"Role, F., & Nadif, M. (2014). Beyond cluster labeling: Semantic interpretation of clusters\u2019 contents using a graph representation. Knowledge-Based Systems, 56, 141\u2013155. https:\/\/doi.org\/10.1016\/j.knosys.2013.11.005","journal-title":"Knowledge-Based Systems"},{"issue":"7987","key":"5445_CR33","doi-asserted-by":"publisher","first-page":"493","DOI":"10.1038\/s41586-023-06647-8","volume":"623","author":"M Shanahan","year":"2023","unstructured":"Shanahan, M., McDonell, K., & Reynolds, L. (2023). Role play with large nolanguage models. Nature, 623(7987), 493\u2013498.","journal-title":"Nature"},{"key":"5445_CR34","doi-asserted-by":"crossref","unstructured":"Singhal K, Azizi S, Tu T, Mahdavi SS, Wei J, Chung HW, Scales N, Tanwani A, Cole-Lewis H, Pfohl S, Payne P, Seneviratne M, Gamble P, Kelly C, Babiker A, Sch\u00e4rli N, Chowdhery A, Mansfield P, Demner-Fushman D, Ag\u00fcera Y Arcas B, Webster D, Corrado GS, Matias Y, Chou K, Gottweis J, Tomasev N, Liu Y, Rajkomar A, Barral J, Semturs C, Karthikesalingam A, Natarajan V. (2023). Large language models encode clinical knowledge. Nature, 620(7972), 172\u2013180.","DOI":"10.1038\/s41586-023-06291-2"},{"issue":"7","key":"5445_CR35","doi-asserted-by":"publisher","first-page":"853","DOI":"10.1002\/asi.24452","volume":"72","author":"P Sj\u00f6g\u00e5rde","year":"2021","unstructured":"Sj\u00f6g\u00e5rde, P., Ahlgren, P., & Waltman, L. (2021). Algorithmic labeling in hierarchical classifications of publications: Evaluation of bibliographic fields and term weighting approaches. Journal of the Association for Information Science and Technology, 72(7), 853\u2013869.","journal-title":"Journal of the Association for Information Science and Technology"},{"key":"5445_CR36","doi-asserted-by":"crossref","unstructured":"\u0160ubelj, L., van Eck, N. J., & Waltman, L. (2016). Clustering scientific publications based on citation relations: A systematic comparison of different methods. PLoS ONE, 11(4), e0154404.","DOI":"10.1371\/journal.pone.0154404"},{"issue":"10","key":"5445_CR37","doi-asserted-by":"publisher","first-page":"2464","DOI":"10.1002\/asi.23596","volume":"67","author":"A Suominen","year":"2015","unstructured":"Suominen, A., & Toivanen, H. (2015). Map of science with topic modeling: Comparison of unsupervised learning and human-assigned subject classification. Journal of the Association for Information Science and Technology, 67(10), 2464\u20132476. https:\/\/doi.org\/10.1002\/asi.23596","journal-title":"Journal of the Association for Information Science and Technology"},{"key":"5445_CR38","unstructured":"T\u00f6rnberg, P. (2023). ChatGPT-4 outperforms experts and crowd workers in annotating political twitter messages with zero-shot learning. ArXiv. https:\/\/arxiv.org\/abs\/2304.06588."},{"key":"5445_CR39","doi-asserted-by":"crossref","unstructured":"Van Eck, N. J., & Waltman, L. (2024). An open approach for classifying research publications. Front Matter.","DOI":"10.59350\/gygeh-nv475"},{"key":"5445_CR40","unstructured":"Vahidnia, S., Abbasi, A., & Abbass, H.A. (2020). Document clustering and labeling for research trend extraction and evolution mapping. In Eeke@jcdl 2020 (Vol. 2658, pp. 54\u201362). CEUR-WS.org."},{"issue":"2","key":"5445_CR41","doi-asserted-by":"publisher","first-page":"1033","DOI":"10.1007\/s11192-017-2299-9","volume":"111","author":"T Velden","year":"2017","unstructured":"Velden, T., Yan, S., & Lagoze, C. (2017). Mapping the cognitive structure of astrophysics by infomap clustering of the citation network and topic affinity analysis. Scientometrics, 111(2), 1033\u20131051.","journal-title":"Scientometrics"},{"key":"5445_CR42","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1162\/tacl_a_00632","volume":"12","author":"T Zhang","year":"2024","unstructured":"Zhang, T., Ladhak, F., Durmus, E., Liang, P., McKeown, K., & Hashimoto, T. B. (2024). Benchmarking large nolanguage models for news summarization. Transactions of the Association for Computational Linguistics, 12, 39\u201357.","journal-title":"Transactions of the Association for Computational Linguistics"},{"issue":"1","key":"5445_CR43","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1162\/coli_a_00502","volume":"50","author":"C Ziems","year":"2024","unstructured":"Ziems, C., Held, W., Shaikh, O., Chen, J., Zhang, Z., & Yang, D. (2024). Can large nolanguage models transform computational social science? Computational Linguistics, 50(1), 237\u2013291.","journal-title":"Computational Linguistics"}],"container-title":["Scientometrics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-025-05445-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s11192-025-05445-5","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-025-05445-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,25]],"date-time":"2026-04-25T06:57:01Z","timestamp":1777100221000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s11192-025-05445-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,8]]},"references-count":43,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2026,4]]}},"alternative-id":["5445"],"URL":"https:\/\/doi.org\/10.1007\/s11192-025-05445-5","relation":{},"ISSN":["0138-9130","1588-2861"],"issn-type":[{"value":"0138-9130","type":"print"},{"value":"1588-2861","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11,8]]},"assertion":[{"value":"24 April 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 October 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 November 2025","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"T.H. is a current employee and D.M. a former employee of\n                      Digital Science\n                      , the company that produced the bibliographic database Dimensions. The remaining authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}