{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,25]],"date-time":"2026-10-25T05:34:39Z","timestamp":1792906479967,"version":"4.5.0"},"reference-count":98,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"DOI":"10.13039\/501100008530","name":"European Regional Development Fund","doi-asserted-by":"publisher","award":["KK.01.1.1.01.0009 (DATACROSS)"],"award-info":[{"award-number":["KK.01.1.1.01.0009 (DATACROSS)"]}],"id":[{"id":"10.13039\/501100008530","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2021]]},"DOI":"10.1109\/access.2021.3109425","type":"journal-article","created":{"date-parts":[[2021,8,31]],"date-time":"2021-08-31T16:05:56Z","timestamp":1630425956000},"page":"123280-123312","source":"Crossref","is-referenced-by-count":23,"title":["A Topic Coverage Approach to Evaluation of Topic Models"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4645-2937","authenticated-orcid":false,"given":"Damir","family":"Korencic","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6039-0838","authenticated-orcid":false,"given":"Strahil","family":"Ristov","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jelena","family":"Repar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8942-5301","authenticated-orcid":false,"given":"Jan","family":"Snajder","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","first-page":"993","article-title":"Latent Dirichlet allocation","volume":"3","author":"Blei","year":"2003","journal-title":"J. Mach. Learn. Res."},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207738"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148204"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.3115\/1621474.1621534"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0087908"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553515"},{"key":"ref7","first-page":"288","article-title":"Reading tea leaves: How humans interpret topic models","volume-title":"Proc. 22nd Int. Conf. Neural Inf. Process. Syst.","author":"Chang"},{"key":"ref8","first-page":"79","article-title":"Evaluating topic models with stability","volume-title":"Proc. 19th Annu. Symp. Pattern Recognit. Assoc. South Afr.","author":"De Waal"},{"key":"ref9","first-page":"100","article-title":"Automatic evaluation of topic coherence","volume-title":"Proc. Hum. Lang. Technol., Annu. Conf. North Amer. Chapter Assoc. Comput. Linguistics (ACL)","author":"Newman"},{"key":"ref10","first-page":"612","article-title":"Topic model diagnostics: Assessing domain relevance via topical alignment","volume-title":"Proc. 30th Int. Conf. Mach. Learn. (ICML)","author":"Chuang"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685324"},{"key":"ref12","article-title":"Inferring concepts from topics: Towards procedures for validating topics as measures","volume-title":"Proc. 36th PolMeth","volume":"5","author":"Ying"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2017.08.047"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1038\/44565"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1016\/j.poetic.2013.06.003"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1080\/21670811.2015.1093271"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1111\/j.1540-5907.2009.00427.x"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1093\/pan\/mpp034"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2005.04.034"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkw964"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/FOCS.2012.49"},{"key":"ref22","first-page":"3824","article-title":"Topic model or topic twaddle? Re-evaluating semantic interpretability measures","volume-title":"Proc. Conf. North Amer. Chapter Assoc. Comput. Linguistics, Hum. Lang. Technol.","author":"Doogan"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1145\/2615569.2615680"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/N15-1018"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2809936.2809942"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/2133806.2133826"},{"key":"ref27","first-page":"3881","article-title":"Conic scan-and-cover algorithms for nonparametric topic modeling","volume-title":"Proc. 31st Int. Conf. Neural Inf. Process. Syst. (NIPS)","author":"Yurochkin"},{"key":"ref28","article-title":"Topic modelling meets deep neural networks: A survey","author":"Zhao","year":"2021","journal-title":"arXiv:2103.00498"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1093\/pan\/mps028"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.0307752101"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1016\/j.poetic.2013.08.004"},{"key":"ref32","volume-title":"Topic Modeling for Media and Communication Research: A Short Primer","author":"Puschmann","year":"2016"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1080\/19312458.2018.1430754"},{"key":"ref34","first-page":"1973","article-title":"Rethinking LDA: Why priors matter","volume-title":"Proc. Adv. Neural Inf. Process. Syst.","author":"Wallach"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/2700497"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623691"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1198\/016214506000000302"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1214\/aop\/1024404422"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/2567948.2577379"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2013.212"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/2786451.2786464"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2015.02.055"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-44848-9_32"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-47650-6_6"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/2907070"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1007\/s13748-016-0094-0"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.4135\/9781071878781"},{"key":"ref48","volume-title":"Machine Learning: A Probabilistic Perspective","author":"Murphy","year":"2012"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1007\/BF00994018"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.29172\/7c2a6982-6d72-4cd8-bba6-2fccb06a7011"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-45167-9_6"},{"key":"ref52","first-page":"2079","article-title":"On over-fitting in model selection and subsequent selection bias in performance evaluation","volume":"11","author":"Cawley","year":"2010","journal-title":"J. Mach. Learn. Res."},{"key":"ref53","first-page":"519","article-title":"AUC: A statistically consistent and more discriminating measure than accuracy","volume-title":"Proc. IJCAI","volume":"3","author":"Ling"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-20161-5_34"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781316257340.004"},{"key":"ref56","volume-title":"Introduction to Data Mining","author":"Tan","year":"2005"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2783264"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2018.07.063"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1080\/1369118X.2016.1139615"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1086\/267990"},{"key":"ref61","volume-title":"Policy Diffusion: The Issue-Definition Stage","author":"Gilardi","year":"2018"},{"key":"ref62","first-page":"96","article-title":"Topic modeling on historical newspapers","volume-title":"Proc. 5th ACL-HLT Workshop Lang. Technol. Cultural Heritage, Social Sci., Humanities (LaTeCH)","author":"Yang"},{"key":"ref63","first-page":"13","article-title":"Evaluating topic coherence using distributional semantics","volume-title":"Proc. 10th Int. Conf. Comput. Semantics (IWCS)","author":"Aletras"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1093\/analys\/63.3.194"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.4324\/9780203936399.ch21"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1002\/nav.3800020109"},{"key":"ref67","first-page":"1776","article-title":"Short text classification improved by learning multi-granularity topics","volume-title":"Proc. 22nd Int. Joint Conf. Artif. Intell.","author":"Chen"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646003"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199009)41:6<391::AID-ASI1>3.0.CO;2-9"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2012.51"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-99365-2_23"},{"key":"ref72","first-page":"227","article-title":"Bayesian checking for topic models","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Mimno"},{"key":"ref73","first-page":"262","article-title":"Optimizing semantic coherence in topic models","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Mimno"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/E14-1056"},{"key":"ref75","first-page":"1","article-title":"Evaluating topic coherence measures","volume":"abs\/1403.6397","author":"Rosner","year":"2014","journal-title":"CoRR"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1177\/0165551515617393"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914720"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/E17-2070"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-04180-8_22"},{"key":"ref80","first-page":"1866","article-title":"Improving topic evaluation using conceptual knowledge","volume-title":"Proc. 22nd Int. Joint Conf. Artif. Intell.","volume":"3","author":"Musat"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2020.2985079"},{"key":"ref82","first-page":"1024","article-title":"Probabalistic walks in semantic hierarchies as a topic model for WSD","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"Boyd-Graber"},{"key":"ref83","first-page":"308","article-title":"A joint model of text and aspect ratings for sentiment summarization","volume-title":"Proc. ACL","author":"Titov"},{"key":"ref84","first-page":"952","article-title":"Exploring topic coherence over many models and many topics","volume-title":"Proc. Joint Conf. Empirical Methods Natural Lang. Process. Comput. Natural Lang. Learn.","author":"Stevens"},{"key":"ref85","first-page":"1","article-title":"Computer-assisted content analysis: Topic models for exploring multiple subjective interpretations","volume-title":"Proc. Adv. Neural Inf. Process. Syst. Workshop Hum.-Propelled Mach. Learn.","author":"Chuang"},{"key":"ref86","article-title":"A new evaluation framework for topic modeling algorithms based on synthetic corpora","author":"Shi","year":"2019","journal-title":"arXiv:1901.09848"},{"key":"ref87","first-page":"2377","article-title":"Topic identification and discovery on text and speech","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process.","author":"May"},{"key":"ref88","first-page":"211","article-title":"Topic modelling in social sciences: Case study of web of science","volume-title":"Proc. 31st Central Eur. Conf. Inf. Intell. Syst.","author":"Pandur"},{"key":"ref89","article-title":"Active learning literature survey","author":"Settles","year":"2009"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1007\/s10115-017-1147-9"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2020.113709"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2019.2944973"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.3115\/1699510.1699543"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-23780-5_29"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.2307\/2347565"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1162\/neco.2007.19.10.2756"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/j.patcog.2007.09.010"},{"key":"ref98","article-title":"API design for machine learning software: Experiences from the scikit-learn project","author":"Buitinck","year":"2013","journal-title":"arXiv:1309.0238"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9312710\/09526605.pdf?arnumber=9526605","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,10,25]],"date-time":"2026-10-25T04:36:31Z","timestamp":1792902991000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9526605\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":98,"URL":"https:\/\/doi.org\/10.1109\/access.2021.3109425","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]}}}