{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,11]],"date-time":"2026-02-11T05:50:09Z","timestamp":1770789009428,"version":"3.50.0"},"reference-count":59,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2024,1,22]],"date-time":"2024-01-22T00:00:00Z","timestamp":1705881600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,22]],"date-time":"2024-01-22T00:00:00Z","timestamp":1705881600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Cogn Comput"],"published-print":{"date-parts":[[2024,5]]},"DOI":"10.1007\/s12559-023-10235-3","type":"journal-article","created":{"date-parts":[[2024,1,22]],"date-time":"2024-01-22T09:02:18Z","timestamp":1705914138000},"page":"949-963","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["A Fistful of Vectors: A Tool for Intrinsic Evaluation of Word Embeddings"],"prefix":"10.1007","volume":"16","author":[{"given":"Roberto","family":"Ascari","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9419-4800","authenticated-orcid":false,"given":"Anna","family":"Giabelli","sequence":"additional","affiliation":[]},{"given":"Lorenzo","family":"Malandri","sequence":"additional","affiliation":[]},{"given":"Fabio","family":"Mercorio","sequence":"additional","affiliation":[]},{"given":"Mario","family":"Mezzanzanica","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,1,22]]},"reference":[{"key":"10235_CR1","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: NeurIPS (2013)"},{"key":"10235_CR2","doi-asserted-by":"crossref","unstructured":"Wang, B., Wang, A., Chen, F., Wang, Y., Kuo, C.-C.J.: Evaluating word embedding models: methods and experimental results. APSIPA transactions on signal and information processing 8 (2019)","DOI":"10.1017\/ATSIP.2019.12"},{"key":"10235_CR3","doi-asserted-by":"crossref","unstructured":"Schnabel, T., Labutov, I., Mimno, D., Joachims, T.: Evaluation methods for unsupervised word embeddings. In: EMNLP (2015)","DOI":"10.18653\/v1\/D15-1036"},{"key":"10235_CR4","doi-asserted-by":"crossref","unstructured":"Camacho-Collados, J., Pilehvar, M.T.: From word to sense embeddings: a survey on vector representations of meaning. Journal of Artificial Intelligence Research 63, 743\u2013788 (2018)","DOI":"10.1613\/jair.1.11259"},{"key":"10235_CR5","doi-asserted-by":"crossref","unstructured":"Baroni, M., Dinu, G., Kruszewski, G.: Don\u2019t count, predict! a systematic comparison of context-counting vs. context-predicting semantic vectors. In: ACL (2014)","DOI":"10.3115\/v1\/P14-1023"},{"key":"10235_CR6","unstructured":"Bakarov, A.: A survey of word embeddings evaluation methods (2018)"},{"key":"10235_CR7","doi-asserted-by":"crossref","unstructured":"Giabelli, A., Malandri, L., Mercorio, F., Mezzanzanica, M., Nobani, N.: Embeddings evaluation using a novel measure of semantic similarity. Cognitive Computation, 1\u201315 (2022)","DOI":"10.1007\/s12559-022-10067-7"},{"key":"10235_CR8","doi-asserted-by":"crossref","unstructured":"Giabelli, A., Malandri, L., Mercorio, F., Mezzanzanica, M., Seveso, A.: Neo: A tool for taxonomy enrichment with new emerging occupations. In: The Semantic Web\u2013ISWC 2020: 19th International Semantic Web Conference, Athens, Greece, November 2\u20136, 2020, Proceedings, Part II 19, pp. 568\u2013584 (2020). Springer","DOI":"10.1007\/978-3-030-62466-8_35"},{"key":"10235_CR9","doi-asserted-by":"crossref","unstructured":"Gladkova, A., Drozd, A.: Intrinsic evaluations of word embeddings: what can we do better? In: Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for NLP, pp. 36\u201342 (2016)","DOI":"10.18653\/v1\/W16-2507"},{"key":"10235_CR10","doi-asserted-by":"crossref","unstructured":"Levy, O., Goldberg, Y., Dagan, I.: Improving distributional similarity with lessons learned from word embeddings. TACL 3 (2015)","DOI":"10.1162\/tacl_a_00134"},{"key":"10235_CR11","doi-asserted-by":"crossref","unstructured":"Caselles-Dupr\u00e9, H., Lesaint, F., Royo-Letelier, J.: Word2vec applied to recommendation: Hyperparameters matter. In: RECSYS (2018)","DOI":"10.1145\/3240323.3240377"},{"key":"10235_CR12","unstructured":"Torregrossa, F., Claveau, V., Kooli, N., Gravier, G., Allesiardo, R.: On the correlation of word embedding evaluation metrics. In: Proceedings of the 12th Conference on Language Resources and Evaluation (LREC 2020), pp. 4789\u20134797 (2020)"},{"key":"10235_CR13","doi-asserted-by":"crossref","unstructured":"Torregrossa, F., Allesiardo, R., Claveau, V., Kooli, N., Gravier, G.: A survey on training and evaluation of word embeddings. International Journal of Data Science and Analytics 11, 85\u2013103 (2021)","DOI":"10.1007\/s41060-021-00242-8"},{"key":"10235_CR14","doi-asserted-by":"crossref","unstructured":"Lai, S., Liu, K., He, S., Zhao, J.: How to generate a good word embedding. IEEE Intelligent Systems 31(6), 5\u201314 (2016) 10.1109\/MIS.2016.45","DOI":"10.1109\/MIS.2016.45"},{"key":"10235_CR15","doi-asserted-by":"crossref","unstructured":"Faruqui, M., Dyer, C.: Community evaluation and exchange of word vectors at wordvectors. org. In: Proceedings of 52nd Annual Meeting of the Association for Computational Linguistics: System Demonstrations, pp. 19\u201324 (2014)","DOI":"10.3115\/v1\/P14-5004"},{"key":"10235_CR16","doi-asserted-by":"crossref","unstructured":"Bojanowski, P., Grave, E., Joulin, A., Mikolov, T.: Enriching word vectors with subword information. Transactions of the Association for Computational Linguistics 5, 135\u2013146 (2017)","DOI":"10.1162\/tacl_a_00051"},{"key":"10235_CR17","doi-asserted-by":"crossref","unstructured":"Pennington, J., Socher, R., Manning, C.: Glove: global vectors for word representation. In: EMNLP (2014)","DOI":"10.3115\/v1\/D14-1162"},{"key":"10235_CR18","doi-asserted-by":"crossref","unstructured":"Peters, M.E., Neumann, M., Iyyer, M., Gardner, M., Clark, C., Lee, K., Zettlemoyer, L.: Deep contextualized word representations. In NAACL. Association for Computational Linguistics New Orleans, Louisiana, USA (2018)","DOI":"10.18653\/v1\/N18-1202"},{"key":"10235_CR19","unstructured":"Devlin, J., Chang, M.-W., Lee, K., Toutanova, K.: Bert: pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)"},{"key":"10235_CR20","doi-asserted-by":"crossref","unstructured":"Roy, A., Pan, S.: Incorporating extra knowledge to enhance word embedding. In: Proceedings of the Twenty-Ninth International Conference on International Joint Conferences on Artificial Intelligence, pp. 4929\u20134935 (2021)","DOI":"10.24963\/ijcai.2020\/686"},{"key":"10235_CR21","doi-asserted-by":"crossref","unstructured":"Asudani, D.S., Nagwani, N.K., Singh, P.: Impact of word embedding models on text analytics in deep learning environment: a review. Artificial Intelligence Review, 1\u201381 (2023)","DOI":"10.1007\/s10462-023-10419-1"},{"key":"10235_CR22","doi-asserted-by":"crossref","unstructured":"Formica, A., Taglino, F.: Semantic relatedness in DBpedia: a comparative and experimental assessment. Information Sciences 621, 474\u2013505 (2023)","DOI":"10.1016\/j.ins.2022.11.025"},{"key":"10235_CR23","doi-asserted-by":"crossref","unstructured":"Zhang, M., Palade, V., Wang, Y., Ji, Z.: Word representation using refined contexts. Applied Intelligence 52(11), 12347\u201312368 (2022)","DOI":"10.1007\/s10489-021-02898-y"},{"key":"10235_CR24","doi-asserted-by":"crossref","unstructured":"Jameel, S., Schockaert, S.: Word and document embedding with vMF-mixture priors on context word vectors. (2019). ACL","DOI":"10.18653\/v1\/P19-1321"},{"key":"10235_CR25","doi-asserted-by":"crossref","unstructured":"Yang, D., Li, N., Zou, L., Ma, H.: Lexical semantics enhanced neural word embeddings. Knowledge-Based Systems 252, 109298 (2022)","DOI":"10.1016\/j.knosys.2022.109298"},{"key":"10235_CR26","doi-asserted-by":"crossref","unstructured":"An, H., Liu, X., Zhang, D.: Learning bias-reduced word embeddings using dictionary definitions. In: Findings of the Association for Computational Linguistics: ACL 2022, pp. 1139\u20131152 (2022)","DOI":"10.18653\/v1\/2022.findings-acl.90"},{"key":"10235_CR27","doi-asserted-by":"crossref","unstructured":"Zheng, J., Wang, Y., Wang, G., Xia, J., Huang, Y., Zhao, G., Zhang, Y., Li, S.: Using context-to-vector with graph retrofitting to improve word embeddings. In: Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8154\u20138163. Association for Computational Linguistics, Dublin, Ireland (2022). 10.18653\/v1\/2022.acl-long.561 . https:\/\/aclanthology.org\/2022.acl-long.561","DOI":"10.18653\/v1\/2022.acl-long.561"},{"key":"10235_CR28","doi-asserted-by":"crossref","unstructured":"Camacho-Collados, J., Navigli, R.: Find the word that does not belong: a framework for an intrinsic evaluation of word vector representations. In: Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for NLP, pp. 43\u201350 (2016)","DOI":"10.18653\/v1\/W16-2508"},{"key":"10235_CR29","doi-asserted-by":"crossref","unstructured":"Corcoran, P., Palmer, G., Arman, L., Knight, D., Spasi\u0107, I.: Creating welsh language word embeddings. Applied Sciences 11(15), 6896 (2021)","DOI":"10.3390\/app11156896"},{"key":"10235_CR30","doi-asserted-by":"crossref","unstructured":"Jolliffe, I.T., Cadima, J.: Principal component analysis: a review and recent developments. Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences 374(2065), 20150202 (2016)","DOI":"10.1098\/rsta.2015.0202"},{"key":"10235_CR31","doi-asserted-by":"crossref","unstructured":"Hadj\u00a0Taieb, M.A., Zesch, T., Ben\u00a0Aouicha, M.: A survey of semantic relatedness evaluation datasets and procedures. Artificial Intelligence Review 53(6), 4407\u20134448 (2020)","DOI":"10.1007\/s10462-019-09796-3"},{"key":"10235_CR32","doi-asserted-by":"crossref","unstructured":"Finkelstein, L., Gabrilovich, E., Matias, Y., Rivlin, E., Solan, Z., Wolfman, G., Ruppin, E.: Placing search in context: the concept revisited. In: Proceedings of the 10th International Conference on World Wide Web, pp. 406\u2013414 (2001)","DOI":"10.1145\/371920.372094"},{"key":"10235_CR33","doi-asserted-by":"crossref","unstructured":"Rubenstein, H., Goodenough, J.B.: Contextual correlates of synonymy. Communications of the ACM 8(10), 627\u2013633 (1965)","DOI":"10.1145\/365628.365657"},{"key":"10235_CR34","unstructured":"Luong, M.-T., Socher, R., Manning, C.D.: Better word representations with recursive neural networks for morphology. In: Proceedings of the Seventeenth Conference on Computational Natural Language Learning, pp. 104\u2013113 (2013)"},{"key":"10235_CR35","doi-asserted-by":"crossref","unstructured":"Bruni, E., Tran, N.-K., Baroni, M.: Multimodal distributional semantics. Journal of artificial intelligence research 49, 1\u201347 (2014)","DOI":"10.1613\/jair.4135"},{"key":"10235_CR36","doi-asserted-by":"crossref","unstructured":"Radinsky, K., Agichtein, E., Gabrilovich, E., Markovitch, S.: A word at a time: computing word relatedness using temporal semantic analysis. In: Proceedings of the 20th International Conference on World Wide Web, pp. 337\u2013346 (2011)","DOI":"10.1145\/1963405.1963455"},{"key":"10235_CR37","doi-asserted-by":"crossref","unstructured":"Hill, F., Reichart, R., Korhonen, A.: Simlex-999: evaluating semantic models with (genuine) similarity estimation. Computational Linguistics 41(4) (2015)","DOI":"10.1162\/COLI_a_00237"},{"key":"10235_CR38","doi-asserted-by":"crossref","unstructured":"Miller, G.A., Charles, W.G.: Contextual correlates of semantic similarity. Language and cognitive processes 6(1), 1\u201328 (1991)","DOI":"10.1080\/01690969108406936"},{"key":"10235_CR39","doi-asserted-by":"crossref","unstructured":"Halawi, G., Dror, G., Gabrilovich, E., Koren, Y.: Large-scale learning of word relatedness with constraints. In: Proceedings of the 18th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 1406\u20131414 (2012)","DOI":"10.1145\/2339530.2339751"},{"key":"10235_CR40","unstructured":"Yang, D., Powers, D.: Verb similarity on the taxonomy of wordnet. In: The Third International WordNet Conference: GWC 2006 (2006). Masaryk University"},{"key":"10235_CR41","doi-asserted-by":"crossref","unstructured":"Baker, S., Reichart, R., Korhonen, A.: An unsupervised model for instance level subcategorization acquisition. In: EMNLP, pp. 278\u2013289 (2014)","DOI":"10.3115\/v1\/D14-1034"},{"key":"10235_CR42","doi-asserted-by":"crossref","unstructured":"Gerz, D., Vuli\u0107, I., Hill, F., Reichart, R., Korhonen, A.: Simverb-3500: a large-scale evaluation set of verb similarity. arXiv preprint arXiv:1608.00869 (2016)","DOI":"10.18653\/v1\/D16-1235"},{"key":"10235_CR43","doi-asserted-by":"crossref","unstructured":"Camacho-Collados, J., Pilehvar, M.T., Collier, N., Navigli, R.: Semeval-2017 task 2: multilingual and cross-lingual semantic word similarity. In: Proceedings of the 11th International Workshop on Semantic Evaluation (SemEval-2017), pp. 15\u201326 (2017)","DOI":"10.18653\/v1\/S17-2002"},{"key":"10235_CR44","doi-asserted-by":"crossref","unstructured":"Agirre, E., Alfonseca, E., Hall, K., Kravalova, J., Pasca, M., Soroa, A.: A study on similarity and relatedness using distributional and wordnet-based approaches (2009)","DOI":"10.3115\/1620754.1620758"},{"key":"10235_CR45","unstructured":"Allen, C., Hospedales, T.: Analogies explained: towards understanding word embeddings. In: International Conference on Machine Learning, pp. 223\u2013231 (2019). PMLR"},{"key":"10235_CR46","unstructured":"Mikolov, T., Chen, K., Corrado, G., Dean, J.: Efficient estimation of word representations in vector space. arXiv preprint arXiv:1301.3781 (2013)"},{"key":"10235_CR47","unstructured":"Mikolov, T., Yih, W.-t., Zweig, G.: Linguistic regularities in continuous space word representations. In: Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 746\u2013751 (2013)"},{"key":"10235_CR48","unstructured":"Jurgens, D., Mohammad, S., Turney, P., Holyoak, K.: Semeval-2012 task 2: measuring degrees of relational similarity. In: * SEM 2012: The First Joint Conference on Lexical and Computational Semantics\u2013Volume 1: Proceedings of the Main Conference and the Shared Task, and Volume 2: Proceedings of the Sixth International Workshop on Semantic Evaluation (SemEval 2012), pp. 356\u2013364 (2012)"},{"key":"10235_CR49","doi-asserted-by":"crossref","unstructured":"Hanson, S.J., Bauer, M.: Conceptual clustering, categorization, and polymorphy. Machine Learning 3, 343\u2013372 (1989)","DOI":"10.1007\/BF00116838"},{"key":"10235_CR50","unstructured":"Almuhareb, A.: Attributes in lexical acquisition. PhD thesis, University of Essex (2006)"},{"key":"10235_CR51","unstructured":"Baroni, M., Lenci, A.: How we blessed distributional semantic evaluation. In: Proceedings of the GEMS 2011 Workshop on GEometrical Models of Natural Language Semantics, pp. 1\u201310 (2011)"},{"key":"10235_CR52","doi-asserted-by":"crossref","unstructured":"Baroni, M., Murphy, B., Barbu, E., Poesio, M.: Strudel: a corpus-based semantic model based on properties and types. Cognitive science 34(2), 222\u2013254 (2010)","DOI":"10.1111\/j.1551-6709.2009.01068.x"},{"key":"10235_CR53","unstructured":"Baroni, M., Evert, S., Lenci, A.: Bridging the gap between semantic theory and computational simulations: Proceedings of the esslli workshop on distributional lexical semantics. Hamburg, Germany: FOLLI (2008)"},{"key":"10235_CR54","unstructured":"Blair, P., Merhav, Y., Barry, J.: Automated generation of multilingual clusters for the evaluation of distributed representations. arXiv preprint arXiv:1611.01547 (2016)"},{"key":"10235_CR55","doi-asserted-by":"crossref","unstructured":"Ninio, F.: A simple proof of the Perron-Frobenius theorem for positive symmetric matrices. Journal of Physics A: General Physics 9(8), 1281\u20131282 (1976) 10.1088\/0305-4470\/9\/8\/017","DOI":"10.1088\/0305-4470\/9\/8\/017"},{"key":"10235_CR56","unstructured":"Jastrzebski, S., Le\u015bniak, D., Czarnecki, W.M.: How to evaluate word embeddings? on importance of data efficiency and simple supervised tasks. arXiv preprint arXiv:1702.02170 (2017)"},{"key":"10235_CR57","unstructured":"Guo, Y., Langer, C., Mercorio, F., Trentini, F.: Skills mismatch, automation, and training: evidence from 17 European countries using survey data and online job ads. In: CESifo Forum, vol. 23, pp. 11\u201315 (2022)"},{"key":"10235_CR58","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1016\/j.future.2018.03.035","volume":"86","author":"R Boselli","year":"2018","unstructured":"Boselli R, Cesarini M, Mercorio F, Mezzanzanica M. Classifying online Job advertisements through machine learning. Future Gener Comput Syst. 2018;86:319\u201328.","journal-title":"Future Gener Comput Syst."},{"key":"10235_CR59","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1016\/j.infoecopol.2019.05.003","volume":"47","author":"E Colombo","year":"2019","unstructured":"Colombo E, Mercorio F, Mezzanzanica M. AI meets labor market: Exploring the link between automation and skills. Inf Econ Policy. 2019;47:27\u201337.","journal-title":"Inf Econ Policy"}],"container-title":["Cognitive Computation"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12559-023-10235-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s12559-023-10235-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s12559-023-10235-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,1]],"date-time":"2024-06-01T09:30:44Z","timestamp":1717234244000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s12559-023-10235-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,1,22]]},"references-count":59,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2024,5]]}},"alternative-id":["10235"],"URL":"https:\/\/doi.org\/10.1007\/s12559-023-10235-3","relation":{},"ISSN":["1866-9956","1866-9964"],"issn-type":[{"value":"1866-9956","type":"print"},{"value":"1866-9964","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,1,22]]},"assertion":[{"value":"3 July 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 December 2023","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"22 January 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of Interest"}},{"value":"This article does not contain any studies with human participants or animals performed by any of the authors.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"Research Involving Human Participants or Animals"}}]}}