{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T06:19:16Z","timestamp":1777357156978,"version":"3.51.4"},"reference-count":77,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2020,3,2]],"date-time":"2020-03-02T00:00:00Z","timestamp":1583107200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,3,2]],"date-time":"2020-03-02T00:00:00Z","timestamp":1583107200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Int J Multimed Info Retr"],"published-print":{"date-parts":[[2020,9]]},"DOI":"10.1007\/s13735-020-00193-z","type":"journal-article","created":{"date-parts":[[2020,3,2]],"date-time":"2020-03-02T18:03:41Z","timestamp":1583172221000},"page":"191-203","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":13,"title":["Image annotation: the effects of content, lexicon and annotation method"],"prefix":"10.1007","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3168-2350","authenticated-orcid":false,"given":"Zenonas","family":"Theodosiou","sequence":"first","affiliation":[]},{"given":"Nicolas","family":"Tsapatsoulis","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,3,2]]},"reference":[{"key":"193_CR1","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1007\/978-981-10-6759-4_2","volume-title":"Content-based image retrieval techniques: a review","author":"V Tyagi","year":"2017","unstructured":"Tyagi V (2017) Content-based image retrieval techniques: a review. Springer, Singapore, pp 29\u201348"},{"issue":"2","key":"193_CR2","doi-asserted-by":"crossref","first-page":"472","DOI":"10.1109\/TKDE.2016.2562624","volume":"29","author":"L Zhu","year":"2017","unstructured":"Zhu L, Shen J, Xie L, Cheng Z (2017) Unsupervised visual hashing with semantic assistant for content-based image retrieval. IEEE Trans Knowl Data Eng 29(2):472\u2013486","journal-title":"IEEE Trans Knowl Data Eng"},{"key":"193_CR3","doi-asserted-by":"crossref","unstructured":"Nazir A, Ashraf R, Hamdani T, Ali N (2018) Content based image retrieval system by using HSV color histogram, discrete wavelet transform and edge histogram descriptor. In: 2018 international conference on computing, mathematics and engineering technologies (iCoMET), pp 1\u20136","DOI":"10.1109\/ICOMET.2018.8346343"},{"key":"193_CR4","doi-asserted-by":"crossref","unstructured":"Li A, Sun J, Ng JY, Yu R, Morariu VI, Davis LS (2017) Generating holistic 3D scene abstractions for text-based image retrieval. In: 2017 IEEE conference on computer vision and pattern recognition (CVPR), pp 1942\u20131950","DOI":"10.1109\/CVPR.2017.210"},{"issue":"24","key":"193_CR5","doi-asserted-by":"crossref","first-page":"31991","DOI":"10.1007\/s11042-018-6247-3","volume":"77","author":"A Dutta","year":"2018","unstructured":"Dutta A, Verma Y, Jawahar CV (2018) Automatic image annotation: the quirks and what works. Multimed Tools Appl 77(24):31991\u201332011","journal-title":"Multimed Tools Appl"},{"issue":"12","key":"193_CR6","doi-asserted-by":"crossref","first-page":"3005","DOI":"10.1109\/TVCG.2017.2772238","volume":"24","author":"DT Nguyen","year":"2018","unstructured":"Nguyen DT, Hua B, Yu L, Yeung S (2018) A robust 3D\u20132D interactive tool for scene segmentation and annotation. IEEE Trans Vis Comput Graph 24(12):3005\u20133018","journal-title":"IEEE Trans Vis Comput Graph"},{"key":"193_CR7","doi-asserted-by":"crossref","unstructured":"Yang CM, Choo Y, Park S (2018) Semi-automatic image and video annotation system for generating ground truth information. In: 2018 International conference on information networking (ICOIN), pp 821\u2013824","DOI":"10.1109\/ICOIN.2018.8343233"},{"key":"193_CR8","doi-asserted-by":"publisher","unstructured":"Dutta A, Zisserman A (2019) The VIA annotation software for images, audio and video. In: Proceedings of the 27th ACM international conference on multimedia, MM \u201919, Nice, France. ACM, New York, NY. https:\/\/doi.org\/10.1145\/3343031.3350535","DOI":"10.1145\/3343031.3350535"},{"key":"193_CR9","doi-asserted-by":"crossref","first-page":"242","DOI":"10.1016\/j.patcog.2018.02.017","volume":"79","author":"Q Cheng","year":"2018","unstructured":"Cheng Q, Zhang Q, Fu P, Tu C, Li S (2018) A survey and analysis on automatic image annotation. Pattern Recognit 79:242\u2013259","journal-title":"Pattern Recognit"},{"issue":"3","key":"193_CR10","doi-asserted-by":"crossref","first-page":"3767","DOI":"10.1007\/s11042-018-6038-x","volume":"78","author":"Y Ma","year":"2019","unstructured":"Ma Y, Liu Y, Xie Q (2019) CNN-feature based automatic image annotation method. Multimed Tools Appl 78(3):3767\u20133780","journal-title":"Multimed Tools Appl"},{"issue":"9","key":"193_CR11","doi-asserted-by":"crossref","first-page":"11815","DOI":"10.1007\/s11042-018-6742-6","volume":"78","author":"C Jin","year":"2019","unstructured":"Jin C, Sun QM, Jin SW (2019) A hybrid automatic image annotation approach. Multimed Tools Appl 78(9):11815\u201311834","journal-title":"Multimed Tools Appl"},{"key":"193_CR12","doi-asserted-by":"crossref","first-page":"346","DOI":"10.1016\/j.patcog.2011.05.013","volume":"45","author":"D Zhang","year":"2012","unstructured":"Zhang D, Islam MM, Lu G (2012) A review on automatic image annotation techniques. Pattern Recognit 45:346\u2013362","journal-title":"Pattern Recognit"},{"key":"193_CR13","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1007\/s00530-006-0025-1","volume":"12","author":"R Zhang","year":"2006","unstructured":"Zhang R, Zhang Z, Li M, Zhang HJ (2006) A probabilistic semantic model for image annotation and multi-modal image retrieval. Multimed Syst 12:27\u201333","journal-title":"Multimed Syst"},{"key":"193_CR14","unstructured":"Kwasnicka H, Paradowski M (2010) Machine learning methods in automatic image annotation. In: Advances in machine learning II. Studies in computational intelligence, vol 263, pp 387\u2013411"},{"issue":"1","key":"193_CR15","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1007\/s11263-017-1039-1","volume":"126","author":"M Wigness","year":"2018","unstructured":"Wigness M, Draper BA, Beveridge JR (2018) Efficient label collection for image datasets via hierarchical clustering. Int J Comput Vis 126(1):59\u201385","journal-title":"Int J Comput Vis"},{"issue":"7","key":"193_CR16","doi-asserted-by":"crossref","first-page":"1411","DOI":"10.1109\/TPAMI.2015.2487982","volume":"38","author":"S Hong","year":"2016","unstructured":"Hong S, Choi J, Feyereisl J, Han B, Davis LS (2016) Joint image clustering and labeling by matrix factorization. IEEE Trans Pattern Anal Mach Intell 38(7):1411\u20131424","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"issue":"11","key":"193_CR17","doi-asserted-by":"crossref","first-page":"299","DOI":"10.3390\/electronics7110299","volume":"7","author":"A Glowacz","year":"2018","unstructured":"Glowacz A (2018) Acoustic-based fault diagnosis of commutator motor. Electronics 7(11):299","journal-title":"Electronics"},{"key":"193_CR18","doi-asserted-by":"crossref","first-page":"65","DOI":"10.1016\/j.ymssp.2018.07.044","volume":"117","author":"A Glowacz","year":"2019","unstructured":"Glowacz A (2019) Fault diagnosis of single-phase induction motor based on acoustic signals. Mech Syst Signal Process 117:65\u201380","journal-title":"Mech Syst Signal Process"},{"issue":"2","key":"193_CR19","first-page":"1","volume":"13","author":"Y Huang","year":"2018","unstructured":"Huang Y, Yang H, Qi X, Malekian R, Pfeiffer O, Li Z (2018) A novel selection method of seismic attributes based on gray relational degree and support vector machine. PLoS ONE 13(2):1\u201316","journal-title":"PLoS ONE"},{"key":"193_CR20","doi-asserted-by":"crossref","first-page":"120","DOI":"10.1016\/j.infrared.2017.12.015","volume":"89","author":"YL dit Leksir","year":"2018","unstructured":"dit Leksir YL, Mansour M, Moussaoui A (2018) Localization of thermal anomalies in electrical equipment using infrared thermography and support vector machine. Infrared Phys Technol 89:120\u2013128","journal-title":"Infrared Phys Technol"},{"issue":"3","key":"193_CR21","doi-asserted-by":"crossref","first-page":"490","DOI":"10.1109\/TPAMI.2015.2459678","volume":"38","author":"M Ristin","year":"2016","unstructured":"Ristin M, Guillaumin M, Gall J, Gool LV (2016) Incremental learning of random forests for large-scale image classification. IEEE Trans Pattern Anal Mach Intell 38(3):490\u2013503","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"193_CR22","doi-asserted-by":"publisher","unstructured":"Piramanayagam S, Schwartzkopf W, Koehler FW, Saber E (2016) Classification of remote sensed images using random forests and deep learning framework. In: Bruzzone L, Bovolo F (eds) Image and signal processing for remote sensing XXII, vol 10004. SPIE, pp 205\u2013212. https:\/\/doi.org\/10.1117\/12.2243169","DOI":"10.1117\/12.2243169"},{"key":"193_CR23","doi-asserted-by":"crossref","unstructured":"Quintero R, Parra I, Lorenzo J, Fern\u00e1ndez-Llorca D, Sotelo MA (2017) Pedestrian intention recognition by means of a hidden Markov model and body language. In: 2017 IEEE 20th international conference on intelligent transportation systems (ITSC), pp 1\u20137","DOI":"10.1109\/ITSC.2017.8317766"},{"issue":"3","key":"193_CR24","doi-asserted-by":"crossref","first-page":"849","DOI":"10.1109\/TMI.2016.2633551","volume":"36","author":"F Xie","year":"2017","unstructured":"Xie F, Fan H, Li Y, Jiang Z, Meng R, Bovik A (2017) Melanoma classification on dermoscopy images using a neural network ensemble model. IEEE Trans Med Imaging 36(3):849\u2013858","journal-title":"IEEE Trans Med Imaging"},{"issue":"7553","key":"193_CR25","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1038\/nature14539","volume":"521","author":"Y Lecun","year":"2015","unstructured":"Lecun Y, Bengio Y, Hinton G (2015) Deep learning. Nature 521(7553):436\u2013444","journal-title":"Nature"},{"key":"193_CR26","unstructured":"Perina A, Mohammadi S, Jojic N, Murino V (2017) Summarization and classification of wearable camera streams by learning the distributions over deep features of out-of-sample image sequences. In: The IEEE international conference on computer vision (ICCV)"},{"issue":"2","key":"193_CR27","doi-asserted-by":"crossref","first-page":"645","DOI":"10.1109\/TGRS.2016.2612821","volume":"55","author":"E Maggiori","year":"2017","unstructured":"Maggiori E, Tarabalka Y, Charpiat G, Alliez P (2017) Convolutional neural networks for large-scale remote-sensing image classification. IEEE Trans Geosci Remote Sens 55(2):645\u2013657","journal-title":"IEEE Trans Geosci Remote Sens"},{"key":"193_CR28","doi-asserted-by":"crossref","first-page":"834","DOI":"10.1109\/TPAMI.2017.2699184","volume":"40","author":"LC Chen","year":"2018","unstructured":"Chen LC, Papandreou G, Kokkinos I, Murphy K, Yuille AL (2018) Deeplab: semantic image segmentation with deep convolutional nets, atrous convolution, and fully connected crfs. IEEE Trans Pattern Anal Mach Intell 40:834\u2013848","journal-title":"IEEE Trans Pattern Anal Mach Intell"},{"key":"193_CR29","doi-asserted-by":"crossref","unstructured":"Wang J, Yang Y, Mao J, Huang Z, Huang C, Xu W (2016) CNN-RNN: a unified framework for multi-label image classification. In: The IEEE conference on computer vision and pattern recognition (CVPR)","DOI":"10.1109\/CVPR.2016.251"},{"issue":"6","key":"193_CR30","doi-asserted-by":"crossref","first-page":"2712","DOI":"10.1109\/TIP.2016.2549459","volume":"25","author":"X Jing","year":"2016","unstructured":"Jing X, Wu F, Li Z, Hu R, Zhang D (2016) Multi-label dictionary learning for image annotation. IEEE Trans Image Process 25(6):2712\u20132725","journal-title":"IEEE Trans Image Process"},{"key":"193_CR31","doi-asserted-by":"crossref","unstructured":"Penna A, Mohammadi S, Jojic N, Murino V (2017) Summarization and classification of wearable camera streams by learning the distributions over deep features of out-of-sample image sequences. In: 2017 IEEE international conference on computer vision (ICCV), pp 4336\u20134344","DOI":"10.1109\/ICCV.2017.464"},{"issue":"2","key":"193_CR32","first-page":"303","volume":"48","author":"PB Heidorn","year":"1999","unstructured":"Heidorn PB (1999) Image retrieval as linguistic and nonlinguistic visual model matching. Libr Trends 48(2):303\u2013325","journal-title":"Libr Trends"},{"key":"193_CR33","doi-asserted-by":"crossref","unstructured":"Hare JS, Lewis PH, Esner PGB, Sandom CJ (2006) Mind the gap: another look at the problem of the semantic gap in image retrieval. In: Proceedings of multimedia content analysis, management and retrieval 2006 SPIE, San Jose, California, USA","DOI":"10.1117\/12.647755"},{"key":"193_CR34","doi-asserted-by":"crossref","unstructured":"Theodosiou Z, Kasapi C, Tsapatsoulis N (2012) Semantic gap between people: an experimental investigation based on image annotation. In: Seventh international workshop on semantic and social media adaptation and personalization (SMAP), Luxembourg, pp 73\u201377","DOI":"10.1109\/SMAP.2012.23"},{"issue":"3","key":"193_CR35","doi-asserted-by":"crossref","first-page":"177","DOI":"10.1561\/0600000071","volume":"10","author":"A Kovashka","year":"2016","unstructured":"Kovashka A, Russakovsky O, Fei-Fei L, Grauman K (2016) Crowdsourcing in computer vision. Found Trends Comput Graph Vis 10(3):177\u2013243","journal-title":"Found Trends Comput Graph Vis"},{"key":"193_CR36","doi-asserted-by":"crossref","unstructured":"Makadia A, Pavlovic V, Kumar S (2008) A new baseline for image annotation. In: Proceedings of European conference on computer vision, Marseille, France, pp 316\u2013329","DOI":"10.1007\/978-3-540-88690-7_24"},{"issue":"5","key":"193_CR37","doi-asserted-by":"crossref","first-page":"617","DOI":"10.1016\/j.jvlc.2008.01.002","volume":"19","author":"A Hanbury","year":"2008","unstructured":"Hanbury A (2008) A survey of methods for image annotation. J Vis Lang Comput 19(5):617\u2013627","journal-title":"J Vis Lang Comput"},{"key":"193_CR38","doi-asserted-by":"crossref","first-page":"173","DOI":"10.1007\/978-981-10-8848-3_17","volume-title":"Software engineering","author":"P Gulati","year":"2019","unstructured":"Gulati P, Yadav M (2019) A novel approach for extracting pertinent keywords for web image annotation using semantic distance and euclidean distance. In: Hoda MN, Chauhan N, Quadri SMK, Srivastava PR (eds) Software engineering. Springer, Singapore, pp 173\u2013183"},{"issue":"4","key":"193_CR39","doi-asserted-by":"crossref","first-page":"283","DOI":"10.1108\/10650750610706998","volume":"22","author":"KK Matusiak","year":"2006","unstructured":"Matusiak KK (2006) Towards user-centered indexing in digital image collections. OCLC Syst Serv 22(4):283\u2013298","journal-title":"OCLC Syst Serv"},{"key":"193_CR40","doi-asserted-by":"crossref","unstructured":"Joachims T, Granka L, Pang B, Hembrooke H, Gay G (2005) Accurately interpreting clickthrough data as implicit feedback. In: Proceedings of the 28th annual international ACM SIGIR conference, Salvador, Brazil, pp 154\u2013161","DOI":"10.1145\/1076034.1076063"},{"key":"193_CR41","doi-asserted-by":"crossref","unstructured":"Macdonald C, Ounis I (2009) Usefulness of quality clickthrough data for training. In: Proceedings of the 2009 workshop on web search click data, Barcelona, Spain, pp 75\u201379","DOI":"10.1145\/1507509.1507521"},{"key":"193_CR42","doi-asserted-by":"crossref","unstructured":"Tsikrika T, Diou C, De Vries AP, Delopoulos A (2009) Image annotation using clickthrough data. In: Proceedings of the 8th international conference on image and video retrieval, Santorini, Greece, pp 1\u20138","DOI":"10.1145\/1646396.1646415"},{"key":"193_CR43","doi-asserted-by":"crossref","unstructured":"Kittur A, Kraut RE (2008) Harnessing the wisdom of crowds in Wikipedia: quality through coordination. In: Proceedings of the 2008 ACM conference on computer supported cooperative work, San Diego, CA, USA, pp 37\u201346","DOI":"10.1145\/1460563.1460572"},{"key":"193_CR44","doi-asserted-by":"crossref","unstructured":"Theodosiou Z, Tsapatsoulis N (2011) Crowdsourcing annotation: modelling keywords using low level features. In: Proceedings of the 5th international conference on internet multimedia systems architecture and application, Bangalore, India","DOI":"10.1109\/IMSAA.2011.6156351"},{"key":"193_CR45","doi-asserted-by":"crossref","unstructured":"Chen KT, Wu CC, Chang YC, Lei CL (2009) A crowdsourceable QoE evaluation framework for multimedia content. In: Proceedings of the 17th ACM international conference on multimedia, Beijing, China, pp 491\u2013500","DOI":"10.1145\/1631272.1631339"},{"key":"193_CR46","unstructured":"Brants T (2000) Inter-annotator agreement for a German newspaper corpus. In: Proceedings of the 2nd international conference on language resources and evaluation, Athens, Greece, pp 1\u20135"},{"issue":"3","key":"193_CR47","doi-asserted-by":"crossref","first-page":"453","DOI":"10.1006\/csla.1998.0108","volume":"12","author":"A Kilgarriff","year":"1998","unstructured":"Kilgarriff A (1998) Gold standard datasets for evaluating word sense disambiguation programs. Comput Speech Lang 12(3):453\u2013472","journal-title":"Comput Speech Lang"},{"key":"193_CR48","volume-title":"Crowdsourcing: why the power of the crowd is driving the future of business","author":"J Howe","year":"2008","unstructured":"Howe J (2008) Crowdsourcing: why the power of the crowd is driving the future of business. Crown Business, New York"},{"issue":"2","key":"193_CR49","doi-asserted-by":"crossref","first-page":"343","DOI":"10.1111\/ijmr.12135","volume":"20","author":"A Ghezzi","year":"2018","unstructured":"Ghezzi A, Gabelloni D, Martini A, Natalicchio A (2018) Crowdsourcing: a review and suggestions for future research. Int J Manag Rev 20(2):343\u2013363","journal-title":"Int J Manag Rev"},{"key":"193_CR50","doi-asserted-by":"crossref","unstructured":"Welinder P, Perona P (2010) Online crowdsourcing: rating annotators and obtaining cost effective labels. In: Proceedings of IEEE conference on computer vision and pattern recognition, San Francisco, CA, USA, pp 25\u201332","DOI":"10.1109\/CVPRW.2010.5543189"},{"issue":"6","key":"193_CR51","first-page":"176","volume":"14","author":"J Howe","year":"2006","unstructured":"Howe J (2006) The rise of crowdsourcing. Wired Mag 14(6):176\u2013183","journal-title":"Wired Mag"},{"issue":"1","key":"193_CR52","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1177\/1354856507084420","volume":"14","author":"D Brabham","year":"2008","unstructured":"Brabham D (2008) Crowdsourcing as a model for problem solving: an introduction and cases. Convergence 14(1):75\u201390","journal-title":"Convergence"},{"key":"193_CR53","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1016\/j.chb.2015.08.031","volume":"54","author":"AM Brawley","year":"2016","unstructured":"Brawley AM, Pury CLS (2016) Work experiences on mturk: job satisfaction, turnover, and information sharing. Comput Hum Behav 54:531\u2013546","journal-title":"Comput Hum Behav"},{"key":"193_CR54","volume-title":"Survey research methods","author":"F Fowler Jr","year":"2014","unstructured":"Fowler F Jr (2014) Survey research methods, 5th edn. SAGE Publications Inc, Thousand Oaks","edition":"5"},{"issue":"2","key":"193_CR55","doi-asserted-by":"crossref","first-page":"76","DOI":"10.1109\/MIC.2013.20","volume":"17","author":"M Allahbakhsh","year":"2013","unstructured":"Allahbakhsh M, Benatallah B, Ignjatovic A, Motahari-Nezhad HR, Bertino E, Dustdar S (2013) Quality control in crowdsourcing systems: issues and directions. IEEE Internet Comput 17(2):76\u201381","journal-title":"IEEE Internet Comput"},{"key":"193_CR56","doi-asserted-by":"crossref","first-page":"759","DOI":"10.1177\/1073191118760709","volume":"26","author":"MN McCredie","year":"2018","unstructured":"McCredie MN, Morey LC (2018) Who are the turkers? A characterization of mturk workers using the personality assessment inventory. Assessment 26:759\u2013766","journal-title":"Assessment"},{"issue":"2","key":"193_CR57","doi-asserted-by":"crossref","first-page":"339","DOI":"10.1111\/apps.12124","volume":"67","author":"M Lovett","year":"2017","unstructured":"Lovett M, Bajaba S, Lovett M, Simmering MJ (2017) Data quality from crowdsourced surveys: a mixed method inquiry into perceptions of Amazon\u2019s mechanical turk masters. Appl Psychol 67(2):339\u2013366","journal-title":"Appl Psychol"},{"key":"193_CR58","doi-asserted-by":"crossref","unstructured":"Snow R, O\u2019Connor B, Jurafsky D, Ng A (2008) Cheap and fast but is it good evaluating nonexpert annotations for natural language tasks. In: Proceedings of the conference on empirical methods in natural language processing, Honolulu, HI, USA, pp 254\u2013263","DOI":"10.3115\/1613715.1613751"},{"key":"193_CR59","doi-asserted-by":"crossref","unstructured":"Raykar V, Zhao S, Yu L, Jerebko A, Florin C, Valadez G, Bogoni L, Moy L (2009) Supervised learning from multiple experts: whom to trust when everyone lies a bit. In: Proceedings of the 26th annual international conference on machine learning, Montreal, Canada, pp 889\u2013896","DOI":"10.1145\/1553374.1553488"},{"key":"193_CR60","first-page":"1085","volume":"7","author":"P Smyth","year":"1995","unstructured":"Smyth P, Fayyad UM, Burl M, Perona P, Baldi P (1995) Inferring ground truth from subjective labeling of venus images. Adv Neural Inf Process Syst 7:1085\u20131092","journal-title":"Adv Neural Inf Process Syst"},{"key":"193_CR61","doi-asserted-by":"crossref","unstructured":"Sheng VS, Provost F, Ipeirotis PG (2008) Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD international conference on knowledge discovery and data mining, Las Vegas, NV, USA, pp 614\u2013622","DOI":"10.1145\/1401890.1401965"},{"issue":"5895","key":"193_CR62","doi-asserted-by":"crossref","first-page":"1465","DOI":"10.1126\/science.1160379","volume":"321","author":"LV Ahn","year":"2008","unstructured":"Ahn LV, Maurer B, McMillen C, Abraham D, Blum M (2008) Recaptcha: human-based character recognition via web security measures. Science 321(5895):1465\u20131468","journal-title":"Science"},{"key":"193_CR63","unstructured":"Whitehill J, Ruvolo P, Bergsma T Wu J, Movellan J (2009) Whose vote should count more: optimal integration of labels from labelers of unknown expertise. In: Proceedings of the 23rd annual conference on neural information processing systems, Vancouver, Canada, pp 2035\u20132043"},{"key":"193_CR64","unstructured":"Vijayanarasimhan S, Grauman K (2009) What\u2019s it going to cost you? Predicting effort vs. informativeness for multi-label image annotations. In: Proceedings of IEEE computer society conference on computer vision and pattern recognition, Miami, FL, USA, pp 2262\u20132269"},{"issue":"1","key":"193_CR65","doi-asserted-by":"crossref","first-page":"15","DOI":"10.1609\/aimag.v36i1.2564","volume":"36","author":"L Aroyo","year":"2015","unstructured":"Aroyo L, Welty C (2015) Truth is a lie: crowd truth and the seven myths of human annotation. AI Mag 36(1):15\u201324","journal-title":"AI Mag"},{"key":"193_CR66","volume-title":"Handbook of linguistic annotation","author":"R Artstein","year":"2017","unstructured":"Artstein R (2017) Inter-annotator agreement. In: Ide N, Pustejovsky J (eds) Handbook of linguistic annotation. Springer, Dordrecht"},{"key":"193_CR67","doi-asserted-by":"crossref","unstructured":"Callison-Burch C (2009) Fast, cheap, and creative: evaluating translation quality using Amazon\u2019s mechanical turk. In: Proceedings of conference on empirical methods in natural language processing, Singapore, pp 286\u2013295","DOI":"10.3115\/1699510.1699548"},{"key":"193_CR68","doi-asserted-by":"crossref","unstructured":"Nowak S, Ruger S (2010) How reliable are annotations via crowdsourcing a study about inter-annotator agreement for multi-label image annotation. In: Proceedings of the international conference on multimedia information retrieval, Philadelphia, PA, USA, pp 557\u2013566","DOI":"10.1145\/1743384.1743478"},{"key":"193_CR69","first-page":"644","volume":"245","author":"P Yadav","year":"2017","unstructured":"Yadav P, Jezek E, Bouillon P, Callahan T, Bada M, Hunter L, Cohen KB (2017) Semantic relations in compound nouns: perspectives from inter-annotator agreement. Stud Health Technol Inform 245:644\u2013648","journal-title":"Stud Health Technol Inform"},{"key":"193_CR70","unstructured":"https:\/\/commandaria.cut.ac.cy\/\/"},{"key":"193_CR71","unstructured":"Papadopoulos K, Tsapatsoulis N, Lanitis A, Kounoudes A (2008) The history of commandaria: digital journeys back to time. In: Proceedings of the 14th international conference on virtual systems and multimedia, Limassol, Cyprus"},{"issue":"1","key":"193_CR72","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen J (1960) A coefficient of agreement for nomimal scales. Educ Phsychol Meas 20(1):37\u201346","journal-title":"Educ Phsychol Meas"},{"key":"193_CR73","doi-asserted-by":"crossref","first-page":"159","DOI":"10.2307\/2529310","volume":"33","author":"JR Landis","year":"1977","unstructured":"Landis JR, Koch GK (1977) The measurement of observer agreement for categorical data. Biometrics 33:159\u2013174","journal-title":"Biometrics"},{"key":"193_CR74","unstructured":"Randolph JJ (2005) Free-marginal multirater kappa: an alternative to Fleiss\u2019 fixed-marginal multirater kappa. In: Joensuu University learning and instruction symposium, Joensuu, Finland"},{"issue":"5","key":"193_CR75","doi-asserted-by":"crossref","first-page":"553","DOI":"10.1037\/0003-066X.37.5.553","volume":"37","author":"M Cowles","year":"1982","unstructured":"Cowles M, Davis C (1982) On the origins of the.05 level of statistical significance. Am Psychol 37(5):553\u2013558","journal-title":"Am Psychol"},{"key":"193_CR76","unstructured":"Fujisawa S (2007) Automatic creation and enhancement of metadata for cultural heritage. In: Bulletin of IEEE technical committee on digital libraries (TCDL)"},{"key":"193_CR77","unstructured":"Randolph JJ (2008) Online kappa calculator. http:\/\/justusrandolph.net\/kappa\/. Retrieved 5 Apr 2019"}],"container-title":["International Journal of Multimedia Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-020-00193-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13735-020-00193-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-020-00193-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,3,2]],"date-time":"2021-03-02T01:11:50Z","timestamp":1614647510000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13735-020-00193-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,3,2]]},"references-count":77,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2020,9]]}},"alternative-id":["193"],"URL":"https:\/\/doi.org\/10.1007\/s13735-020-00193-z","relation":{},"ISSN":["2192-6611","2192-662X"],"issn-type":[{"value":"2192-6611","type":"print"},{"value":"2192-662X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,3,2]]},"assertion":[{"value":"23 July 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 December 2019","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 February 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 March 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}