{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,22]],"date-time":"2026-03-22T16:04:03Z","timestamp":1774195443459,"version":"3.50.1"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2010,8,17]],"date-time":"2010-08-17T00:00:00Z","timestamp":1282003200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Multimed Tools Appl"],"published-print":{"date-parts":[[2011,10]]},"DOI":"10.1007\/s11042-010-0584-1","type":"journal-article","created":{"date-parts":[[2010,8,16]],"date-time":"2010-08-16T06:36:30Z","timestamp":1281940590000},"page":"27-52","source":"Crossref","is-referenced-by-count":17,"title":["Reliability and effectiveness of clickthrough data for automatic image annotation"],"prefix":"10.1007","volume":"55","author":[{"given":"Theodora","family":"Tsikrika","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christos","family":"Diou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Arjen P.","family":"de Vries","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anastasios","family":"Delopoulos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2010,8,17]]},"reference":[{"key":"584_CR1","doi-asserted-by":"crossref","unstructured":"Ashman H, Antunovic M, Donner C, Frith R, Rebelos E, Schmakeit JF, Smith G, Truran M (2009) Are clickthroughs useful for image labelling? In: Pasi G, Bordogna G, Mauri G, Baeza-Yates R (eds) Proceedings of the 2009 IEEE\/WIC\/ACM international conference on web intelligence (WI 2009), pp 191\u2013197","DOI":"10.1109\/WI-IAT.2009.35"},{"key":"584_CR2","doi-asserted-by":"crossref","unstructured":"Ayache S, Qu\u00e9not G (2008) Video corpus annotation using active learning. In: Boughanem M, Berrut C, Mothe J, Soul\u00e9-Dupuy C (eds) Proceedings of the 30th European conference on IR research, pp 187\u2013198","DOI":"10.1007\/978-3-540-78646-7_19"},{"issue":"12","key":"584_CR3","doi-asserted-by":"crossref","first-page":"1793","DOI":"10.1002\/asi.20627","volume":"58","author":"RA Baeza-Yates","year":"2007","unstructured":"Baeza-Yates RA, Hurtado CA, Mendoza M (2007) Improving search engines by query clustering. J Am Soc Inf Sci Technol 58(12):1793\u20131804","journal-title":"J Am Soc Inf Sci Technol"},{"key":"584_CR4","unstructured":"Chang CC, Lin CJ (2001) Libsvm: a library for support vector machines. Available at http:\/\/www.csie.ntu.edu.tw\/~cjlin\/libsvm"},{"key":"584_CR5","unstructured":"Chang SF, He J, Jiang YG, El\u00a0Khoury E, Ngo CW, Yanagawa A, Zavesky E (2008) Columbia University\/VIREO-CityU\/IRIT TRECVID2008 high-level feature extraction and interactive video search. In: Proceedings of TRECVID 2008"},{"key":"584_CR6","doi-asserted-by":"crossref","unstructured":"Chua TS, Tang J, Hong R, Li H, Luo H, Zheng YT (2009) NUS-WIDE: A real-world Web image database from National University of Singapore. In: Marchand-Maillet S, Kompatsiaris Y (eds) Proceedings of the 8th international conference on content-based image and video retrieval (CIVR 2009). ACM Press","DOI":"10.1145\/1646396.1646452"},{"key":"584_CR7","doi-asserted-by":"crossref","unstructured":"Craswell N, Szummer M (2007) Random walks on the click graph. In: Proceedings of the 30th ACM SIGIR conference on research and development in information retrieval, pp 239\u2013246","DOI":"10.1145\/1277741.1277784"},{"key":"584_CR8","unstructured":"Faymonville P, Wang K, Miller J, Belongie SJ (2009) CAPTCHA-based image labeling on the Soylent Grid. In: Bennett PN, Chandrasekar R, Chickering M, Ipeirotis PG, Law E, Mityagin A, Provost FJ, von Ahn L (eds) (2009) Proceedings of the ACM SIGKDD workshop on human computation. ACM Press, pp 46\u201349"},{"issue":"2","key":"584_CR9","doi-asserted-by":"crossref","first-page":"147","DOI":"10.1145\/1059981.1059982","volume":"23","author":"S Fox","year":"2005","unstructured":"Fox S, Karnawat K, Mydland M, Dumais ST, White T (2005) Evaluating implicit measures to improve web search. ACM Trans Inf Syst 23(2):147\u2013168","journal-title":"ACM Trans Inf Syst"},{"key":"584_CR10","doi-asserted-by":"crossref","unstructured":"van Gemert JC, Geusebroek JM, Veenman CJ, Snoek CGM, Smeulders AWM (2006) Robust scene categorization by learning image statistics in context. In: International workshop on semantic learning applications in multimedia, p 105","DOI":"10.1109\/CVPRW.2006.177"},{"key":"584_CR11","unstructured":"Hauptmann A, Yan R, Lin WH (2007) How many high-level concepts will fill the semantic gap in news video retrieval? In: Sebe N, Worring M (eds) Proceedings of the 6th international conference on content-based image and video retrieval (CIVR 2007). ACM Press, pp 627\u2013634"},{"key":"584_CR12","doi-asserted-by":"crossref","unstructured":"Hiemstra D (1998) A linguistically motivated probabilistic model of information retrieval. In: Proceedings of the 2nd European conference on research and advanced technology for digital libraries (ECDL 1998), pp 569\u2013584","DOI":"10.1007\/3-540-49653-X_34"},{"key":"584_CR13","unstructured":"Hiemstra D, Rode H, van Os R, Flokstra J (2006) PF\/Tijah: text search in an XML database system. In: Proceedings of the 2nd international workshop on open source information retrieval (OSIR 2006), pp 12\u201317"},{"key":"584_CR14","unstructured":"Ho CJ, Chang TH, Lee JC, Hsu JYJ, Chen KT (2009) KissKissBan: a competitive human computation game for image annotation. In: Bennett PN, Chandrasekar R, Chickering M, Ipeirotis PG, Law E, Mityagin A, Provost FJ, von Ahn L (eds) (2009) Proceedings of the ACM SIGKDD workshop on human computation. ACM Press, pp 11\u201314"},{"key":"584_CR15","doi-asserted-by":"crossref","unstructured":"Joachims T (2002) Optimizing search engines using clickthrough data. In: Proceedings of the 8th annual international ACM SIGKDD conference on knowledge discovery and data mining, pp 133\u2013142","DOI":"10.1145\/775047.775067"},{"key":"584_CR16","author":"T Joachims","year":"2007","unstructured":"Joachims T, Granka L, Pan B, Hembrooke H, Radlinski F, Gay G (2007) Evaluating the accuracy of implicit feedback from clicks and query reformulations in Web search. ACM Trans Inf Syst 25(2). doi: 10.1145\/1229179.1229181","journal-title":"ACM Trans Inf Syst"},{"issue":"2","key":"584_CR17","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1328964.1328974","volume":"41","author":"T Joachims","year":"2007","unstructured":"Joachims T, Li H, Liu TY, Zhai C (2007) Learning to rank for information retrieval (lr4ir 2007). SIGIR Forum 41(2):58\u201362","journal-title":"SIGIR Forum"},{"issue":"2","key":"584_CR18","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1145\/959258.959260","volume":"37","author":"D Kelly","year":"2003","unstructured":"Kelly D, Teevan J (2003) Implicit feedback for inferring user preference: a bibliography. SIGIR Forum 37(2):18\u201328","journal-title":"SIGIR Forum"},{"key":"584_CR19","doi-asserted-by":"crossref","unstructured":"Li X, Snoek CGM (2009) Visual categorization with negative examples for free. In: Gao W, Rui Y, Hanjalic A, Xu C, Steinbach EG, El-Saddik A, Zhou MX (eds) Proceedings of the 17th international conference on multimedia. ACM Press, pp 661\u2013664","DOI":"10.1145\/1631272.1631382"},{"key":"584_CR20","unstructured":"LSCOM Lexicon definitions and annotations version 1.0. Tech. rep., Columbia University (2006)"},{"key":"584_CR21","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1145\/1507509.1507521","volume-title":"Proceedings of the 2009 workshop on Web search click data (WSCD 2009).","author":"C Macdonald","year":"2009","unstructured":"Macdonald C, Ounis I (2009) Usefulness of quality click-through data for training. In: Craswell N, Jones R, Dupret G, Viegas E (eds) Proceedings of the 2009 workshop on Web search click data (WSCD 2009). ACM, New York, pp 75\u201379"},{"key":"584_CR22","doi-asserted-by":"crossref","unstructured":"Morrison D, Marchand-Maillet S, Bruno E (2009) TagCaptcha: annotating images with CAPTCHAs. In: Bennett PN, Chandrasekar R, Chickering M, Ipeirotis PG, Law E, Mityagin A, Provost FJ, von Ahn L (eds) Proceedings of the ACM SIGKDD workshop on human computation. ACM Press, pp 44\u201345","DOI":"10.1145\/1600150.1600166"},{"key":"584_CR23","unstructured":"Palomino MA, Oakes MP, Wuytack T (2009) Automatic extraction of keywords for a multimedia search engine using the chi-square test. In: Proceedings of the 9th Dutch\u2013Belgian information retrieval workshop (DIR 2009), pp 3\u201310"},{"key":"584_CR24","doi-asserted-by":"crossref","unstructured":"Poblete B, Baeza-Yates RA (2008) Query-sets: using implicit feedback and query patterns to organize Web documents. In: Huai J, Chen R, Hon HW, Liu Y, Ma WY, Tomkins A, Zhang X (eds) Proceedings of the 17th international conference on World Wide Web, pp 41\u201350","DOI":"10.1145\/1367497.1367504"},{"key":"584_CR25","doi-asserted-by":"crossref","unstructured":"Scholer F, Shokouhi M, Billerbeck B, Turpin A (2008) Using clicks as implicit judgments: expectations versus observations. In: Boughanem M, Berrut C, Mothe J, Soul\u00e9-Dupuy C (eds) Proceedings of the 30th European conference on IR research, pp 28\u201339","DOI":"10.1007\/978-3-540-78646-7_6"},{"key":"584_CR26","doi-asserted-by":"crossref","unstructured":"Setz AT, Snoek CGM (2009) Can social tagged images aid concept-based video search? In: Proceedings of the IEEE international conference on multimedia & expo (ICME 2009), pp 1460\u20131463","DOI":"10.1109\/ICME.2009.5202778"},{"key":"584_CR27","doi-asserted-by":"crossref","unstructured":"Sigurbj\u00f6rnsson B, van Zwol R (2008) Flickr tag recommendation based on collective knowledge. In: Huai J, Chen R, Hon HW, Liu Y, Ma WY, Tomkins A, Zhang X (eds) Proceedings of the 17th international conference on World Wide Web, pp 327\u2013336","DOI":"10.1145\/1367497.1367542"},{"key":"584_CR28","unstructured":"Smith G, Ashman H (2009) Evaluating implicit judgements from image search interactions. In: Proceedings of the Web science conference: society on-line (WebSci 2009)"},{"key":"584_CR29","unstructured":"Snoek CGM, Worring M, van Gemert JC, Geusebroek JM, Smeulders AWM (2004) The challenge problem for automated detection of 101 semantic concepts in multimedia. In: Proceedings of the 14th ACM international conference on multimedia, pp 421\u2013430"},{"key":"584_CR30","doi-asserted-by":"crossref","unstructured":"Tsikrika T, Diou C, de\u00a0Vries AP, Delopoulos A (2009) Image annotation using clickthrough data. In: Marchand-Maillet S, Kompatsiaris Y (eds) Proceedings of the 8th international conference on content-based image and video retrieval (CIVR 2009). ACM Press","DOI":"10.1145\/1646396.1646415"},{"key":"584_CR31","unstructured":"Ulges A, Koch M, Schulze C, Breuel T (2008) Learning TRECVID\u201908 high-level features from YouTubeTM. In: Proceedings of TRECVID 2008"},{"key":"584_CR32","doi-asserted-by":"crossref","unstructured":"Ulges A, Schulze C, Keysers D, Breuel TM (2008) Identifying relevant frames in weakly labeled videos for training concept detectors. In: Luo J, Guan L, Hanjalic A, Kankanhalli MS, Lee I (eds) Proceedings of the 7th international conference on content-based image and video retrieval (CIVR 2008). ACM Press, pp 9\u201316","DOI":"10.1145\/1386352.1386358"},{"key":"584_CR33","doi-asserted-by":"crossref","unstructured":"Ulges A, Schulze C, Keysers D, Breuel TM (2008) A system that learns to tag videos by watching YouTube. In: Gasteratos A, Vincze M, Tsotsos JK (eds) Proceedings of the 6th international conference of computer vision systems (ICVS 2008). Lecture Notes in Computer Science, vol 5008. Springer, pp 415\u2013424","DOI":"10.1007\/978-3-540-79547-6_40"},{"issue":"2","key":"584_CR34","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1145\/966389.966390","volume":"47","author":"L Ahn von","year":"2004","unstructured":"von Ahn L, Blum M, Langford J (2004) Telling humans and computers apart automatically. Commun ACM 47(2):56\u201360","journal-title":"Commun ACM"},{"key":"584_CR35","unstructured":"von Ahn L, Dabbish L (2004) Labeling images with a computer game. In: Proceedings of the ACM SIGCHI conference on human factors in computing systems (CHI 2004). ACM Press, pp 319\u2013326"},{"issue":"8","key":"584_CR36","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1145\/1378704.1378719","volume":"51","author":"L Ahn von","year":"2008","unstructured":"von Ahn L, Dabbish L (2008) Designing games with a purpose. Commun ACM 51(8):58\u201367","journal-title":"Commun ACM"},{"key":"584_CR37","doi-asserted-by":"crossref","unstructured":"von Ahn L, Liu R, Blum M (2006) Peekaboom: a game for locating objects in images. In: Grinter RE, Rodden T, Aoki PM, Cutrell E, Jeffries R, Olson GM (eds) Proceedings of the ACM SIGCHI conference on human factors in computing systems (CHI 2006). ACM Press, pp 55\u201364","DOI":"10.1145\/1124772.1124782"},{"issue":"5895","key":"584_CR38","doi-asserted-by":"crossref","first-page":"1465","DOI":"10.1126\/science.1160379","volume":"321","author":"L Ahn von","year":"2008","unstructured":"von Ahn L, Maurer B, Mcmillen C, Abraham D, Blum M (2008) reCAPTCHA: Human-based character recognition via web security measures. Science 321(5895):1465\u20131468","journal-title":"Science"},{"key":"584_CR39","doi-asserted-by":"crossref","unstructured":"Yang J, Hauptmann AG (2008) (Un)Reliability of video concept detection. In: Luo J, Guan L, Hanjalic A, Kankanhalli MS, Lee I (eds) Proceedings of the 7th international conference on content-based image and video retrieval (CIVR 2008). ACM Press, pp 85\u201394","DOI":"10.1145\/1386352.1386367"}],"container-title":["Multimedia Tools and Applications"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-010-0584-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11042-010-0584-1\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11042-010-0584-1","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,6,1]],"date-time":"2019-06-01T23:31:23Z","timestamp":1559431883000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11042-010-0584-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2010,8,17]]},"references-count":39,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2011,10]]}},"alternative-id":["584"],"URL":"https:\/\/doi.org\/10.1007\/s11042-010-0584-1","relation":{},"ISSN":["1380-7501","1573-7721"],"issn-type":[{"value":"1380-7501","type":"print"},{"value":"1573-7721","type":"electronic"}],"subject":[],"published":{"date-parts":[[2010,8,17]]}}}