{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,25]],"date-time":"2025-10-25T14:18:01Z","timestamp":1761401881655,"version":"3.37.3"},"reference-count":35,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,7,3]],"date-time":"2017-07-03T00:00:00Z","timestamp":1499040000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100004963","name":"Seventh Framework Programme","doi-asserted-by":"publisher","award":["296170","611396"],"award-info":[{"award-number":["296170","611396"]}],"id":[{"id":"10.13039\/501100004963","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Government of Spain","award":["TIN2014-54288-C4-3-R"],"award-info":[{"award-number":["TIN2014-54288-C4-3-R"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Lang Resources & Evaluation"],"published-print":{"date-parts":[[2018,3]]},"DOI":"10.1007\/s10579-017-9396-5","type":"journal-article","created":{"date-parts":[[2017,7,3]],"date-time":"2017-07-03T09:13:56Z","timestamp":1499073236000},"page":"341-364","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Cross-language transfer of semantic annotation via targeted crowdsourcing: task design and evaluation"],"prefix":"10.1007","volume":"52","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5917-9688","authenticated-orcid":false,"given":"Evgeny A.","family":"Stepanov","sequence":"first","affiliation":[]},{"given":"Shammur Absar","family":"Chowdhury","sequence":"additional","affiliation":[]},{"given":"Ali Orkan","family":"Bayer","sequence":"additional","affiliation":[]},{"given":"Arindam","family":"Ghosh","sequence":"additional","affiliation":[]},{"given":"Ioannis","family":"Klasinas","sequence":"additional","affiliation":[]},{"given":"Marcos","family":"Calvo","sequence":"additional","affiliation":[]},{"given":"Emilio","family":"Sanchis","sequence":"additional","affiliation":[]},{"given":"Giuseppe","family":"Riccardi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,7,3]]},"reference":[{"issue":"2","key":"9396_CR1","doi-asserted-by":"crossref","first-page":"76","DOI":"10.1109\/MIC.2013.20","volume":"17","author":"M Allahbakhsh","year":"2013","unstructured":"Allahbakhsh, M., Benatallah, B., Ignjatovic, A., Motahari-Nezhad, H., Bertino, E., & Dustdar, S. (2013). Quality control in crowdsourcing systems. IEEE Internet Computing, 17(2), 76\u201381.","journal-title":"IEEE Internet Computing"},{"key":"9396_CR2","doi-asserted-by":"crossref","unstructured":"Bayer, A. O., & Riccardi, G. (2012). Joint language models for automatic speech recognition and understanding. In Proceeding of the IEEE spoken language technology workshop.","DOI":"10.1109\/SLT.2012.6424222"},{"key":"9396_CR3","doi-asserted-by":"crossref","unstructured":"Bentivogli, L., Forner, P., & Pianta, E. (2004). Evaluating cross-language annotation transfer in the multisemcor corpus. In Proceedings of the 20th international conference on computational linguistics, association for computational linguistics.","DOI":"10.3115\/1220355.1220408"},{"key":"9396_CR4","doi-asserted-by":"crossref","first-page":"86","DOI":"10.1016\/j.csl.2016.01.002","volume":"38","author":"M Calvo","year":"2016","unstructured":"Calvo, M., Hurtado, L. F., Garcia, F., Sanchis, E., & Segarra, E. (2016). Multilingual spoken language understanding using graphs and multiple translations. Computer Speech and Language, 38, 86\u2013103.","journal-title":"Computer Speech and Language"},{"key":"9396_CR5","unstructured":"Chowdhury, S. A., Calvo, M., Ghosh, A., Stepanov, E. A., Bayer, A. O., Riccardi, G., et al. (2015). Selection and aggregation techniques for crowdsourced semantic annotation task. In The 16th annual conference of the international speech communication association (INTERSPEECH) (pp. 2779\u20132783). Dresden: ISCA."},{"key":"9396_CR6","unstructured":"Chowdhury, S. A., Ghosh, A., Stepanov, E. A., Bayer, A. O., Riccardi, G., & Klasinas, I. (2014). Cross-language transfer of semantic annotation via targeted crowdsourcing. In The 15th annual conference of the international speech communication association (INTERSPEECH) (pp. 2108\u20132112). Singapore: ISCA."},{"key":"9396_CR7","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J. (1960). A coefficient of agreement for nominal scales. Educational and Psychological Measurement, 20, 37\u201346.","journal-title":"Educational and Psychological Measurement"},{"issue":"3","key":"9396_CR8","doi-asserted-by":"crossref","first-page":"297","DOI":"10.2307\/1932409","volume":"26","author":"LR Dice","year":"1945","unstructured":"Dice, L. R. (1945). Measures of the amount of ecologic association between species. Ecology, 26(3), 297\u2013302.","journal-title":"Ecology"},{"key":"9396_CR9","doi-asserted-by":"crossref","unstructured":"Dinarelli, M., Quarteroni, S., Tonelli, S., Moschitti, A., & Riccardi, G. (2009). Annotating spoken dialogs: From speech segments to dialog acts and frame semantics. In Proceedings of EACL workshop on the semantic representation of spoken language. Athens.","DOI":"10.3115\/1626296.1626301"},{"key":"9396_CR10","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J., & Dredze, M. (2010). Annotating named entities in twitter data with crowdsourcing. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk, association for computational linguistics (pp. 80\u201388)."},{"issue":"5","key":"9396_CR11","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss, J. L. (1971). Measuring nominal scale agreement among many raters. Psychological Bulletin, 76(5), 378\u2013382.","journal-title":"Psychological Bulletin"},{"key":"9396_CR12","doi-asserted-by":"crossref","first-page":"651","DOI":"10.2307\/2529549","volume":"31","author":"JL Fleiss","year":"1975","unstructured":"Fleiss, J. L. (1975). Measuring agreement between two judges on the presence or absence of a trait. Biometrics, 31, 651\u2013659.","journal-title":"Biometrics"},{"issue":"2","key":"9396_CR13","doi-asserted-by":"crossref","first-page":"413","DOI":"10.1162\/COLI_a_00057","volume":"37","author":"K Fort","year":"2011","unstructured":"Fort, K., Adda, G., & Cohen, K. B. (2011). Amazon mechanical turk: Gold mine or coal mine? Computational Linguistics, 37(2), 413\u2013420.","journal-title":"Computational Linguistics"},{"key":"9396_CR14","unstructured":"Gonz\u00e0lez, M., Mateva, M., Enache, R., Na, C. E., Arquez, L. M., Popov, B., & Ranta, A. (2013). MT techniques in a retrieval system of semantically enriched patents. In MT Summit."},{"issue":"3","key":"9396_CR15","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1197\/jamia.M1733","volume":"12","author":"G Hripcsak","year":"2005","unstructured":"Hripcsak, G., & Rothschild, A. S. (2005). Agreement, the f-measure, and reliability in information retrieval. Journal of the American Medical Informatics Association, 12(3), 296\u2013298.","journal-title":"Journal of the American Medical Informatics Association"},{"key":"9396_CR16","unstructured":"Hwa, R., Resnik, P., Weinberg, A., & Kolak, O. (2002). Evaluating translational correspondence using annotation projection. In Proceedings of the 40th annual meeting on association for computational linguistics, association for computational linguistics (pp. 392\u2013399)."},{"key":"9396_CR17","doi-asserted-by":"crossref","unstructured":"Jabaian, B., Besacier, L., & Lef\u00e8vre, F. (2010). Investigating multiple approaches for SLU portability to a new language. In Proceedings of INTERSPEECH.","DOI":"10.21437\/Interspeech.2010-672"},{"key":"9396_CR18","doi-asserted-by":"crossref","unstructured":"Jabaian, B., Besacier, L., & Lef\u00e8vre, F. (2011). Combination of stochastic understanding and machine translation systems for language portability of dialogue systems. In Proceedings of the IEEE international conference on acoustics, speech, and signal processing (ICASSP).","DOI":"10.1109\/ICASSP.2011.5947632"},{"issue":"3","key":"9396_CR19","doi-asserted-by":"crossref","first-page":"636","DOI":"10.1109\/TASL.2012.2229983","volume":"21","author":"B Jabaian","year":"2013","unstructured":"Jabaian, B., Besacier, L., & Lef\u00e8vre, F. (2013). Comparison and combination of lightly supervised approaches for language portability of a spoken language understanding system. IEEE Transactions on Audio, Speech, and Language Processing, 21(3), 636\u2013648.","journal-title":"IEEE Transactions on Audio, Speech, and Language Processing"},{"key":"9396_CR20","unstructured":"Johansson, R., & Moschitti, A. (2010). Syntactic and semantic structure for opinion expression detection. In Proceedings of the 40th conference on computational natural language learning (pp. 67\u201376)."},{"key":"9396_CR21","unstructured":"Lawson, N., Eustice, K., Perkowitz, M., & Yetisgen-Yildiz, M. (2010). Annotating large email datasets for named entity recognition with mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with amazon\u2019s mechanical turk, association for computational linguistics (pp. 71\u201379)."},{"key":"9396_CR22","doi-asserted-by":"crossref","unstructured":"Lef\u00e8vre, F., Mairesse, F., & Young, S. (2010). Cross-lingual spoken language understanding from unaligned data using discriminative classification models and machine translation. In Proceedings of INTERSPEECH.","DOI":"10.21437\/Interspeech.2010-38"},{"key":"9396_CR23","doi-asserted-by":"crossref","unstructured":"Marge, M., Banerjee, S., & Rudnicky, A. I. (2010). Using the amazon mechanical turk for transcription of spoken language. In 2010 IEEE international conference on acoustics speech and signal processing (ICASSP) (pp. 5270\u20135273). IEEE.","DOI":"10.1109\/ICASSP.2010.5494979"},{"issue":"1","key":"9396_CR24","doi-asserted-by":"crossref","first-page":"307","DOI":"10.1613\/jair.2863","volume":"36","author":"S Pad\u00f3","year":"2009","unstructured":"Pad\u00f3, S., & Lapata, M. (2009). Cross-lingual annotation projection for semantic roles. Journal of Artificial Intelligence Research, 36(1), 307\u2013340.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"9396_CR25","doi-asserted-by":"crossref","unstructured":"Parent, G., & Eskenazi, M. (2010). Toward better crowdsourced transcription: Transcription of a year of the let\u2019s go bus information system data. In IEEE spoken language technology workshop (SLT) (pp. 312\u2013317). IEEE.","DOI":"10.1109\/SLT.2010.5700870"},{"key":"9396_CR26","unstructured":"Pustejovsky, J., & Rumshisky, A. (2014). Deep semantic annotation with shallow methods. LREC 2014 Tutorial."},{"key":"9396_CR27","unstructured":"Rigo, S., Stepanov, E. A., Roberti, P., Quarteroni, S., & Riccardi, G. (2009). The 2009 UNITN EVALITA Italian spoken dialogue system. In Evaluation of NLP and speech tools for Italian workshop (EVALITA). Reggio Emilia."},{"key":"9396_CR28","doi-asserted-by":"crossref","unstructured":"Riloff, E., Schafer, C., & Yarowsky, D. (2002). Inducing information extraction systems for new languages via cross-language projection. In: Proceedings of the 19th international conference on computational linguistics\u2014Volume 1, association for computational linguistics (pp. 1\u20137).","DOI":"10.3115\/1072228.1072298"},{"key":"9396_CR29","unstructured":"Ross, J., Zaldivar, A., Irani, L., & Tomlinson, B. (2009). Who are the turkers? Worker demographics in amazon mechanical turk. Tech Rep. Irvine: Department of Informatics: University of California."},{"key":"9396_CR30","unstructured":"Spreyer, K., & Frank, A. (2008). Projection-based acquisition of a temporal labeller. In Proceedings of the international joint conference on natural language processing (pp. 489\u2013496)."},{"key":"9396_CR31","doi-asserted-by":"crossref","unstructured":"Stepanov, E. A., Kashkarev, I., Bayer, A. O., Riccardi, G., & Ghosh, A. (2013). Language style and domain adaptation for cross-language SLU porting. In IEEE workshop on automatic speech recognition and understanding (ASRU) (pp. 144\u2013149). Olomouc: IEEE.","DOI":"10.1109\/ASRU.2013.6707720"},{"key":"9396_CR32","unstructured":"Stepanov, E. A., Riccardi, G., & Bayer, A. O. (2014). The development of the multilingual LUNA corpus for spoken language system porting. In The 9th international conference on language resources and evaluation (LREC\u201914) (pp. 2675\u20132678). Reykjavik: ELRA."},{"key":"9396_CR33","doi-asserted-by":"crossref","unstructured":"Xi, C., & Hwa, R. (2005). A Backoff model for bootstrapping resources for non-english languages. In Proceedings of the conference on human language technology and empirical methods in natural language processing, association for computational linguistics (pp. 851\u2013858).","DOI":"10.3115\/1220575.1220682"},{"key":"9396_CR34","doi-asserted-by":"crossref","unstructured":"Yarowsky, D., Ngai, G., & Wicentowski, R. (2001). Inducing multilingual text analysis tools via robust projection across aligned corpora. In Proceedings of the 1st international conference on human language technology research, association for computational linguistics (pp. 1\u20138).","DOI":"10.3115\/1072133.1072187"},{"key":"9396_CR35","unstructured":"Zaidan, O. F., & Callison-Burch, C. (2011). Crowdsourcing translation: Professional quality from non-professionals. In Proceedings of the 49th annual meeting of the association for computational linguistics: human language technologies-volume 1, association for computational linguistics (pp. 1220\u20131229)."}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10579-017-9396-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-017-9396-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-017-9396-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,7,30]],"date-time":"2022-07-30T10:00:34Z","timestamp":1659175234000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10579-017-9396-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,7,3]]},"references-count":35,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2018,3]]}},"alternative-id":["9396"],"URL":"https:\/\/doi.org\/10.1007\/s10579-017-9396-5","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"type":"print","value":"1574-020X"},{"type":"electronic","value":"1574-0218"}],"subject":[],"published":{"date-parts":[[2017,7,3]]}}}