{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,17]],"date-time":"2026-04-17T02:35:05Z","timestamp":1776393305563,"version":"3.51.2"},"publisher-location":"New York, NY, USA","reference-count":28,"publisher":"ACM","license":[{"start":{"date-parts":[[2010,3,29]],"date-time":"2010-03-29T00:00:00Z","timestamp":1269820800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2010,3,29]]},"DOI":"10.1145\/1743384.1743478","type":"proceedings-article","created":{"date-parts":[[2010,3,30]],"date-time":"2010-03-30T12:32:28Z","timestamp":1269952348000},"page":"557-566","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":210,"title":["How reliable are annotations via crowdsourcing"],"prefix":"10.1145","author":[{"given":"Stefanie","family":"Nowak","sequence":"first","affiliation":[{"name":"Fraunhofer IDMT, Ilmenau, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefan","family":"R\u00fcger","sequence":"additional","affiliation":[{"name":"Open University, Milton Keynes, England UK"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2010,3,29]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"SIGIR 2009 Workshop on the Future of IR Evaluation.","author":"Alonso O.","unstructured":"O. Alonso and S. Mizzaro . Can we get rid of TREC assessors? Using Mechanical Turk for relevance assessment . In SIGIR 2009 Workshop on the Future of IR Evaluation. O. Alonso and S. Mizzaro. Can we get rid of TREC assessors? Using Mechanical Turk for relevance assessment. In SIGIR 2009 Workshop on the Future of IR Evaluation."},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. of the 2nd Intern. Conf. on Language Resources and Evaluation","author":"Brants T.","year":"2000","unstructured":"T. Brants . Inter-annotator agreement for a German newspaper corpus . In Proc. of the 2nd Intern. Conf. on Language Resources and Evaluation , 2000 . T. Brants. Inter-annotator agreement for a German newspaper corpus. In Proc. of the 2nd Intern. Conf. on Language Resources and Evaluation, 2000."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316448104100307"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1631272.1631339"},{"key":"e_1_3_2_1_5_1","volume-title":"Proceedings of RANLP 2003","author":"Chklovski T.","year":"2003","unstructured":"T. Chklovski and R. Mihalcea . Exploiting agreement and disagreement of human annotators for word sense disambiguation . In Proceedings of RANLP 2003 , 2003 . T. Chklovski and R. Mihalcea. Exploiting agreement and disagreement of human annotators for word sense disambiguation. In Proceedings of RANLP 2003, 2003."},{"key":"e_1_3_2_1_6_1","volume-title":"A coefficient of agreement for nominal scales. Educational and psychological measurement, 20(1):37--46","author":"Cohen J.","year":"1960","unstructured":"J. Cohen A coefficient of agreement for nominal scales. Educational and psychological measurement, 20(1):37--46 , 1960 . J. Cohen et al. A coefficient of agreement for nominal scales. Educational and psychological measurement, 20(1):37--46, 1960."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1557019.1557053"},{"key":"e_1_3_2_1_9_1","volume-title":"Acquiring High Quality Non-Expert Knowledge from On-demand Workforce. ACL-IJCNLP","author":"Feng D.","year":"2009","unstructured":"D. Feng and S. Zajac . Acquiring High Quality Non-Expert Knowledge from On-demand Workforce. ACL-IJCNLP 2009 . D. Feng and S. Zajac. Acquiring High Quality Non-Expert Knowledge from On-demand Workforce. ACL-IJCNLP 2009."},{"key":"e_1_3_2_1_10_1","volume-title":"http:\/\/crowdsourcing.typepad.com\/cs\/2006\/06\/, last accessed 24.11.2009","author":"Howe J.","year":"2006","unstructured":"J. Howe . Crowdsourcing - A Definition . http:\/\/crowdsourcing.typepad.com\/cs\/2006\/06\/, last accessed 24.11.2009 , 2006 . J. Howe. Crowdsourcing - A Definition. http:\/\/crowdsourcing.typepad.com\/cs\/2006\/06\/, last accessed 24.11.2009, 2006."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.5555\/1564131.1564137"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1460096.1460104"},{"key":"e_1_3_2_1_13_1","volume-title":"SIGIR 2009 Workshop on the Future of IR Evaluation.","author":"Kazai G.","unstructured":"G. Kazai and N. Milic-Frayling . On the Evaluation of the Quality of Relevance Assessments Collected through Crowdsourcing . In SIGIR 2009 Workshop on the Future of IR Evaluation. G. Kazai and N. Milic-Frayling. On the Evaluation of the Quality of Relevance Assessments Collected through Crowdsourcing. In SIGIR 2009 Workshop on the Future of IR Evaluation."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1006\/csla.1998.0108"},{"issue":"1","key":"e_1_3_2_1_16_1","first-page":"83","article-title":"Sulla determinazione empirica di una legge di distribuzione","volume":"4","author":"Kolmogoroff A.","year":"1933","unstructured":"A. Kolmogoroff . Sulla determinazione empirica di una legge di distribuzione . Giorn. Ist. Ital. Attuari , 4 ( 1 ): 83 -- 91 , 1933 . A. Kolmogoroff. Sulla determinazione empirica di una legge di distribuzione. Giorn. Ist. Ital. Attuari, 4(1):83--91, 1933.","journal-title":"Giorn. Ist. Ital. Attuari"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.2307\/2529310"},{"key":"e_1_3_2_1_18_1","volume-title":"THESEUS-ImageCLEF Pre-Workshop 2009","author":"Nowak S.","year":"2009","unstructured":"S. Nowak and P. Dunker . A Consumer Photo Tagging Ontology: Concepts and Annotations . In THESEUS-ImageCLEF Pre-Workshop 2009 , Corfu, Greece , 2009 . S. Nowak and P. Dunker. A Consumer Photo Tagging Ontology: Concepts and Annotations. In THESEUS-ImageCLEF Pre-Workshop 2009, Corfu, Greece, 2009."},{"key":"e_1_3_2_1_19_1","volume-title":"Overview of the CLEF 2009 Large-Scale Visual Concept Detection and Annotation Task. CLEF working notes 2009","author":"Nowak S.","year":"2009","unstructured":"S. Nowak and P. Dunker . Overview of the CLEF 2009 Large-Scale Visual Concept Detection and Annotation Task. CLEF working notes 2009 , Corfu, Greece , 2009 . S. Nowak and P. Dunker. Overview of the CLEF 2009 Large-Scale Visual Concept Detection and Annotation Task. CLEF working notes 2009, Corfu, Greece, 2009."},{"key":"e_1_3_2_1_20_1","volume-title":"Proc. of IRMLeS Workshop, ESWC","author":"Nowak S.","year":"2009","unstructured":"S. Nowak and H. Lukashevich . Multilabel Classification Evaluation using Ontology Information . In Proc. of IRMLeS Workshop, ESWC , Greece , 2009 . S. Nowak and H. Lukashevich. Multilabel Classification Evaluation using Ontology Information. In Proc. of IRMLeS Workshop, ESWC, Greece, 2009."},{"key":"e_1_3_2_1_21_1","volume-title":"Joensuu Learning and Instruction Symposium","author":"Randolph J.","year":"2005","unstructured":"J. Randolph . Free-Marginal Multirater Kappa (multirater \u03ba free): An Alternative to Fleiss' Fixed-Marginal Multirater Kappa . In Joensuu Learning and Instruction Symposium , 2005 . J. Randolph. Free-Marginal Multirater Kappa (multirater \u03ba free): An Alternative to Fleiss' Fixed-Marginal Multirater Kappa. In Joensuu Learning and Instruction Symposium, 2005."},{"key":"e_1_3_2_1_22_1","unstructured":"J. Randolph. Online Kappa Calculator. http:\/\/justus.randolph.name\/kappa 2008.  J. Randolph. Online Kappa Calculator. http:\/\/justus.randolph.name\/kappa 2008."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1401890.1401965"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"e_1_3_2_1_26_1","volume-title":"Programme and advanced papers of the Senseval workshop","author":"V\u00e9ronis J.","year":"1998","unstructured":"J. V\u00e9ronis . A study of polysemy judgements and inter-annotator agreement . In Programme and advanced papers of the Senseval workshop , 1998 . J. V\u00e9ronis. A study of polysemy judgements and inter-annotator agreement. In Programme and advanced papers of the Senseval workshop, 1998."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2007.900153"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"}],"event":{"name":"MIR '10: International Conference on Multimedia Information Retrieval","location":"Philadelphia Pennsylvania USA","acronym":"MIR '10","sponsor":["SIGMM ACM Special Interest Group on Multimedia"]},"container-title":["Proceedings of the international conference on Multimedia information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1743384.1743478","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1743384.1743478","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T11:23:19Z","timestamp":1750245799000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1743384.1743478"}},"subtitle":["a study about inter-annotator agreement for multi-label image annotation"],"short-title":[],"issued":{"date-parts":[[2010,3,29]]},"references-count":28,"alternative-id":["10.1145\/1743384.1743478","10.1145\/1743384"],"URL":"https:\/\/doi.org\/10.1145\/1743384.1743478","relation":{},"subject":[],"published":{"date-parts":[[2010,3,29]]},"assertion":[{"value":"2010-03-29","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}