{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,16]],"date-time":"2026-01-16T18:51:33Z","timestamp":1768589493737,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,10,17]],"date-time":"2018-10-17T00:00:00Z","timestamp":1539734400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"QROWD - Because Big Data Integration is Humanly Possible","award":["732194"],"award-info":[{"award-number":["732194"]}]},{"name":"IBM PhD Fellowship"},{"name":"STARS4ALL - A Collective Awareness Platform for Promoting Dark Skies in Europe","award":["688135"],"award-info":[{"award-number":["688135"]}]},{"name":"Google Faculty Research Award"},{"name":"Bloomberg Research Grant"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,10,17]]},"DOI":"10.1145\/3269206.3271779","type":"proceedings-article","created":{"date-parts":[[2018,10,22]],"date-time":"2018-10-22T12:08:27Z","timestamp":1540210107000},"page":"1253-1262","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Studying Topical Relevance with Evidence-based Crowdsourcing"],"prefix":"10.1145","author":[{"given":"Oana","family":"Inel","sequence":"first","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}]},{"given":"Giannis","family":"Haralabopoulos","sequence":"additional","affiliation":[{"name":"University of Nottingham, Nottingham, England UK"}]},{"given":"Dan","family":"Li","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Christophe","family":"Van Gysel","sequence":"additional","affiliation":[{"name":"Apple Inc., Cupertino, CA, USA"}]},{"given":"Zolt\u00e1n","family":"Szl\u00e1vik","sequence":"additional","affiliation":[{"name":"IBM CAS Benelux, Amsterdam, Netherlands"}]},{"given":"Elena","family":"Simperl","sequence":"additional","affiliation":[{"name":"University of Southampton, Southampton, England UK"}]},{"given":"Evangelos","family":"Kanoulas","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, Netherlands"}]},{"given":"Lora","family":"Aroyo","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2018,10,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390450"},{"key":"e_1_3_2_1_2_1","volume-title":"TREC","volume":"13","author":"Allan J.","year":"2004","unstructured":"J. Allan . Overview of the trec 2004 robust retrieval track . In TREC , volume 13 , 2004 . J. Allan. Overview of the trec 2004 robust retrieval track. In TREC, volume 13, 2004."},{"key":"e_1_3_2_1_3_1","volume-title":"TREC","author":"Allan J.","year":"2017","unstructured":"J. Allan , E. K. Donna Harman , D. Li , C. V. Gysel , and E. Vorhees . Trec 2017 common core track overview . In TREC , 2017 . J. Allan, E. K. Donna Harman, D. Li, C. V. Gysel, and E. Vorhees. Trec 2017 common core track overview. In TREC, 2017."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.5555\/1996889.1996910"},{"key":"e_1_3_2_1_5_1","first-page":"16","volume-title":"SIGIR Workshop on the Future of IR Evaluation","volume":"15","author":"Alonso O.","year":"2009","unstructured":"O. Alonso and S. Mizzaro . Can we get rid of trec assessors? using mechanical turk for relevance assessment . In SIGIR Workshop on the Future of IR Evaluation , volume 15 , page 16 , 2009 . O. Alonso and S. Mizzaro. Can we get rid of trec assessors? using mechanical turk for relevance assessment. In SIGIR Workshop on the Future of IR Evaluation, volume 15, page 16, 2009."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2012.01.004"},{"key":"e_1_3_2_1_7_1","volume-title":"ACM Web Science","author":"Aroyo L.","year":"2013","unstructured":"L. Aroyo and C. Welty . Crowd Truth: Harnessing disagreement in crowdsourcing a relation extraction gold standard . ACM Web Science , 2013 . L. Aroyo and C. Welty. Crowd Truth: Harnessing disagreement in crowdsourcing a relation extraction gold standard. ACM Web Science, 2013."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.15346\/hc.v1i1.3"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080763"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.5555\/188490.188589"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277754"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2012.95"},{"key":"e_1_3_2_1_14_1","volume-title":"Trec 2014 web track overview. Techn. report","author":"Collins-Thompson K.","year":"2015","unstructured":"K. Collins-Thompson , C. Macdonald , P. Bennett , F. Diaz , and E. M. Voorhees . Trec 2014 web track overview. Techn. report , University of Michigan , Ann Arbor , 2015 . K. Collins-Thompson, C. Macdonald, P. Bennett, F. Diaz, and E. M. Voorhees. Trec 2014 web track overview. Techn. report, University of Michigan, Ann Arbor, 2015."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080729"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3152889"},{"key":"e_1_3_2_1_17_1","volume-title":"Special Issue on Human Computation and Crowdsourcing (in review)","author":"Dumitrache A.","year":"2017","unstructured":"Dumitrache, Inel, Timmermans, Ortiz, Sips, and Aroyo}swj A. Dumitrache , O. Inel , B. Timmermans , C. Ortiz , R.-J. Sips , and L. Aroyo . Empirical methodology for crowdsourcing ground truth. Semantic Web Journal , Special Issue on Human Computation and Crowdsourcing (in review) , 2017 b . Dumitrache, Inel, Timmermans, Ortiz, Sips, and Aroyo}swjA. Dumitrache, O. Inel, B. Timmermans, C. Ortiz, R.-J. Sips, and L. Aroyo. Empirical methodology for crowdsourcing ground truth. Semantic Web Journal, Special Issue on Human Computation and Crowdsourcing (in review), 2017 b ."},{"key":"e_1_3_2_1_18_1","volume-title":"Crowdtruth 2.0: Quality metrics for crowdsourcing with disagreement","author":"Dumitrache A.","year":"2018","unstructured":"A. Dumitrache , O. Inel , L. Aroyo , B. Timmermans , and C. Welty . Crowdtruth 2.0: Quality metrics for crowdsourcing with disagreement . 2018 . A. Dumitrache, O. Inel, L. Aroyo, B. Timmermans, and C. Welty. Crowdtruth 2.0: Quality metrics for crowdsourcing with disagreement. 2018."},{"key":"e_1_3_2_1_19_1","first-page":"172","volume-title":"NAACL HLT Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk","author":"Grady C.","year":"2010","unstructured":"C. Grady and M. Lease . Crowdsourcing document relevance assessment with mechanical turk . In NAACL HLT Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk , pages 172 -- 179 , 2010 . C. Grady and M. Lease. Crowdsourcing document relevance assessment with mechanical turk. In NAACL HLT Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk, pages 172--179, 2010."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02769550"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3002172"},{"key":"e_1_3_2_1_22_1","volume-title":"HCOMP, page 10","author":"McDonnell T.","year":"2016","unstructured":"T. McDonnell , M. Lease , T. Elsayad , and M. Kutlu . Why is that relevant? collecting annotator rationales for relevance judgments . In HCOMP, page 10 , 2016 . T. McDonnell, M. Lease, T. Elsayad, and M. Kutlu. Why is that relevant? collecting annotator rationales for relevance judgments. In HCOMP, page 10, 2016."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02295996"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.joi.2012.12.001"},{"issue":"4","key":"e_1_3_2_1_25_1","first-page":"247","article-title":"Test collection based evaluation of information retrieval systems","volume":"4","author":"Sanderson M.","year":"2010","unstructured":"M. Sanderson Test collection based evaluation of information retrieval systems . FNTIR , 4 ( 4 ): 247 -- 375 , 2010 . M. Sanderson et al. Test collection based evaluation of information retrieval systems. FNTIR, 4 (4): 247--375, 2010.","journal-title":"FNTIR"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.5555\/1613715.1613751"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026526"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/297427.297450"},{"key":"e_1_3_2_1_30_1","first-page":"49","volume-title":"SIGIR Workshop on Focused Retrieval","author":"Trotman A.","year":"2007","unstructured":"A. Trotman , N. Pharo , and D. Jenkinson . Can we at least agree on something ? In SIGIR Workshop on Focused Retrieval , pages 49 -- 56 , 2007 . A. Trotman, N. Pharo, and D. Jenkinson. Can we at least agree on something? In SIGIR Workshop on Focused Retrieval, pages 49--56, 2007."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_1_32_1","first-page":"355","volume-title":"CLEF","volume":"1","author":"Voorhees E. M.","year":"2001","unstructured":"E. M. Voorhees . The philosophy of information retrieval evaluation . In CLEF , volume 1 , pages 355 -- 370 . Springer , 2001 . E. M. Voorhees. The philosophy of information retrieval evaluation. In CLEF, volume 1, pages 355--370. Springer, 2001."},{"key":"e_1_3_2_1_33_1","volume-title":"TREC","author":"Voorhees E. M.","year":"2001","unstructured":"E. M. Voorhees and D. Harman . Overview of trec 2001 . In TREC , 2001 . E. M. Voorhees and D. Harman. Overview of trec 2001. In TREC, 2001."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564389"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"}],"event":{"name":"CIKM '18: The 27th ACM International Conference on Information and Knowledge Management","location":"Torino Italy","acronym":"CIKM '18","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 27th ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3269206.3271779","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3269206.3271779","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T00:44:29Z","timestamp":1750207469000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3269206.3271779"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,10,17]]},"references-count":35,"alternative-id":["10.1145\/3269206.3271779","10.1145\/3269206"],"URL":"https:\/\/doi.org\/10.1145\/3269206.3271779","relation":{},"subject":[],"published":{"date-parts":[[2018,10,17]]},"assertion":[{"value":"2018-10-17","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}