{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:09:30Z","timestamp":1750306170181,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":14,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,9,12]],"date-time":"2016-09-12T00:00:00Z","timestamp":1473638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DE140100275"],"award-info":[{"award-number":["DE140100275"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,9,12]]},"DOI":"10.1145\/2970398.2970431","type":"proceedings-article","created":{"date-parts":[[2016,9,9]],"date-time":"2016-09-09T14:28:57Z","timestamp":1473431337000},"page":"73-76","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["The Effect of Document Order and Topic Difficulty on Assessor Agreement"],"prefix":"10.1145","author":[{"given":"Tadele T.","family":"Damessie","sequence":"first","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]},{"given":"Kalvero","family":"J\u00e4rvelin","sequence":"additional","affiliation":[{"name":"University of Tampere, Tampere, Finland"}]},{"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}]}],"member":"320","published-online":{"date-parts":[[2016,9,12]]},"reference":[{"doi-asserted-by":"publisher","key":"e_1_3_2_1_1_1","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_2_1","volume-title":"TREC","author":"Carterette B.","year":"2009","unstructured":"B. Carterette , V. Pavlu , H. Fang , and E. Kanoulas . Million query track 2009 overview . In TREC , 2009 . B. Carterette, V. Pavlu, H. Fang, and E. Kanoulas. Million query track 2009 overview. In TREC, 2009."},{"key":"e_1_3_2_1_3_1","volume-title":"Experimental Studies of Relevance Judgments. Final Report. 3 Volumes","author":"Cuadra C. A.","year":"1967","unstructured":"C. A. Cuadra , R. V. Katter , E. H. Holmes , and E. M. Wallace . Experimental Studies of Relevance Judgments. Final Report. 3 Volumes . System Development Corporation , 1967 . C. A. Cuadra, R. V. Katter, E. H. Holmes, and E. M. Wallace. Experimental Studies of Relevance Judgments. Final Report. 3 Volumes. System Development Corporation, 1967."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_4_1","DOI":"10.1002\/asi.4630280507"},{"key":"e_1_3_2_1_5_1","volume-title":"Calculating inter-coder reliability in media content analysis using Krippendorff's alpha","author":"De Swert K.","year":"2012","unstructured":"K. De Swert . Calculating inter-coder reliability in media content analysis using Krippendorff's alpha . Center for Politics and Communication , 2012 . K. De Swert. Calculating inter-coder reliability in media content analysis using Krippendorff's alpha. Center for Politics and Communication, 2012."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_6_1","DOI":"10.1002\/(SICI)1097-4571(198809)39:5<293::AID-ASI1>3.0.CO;2-I"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_7_1","DOI":"10.1002\/asi.5090070408"},{"key":"e_1_3_2_1_8_1","volume-title":"Answering the call for a standard reliability measure for coding data. Communication methods and measures, 1 (1): 77--89","author":"Hayes A. F.","year":"2007","unstructured":"A. F. Hayes and K. Krippendorff . Answering the call for a standard reliability measure for coding data. Communication methods and measures, 1 (1): 77--89 , 2007 . A. F. Hayes and K. Krippendorff. Answering the call for a standard reliability measure for coding data. Communication methods and measures, 1 (1): 77--89, 2007."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_9_1","DOI":"10.1002\/asi.20047"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_10_1","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_11_1","volume-title":"I","author":"Rees A. M.","year":"1967","unstructured":"A. M. Rees and D. G. Schultz . A field experimental approach to the study of relevance assessments in relation to document searching. Final Report to the National Science Foundation , I , 1967 . A. M. Rees and D. G. Schultz. A field experimental approach to the study of relevance assessments in relation to document searching. Final Report to the National Science Foundation, I, 1967."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_12_1","DOI":"10.1002\/asi.20046"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_13_1","DOI":"10.1016\/S0306-4573(00)00010-8"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_14_1","DOI":"10.1002\/asi.v59:8"}],"event":{"sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"acronym":"ICTIR '16","name":"ICTIR '16: ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Newark Delaware USA"},"container-title":["Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2970398.2970431","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2970398.2970431","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:40:08Z","timestamp":1750218008000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2970398.2970431"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,9,12]]},"references-count":14,"alternative-id":["10.1145\/2970398.2970431","10.1145\/2970398"],"URL":"https:\/\/doi.org\/10.1145\/2970398.2970431","relation":{},"subject":[],"published":{"date-parts":[[2016,9,12]]},"assertion":[{"value":"2016-09-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}