{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T22:27:35Z","timestamp":1775255255470,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,10,17]],"date-time":"2018-10-17T00:00:00Z","timestamp":1539734400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100012537","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP170102231, DP170102726, DP180102687"],"award-info":[{"award-number":["DP170102231, DP170102726, DP180102687"]}],"id":[{"id":"10.13039\/100012537","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,10,17]]},"DOI":"10.1145\/3269206.3271750","type":"proceedings-article","created":{"date-parts":[[2018,10,22]],"date-time":"2018-10-22T08:08:27Z","timestamp":1540195707000},"page":"723-732","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Presentation Ordering Effects On Assessor Agreement"],"prefix":"10.1145","author":[{"given":"Tadele T.","family":"Damessie","sequence":"first","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jaewon","family":"Kim","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2018,10,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390450"},{"key":"e_1_3_2_1_2_1","volume-title":"Proc. ECIR . 153--164","author":"Alonso O."},{"key":"e_1_3_2_1_3_1","first-page":"411","article-title":"Structural equation modeling in practice: A review and recommended two-step approach. Psy","volume":"103","author":"Anderson J. C.","year":"1988","journal-title":"Bulletin"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_5_1","volume-title":"CLEF 2003--Overview of results. In Workshop of Cross-Lang. Eval. Forum on EU. Lang. Springer, 44--63","author":"Braschler M.","year":"2003"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1080\/01621459.1993.10594284","article-title":"Approximate inference in generalized linear mixed models","volume":"88","author":"Breslow N. E.","year":"1993","journal-title":"JASA"},{"key":"e_1_3_2_1_7_1","volume-title":"Proc. ECIR. 16--27","author":"Carterette B."},{"key":"e_1_3_2_1_8_1","volume-title":"Proc. TREC .","author":"Carterette B."},{"key":"e_1_3_2_1_9_1","volume-title":"Aslib proceedings","author":"Cleverdon C."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2854946.2854976"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609531"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080729"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3015022.3015033"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970398.2970431"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(198809)39:5<293::AID-ASI1>3.0.CO;2-I"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.sapharm.2012.04.004"},{"key":"e_1_3_2_1_17_1","volume-title":"Proc. CSLDAMT. Association for Computational Linguistics, 172--179","author":"Grady C."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.5555\/2031434"},{"key":"e_1_3_2_1_19_1","volume-title":"Proc. TREC .","author":"Hawking D.","year":"2000"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1080\/19312450709336664"},{"key":"e_1_3_2_1_21_1","unstructured":"J. J. Hox and T. M. Bechger. 2007. An introduction to structural equation modeling. (2007). J. J. Hox and T. M. Bechger. 2007. An introduction to structural equation modeling. (2007)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20047"},{"key":"e_1_3_2_1_23_1","volume-title":"Proc. NTCIR. 11--44","author":"Kando N."},{"key":"e_1_3_2_1_24_1","volume-title":"CLEF 2017 technologically assisted reviews in empirical medicine overview. In CEUR Workshop Proceedings","volume":"1866","author":"Kanoulas E."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(68)90002-8"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.5555\/1996889.1996911"},{"key":"e_1_3_2_1_27_1","volume-title":"Inf. Proc. & Man.","volume":"41","author":"J.","year":"2005"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijnurstu.2011.01.016"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277824"},{"key":"e_1_3_2_1_31_1","unstructured":"A. M. Rees and D. G. Schultz. 1967. A Field Experimental Approach to the Study of Relevance Assessments in Relation to Document Searching. Final Report to the NSF Vol. I (1967). A. M. Rees and D. G. Schultz. 1967. A Field Experimental Approach to the Study of Relevance Assessments in Relation to Document Searching. Final Report to the NSF Vol. I (1967)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551507086989"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2701336.2701636"},{"key":"e_1_3_2_1_34_1","volume-title":"Proc. NTCIR .","author":"Sakai T."},{"key":"e_1_3_2_1_35_1","volume-title":"Relevance Assessments: Revisited. In Proc. EVIA . 25--33.","author":"Sakai T.","year":"2010"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"M. Sanderson. 2010. Test collection based evaluation of information retrieval systems . M. Sanderson. 2010. Test collection based evaluation of information retrieval systems .","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.5555\/1315930.1315947"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484090"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-04769-5_5"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"M. D. Smucker G. Kazai and M. Lease. 2012. Overview of the trec 2012 crowdsourcing track . Technical Report. TEXAS UNIV AT AUSTIN SCHOOL OF INFORMATION. M. D. Smucker G. Kazai and M. Lease. 2012. Overview of the trec 2012 crowdsourcing track . Technical Report. TEXAS UNIV AT AUSTIN SCHOOL OF INFORMATION.","DOI":"10.6028\/NIST.SP.500-298.crowd-overview"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.5555\/262192.262203"},{"key":"e_1_3_2_1_43_1","volume-title":"Calculating inter-coder reliability in media content analysis using Krippendorff's Alpha","author":"Swert K. D.","year":"2012"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.5116\/ijme.4dfb.8dfd"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2011.04.005"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20046"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"e_1_3_2_1_48_1","volume-title":"TREC: Experiment and evaluation in information retrieval.","author":"Voorhees E. M.","year":"2005"}],"event":{"name":"CIKM '18: The 27th ACM International Conference on Information and Knowledge Management","location":"Torino Italy","acronym":"CIKM '18","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 27th ACM International Conference on Information and Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3269206.3271750","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3269206.3271750","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,3]],"date-time":"2026-04-03T21:06:31Z","timestamp":1775250391000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3269206.3271750"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,10,17]]},"references-count":48,"alternative-id":["10.1145\/3269206.3271750","10.1145\/3269206"],"URL":"https:\/\/doi.org\/10.1145\/3269206.3271750","relation":{},"subject":[],"published":{"date-parts":[[2018,10,17]]},"assertion":[{"value":"2018-10-17","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}