{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T15:21:29Z","timestamp":1759936889466,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":35,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T00:00:00Z","timestamp":1650844800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"ARC Discovery Project","award":["DP190102141"],"award-info":[{"award-number":["DP190102141"]}]},{"name":"ARC Training Centre for Information Resilience","award":["IC200100022"],"award-info":[{"award-number":["IC200100022"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,4,25]]},"DOI":"10.1145\/3485447.3511960","type":"proceedings-article","created":{"date-parts":[[2022,4,25]],"date-time":"2022-04-25T05:13:07Z","timestamp":1650863587000},"page":"319-327","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Preferences on a Budget: Prioritizing Document Pairs when Crowdsourcing Relevance Judgments"],"prefix":"10.1145","author":[{"given":"Kevin","family":"Roitero","sequence":"first","affiliation":[{"name":"University of Udine, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alessandro","family":"Checco","sequence":"additional","affiliation":[{"name":"University of Sheffield, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefano","family":"Mizzaro","sequence":"additional","affiliation":[{"name":"University of Udine, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gianluca","family":"Demartini","sequence":"additional","affiliation":[{"name":"University of Queensland, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,4,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.384007"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484170"},{"key":"e_1_3_2_1_3_1","first-page":"324","article-title":"Rank Analysis of Incomplete Block Designs","volume":"39","author":"Bradley Ralph\u00a0Allan","year":"1952","unstructured":"Ralph\u00a0Allan Bradley and Milton\u00a0E. Terry. 1952. Rank Analysis of Incomplete Block Designs: I. The Method of Paired Comparisons. Biometrika 39, 3\/4 (1952), 324\u2013345.","journal-title":"I. The Method of Paired Comparisons. Biometrika"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390451"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-78646-7_5"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348398"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2433396.2433420"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/3409256.3409816"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3411915"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3451161"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1572114"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.5923\/j.computer.20120205.01"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00355-011-0603-9"},{"key":"e_1_3_2_1_14_1","volume-title":"Proceedings of The Eighth Text REtrieval Conference, TREC 1999","author":"Hawking David","year":"1999","unstructured":"David Hawking, Ellen\u00a0M. Voorhees, Nick Craswell, and Peter Bailey. 1999. Overview of the TREC-8 Web Track. In Proceedings of The Eighth Text REtrieval Conference, TREC 1999, Gaithersburg, Maryland, USA, November 17-19, 1999(NIST Special Publication, Vol.\u00a0500-246), Ellen\u00a0M. Voorhees and Donna\u00a0K. Harman (Eds.). National Institute of Standards and Technology (NIST)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.7465\/jkdi.2013.24.5.1089"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210186"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2970398.2970429"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000016"},{"volume-title":"Individual choice behavior","author":"Luce Duncan","key":"e_1_3_2_1_20_1","unstructured":"R.\u00a0Duncan Luce. 1959. Individual choice behavior.John Wiley, Oxford, England. xii\u2013153 pages."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3002172"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2005.03.023"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1935826.1935850"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2019.102149"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210052"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102688"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199012)41:8<590::AID-ASI5>3.0.CO;2-T"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401115"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3431813"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341981.3344246"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3291992.3291995"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390437"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10844-009-0096-5"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation","author":"Zhu Dongqing","year":"2010","unstructured":"Dongqing Zhu and Ben Carterette. 2010. An Analysis of Assessor Behavior in Crowdsourced Preference Judgments. In Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (Geneva). 17\u201320."}],"event":{"name":"WWW '22: The ACM Web Conference 2022","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Virtual Event, Lyon France","acronym":"WWW '22"},"container-title":["Proceedings of the ACM Web Conference 2022"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3511960","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3485447.3511960","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:12:09Z","timestamp":1750191129000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485447.3511960"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,25]]},"references-count":35,"alternative-id":["10.1145\/3485447.3511960","10.1145\/3485447"],"URL":"https:\/\/doi.org\/10.1145\/3485447.3511960","relation":{},"subject":[],"published":{"date-parts":[[2022,4,25]]},"assertion":[{"value":"2022-04-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}