{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:11:45Z","timestamp":1775283105289,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":21,"publisher":"ACM","license":[{"start":{"date-parts":[[2007,7,23]],"date-time":"2007-07-23T00:00:00Z","timestamp":1185148800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2007,7,23]]},"DOI":"10.1145\/1277741.1277754","type":"proceedings-article","created":{"date-parts":[[2008,9,18]],"date-time":"2008-09-18T11:59:52Z","timestamp":1221739192000},"page":"55-62","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":33,"title":["Robust test collections for retrieval evaluation"],"prefix":"10.1145","author":[{"given":"Ben","family":"Carterette","sequence":"first","affiliation":[{"name":"University of Massachusetts Amherst"}]}],"member":"320","published-online":{"date-parts":[[2007,7,23]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.384007"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/860435.860501"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/956863.956953"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148263"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.5555\/234285.234289"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1063\/1.1751354"},{"key":"e_1_3_2_1_7_1","volume-title":"Proceedings of RIAO","author":"Carterette B.","year":"2007","unstructured":"B. Carterette and J. Allan . Research methodology in studies of assessor effort for retrieval evaluation . In Proceedings of RIAO 2007 . B. Carterette and J. Allan. Research methodology in studies of assessor effort for retrieval evaluation. In Proceedings of RIAO 2007."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148219"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148289"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.36.7.767"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"e_1_3_2_1_12_1","volume-title":"Bayesian Data Analysis Chapman &amp","author":"Gelman A.","year":"2004","unstructured":"A. Gelman , J. B. Carlin , H. S. Stern , and D. B. Rubin . Bayesian Data Analysis Chapman &amp ; Hall\/CRC , 2004 . A. Gelman, J. B. Carlin, H. S. Stern, and D. B. Rubin. Bayesian Data Analysis Chapman & Hall\/CRC, 2004."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511790423","volume-title":"Probability Theory: The Logic of Science Cambridge University Press","author":"Jaynes E. T.","year":"2003","unstructured":"E. T. Jaynes . Probability Theory: The Logic of Science Cambridge University Press , 2003 . E. T. Jaynes. Probability Theory: The Logic of Science Cambridge University Press, 2003."},{"key":"e_1_3_2_1_14_1","first-page":"88","volume-title":"Proceedings of HLT","author":"Manmatha R.","year":"2002","unstructured":"R. Manmatha and H. Sever . A Formal Approach to Score Normalization for Metasearch . In Proceedings of HLT pages 88 -- 93 , 2002 . R. Manmatha and H. Sever. A Formal Approach to Score Normalization for Metasearch. In Proceedings of HLT pages 88--93, 2002."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.42.10.1420"},{"key":"e_1_3_2_1_16_1","first-page":"61","volume-title":"Probabilistic outputs for support vector machines and comparison to regularized likelihood methods","author":"Platt J.","year":"2000","unstructured":"J. Platt . Probabilistic outputs for support vector machines and comparison to regularized likelihood methods . pages 61 -- 74 , 2000 . J. Platt. Probabilistic outputs for support vector machines and comparison to regularized likelihood methods. pages 61--74, 2000."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009001"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148220"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb026616"},{"key":"e_1_3_2_1_20_1","volume-title":"TREC: Experiment and Evaluation in Information Retrieval MIT Press","author":"Voorhees E. M.","year":"2005","unstructured":"E. M. Voorhees and D. K. Harman , editors . TREC: Experiment and Evaluation in Information Retrieval MIT Press , 2005 . E. M. Voorhees and D. K. Harman, editors. TREC: Experiment and Evaluation in Information Retrieval MIT Press, 2005."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR07: The 30th Annual International SIGIR Conference","location":"Amsterdam The Netherlands","acronym":"SIGIR07","sponsor":["ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 30th annual international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1277741.1277754","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1277741.1277754","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T14:58:14Z","timestamp":1750258694000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1277741.1277754"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2007,7,23]]},"references-count":21,"alternative-id":["10.1145\/1277741.1277754","10.1145\/1277741"],"URL":"https:\/\/doi.org\/10.1145\/1277741.1277754","relation":{},"subject":[],"published":{"date-parts":[[2007,7,23]]},"assertion":[{"value":"2007-07-23","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}