{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:15:01Z","timestamp":1775283301675,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":28,"publisher":"ACM","license":[{"start":{"date-parts":[[2009,7,19]],"date-time":"2009-07-19T00:00:00Z","timestamp":1247961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2009,7,19]]},"DOI":"10.1145\/1571941.1572029","type":"proceedings-article","created":{"date-parts":[[2009,7,28]],"date-time":"2009-07-28T12:45:28Z","timestamp":1248785128000},"page":"508-515","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":34,"title":["Including summaries in system evaluation"],"prefix":"10.1145","author":[{"given":"Andrew","family":"Turpin","sequence":"first","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kalvero","family":"Jarvelin","sequence":"additional","affiliation":[{"name":"University of Tampere, Tampere, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mingfang","family":"Wu","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[{"name":"RMIT University, Melbourne, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2009,7,19]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390347"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076109"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"e_1_3_2_1_4_1","volume-title":"Retrieval system evaluation","author":"Buckley C.","year":"2005","unstructured":"C. Buckley and E.M. Voorhees . Retrieval system evaluation . In Ellen M. Voorhees and Donna K. Harman, editors, TREC: experiment and evaluation in information retrieval. MIT Press , 2005 . C. Buckley and E.M. Voorhees. Retrieval system evaluation. In Ellen M. Voorhees and Donna K. Harman, editors, TREC: experiment and evaluation in information retrieval. MIT Press, 2005."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1099554.1099723"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/275537.275544"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.5555\/631.634"},{"key":"e_1_3_2_1_8_1","first-page":"87","volume-title":"TREC-9","author":"Hawking D.","year":"2000","unstructured":"D. Hawking . Overview of the TREC-9 Web track . In TREC-9 , pages 87 -- 102 , Gaithersburg , MD , 2000 . D. Hawking. Overview of the TREC-9 Web track. In TREC-9, pages 87--102, Gaithersburg, MD, 2000."},{"key":"e_1_3_2_1_9_1","first-page":"61","volume-title":"Overview of TREC 2001 Web track. In TREC 2001","author":"Hawking D.","year":"2001","unstructured":"D. Hawking and N. Craswell . Overview of TREC 2001 Web track. In TREC 2001 , pages 61 -- 67 , Gaithersburg, MD , 2001 . D. Hawking and N. Craswell. Overview of TREC 2001 Web track. In TREC 2001, pages 61--67, Gaithersburg, MD, 2001."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345539"},{"key":"e_1_3_2_1_11_1","volume-title":"The Turn: Integration of Information Seeking and Retrieval in Context","author":"Ingwersen P.","year":"2005","unstructured":"P. Ingwersen and K. J\u00e4rvelin . The Turn: Integration of Information Seeking and Retrieval in Context . Kluwer Academic Publishers , 2005 . P. Ingwersen and K. J\u00e4rvelin. The Turn: Integration of Information Seeking and Retrieval in Context. Kluwer Academic Publishers, 2005."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_15_1","volume-title":"R: A Language and Environment for Statistical Computing","author":"Development Core Team R","year":"2008","unstructured":"R Development Core Team . R: A Language and Environment for Statistical Computing . R Foundation for Statistical Computing, Vienna, Austria , 2008 . ISBN 3-900051-07-0. R Development Core Team. R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria, 2008. ISBN 3-900051-07-0."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009001"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277935"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/584792.584846"},{"key":"e_1_3_2_1_19_1","volume-title":"Handbook of parametric and nonparametric statistical procedures","author":"Sheskin D.","year":"1997","unstructured":"D. Sheskin . Handbook of parametric and nonparametric statistical procedures . CRC Press , 1997 . D. Sheskin. Handbook of parametric and nonparametric statistical procedures. CRC Press, 1997."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.290947"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383992"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148176"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.datak.2008.10.002"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383963"},{"key":"e_1_3_2_1_27_1","volume-title":"TREC: experiment and evaluation in information retrieval","author":"Voorhees E.M.","year":"2005","unstructured":"E.M. Voorhees and D.K. Harman . TREC: experiment and evaluation in information retrieval . MIT Press , 2005 . E.M. Voorhees and D.K. Harman. TREC: experiment and evaluation in information retrieval. MIT Press, 2005."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.384028"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183633"}],"event":{"name":"SIGIR '09: The 32nd International ACM SIGIR conference on research and development in Information Retrieval","location":"Boston MA USA","acronym":"SIGIR '09","sponsor":["ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 32nd international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1571941.1572029","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1571941.1572029","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T12:23:33Z","timestamp":1750249413000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1571941.1572029"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2009,7,19]]},"references-count":28,"alternative-id":["10.1145\/1571941.1572029","10.1145\/1571941"],"URL":"https:\/\/doi.org\/10.1145\/1571941.1572029","relation":{},"subject":[],"published":{"date-parts":[[2009,7,19]]},"assertion":[{"value":"2009-07-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}