{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:11:43Z","timestamp":1775283103900,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":22,"publisher":"ACM","license":[{"start":{"date-parts":[[2004,7,25]],"date-time":"2004-07-25T00:00:00Z","timestamp":1090713600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2004,7,25]]},"DOI":"10.1145\/1008992.1008997","type":"proceedings-article","created":{"date-parts":[[2004,10,7]],"date-time":"2004-10-07T17:39:48Z","timestamp":1097170788000},"page":"10-17","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":20,"title":["Scaling IR-system evaluation using term relevance sets"],"prefix":"10.1145","author":[{"given":"Einat","family":"Amitay","sequence":"first","affiliation":[{"name":"IBM Haifa Research Lab, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David","family":"Carmel","sequence":"additional","affiliation":[{"name":"IBM Haifa Research Lab, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ronny","family":"Lempel","sequence":"additional","affiliation":[{"name":"IBM Haifa Research Lab, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Aya","family":"Soffer","sequence":"additional","affiliation":[{"name":"IBM Haifa Research Lab, Haifa, Israel"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2004,7,25]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/956863.956868"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5555\/283554.283370"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345543"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceeding of Tenth Text REtrieval Conference(TREC-10)","author":"Carmel David","year":"2001","unstructured":"David Carmel , Einat Amitay , Miki Herscovici , Yoelle S. Maarek , Yael Petruschka , and Aya Soffer . Juru at TREC 10 - Experiments with Index Pruning . In Proceeding of Tenth Text REtrieval Conference(TREC-10) . National Institute of Standards and Technology. NIST , 2001 . David Carmel, Einat Amitay, Miki Herscovici, Yoelle S. Maarek, Yael Petruschka, and Aya Soffer. Juru at TREC 10 - Experiments with Index Pruning. In Proceeding of Tenth Text REtrieval Conference(TREC-10). National Institute of Standards and Technology. NIST, 2001."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0169-7552(98)00108-1"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.5555\/644108.644113"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/775152.775246"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the Tenth Text Retrieval Conference(TREC-10)","author":"Hawking D.","year":"2001","unstructured":"D. Hawking and N. Craswell . Overview of the TREC-2001 Web Track. In E. M. Voorhees and D. K. Harman, editors , Proceedings of the Tenth Text Retrieval Conference(TREC-10) . National Institute of Standards and Technology. NIST , 2001 . D. Hawking and N. Craswell. Overview of the TREC-2001 Web Track. In E. M. Voorhees and D. K. Harman, editors, Proceedings of the Tenth Text Retrieval Conference(TREC-10). National Institute of Standards and Technology. NIST, 2001."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/S1389-1286(99)00024-9"},{"key":"e_1_3_2_1_11_1","volume-title":"Science","author":"Lawrence Steve","year":"1998","unstructured":"Steve Lawrence and C. Lee Giles . Searching the world wide web . Science , 280, April 1998 . Steve Lawrence and C. Lee Giles. Searching the world wide web. Science, 280, April 1998."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/75334.75355"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/371920.371965"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/860435.860510"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/321296.321299"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/383952.383961"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/959258.959267"},{"key":"e_1_3_2_1_18_1","volume-title":"Proceeding of Tenth Text REtrieval Conference(TREC-10)","author":"Soubbotin M. M.","year":"2001","unstructured":"M. M. Soubbotin . Patterns of potential answer expressions as clues to the right answers . In Proceeding of Tenth Text REtrieval Conference(TREC-10) . National Institute of Standards and Technology. NIST , 2001 . M. M. Soubbotin. Patterns of potential answer expressions as clues to the right answers. In Proceeding of Tenth Text REtrieval Conference(TREC-10). National Institute of Standards and Technology. NIST, 2001."},{"key":"e_1_3_2_1_19_1","first-page":"355","volume-title":"Proceedings of the Second Workshop of the Cross-Language Evaluation Forum,(CLEF 2001","author":"Voorhees Ellen","year":"2001","unstructured":"Ellen Voorhees . The philosophy of information retrieval evaluation . In Proceedings of the Second Workshop of the Cross-Language Evaluation Forum,(CLEF 2001 ), pages 355 -- 370 , 2001 . Ellen Voorhees. The philosophy of information retrieval evaluation. In Proceedings of the Second Workshop of the Cross-Language Evaluation Forum,(CLEF 2001), pages 355--370, 2001."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(99)00043-6"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/952532.952693"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR04: The 27th ACM\/SIGIR International Symposium on Information Retrieval 2004","location":"Sheffield United Kingdom","acronym":"SIGIR04","sponsor":["ACM Association for Computing Machinery","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 27th annual international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1008992.1008997","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/1008992.1008997","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T21:41:08Z","timestamp":1750282868000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/1008992.1008997"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2004,7,25]]},"references-count":22,"alternative-id":["10.1145\/1008992.1008997","10.1145\/1008992"],"URL":"https:\/\/doi.org\/10.1145\/1008992.1008997","relation":{},"subject":[],"published":{"date-parts":[[2004,7,25]]},"assertion":[{"value":"2004-07-25","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}