{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,5]],"date-time":"2025-10-05T19:50:00Z","timestamp":1759693800652,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":14,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,7]],"date-time":"2017-08-07T00:00:00Z","timestamp":1502064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,7]]},"DOI":"10.1145\/3077136.3080644","type":"proceedings-article","created":{"date-parts":[[2017,7,28]],"date-time":"2017-07-28T19:35:01Z","timestamp":1501270501000},"page":"793-796","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["On the Reusability of \"Living Labs\" Test Collections"],"prefix":"10.1145","author":[{"given":"Luchen","family":"Tan","sequence":"first","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Gaurav","family":"Baruah","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]},{"given":"Jimmy","family":"Lin","sequence":"additional","affiliation":[{"name":"University of Waterloo, Waterloo, ON, Canada"}]}],"member":"320","published-online":{"date-parts":[[2017,8,7]]},"reference":[{"volume-title":"Overview of the TREC 2016 Open Search Track TREC.","year":"2016","author":"Balog Krisztian","key":"e_1_3_2_1_1_1"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-007-9032-x"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Stefan B\u00fcttcher Charles L. A. Clarke Peter C. K. Yeung and Ian Soboroff. 2007. Reliable Information Retrieval Evaluation with Incomplete and Biased Judgements SIGIR. 63--70.  Stefan B\u00fcttcher Charles L. A. Clarke Peter C. K. Yeung and Ian Soboroff. 2007. Reliable Information Retrieval Evaluation with Incomplete and Biased Judgements SIGIR. 63--70.","DOI":"10.1145\/1277741.1277755"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1718487.1718516"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_1_6_1","unstructured":"Allan Hanbury Henning M\u00fcller Krisztian Balog Torben Brodt Gordon V. Cormack Ivan Eggel Tim Gollub Frank Hopfgartner Jayashree Kalpathy-Cramer Noriko Kando Anastasia Krithara Jimmy Lin Simon Mercer and Martin Potthast. 2015. Evaluation-as-a-Service: Overview and Outlook. arXiv:1512.07454.  Allan Hanbury Henning M\u00fcller Krisztian Balog Torben Brodt Gordon V. Cormack Ivan Eggel Tim Gollub Frank Hopfgartner Jayashree Kalpathy-Cramer Noriko Kando Anastasia Krithara Jimmy Lin Simon Mercer and Martin Potthast. 2015. Evaluation-as-a-Service: Overview and Outlook. arXiv:1512.07454."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"William Hersh Andrew Turpin Susan Price Benjamin Chan Dale Kramer Lynetta Sacherek and Daniel Olson. 2000. Do Batch and User Evaluations Give the Same Results? SIGIR. 17--24.  William Hersh Andrew Turpin Susan Price Benjamin Chan Dale Kramer Lynetta Sacherek and Daniel Olson. 2000. Do Batch and User Evaluations Give the Same Results? SIGIR. 17--24.","DOI":"10.1145\/345508.345539"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Ron Kohavi Randal M. Henne and Dan Sommerfield. 2007. Practical Guide to Controlled Experiments on the Web: Listen to Your Customers not to the HiPPO. In KDD. 959--967.  Ron Kohavi Randal M. Henne and Dan Sommerfield. 2007. Practical Guide to Controlled Experiments on the Web: Listen to Your Customers not to the HiPPO. In KDD. 959--967.","DOI":"10.1145\/1281192.1281295"},{"volume-title":"Overview of the TREC 2016 Real-Time Summarization Track TREC.","year":"2016","author":"Lin Jimmy","key":"e_1_3_2_1_9_1"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911494"},{"volume-title":"de Vries","year":"2013","author":"Said Alan","key":"e_1_3_2_1_11_1"},{"volume-title":"Overview of the Living Labs for Information Retrieval Evaluation (LL4IR) CLEF Lab","year":"2015","author":"Schuth Anne","key":"e_1_3_2_1_12_1"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Ellen M. Voorhees. 2002. The Philosophy of Information Retrieval Evaluation CLEF. 355--370.  Ellen M. Voorhees. 2002. The Philosophy of Information Retrieval Evaluation CLEF. 355--370.","DOI":"10.1007\/3-540-45691-0_34"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"Justin Zobel. 1998. How Reliable Are the Results of Large-Scale Information Retrieval Experiments? SIGIR. 307--314.  Justin Zobel. 1998. How Reliable Are the Results of Large-Scale Information Retrieval Experiments? SIGIR. 307--314.","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '17: The 40th International ACM SIGIR conference on research and development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Shinjuku Tokyo Japan","acronym":"SIGIR '17"},"container-title":["Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080644","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3080644","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T03:37:11Z","timestamp":1750217831000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080644"}},"subtitle":["A Case Study of Real-Time Summarization"],"short-title":[],"issued":{"date-parts":[[2017,8,7]]},"references-count":14,"alternative-id":["10.1145\/3077136.3080644","10.1145\/3077136"],"URL":"https:\/\/doi.org\/10.1145\/3077136.3080644","relation":{},"subject":[],"published":{"date-parts":[[2017,8,7]]},"assertion":[{"value":"2017-08-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}