{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:15:04Z","timestamp":1775283304333,"version":"3.50.1"},"reference-count":18,"publisher":"Association for Computing Machinery (ACM)","issue":"3","license":[{"start":{"date-parts":[[2018,9,29]],"date-time":"2018-09-29T00:00:00Z","timestamp":1538179200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Ahold Delhaize, Amsterdam Data Science, the Bloomberg Research Grant Program, the Criteo Faculty Research Award Program, Elsevier"},{"name":"the Microsoft Research Ph.D. Program, the Netherlands Institute for Sound and Vision, the Netherlands Organisation for Scientific Research","award":["CI-14-25, 652.002.001, 612.001.551 and 652.001.003"],"award-info":[{"award-number":["CI-14-25, 652.002.001, 612.001.551 and 652.001.003"]}]},{"name":"Yandex"},{"name":"European Community\u2019s Seventh Framework Programme","award":["312827 (VOX-Pol)"],"award-info":[{"award-number":["312827 (VOX-Pol)"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["J. Data and Information Quality"],"published-print":{"date-parts":[[2018,9,30]]},"abstract":"<jats:p>\n            We report on our experience with TREC OpenSearch, an online evaluation campaign that enabled researchers to evaluate their experimental retrieval methods using real users of a live website. Specifically, we focus on the task of\n            <jats:italic>ad hoc<\/jats:italic>\n            document retrieval within the academic search domain, and work with two search engines, CiteSeerX and SSOAR, that provide us with traffic. We describe our experimental platform, which is based on the living labs methodology, and report on the experimental results obtained. We also share our experiences, challenges, and the lessons learned from running this track in 2016 and 2017.\n          <\/jats:p>","DOI":"10.1145\/3239575","type":"journal-article","created":{"date-parts":[[2018,10,1]],"date-time":"2018-10-01T12:15:55Z","timestamp":1538396155000},"page":"1-15","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":17,"title":["OpenSearch"],"prefix":"10.1145","volume":"10","author":[{"given":"Rolf","family":"Jagerman","sequence":"first","affiliation":[{"name":"University of Amsterdam, Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Krisztian","family":"Balog","sequence":"additional","affiliation":[{"name":"University of Stavanger, Stavanger, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Maarten De","family":"Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2018,9,29]]},"reference":[{"key":"e_1_2_1_1_1","volume-title":"CLEF 2011: Conference on Multilingual and Multimodal Information Access Evaluation. Springer","author":"Azzopardi Leif","year":"2011"},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661962"},{"key":"e_1_2_1_3_1","volume-title":"Overview of the TREC 2016 open search track: Academic search edition. In Proceedings of the 25th Text REtrieval Conference (TREC\u201916)","author":"Balog Krisztian"},{"key":"e_1_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2888422.2888443"},{"key":"e_1_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/775047.775067"},{"key":"e_1_2_1_9_1","unstructured":"Makoto P. Kato Takehiro Yamamoto Sumio Fujita Akiomi Nishida and Tomohiro Manabe. 2017. NTCIR-13 OpenLiveQ. Retrieved October 8 2017 from http:\/\/www.openliveq.net\/.  Makoto P. Kato Takehiro Yamamoto Sumio Fujita Akiomi Nishida and Tomohiro Manabe. 2017. NTCIR-13 OpenLiveQ. Retrieved October 8 2017 from http:\/\/www.openliveq.net\/."},{"key":"e_1_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_2_1_11_1","unstructured":"Julia Kiseleva and Maarten de Rijke. 2017. Evaluating personal assistants on mobile devices. arXiv:1706.04524.  Julia Kiseleva and Maarten de Rijke. 2017. Evaluating personal assistants on mobile devices. arXiv:1706.04524."},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2488217"},{"key":"e_1_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458092"},{"key":"e_1_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-24027-5_47"},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661952"},{"key":"e_1_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1080343.1080347"},{"key":"e_1_2_1_18_1","volume-title":"Proceedings of the 28th AAAI Conference on Artificial Intelligence, Innovative Applications of Artificial Intelligence","author":"Wu Jian"}],"container-title":["Journal of Data and Information Quality"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3239575","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3239575","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T01:08:20Z","timestamp":1750208900000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3239575"}},"subtitle":["Lessons Learned from an Online Evaluation Campaign"],"short-title":[],"issued":{"date-parts":[[2018,9,29]]},"references-count":18,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2018,9,30]]}},"alternative-id":["10.1145\/3239575"],"URL":"https:\/\/doi.org\/10.1145\/3239575","relation":{},"ISSN":["1936-1955","1936-1963"],"issn-type":[{"value":"1936-1955","type":"print"},{"value":"1936-1963","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,9,29]]},"assertion":[{"value":"2017-10-01","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2018-07-01","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2018-09-29","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}