{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:51:24Z","timestamp":1750308684762,"version":"3.41.0"},"reference-count":4,"publisher":"Association for Computing Machinery (ACM)","issue":"1","license":[{"start":{"date-parts":[[2014,6,26]],"date-time":"2014-06-26T00:00:00Z","timestamp":1403740800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["SIGIR Forum"],"published-print":{"date-parts":[[2014,6,26]]},"abstract":"<jats:p>Evaluation is a central aspect of information retrieval (IR) research. In the past few years, a new evaluation methodology known as living labs has been proposed as a way for researchers to be able to perform in-situ evaluation. The first CIKM workshop on Living Labs for IR evaluation (LL'13) was held on 1st November 2013 in San Francisco, USA. The workshop consisted of an industrial keynote, four oral paper presentations, three demo presentations, and a discussion session. This report presents an overview of the scope and contents of the workshop and outlines the major outcomes.<\/jats:p>","DOI":"10.1145\/2641383.2641388","type":"journal-article","created":{"date-parts":[[2014,7,7]],"date-time":"2014-07-07T11:55:18Z","timestamp":1404734118000},"page":"21-28","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Report on the CIKM workshop on living labs for information retrieval evaluation"],"prefix":"10.1145","volume":"48","author":[{"given":"Krisztian","family":"Balog","sequence":"first","affiliation":[{"name":"University of Stavanger, Norway"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David","family":"Elsweiler","sequence":"additional","affiliation":[{"name":"University of Regensburg, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Evangelos","family":"Kanoulas","sequence":"additional","affiliation":[{"name":"Google Inc., Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liadh","family":"Kelly","sequence":"additional","affiliation":[{"name":"Dublin City University, Ireland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mark D.","family":"Smucker","sequence":"additional","affiliation":[{"name":"University of Waterloo, Canada"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2014,6,26]]},"reference":[{"key":"e_1_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/2045274.2045281"},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2422256.2422258"},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2009.82"},{"key":"e_1_2_1_4_1","first-page":"101","article-title":"Securing trust online: Wisdom or oxymoron?","volume":"81","author":"Nissenbaum H.","year":"2001","unstructured":"H. Nissenbaum . Securing trust online: Wisdom or oxymoron? Boston University Law Review , 81 : 101 -- 131 , 2001 . H. Nissenbaum. Securing trust online: Wisdom or oxymoron? Boston University Law Review, 81:101--131, 2001.","journal-title":"Boston University Law Review"}],"container-title":["ACM SIGIR Forum"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2641383.2641388","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2641383.2641388","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T20:01:07Z","timestamp":1750276867000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2641383.2641388"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,6,26]]},"references-count":4,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2014,6,26]]}},"alternative-id":["10.1145\/2641383.2641388"],"URL":"https:\/\/doi.org\/10.1145\/2641383.2641388","relation":{},"ISSN":["0163-5840"],"issn-type":[{"type":"print","value":"0163-5840"}],"subject":[],"published":{"date-parts":[[2014,6,26]]},"assertion":[{"value":"2014-06-26","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}