{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T04:51:51Z","timestamp":1750308711758,"version":"3.41.0"},"reference-count":57,"publisher":"Association for Computing Machinery (ACM)","issue":"2","license":[{"start":{"date-parts":[[2012,12,21]],"date-time":"2012-12-21T00:00:00Z","timestamp":1356048000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["SIGIR Forum"],"published-print":{"date-parts":[[2012,12,21]]},"abstract":"<jats:p>The PROMISE network of excellence organized a two-days brainstorming workshop on 30th and 31st May 2012 in Padua, Italy, to discuss and envisage future directions and perspectives for the evaluation of information access and retrieval systems in multiple languages and multiple media. This document reports on the outcomes of this event and provides details about the six envisaged research lines: search applications; contextual evaluation; challenges in test collection design and exploitation; component-based evaluation; ongoing evaluation; and signal-aware evaluation. The ultimate goal of the PROMISE retreat is to stimulate and involve the research community along these research lines and to provide funding agencies with effective and scientifically sound ideas for coordinating and supporting information access research.<\/jats:p>","DOI":"10.1145\/2422256.2422265","type":"journal-article","created":{"date-parts":[[2013,1,2]],"date-time":"2013-01-02T13:23:15Z","timestamp":1357132995000},"page":"60-84","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["PROMISE retreat report prospects and opportunities for information access evaluation"],"prefix":"10.1145","volume":"46","author":[{"given":"Maristella","family":"Agosti","sequence":"first","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Richard","family":"Berendsen","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Toine","family":"Bogers","sequence":"additional","affiliation":[{"name":"Royal School of Library and Information Science, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Martin","family":"Braschler","sequence":"additional","affiliation":[{"name":"Zurich University of Applied Sciences, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Paul","family":"Buitelaar","sequence":"additional","affiliation":[{"name":"National University of Ireland, Galway Ireland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Khalid","family":"Choukri","sequence":"additional","affiliation":[{"name":"Evaluations and Language resources Distribution Agency (ELDA), France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Giorgio","family":"Maria Di Nunzio","sequence":"additional","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Nicola","family":"Ferro","sequence":"additional","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Pamela","family":"Forner","sequence":"additional","affiliation":[{"name":"Centre for the Evaluation of Language and Communication Technologies (CELCT), Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Karin Friberg","family":"Heppin","sequence":"additional","affiliation":[{"name":"University of Gothenburg, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Preben","family":"Hansen","sequence":"additional","affiliation":[{"name":"Swedish Institute of Computer Science, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anni","family":"J\u00e4rvelin","sequence":"additional","affiliation":[{"name":"Swedish Institute of Computer Science, Sweden"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Birger","family":"Larsen","sequence":"additional","affiliation":[{"name":"Royal School of Library and Information Science, Denmark"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mihai","family":"Lupu","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ivano","family":"Masiero","sequence":"additional","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Henning","family":"M\u00fcller","sequence":"additional","affiliation":[{"name":"University of Applied Sciences Western Switzerland (HES-SO), Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Simone","family":"Peruzzo","sequence":"additional","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Vivien","family":"Petras","sequence":"additional","affiliation":[{"name":"Humboldt University Berlin, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Florina","family":"Piroi","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Giuseppe","family":"Santucci","sequence":"additional","affiliation":[{"name":"Sapienza, University of Rome, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gianmaria","family":"Silvello","sequence":"additional","affiliation":[{"name":"University of Padua, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Elaine","family":"Toms","sequence":"additional","affiliation":[{"name":"University of Sheffield, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2012,12,21]]},"reference":[{"key":"e_1_2_1_1_1","doi-asserted-by":"crossref","unstructured":"M. Agosti R. Berendsen T. Bogers M. Braschler P. Buitelaar K. Choukri G. M. Di Nunzio N. Ferro P. Forner A. Hanbury K. Friberg Heppin P. Hansen A. J\u00e4rvelin B. Larsen M. Lupu I. Masiero H. M\u00fcller S. Peruzzo V. Petras F. Piroi M. de Rijke G. Santucci G. Silvello and E. Toms. PROMISE Retreat Report -- Prospects and Opportunities for Information Access Evaluation. PROMISE network of excellence ISBN 978-88-6321-039-2 http:\/\/www.promise-noe.eu\/promise-retreat-report-2012\/ September 2012  M. Agosti R. Berendsen T. Bogers M. Braschler P. Buitelaar K. Choukri G. M. Di Nunzio N. Ferro P. Forner A. Hanbury K. Friberg Heppin P. Hansen A. J\u00e4rvelin B. Larsen M. Lupu I. Masiero H. M\u00fcller S. Peruzzo V. Petras F. Piroi M. de Rijke G. Santucci G. Silvello and E. Toms. PROMISE Retreat Report -- Prospects and Opportunities for Information Access Evaluation. PROMISE network of excellence ISBN 978-88-6321-039-2 http:\/\/www.promise-noe.eu\/promise-retreat-report-2012\/ September 2012","DOI":"10.1145\/2422256.2422265"},{"key":"e_1_2_1_2_1","unstructured":"M. Agosti M. Braschler E. Di Buccio M. Dussin N. Ferro G. L. Granato I. Masiero E. Pianta G. Santucci G. Silvello and G. Tino. Deliverable D3.2 -- Specification of the evaluation infrastructure based on user requirements. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/fdf43394-0997-4638-9f99-38b2e9c63802 August 2011.  M. Agosti M. Braschler E. Di Buccio M. Dussin N. Ferro G. L. Granato I. Masiero E. Pianta G. Santucci G. Silvello and G. Tino. Deliverable D3.2 -- Specification of the evaluation infrastructure based on user requirements. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/fdf43394-0997-4638-9f99-38b2e9c63802 August 2011."},{"key":"e_1_2_1_3_1","unstructured":"M. Agosti E. Di Buccio N. Ferro I. Masiero M. Nicchio S. Peruzzo and G. Silvello. Deliverable D3.3 -- Prototype of the Evaluation Infrastructure. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/3783730a-bce3-481b-83df-48e209c6286a September 2012.  M. Agosti E. Di Buccio N. Ferro I. Masiero M. Nicchio S. Peruzzo and G. Silvello. Deliverable D3.3 -- Prototype of the Evaluation Infrastructure. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/3783730a-bce3-481b-83df-48e209c6286a September 2012."},{"key":"e_1_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-33247-0_11"},{"key":"e_1_2_1_5_1","first-page":"62","volume-title":"Proc. 1st International Workshop on Evaluating Information Access (EVIA 2007","author":"Agosti M.","year":"2007","unstructured":"M. Agosti , G. M. Di Nunzio , and N. Ferro . A Proposal to Extend and Enrich the Scientific Data Curation of Evaluation Campaigns. In T. Sakay, M. Sanderson, and D. K. Evans, editors , Proc. 1st International Workshop on Evaluating Information Access (EVIA 2007 ), pages 62 -- 73 . National Institute of Informatics, Tokyo, Japan , 2007 . M. Agosti, G. M. Di Nunzio, and N. Ferro. A Proposal to Extend and Enrich the Scientific Data Curation of Evaluation Campaigns. In T. Sakay, M. Sanderson, and D. K. Evans, editors, Proc. 1st International Workshop on Evaluating Information Access (EVIA 2007), pages 62--73. National Institute of Informatics, Tokyo, Japan, 2007."},{"key":"e_1_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/1782334.1782354"},{"key":"e_1_2_1_7_1","doi-asserted-by":"crossref","first-page":"93","DOI":"10.1016\/B978-1-84334-484-1.50006-5","volume-title":"Evaluation of Digital Libraries: An insight into useful applications and methods","author":"Agosti M.","year":"2009","unstructured":"M. Agosti and N. Ferro . Towards an Evaluation Infrastructure for DL Performance Evaluation . In G. Tsakonas and C. Papatheodorou, editors, Evaluation of Digital Libraries: An insight into useful applications and methods , pages 93 -- 120 . Chandos Publishing , Oxford, UK , 2009 . M. Agosti and N. Ferro. Towards an Evaluation Infrastructure for DL Performance Evaluation. In G. Tsakonas and C. Papatheodorou, editors, Evaluation of Digital Libraries: An insight into useful applications and methods, pages 93--120. Chandos Publishing, Oxford, UK, 2009."},{"key":"e_1_2_1_8_1","volume-title":"Proceedings of the International Conference of the Cross-Language Evaluation Forum (CLEF 2010","author":"Agosti M.","year":"2010","unstructured":"M. Agosti , N. Ferro , C. Peters , M. de Rijke , and A. Smeaton , editors. Multilingual and Multimodal Information Access Evaluation . Proceedings of the International Conference of the Cross-Language Evaluation Forum (CLEF 2010 ). Lecture Notes in Computer Science (LNCS) 6360, Springer, Heidelberg, Germany , 2010 . M. Agosti, N. Ferro, C. Peters, M. de Rijke, and A. Smeaton, editors. Multilingual and Multimodal Information Access Evaluation. Proceedings of the International Conference of the Cross-Language Evaluation Forum (CLEF 2010). Lecture Notes in Computer Science (LNCS) 6360, Springer, Heidelberg, Germany, 2010."},{"key":"e_1_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2215676.2215681"},{"key":"e_1_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2215676.2215678"},{"key":"e_1_2_1_11_1","unstructured":"M. Angelini N. Ferro G. L. Granato and G. Santucci. Deliverable D5.3 -- Collaborative User Interface Prototype with Annotation Functionalities. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/8c475e6c-36b5-4822-9fbc-d7d116b3a897 September 2012.  M. Angelini N. Ferro G. L. Granato and G. Santucci. Deliverable D5.3 -- Collaborative User Interface Prototype with Annotation Functionalities. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/8c475e6c-36b5-4822-9fbc-d7d116b3a897 September 2012."},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2362724.2362757"},{"key":"e_1_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010058"},{"key":"e_1_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277820"},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/956863.956868"},{"key":"e_1_2_1_17_1","unstructured":"R. Berendsen M. Braschler M. G\u00e4de M. Kleineberg M. Lupu V. Petras and S. Reitberger. Deliverable D4.3 -- Final Report on Alternative Evaluation Methodology. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/0092298d-892b-45c0-a534-b9a3d0c717b1 September 2012.  R. Berendsen M. Braschler M. G\u00e4de M. Kleineberg M. Lupu V. Petras and S. Reitberger. Deliverable D4.3 -- Final Report on Alternative Evaluation Methodology. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/0092298d-892b-45c0-a534-b9a3d0c717b1 September 2012."},{"key":"e_1_2_1_18_1","unstructured":"R. Berendsen E. Tsagkias M. de Rijke and E. Meij. Generating pseudo test collections for learning to rank scientific articles. In Catarci et al. {24}.  R. Berendsen E. Tsagkias M. de Rijke and E. Meij. Generating pseudo test collections for learning to rank scientific articles. In Catarci et al. {24}."},{"key":"e_1_2_1_19_1","first-page":"140","volume-title":"Agosti et al. {8}","author":"Braschler M.","unstructured":"M. Braschler , K. Choukri , N. Ferro , A. Hanbury , J. Karlgren , H. M\u00fcller , V. Petras , E. Pianta , M. de Rijke , and G. Santucci . A PROMISE for Experimental Evaluation . In Agosti et al. {8} , pages 140 -- 144 . M. Braschler, K. Choukri, N. Ferro, A. Hanbury, J. Karlgren, H. M\u00fcller, V. Petras, E. Pianta, M. de Rijke, and G. Santucci. A PROMISE for Experimental Evaluation. In Agosti et al. {8}, pages 140--144."},{"key":"e_1_2_1_20_1","volume-title":"CLEF 2010 Labs and Workshops, Notebook Papers. MINT srl","author":"Braschler M.","year":"2010","unstructured":"M. Braschler , D. K. Harman , and E. Pianta , editors . CLEF 2010 Labs and Workshops, Notebook Papers. MINT srl , Trento, Italy. ISBN 978-88-904810-0-0. , 2010 . M. Braschler, D. K. Harman, and E. Pianta, editors. CLEF 2010 Labs and Workshops, Notebook Papers. MINT srl, Trento, Italy. ISBN 978-88-904810-0-0., 2010."},{"key":"e_1_2_1_21_1","unstructured":"M. Braschler S. Reitberger M. Imhof A. J\u00e4rvelin P. Hansen M. Lupu M. G\u00e4de R. Berendsen and A. Garcia Seco de Herrera. Deliverable D2.3 -- Best Practices Report. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/086010bb-0d3f-46ef-946f-f0bbeef305e8 August 2012.  M. Braschler S. Reitberger M. Imhof A. J\u00e4rvelin P. Hansen M. Lupu M. G\u00e4de R. Berendsen and A. Garcia Seco de Herrera. Deliverable D2.3 -- Best Practices Report. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/086010bb-0d3f-46ef-946f-f0bbeef305e8 August 2012."},{"key":"e_1_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2006.07.009"},{"key":"e_1_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1924475.1924481"},{"key":"e_1_2_1_24_1","volume-title":"Proceedings of the Third International Conference of the CLEF Initiative (CLEF 2012","author":"Catarci T.","year":"2012","unstructured":"T. Catarci , P. Forner , D. Hiemstra , A. Pe\u00f1as , and G. Santucci , editors. Information Access Evaluation. Multilinguality, Multimodality, and Visual Analytics . Proceedings of the Third International Conference of the CLEF Initiative (CLEF 2012 ). Lecture Notes in Computer Science (LNCS) 7488, Springer, Heidelberg, Germany , 2012 . T. Catarci, P. Forner, D. Hiemstra, A. Pe\u00f1as, and G. Santucci, editors. Information Access Evaluation. Multilinguality, Multimodality, and Visual Analytics. Proceedings of the Third International Conference of the CLEF Initiative (CLEF 2012). Lecture Notes in Computer Science (LNCS) 7488, Springer, Heidelberg, Germany, 2012."},{"key":"e_1_2_1_25_1","volume-title":"Aslib Cranfield Research Project","author":"Cleverdon C. W.","year":"1962","unstructured":"C. W. Cleverdon . Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems. Technical report , Aslib Cranfield Research Project , 1962 . C. W. Cleverdon. Report on the testing and analysis of an investigation into the comparative efficiency of indexing systems. Technical report, Aslib Cranfield Research Project, 1962."},{"key":"e_1_2_1_26_1","first-page":"47","volume-title":"K","author":"Cleverdon C. W.","year":"1997","unstructured":"C. W. Cleverdon . The Cranfield Tests on Index Languages Devices . In K . Sp\u00e4rck Jones and P. Willett, editors, Readings in Information Retrieval, pages 47 -- 60 . Morgan Kaufmann Publisher , Inc., San Francisco, CA, USA, 1997 . C. W. Cleverdon. The Cranfield Tests on Index Languages Devices. In K. Sp\u00e4rck Jones and P. Willett, editors, Readings in Information Retrieval, pages 47--60. Morgan Kaufmann Publisher, Inc., San Francisco, CA, USA, 1997."},{"key":"e_1_2_1_27_1","unstructured":"M. Croce E. Di Buccio E. Di Reto M. Dussin N. Ferro G. L. Granato P. Hansen M. Lupu M. Perlorca A. Pronesti A. Sabetta G. Santucci G. Silvello G. Tino and T. Tsikrika. Deliverable D5.2 -- User interface and Visual analytics environment requirements. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/21f1512a-5b47-48ae-834a-89d6441d079e August 2011.  M. Croce E. Di Buccio E. Di Reto M. Dussin N. Ferro G. L. Granato P. Hansen M. Lupu M. Perlorca A. Pronesti A. Sabetta G. Santucci G. Silvello G. Tino and T. Tsikrika. Deliverable D5.2 -- User interface and Visual analytics environment requirements. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/21f1512a-5b47-48ae-834a-89d6441d079e August 2011."},{"key":"e_1_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2008.06.012"},{"key":"e_1_2_1_29_1","first-page":"54","article-title":"the First Prototype of the PROMISE Evaluation Infrastructure for Information Retrieval Experimental Evaluation","volume":"86","author":"Ferro N.","year":"2011","unstructured":"N. Ferro . DIRECT : the First Prototype of the PROMISE Evaluation Infrastructure for Information Retrieval Experimental Evaluation . ERCIM News , 86 : 54 -- 55 , July 2011 . N. Ferro. DIRECT: the First Prototype of the PROMISE Evaluation Infrastructure for Information Retrieval Experimental Evaluation. ERCIM News, 86:54--55, July 2011.","journal-title":"ERCIM News"},{"key":"e_1_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.procs.2011.04.078"},{"key":"e_1_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2390803.2390808"},{"key":"e_1_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/2045274"},{"key":"e_1_2_1_33_1","volume-title":"CLEF 2012 Labs and Workshops, Notebook Papers. MINT srl","author":"Forner P.","year":"2012","unstructured":"P. Forner , J. Karlgren , and C. Womser-Hacker , editors . CLEF 2012 Labs and Workshops, Notebook Papers. MINT srl , Trento, Italy. ISBN 978-88-904810-1-7. , 2012 . P. Forner, J. Karlgren, and C. Womser-Hacker, editors. CLEF 2012 Labs and Workshops, Notebook Papers. MINT srl, Trento, Italy. ISBN 978-88-904810-1-7., 2012."},{"key":"e_1_2_1_34_1","doi-asserted-by":"crossref","unstructured":"A. Hanbury H. M\u00fcller G. Langs M. A. Weber B. H. Menze and T. S. Fernandez. Bringing the algorithms to the data: cloud-based benchmarking for medical image analysis. In Catarci et al. {24}.  A. Hanbury H. M\u00fcller G. Langs M. A. Weber B. H. Menze and T. S. Fernandez. Bringing the algorithms to the data: cloud-based benchmarking for medical image analysis. In Catarci et al. {24}.","DOI":"10.1007\/978-3-642-33247-0_3"},{"key":"e_1_2_1_35_1","first-page":"124","volume-title":"Agosti et al. {8}","author":"Hanbury Allan","unstructured":"Allan Hanbury and Henning M\u00fcller . Automated component-level evaluation: Present and future . In Agosti et al. {8} , pages 124 -- 135 . Allan Hanbury and Henning M\u00fcller. Automated component-level evaluation: Present and future. In Agosti et al. {8}, pages 124--135."},{"key":"e_1_2_1_36_1","volume-title":"Proc. Workshop on Collaborative Information Seeking: Briding the Gap between Theory and Practice (CIS 2011)","author":"Hansen P.","year":"2011","unstructured":"P. Hansen , G. L. Granato , and G. Santucci . Collecting and Assessing Collaborative Requirements. In C. Shah, P. Hansen, and R. Capra, editors , Proc. Workshop on Collaborative Information Seeking: Briding the Gap between Theory and Practice (CIS 2011) , 2011 . P. Hansen, G. L. Granato, and G. Santucci. Collecting and Assessing Collaborative Requirements. In C. Shah, P. Hansen, and R. Capra, editors, Proc. Workshop on Collaborative Information Seeking: Briding the Gap between Theory and Practice (CIS 2011), 2011."},{"key":"e_1_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2004.04.016"},{"key":"e_1_2_1_38_1","doi-asserted-by":"crossref","unstructured":"D. K. Harman. Information Retrieval Evaluation. Morgan & Claypool Publishers USA 2011.   D. K. Harman. Information Retrieval Evaluation. Morgan & Claypool Publishers USA 2011.","DOI":"10.1007\/978-3-031-02276-0"},{"key":"e_1_2_1_39_1","volume-title":"TREC. Experiment and Evaluation in Information Retrieval","author":"Harman D. K.","year":"2005","unstructured":"D. K. Harman and E. M. Voorhees , editors . TREC. Experiment and Evaluation in Information Retrieval . MIT Press , Cambridge (MA), USA, 2005 . D. K. Harman and E. M. Voorhees, editors. TREC. Experiment and Evaluation in Information Retrieval. MIT Press, Cambridge (MA), USA, 2005."},{"key":"e_1_2_1_40_1","doi-asserted-by":"crossref","DOI":"10.1007\/978-0-387-76578-5","volume-title":"Service Science, Management, and Engineering: Education for the 21st Century","author":"Hefley B.","year":"2008","unstructured":"B. Hefley and W. Murphy , editors . Service Science, Management, and Engineering: Education for the 21st Century . Springer , Heidelberg, Germany , 2008 . B. Hefley and W. Murphy, editors. Service Science, Management, and Engineering: Education for the 21st Century. Springer, Heidelberg, Germany, 2008."},{"key":"e_1_2_1_41_1","first-page":"40","volume-title":"Agosti et al. {8}","author":"Huurnink B.","unstructured":"B. Huurnink , K. Hofmann , M. de Rijke , and M. Bron . Validating query simulators: An experiment using commercial searches and purchases . In Agosti et al. {8} , pages 40 -- 51 . B. Huurnink, K. Hofmann, M. de Rijke, and M. Bron. Validating query simulators: An experiment using commercial searches and purchases. In Agosti et al. {8}, pages 40--51."},{"key":"e_1_2_1_42_1","unstructured":"A. J\u00e4rvelin G. Eriksson P. Hansen T. Tsikrika A. Garcia Seco de Herrera M. Lupu M. G\u00e4de V. Petras S. Rietberger M. Braschler and R. Berendsen. Deliverable D2.2 -- Revised Specification of Evaluation Tasks. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/a0d664fe-16e4-4df6-bcf9-1dc3e5e8c18e February 2012.  A. J\u00e4rvelin G. Eriksson P. Hansen T. Tsikrika A. Garcia Seco de Herrera M. Lupu M. G\u00e4de V. Petras S. Rietberger M. Braschler and R. Berendsen. Deliverable D2.2 -- Revised Specification of Evaluation Tasks. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/a0d664fe-16e4-4df6-bcf9-1dc3e5e8c18e February 2012."},{"key":"e_1_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/1734160.1734166"},{"key":"e_1_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btq464"},{"key":"e_1_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1840784.1840800"},{"key":"e_1_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2093346.2093356"},{"key":"e_1_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1038\/ng0411-281"},{"key":"e_1_2_1_49_1","volume-title":"CLEF 2011 Labs and Workshops, Notebook Papers. MINT srl","author":"Petras V.","year":"2011","unstructured":"V. Petras , P. Forner , and P. Clough , editors . CLEF 2011 Labs and Workshops, Notebook Papers. MINT srl , Trento, Italy. ISBN 978-88-904810-1-7. , 2011 . V. Petras, P. Forner, and P. Clough, editors. CLEF 2011 Labs and Workshops, Notebook Papers. MINT srl, Trento, Italy. ISBN 978-88-904810-1-7., 2011."},{"key":"e_1_2_1_50_1","unstructured":"S. Reitberger M. Imhof M. Braschler R. Berendsen A. J\u00e4rvelin P. Hansen A. Garcia Seco de Herrera T. Tsikrika M. Lupu V. Petras M. G\u00e4de M. Kleineberg and K. Choukri. Deliverable D4.2 -- Tutorial on Evaluation in the Wild. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/3f546a0b-be7c-48df-b228-924cc5e185cb August 2012.  S. Reitberger M. Imhof M. Braschler R. Berendsen A. J\u00e4rvelin P. Hansen A. Garcia Seco de Herrera T. Tsikrika M. Lupu V. Petras M. G\u00e4de M. Kleineberg and K. Choukri. Deliverable D4.2 -- Tutorial on Evaluation in the Wild. PROMISE Network of Excellence EU 7FP Contract N. 258191. http:\/\/www.promise-noe.eu\/documents\/10156\/3f546a0b-be7c-48df-b228-924cc5e185cb August 2012."},{"key":"e_1_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551507086989"},{"key":"e_1_2_1_52_1","unstructured":"B. R. Rowe D. W. Wood A. L. Link and D. A. Simoni. Economic Impact Assessment of NIST's Text REtrieval Conference (TREC) Program. RTI Project Number 0211875 RTI International USA. http:\/\/trec.nist.gov\/pubs\/2010.economic.impact.pdf July 2010.  B. R. Rowe D. W. Wood A. L. Link and D. A. Simoni. Economic Impact Assessment of NIST's Text REtrieval Conference (TREC) Program. RTI Project Number 0211875 RTI International USA. http:\/\/trec.nist.gov\/pubs\/2010.economic.impact.pdf July 2010."},{"key":"e_1_2_1_53_1","volume-title":"Foundations and Trends in Information Retrieval (FnTIR), 4(4):247--375","author":"Sanderson M.","year":"2010","unstructured":"M. Sanderson . Test Collection Based Evaluation of Information Retrieval Systems . Foundations and Trends in Information Retrieval (FnTIR), 4(4):247--375 , 2010 . M. Sanderson. Test Collection Based Evaluation of Information Retrieval Systems. Foundations and Trends in Information Retrieval (FnTIR), 4(4):247--375, 2010."},{"key":"e_1_2_1_54_1","volume-title":"Editor","author":"Spohrer J.","year":"2009","unstructured":"J. Spohrer . Editor ial Column-Welcome to Our Declaration of Interdependence. Service Science , 1(1):i--ii, 2009 . J. Spohrer. Editorial Column-Welcome to Our Declaration of Interdependence. Service Science, 1(1):i--ii, 2009."},{"key":"e_1_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.21494"},{"key":"e_1_2_1_56_1","first-page":"95","volume-title":"Forner et al. {32}","author":"Tsikrika T.","unstructured":"T. Tsikrika , A. Garcia Seco de Herrera, and H. M\u00fcller. Assessing the Scholarly Impact of Image-CLEF . In Forner et al. {32} , pages 95 -- 106 . T. Tsikrika, A. Garcia Seco de Herrera, and H. M\u00fcller. Assessing the Scholarly Impact of Image-CLEF. In Forner et al. {32}, pages 95--106."},{"key":"e_1_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.5555\/1940006.1940067"}],"container-title":["ACM SIGIR Forum"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2422256.2422265","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2422256.2422265","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T20:14:14Z","timestamp":1750277654000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2422256.2422265"}},"subtitle":[],"editor":[{"given":"Richard","family":"Berendsen","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Mihai","family":"Lupu","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Vivien","family":"Petras","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]},{"given":"Gianmaria","family":"Silvello","sequence":"additional","affiliation":[],"role":[{"role":"editor","vocabulary":"crossref"}]}],"short-title":[],"issued":{"date-parts":[[2012,12,21]]},"references-count":57,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2012,12,21]]}},"alternative-id":["10.1145\/2422256.2422265"],"URL":"https:\/\/doi.org\/10.1145\/2422256.2422265","relation":{},"ISSN":["0163-5840"],"issn-type":[{"type":"print","value":"0163-5840"}],"subject":[],"published":{"date-parts":[[2012,12,21]]},"assertion":[{"value":"2012-12-21","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}