{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,1,29]],"date-time":"2025-01-29T05:57:18Z","timestamp":1738130238323,"version":"3.33.0"},"publisher-location":"Berlin, Heidelberg","reference-count":21,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"type":"print","value":"9783540781585"},{"type":"electronic","value":"9783540781592"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"DOI":"10.1007\/978-3-540-78159-2_31","type":"book-chapter","created":{"date-parts":[[2008,2,29]],"date-time":"2008-02-29T10:26:29Z","timestamp":1204280789000},"page":"344-350","source":"Crossref","is-referenced-by-count":0,"title":["Towards Better Evaluation for Human Language Technology"],"prefix":"10.1007","author":[{"given":"Donna","family":"Harman","sequence":"first","affiliation":[]}],"member":"297","reference":[{"key":"31_CR1","unstructured":"Cleverdon, C.W., Mills, J., Keen, E.M.: Factors determining the performance of indexing systems, vol. 1: Design, vol. 2: Test Results. Aslib Cranfield Research Project (1966)"},{"key":"31_CR2","unstructured":"Text REtrieval Conference, http:\/\/trec.nist.gov\/"},{"key":"31_CR3","volume-title":"TREC: Experiment in Evaluation in Information Retrieval","author":"E.M. Voorhees","year":"2005","unstructured":"Voorhees, E.M., Harman, D.K.: TREC: Experiment in Evaluation in Information Retrieval. MIT Press, Cambridge (2005)"},{"key":"31_CR4","unstructured":"Cross-Language Evaluation Forum, http:\/\/www.clef-campaign.org\/"},{"key":"31_CR5","unstructured":"NII-NACSIS Test Collection for IR Systems, http:\/\/research.nii.ac.jp\/ntcir\/"},{"key":"31_CR6","unstructured":"Francis, W.N., Kucera, H.: Brown Corpus Manual. Brown University (1964)"},{"key":"31_CR7","unstructured":"The Penn Treebank Project, http:\/\/www.cis.upenn.edu\/~treebank\/"},{"key":"31_CR8","unstructured":"Miller, G.: WordNet, http:\/\/wordnet.princeton.edu\/"},{"key":"31_CR9","unstructured":"Message Understanding Conferences, http:\/\/www.aclweb.org\/anthology-new\/"},{"key":"31_CR10","unstructured":"SemEval (2007), http:\/\/nlp.cs.swarthmore.edu\/semeval\/"},{"key":"31_CR11","unstructured":"Document Understanding Conferences, http:\/\/duc.nist.gov\/"},{"key":"31_CR12","unstructured":"NIST speech evaluation benchmarks, http:\/\/www.nist.gov\/speech\/history\/index.htm"},{"key":"31_CR13","unstructured":"Falkedal, I.: Evaluation methods for machine translation systems; a historical overview and critical account. Report, ISSCO, Universite de Geneve (1991)"},{"key":"31_CR14","doi-asserted-by":"crossref","unstructured":"Papineni, K., et al.: Bleu: a method for automatic evaluation of machine translation. IBM Research Report, RC22176 (2001)","DOI":"10.3115\/1073083.1073135"},{"key":"31_CR15","unstructured":"The Meeting of the MINDS: Future directions for human language technology Executive summary, http:\/\/www-nlpir.nist.gov\/MINDS\/FINAL\/exec.summary.pdf"},{"key":"31_CR16","doi-asserted-by":"crossref","unstructured":"Voorhees, Ellen: Variations in Relevance Judgments and the Measurement of Retrieval Effectiveness. Proceedings of SIGIR 1998, pp. 315\u2013323 (1998)","DOI":"10.1145\/290941.291017"},{"key":"31_CR17","doi-asserted-by":"crossref","unstructured":"Buttcher, S., et al.: Reliable Information Retrieval Evaluation with Incomplete and Biased Judgements. Proceedings of SIGIR 2007, pp. 63\u201370 (2007)","DOI":"10.1145\/1277741.1277755"},{"key":"31_CR18","doi-asserted-by":"crossref","unstructured":"Sakai, T.: Alternatives to Bpref. Proceedings of SIGIR 2007, pp. 71\u201378 (2007)","DOI":"10.1145\/1277741.1277756"},{"key":"31_CR19","series-title":"Lecture Notes in Computer Science","volume-title":"Evaluating Natural Language Processing Systems","year":"1996","unstructured":"Sparck Jones, K., Galliers, J.R. (eds.): Evaluating Natural Language Processing Systems. LNCS, vol.\u00a01083. Springer, Heidelberg (1996)"},{"key":"31_CR20","doi-asserted-by":"crossref","unstructured":"Garofolo, G., et al.: The 1998 TREC-7 Spoken Document Retrieval Track Overview, http:\/\/trec.nist.gov\/pubs\/trec7\/t7_proceedings.html","DOI":"10.6028\/NIST.SP.500-246.sdr-overview"},{"key":"31_CR21","doi-asserted-by":"crossref","unstructured":"Buckley, C.: Why current IR engines fail. In: Proceedings of SIGIR 2004, pp. 584\u2013585 (2004)","DOI":"10.1145\/1008992.1009132"}],"container-title":["Lecture Notes in Computer Science","Large-Scale Knowledge Resources. Construction and Application"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-540-78159-2_31.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,28]],"date-time":"2025-01-28T21:01:07Z","timestamp":1738098067000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-540-78159-2_31"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[null]]},"ISBN":["9783540781585","9783540781592"],"references-count":21,"URL":"https:\/\/doi.org\/10.1007\/978-3-540-78159-2_31","relation":{},"subject":[]}}