{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,24]],"date-time":"2026-01-24T19:24:44Z","timestamp":1769282684545,"version":"3.49.0"},"reference-count":23,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2006,9,1]],"date-time":"2006-09-01T00:00:00Z","timestamp":1157068800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2006,9,1]],"date-time":"2006-09-01T00:00:00Z","timestamp":1157068800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval"],"published-print":{"date-parts":[[2006,11]]},"DOI":"10.1007\/s10791-006-9003-7","type":"journal-article","created":{"date-parts":[[2006,8,31]],"date-time":"2006-08-31T17:10:36Z","timestamp":1157044236000},"page":"565-587","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":21,"title":["Methods for automatically evaluating answers to complex questions"],"prefix":"10.1007","volume":"9","author":[{"given":"Jimmy","family":"Lin","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dina","family":"Demner-Fushman","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2006,9,1]]},"reference":[{"key":"9003_CR1","doi-asserted-by":"crossref","unstructured":"Amig\u00f3, E., Gonzalo, J., Peinado, V., Pe\u00f1as, A., & Verdejo, F. (2004). An empirical study of information synthesis task. In Proceedings of the 42nd Annual Meeting of the Association for Computational Linguistics (ACL 2004).","DOI":"10.3115\/1218955.1218982"},{"key":"9003_CR2","doi-asserted-by":"crossref","unstructured":"Babych, B., & Hartley, A. (2004). Extending the BLEU MT evaluation method with frequency weightings. In Proceedings of the 42nd Annual Meeting of the Association for Computational Linguistics (ACL 2004).","DOI":"10.3115\/1218955.1219034"},{"key":"9003_CR3","unstructured":"Banerjee, S., & Lavie, A. (2005). METEOR: An automatic metric for mt evaluation with improved correlation with human judgments. In Proceedings of ACL 2005 Workshop on Intrinsic and Extrinsic Evaluation Measures for MT and\/or Summarization."},{"issue":"4","key":"9003_CR4","doi-asserted-by":"crossref","first-page":"329","DOI":"10.1080\/00031305.1995.10476177","volume":"49","author":"S. Chib","year":"1995","unstructured":"Chib, S., & Greenberg, E. (1995). Understanding the Metropolis-Hastings algorithm. American Statistician, 49(4), 329\u2013345.","journal-title":"American Statistician"},{"key":"9003_CR5","unstructured":"Dang, H. (2005). Overview of DUC 2005. In Proceedings of the 2005 Document Understanding Conference (DUC 2005) at NLT\/EMNLP 2005."},{"key":"9003_CR6","doi-asserted-by":"crossref","unstructured":"Doddington, G. (2002). Automatic evaluation of machine translation quality using n-gram co-occurrence statistics. In Proceeding of 2002 Human Language Technology Conference (HLT 2002).","DOI":"10.3115\/1289189.1289273"},{"key":"9003_CR7","unstructured":"Hildebrandt, W., Katz, B., & Lin, J. (2004). Answering definition questions with multiple knowledge sources. In Proceedings of the 2004 Human Language Technology Conference and the North American Chapter of the Association for Computational Linguistics Annual Meeting (HLT\/NAACL 2004)."},{"key":"9003_CR8","unstructured":"Hovy, E., Lin, C.-Y., & Zhou, L. (2005). Evaluating DUC 2005 using Basic Elements. In Proceedings of the 2005 Document Understanding Conference (DUC 2005) at NLT\/EMNLP 2005."},{"key":"9003_CR9","doi-asserted-by":"crossref","unstructured":"Kraaij, W., & Pohlmann, R. (1996). Viewing stemming as recall enhancement. In Proceedings of the 19th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 1996).","DOI":"10.1145\/243199.243209"},{"key":"9003_CR10","unstructured":"Kulesza, A., & Shieber, S. M. (2004). A learning approach to improving sentence level MT evaluation. In Proceedings of the 10th International Conference on Theoretical and Methodological Issues in Machine Translation."},{"key":"9003_CR11","unstructured":"Lin, J., & Demner-Fushman, D. (2005). Evaluating summaries and answers: Two sides of the same coin? In Proceedings of the ACL 2005 Workshop on Intrinsic and Extrinsic Evaluation Measures for MT and\/or Summarization."},{"key":"9003_CR12","doi-asserted-by":"crossref","unstructured":"Lin, J., & Demner-Fushman, D. (2006). Will pyramids built of nuggets topple over? In Proceedings of the 2006 Human Language Technology Conference and the North American Chapter of the Association for Computational Linguistics Annual Meeting (HLT\/NAACL 2006).","DOI":"10.3115\/1220835.1220884"},{"key":"9003_CR13","doi-asserted-by":"crossref","unstructured":"Lin, C.-Y., & Hovy, E. (2003). Automatic evaluation of summaries using n-gram co-occurrence statistics. In Proceedings of the 2003 Human Language Technology Conference and the North American Chapter of the Association for Computational Linguistics Annual Meeting (HLT\/NAACL 2003).","DOI":"10.3115\/1073445.1073465"},{"key":"9003_CR14","doi-asserted-by":"crossref","unstructured":"Lin, C.-Y., & Och., F. J. (2004). Orange: A method for evaluating automatic evaluation metrics for machine translation. In Proceedings of the 20th International Conference on Computational Linguistics (COLING 2004).","DOI":"10.3115\/1220355.1220427"},{"key":"9003_CR15","doi-asserted-by":"crossref","unstructured":"Lin, J., Abels, E., Demner-Fushman, D., Oard, D. W., Wu, P., & Wu, Y. (2005). A menagerie of tracks at Maryland: HARD, Enterprise, QA, and Genomics, oh my! In Proceedings of the Fourteenth Text REtrieval Conference (TREC 2005).","DOI":"10.21236\/ADA456308"},{"key":"9003_CR16","unstructured":"Nenkova, A., & Passonneau, R. (2004). Evaluating content selection in summarization: The pyramid method. In Proceedings of the 2004 Human Language Technology Conference and the North American Chapter of the Association for Computational Linguistics Annual Meeting (HLT\/NAACL 2004)."},{"key":"9003_CR17","doi-asserted-by":"crossref","unstructured":"Papineni, K., Roukos, S., Ward, T., & Zhu, W.-J. (2002). BLEU: A method for automatic evaluation of machine translation. In Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics (ACL 2002).","DOI":"10.3115\/1073083.1073135"},{"key":"9003_CR18","doi-asserted-by":"crossref","unstructured":"Soricut, R., & Brill, E. (2004). A unified framework for automatic evaluation using n-gram co-occurrence statistics. In Proceedings of the 42nd Annual Meeting of the Association for Computational Linguistics (ACL 2004).","DOI":"10.3115\/1218955.1219033"},{"issue":"5","key":"9003_CR19","doi-asserted-by":"crossref","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"E. M. Voorhees","year":"2000","unstructured":"Voorhees, E. M. (2000). Variations in relevance judgments and the measurement of retrieval effectiveness. Information Processing and Management, 36(5), 697\u2013716.","journal-title":"Information Processing and Management"},{"key":"9003_CR20","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2003). Overview of the TREC 2003 question answering track. In Proceedings of the Twelfth Text REtrieval Conference (TREC 2003).","DOI":"10.6028\/NIST.SP.500-255"},{"key":"9003_CR21","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M. (2005). Using question series to evaluate question answering system effectiveness. In Proceedings of the 2005 Human Language Technology Conference and Conference on Empirical Methods in Natural Language Processing (HLT\/EMNLP 2005).","DOI":"10.3115\/1220575.1220613"},{"key":"9003_CR22","doi-asserted-by":"crossref","unstructured":"Voorhees, E. M., Tice, D. M. (2000). Building a question answering test collection. In Proceedings of the 23rd Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2000).","DOI":"10.1145\/345508.345577"},{"key":"9003_CR23","doi-asserted-by":"crossref","unstructured":"Xu, J., Weischedel, R., & Licuanan, A. (2004). Evaluation of an extraction-based approach to answering definition questions. In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2004).","DOI":"10.1145\/1008992.1009064"}],"container-title":["Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-006-9003-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-006-9003-7\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-006-9003-7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-006-9003-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,1,10]],"date-time":"2025-01-10T18:44:20Z","timestamp":1736534660000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-006-9003-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2006,9,1]]},"references-count":23,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2006,11]]}},"alternative-id":["9003"],"URL":"https:\/\/doi.org\/10.1007\/s10791-006-9003-7","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2006,9,1]]},"assertion":[{"value":"19 November 2005","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 May 2006","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"1 September 2006","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}