{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:26:57Z","timestamp":1750220817072,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":23,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,3,14]],"date-time":"2020-03-14T00:00:00Z","timestamp":1584144000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100001602","name":"Science Foundation Ireland","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001602","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,3,14]]},"DOI":"10.1145\/3343413.3377996","type":"proceedings-article","created":{"date-parts":[[2020,3,12]],"date-time":"2020-03-12T18:39:33Z","timestamp":1584038373000},"page":"348-352","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Contrasting Human Opinion of Non-factoid Question Answering with Automatic Evaluation"],"prefix":"10.1145","author":[{"given":"Tianbo","family":"Ji","sequence":"first","affiliation":[{"name":"Dublin City University, Dublin, Ireland"}]},{"given":"Yvette","family":"Graham","sequence":"additional","affiliation":[{"name":"Dublin City University, Dublin, Ireland"}]},{"given":"Gareth J.F.","family":"Jones","sequence":"additional","affiliation":[{"name":"Dublin City University, Dublin, Ireland"}]}],"member":"320","published-online":{"date-parts":[[2020,3,14]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"TRECVID 2018: Benchmarking Video Activity Detection, Video Captioning and Matching, Video Storytelling Linking and Video Search. In Proceedings of TRECVID 2018","author":"Awad George","year":"2018","unstructured":"George Awad , Asad A. Butt , Keith Curtis , Yooyoung Lee , Jonathan Fiscus , Afzal Godil , David Joy , Andrew Delgado , Alan F. Smeaton , Yvette Graham , Wessel Kraaij , Georges Qu\u00e9not , Joao Magalhaes , David Semedo , and Saverio Blasi . 2018 . TRECVID 2018: Benchmarking Video Activity Detection, Video Captioning and Matching, Video Storytelling Linking and Video Search. In Proceedings of TRECVID 2018 . Gaithersburg, MD. George Awad, Asad A. Butt, Keith Curtis, Yooyoung Lee, Jonathan Fiscus, Afzal Godil, David Joy, Andrew Delgado, Alan F. Smeaton, Yvette Graham, Wessel Kraaij, Georges Qu\u00e9not, Joao Magalhaes, David Semedo, and Saverio Blasi. 2018. TRECVID 2018: Benchmarking Video Activity Detection, Video Captioning and Matching, Video Storytelling Linking and Video Search. In Proceedings of TRECVID 2018 . Gaithersburg, MD."},{"key":"e_1_3_2_2_2_1","volume-title":"Proceedings of TRECVID 2017","author":"Awad George","year":"2017","unstructured":"George Awad , Asad A. Butt , Jonathan Fiscus , David Joy , Andrew Delgado , Martial Michel , Alan F. Smeaton , Yvette Graham , Wessel Kraaij , Georges Quenot , Maria Eskevich , Roeland Ordelman , Gareth J. F. Jones , and Benoit Huet . 2017 . Trecvid 2017: Evaluating Ad-hoc and Instance Video Search, Event Detection, Video Captioning and Hyperlinking . In Proceedings of TRECVID 2017 . Gaithersburg, MD. George Awad, Asad A. Butt, Jonathan Fiscus, David Joy, Andrew Delgado, Martial Michel, Alan F. Smeaton, Yvette Graham, Wessel Kraaij, Georges Quenot, Maria Eskevich, Roeland Ordelman, Gareth J. F. Jones, and Benoit Huet. 2017. Trecvid 2017: Evaluating Ad-hoc and Instance Video Search, Event Detection, Video Captioning and Hyperlinking. In Proceedings of TRECVID 2017 . Gaithersburg, MD."},{"volume-title":"Proceedings of the Fourth Conference on Machine Translation (Volume 2: Shared Task Papers, Day 1)","author":"Barrault Loic","key":"e_1_3_2_2_3_1","unstructured":"Loic Barrault , Ondrej Bojar , Marta R. Costa-jussa, Christian Federmann , Mark Fishel , Yvette Graham , Barry Haddow , Matthias Huck , Philipp Koehn , Shervin Malmasi , Christof Monz , Mathias Muller , Santanu Pal , Matt Post , and Marcos Zampieri . 2019. Findings of the 2019 Conference on Machine Translation (WMT19) . In Proceedings of the Fourth Conference on Machine Translation (Volume 2: Shared Task Papers, Day 1) . Association for Computational Linguistics , Florence, Italy , 1--61. http:\/\/www.aclweb.org\/anthology\/W19--5301 Loic Barrault, Ondrej Bojar, Marta R. Costa-jussa, Christian Federmann, Mark Fishel, Yvette Graham, Barry Haddow, Matthias Huck, Philipp Koehn, Shervin Malmasi, Christof Monz, Mathias Muller, Santanu Pal, Matt Post, and Marcos Zampieri. 2019. Findings of the 2019 Conference on Machine Translation (WMT19). In Proceedings of the Fourth Conference on Machine Translation (Volume 2: Shared Task Papers, Day 1) . Association for Computational Linguistics, Florence, Italy, 1--61. http:\/\/www.aclweb.org\/anthology\/W19--5301"},{"key":"e_1_3_2_2_4_1","volume-title":"Commonsense for Generative Multi-Hop Question Answering Tasks. CoRR","author":"Bauer Lisa","year":"2018","unstructured":"Lisa Bauer , Yicheng Wang , and Mohit Bansal . 2018. Commonsense for Generative Multi-Hop Question Answering Tasks. CoRR , Vol. abs\/ 1809 .06309 ( 2018 ). arxiv: 1809.06309 http:\/\/arxiv.org\/abs\/1809.06309 Lisa Bauer, Yicheng Wang, and Mohit Bansal. 2018. Commonsense for Generative Multi-Hop Question Answering Tasks. CoRR , Vol. abs\/1809.06309 (2018). arxiv: 1809.06309 http:\/\/arxiv.org\/abs\/1809.06309"},{"key":"e_1_3_2_2_5_1","volume-title":"Findings of the 2017 Conference on Machine Translation (WMT17). In Proceedings of the Second Conference on Machine Translation","volume":"214","author":"Bojar Ondvrej","year":"2017","unstructured":"Ondvrej Bojar , Rajen Chatterjee , Christian Federmann , Yvette Graham , Barry Haddow , Shujian Huang , Matthias Huck , Philipp Koehn , Qun Liu , Varvara Logacheva , Christof Monz , Matteo Negri , Matt Post , Raphael Rubino , Lucia Specia , and Marco Turchi . 2017 . Findings of the 2017 Conference on Machine Translation (WMT17). In Proceedings of the Second Conference on Machine Translation , Volume 2: Shared Task Papers . Association for Computational Linguistics, Copenhagen, Denmark, 169-- 214 . http:\/\/www.aclweb.org\/anthology\/W17--4717 Ondvrej Bojar, Rajen Chatterjee, Christian Federmann, Yvette Graham, Barry Haddow, Shujian Huang, Matthias Huck, Philipp Koehn, Qun Liu, Varvara Logacheva, Christof Monz, Matteo Negri, Matt Post, Raphael Rubino, Lucia Specia, and Marco Turchi. 2017. Findings of the 2017 Conference on Machine Translation (WMT17). In Proceedings of the Second Conference on Machine Translation, Volume 2: Shared Task Papers . Association for Computational Linguistics, Copenhagen, Denmark, 169--214. http:\/\/www.aclweb.org\/anthology\/W17--4717"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W16-2301"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.5555\/2132960.2132962"},{"key":"e_1_3_2_2_8_1","volume-title":"Findings of the 2018 Conference on Machine Translation (WMT18). In Proceedings of the Third Conference on Machine Translation. Association for Computational Linguistics","author":"Bojar Ondrej","year":"2018","unstructured":"Ondrej Bojar , Christian Federmann , Mark Fishel , Yvette Graham , Barry Haddow , Matthias Huck , Philipp Koehn , and Christof Monz . 2018 . Findings of the 2018 Conference on Machine Translation (WMT18). In Proceedings of the Third Conference on Machine Translation. Association for Computational Linguistics , Belgium, Brussels, 272--307. http:\/\/www.aclweb.org\/anthology\/W18--64028 Ondrej Bojar, Christian Federmann, Mark Fishel, Yvette Graham, Barry Haddow, Matthias Huck, Philipp Koehn, and Christof Monz. 2018. Findings of the 2018 Conference on Machine Translation (WMT18). In Proceedings of the Third Conference on Machine Translation. Association for Computational Linguistics, Belgium, Brussels, 272--307. http:\/\/www.aclweb.org\/anthology\/W18--64028"},{"volume-title":"Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics, 85--91","author":"Denkowski M.","key":"e_1_3_2_2_9_1","unstructured":"M. Denkowski and A. Lavie . 2011. Meteor 1.3: Automatic metric for reliable optimization and evaluation of machine translation systems . In Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics, 85--91 . M. Denkowski and A. Lavie. 2011. Meteor 1.3: Automatic metric for reliable optimization and evaluation of machine translation systems. In Proceedings of the Sixth Workshop on Statistical Machine Translation. Association for Computational Linguistics, 85--91."},{"key":"e_1_3_2_2_10_1","volume-title":"Gated-Attention Readers for Text Comprehension. CoRR","author":"Dhingra Bhuwan","year":"2016","unstructured":"Bhuwan Dhingra , Hanxiao Liu , William W. Cohen , and Ruslan Salakhutdinov . 2016. Gated-Attention Readers for Text Comprehension. CoRR , Vol. abs\/ 1606 .01549 ( 2016 ). arxiv: 1606.01549 http:\/\/arxiv.org\/abs\/1606.01549 Bhuwan Dhingra, Hanxiao Liu, William W. Cohen, and Ruslan Salakhutdinov. 2016. Gated-Attention Readers for Text Comprehension. CoRR , Vol. abs\/1606.01549 (2016). arxiv: 1606.01549 http:\/\/arxiv.org\/abs\/1606.01549"},{"key":"e_1_3_2_2_11_1","volume-title":"Can machine translation systems be evaluated by the crowd alone. Natural Language Engineering","author":"Graham Yvette","year":"2016","unstructured":"Yvette Graham , Timothy Baldwin , Alistair Moffat , and Justin Zobel . 2016. Can machine translation systems be evaluated by the crowd alone. Natural Language Engineering , Vol. FirstView ( 1 2016 ), 1--28. https:\/\/doi.org\/10.1017\/S1351324915000339 10.1017\/S1351324915000339 Yvette Graham, Timothy Baldwin, Alistair Moffat, and Justin Zobel. 2016. Can machine translation systems be evaluated by the crowd alone. Natural Language Engineering , Vol. FirstView (1 2016), 1--28. https:\/\/doi.org\/10.1017\/S1351324915000339"},{"key":"e_1_3_2_2_12_1","volume-title":"Translationese in Machine Translation Evaluation. CoRR","author":"Graham Yvette","year":"2019","unstructured":"Yvette Graham , Barry Haddow , and Philipp Koehn . 2019. Translationese in Machine Translation Evaluation. CoRR , Vol. abs\/ 1906 .09833 ( 2019 ). arxiv: 1906.09833 http:\/\/arxiv.org\/abs\/1906.09833 Yvette Graham, Barry Haddow, and Philipp Koehn. 2019. Translationese in Machine Translation Evaluation. CoRR , Vol. abs\/1906.09833 (2019). arxiv: 1906.09833 http:\/\/arxiv.org\/abs\/1906.09833"},{"key":"e_1_3_2_2_13_1","volume-title":"Achieving Human Parity on Automatic Chinese to English News Translation. CoRR","author":"Hassan Hany","year":"2018","unstructured":"Hany Hassan , Anthony Aue , Chang Chen , Vishal Chowdhary , Jonathan Clark , Christian Federmann , Xuedong Huang , Marcin Junczys-Dowmunt , William Lewis , Mu Li , Shujie Liu , Tie-Yan Liu , Renqian Luo , Arul Menezes , Tao Qin , Frank Seide , Xu Tan , Fei Tian , Lijun Wu , Shuangzhi Wu , Yingce Xia , Dongdong Zhang , Zhirui Zhang , and Ming Zhou . 2018. Achieving Human Parity on Automatic Chinese to English News Translation. CoRR , Vol. abs\/ 1803 .05567 ( 2018 ). arxiv: 1803.05567 http:\/\/arxiv.org\/abs\/1803.05567 Hany Hassan, Anthony Aue, Chang Chen, Vishal Chowdhary, Jonathan Clark, Christian Federmann, Xuedong Huang, Marcin Junczys-Dowmunt, William Lewis, Mu Li, Shujie Liu, Tie-Yan Liu, Renqian Luo, Arul Menezes, Tao Qin, Frank Seide, Xu Tan, Fei Tian, Lijun Wu, Shuangzhi Wu, Yingce Xia, Dongdong Zhang, Zhirui Zhang, and Ming Zhou. 2018. Achieving Human Parity on Automatic Chinese to English News Translation. CoRR , Vol. abs\/1803.05567 (2018). arxiv: 1803.05567 http:\/\/arxiv.org\/abs\/1803.05567"},{"key":"e_1_3_2_2_14_1","volume-title":"Attention-Guided Answer Distillation for Machine Reading Comprehension. CoRR","author":"Hu Minghao","year":"2018","unstructured":"Minghao Hu , Yuxing Peng , Furu Wei , Zhen Huang , Dongsheng Li , Nan Yang , and Ming Zhou . 2018. Attention-Guided Answer Distillation for Machine Reading Comprehension. CoRR , Vol. abs\/ 1808 .07644 ( 2018 ). arxiv: 1808.07644 http:\/\/arxiv.org\/abs\/1808.07644 Minghao Hu, Yuxing Peng, Furu Wei, Zhen Huang, Dongsheng Li, Nan Yang, and Ming Zhou. 2018. Attention-Guided Answer Distillation for Machine Reading Comprehension. CoRR , Vol. abs\/1808.07644 (2018). arxiv: 1808.07644 http:\/\/arxiv.org\/abs\/1808.07644"},{"key":"e_1_3_2_2_15_1","volume-title":"G\u00e1 bor Melis, and Edward Grefenstette.","author":"Kocisk\u00fd Tom\u00e1","year":"2017","unstructured":"Tom\u00e1 s Kocisk\u00fd , Jonathan Schwarz , Phil Blunsom , Chris Dyer , Karl Moritz Hermann , G\u00e1 bor Melis, and Edward Grefenstette. 2017 . The NarrativeQA Reading Comprehension Challenge. CoRR , Vol. abs\/ 1712 .07040 (2017). arxiv: 1712.07040 http:\/\/arxiv.org\/abs\/1712.07040 Tom\u00e1 s Kocisk\u00fd , Jonathan Schwarz, Phil Blunsom, Chris Dyer, Karl Moritz Hermann, G\u00e1 bor Melis, and Edward Grefenstette. 2017. The NarrativeQA Reading Comprehension Challenge. CoRR , Vol. abs\/1712.07040 (2017). arxiv: 1712.07040 http:\/\/arxiv.org\/abs\/1712.07040"},{"volume-title":"Proceedings of the 2003 Conference of the North American","author":"Lin Chin-Yew","key":"e_1_3_2_2_16_1","unstructured":"Chin-Yew Lin and Eduard Hovy . 2003. Automatic evaluation of summaries using n-gram co-occurrence statistics . In Proceedings of the 2003 Conference of the North American Chapter of the Association for Computational Linguistics on Human Language Technology-Volume 1 . Association for Computational Linguistics , 71--78. Chin-Yew Lin and Eduard Hovy. 2003. Automatic evaluation of summaries using n-gram co-occurrence statistics. In Proceedings of the 2003 Conference of the North American Chapter of the Association for Computational Linguistics on Human Language Technology-Volume 1 . Association for Computational Linguistics, 71--78."},{"key":"e_1_3_2_2_17_1","volume-title":"Neural Machine Reading Comprehension: Methods and Trends. CoRR","author":"Liu Shanshan","year":"2019","unstructured":"Shanshan Liu , Xin Zhang , Sheng Zhang , Hui Wang , and Weiming Zhang . 2019. Neural Machine Reading Comprehension: Methods and Trends. CoRR , Vol. abs\/ 1907 .01118 ( 2019 ). arxiv: 1907.01118 http:\/\/arxiv.org\/abs\/1907.01118 Shanshan Liu, Xin Zhang, Sheng Zhang, Hui Wang, and Weiming Zhang. 2019. Neural Machine Reading Comprehension: Methods and Trends. CoRR , Vol. abs\/1907.01118 (2019). arxiv: 1907.01118 http:\/\/arxiv.org\/abs\/1907.01118"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-3601"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-6301"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/0304-3959(85)90287-8"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1453"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/W18-2611"}],"event":{"name":"CHIIR '20: Conference on Human Information Interaction and Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval","SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Vancouver BC Canada","acronym":"CHIIR '20"},"container-title":["Proceedings of the 2020 Conference on Human Information Interaction and Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3343413.3377996","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3343413.3377996","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:13:07Z","timestamp":1750201987000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3343413.3377996"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,3,14]]},"references-count":23,"alternative-id":["10.1145\/3343413.3377996","10.1145\/3343413"],"URL":"https:\/\/doi.org\/10.1145\/3343413.3377996","relation":{},"subject":[],"published":{"date-parts":[[2020,3,14]]},"assertion":[{"value":"2020-03-14","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}