{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,12]],"date-time":"2025-12-12T13:08:23Z","timestamp":1765544903705,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"FCT","award":["PTDC\/CCI-COM\/31857\/2017"],"award-info":[{"award-number":["PTDC\/CCI-COM\/31857\/2017"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591892","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:59Z","timestamp":1689726179000},"page":"2871-2879","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["tieval: An Evaluation Framework for Temporal Information Extraction Systems"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3226-9189","authenticated-orcid":false,"given":"Hugo","family":"Sousa","sequence":"first","affiliation":[{"name":"INESC TEC & University of Porto, Porto, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8767-8126","authenticated-orcid":false,"given":"Ricardo","family":"Campos","sequence":"additional","affiliation":[{"name":"INESC TEC & Polytechnic Institute of Tomar; Ci2 - Smart Cities Research Center, Porto, Portugal"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5475-1382","authenticated-orcid":false,"given":"Al\u00edpio M\u00e1rio","family":"Jorge","sequence":"additional","affiliation":[{"name":"INESC TEC & University of Porto, Porto, Portugal"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"doi-asserted-by":"publisher","key":"e_1_3_2_1_1_1","DOI":"10.1145\/182.358434"},{"key":"e_1_3_2_1_2_1","volume-title":"Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics","author":"Bittar Andr\u00e9","year":"2011","unstructured":"Andr\u00e9 Bittar, Pascal Amsili, Pascal Denis, and Laurence Danlos. 2011. French TimeBank: An ISO-TimeML Annotated Reference Corpus. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, Portland, Oregon, USA, 130--134. https:\/\/aclanthology.org\/P11-2023"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_3_1","DOI":"10.4000\/books.aaccademia.1732"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_4_1","DOI":"10.1145\/2619088"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_5_1","DOI":"10.1007\/s10791-017-9302-1"},{"key":"e_1_3_2_1_6_1","volume-title":"Proceedings of the 5th Linguistic Annotation Workshop. Association for Computational Linguistics","author":"Caselli Tommaso","year":"2011","unstructured":"Tommaso Caselli, Valentina Bartalesi Lenzi, Rachele Sprugnoli, Emanuele Pianta, and Irina Prodanof. 2011. Annotating Events, Temporal Expressions and Relations in Italian: the It-Timeml Experience for the Ita-TimeBank. In Proceedings of the 5th Linguistic Annotation Workshop. Association for Computational Linguistics, Portland, Oregon, USA, 143--151. https:\/\/aclanthology.org\/W11-0418"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_7_1","DOI":"10.3115\/v1\/P14-2082"},{"key":"e_1_3_2_1_8_1","volume-title":"Proceedings of the 2013 NAACL HLT Demonstration Session. Association for Computational Linguistics","author":"Chen Wei-Te","year":"2013","unstructured":"Wei-Te Chen and Will Styler. 2013. Anafora: A Web-based General Purpose Annotation Tool. In Proceedings of the 2013 NAACL HLT Demonstration Session. Association for Computational Linguistics, Atlanta, Georgia, 14--19. https:\/\/aclanthology.org\/N13-3004"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the Eighth International Conference on Language Resources and Evaluation (LREC'12)","author":"Costa Francisco","year":"2012","unstructured":"Francisco Costa and Ant\u00f3nio Branco. 2012. TimeBankPT: A TimeML Annotated Corpus of Portuguese. In Proceedings of the Eighth International Conference on Language Resources and Evaluation (LREC'12). European Language Resources Association (ELRA), Istanbul, Turkey, 3727--3734. http:\/\/www.lrec-conf.org\/proceedings\/lrec2012\/pdf\/246_Paper.pdf"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_10_1","DOI":"10.1016\/j.ipm.2015.05.002"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_11_1","DOI":"10.1145\/152947.152953"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_12_1","DOI":"10.1017\/S1351324914000060"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_13_1","DOI":"10.35111\/pcbv-jq63"},{"key":"e_1_3_2_1_14_1","volume-title":"ModeS TimeBank: A modern Spanish TimeBank Corpus. Revista de la Sociedad Espanola del Procesamiento del Lenguaje Natural","author":"Nieto Marta Guerrero","year":"2011","unstructured":"Marta Guerrero Nieto, Roser Saur\u00ed, and Miguel-Angel Bernab\u00e9 -Poveda. 2011. ModeS TimeBank: A modern Spanish TimeBank Corpus. Revista de la Sociedad Espanola del Procesamiento del Lenguaje Natural, Vol. 47 (4 2011), 259--267."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_15_1","DOI":"10.1613\/jair.1.11727"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_16_1","DOI":"10.1109\/TASLP.2020.3027201"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_17_1","DOI":"10.3115\/v1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_18_1","DOI":"10.1609\/aaai.v24i1.7512"},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of the 2010 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics","author":"Mazur Pawel","year":"2010","unstructured":"Pawel Mazur and Robert Dale. 2010. WikiWars: A New Corpus for Research on Temporal Expressions. In Proceedings of the 2010 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, Cambridge, MA, 913--922. https:\/\/aclanthology.org\/D10-1089"},{"key":"e_1_3_2_1_20_1","volume-title":"Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC'16)","author":"Minard Anne-Lyse","year":"2016","unstructured":"Anne-Lyse Minard, Manuela Speranza, Ruben Urizar, Bego\u00f1a Altuna, Marieke van Erp, Anneleen Schoen, and Chantal van Son. 2016. MEANTIME, the NewsReader Multilingual Event and Time Corpus. In Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC'16). European Language Resources Association (ELRA), Portorovz, Slovenia, 4417--4422. https:\/\/aclanthology.org\/L16-1699"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_21_1","DOI":"10.18653\/v1\/W19-5929"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_22_1","DOI":"10.18653\/v1\/P18-1212"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_23_1","DOI":"10.18653\/v1"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_24_1","DOI":"10.18653\/v1\/2020.emnlp-main.88"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_25_1","DOI":"10.18653\/v1\/N18-1077"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_26_1","DOI":"10.18653\/v1\/P18-1122"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_27_1","DOI":"10.18653\/v1\/W16-5706"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_28_1","DOI":"10.18653\/v1\/N18-1202"},{"key":"e_1_3_2_1_29_1","first-page":"28","article-title":"a. TimeML: Robust Specification of Event and Temporal Expressions in Text","volume":"3","author":"Pustejovsky James","year":"2003","unstructured":"James Pustejovsky, Jos\u00e9 M Castano, Robert Ingria, Roser Sauri, Robert J Gaizauskas, Andrea Setzer, Graham Katz, and Dragomir R Radev. 2003 a. TimeML: Robust Specification of Event and Temporal Expressions in Text. New Directions in Question Answering, Vol. 3 (2003), 28--34.","journal-title":"New Directions in Question Answering"},{"key":"e_1_3_2_1_30_1","volume-title":"Proceedings of Corpus Linguistics","volume":"2003","author":"Gaizauskas Rob","year":"2003","unstructured":"James Pustejovsky, Patrick Hanks, Roser Saur\u00ed, Andrew See, Rob Gaizauskas, Andrea Setzer, Dragomir Radev, Beth Sundheim, David Day, Lisa Ferro, and Marcia Lazo. 2003 b. The TimeBank corpus. Proceedings of Corpus Linguistics, Vol. 2003 (4 2003)."},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_31_1","DOI":"10.18653\/v1\/P16-1207"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_32_1","DOI":"10.1007\/s10462-022-10338-7"},{"doi-asserted-by":"crossref","unstructured":"Hugo Sousa Ricardo Campos and Al\u00edpio Jorge. 2023. TEI2GO: A Multilingual Approach for Fast Temporal Expression Identification. (2023).","key":"e_1_3_2_1_33_1","DOI":"10.1145\/3583780.3615130"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC'14)","author":"Str\u00f6tgen Jannik","year":"2014","unstructured":"Jannik Str\u00f6tgen, Thomas B\u00f6gel, Julian Zell, Ayser Armiti, Tran Van Canh, and Michael Gertz. 2014. Extending HeidelTime for Temporal Expressions Referring to Historic Dates. In Proceedings of the Ninth International Conference on Language Resources and Evaluation (LREC'14). European Language Resources Association (ELRA), Reykjavik, Iceland, 2390--2397. http:\/\/www.lrec-conf.org\/proceedings\/lrec2014\/pdf\/849_Paper.pdf"},{"key":"e_1_3_2_1_35_1","volume-title":"Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC","author":"Str\u00f6tgen Jannik","year":"2018","unstructured":"Jannik Str\u00f6tgen, Anne-Lyse Minard, Lukas Lange, Manuela Speranza, and Bernardo Magnini. 2018. KRAUTS: A German Temporally Annotated News Corpus. In Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018), Nicoletta Calzolari, Khalid Choukri, Christopher Cieri, Thierry Declerck, Koiti Hasida, Hitoshi Isahara, Bente Maegaard, Joseph Mariani, Asuncion Moreno, Jan Odijk, Stelios Piperidis, and Takenobu Tokunaga (Eds.). European Language Resources Association (ELRA), Miyazaki, Japan, 536--540. https:\/\/aclanthology.org\/L18-1085"},{"key":"e_1_3_2_1_36_1","first-page":"13","volume-title":"Second Joint Conference on Lexical and Computational Semantics (*SEM)","volume":"19","author":"Str\u00f6tgen Jannik","year":"2013","unstructured":"Jannik Str\u00f6tgen, Julian Zell, and Michael Gertz. 2013. HeidelTime: Tuning English and Developing Spanish Resources for TempEval-3. In Second Joint Conference on Lexical and Computational Semantics (*SEM), Volume 2: Proceedings of the Seventh International Workshop on Semantic Evaluation (SemEval 2013). Association for Computational Linguistics, Atlanta, Georgia, USA, 15--19. https:\/\/aclanthology.org\/S13-2003"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_37_1","DOI":"10.1162\/tacl_a_00172"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_38_1","DOI":"10.1136\/amiajnl-2013-001628"},{"key":"e_1_3_2_1_39_1","volume-title":"Temporal Evaluation. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics","author":"UzZaman Naushad","year":"2011","unstructured":"Naushad UzZaman and James Allen. 2011. Temporal Evaluation. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, Portland, Oregon, USA, 351--356. https:\/\/aclanthology.org\/P11-2061"},{"key":"e_1_3_2_1_40_1","first-page":"13","volume-title":"Second Joint Conference on Lexical and Computational Semantics (*SEM)","volume":"9","author":"UzZaman Naushad","year":"2013","unstructured":"Naushad UzZaman, Hector Llorens, Leon Derczynski, James Allen, Marc Verhagen, and James Pustejovsky. 2013. SemEval-2013 Task 1: TempEval-3: Evaluating Time Expressions, Events, and Temporal Relations. In Second Joint Conference on Lexical and Computational Semantics (*SEM), Volume 2: Proceedings of the Seventh International Workshop on Semantic Evaluation (SemEval 2013). Association for Computational Linguistics, Atlanta, Georgia, USA, 1--9. https:\/\/aclanthology.org\/S13-2001"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_41_1","DOI":"10.3115\/1621474.1621488"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_42_1","DOI":"10.5555\/1859664.1859674"},{"doi-asserted-by":"publisher","key":"e_1_3_2_1_43_1","DOI":"10.18653\/v1"}],"event":{"sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"acronym":"SIGIR '23","name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Taipei Taiwan"},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591892","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591892","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:07Z","timestamp":1750178287000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591892"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":43,"alternative-id":["10.1145\/3539618.3591892","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591892","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}