{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,11]],"date-time":"2026-02-11T19:14:44Z","timestamp":1770837284536,"version":"3.50.1"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030916688","type":"print"},{"value":"9783030916695","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-91669-5_35","type":"book-chapter","created":{"date-parts":[[2021,11,30]],"date-time":"2021-11-30T00:03:08Z","timestamp":1638230588000},"page":"453-470","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":17,"title":["Automated Mining of\u00a0Leaderboards for\u00a0Empirical AI Research"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0021-9729","authenticated-orcid":false,"given":"Salomon","family":"Kabongo","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6616-9509","authenticated-orcid":false,"given":"Jennifer","family":"D\u2019Souza","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0698-2864","authenticated-orcid":false,"given":"S\u00f6ren","family":"Auer","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,11,30]]},"reference":[{"key":"35_CR1","unstructured":"AI metrics. https:\/\/www.eff.org\/ai\/metrics. Accessed 26 Apr 2021"},{"key":"35_CR2","unstructured":"Natural Language Inference. https:\/\/paperswithcode.com\/task\/natural-language-inference. Accessed 22 Apr 2021"},{"key":"35_CR3","unstructured":"Nlp-progress. http:\/\/nlpprogress.com\/. Accessed 26 Apr 2021"},{"key":"35_CR4","unstructured":"paperswithcode.com. https:\/\/paperswithcode.com\/. Accessed 26 Apr 2021"},{"key":"35_CR5","unstructured":"Reddit sota. https:\/\/github.com\/RedditSota\/state-of-the-art-result-for-machine-learning-problems. Accessed 26 Apr 2021"},{"key":"35_CR6","unstructured":"Squad explorer. https:\/\/rajpurkar.github.io\/SQuAD-explorer\/. Accessed 26 Apr 2021"},{"key":"35_CR7","unstructured":"Anteghini, M., D\u2019Souza, J., Dos Santos, V.A.M., Auer, S.: SciBERT-based semantification of bioassays in the open research knowledge graph. In: EKAW-PD 2020, pp. 22\u201330 (2020)"},{"key":"35_CR8","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1007\/978-3-030-64452-9_8","volume-title":"Digital Libraries at Times of Massive Societal Transition","author":"M Anteghini","year":"2020","unstructured":"Anteghini, M., D\u2019Souza, J., Martins\u00a0dos\u00a0Santos, V.A.P., Auer, S.: Representing semantified biological assays in the open research knowledge graph. In: Ishita, E., Pang, N.L.S., Zhou, L. (eds.) ICADL 2020. LNCS, vol. 12504, pp. 89\u201398. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-64452-9_8"},{"key":"35_CR9","doi-asserted-by":"publisher","unstructured":"Auer, S.: Towards an open research knowledge graph, January 2018. https:\/\/doi.org\/10.5281\/zenodo.1157185","DOI":"10.5281\/zenodo.1157185"},{"key":"35_CR10","doi-asserted-by":"crossref","unstructured":"Augenstein, I., Das, M., Riedel, S., Vikraman, L., McCallum, A.: SemEval 2017 task 10: ScienceIE - extracting keyphrases and relations from scientific publications. In: SemEval@ACL (2017)","DOI":"10.18653\/v1\/S17-2091"},{"key":"35_CR11","doi-asserted-by":"crossref","unstructured":"Beltagy, I., Lo, K., Cohan, A.: SciBERT: a pretrained language model for scientific text. arXiv preprint arXiv:1903.10676 (2019)","DOI":"10.18653\/v1\/D19-1371"},{"key":"35_CR12","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"251","DOI":"10.1007\/978-3-030-45439-5_17","volume-title":"Advances in Information Retrieval","author":"A Brack","year":"2020","unstructured":"Brack, A., D\u2019Souza, J., Hoppe, A., Auer, S., Ewerth, R., et al.: Domain-independent extraction of scientific concepts from research articles. In: Jose, J.M. (ed.) ECIR 2020. LNCS, vol. 12035, pp. 251\u2013266. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-45439-5_17"},{"key":"35_CR13","unstructured":"Chiarelli, A., Johnson, R., Richens, E., Pinfield, S.: Accelerating scholarly communication: the transformative role of preprints (2019)"},{"key":"35_CR14","doi-asserted-by":"crossref","unstructured":"Dai, Z., Yang, Z., Yang, Y., Carbonell, J.G., Le, Q., Salakhutdinov, R.: Transformer-XL: attentive language models beyond a fixed-length context. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 2978\u20132988 (2019)","DOI":"10.18653\/v1\/P19-1285"},{"key":"35_CR15","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)"},{"key":"35_CR16","doi-asserted-by":"crossref","unstructured":"D\u2019Souza, J., Auer, S., Pedersen, T.: SemEval-2021 task 11: NLPcontributiongraph - structuring scholarly NLP contributions for a research knowledge graph. In: Proceedings of the Fifteenth Workshop on Semantic Evaluation. Association for Computational Linguistics, Bangkok, August 2021","DOI":"10.18653\/v1\/2021.semeval-1.44"},{"key":"35_CR17","doi-asserted-by":"crossref","unstructured":"D\u2019Souza, J., Auer, S., Pederson, T.: SemEval-2021 task 11: NLPContributionGraph - structuring scholarly NLP contributions for a research knowledge graph, May 2021. https:\/\/zenodo.org\/record\/4737071","DOI":"10.18653\/v1\/2021.semeval-1.44"},{"key":"35_CR18","unstructured":"D\u2019Souza, J., Hoppe, A., Brack, A., Jaradeh, M.Y., Auer, S., Ewerth, R.: The STEM-ECR dataset: grounding scientific entity references in stem scholarly content to authoritative encyclopedic and lexicographic sources. In: LREC, Marseille, France, pp. 2192\u20132203, May 2020"},{"key":"35_CR19","doi-asserted-by":"crossref","unstructured":"D\u2019Souza, J., Auer, S.: Sentence, phrase, and triple annotations to build a knowledge graph of natural language processing contributions\u2013a trial dataset. J. Data Inf. Sci. 20210429 (2021)","DOI":"10.2478\/jdis-2021-0023"},{"key":"35_CR20","doi-asserted-by":"crossref","unstructured":"G\u00e1bor, K., Buscaldi, D., Schumann, A.K., QasemiZadeh, B., Zargayouna, H., Charnois, T.: SemEval-2018 task 7: semantic relation extraction and classification in scientific papers. In: Proceedings of The 12th International Workshop on Semantic Evaluation, pp. 679\u2013688 (2018)","DOI":"10.18653\/v1\/S18-1111"},{"key":"35_CR21","unstructured":"Ghasemi-Gol, M., Szekely, P.: TabVec: table vectors for classification of web tables. arXiv preprint arXiv:1802.06290 (2018)"},{"key":"35_CR22","unstructured":"Handschuh, S., QasemiZadeh, B.: The ACL RD-TEC: a dataset for benchmarking terminology extraction and classification in computational linguistics. In: COLING 2014: 4th International Workshop on Computational Terminology (2014)"},{"key":"35_CR23","doi-asserted-by":"crossref","unstructured":"Herzig, J., Nowak, P.K., Mueller, T., Piccinno, F., Eisenschlos, J.: TaPas: weakly supervised table parsing via pre-training. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 4320\u20134333 (2020)","DOI":"10.18653\/v1\/2020.acl-main.398"},{"key":"35_CR24","doi-asserted-by":"crossref","unstructured":"Hou, Y., Jochim, C., Gleize, M., Bonin, F., Ganguly, D.: Identification of tasks, datasets, evaluation metrics, and numeric scores for scientific leaderboards construction. arXiv preprint arXiv:1906.09317 (2019)","DOI":"10.18653\/v1\/P19-1513"},{"key":"35_CR25","doi-asserted-by":"crossref","unstructured":"Hou, Y., Jochim, C., Gleize, M., Bonin, F., Ganguly, D.: TDMSci: a specialized corpus for scientific literature entity tagging of tasks datasets and metrics. In: Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Main Volume, pp. 707\u2013714 (2021)","DOI":"10.18653\/v1\/2021.eacl-main.59"},{"key":"35_CR26","doi-asserted-by":"crossref","unstructured":"Jain, S., van Zuylen, M., Hajishirzi, H., Beltagy, I.: SciREX: a challenge dataset for document-level information extraction. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7506\u20137516 (2020)","DOI":"10.18653\/v1\/2020.acl-main.670"},{"key":"35_CR27","doi-asserted-by":"crossref","unstructured":"Jaradeh, M.Y., et al.: Open research knowledge graph: next generation infrastructure for semantic scholarly knowledge. In: Proceedings of the 10th International Conference on Knowledge Capture, pp. 243\u2013246 (2019)","DOI":"10.1145\/3360901.3364435"},{"key":"35_CR28","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-3-030-64452-9_1","volume-title":"Digital Libraries at Times of Massive Societal Transition","author":"M Jiang","year":"2020","unstructured":"Jiang, M., D\u2019Souza, J., Auer, S., Downie, J.S.: Improving scholarly knowledge representation: evaluating BERT-based models for scientific relation classification. In: Ishita, E., Pang, N.L.S., Zhou, L. (eds.) ICADL 2020. LNCS, vol. 12504, pp. 3\u201319. Springer, Cham (2020). https:\/\/doi.org\/10.1007\/978-3-030-64452-9_1"},{"issue":"3","key":"35_CR29","doi-asserted-by":"publisher","first-page":"258","DOI":"10.1087\/20100308","volume":"23","author":"AE Jinha","year":"2010","unstructured":"Jinha, A.E.: Article 50 million: an estimate of the number of scholarly articles in existence. Learn. Publ. 23(3), 258\u2013263 (2010)","journal-title":"Learn. Publ."},{"key":"35_CR30","doi-asserted-by":"crossref","unstructured":"Kardas, M., et al.: AxCell: automatic extraction of results from machine learning papers. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 8580\u20138594 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.692"},{"issue":"1","key":"35_CR31","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41597-018-0005-2","volume":"6","author":"O Kononova","year":"2019","unstructured":"Kononova, O., et al.: Text-mined dataset of inorganic materials synthesis recipes. Sci. Data 6(1), 1\u201311 (2019)","journal-title":"Sci. Data"},{"key":"35_CR32","doi-asserted-by":"crossref","unstructured":"Kulkarni, C., Xu, W., Ritter, A., Machiraju, R.: An annotated corpus for machine reading of instructions in wet lab protocols. In: NAACL: HLT, Volume 2 (Short Papers), New Orleans, Louisiana, pp. 97\u2013106, June 2018. https:\/\/doi.org\/10.18653\/v1\/N18-2016","DOI":"10.18653\/v1\/N18-2016"},{"key":"35_CR33","unstructured":"Kuniyoshi, F., Makino, K., Ozawa, J., Miwa, M.: Annotating and extracting synthesis process of all-solid-state batteries from scientific literature. In: LREC, pp. 1941\u20131950 (2020)"},{"key":"35_CR34","doi-asserted-by":"crossref","unstructured":"Liu, Y., Bai, K., Mitra, P., Giles, C.L.: TableSeer: automatic table metadata extraction and searching in digital libraries. In: Proceedings of the 7th ACM\/IEEE-CS Joint Conference on Digital Libraries, pp. 91\u2013100 (2007)","DOI":"10.1145\/1255175.1255193"},{"key":"35_CR35","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"473","DOI":"10.1007\/978-3-642-04346-8_62","volume-title":"Research and Advanced Technology for Digital Libraries","author":"P Lopez","year":"2009","unstructured":"Lopez, P.: GROBID: combining automatic bibliographic data recognition and term extraction for scholarship publications. In: Agosti, M., Borbinha, J., Kapidakis, S., Papatheodorou, C., Tsakonas, G. (eds.) ECDL 2009. LNCS, vol. 5714, pp. 473\u2013474. Springer, Heidelberg (2009). https:\/\/doi.org\/10.1007\/978-3-642-04346-8_62"},{"key":"35_CR36","doi-asserted-by":"crossref","unstructured":"Luan, Y., He, L., Ostendorf, M., Hajishirzi, H.: Multi-task identification of entities, relations, and coreference for scientific knowledge graph construction. arXiv preprint arXiv:1808.09602 (2018)","DOI":"10.18653\/v1\/D18-1360"},{"key":"35_CR37","doi-asserted-by":"crossref","unstructured":"Luan, Y., He, L., Ostendorf, M., Hajishirzi, H.: Multi-task identification of entities, relations, and coreference for scientific knowledge graph construction. In: EMNLP (2018)","DOI":"10.18653\/v1\/D18-1360"},{"issue":"4","key":"35_CR38","doi-asserted-by":"publisher","first-page":"701","DOI":"10.1162\/COLI_a_00239","volume":"41","author":"CD Manning","year":"2015","unstructured":"Manning, C.D.: Computational linguistics and deep learning. Comput. Linguist. 41(4), 701\u2013707 (2015)","journal-title":"Comput. Linguist."},{"issue":"1","key":"35_CR39","doi-asserted-by":"publisher","first-page":"55","DOI":"10.1007\/s10032-019-00317-0","volume":"22","author":"N Milosevic","year":"2019","unstructured":"Milosevic, N., Gregson, C., Hernandez, R., Nenadic, G.: A framework for information extraction from tables in biomedical literature. Int. J. Doc. Anal. Recognit. 22(1), 55\u201378 (2019). https:\/\/doi.org\/10.1007\/s10032-019-00317-0","journal-title":"Int. J. Doc. Anal. Recognit."},{"key":"35_CR40","unstructured":"Mondal, I., Hou, Y., Jochim, C.: End-to-end NLP knowledge graph construction. arXiv preprint arXiv:2106.01167 (2021)"},{"key":"35_CR41","doi-asserted-by":"crossref","unstructured":"Mysore, S., et al.: The materials science procedural text corpus: annotating materials synthesis procedures with shallow semantic structures. In: Proceedings of the 13th Linguistic Annotation Workshop, pp. 56\u201364 (2019)","DOI":"10.18653\/v1\/W19-4007"},{"key":"35_CR42","doi-asserted-by":"crossref","unstructured":"Oelen, A., Stocker, M., Auer, S.: Crowdsourcing scholarly discourse annotations. In: 26th International Conference on Intelligent User Interfaces, pp. 464\u2013474 (2021)","DOI":"10.1145\/3397481.3450685"},{"issue":"5942","key":"35_CR43","doi-asserted-by":"publisher","first-page":"828","DOI":"10.1126\/science.1157784","volume":"325","author":"AH Renear","year":"2009","unstructured":"Renear, A.H., Palmer, C.L.: Strategic reading, ontologies, and the future of scientific publishing. Science 325(5942), 828\u2013832 (2009)","journal-title":"Science"},{"key":"35_CR44","unstructured":"Vaswani, A., et al.: Attention is all you need. In: Advances in Neural Information Processing Systems, pp. 5998\u20136008 (2017)"},{"key":"35_CR45","unstructured":"Ware, M., Mabe, M.: The STM report: an overview of scientific and scholarly journal publishing, March 2015"},{"issue":"5","key":"35_CR46","doi-asserted-by":"publisher","first-page":"589","DOI":"10.1007\/s10791-006-9005-5","volume":"9","author":"X Wei","year":"2006","unstructured":"Wei, X., Croft, B., Mccallum, A.: Table extraction for answer retrieval. Inf. Retr. 9(5), 589\u2013611 (2006). https:\/\/doi.org\/10.1007\/s10791-006-9005-5","journal-title":"Inf. Retr."},{"key":"35_CR47","unstructured":"Yang, Z., Dai, Z., Yang, Y., Carbonell, J., Salakhutdinov, R., Le, Q.V.: XLNet: generalized autoregressive pretraining for language understanding. arXiv preprint arXiv:1906.08237 (2019)"}],"container-title":["Lecture Notes in Computer Science","Towards Open and Trustworthy Digital Societies"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-91669-5_35","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,7]],"date-time":"2024-03-07T15:59:57Z","timestamp":1709827197000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-91669-5_35"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030916688","9783030916695"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-91669-5_35","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"30 November 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICADL","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Asian Digital Libraries","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"1 December 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 December 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"icadl2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/icadl.net\/icadl2021\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Easychair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"87","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"17","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"14","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"20% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.01","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4.61","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}