{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,25]],"date-time":"2025-03-25T14:37:05Z","timestamp":1742913425962,"version":"3.40.3"},"publisher-location":"Cham","reference-count":36,"publisher":"Springer Nature Switzerland","isbn-type":[{"type":"print","value":"9783031545337"},{"type":"electronic","value":"9783031545344"}],"license":[{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,1,1]],"date-time":"2024-01-01T00:00:00Z","timestamp":1704067200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2024]]},"DOI":"10.1007\/978-3-031-54534-4_8","type":"book-chapter","created":{"date-parts":[[2024,3,19]],"date-time":"2024-03-19T10:39:18Z","timestamp":1710844758000},"page":"109-125","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Less than Necessary or\u00a0More than Sufficient: Validating Probing Dataset Size"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-1660-2810","authenticated-orcid":false,"given":"Evgeny","family":"Orlov","sequence":"first","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3746-2642","authenticated-orcid":false,"given":"Oleg","family":"Serikov","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,3,12]]},"reference":[{"key":"8_CR1","doi-asserted-by":"publisher","unstructured":"Adcock, C.J.: Sample size determination: a review. J. Roy. Stat. Soc.: Ser. D (Stat.) 46(2), 261\u2013283 (1997). https:\/\/doi.org\/10.1111\/1467-9884.00082, https:\/\/onlinelibrary.wiley.com\/doi\/abs\/10.1111\/1467-9884.00082, _eprint: https:\/\/onlinelibrary.wiley.com\/doi\/pdf\/10.1111\/1467-9884.00082","DOI":"10.1111\/1467-9884.00082"},{"key":"8_CR2","doi-asserted-by":"crossref","unstructured":"Belinkov, Y.: Probing classifiers: promises, shortcomings, and advances. arXiv:2102.12452 [cs] (2021)","DOI":"10.1162\/coli_a_00422"},{"key":"8_CR3","series-title":"Lecture Notes in Computer Science (Lecture Notes in Artificial Intelligence)","doi-asserted-by":"publisher","first-page":"529","DOI":"10.1007\/978-3-540-30134-9_71","volume-title":"Knowledge-Based Intelligent Information and Engineering Systems","author":"N Boonyanunta","year":"2004","unstructured":"Boonyanunta, N., Zeephongsekul, P.: Predicting the relationship between the size of training sample and the predictive power of classifiers. In: Negoita, M.G., Howlett, R.J., Jain, L.C. (eds.) KES 2004. LNCS (LNAI), vol. 3215, pp. 529\u2013535. Springer, Heidelberg (2004). https:\/\/doi.org\/10.1007\/978-3-540-30134-9_71"},{"issue":"2 Suppl","key":"8_CR4","doi-asserted-by":"publisher","first-page":"S81","DOI":"10.1177\/0272989X98018002S10","volume":"18","author":"AH Briggs","year":"1998","unstructured":"Briggs, A.H., Gray, A.M.: Power and sample size calculations for stochastic cost-effectiveness analysis. Med. Decis. Making: Int. J. Soc. Med. Decis. Making 18(2 Suppl), S81-92 (1998). https:\/\/doi.org\/10.1177\/0272989X98018002S10","journal-title":"Med. Decis. Making: Int. J. Soc. Med. Decis. Making"},{"key":"8_CR5","unstructured":"Brinker, K.: Incorporating diversity in active learning with support vector machines, pp. 59\u201366 (2003)"},{"key":"8_CR6","unstructured":"Carneiro, A.V.: Estimating sample size in clinical studies: basic methodological principles. Revista Portuguesa De Cardiologia: Orgao Oficial Da Sociedade Portuguesa De Cardiologia = Portuguese J. Cardiol.: Off. J. Portuguese Soc. Cardiol. 22(12), 1513\u20131521 (2003)"},{"key":"8_CR7","doi-asserted-by":"crossref","unstructured":"Conneau, A., Kruszewski, G., Lample, G., Barrault, L., Baroni, M.: What you can cram into a single vector: probing sentence embeddings for linguistic properties. arXiv:1805.01070 [cs] (2018)","DOI":"10.18653\/v1\/P18-1198"},{"key":"8_CR8","unstructured":"Cortes, C., Jackel, L., Solla, S., Vapnik, V., Denker, J.: Learning curves: asymptotic values and rate of convergence. In: NIPS (1993)"},{"key":"8_CR9","doi-asserted-by":"crossref","unstructured":"Dalvi, F., et al.: NeuroX: a toolkit for analyzing individual neurons in neural networks. In: AAAI Conference on Artificial Intelligence (AAAI) (2019). https:\/\/www.aaai.org\/ojs\/index.php\/AAAI\/article\/view\/5063","DOI":"10.1609\/aaai.v33i01.33019851"},{"key":"8_CR10","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. arXiv:1810.04805 [cs] (2019)"},{"issue":"1","key":"8_CR11","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1158\/1078-0432.CCR-07-0443","volume":"14","author":"KK Dobbin","year":"2008","unstructured":"Dobbin, K.K., Zhao, Y., Simon, R.M.: How large a training set is needed to develop a classifier for microarray data? Clin. Cancer Res.: Off. J. Am. Assoc. Cancer Res. 14(1), 108\u2013114 (2008). https:\/\/doi.org\/10.1158\/1078-0432.CCR-07-0443","journal-title":"Clin. Cancer Res.: Off. J. Am. Assoc. Cancer Res."},{"key":"8_CR12","doi-asserted-by":"publisher","unstructured":"Eger, S., Daxenberger, J., Gurevych, I.: How to probe sentence embeddings in low-resource languages: on structural design choices for probing task evaluation. In: Proceedings of the 24th Conference on Computational Natural Language Learning, pp. 108\u2013118. Association for Computational Linguistics, Online (2020). https:\/\/doi.org\/10.18653\/v1\/2020.conll-1.8, https:\/\/aclanthology.org\/2020.conll-1.8","DOI":"10.18653\/v1\/2020.conll-1.8"},{"key":"8_CR13","doi-asserted-by":"publisher","unstructured":"Elazar, Y., Ravfogel, S., Jacovi, A., Goldberg, Y.: Amnesic probing: behavioral explanation with amnesic counterfactuals. Trans. Assoc. Comput. Linguist. 9, 160\u2013175 (2021). https:\/\/doi.org\/10.1162\/tacl_a_00359, _eprint: https:\/\/direct.mit.edu\/tacl\/article-pdf\/doi\/10.1162\/tacl_a_00359\/1924189\/tacl_a_00359.pdf","DOI":"10.1162\/tacl_a_00359"},{"key":"8_CR14","doi-asserted-by":"publisher","unstructured":"Ethayarajh, K., Jurafsky, D.: Utility is in the eye of the user: a critique of NLP leaderboards. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 4846\u20134853. Association for Computational Linguistics, Online (2020). https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.393, https:\/\/aclanthology.org\/2020.emnlp-main.393","DOI":"10.18653\/v1\/2020.emnlp-main.393"},{"issue":"1","key":"8_CR15","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1186\/1472-6947-12-8","volume":"12","author":"RL Figueroa","year":"2012","unstructured":"Figueroa, R.L., Zeng-Treitler, Q., Kandula, S., Ngo, L.H.: Predicting sample size required for classification performance. BMC Med. Inform. Decis. Making 12(1), 8 (2012). https:\/\/doi.org\/10.1186\/1472-6947-12-8","journal-title":"BMC Med. Inform. Decis. Making"},{"key":"8_CR16","doi-asserted-by":"crossref","unstructured":"Fr\u00e9chet, M.: Sur quelques points du calcul fonctionnel. Rendiconti Circolo Mat. Palermo 22, 1\u201372 (1884\u20131940)","DOI":"10.1007\/BF03018603"},{"key":"8_CR17","doi-asserted-by":"publisher","first-page":"873","DOI":"10.1109\/34.31448","volume":"11","author":"K Fukunaga","year":"1989","unstructured":"Fukunaga, K., Hayes, R.: Effects of sample size in classifier design. IEEE Trans. Pattern Anal. Mach. Intell. 11, 873\u2013885 (1989). https:\/\/doi.org\/10.1109\/34.31448","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"8_CR18","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-84858-7","volume-title":"The Elements of Statistical Learning: Data Mining, Inference, and Prediction","author":"T Hastie","year":"2009","unstructured":"Hastie, T., Tibshirani, R., Friedman, J.H., Friedman, J.H.: The Elements of Statistical Learning: Data Mining, Inference, and Prediction, vol. 2. Springer, Heidelberg (2009). https:\/\/doi.org\/10.1007\/978-0-387-84858-7"},{"issue":"1","key":"8_CR19","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1053\/j.seminoncol.2009.12.002","volume":"37","author":"KR Hess","year":"2010","unstructured":"Hess, K.R., Wei, C.: Learning curves in classification with microarray data. Semin. Oncol. 37(1), 65\u201368 (2010). https:\/\/doi.org\/10.1053\/j.seminoncol.2009.12.002","journal-title":"Semin. Oncol."},{"key":"8_CR20","doi-asserted-by":"publisher","first-page":"147","DOI":"10.1186\/1471-2105-10-147","volume":"10","author":"SY Kim","year":"2009","unstructured":"Kim, S.Y.: Effects of sample size on robustness and prediction accuracy of a prognostic gene signature. BMC Bioinform. 10, 147 (2009). https:\/\/doi.org\/10.1186\/1471-2105-10-147","journal-title":"BMC Bioinform."},{"key":"8_CR21","doi-asserted-by":"publisher","unstructured":"Lenth, R.: Some practical guidelines for effective sample-size determination. Am. Stat. 55 (2001). https:\/\/doi.org\/10.1198\/000313001317098149","DOI":"10.1198\/000313001317098149"},{"key":"8_CR22","doi-asserted-by":"publisher","first-page":"1251","DOI":"10.1109\/TPAMI.2006.156","volume":"28","author":"M Li","year":"2006","unstructured":"Li, M., Sethi, I.: Confidence-based active learning. IEEE Trans. Pattern Anal. Mach. Intell. 28, 1251\u201361 (2006). https:\/\/doi.org\/10.1109\/TPAMI.2006.156","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"6","key":"8_CR23","doi-asserted-by":"publisher","first-page":"1936","DOI":"10.1021\/ci049810a","volume":"44","author":"Y Liu","year":"2004","unstructured":"Liu, Y.: Active learning with support vector machine applied to gene expression data for cancer classification. J. Chem. Inf. Comput. Sci. 44(6), 1936\u20131941 (2004). https:\/\/doi.org\/10.1021\/ci049810a","journal-title":"J. Chem. Inf. Comput. Sci."},{"key":"8_CR24","doi-asserted-by":"publisher","unstructured":"Liu, Y., et al.: RoBERTa: a robustly optimized BERT pretraining approach (2019). https:\/\/doi.org\/10.48550\/arXiv.1907.11692, http:\/\/arxiv.org\/abs\/1907.11692","DOI":"10.48550\/arXiv.1907.11692"},{"key":"8_CR25","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1146\/annurev.psych.59.103006.093735","volume":"59","author":"SE Maxwell","year":"2008","unstructured":"Maxwell, S.E., Kelley, K., Rausch, J.R.: Sample size planning for statistical power and accuracy in parameter estimation. Annu. Rev. Psychol. 59, 537\u2013563 (2008). https:\/\/doi.org\/10.1146\/annurev.psych.59.103006.093735","journal-title":"Annu. Rev. Psychol."},{"key":"8_CR26","unstructured":"Mikhailov, V., Taktasheva, E., Sigdel, E., Artemova, E.: RuSentEval: linguistic source, encoder force! arXiv:2103.00573 [cs] (2021)"},{"issue":"2","key":"8_CR27","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1089\/106652703321825928","volume":"10","author":"S Mukherjee","year":"2003","unstructured":"Mukherjee, S., et al.: Estimating dataset size requirements for classifying DNA microarray data. J. Computat. Biol.: J. Comput. Mol. Cell Biol. 10(2), 119\u2013142 (2003). https:\/\/doi.org\/10.1089\/106652703321825928","journal-title":"J. Computat. Biol.: J. Comput. Mol. Cell Biol."},{"key":"8_CR28","doi-asserted-by":"publisher","unstructured":"Perlich, C.: Learning curves in machine learning (2011). https:\/\/doi.org\/10.1007\/978-0-387-30164-8_452","DOI":"10.1007\/978-0-387-30164-8_452"},{"key":"8_CR29","doi-asserted-by":"publisher","unstructured":"Provost, F., Jensen, D., Oates, T.: Efficient progressive sampling. In: Proceedings of the fifth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, KDD 1999, pp. 23\u201332. Association for Computing Machinery, New York (1999). https:\/\/doi.org\/10.1145\/312129.312188","DOI":"10.1145\/312129.312188"},{"key":"8_CR30","doi-asserted-by":"crossref","unstructured":"Ravishankar, V., \u00d8vrelid, L., Velldal, E.: Probing multilingual sentence representations with x-probe. In: RepL4NLP@ACL (2019)","DOI":"10.18653\/v1\/W19-4318"},{"key":"8_CR31","doi-asserted-by":"publisher","unstructured":"Rodriguez, P., Barrow, J., Hoyle, A.M., Lalor, J.P., Jia, R., Boyd-Graber, J.: Evaluation examples are not equally informative: how should that change NLP leaderboards? In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers), pp. 4486\u20134503. Association for Computational Linguistics, Online (2021). https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.346, https:\/\/aclanthology.org\/2021.acl-long.346","DOI":"10.18653\/v1\/2021.acl-long.346"},{"key":"8_CR32","doi-asserted-by":"crossref","unstructured":"Rogers, A., Kovaleva, O., Rumshisky, A.: A primer in BERTology: what we know about how BERT works. arXiv:2002.12327 [cs] (2020)","DOI":"10.1162\/tacl_a_00349"},{"key":"8_CR33","unstructured":"Vaswani, A., et al.: Attention is all you need. arXiv:1706.03762 [cs] (2017)"},{"key":"8_CR34","doi-asserted-by":"publisher","unstructured":"Voita, E., Titov, I.: Information-theoretic probing with minimum description length. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 183\u2013196. Association for Computational Linguistics, Online (2020). https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.14, https:\/\/aclanthology.org\/2020.emnlp-main.14","DOI":"10.18653\/v1\/2020.emnlp-main.14"},{"issue":"2","key":"8_CR35","doi-asserted-by":"publisher","first-page":"667","DOI":"10.1021\/ci025620t","volume":"43","author":"MK Warmuth","year":"2003","unstructured":"Warmuth, M.K., Liao, J., R\u00e4tsch, G., Mathieson, M., Putta, S., Lemmen, C.: Active learning with support vector machines in the drug discovery process. J. Chem. Inf. Comput. Sci. 43(2), 667\u2013673 (2003). https:\/\/doi.org\/10.1021\/ci025620t","journal-title":"J. Chem. Inf. Comput. Sci."},{"key":"8_CR36","doi-asserted-by":"publisher","unstructured":"Zhu, Z., Wang, J., Li, B., Rudzicz, F.: On the data requirements of probing. In: Findings of the Association for Computational Linguistics: ACL 2022, pp. 4132\u20134147. Association for Computational Linguistics, Dublin (2022). https:\/\/doi.org\/10.18653\/v1\/2022.findings-acl.326, https:\/\/aclanthology.org\/2022.findings-acl.326","DOI":"10.18653\/v1\/2022.findings-acl.326"}],"container-title":["Lecture Notes in Computer Science","Analysis of Images, Social Networks and Texts"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-54534-4_8","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,19]],"date-time":"2024-03-19T10:40:38Z","timestamp":1710844838000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-54534-4_8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024]]},"ISBN":["9783031545337","9783031545344"],"references-count":36,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-54534-4_8","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2024]]},"assertion":[{"value":"12 March 2024","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"AIST","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Analysis of Images, Social Networks and Texts","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Yerevan","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Armenia","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"28 September 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 September 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"aist2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/aistconf.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Easy Chair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"93","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"24","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"26% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"1.62","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Out of the 93 submission, 17 were rejected before being sent to peer review.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}