{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T04:29:14Z","timestamp":1772080154410,"version":"3.50.1"},"publisher-location":"Cham","reference-count":51,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031359941","type":"print"},{"value":"9783031359958","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-35995-8_23","type":"book-chapter","created":{"date-parts":[[2023,6,28]],"date-time":"2023-06-28T07:02:24Z","timestamp":1687935744000},"page":"321-335","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":11,"title":["Variable Discovery with\u00a0Large Language Models for\u00a0Metamorphic Testing of\u00a0Scientific Software"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9493-3404","authenticated-orcid":false,"given":"Christos","family":"Tsigkanos","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5127-4042","authenticated-orcid":false,"given":"Pooja","family":"Rani","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3057-1125","authenticated-orcid":false,"given":"Sebastian","family":"M\u00fcller","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2582-5557","authenticated-orcid":false,"given":"Timo","family":"Kehrer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,6,26]]},"reference":[{"key":"23_CR1","unstructured":"Implementation of the LLM-based workflow and reproduction kit (2023). https:\/\/seg.inf.unibe.ch\/papers\/mt-varextract-gpt-0.7.tar.gz"},{"key":"23_CR2","unstructured":"Arnold, J.G., Kiniry, J.R., Srinivasan, R., Williams, J.R., Haney, E.B., Neitsch, S.L.: United States department of agriculture. Soil and Water Assessment Tool (SWAT). Texas Water Resources Institute (2012)"},{"key":"23_CR3","unstructured":"Balog, M., Gaunt, A.L., Brockschmidt, M., Nowozin, S., Tarlow, D.: DeepCoder: learning to write programs. arXiv preprint arXiv:1611.01989 (2016)"},{"issue":"5","key":"23_CR4","doi-asserted-by":"publisher","first-page":"507","DOI":"10.1109\/TSE.2014.2372785","volume":"41","author":"ET Barr","year":"2014","unstructured":"Barr, E.T., Harman, M., McMinn, P., Shahbaz, M., Yoo, S.: The oracle problem in software testing: a survey. IEEE Trans. Softw. Eng. 41(5), 507\u2013525 (2014)","journal-title":"IEEE Trans. Softw. Eng."},{"key":"23_CR5","doi-asserted-by":"crossref","unstructured":"Black, S., et al.: GPT-NeoX-20B: an open-source autoregressive language model. arXiv preprint arXiv:2204.06745 (2022)","DOI":"10.18653\/v1\/2022.bigscience-1.9"},{"key":"23_CR6","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T., et al.: Language models are few-shot learners. Adv. Neural. Inf. Process. Syst. 33, 1877\u20131901 (2020)","journal-title":"Adv. Neural. Inf. Process. Syst."},{"key":"23_CR7","doi-asserted-by":"crossref","unstructured":"Carlson, A., Fette, I.: Memory-based context-sensitive spelling correction at web scale. In: International Conference on Machine Learning and Applications, pp. 166\u2013171. IEEE (2007)","DOI":"10.1109\/ICMLA.2007.50"},{"key":"23_CR8","doi-asserted-by":"crossref","unstructured":"Carver, J.C., Hong, N.P.C., Thiruvathukal, G.K.: Software Engineering for Science. CRC Press (2016)","DOI":"10.1201\/9781315368924"},{"key":"23_CR9","doi-asserted-by":"crossref","unstructured":"Carver, J.C., Kendall, R.P., Squires, S.E., Post, D.E.: Software development environments for scientific and engineering software: a series of case studies. In: International Conference on Software Engineering, pp. 550\u2013559. IEEE (2007)","DOI":"10.1109\/ICSE.2007.77"},{"key":"23_CR10","unstructured":"Chen, M., et al.: Evaluating large language models trained on code. arXiv:2107.03374 (2021)"},{"key":"23_CR11","doi-asserted-by":"crossref","unstructured":"Chen, T.Y., Feng, J., Tse, T.H.: Metamorphic testing of programs on partial differential equations: a case study. In: International Computer Software and Applications, pp. 327\u2013333. IEEE (2002)","DOI":"10.1109\/CMPSAC.2002.1045022"},{"issue":"1","key":"23_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/1471-2105-10-24","volume":"10","author":"TY Chen","year":"2009","unstructured":"Chen, T.Y., Ho, J.W.K., Liu, H., Xie, X.: An innovative approach for testing bioinformatics programs using metamorphic testing. BMC Bioinform. 10(1), 1\u201312 (2009)","journal-title":"BMC Bioinform."},{"key":"23_CR13","doi-asserted-by":"crossref","unstructured":"Colavito, G., Lanubile, F., Novielli, N.: Issue report classification using pre-trained language models. In: International Workshop on Natural Language-Based Software Engineering, pp. 29\u201332 (2022)","DOI":"10.1145\/3528588.3528659"},{"key":"23_CR14","doi-asserted-by":"crossref","unstructured":"Ding, J., Zhang, D., Hu, X.-H.: An application of metamorphic testing for testing scientific software. In: International Workshop on Metamorphic Testing, pp. 37\u201343 (2016)","DOI":"10.1145\/2896971.2896981"},{"key":"23_CR15","unstructured":"Gao, L., et al.: The pile: An 800gb dataset of diverse text for language modeling. arXiv preprint arXiv:2101.00027 (2020)"},{"key":"23_CR16","doi-asserted-by":"crossref","unstructured":"Han, C., Fan, Z., Zhang, D., Qiu, M., Gao, M., Zhou, A.: Meta-learning adversarial domain adaptation network for few-shot text classification. arXiv preprint arXiv:2107.12262 (2021)","DOI":"10.18653\/v1\/2021.findings-acl.145"},{"key":"23_CR17","doi-asserted-by":"crossref","unstructured":"Hardin, B., Kanewala, U.: Using semi-supervised learning for predicting metamorphic relations. In: International Workshop on Metamorphic Testing, pp. 14\u201317. IEEE (2018)","DOI":"10.1145\/3193977.3193985"},{"key":"23_CR18","doi-asserted-by":"crossref","unstructured":"Hiremath, D.J., Claus, M., Hasselbring, W., Rath, W.: Towards automated metamorphic test identification for ocean system models. In: International Workshop on Metamorphic Testing, pp. 42\u201346. IEEE (2021)","DOI":"10.1109\/MET52542.2021.00014"},{"key":"23_CR19","doi-asserted-by":"crossref","unstructured":"Howard, J., Ruder, S.: Universal language model fine-tuning for text classification (2018)","DOI":"10.18653\/v1\/P18-1031"},{"key":"23_CR20","unstructured":"Jiang, X., et al.: On the importance of attention in meta-learning for few-shot text classification. arXiv preprint arXiv:1806.00852 (2018)"},{"key":"23_CR21","doi-asserted-by":"crossref","unstructured":"Kanewala, U., Bieman, J.M.: Techniques for testing scientific programs without an oracle. In: International Workshop on Software Engineering for Computational Science and Engineering, pp. 48\u201357. IEEE (2013)","DOI":"10.1109\/SECSE.2013.6615099"},{"key":"23_CR22","doi-asserted-by":"crossref","unstructured":"Kanewala, U., Bieman, J.M.: Using machine learning techniques to detect metamorphic relations for programs without test oracles. In: International Symposium on Software Reliability Engineering, pp. 1\u201310. IEEE (2013)","DOI":"10.1109\/ISSRE.2013.6698899"},{"issue":"10","key":"23_CR23","doi-asserted-by":"publisher","first-page":"1219","DOI":"10.1016\/j.infsof.2014.05.006","volume":"56","author":"U Kanewala","year":"2014","unstructured":"Kanewala, U., Bieman, J.M.: Testing scientific software: a systematic literature review. Inf. Softw. Technol. 56(10), 1219\u20131232 (2014)","journal-title":"Inf. Softw. Technol."},{"issue":"3","key":"23_CR24","doi-asserted-by":"publisher","first-page":"245","DOI":"10.1002\/stvr.1594","volume":"26","author":"U Kanewala","year":"2016","unstructured":"Kanewala, U., Bieman, J.M., Ben-Hur, A.: Predicting metamorphic relations for testing scientific software: a machine learning approach using graph kernels. Softw. Test. Verif. Reliab. 26(3), 245\u2013269 (2016)","journal-title":"Softw. Test. Verif. Reliab."},{"key":"23_CR25","unstructured":"Kaplan, J., et al.: Scaling laws for neural language models. arXiv preprint arXiv:2001.08361 (2020)"},{"key":"23_CR26","unstructured":"Kelly, D., Sanders, R.: The challenge of testing scientific software. In: Annual Conference of the Association for Software Testing, pp. 30\u201336 (2008)"},{"key":"23_CR27","unstructured":"Kelly, D., Sanders, R., et al.: Assessing the quality of scientific software. In: International Workshop on Software Engineering for Computational Science and Engineering (2008)"},{"issue":"05","key":"23_CR28","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1109\/MCSE.2011.86","volume":"13","author":"D Kelly","year":"2011","unstructured":"Kelly, D., Smith, S., Meng, N.: Software engineering for scientists. Comput. Sci. Eng. 13(05), 7\u201311 (2011)","journal-title":"Comput. Sci. Eng."},{"key":"23_CR29","doi-asserted-by":"publisher","unstructured":"Langevin, C.D., Hughes, J.D., Banta, E.R., Provost, A.M., Niswonger, R.G., Panday, S.: MODFLOW 6 modular hydrologic model version 6.2. 1: US geological survey software release (2021). https:\/\/doi.org\/10.5066\/F76Q1VQV","DOI":"10.5066\/F76Q1VQV"},{"key":"23_CR30","unstructured":"Leser, U., et al.: The Collaborative Research Center FONDA. Datenbank-Spektrum, (1610\u20131995) (2021)"},{"issue":"2","key":"23_CR31","doi-asserted-by":"publisher","first-page":"78","DOI":"10.1109\/MCSE.2018.2880577","volume":"22","author":"X Lin","year":"2018","unstructured":"Lin, X., Simon, M., Niu, N.: Exploratory metamorphic testing for scientific software. Comput. Sci. Eng. 22(2), 78\u201387 (2018)","journal-title":"Comput. Sci. Eng."},{"key":"23_CR32","doi-asserted-by":"crossref","unstructured":"Lin, X., Simon, M., Niu, N.: Hierarchical metamorphic relations for testing scientific software. In: International Workshop on Software Engineering for Science, pp. 1\u20138 (2018)","DOI":"10.1145\/3194747.3194750"},{"issue":"5807","key":"23_CR33","doi-asserted-by":"publisher","first-page":"1856","DOI":"10.1126\/science.314.5807.1856","volume":"314","author":"G Miller","year":"2006","unstructured":"Miller, G.: A scientist\u2019s nightmare: software problem leads to five retractions. Science 314(5807), 1856\u20131857 (2006)","journal-title":"Science"},{"key":"23_CR34","doi-asserted-by":"crossref","unstructured":"M\u00fcller, S., Gogoll, V., Vu, A.D., Kehrer, T., Grunske, L.: Automatically finding metamorphic relations in computational material science parsers. In: International Workshop on Software Engineering for eScience (2022)","DOI":"10.1109\/eScience55777.2022.00092"},{"key":"23_CR35","doi-asserted-by":"crossref","unstructured":"Nguyen-Hoan, L., Flint, S., Sankaranarayana, R.: A survey of scientific software development. In: International Symposium on Empirical Software Engineering and Measurement, pp. 1\u201310 (2010)","DOI":"10.1145\/1852786.1852802"},{"key":"23_CR36","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"375","DOI":"10.1007\/978-3-030-77980-1_29","volume-title":"Computational Science \u2013 ICCS 2021","author":"Z Peng","year":"2021","unstructured":"Peng, Z., Lin, X., Niu, N., Abdul-Aziz, O.I.: I\/O associations in scientific software: a study of SWMM. In: Paszynski, M., Kranzlm\u00fcller, D., Krzhizhanovskaya, V.V., Dongarra, J.J., Sloot, P.M.A. (eds.) ICCS 2021. LNCS, vol. 12747, pp. 375\u2013389. Springer, Cham (2021). https:\/\/doi.org\/10.1007\/978-3-030-77980-1_29"},{"key":"23_CR37","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"503","DOI":"10.1007\/978-3-031-08760-8_42","volume-title":"Computational Science - ICCS 2022","author":"Z Peng","year":"2022","unstructured":"Peng, Z., Lin, X., Santhoshkumar, S.N., Niu, N., Kanewala, U.: Learning I\/O variables from scientific software\u2019s user manuals. In: Groen, D., de Mulatier, C., Paszynski, M., Krzhizhanovskaya, V.V., Dongarra, J.J., Sloot, P.M.A. (eds.) ICCS 2022. LNCS, vol. 13353, pp. 503\u2013516. Springer, Cham (2022). https:\/\/doi.org\/10.1007\/978-3-031-08760-8_42"},{"issue":"8","key":"23_CR38","first-page":"9","volume":"1","author":"A Radford","year":"2019","unstructured":"Radford, A., et al.: Language models are unsupervised multitask learners. OpenAI Blog 1(8), 9 (2019)","journal-title":"OpenAI Blog"},{"key":"23_CR39","doi-asserted-by":"crossref","unstructured":"Ren, X., He, W., Qu, M., Huang, L., Ji, H., Han, J.: AFET: automatic fine-grained entity typing by hierarchical partial-label embedding. In: Conference on empirical methods in natural language processing, pp. 1369\u20131378 (2016)","DOI":"10.18653\/v1\/D16-1144"},{"key":"23_CR40","doi-asserted-by":"crossref","unstructured":"Reynolds, L., McDonell, K.: Prompt programming for large language models: beyond the few-shot paradigm. In: Extended Abstracts, CHI, pp. 1\u20137 (2021)","DOI":"10.1145\/3411763.3451760"},{"key":"23_CR41","unstructured":"Rossman, L.A.: Storm water management model user\u2019s manual, version 5.0. Cincinnati: National Risk Management Research Laboratory, Office of Research and Development, US Environmental Protection Agency (2010)"},{"issue":"4","key":"23_CR42","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1109\/MS.2008.84","volume":"25","author":"R Sanders","year":"2008","unstructured":"Sanders, R., Kelly, D.: Dealing with risk in scientific software development. IEEE Softw. 25(4), 21\u201328 (2008)","journal-title":"IEEE Softw."},{"issue":"9","key":"23_CR43","doi-asserted-by":"publisher","first-page":"805","DOI":"10.1109\/TSE.2016.2532875","volume":"42","author":"S Segura","year":"2016","unstructured":"Segura, S., Fraser, G., Sanchez, A.B., Ruiz-Cort\u00e9s, A.: A survey on metamorphic testing. IEEE Trans. Softw. Eng. 42(9), 805\u2013824 (2016)","journal-title":"IEEE Trans. Softw. Eng."},{"issue":"3","key":"23_CR44","doi-asserted-by":"publisher","first-page":"46","DOI":"10.1109\/MS.2018.2875968","volume":"37","author":"S Segura","year":"2018","unstructured":"Segura, S., Towey, D., Zhou, Z.Q., Chen, T.Y.: Metamorphic testing: testing the untestable. IEEE Softw. 37(3), 46\u201353 (2018)","journal-title":"IEEE Softw."},{"key":"23_CR45","doi-asserted-by":"crossref","unstructured":"Su, F-H, Bell, J, Murphy, C., Kaiser, G.: Dynamic inference of likely metamorphic properties to support differential testing. In: International Workshop on Automation of Software Test, pp. 55\u201359. IEEE (2015)","DOI":"10.1109\/AST.2015.19"},{"key":"23_CR46","doi-asserted-by":"crossref","unstructured":"Tsigkanos, C., Rani, P., M\u00fcller, S., Kehrer, T.: Large language models: the next frontier for variable discovery within metamorphic testing? In: International Conference on Software Analysis, Evolution and Reengineering, Early Research Achievements (ERA) track. IEEE Computer Society (2023)","DOI":"10.1109\/SANER56733.2023.00070"},{"key":"23_CR47","doi-asserted-by":"crossref","unstructured":"Vu, A.D., Kehrer, T., Tsigkanos, C.: Outcome-preserving input reduction for scientific data analysis workflows. In: 37th IEEE\/ACM International Conference on Automated Software Engineering, ASE 2022, Rochester, MI, USA, 10\u201314 October 2022, pp. 182:1\u2013182:5. ACM (2022)","DOI":"10.1145\/3551349.3559558"},{"key":"23_CR48","unstructured":"Wang, B.: Mesh-transformer-JAX: model-parallel implementation of transformer language model with JAX. https:\/\/github.com\/kingoflolz\/mesh-transformer-jax"},{"key":"23_CR49","unstructured":"Wang, B., Komatsuzaki, A.: GPT-J-6B: a 6 billion parameter autoregressive language model (2022). https:\/\/github.com\/kingoflolz\/mesh-transformer-jax"},{"issue":"4","key":"23_CR50","doi-asserted-by":"publisher","first-page":"465","DOI":"10.1093\/comjnl\/25.4.465","volume":"25","author":"EJ Weyuker","year":"1982","unstructured":"Weyuker, E.J.: On testing non-testable programs. Comput. J. 25(4), 465\u2013470 (1982)","journal-title":"Comput. J."},{"key":"23_CR51","doi-asserted-by":"crossref","unstructured":"Xu, F.F., Alon, U., Neubig, G., Hellendoorn, V.J.: A systematic evaluation of large language models of code. In: ACM SIGPLAN International Symposium on Machine Programming, pp. 1\u201310 (2022)","DOI":"10.1145\/3520312.3534862"}],"container-title":["Lecture Notes in Computer Science","Computational Science \u2013 ICCS 2023"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-35995-8_23","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,10,23]],"date-time":"2024-10-23T05:51:35Z","timestamp":1729662695000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-35995-8_23"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031359941","9783031359958"],"references-count":51,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-35995-8_23","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"26 June 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ICCS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Computational Science","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Prague","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Czech Republic","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"3 July 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"5 July 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"iccs-computsci2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.iccs-meeting.org\/iccs2023\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"530","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"188","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"94","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"2,8","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3,2","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}