{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,28]],"date-time":"2026-01-28T20:35:37Z","timestamp":1769632537209,"version":"3.49.0"},"reference-count":38,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2023,1,7]],"date-time":"2023-01-07T00:00:00Z","timestamp":1673049600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,7]],"date-time":"2023-01-07T00:00:00Z","timestamp":1673049600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/501100001537","name":"University of Auckland","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001537","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2023,3]]},"DOI":"10.1007\/s10664-022-10254-y","type":"journal-article","created":{"date-parts":[[2023,1,7]],"date-time":"2023-01-07T06:04:01Z","timestamp":1673071441000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":10,"title":["Evaluating software user feedback classifier performance on unseen apps, datasets, and metadata"],"prefix":"10.1007","volume":"28","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8083-320X","authenticated-orcid":false,"given":"Peter","family":"Devine","sequence":"first","affiliation":[]},{"given":"Yun Sing","family":"Koh","sequence":"additional","affiliation":[]},{"given":"Kelly","family":"Blincoe","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2023,1,7]]},"reference":[{"issue":"12","key":"10254_CR1","first-page":"e2309","volume":"32","author":"J Ali Khan","year":"2020","unstructured":"Ali Khan J, Liu L, Wen L, Ali R (2020) Conceptualising, extracting and analysing requirements arguments in users\u2019 forums: the crowdre-arg framework. J Softw: Evol Process 32(12):e2309","journal-title":"J Softw: Evol Process"},{"key":"10254_CR2","doi-asserted-by":"crossref","unstructured":"Araujo A, Golo M, Viana B, Sanches F, Romero R, Marcacini R (2020) From bag-of-words to pre-trained neural language models: Improving automatic classification of app reviews for requirements engineering. In: Anais do XVII encontro nacional de intelig\u00eancia artificial e computacional. SBC, pp 378\u2013389","DOI":"10.5753\/eniac.2020.12144"},{"issue":"1","key":"10254_CR3","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1145\/1007730.1007735","volume":"6","author":"GE Batista","year":"2004","unstructured":"Batista G E, Prati R C, Monard M C (2004) A study of the behavior of several methods for balancing machine learning training data. ACM SIGKDD Expl Newsl 6(1):20\u201329","journal-title":"ACM SIGKDD Expl Newsl"},{"issue":"3","key":"10254_CR4","doi-asserted-by":"publisher","first-page":"265","DOI":"10.1023\/B:SQJO.0000034711.87241.f0","volume":"12","author":"E Berki","year":"2004","unstructured":"Berki E, Georgiadou E, Holcombe M (2004) Requirements engineering and process modelling in software quality management\u2014towards a generic process metamodel. Softw Qual J 12(3):265\u2013283","journal-title":"Softw Qual J"},{"key":"10254_CR5","doi-asserted-by":"crossref","unstructured":"Broy M (2006) Requirements engineering as a key to holistic software quality. In: International symposium on computer and information sciences. Springer, pp 24\u201334","DOI":"10.1007\/11902140_3"},{"key":"10254_CR6","doi-asserted-by":"crossref","unstructured":"Ciurumelea A, Schaufelb\u00fchl A, Panichella S, Gall H C (2017) Analyzing reviews and code of mobile apps for better release planning. In: 2017 IEEE 24th international conference on software analysis, evolution and reengineering (SANER). IEEE, pp 91\u2013102","DOI":"10.1109\/SANER.2017.7884612"},{"issue":"2","key":"10254_CR7","doi-asserted-by":"publisher","first-page":"103","DOI":"10.1007\/s00766-007-0045-1","volume":"12","author":"J Cleland-Huang","year":"2007","unstructured":"Cleland-Huang J, Settimi R, Zou X, Solc P (2007) Automated classification of non-functional requirements. Requir Eng 12(2):103\u2013120","journal-title":"Requir Eng"},{"issue":"7","key":"10254_CR8","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1109\/TSE.2006.61","volume":"32","author":"D Damian","year":"2006","unstructured":"Damian D, Chisan J (2006) An empirical study of the complex relationships between requirements engineering processes and other processes that lead to payoffs in productivity, quality, and risk management. IEEE Trans Softw Eng 32 (7):433\u2013453","journal-title":"IEEE Trans Softw Eng"},{"key":"10254_CR9","unstructured":"Devlin J, Chang M W, Lee K, Toutanova K (2019) BERT: pre-training of deep bidirectional transformers for language understanding. In: Proceedings of the 2019 conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, vol 1 (Long and Short Papers). https:\/\/aclanthology.org\/N19-1423. Association for Computational Linguistics, Minneapolis, pp 4171\u20134186"},{"key":"10254_CR10","doi-asserted-by":"crossref","unstructured":"Dhinakaran V T, Pulle R, Ajmeri N, Murukannaiah P K (2018) App review analysis via active learning: reducing supervision effort without compromising classification accuracy. In: 2018 IEEE 26th international requirements engineering conference (RE). IEEE, pp 170\u2013181","DOI":"10.1109\/RE.2018.00026"},{"issue":"3","key":"10254_CR11","first-page":"e2316","volume":"33","author":"A Di Sorbo","year":"2021","unstructured":"Di Sorbo A, Grano G, Aaron Visaggio C, Panichella S (2021) Investigating the criticality of user-reported issues through their relations with app rating. J Softw: Evol Process 33(3):e2316","journal-title":"J Softw: Evol Process"},{"key":"10254_CR12","unstructured":"Gillies A (2011) Software quality: theory and management. Lulu com"},{"key":"10254_CR13","doi-asserted-by":"crossref","unstructured":"Guzman E, El-Haliby M, Bruegge B (2015) Ensemble methods for app review classification: an approach for software evolution (n). In: 2015 30th IEEE\/ACM international conference on automated software engineering (ASE). IEEE, pp 771\u2013776","DOI":"10.1109\/ASE.2015.88"},{"key":"10254_CR14","doi-asserted-by":"crossref","unstructured":"Guzman E, Alkadhi R, Seyff N (2016) A needle in a haystack: what do twitter users say about software?. In: 2016 IEEE 24th international requirements engineering conference (RE). IEEE, pp 96\u2013105","DOI":"10.1109\/RE.2016.67"},{"key":"10254_CR15","doi-asserted-by":"crossref","unstructured":"Guzman E, Ibrahim M, Glinz M (2017) A little bird told me: mining tweets for requirements and software evolution. In: 2017 IEEE 25th international requirements engineering conference (RE). IEEE, pp 11\u201320","DOI":"10.1109\/RE.2017.88"},{"key":"10254_CR16","unstructured":"Hadi M A, Fard F H (2021) Evaluating pre-trained models for user feedback analysis in software engineering: a study on classification of app-reviews. arXiv:2104.05861"},{"key":"10254_CR17","doi-asserted-by":"crossref","unstructured":"Henao P R, Fischbach J, Spies D, Frattini J, Vogelsang A (2021) Transfer learning for mining feature requests and bug reports from tweets and app store reviews. In: 2021 IEEE 29th international requirements engineering conference workshops (REW). IEEE, pp 80\u201386","DOI":"10.1109\/REW53955.2021.00019"},{"key":"10254_CR18","doi-asserted-by":"crossref","unstructured":"Iacob C, Harrison R, Faily S (2013) Online reviews as first class artifacts in mobile app development. In: International conference on mobile computing, applications, and services. Springer, pp 47\u201353","DOI":"10.1007\/978-3-319-05452-0_4"},{"key":"10254_CR19","doi-asserted-by":"crossref","unstructured":"Iqbal T, Khan M, Taveter K, Seyff N (2021) Mining reddit as a new source for software requirements. In: 2021 IEEE 29th international requirements engineering conference (RE). IEEE, pp 128\u2013138","DOI":"10.1109\/RE51729.2021.00019"},{"issue":"4","key":"10254_CR20","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1007\/s11334-014-0232-4","volume":"10","author":"M Kassab","year":"2014","unstructured":"Kassab M, Neill C, Laplante P (2014) State of practice in requirements engineering: contemporary data. Innov Syst Softw Eng 10(4):235\u2013241","journal-title":"Innov Syst Softw Eng"},{"issue":"1","key":"10254_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s42979-020-00416-4","volume":"2","author":"S Lim","year":"2021","unstructured":"Lim S, Henriksson A, Zdravkovic J (2021) Data-driven requirements elicitation: a systematic literature review. SN Comput Sci 2(1):1\u201335","journal-title":"SN Comput Sci"},{"issue":"1","key":"10254_CR22","doi-asserted-by":"publisher","first-page":"170","DOI":"10.1007\/s10664-018-9627-4","volume":"24","author":"D Lin","year":"2019","unstructured":"Lin D, Bezemer C P, Zou Y, Hassan A E (2019) An empirical study of game reviews on the steam platform. Empir Softw Eng 24(1):170\u2013207","journal-title":"Empir Softw Eng"},{"issue":"3","key":"10254_CR23","doi-asserted-by":"publisher","first-page":"311","DOI":"10.1007\/s00766-016-0251-9","volume":"21","author":"W Maalej","year":"2016","unstructured":"Maalej W, Kurtanovi\u0107 Z, Nabil H, Stanik C (2016) On the automatic classification of app reviews. Requir Eng 21(3):311\u2013331","journal-title":"Requir Eng"},{"key":"10254_CR24","doi-asserted-by":"crossref","unstructured":"Magalh\u00e3es C, Sardinha A, Ara\u00fajo J (2021) Mare: an active learning approach for requirements classification. In: RE@Next! track of the 29th IEEE international requirements engineering conference","DOI":"10.1109\/RE51729.2021.9714537"},{"issue":"5","key":"10254_CR25","doi-asserted-by":"publisher","first-page":"2764","DOI":"10.1007\/s10664-018-9601-1","volume":"23","author":"M Nayebi","year":"2018","unstructured":"Nayebi M, Cho H, Ruhe G (2018) App store mining is not enough for app improvement. Empir Softw Eng 23(5):2764\u20132794","journal-title":"Empir Softw Eng"},{"key":"10254_CR26","doi-asserted-by":"crossref","unstructured":"Nuseibeh B, Easterbrook S (2000) Requirements engineering: a roadmap. In: Proceedings of the conference on the future of software engineering, pp 35\u201346","DOI":"10.1145\/336512.336523"},{"key":"10254_CR27","doi-asserted-by":"crossref","unstructured":"Pagano D, Maalej W (2013) User feedback in the appstore: an empirical study. In: 2013 21st IEEE international requirements engineering conference (RE). IEEE, pp 125\u2013134","DOI":"10.1109\/RE.2013.6636712"},{"key":"10254_CR28","doi-asserted-by":"crossref","unstructured":"Panichella S, Di Sorbo A, Guzman E, Visaggio C A, Canfora G, Gall H C (2016) Ardoc: app reviews development oriented classifier. In: Proceedings of the 2016 24th ACM SIGSOFT international symposium on foundations of software engineering, pp 1023\u20131027","DOI":"10.1145\/2950290.2983938"},{"key":"10254_CR29","doi-asserted-by":"crossref","unstructured":"Radli\u0144ski \u0141 (2012) Empirical analysis of the impact of requirements engineering on software quality. In: International working conference on requirements engineering: foundation for software quality. Springer, pp 232\u2013238","DOI":"10.1007\/978-3-642-28714-5_21"},{"issue":"8","key":"10254_CR30","doi-asserted-by":"publisher","first-page":"777","DOI":"10.1109\/TSE.2016.2622264","volume":"43","author":"P Rempel","year":"2016","unstructured":"Rempel P, M\u00e4der P (2016) Preventing defects: the impact of requirements traceability completeness on software quality. IEEE Trans Softw Eng 43 (8):777\u2013797","journal-title":"IEEE Trans Softw Eng"},{"key":"10254_CR31","unstructured":"Sanh V, Debut L, Chaumond J, Wolf T (2019) Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter. arXiv:1910.01108"},{"issue":"1","key":"10254_CR32","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1109\/TSE.2017.2759112","volume":"45","author":"S Scalabrino","year":"2017","unstructured":"Scalabrino S, Bavota G, Russo B, Di Penta M, Oliveto R (2017) Listening to the crowd for the release planning of mobile apps. IEEE Trans Softw Eng 45(1):68\u201386","journal-title":"IEEE Trans Softw Eng"},{"key":"10254_CR33","doi-asserted-by":"crossref","unstructured":"Stanik C, Haering M, Maalej W (2019) Classifying multilingual user feedback using traditional machine learning and deep learning. In: 2019 IEEE 27th international requirements engineering conference workshops (REW). IEEE, pp 220\u2013226","DOI":"10.1109\/REW.2019.00046"},{"key":"10254_CR34","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1162\/tacl_a_00178","volume":"2","author":"MA Sultan","year":"2014","unstructured":"Sultan M A, Bethard S, Sumner T (2014) Back to basics for monolingual alignment: exploiting word similarity and contextual evidence. Trans Assoc Comput Linguist 2:219\u2013230","journal-title":"Trans Assoc Comput Linguist"},{"key":"10254_CR35","doi-asserted-by":"crossref","unstructured":"Tizard J, Wang H, Yohannes L, Blincoe K (2019) Can a conversation paint a picture? Mining requirements in software forums. In: 2019 IEEE 27th international requirements engineering conference (RE). IEEE, pp 17\u201327","DOI":"10.1109\/RE.2019.00014"},{"key":"10254_CR36","doi-asserted-by":"crossref","unstructured":"Tizard J, Rietz T, Liu X, Blincoe K (2021) Voice of the users: an extended study of software feedback engagement. Requir Eng 1\u201323","DOI":"10.1007\/s00766-021-00357-1"},{"key":"10254_CR37","doi-asserted-by":"crossref","unstructured":"Williams G, Mahmoud A (2017) Mining twitter feeds for software user requirements. In: 2017 IEEE 25th international requirements engineering conference (RE). IEEE, pp 1\u201310","DOI":"10.1109\/RE.2017.14"},{"key":"10254_CR38","doi-asserted-by":"crossref","unstructured":"Yin W, Hay J, Roth D (2019) Benchmarking zero-shot text classification: datasets, evaluation and entailment approach. arXiv:1909.00161","DOI":"10.18653\/v1\/D19-1404"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-022-10254-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-022-10254-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-022-10254-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,4,3]],"date-time":"2023-04-03T06:54:53Z","timestamp":1680504893000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-022-10254-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,1,7]]},"references-count":38,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2023,3]]}},"alternative-id":["10254"],"URL":"https:\/\/doi.org\/10.1007\/s10664-022-10254-y","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,1,7]]},"assertion":[{"value":"31 October 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 January 2023","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"Matters of consent are not applicable to this work due to the fact that no human participants were involved.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Consent"}},{"value":"None of the authors listed have a declared conflict of interest related to this work.","order":3,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Conflict of Interest"}},{"value":"One of the authors of this paper (Kelly Blincoe) is the editorial boards of the IEEE Transactions on Software Engineering, the Empirical Software Engineering Journal, and the Journal of Systems and Software.","order":4,"name":"Ethics","group":{"name":"EthicsHeading","label":"<!--Emphasis Type='Bold' removed-->Competing Interests"}}],"article-number":"26"}}