{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,3]],"date-time":"2026-03-03T00:49:50Z","timestamp":1772498990955,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":48,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,16]],"date-time":"2024-04-16T00:00:00Z","timestamp":1713225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,16]]},"DOI":"10.1145\/3643664.3648213","type":"proceedings-article","created":{"date-parts":[[2024,8,9]],"date-time":"2024-08-09T15:48:13Z","timestamp":1723218493000},"page":"42-47","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Apples, Oranges, and Software Engineering: Study Selection Challenges for Secondary Research on Latent Variables"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8506-3294","authenticated-orcid":false,"given":"Marvin","family":"Wyrich","sequence":"first","affiliation":[{"name":"Saarland University, Saarbrucken, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5991-3072","authenticated-orcid":false,"given":"Marvin","family":"Munoz Baron","sequence":"additional","affiliation":[{"name":"University of Stuttgart, Stuttgart, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5788-0991","authenticated-orcid":false,"given":"Justus","family":"Bogner","sequence":"additional","affiliation":[{"name":"Vrije Universiteit Amsterdam, Amsterdam, Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2024,8,9]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"crossref","first-page":"44","DOI":"10.26407\/2018JRTDD.1.6","article-title":"An Overview of the History and Methodological Aspects of Psychometrics: History and Methodological Aspects of Psychometrics","volume":"1","author":"Anunciacao Luis","year":"2018","unstructured":"Luis Anunciacao. 2018. An Overview of the History and Methodological Aspects of Psychometrics: History and Methodological Aspects of Psychometrics. Journal for ReAttach Therapy and Developmental Diversities 1, 1 (2018), 44--58.","journal-title":"Journal for ReAttach Therapy and Developmental Diversities"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","first-page":"571","DOI":"10.1016\/j.jss.2006.07.009","article-title":"Lessons from Applying the Systematic Literature Review Process within the Software Engineering Domain","volume":"80","author":"Brereton Pearl","year":"2007","unstructured":"Pearl Brereton, Barbara A. Kitchenham, David Budgen, Mark Turner, and Mohamed Khalil. 2007. Lessons from Applying the Systematic Literature Review Process within the Software Engineering Domain. Journal of systems and software 80, 4 (2007), 571--583.","journal-title":"Journal of systems and software"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00125812"},{"key":"e_1_3_2_1_4_1","volume-title":"Confirmatory Factor Analysis for Applied Research","author":"Brown Timothy A.","unstructured":"Timothy A. Brown. 2015. Confirmatory Factor Analysis for Applied Research. Guilford publications."},{"key":"e_1_3_2_1_5_1","volume-title":"2015 IEEE 23rd International Conference on Program Comprehension. IEEE, 255--265","author":"Busjahn Teresa","year":"2015","unstructured":"Teresa Busjahn, Roman Bednarik, Andrew Begel, Martha Crosby, James H Paterson, Carsten Schulte, Bonita Sharif, and Sascha Tamm. 2015. Eye movements in code reading: Relaxing the linear order. In 2015 IEEE 23rd International Conference on Program Comprehension. IEEE, 255--265."},{"key":"e_1_3_2_1_6_1","volume-title":"The three-stratum theory of cognitive abilities. Contemporary intellectual assessment: theories, tests, and issues","author":"Carroll John B","year":"2005","unstructured":"John B Carroll. 2005. The three-stratum theory of cognitive abilities. Contemporary intellectual assessment: theories, tests, and issues (2005)."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-013-9290-8"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"crossref","first-page":"1","DOI":"10.20982\/tqmp.01.1.p001","article-title":"The Rise of Quantitative Methods in Psychology","volume":"1","author":"Cousineau Denis","year":"2005","unstructured":"Denis Cousineau. 2005. The Rise of Quantitative Methods in Psychology. Tutorials in Quantitative Methods for Psychology 1, 1 (2005), 1--3.","journal-title":"Tutorials in Quantitative Methods for Psychology"},{"key":"e_1_3_2_1_9_1","volume-title":"Meehl","author":"Cronbach Lee J.","year":"1955","unstructured":"Lee J. Cronbach and Paul E. Meehl. 1955. Construct Validity in Psychological Tests. Psychological bulletin 52, 4 (1955), 281."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","first-page":"440","DOI":"10.1016\/j.infsof.2011.01.004","article-title":"Research Synthesis in Software Engineering: A Tertiary Study","volume":"53","author":"Cruzes Daniela S.","year":"2011","unstructured":"Daniela S. Cruzes and Tore Dyb\u00e5. 2011. Research Synthesis in Software Engineering: A Tertiary Study. Information and Software Technology 53, 5 (2011), 440--455.","journal-title":"Information and Software Technology"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","first-page":"899","DOI":"10.1016\/j.infsof.2011.04.004","article-title":"Six Years of Systematic Literature Reviews in Software Engineering: An Updated Tertiary Study","volume":"53","author":"Da Silva Fabio QB","year":"2011","unstructured":"Fabio QB Da Silva, Andre LM Santos, S\u00e9rgio Soares, A. C\u00e9sar C. Fran\u00e7a, Cleviton VF Monteiro, and Felipe Farias Maciel. 2011. Six Years of Systematic Literature Reviews in Software Engineering: An Updated Tertiary Study. Information and Software Technology 53, 9 (2011), 899--913.","journal-title":"Information and Software Technology"},{"key":"e_1_3_2_1_12_1","volume-title":"Empirical Methodologies in Software Engineering. In Eleventh Annual International Workshop on Software Technology and Engineering Practice. IEEE, 52--58","author":"Dawson Ray","year":"2003","unstructured":"Ray Dawson, Phil Bones, Briony J. Oates, Pearl Brereton, Motoei Azuma, and Mary Lou Jackson. 2003. Empirical Methodologies in Software Engineering. In Eleventh Annual International Workshop on Software Technology and Engineering Practice. IEEE, 52--58."},{"key":"e_1_3_2_1_13_1","volume-title":"Meta-Analysis in Clinical Trials. Controlled clinical trials 7, 3","author":"DerSimonian Rebecca","year":"1986","unstructured":"Rebecca DerSimonian and Nan Laird. 1986. Meta-Analysis in Clinical Trials. Controlled clinical trials 7, 3 (1986), 177--188."},{"key":"e_1_3_2_1_14_1","volume-title":"Meta-Analysis in Clinical Trials Revisited. Contemporary clinical trials 45","author":"DerSimonian Rebecca","year":"2015","unstructured":"Rebecca DerSimonian and Nan Laird. 2015. Meta-Analysis in Clinical Trials Revisited. Contemporary clinical trials 45 (2015), 139--145."},{"key":"e_1_3_2_1_15_1","volume-title":"Classical Test Theory. Medical care","author":"DeVellis Robert F.","year":"2006","unstructured":"Robert F. DeVellis. 2006. Classical Test Theory. Medical care (2006), S50--S59. jstor:41219505"},{"key":"e_1_3_2_1_16_1","volume-title":"Reise","author":"Embretson Susan E.","year":"2013","unstructured":"Susan E. Embretson and Steven P. Reise. 2013. Item Response Theory. Psychology Press."},{"key":"e_1_3_2_1_17_1","volume-title":"Psychometrics: An Introduction. SAGE publications.","author":"Furr R. Michael","year":"2021","unstructured":"R. Michael Furr. 2021. Psychometrics: An Introduction. SAGE publications."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3469888"},{"key":"e_1_3_2_1_19_1","unstructured":"Thom Holwerda. 2008. WTFs\/m. https:\/\/web.archive.org\/web\/20231122170548\/ https:\/\/www.osnews.com\/story\/19266\/wtfsm\/"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","first-page":"111","DOI":"10.1109\/TSE.2017.2770124","article-title":"A Systematic Literature Review and Meta-Analysis on Cross Project Defect Prediction","volume":"45","author":"Hosseini Seyedrebvar","year":"2017","unstructured":"Seyedrebvar Hosseini, Burak Turhan, and Dimuthu Gunarathna. 2017. A Systematic Literature Review and Meta-Analysis on Cross Project Defect Prediction. IEEE Transactions on Software Engineering 45, 2 (2017), 111--147.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_21_1","volume-title":"Systematizing the Meta-Analytical Process in Software Engineering. In 2021 2nd European Symposium on Software Engineering. ACM, Larissa Greece, 1--5. 10","author":"Jaccheri Letizia","year":"2021","unstructured":"Letizia Jaccheri, Zamira Kholmatova, and Giancarlo Succi. 2021. Systematizing the Meta-Analytical Process in Software Engineering. In 2021 2nd European Symposium on Software Engineering. ACM, Larissa Greece, 1--5. 10.1145\/3501774.3501775"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings. 2004 International Symposium on Empirical Software Engineering, 2004. ISESE'04. IEEE, 261--270","author":"Jedlitschka Andreas","year":"2004","unstructured":"Andreas Jedlitschka and Marcus Ciolkowski. 2004. Towards Evidence in Software Engineering. In Proceedings. 2004 International Symposium on Empirical Software Engineering, 2004. ISESE'04. IEEE, 261--270."},{"key":"e_1_3_2_1_23_1","first-page":"11","article-title":"A Systematic Review of Effect Size in Software Engineering Experiments","volume":"49","author":"Kampenes Vigdis By","year":"2007","unstructured":"Vigdis By Kampenes, Tore Dyb\u00e5, Jo E. Hannay, and Dag IK Sj\u00f8berg. 2007. A Systematic Review of Effect Size in Software Engineering Experiments. Information and Software Technology 49, 11-12 (2007), 1073--1086.","journal-title":"Information and Software Technology"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9061-0"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","first-page":"1273","DOI":"10.1109\/TSE.2022.3174092","article-title":"SEGRESS: Software Engineering Guidelines for Reporting Secondary Studies","volume":"49","author":"Kitchenham Barbara","year":"2022","unstructured":"Barbara Kitchenham, Lech Madeyski, and David Budgen. 2022. SEGRESS: Software Engineering Guidelines for Reporting Secondary Studies. IEEE Transactions on Software Engineering 49, 3 (2022), 1273--1298.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"e_1_3_2_1_26_1","volume-title":"Systematic Literature Reviews in Software Engineering-a Tertiary Study. Information and software technology 52, 8","author":"Kitchenham Barbara","year":"2010","unstructured":"Barbara Kitchenham, Rialette Pretorius, David Budgen, O. Pearl Brereton, Mark Turner, Mahmood Niazi, and Stephen Linkman. 2010. Systematic Literature Reviews in Software Engineering-a Tertiary Study. Information and software technology 52, 8 (2010), 792--805."},{"key":"e_1_3_2_1_27_1","first-page":"4","article-title":"Evidence-Based Software Engineering and Systematic Reviews","author":"Kitchenham Barbara Ann","year":"2015","unstructured":"Barbara Ann Kitchenham. 2015. Evidence-Based Software Engineering and Systematic Reviews. Chapman & Hall \/ CRC Innovations in Software Engineering and Software Development Series, Vol. v.4. CRC Press, Boca Raton.","journal-title":"Chapman & Hall \/ CRC Innovations in Software Engineering and Software Development Series"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","first-page":"1109","DOI":"10.1007\/s10664-018-9649-y","article-title":"Eye tracking analysis ofcomputer program comprehension in programmers with dyslexia","volume":"24","author":"McChesney Ian","year":"2019","unstructured":"Ian McChesney and Raymond Bond. 2019. Eye tracking analysis ofcomputer program comprehension in programmers with dyslexia. Empirical Software Engineering 24, 3 (2019), 1109--1154.","journal-title":"Empirical Software Engineering"},{"key":"e_1_3_2_1_29_1","volume-title":"The Five-Factor Model of personality traits: consensus and controversy","author":"McCrae Robert R.","unstructured":"Robert R. McCrae. 2020. The Five-Factor Model of personality traits: consensus and controversy. In The Cambridge Handbook of Personality Psychology (2nd ed.). Cambridge University Press."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0164-1212(00)00024-8"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.285.15.1992"},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the 14th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","author":"Bar\u00f3n Marvin Mu\u00f1oz","year":"2020","unstructured":"Marvin Mu\u00f1oz Bar\u00f3n, Marvin Wyrich, and Stefan Wagner. 2020. An empirical validation of cognitive complexity as a measure of source code understandability. In Proceedings of the 14th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM) (Bari, Italy). Association for Computing Machinery, New York, NY, USA, Article 5, 12 pages. 10.1145\/3382494.3410636"},{"key":"e_1_3_2_1_33_1","volume-title":"Fifty years of reading comprehension theory and practice. Research-based practices for teaching Common Core literacy","author":"David Pearson P","year":"2015","unstructured":"P David Pearson and Gina N Cervetti. 2015. Fifty years of reading comprehension theory and practice. Research-based practices for teaching Common Core literacy (2015), 1--24."},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 28th International Conference on Program Comprehension","author":"Peitek Norman","year":"2020","unstructured":"Norman Peitek, Janet Siegmund, and Sven Apel. 2020. What Drives the Reading Order of Programmers? An Eye Tracking Study. In Proceedings of the 28th International Conference on Program Comprehension (Seoul, Republic of Korea) (ICPC '20). Association for Computing Machinery, New York, NY, USA, 342--353. 10.1145\/3387904.3389279"},{"key":"e_1_3_2_1_35_1","volume-title":"Systematic Mapping Studies in Software Engineering. In 12th International Conference on Evaluation and Assessment in Software Engineering (EASE) 12","author":"Petersen Kai","year":"2008","unstructured":"Kai Petersen, Robert Feldt, Shahid Mujtaba, and Michael Mattsson. 2008. Systematic Mapping Studies in Software Engineering. In 12th International Conference on Evaluation and Assessment in Software Engineering (EASE) 12. 1--10."},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","author":"Ralph Paul","year":"2022","unstructured":"Paul Ralph and Sebastian Baltes. 2022. Paving the Way for Mature Secondary Research: The Seven Types of Literature Review. In Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (Singapore) (ESEC\/FSE 2022). Association for Computing Machinery, New York, NY, USA, 1632--1636. 10.1145\/3540250.3560877"},{"key":"e_1_3_2_1_37_1","volume-title":"Proceedings of the 22nd International Conference on Evaluation and Assessment in Software Engineering 2018","author":"Ralph Paul","year":"2018","unstructured":"Paul Ralph and Ewan Tempero. 2018. Construct Validity in Software Engineering Research and Software Metrics. In Proceedings of the 22nd International Conference on Evaluation and Assessment in Software Engineering 2018 (Christchurch, New Zealand) (EASE '18). Association for Computing Machinery, New York, NY, USA, 13--23. 10.1145\/3210459.3210461"},{"key":"e_1_3_2_1_38_1","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. ACM, Florence Italy, 893--902","author":"Schmettow Martin","year":"2008","unstructured":"Martin Schmettow and Wolfgang Vietze. 2008. Introducing Item Response Theory for Measuring Usability Inspection Processes. In Proceedings of the SIGCHI Conference on Human Factors in Computing Systems. ACM, Florence Italy, 893--902. 10.1145\/1357054.1357196"},{"key":"e_1_3_2_1_39_1","volume-title":"MIP Talk: Measuring Programming Experience: 2012 vs","author":"Siegmund Janet","year":"2022","unstructured":"Janet Siegmund. 2022. MIP Talk: Measuring Programming Experience: 2012 vs. 2022. https:\/\/www.youtube.com\/watch?v=_OwQiUyVp-M"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","first-page":"1299","DOI":"10.1007\/s10664-013-9286-4","article-title":"Measuring and modeling programming experience","volume":"19","author":"Siegmund Janet","year":"2014","unstructured":"Janet Siegmund, Christian K\u00e4stner, J\u00f6rg Liebig, Sven Apel, and Stefan Hanenberg. 2014. Measuring and modeling programming experience. Empirical Software Engineering 19 (2014), 1299--1334.","journal-title":"Empirical Software Engineering"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2362.2009.02234.x"},{"key":"e_1_3_2_1_42_1","volume-title":"Bergersen","author":"Sj\u00f8berg Dag IK","year":"2023","unstructured":"Dag IK Sj\u00f8berg and Gunnar R. Bergersen. 2023. Improving the Reporting of Threats to Construct Validity. arXiv preprint arXiv:2306.05336 (2023). arXiv:2306.05336"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2022.3176725"},{"key":"e_1_3_2_1_44_1","volume-title":"On Construct Validity: Issues of Method and Measurement. Psychological assessment 17, 4","author":"Smith Gregory T.","year":"2005","unstructured":"Gregory T. Smith. 2005. On Construct Validity: Issues of Method and Measurement. Psychological assessment 17, 4 (2005), 396."},{"key":"e_1_3_2_1_45_1","volume-title":"2016 23rd Asia-Pacific Software Engineering Conference (APSEC). IEEE, 145--152","author":"Tempero Ewan","year":"2016","unstructured":"Ewan Tempero and Paul Ralph. 2016. A Model for Defining Coupling Metrics. In 2016 23rd Asia-Pacific Software Engineering Conference (APSEC). IEEE, 145--152. 10.1109\/APSEC.2016.030"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","first-page":"608","DOI":"10.1037\/0022-3514.84.3.608","article-title":"Quantifying Construct Validity: Two Simple Measures","volume":"84","author":"Westen Drew","year":"2003","unstructured":"Drew Westen and Robert Rosenthal. 2003. Quantifying Construct Validity: Two Simple Measures. Journal of personality and social psychology 84, 3 (2003), 608.","journal-title":"Journal of personality and social psychology"},{"key":"e_1_3_2_1_47_1","unstructured":"Marvin Wyrich. 2023. Source Code Comprehension: A Contemporary Definition and Conceptual Model for Empirical Investigation. arXiv:2310.11301 [cs.SE]"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","unstructured":"Marvin Wyrich Justus Bogner and Stefan Wagner. 2023. 40 Years of Designing Code Comprehension Experiments: A Systematic Mapping Study. ACM Comput. Surv. (2023) 42 pages. 10.1145\/3626522","DOI":"10.1145\/3626522"}],"event":{"name":"WSESE '24: 1st IEEE\/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering","location":"Lisbon Portugal","acronym":"WSESE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the 1st IEEE\/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643664.3648213","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3643664.3648213","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T00:05:32Z","timestamp":1750291532000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3643664.3648213"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,16]]},"references-count":48,"alternative-id":["10.1145\/3643664.3648213","10.1145\/3643664"],"URL":"https:\/\/doi.org\/10.1145\/3643664.3648213","relation":{},"subject":[],"published":{"date-parts":[[2024,4,16]]},"assertion":[{"value":"2024-08-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}