{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,15]],"date-time":"2026-06-15T22:57:22Z","timestamp":1781564242159,"version":"3.54.5"},"reference-count":41,"publisher":"Association for Computing Machinery (ACM)","issue":"5","license":[{"start":{"date-parts":[[2024,6,4]],"date-time":"2024-06-04T00:00:00Z","timestamp":1717459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"crossref"}]},{"name":"European Union\u2014NextGenerationEU through the Italian Ministry of University"},{"DOI":"10.13039\/100000001","name":"U.S. National Science Foundation","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100001602","name":"Science Foundation Ireland","doi-asserted-by":"crossref","id":[{"id":"10.13039\/501100001602","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["ACM Trans. Softw. Eng. Methodol."],"published-print":{"date-parts":[[2024,6,30]]},"abstract":"<jats:p>\n            Reflecting on the limitations of a study is a crucial part of the research process. In software engineering studies, this reflection is typically conveyed through discussions of study limitations or threats to validity. In current practice, such discussions seldom provide sufficient insight to understand the rationale for decisions taken before and during the study, and their implications. We revisit the practice of discussing study limitations and threats to validity and identify its weaknesses. We propose to refocus this practice of self-reflection to a discussion centered on the notion of\n            <jats:italic>trade-offs<\/jats:italic>\n            . We argue that documenting trade-offs allows researchers to clarify how the benefits of their study design decisions outweigh the costs of possible alternatives. We present guidelines for reporting trade-offs in a way that promotes a fair and dispassionate assessment of researchers\u2019 work.\n          <\/jats:p>","DOI":"10.1145\/3649598","type":"journal-article","created":{"date-parts":[[2024,3,2]],"date-time":"2024-03-02T11:23:26Z","timestamp":1709378606000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":27,"title":["Communicating Study Design Trade-offs in Software Engineering"],"prefix":"10.1145","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0248-1384","authenticated-orcid":false,"given":"Martin P.","family":"Robillard","sequence":"first","affiliation":[{"name":"McGill University, Montr\u00e9al, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3719-5011","authenticated-orcid":false,"given":"Deeksha M.","family":"Arya","sequence":"additional","affiliation":[{"name":"McGill University, Montr\u00e9al, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5992-2366","authenticated-orcid":false,"given":"Neil A.","family":"Ernst","sequence":"additional","affiliation":[{"name":"University of Victoria, Victoria, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1782-1545","authenticated-orcid":false,"given":"Jin L. C.","family":"Guo","sequence":"additional","affiliation":[{"name":"McGill University, Montr\u00e9al, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3705-6238","authenticated-orcid":false,"given":"Maxime","family":"Lamothe","sequence":"additional","affiliation":[{"name":"Polytechnique Montr\u00e9al, Montr\u00e9al, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0211-7256","authenticated-orcid":false,"given":"Mathieu","family":"Nassif","sequence":"additional","affiliation":[{"name":"McGill University, Montr\u00e9al, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1160-2608","authenticated-orcid":false,"given":"Nicole","family":"Novielli","sequence":"additional","affiliation":[{"name":"University of Bari, Bari, Italy"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1418-0095","authenticated-orcid":false,"given":"Alexander","family":"Serebrenik","sequence":"additional","affiliation":[{"name":"Eindhoven University of Technology, Eindhoven, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0612-5790","authenticated-orcid":false,"given":"Igor","family":"Steinmacher","sequence":"additional","affiliation":[{"name":"Northern Arizona University, Flagstaff, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1038-5050","authenticated-orcid":false,"given":"Klaas-Jan","family":"Stol","sequence":"additional","affiliation":[{"name":"University College Cork and Lero, Cork, Ireland"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,6,4]]},"reference":[{"key":"e_1_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2018.10.006"},{"key":"e_1_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-021-10072-8"},{"key":"e_1_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.leaqua.2011.11.015"},{"key":"e_1_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1080\/13876988.2020.1871297"},{"key":"e_1_3_3_6_2","doi-asserted-by":"publisher","DOI":"10.1201\/9781315154855-10"},{"key":"e_1_3_3_7_2","doi-asserted-by":"publisher","DOI":"10.1109\/icse43902.2021.00057"},{"key":"e_1_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-84800-044-5_11"},{"key":"e_1_3_3_9_2","unstructured":"Felipe Ebert Alexander Serebrenik Christoph Treude Nicole Novielli and Fernando Castor. 2022. On recruiting experienced GitHub contributors for interviews and surveys on prolific. Retrieved 2023-07-07 from DOI:https:\/\/www.win.tue.nl\/aserebre\/ROPES2022.pdf."},{"key":"e_1_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-017-9523-3"},{"key":"e_1_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-018-9655-0"},{"key":"e_1_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1080\/13645579.2015.1005453"},{"key":"e_1_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00123"},{"key":"e_1_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021.3087906"},{"key":"e_1_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1080\/10511253.2017.1420809"},{"key":"e_1_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-005-1290-x"},{"key":"e_1_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1016\/0147-1767(85)90062-8"},{"key":"e_1_3_3_18_2","doi-asserted-by":"publisher","DOI":"10.1177\/109634809702100103"},{"key":"e_1_3_3_19_2","doi-asserted-by":"publisher","DOI":"10.1007\/BF00995569"},{"key":"e_1_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1177\/000276428102500205"},{"key":"e_1_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1108\/EUM0000000004619"},{"key":"e_1_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1007\/BF03219568"},{"key":"e_1_3_3_23_2","doi-asserted-by":"publisher","DOI":"10.1016\/S0278-4254(00)00018-1"},{"key":"e_1_3_3_24_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.amepre.2007.04.005"},{"issue":"4","key":"e_1_3_3_25_2","first-page":"826","article-title":"A decision tree approach to the methodological decision process","volume":"47","author":"Mock Theodore J.","year":"1972","unstructured":"Theodore J. Mock. 1972. A decision tree approach to the methodological decision process. Account. Rev. 47, 4 (1972), 826\u2013829.","journal-title":"Account. Rev."},{"key":"e_1_3_3_26_2","doi-asserted-by":"publisher","DOI":"10.1002\/ev.364"},{"key":"e_1_3_3_27_2","first-page":"3 pages","volume-title":"Proceedings of the 1st International Workshop on Recruiting Participants for Empirical Software Engineering (RoPES\u201922)","author":"Reid Brittany","year":"2022","unstructured":"Brittany Reid, Markus Wagner, Marcelo d\u2019Amorim, and Christoph Treude. 2022. Software engineering user study recruitment on prolific: An experience report. In Proceedings of the 1st International Workshop on Recruiting Participants for Empirical Software Engineering (RoPES\u201922). 3 pages pages. arxiv:2201.05348"},{"key":"e_1_3_3_28_2","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3473923"},{"key":"e_1_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSME.2018.00009"},{"key":"e_1_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1007\/s40037-019-00530-x"},{"key":"e_1_3_3_31_2","volume-title":"Proceedings of the 1st International Workshop on Recruiting Participants for Empirical Software Engineering (RoPES\u201922)","author":"Russo Daniel","year":"2022","unstructured":"Daniel Russo. 2022. Recruiting software engineers on prolific. In Proceedings of the 1st International Workshop on Recruiting Participants for Empirical Software Engineering (RoPES\u201922). Retrieved from https:\/\/arxiv:2203.14695."},{"key":"e_1_3_3_32_2","volume-title":"Experimental and Quasi-Experimental Designs for Generalized Causal Inference","author":"Shadish William R.","year":"2002","unstructured":"William R. Shadish, Thomas D. Cook, and Donald T. Campbell. 2002. Experimental and Quasi-Experimental Designs for Generalized Causal Inference. Houghton, Mifflin and Company."},{"key":"e_1_3_3_33_2","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2015.24"},{"key":"e_1_3_3_34_2","doi-asserted-by":"publisher","DOI":"10.1177\/104225878901400104"},{"key":"e_1_3_3_35_2","doi-asserted-by":"publisher","DOI":"10.1145\/3241743"},{"key":"e_1_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-020-09858-z"},{"key":"e_1_3_3_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501957"},{"key":"e_1_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1016\/J.INFSOF.2023.107329"},{"key":"e_1_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.74.2.259"},{"key":"e_1_3_3_40_2","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-014-9319-7"},{"key":"e_1_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.cities.2019.102417"},{"key":"e_1_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1109\/APSEC.2016.031"}],"container-title":["ACM Transactions on Software Engineering and Methodology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649598","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3649598","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T23:56:54Z","timestamp":1750291014000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649598"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,6,4]]},"references-count":41,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2024,6,30]]}},"alternative-id":["10.1145\/3649598"],"URL":"https:\/\/doi.org\/10.1145\/3649598","relation":{},"ISSN":["1049-331X","1557-7392"],"issn-type":[{"value":"1049-331X","type":"print"},{"value":"1557-7392","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,6,4]]},"assertion":[{"value":"2023-07-07","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2024-02-17","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2024-06-04","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}