{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T02:11:36Z","timestamp":1773713496697,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,4,4]],"date-time":"2017-04-04T00:00:00Z","timestamp":1491264000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100009226","name":"National Security Agency","doi-asserted-by":"publisher","award":["NCState SoSL Lablet"],"award-info":[{"award-number":["NCState SoSL Lablet"]}],"id":[{"id":"10.13039\/100009226","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,4,4]]},"DOI":"10.1145\/3055305.3055307","type":"proceedings-article","created":{"date-parts":[[2017,3,28]],"date-time":"2017-03-28T17:42:08Z","timestamp":1490722928000},"page":"13-23","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Characterizing Scientific Reporting in Security Literature"],"prefix":"10.1145","author":[{"given":"Morgan","family":"Burcham","sequence":"first","affiliation":[{"name":"University of Alabama"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mahran","family":"Al-Zyoud","sequence":"additional","affiliation":[{"name":"University of Alabama"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jeffrey C.","family":"Carver","sequence":"additional","affiliation":[{"name":"University of Alabama"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mohammed","family":"Alsaleh","sequence":"additional","affiliation":[{"name":"UNC at Charlotte"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hongying","family":"Du","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Fida","family":"Gilani","sequence":"additional","affiliation":[{"name":"UNC at Charlotte"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jun","family":"Jiang","sequence":"additional","affiliation":[{"name":"UNC at Chapel Hill"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Akond","family":"Rahman","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"\u00d6zg\u00fcr","family":"Kafal\u0131","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ehab","family":"Al-Shaer","sequence":"additional","affiliation":[{"name":"UNC at Charlotte"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Laurie","family":"Williams","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2017,4,4]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Nature Genetics, 22","year":"1999","unstructured":"Freely associating. Nature Genetics, 22 , 1999 . Freely associating. Nature Genetics, 22, 1999."},{"issue":"5","key":"e_1_3_2_1_2_1","first-page":"8","article-title":"Computing","volume":"12","year":"2010","unstructured":"Reproducible research. Computing in Science Engineering , 12 ( 5 ): 8 -- 13 , Sept 2010 . Reproducible research. Computing in Science Engineering, 12(5):8--13, Sept 2010.","journal-title":"Science Engineering"},{"key":"e_1_3_2_1_3_1","volume-title":"MITRE","year":"2010","unstructured":"The science of cyber-security. Technical Report JSR-10-102 , MITRE , 2010 . The science of cyber-security. Technical Report JSR-10-102, MITRE, 2010."},{"key":"e_1_3_2_1_4_1","volume-title":"MedlinePlus","year":"2015","unstructured":"How reliable are medical studies? half of findings couldn't be replicated . MedlinePlus , 2015 . How reliable are medical studies? half of findings couldn't be replicated. MedlinePlus, 2015."},{"key":"e_1_3_2_1_5_1","volume-title":"Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials","author":"Abramson J.","year":"2011","unstructured":"J. Abramson and Z. Abramson . Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials . John Wiley & Sons , 2011 . J. Abramson and Z. Abramson. Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials. John Wiley & Sons, 2011."},{"key":"e_1_3_2_1_6_1","volume-title":"Cengage Learning","author":"Babbie E.","year":"2015","unstructured":"E. Babbie . The practice of social research . Cengage Learning , 2015 . E. Babbie. The practice of social research. Cengage Learning, 2015."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1086\/208920"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2898375.2898380"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2812803"},{"key":"e_1_3_2_1_10_1","volume-title":"Structures of scientific theories","author":"Craver C. F.","year":"2008","unstructured":"C. F. Craver . Structures of scientific theories . The Blackwell guide to the philosophy of science, 19:55, 2008 . C. F. Craver. Structures of scientific theories. The Blackwell guide to the philosophy of science, 19:55, 2008."},{"key":"e_1_3_2_1_11_1","volume-title":"Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188","author":"DerSimonian R.","year":"1986","unstructured":"R. DerSimonian and N. Laird . Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188 , 1986 . R. DerSimonian and N. Laird. Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188, 1986."},{"key":"e_1_3_2_1_12_1","first-page":"29","article-title":"Meta-analysis in medical research","volume":"14","author":"Haidich A.","year":"2010","unstructured":"A. Haidich . Meta-analysis in medical research . Hippokratia , 14 : 29 -- 37 , 2010 . A. Haidich. Meta-analysis in medical research. Hippokratia, 14:29--37, 2010.","journal-title":"Hippokratia"},{"key":"e_1_3_2_1_13_1","volume-title":"Designing and reporting experiments in psychology","author":"Harris P.","year":"2008","unstructured":"P. Harris . Designing and reporting experiments in psychology . McGraw-Hill Education (UK) , 2008 . P. Harris. Designing and reporting experiments in psychology. McGraw-Hill Education (UK), 2008."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2743015"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2004.10.019"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1007\/978-1-84800-044-5_8","volume-title":"Guide to Advanced Empirical Software Engineering","author":"Jedlitschka A.","year":"2008","unstructured":"A. Jedlitschka , M. Ciolkowski , and D. Pfahl . Reporting experiments in software engineering . In F. Shull, J. Singer, and D. I. Sj\u00f8berg, editors, Guide to Advanced Empirical Software Engineering , pages 201 -- 228 . Springer London , 2008 . A. Jedlitschka, M. Ciolkowski, and D. Pfahl. Reporting experiments in software engineering. In F. Shull, J. Singer, and D. I. Sj\u00f8berg, editors, Guide to Advanced Empirical Software Engineering, pages 201--228. Springer London, 2008."},{"key":"e_1_3_2_1_17_1","volume-title":"Int'l Symp. on Empir. Soft. Eng., page","author":"Jedlitschka A.","year":"2005","unstructured":"A. Jedlitschka and D. Pfahl . Reporting guidelines for controlled experiments in software engineering . In Int'l Symp. on Empir. Soft. Eng., page 10, Nov 2005 . A. Jedlitschka and D. Pfahl. Reporting guidelines for controlled experiments in software engineering. In Int'l Symp. on Empir. Soft. Eng., page 10, Nov 2005."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1007\/978-1-84800-044-5_3","volume-title":"Guide to Advanced Empirical Software Engineering","author":"Kitchenham B.","year":"2008","unstructured":"B. Kitchenham and S. Pfleeger . Personal opinion surveys . In F. Shull, J. Singer, and D. Sj\u00f8berg, editors, Guide to Advanced Empirical Software Engineering , pages 63 -- 92 . Springer London , 2008 . B. Kitchenham and S. Pfleeger. Personal opinion surveys. In F. Shull, J. Singer, and D. Sj\u00f8berg, editors, Guide to Advanced Empirical Software Engineering, pages 63--92. Springer London, 2008."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1080\/00029890.1995.12004627"},{"key":"e_1_3_2_1_20_1","first-page":"4","article-title":"Most published research findings are false--but a little replication goes a long way","author":"Moonesinghe R.","year":"2007","unstructured":"R. Moonesinghe , M. Khoury , and A. Janssens . Most published research findings are false--but a little replication goes a long way . PLOS Medicine , 4 , 2007 . R. Moonesinghe, M. Khoury, and A. Janssens. Most published research findings are false--but a little replication goes a long way. PLOS Medicine, 4, 2007.","journal-title":"PLOS Medicine"},{"key":"e_1_3_2_1_21_1","volume-title":"Conjectures and refutations","author":"Popper K.","year":"1963","unstructured":"K. Popper . Conjectures and refutations , volume 7 . London : Routledge and Kegan Paul , 1963 . K. Popper. Conjectures and refutations, volume 7. London: Routledge and Kegan Paul, 1963."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1002\/sdr.1481"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"e_1_3_2_1_24_1","first-page":"9","article-title":"Ten simple rules for reproducible computational research","author":"Sandve G.","year":"2013","unstructured":"G. Sandve , A. Nekrutenko , J. Taylor , and E. Hovig . Ten simple rules for reproducible computational research . PLoS Comput Biol , 9 , 2013 . G. Sandve, A. Nekrutenko, J. Taylor, and E. Hovig. Ten simple rules for reproducible computational research. PLoS Comput Biol, 9, 2013.","journal-title":"PLoS Comput Biol"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/5992.881708"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9060-1"},{"key":"e_1_3_2_1_27_1","volume-title":"A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53","author":"Simera I.","year":"2010","unstructured":"I. Simera , D. Moher , J. Hoey , K. Schulz , and D. Altman . A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53 , 2010 . I. Simera, D. Moher, J. Hoey, K. Schulz, and D. Altman. A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53, 2010."},{"key":"e_1_3_2_1_28_1","volume-title":"Qualitative psychology: A practical guide to research methods","author":"Smith J. A.","year":"2015","unstructured":"J. A. Smith . Qualitative psychology: A practical guide to research methods . Sage , 2015 . J. A. Smith. Qualitative psychology: A practical guide to research methods. Sage, 2015."},{"key":"e_1_3_2_1_29_1","volume-title":"Methods for meta-analysis in medical research","author":"Sutton A. J.","year":"2000","unstructured":"A. J. Sutton , K. R. Abrams , D. R. Jones , D. R. Jones , T. A. Sheldon , and F. Song . Methods for meta-analysis in medical research . J. Wiley Chichester ; New York, 2000 . A. J. Sutton, K. R. Abrams, D. R. Jones, D. R. Jones, T. A. Sheldon, and F. Song. Methods for meta-analysis in medical research. J. Wiley Chichester; New York, 2000."},{"key":"e_1_3_2_1_30_1","volume-title":"Applying the scientific method to cybersecurity research","author":"Tardiff M.","year":"2016","unstructured":"M. Tardiff . Applying the scientific method to cybersecurity research . In IEEE Int'l Symp. on Technologies for Homeland Security , 2016 . M. Tardiff. Applying the scientific method to cybersecurity research. In IEEE Int'l Symp. on Technologies for Homeland Security, 2016."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2008.06.040"}],"event":{"name":"HoTSoS '17: Symposium and Bootcamp","location":"Hanover MD USA","acronym":"HoTSoS '17","sponsor":["National Security Agency National Security Agency","Vanderbilt University Vanderbilt University","University of Maryland University of Maryland"]},"container-title":["Proceedings of the Hot Topics in Science of Security: Symposium and Bootcamp"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3055305.3055307","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3055305.3055307","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:23:26Z","timestamp":1750220606000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3055305.3055307"}},"subtitle":["An analysis of ACM CCS and IEEE S&P Papers"],"short-title":[],"issued":{"date-parts":[[2017,4,4]]},"references-count":31,"alternative-id":["10.1145\/3055305.3055307","10.1145\/3055305"],"URL":"https:\/\/doi.org\/10.1145\/3055305.3055307","relation":{},"subject":[],"published":{"date-parts":[[2017,4,4]]},"assertion":[{"value":"2017-04-04","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}