{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,17]],"date-time":"2026-03-17T02:11:36Z","timestamp":1773713496691,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":30,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,4,19]],"date-time":"2016-04-19T00:00:00Z","timestamp":1461024000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100009226","name":"National Security Agency","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009226","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,4,19]]},"DOI":"10.1145\/2898375.2898380","type":"proceedings-article","created":{"date-parts":[[2016,4,14]],"date-time":"2016-04-14T21:09:21Z","timestamp":1460668161000},"page":"38-51","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Establishing a baseline for measuring advancement in the science of security"],"prefix":"10.1145","author":[{"given":"Jeffrey C.","family":"Carver","sequence":"first","affiliation":[{"name":"University of Alabama"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Morgan","family":"Burcham","sequence":"additional","affiliation":[{"name":"University of Alabama"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sedef Akinli","family":"Kocak","sequence":"additional","affiliation":[{"name":"Ryerson University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ayse","family":"Bener","sequence":"additional","affiliation":[{"name":"Ryerson University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Felderer","sequence":"additional","affiliation":[{"name":"University of Innsbruck"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Gander","sequence":"additional","affiliation":[{"name":"University of Innsbruck"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jason","family":"King","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jouni","family":"Markkula","sequence":"additional","affiliation":[{"name":"University of Oulu"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markku","family":"Oivo","sequence":"additional","affiliation":[{"name":"University of Oulu"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Clemens","family":"Sauerwein","sequence":"additional","affiliation":[{"name":"University of Innsbruck"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Laurie","family":"Williams","sequence":"additional","affiliation":[{"name":"NC State University"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2016,4,19]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Nature Genetics, 22","author":"Freely","year":"1999","unstructured":"Freely associating. Nature Genetics, 22 , 1999 . Freely associating. Nature Genetics, 22, 1999."},{"key":"e_1_3_2_1_2_1","volume-title":"MedlinePlus","author":"How","year":"2015","unstructured":"How reliable are medical studies? half of findings couldn't be replicated . MedlinePlus , 2015 . How reliable are medical studies? half of findings couldn't be replicated. MedlinePlus, 2015."},{"key":"e_1_3_2_1_3_1","volume-title":"Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials","author":"Abramson J.","year":"2011","unstructured":"J. Abramson and Z. Abramson . Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials . John Wiley & Sons , 2011 . J. Abramson and Z. Abramson. Research methods in community medicine: surveys, epidemiological research, programme evaluation, clinical trials. John Wiley & Sons, 2011."},{"key":"e_1_3_2_1_4_1","volume-title":"Cengage Learning","author":"Babbie E.","year":"2015","unstructured":"E. Babbie . The practice of social research . Cengage Learning , 2015 . E. Babbie. The practice of social research. Cengage Learning, 2015."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1086\/208920"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/2819009.2819025"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF01096215"},{"key":"e_1_3_2_1_8_1","volume-title":"Structures of scientific theories","author":"Craver C. F.","year":"2008","unstructured":"C. F. Craver . Structures of scientific theories . The Blackwell guide to the philosophy of science, 19:55, 2008 . C. F. Craver. Structures of scientific theories. The Blackwell guide to the philosophy of science, 19:55, 2008."},{"key":"e_1_3_2_1_9_1","volume-title":"Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188","author":"DerSimonian R.","year":"1986","unstructured":"R. DerSimonian and N. Laird . Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188 , 1986 . R. DerSimonian and N. Laird. Meta-analysis in clinical trials. Controlled clinical trials, 7(3):177--188, 1986."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10009-015-0365-2"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1002\/stvr.1580"},{"key":"e_1_3_2_1_12_1","first-page":"29","article-title":"Meta-analysis in medical research","volume":"14","author":"Haidich A.","year":"2010","unstructured":"A. Haidich . Meta-analysis in medical research . Hippokratia , 14 : 29 -- 37 , 2010 . A. Haidich. Meta-analysis in medical research. Hippokratia, 14:29--37, 2010.","journal-title":"Hippokratia"},{"key":"e_1_3_2_1_13_1","volume-title":"Designing and reporting experiments in psychology","author":"Harris P.","year":"2008","unstructured":"P. Harris . Designing and reporting experiments in psychology . McGraw-Hill Education (UK) , 2008 . P. Harris. Designing and reporting experiments in psychology. McGraw-Hill Education (UK), 2008."},{"key":"e_1_3_2_1_14_1","volume-title":"LiveScience","author":"Heger M.","year":"2012","unstructured":"M. Heger . What is a theory ? LiveScience , 2012 . M. Heger. What is a theory? LiveScience, 2012."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2004.10.019"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1007\/978-1-84800-044-5_8","volume-title":"Guide to Advanced Empirical Software Engineering","author":"Jedlitschka A.","year":"2008","unstructured":"A. Jedlitschka , M. Ciolkowski , and D. Pfahl . Reporting experiments in software engineering . In F. Shull, J. Singer, and D. Sj\u00c3\u00ffberg, editors, Guide to Advanced Empirical Software Engineering , pages 201 -- 228 . Springer London , 2008 . A. Jedlitschka, M. Ciolkowski, and D. Pfahl. Reporting experiments in software engineering. In F. Shull, J. Singer, and D. Sj\u00c3\u00ffberg, editors, Guide to Advanced Empirical Software Engineering, pages 201--228. Springer London, 2008."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/ISESE.2005.1541818"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1007\/978-1-84800-044-5_3","volume-title":"Guide to Advanced Empirical Software Engineering","author":"Kitchenham B.","year":"2008","unstructured":"B. Kitchenham and S. Pfleeger . Personal opinion surveys . In F. Shull, J. Singer, and D. Sj\u00c3\u00ffberg, editors, Guide to Advanced Empirical Software Engineering , pages 63 -- 92 . Springer London , 2008 . B. Kitchenham and S. Pfleeger. Personal opinion surveys. In F. Shull, J. Singer, and D. Sj\u00c3\u00ffberg, editors, Guide to Advanced Empirical Software Engineering, pages 63--92. Springer London, 2008."},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1080\/00029890.1995.12004627"},{"key":"e_1_3_2_1_20_1","first-page":"4","article-title":"Most published research findings are false---but a little replication goes a long way","author":"Moonesinghe R.","year":"2007","unstructured":"R. Moonesinghe , M. Khoury , and A. Janssens . Most published research findings are false---but a little replication goes a long way . PLOS Medicine , 4 , 2007 . R. Moonesinghe, M. Khoury, and A. Janssens. Most published research findings are false---but a little replication goes a long way. PLOS Medicine, 4, 2007.","journal-title":"PLOS Medicine"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/ARES.2015.69"},{"key":"e_1_3_2_1_22_1","volume-title":"Conjectures and refutations","author":"Popper K.","year":"1963","unstructured":"K. Popper . Conjectures and refutations , volume 7 . London : Routledge and Kegan Paul , 1963 . K. Popper. Conjectures and refutations, volume 7. London: Routledge and Kegan Paul, 1963."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1002\/sdr.1481"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9102-8"},{"key":"e_1_3_2_1_25_1","volume-title":"Cyber crime costs global economy $445 billion a year: report","author":"Sandle P.","year":"2014","unstructured":"P. Sandle . Cyber crime costs global economy $445 billion a year: report . Reuters 9 June 2014 . Available: http:\/\/www.reuters.com\/article\/2014\/06\/09\/us-cybersecurity-mcafee-csis-idUSKBN0EK0SV20140609. P. Sandle. Cyber crime costs global economy $445 billion a year: report. Reuters 9 June 2014. Available: http:\/\/www.reuters.com\/article\/2014\/06\/09\/us-cybersecurity-mcafee-csis-idUSKBN0EK0SV20140609."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-008-9060-1"},{"key":"e_1_3_2_1_27_1","volume-title":"A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53","author":"Simera I.","year":"2010","unstructured":"I. Simera , D. Moher , J. Hoey , K. Schulz , and D. Altman . A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53 , 2010 . I. Simera, D. Moher, J. Hoey, K. Schulz, and D. Altman. A catalogue of reporting guidelines for health research. European journal of clinical investigation, 40(1):35--53, 2010."},{"key":"e_1_3_2_1_28_1","volume-title":"A practical guide to research methods","author":"Smith J. A.","year":"2015","unstructured":"J. A. Smith . Qualitative psychology : A practical guide to research methods . Sage , 2015 . J. A. Smith. Qualitative psychology: A practical guide to research methods. Sage, 2015."},{"key":"e_1_3_2_1_29_1","volume-title":"Methods for meta-analysis in medical research","author":"Sutton A. J.","year":"2000","unstructured":"A. J. Sutton , K. R. Abrams , D. R. Jones , D. R. Jones , T. A. Sheldon , and F. Song . Methods for meta-analysis in medical research . J. Wiley Chichester ; New York, 2000 . A. J. Sutton, K. R. Abrams, D. R. Jones, D. R. Jones, T. A. Sheldon, and F. Song. Methods for meta-analysis in medical research. J. Wiley Chichester; New York, 2000."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2008.06.040"}],"event":{"name":"HotSoS '16: HotSos 2016 Science of Security","location":"Pittsburgh Pennsylvania","acronym":"HotSoS '16"},"container-title":["Proceedings of the Symposium and Bootcamp on the Science of Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2898375.2898380","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2898375.2898380","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:56:29Z","timestamp":1750222589000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2898375.2898380"}},"subtitle":["an analysis of the 2015 IEEE security & privacy proceedings"],"short-title":[],"issued":{"date-parts":[[2016,4,19]]},"references-count":30,"alternative-id":["10.1145\/2898375.2898380","10.1145\/2898375"],"URL":"https:\/\/doi.org\/10.1145\/2898375.2898380","relation":{},"subject":[],"published":{"date-parts":[[2016,4,19]]},"assertion":[{"value":"2016-04-19","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}