{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:00:54Z","timestamp":1755838854907,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":82,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,8,26]],"date-time":"2019-08-26T00:00:00Z","timestamp":1566777600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,8,26]]},"DOI":"10.1145\/3339252.3340339","type":"proceedings-article","created":{"date-parts":[[2019,8,9]],"date-time":"2019-08-09T12:21:03Z","timestamp":1565353263000},"page":"1-7","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["A Systematic Analysis of User Evaluations in Security Research"],"prefix":"10.1145","author":[{"given":"Peter","family":"Hamm","sequence":"first","affiliation":[{"name":"Goethe University Frankfurt, Frankfurt, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"David","family":"Harborth","sequence":"additional","affiliation":[{"name":"Goethe University Frankfurt, Frankfurt, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sebastian","family":"Pape","sequence":"additional","affiliation":[{"name":"Goethe University Frankfurt, Frankfurt, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2019,8,26]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.65"},{"volume-title":"Comparing the Usability of Cryptographic APIs. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 154--171","year":"2017","author":"Acar Yasemin","key":"e_1_3_2_1_2_1"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.25"},{"key":"e_1_3_2_1_4_1","unstructured":"Association for Computing Machinery. 2016. Artifact Review and Badging. Available online at: https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging.  Association for Computing Machinery. 2016. Artifact Review and Badging. Available online at: https:\/\/www.acm.org\/publications\/policies\/artifact-review-badging."},{"key":"e_1_3_2_1_5_1","unstructured":"Monya Baker. 2016. 1 500 scientists lift the lid on reproducibility. https:\/\/www.nature.com\/news\/1-500-scientists-lift-the-lid-on-reproducibility-1.19970.  Monya Baker. 2016. 1 500 scientists lift the lid on reproducibility. https:\/\/www.nature.com\/news\/1-500-scientists-lift-the-lid-on-reproducibility-1.19970."},{"volume-title":"USENIX Security Symposium. USENIX Association, 239--253","author":"Becker Ingolf","key":"e_1_3_2_1_6_1"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Jeremy Berg. 2018. Progress on reproducibility.  Jeremy Berg. 2018. Progress on reproducibility.","DOI":"10.1126\/science.aar8654"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2015.62"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2018.00019"},{"volume-title":"USENIX Security Symposium. USENIX Association, 607--623","author":"Bonneau Joseph","key":"e_1_3_2_1_10_1"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-018-0399-z"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243746"},{"volume-title":"SysPal: System-Guided Pattern Locks for Android. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 338--356","year":"2017","author":"Cho Geumhwan","key":"e_1_3_2_1_13_1"},{"key":"e_1_3_2_1_14_1","first-page":"4","article-title":"Repeatability and benefaction in computer systems research","volume":"14","author":"Collberg Christian","year":"2015","journal-title":"University of Arizona TR"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2812803"},{"volume-title":"Core Conference Ranking. http:\/\/portal.core.edu.au\/conf-ranks\/.","year":"2019","author":"Research Computing","key":"e_1_3_2_1_16_1"},{"key":"e_1_3_2_1_17_1","first-page":"104","article-title":"Organizing Knowledge Synthesis: A Taxonomy of Literature Reviews","volume":"1","author":"Cooper Harris M.","year":"1988","journal-title":"Knowledge in Society"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660267.2660271"},{"volume-title":"An Empirical Study of Textual Key-Fingerprint Representations. In USENIX Security Symposium. USENIX Association, 193--208","year":"2016","author":"Dechand Sergej","key":"e_1_3_2_1_19_1"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2508859.2516753"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.17705\/1atrr.00001"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243794"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660267.2660273"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1708272114"},{"volume-title":"Anatomization and Protection of Mobile Apps' Location Privacy Threats. In USENIX Security Symposium. USENIX Association, 753--768","author":"Fawaz Kassem","key":"e_1_3_2_1_25_1"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660267.2660270"},{"volume-title":"Shin","year":"2015","author":"Feng Huan","key":"e_1_3_2_1_27_1"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2017.39"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243803"},{"volume-title":"Forgetting of Passwords: Ecological Theory and Data. In USENIX Security Symposium. USENIX Association, 221--238","year":"2018","author":"Gao Xianyi","key":"e_1_3_2_1_30_1"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243767"},{"volume-title":"Polisis: Automated Analysis and Presentation of Privacy Policies Using Deep Learning. In USENIX Security Symposium. USENIX Association, 531--548","year":"2018","author":"Harkous Hamza","key":"e_1_3_2_1_32_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 255--272","year":"2018","author":"He Weijia","key":"e_1_3_2_1_33_1"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"crossref","unstructured":"Matthew Hutson. 2018. Artificial intelligence faces reproducibility crisis.  Matthew Hutson. 2018. Artificial intelligence faces reproducibility crisis.","DOI":"10.1126\/science.359.6377.725"},{"volume-title":"Enabling Fine-Grained Permissions for Augmented Reality Applications with Recognizers. In USENIX Security Symposium. USENIX Association, 415--430","year":"2013","author":"Jana Suman","key":"e_1_3_2_1_35_1"},{"key":"e_1_3_2_1_36_1","unstructured":"Joint Committee for Guides in Metrology. 2006. International Vocabulary of Metrology - Basic and General Concepts and Associated Terms. Available online at: https:\/\/www.nist.gov\/sites\/default\/files\/documents\/pml\/div688\/grp40\/International-Vocabulary-of-Metrology.pdf. 3rd Edition.  Joint Committee for Guides in Metrology. 2006. International Vocabulary of Metrology - Basic and General Concepts and Associated Terms. Available online at: https:\/\/www.nist.gov\/sites\/default\/files\/documents\/pml\/div688\/grp40\/International-Vocabulary-of-Metrology.pdf. 3rd Edition."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243764"},{"volume-title":"Sound-Proof: Usable Two-Factor Authentication Based on Ambient Sound. In USENIX Security Symposium. USENIX Association, 483--498","year":"2015","author":"Karapanos Nikolaos","key":"e_1_3_2_1_38_1"},{"volume-title":"Cormac Herley, and Stuart E. Schechter.","year":"2014","author":"Komanduri Saranga","key":"e_1_3_2_1_39_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 1339--1356","author":"Krombholz Katharina","key":"e_1_3_2_1_40_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 33--48","year":"2014","author":"Lau Billy","key":"e_1_3_2_1_41_1"},{"volume-title":"Towards Security and Privacy for Multi-user Augmented Reality: Foundations with End Users. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 392--408","year":"2018","author":"Lebeck Kiron","key":"e_1_3_2_1_42_1"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2508859.2516747"},{"volume-title":"USENIX Security Symposium. USENIX Association, 1033--1050","year":"2016","author":"Li Frank","key":"e_1_3_2_1_44_1"},{"volume-title":"Studying the Impact of Managers on Password Strength and Reuse. In USENIX Security Symposium. USENIX Association, 203--220","year":"2018","author":"Lyastani Sanam Ghorbani","key":"e_1_3_2_1_45_1"},{"volume-title":"Why Reproducible Research is Beneficial for Security Research. under review","year":"2015","author":"Madeyski Lech","key":"e_1_3_2_1_46_1"},{"volume-title":"USENIX Security Symposium","author":"McGregor Susan E.","key":"e_1_3_2_1_47_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 505--522","year":"2017","author":"McGregor Susan E.","key":"e_1_3_2_1_48_1"},{"volume-title":"Understanding the Reproducibility of Crowd-reported Security Vulnerabilities. In USENIX Security Symposium. USENIX Association, 919--936","year":"2018","author":"Mu Dongliang","key":"e_1_3_2_1_49_1"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134082"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2810103.2813660"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3133977"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Brian A Nosek George Alter George C Banks Denny Borsboom Sara D Bowman Steven J Breckler Stuart Buck Christopher D Chambers Gilbert Chin Garret Christensen etal 2015. Promoting an open research culture. Science 348 6242 (2015) 1422--1425.  Brian A Nosek George Alter George C Banks Denny Borsboom Sara D Bowman Steven J Breckler Stuart Buck Christopher D Chambers Gilbert Chin Garret Christensen et al. 2015. Promoting an open research culture. Science 348 6242 (2015) 1422--1425.","DOI":"10.1126\/science.aab2374"},{"volume-title":"SmarPer: Context-Aware and Automatic Runtime-Permissions for Mobile Devices. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 1058--1076","year":"2017","author":"Olejnik Katarzyna","key":"e_1_3_2_1_54_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 239--254","year":"2015","author":"Oltrogge Marten","key":"e_1_3_2_1_55_1"},{"volume-title":"Estimating the reproducibility of psychological science. Science 349, 6251","year":"2015","author":"Collaboration Open Science","key":"e_1_3_2_1_56_1"},{"volume-title":"Exploring User Perceptions of Discrimination in Online Targeted Advertising. In USENIX Security Symposium. USENIX Association, 935--951","year":"2017","author":"Plane Angelisa C.","key":"e_1_3_2_1_57_1"},{"volume-title":"Reproducibility vs. replicability: a brief history of a confused terminology. Frontiers in neuroinformatics 11","year":"2018","author":"Plesser Hans E","key":"e_1_3_2_1_58_1"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660267.2660317"},{"key":"e_1_3_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1145\/2976749.2978307"},{"volume-title":"IEEE Symposium on Security and Privacy. IEEE Computer Society, 272--288","author":"Redmiles Elissa M.","key":"e_1_3_2_1_61_1"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243740"},{"volume-title":"IEEE Symposium on Security and Privacy. IEEE Computer Society, 872--888","author":"Reynolds Joshua","key":"e_1_3_2_1_63_1"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/2976749.2978334"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"crossref","unstructured":"Kelly Servick. 2018. 'Generous' approach to replication confirms many high-profile social science findings. https:\/\/www.sciencemag.org\/news\/2018\/08\/generous-approach-replication-confirms-many-high-profile-social-science-findings.  Kelly Servick. 2018. 'Generous' approach to replication confirms many high-profile social science findings. https:\/\/www.sciencemag.org\/news\/2018\/08\/generous-approach-replication-confirms-many-high-profile-social-science-findings.","DOI":"10.1126\/science.aav2151"},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3243734.3243779"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/2660267.2660274"},{"volume-title":"Computer Security and Privacy for Refugees in the United States. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 409--423","year":"2018","author":"Simko Lucy","key":"e_1_3_2_1_68_1"},{"volume-title":"Mazurek","year":"2018","author":"Stevens Rock","key":"e_1_3_2_1_69_1"},{"volume-title":"Basics of qualitative research techniques","author":"Strauss Anselm","key":"e_1_3_2_1_70_1"},{"volume-title":"SmartAuth: User-Centered Authorization for the Internet of Things. In USENIX Security Symposium. USENIX Association, 361--378","year":"2017","author":"Tian Yuan","key":"e_1_3_2_1_71_1"},{"volume-title":"Users Really Do Plug in USB Drives They Find. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 306--319","year":"2016","author":"Tischer Matthew","key":"e_1_3_2_1_72_1"},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2015.33"},{"volume-title":"Reconstructing the Giant: On the Importance of Rigour in Documenting the Literature Search Process. In ECIS 2009 Proceedings. 2206--2217","year":"2009","author":"vom Brocke Jan","key":"e_1_3_2_1_74_1"},{"volume-title":"Testers: A Comparison of Software Vulnerability Discovery Processes. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 374--391","author":"Votipka Daniel","key":"e_1_3_2_1_75_1"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.5555\/2017160.2017162"},{"volume-title":"Android Permissions Remystified: A Field Study on Contextual Integrity. In USENIX Security Symposium. USENIX Association, 499--514","year":"2015","author":"Wijesekera Primal","key":"e_1_3_2_1_77_1"},{"volume-title":"The Feasibility of Dynamically Granted Permissions: Aligning Mobile Privacy with User Preferences. In IEEE Symposium on Security and Privacy. IEEE Computer Society, 1077--1093","year":"2017","author":"Wijesekera Primal","key":"e_1_3_2_1_78_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 411--428","year":"2018","author":"Winter Philipp","key":"e_1_3_2_1_79_1"},{"volume-title":"Gunter","year":"2018","author":"Yuan Xuejing","key":"e_1_3_2_1_80_1"},{"volume-title":"All Your GPS Are Belong To Us: Towards Stealthy Manipulation of Road Navigation Systems. In USENIX Security Symposium. USENIX Association, 1527--1544","year":"2018","author":"Zeng Kexiong","key":"e_1_3_2_1_81_1"},{"volume-title":"USENIX Security Symposium. USENIX Association, 1061--1078","year":"2018","author":"Zhang-Kennedy Leah","key":"e_1_3_2_1_82_1"}],"event":{"name":"ARES '19: 14th International Conference on Availability, Reliability and Security","acronym":"ARES '19","location":"Canterbury CA United Kingdom"},"container-title":["Proceedings of the 14th International Conference on Availability, Reliability and Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3339252.3340339","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3339252.3340339","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T23:44:17Z","timestamp":1750203857000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3339252.3340339"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,8,26]]},"references-count":82,"alternative-id":["10.1145\/3339252.3340339","10.1145\/3339252"],"URL":"https:\/\/doi.org\/10.1145\/3339252.3340339","relation":{},"subject":[],"published":{"date-parts":[[2019,8,26]]},"assertion":[{"value":"2019-08-26","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}