{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,7]],"date-time":"2026-10-07T06:36:17Z","timestamp":1791354977062,"version":"4.3.1"},"publisher-location":"New York, NY, USA","reference-count":54,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T00:00:00Z","timestamp":1712880000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,4,12]]},"DOI":"10.1145\/3597503.3639075","type":"proceedings-article","created":{"date-parts":[[2024,4,12]],"date-time":"2024-04-12T16:43:26Z","timestamp":1712940206000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["ChatGPT-Resistant Screening Instrument for Identifying Non-Programmers"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9137-5072","authenticated-orcid":false,"given":"Raphael","family":"Serafini","sequence":"first","affiliation":[{"name":"Ruhr University, Bochum, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-4208-2803","authenticated-orcid":false,"given":"Clemens","family":"Otto","sequence":"additional","affiliation":[{"name":"Ruhr University, Bochum, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4053-0706","authenticated-orcid":false,"given":"Stefan Albert","family":"Horstmann","sequence":"additional","affiliation":[{"name":"Ruhr University, Bochum, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1843-2027","authenticated-orcid":false,"given":"Alena","family":"Naiakshina","sequence":"additional","affiliation":[{"name":"Ruhr University, Bochum, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,4,12]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2016.25"},{"key":"e_1_3_2_1_2_1","volume-title":"n.d.. Alexa. Retrieved","year":"2023","unstructured":"Amazon. n.d.. Alexa. Retrieved July 2023 from https:\/\/developer.amazon.com\/"},{"key":"e_1_3_2_1_3_1","volume-title":"n.d.. Amazon Mechanical Turk. Retrieved","year":"2023","unstructured":"Amazon. n.d.. Amazon Mechanical Turk. Retrieved July 2023 from https:\/\/www.mturk.com\/"},{"key":"e_1_3_2_1_4_1","volume-title":"n.d.. Siri. Retrieved","author":"Apple Inc.","year":"2023","unstructured":"Apple Inc. n.d.. Siri. Retrieved July 2023 from https:\/\/www.apple.com\/siri\/"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300519"},{"key":"e_1_3_2_1_6_1","unstructured":"Eugene Bagdasaryan Tsung-Yin Hsieh Ben Nassi and Vitaly Shmatikov. 2023. Abusing Images and Sounds for Indirect Instruction Injection in Multi-Modal LLMs. arXiv:2307.10490 [cs.CR]"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","unstructured":"Rebecca Balebako Abigail Marsh Jialiu Lin Jason Hong and Lorrie Cranor. 2014. The Privacy and Security Behaviors of Smartphone App Developers. 10.14722\/usec.2014.23006","DOI":"10.14722\/usec.2014.23006"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2961111.2962628"},{"key":"e_1_3_2_1_9_1","unstructured":"Chetana Belagere. 2023. Students have started using ChatGPT to cheat in assignments tests. How are professors catching them? https:\/\/thesouthfirst.com\/karnataka\/students-have-started-using-chatgpt-to-cheat-in-tests-exams-how-are-professors-catching-them\/."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP46214.2022.9833641"},{"key":"e_1_3_2_1_11_1","volume-title":"Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang.","author":"Bubeck S\u00e9bastien","year":"2023","unstructured":"S\u00e9bastien Bubeck, Varun Chandrasekaran, Ronen Eldan, Johannes Gehrke, Eric Horvitz, Ece Kamar, Peter Lee, Yin Tat Lee, Yuanzhi Li, Scott Lundberg, Harsha Nori, Hamid Palangi, Marco Tulio Ribeiro, and Yi Zhang. 2023. Sparks of Artificial General Intelligence: Early experiments with GPT-4. arXiv:2303.12712 [cs.CL]"},{"key":"e_1_3_2_1_12_1","unstructured":"M. Chinmoy. 2016. Cracking the Coding Interview: 189 Programming Questions and Solutions. Blurb."},{"key":"e_1_3_2_1_13_1","volume-title":"ChatGPT goes to law school. Available at SSRN","author":"Choi Jonathan H","year":"2023","unstructured":"Jonathan H Choi, Kristin E Hickman, Amy Monahan, and Daniel Schwarcz. 2023. ChatGPT goes to law school. Available at SSRN (2023)."},{"key":"e_1_3_2_1_14_1","volume-title":"n.d.. Capture2Text. Retrieved","author":"Brochtrup Christopher","year":"2023","unstructured":"Christopher Brochtrup. n.d.. Capture2Text. Retrieved July 2023 from https:\/\/capture2text.sourceforge.net\/"},{"key":"e_1_3_2_1_15_1","volume-title":"n.d.. Clickworker. Retrieved","author":"Clickworker","year":"2023","unstructured":"Clickworker GmbH. n.d.. Clickworker. Retrieved July 2023 from https:\/\/www.clickworker.com\/"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3510003.3510223"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00057"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380387"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174220"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.3758\/BRM.41.4.1149"},{"key":"e_1_3_2_1_21_1","volume-title":"Discovering statistics using R","author":"Field Andy","unstructured":"Andy Field, Jeremy Miles, and Zo\u00eb Field. 2012. Discovering statistics using R. Sage publications."},{"key":"e_1_3_2_1_22_1","volume-title":"n.d.. Retrieved","author":"Hughes Colin","year":"2023","unstructured":"Colin Hughes. n.d.. Retrieved July 2023 from https:\/\/projecteuler.net\/problem=1"},{"key":"e_1_3_2_1_23_1","volume-title":"Bartholomew and Dhrumil Mehta","author":"Jem","year":"2023","unstructured":"Jem, Bartholomew and Dhrumil Mehta. 2023. How the media is covering Chat-GPT. Retrieved July 2023 from https:\/\/www.cjr.org\/tow_center\/media-coverage-chatgpt.php"},{"key":"e_1_3_2_1_24_1","volume-title":"31st USENIX Security Symposium (USENIX Security 22)","author":"Kaur Harjot","year":"2022","unstructured":"Harjot Kaur, Sabrina Amft, Daniel Votipka, Yasemin Acar, and Sascha Fahl. 2022. Where to Recruit for Security Development Studies: Comparing Six Software Developer Samples. In 31st USENIX Security Symposium (USENIX Security 22). USENIX Association, Boston, MA, 4041--4058. https:\/\/www.usenix.org\/conference\/usenixsecurity22\/presentation\/kaur"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the 26th USENIX Conference on Security Symposium","author":"Krombholz Katharina","year":"2017","unstructured":"Katharina Krombholz, Wilfried Mayer, Martin Schmiedecker, and Edgar Weippl. 2017. \"I Have No Idea What i'm Doing\": On the Usability of Deploying HTTPS. In Proceedings of the 26th USENIX Conference on Security Symposium (Vancouver, BC, Canada) (SEC'17). USENIX Association, USA, 1339--1356."},{"key":"e_1_3_2_1_26_1","volume-title":"Camille Elepa\u00f1o, Maria Madriaga, Rimel Aggabao, Giezel Diaz-Candido, James Maningo, et al.","author":"Kung Tiffany H","year":"2023","unstructured":"Tiffany H Kung, Morgan Cheatham, Arielle Medenilla, Czarina Sillos, Lorie De Leon, Camille Elepa\u00f1o, Maria Madriaga, Rimel Aggabao, Giezel Diaz-Candido, James Maningo, et al. 2023. Performance of ChatGPT on USMLE: Potential for AI-assisted medical education using large language models. PLoS digital health 2, 2 (2023), e0000198."},{"key":"e_1_3_2_1_27_1","volume-title":"Retrieved","author":"PowerToys Microsoft","year":"2023","unstructured":"Microsoft. 2023. Microsoft PowerToys. Retrieved July 2023 from https:\/\/learn.microsoft.com\/en-us\/windows\/powertoys\/"},{"key":"e_1_3_2_1_28_1","volume-title":"n.d.. Cortana. Retrieved","year":"2023","unstructured":"Microsoft. n.d.. Cortana. Retrieved July 2023 from https:\/\/www.microsoft.com\/en-us\/cortana"},{"key":"e_1_3_2_1_29_1","volume-title":"n.d.. Microsoft Bing. Retrieved","year":"2023","unstructured":"Microsoft. n.d.. Microsoft Bing. Retrieved July 2023 from https:\/\/www.bing.com\/?\/ai"},{"key":"e_1_3_2_1_30_1","volume-title":"Retrieved","author":"Statistics GPT","year":"2023","unstructured":"Nerdynav. 2023. 97+ ChatGPT Statistics & User Numbers in July 2023 (New Data). Retrieved July 2023 from https:\/\/nerdynav.com\/chatgpt-statistics\/"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0042-6989(02)00077-9"},{"key":"e_1_3_2_1_32_1","volume-title":"ChatGPT: Optimizing language models for dialogue. Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. 2022. ChatGPT: Optimizing language models for dialogue. Retrieved July 2023 from https:\/\/openai.com\/blog\/chatgpt\/"},{"key":"e_1_3_2_1_33_1","volume-title":"ChatGPT --- Release Notes. Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. 2023. ChatGPT --- Release Notes. Retrieved November 2023 from https:\/\/help.openai.com\/en\/articles\/6825453-chatgpt-release-notes"},{"key":"e_1_3_2_1_34_1","volume-title":"Image inputs for ChatGPT - FAQ. Retrieved","author":"AI.","year":"2024","unstructured":"OpenAI. 2023. Image inputs for ChatGPT - FAQ. Retrieved January 2024 from https:\/\/help.openai.com\/en\/articles\/8400551-image-inputs-for-chatgpt-faq"},{"key":"e_1_3_2_1_35_1","volume-title":"n.d.. ChatGPT. Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. n.d.. ChatGPT. Retrieved July 2023 from https:\/\/chat.apps.openai.com\/"},{"key":"e_1_3_2_1_36_1","volume-title":"n.d.. ChatGPT GUI. Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. n.d.. ChatGPT GUI. Retrieved July 2023 from https:\/\/chat.openai.com\/"},{"key":"e_1_3_2_1_37_1","volume-title":"n.d.. OpenAI. Retrieved","author":"AI.","year":"2023","unstructured":"OpenAI. n.d.. OpenAI. Retrieved July 2023 from https:\/\/openai.com\/"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","unstructured":"Long Ouyang Jeff Wu Xu Jiang Diogo Almeida Carroll L. Wainwright Pamela Mishkin Chong Zhang Sandhini Agarwal Katarina Slama Alex Ray John Schulman Jacob Hilton Fraser Kelton Luke Miller Maddie Simens Amanda Askell Peter Welinder Paul Christiano Jan Leike and Ryan Lowe. 2022. Training language models to follow instructions with human feedback. https:\/\/arxiv.org\/abs\/2203.02155. 10.48550\/ARXIV.2203.02155","DOI":"10.48550\/ARXIV.2203.02155"},{"key":"e_1_3_2_1_39_1","unstructured":"Martine Paris. 2023. ChatGPT Hits 100 Million Users Google Invests In AI Bot And CatGPT Goes Viral. https:\/\/www.forbes.com\/sites\/martineparis\/2023\/02\/03\/chatgpt-hits-100-million-microsoft-unleashes-ai-bots-and-catgpt-goes-viral\/."},{"key":"e_1_3_2_1_40_1","unstructured":"Maria Carolina Penteado and F\u00e1bio Perez. 2023. Evaluating GPT-3.5 and GPT-4 on Grammatical Error Correction for Brazilian Portuguese. arXiv:2306.15788 [cs.CL]"},{"key":"e_1_3_2_1_41_1","volume-title":"n.d.. Prolific. Retrieved","year":"2023","unstructured":"Prolific. n.d.. Prolific. Retrieved July 2023 from https:\/\/www.prolific.co"},{"key":"e_1_3_2_1_42_1","volume-title":"n.d.. Qualtrics. Retrieved","year":"2023","unstructured":"Qualtrics. n.d.. Qualtrics. Retrieved July 2023 from https:\/\/www.qualtrics.com"},{"key":"e_1_3_2_1_43_1","volume-title":"Jansen","author":"Salminen Joni","year":"2021","unstructured":"Joni Salminen, Soon-gyo Jung, and Bernard J. Jansen. 2021. Suggestions for Online User Studies. In HCI International 2021 - Late Breaking Papers: Design and User Experience, Constantine Stephanidis, Marcelo M. Soares, Elizabeth Rosenzweig, Aaron Marcus, Sakae Yamamoto, Hirohiko Mori, Pei-Luen Patrick Rau, Gabriele Meiselwitz, Xiaowen Fang, and Abbas Moallem (Eds.). Springer International Publishing, Cham, 127--146."},{"key":"e_1_3_2_1_44_1","unstructured":"John Schulman Filip Wolski Prafulla Dhariwal Alec Radford and Oleg Klimov. 2017. Proximal Policy Optimization Algorithms. arXiv:1707.06347 [cs.LG]"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","unstructured":"Dominik Sobania Martin Briesch Carol Hanna and Justyna Petke. 2023. An Analysis of the Automatic Bug Fixing Performance of ChatGPT. https:\/\/arxiv.org\/abs\/2301.08653. 10.48550\/ARXIV.2301.08653","DOI":"10.48550\/ARXIV.2301.08653"},{"key":"e_1_3_2_1_46_1","unstructured":"Congzheng Song and Vitaly Shmatikov. 2018. Fooling OCR Systems with Adversarial Text Images. arXiv:1802.05385 [cs.LG]"},{"key":"e_1_3_2_1_47_1","volume-title":"Retrieved","author":"Survey Developer","year":"2023","unstructured":"StackOverflow. 2023. 2023 Developer Survey. Retrieved July 2023 from https:\/\/survey.stackoverflow.co\/2023\/#sentiment-and-usage-ai-selectt"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501957"},{"key":"e_1_3_2_1_49_1","unstructured":"Christian Terwiesch. 2023. Would Chat GPT3 Get a Wharton MBA? A Prediction Based on Its Performance in the Operations Management Course. https:\/\/mackinstitute.wharton.upenn.edu\/wp-content\/uploads\/2023\/01\/Christian-Terwiesch-Chat-GTP.pdf"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1177\/1098214005283748"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1186\/s40561-023-00237-x"},{"key":"e_1_3_2_1_52_1","volume-title":"n.d.. Top Rated. Retrieved","author":"Upwork Global Inc.","year":"2023","unstructured":"Upwork Global Inc. n.d.. Top Rated. Retrieved July 2023 from https:\/\/support.upwork.com\/hc\/en-us\/articles\/211068468-Top-Rated"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-49044-7_11"},{"key":"e_1_3_2_1_54_1","unstructured":"Daniel M. Ziegler Nisan Stiennon Jeffrey Wu Tom B. Brown Alec Radford Dario Amodei Paul Christiano and Geoffrey Irving. 2020. Fine-Tuning Language Models from Human Preferences. arXiv:1909.08593 [cs.CL]"}],"event":{"name":"ICSE '24: IEEE\/ACM 46th International Conference on Software Engineering","location":"Lisbon Portugal","acronym":"ICSE '24","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","IEEE CS","Faculty of Engineering of University of Porto"]},"container-title":["Proceedings of the IEEE\/ACM 46th International Conference on Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639075","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3597503.3639075","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T22:49:11Z","timestamp":1750286951000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3597503.3639075"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,4,12]]},"references-count":54,"alternative-id":["10.1145\/3597503.3639075","10.1145\/3597503"],"URL":"https:\/\/doi.org\/10.1145\/3597503.3639075","relation":{},"subject":[],"published":{"date-parts":[[2024,4,12]]},"assertion":[{"value":"2024-04-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}