{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,5]],"date-time":"2026-03-05T15:36:38Z","timestamp":1772724998432,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":31,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,3,10]],"date-time":"2024-03-10T00:00:00Z","timestamp":1710028800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100006374","name":"Australian Research Council","doi-asserted-by":"publisher","award":["DP190101113"],"award-info":[{"award-number":["DP190101113"]}],"id":[{"id":"10.13039\/501100006374","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,3,10]]},"DOI":"10.1145\/3627508.3638322","type":"proceedings-article","created":{"date-parts":[[2024,3,8]],"date-time":"2024-03-08T14:03:01Z","timestamp":1709906581000},"page":"340-345","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":12,"title":["Enhancing Human Annotation: Leveraging Large Language Models and Efficient Batch Processing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1535-0989","authenticated-orcid":false,"given":"Oleg","family":"Zendel","sequence":"first","affiliation":[{"name":"RMIT University, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1902-9087","authenticated-orcid":false,"given":"J. Shane","family":"Culpepper","sequence":"additional","affiliation":[{"name":"The University of Queensland, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9094-0810","authenticated-orcid":false,"given":"Falk","family":"Scholer","sequence":"additional","affiliation":[{"name":"RMIT University, Australia"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2425-3136","authenticated-orcid":false,"given":"Paul","family":"Thomas","sequence":"additional","affiliation":[{"name":"Microsoft, Australia"}]}],"member":"320","published-online":{"date-parts":[[2024,3,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591960"},{"key":"e_1_3_2_1_2_1","volume-title":"Open-Source Large Language Models Outperform Crowd Workers and Approach ChatGPT in Text-Annotation Tasks. arXiv preprint arXiv: 2307.02179","author":"Alizadeh Meysam","year":"2023","unstructured":"Meysam Alizadeh, Ma\u00ebl Kubli, Zeynab Samei, Shirin Dehghani, Juan\u00a0Diego Bermeo, Maria Korobeynikova, and Fabrizio Gilardi. 2023. Open-Source Large Language Models Outperform Crowd Workers and Approach ChatGPT in Text-Annotation Tasks. arXiv preprint arXiv: 2307.02179 (2023)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767728"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclepro.2017.12.239"},{"key":"e_1_3_2_1_6_1","volume-title":"Proc. ECIR. 57\u201371","author":"J.","unstructured":"David\u00a0J. Bell and Ian Ruthven. 2004. Searcher\u2019s Assessments of Task Complexity for Web Searching. In Proc. ECIR. 57\u201371."},{"key":"e_1_3_2_1_7_1","first-page":"191","article-title":"Task Complexity Affects Information Seeking and","volume":"2","author":"Bystr\u00f6m Katriina","year":"1995","unstructured":"Katriina Bystr\u00f6m and Kalervo J\u00e4rvelin. 1995. Task Complexity Affects Information Seeking and Use. Inf. Proc. & Man.2 (1995), 191\u2013213.","journal-title":"Use. Inf. Proc. & Man."},{"key":"e_1_3_2_1_8_1","volume-title":"Yuan Li, Jaime Arguello, and Robert Capra.","author":"Choi Bogeum","year":"2019","unstructured":"Bogeum Choi, Austin Ward, Yuan Li, Jaime Arguello, and Robert Capra. 2019. The Effects of Task Complexity on the Use of Different Types of Information in a Search Assistance Tool. ACM Trans. Inf. Sys. 38 (2019)."},{"key":"e_1_3_2_1_9_1","volume-title":"Using Large Language Models in Psychology. Nature Reviews Psychology","author":"Demszky D.","year":"2023","unstructured":"D. Demszky, D. Yang, D.\u00a0S. Yeager, C.\u00a0J. Bryan, M. Clapper, S. Chandhok, C.\u00a0J. Eichstaedt, C. Hecht, J. Jamieson, M. Johnson, M. Jones, D. Krettek-Cobb, L. Lai, N. JonesMitchell, D.\u00a0C. Ong, C.\u00a0S. Dweck, J.\u00a0J. Gross, and J.\u00a0W. Pennebaker. 2023. Using Large Language Models in Psychology. Nature Reviews Psychology (2023)."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605136"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3176349.3176386"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2305016120"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2305016120"},{"key":"e_1_3_2_1_14_1","volume-title":"Proc. NeurIPS, Vol.\u00a035","author":"Kojima Takeshi","year":"2022","unstructured":"Takeshi Kojima, Shixiang\u00a0(Shane) Gu, Machel Reid, Yutaka Matsuo, and Yusuke Iwasawa. 2022. Large Language Models are Zero-Shot Reasoners. In Proc. NeurIPS, Vol.\u00a035. 22199\u201322213."},{"key":"e_1_3_2_1_15_1","unstructured":"David\u00a0R. Krathwohl Lorin\u00a0W. Anderson and Benjamin\u00a0Samuel Bloom. 2001. A Taxonomy for Learning Teaching and Assessing : A Revision of Bloom\u2019s Taxonomy of Educational Objectives (complete ed.)."},{"key":"e_1_3_2_1_16_1","volume-title":"Content Analysis: An Introduction to Its Methodology","author":"Krippendorff Klaus","year":"2022","unstructured":"Klaus Krippendorff. 2022. Content Analysis: An Introduction to Its Methodology (fourth ed.). SAGE Publications, Inc."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3592032"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340531.3412762"},{"key":"e_1_3_2_1_19_1","volume-title":"Proc. NeurIPS, Vol.\u00a035","author":"Ouyang Long","year":"2022","unstructured":"Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul\u00a0F Christiano, Jan Leike, and Ryan Lowe. 2022. Training Language Models to Follow Instructions With Human Feedback. In Proc. NeurIPS, Vol.\u00a035. 27730\u201327744."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102688"},{"key":"e_1_3_2_1_22_1","volume-title":"Proc. CHIIR. 25\u201335","author":"Thomas Paul","year":"2022","unstructured":"Paul Thomas, Gabriella Kazai, Ryen\u00a0W White, and Nick Craswell. 2022. The Crowd is Made of People Observations from Large-Scale Crowd Labelling. In Proc. CHIIR. 25\u201335."},{"key":"e_1_3_2_1_23_1","volume-title":"Large Language Models Can Accurately Predict Searcher Preferences. arXiv preprint arXiv:2309.10621","author":"Thomas Paul","year":"2023","unstructured":"Paul Thomas, Seth Spielman, Nick Craswell, and Bhaskar Mitra. 2023. Large Language Models Can Accurately Predict Searcher Preferences. arXiv preprint arXiv:2309.10621 (2023)."},{"key":"e_1_3_2_1_24_1","volume-title":"ChatGPT-4 Outperforms Experts and Crowd Workers in Annotating Political Twitter Messages with Zero-Shot Learning. arXiv preprint arXiv: 2304.06588","author":"T\u00f6rnberg Petter","year":"2023","unstructured":"Petter T\u00f6rnberg. 2023. ChatGPT-4 Outperforms Experts and Crowd Workers in Annotating Political Twitter Messages with Zero-Shot Learning. arXiv preprint arXiv: 2304.06588 (2023)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(99)00028-X"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2362724.2362768"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531893"},{"key":"e_1_3_2_1_28_1","volume-title":"Proc. ICLR.","author":"Zhang Tianjun","year":"2023","unstructured":"Tianjun Zhang, Xuezhi Wang, Denny Zhou, Dale Schuurmans, and Joseph\u00a0E. Gonzalez. 2023. TEMPERA: Test-Time Prompt Editing via Reinforcement Learning. In Proc. ICLR."},{"key":"e_1_3_2_1_29_1","volume-title":"Proc. ICLR.","author":"Zhou Yongchao","year":"2023","unstructured":"Yongchao Zhou, Andrei\u00a0Ioan Muresanu, Ziwen Han, Keiran Paster, Silviu Pitis, Harris Chan, and Jimmy Ba. 2023. Large Language Models Are Human-Level Prompt Engineers. In Proc. ICLR."},{"key":"e_1_3_2_1_30_1","volume-title":"Large Language Models for Information Retrieval:A Survey. arXiv preprint arXiv: 2308.07107","author":"Zhu Yutao","year":"2023","unstructured":"Yutao Zhu, Huaying Yuan, Shuting Wang, Jiongnan Liu, Wenhan Liu, Chenlong Deng, Zhicheng Dou, and Ji-Rong Wen. 2023. Large Language Models for Information Retrieval:A Survey. arXiv preprint arXiv: 2308.07107 (2023)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3578337.3605121"}],"event":{"name":"CHIIR '24: 2024 ACM SIGIR Conference on Human Information Interaction and Retrieval","location":"Sheffield United Kingdom","acronym":"CHIIR '24"},"container-title":["Proceedings of the 2024 ACM SIGIR Conference on Human Information Interaction and Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627508.3638322","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3627508.3638322","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,29]],"date-time":"2025-08-29T17:37:01Z","timestamp":1756489021000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3627508.3638322"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,10]]},"references-count":31,"alternative-id":["10.1145\/3627508.3638322","10.1145\/3627508"],"URL":"https:\/\/doi.org\/10.1145\/3627508.3638322","relation":{},"subject":[],"published":{"date-parts":[[2024,3,10]]},"assertion":[{"value":"2024-03-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}