{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T17:15:10Z","timestamp":1776100510987,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":9,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,19]],"date-time":"2023-06-19T00:00:00Z","timestamp":1687132800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Stanford University Human-Centered Artificial Intelligence","award":["283862"],"award-info":[{"award-number":["283862"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,19]]},"DOI":"10.1145\/3594536.3595139","type":"proceedings-article","created":{"date-parts":[[2023,9,7]],"date-time":"2023-09-07T23:40:01Z","timestamp":1694130001000},"page":"427-431","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Conceptual Questions in Developing Expert-Annotated Data"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-9488-6302","authenticated-orcid":false,"given":"Megan","family":"Ma","sequence":"first","affiliation":[{"name":"Stanford Center for Legal Informatics, Stanford Law School, Palo Alto, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8046-1701","authenticated-orcid":false,"given":"Brandon","family":"Waldon","sequence":"additional","affiliation":[{"name":"Stanford University, Palo Alto, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7121-5696","authenticated-orcid":false,"given":"Julian","family":"Nyarko","sequence":"additional","affiliation":[{"name":"Stanford Law School, Palo Alto, California, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,9,7]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Daniel Martin Katz, and Nikolaos Aletras","author":"Chalkidis Ilias","year":"2021","unstructured":"Ilias Chalkidis, Abhik Jana, Dirk Hartung, Michael Bommarito, Ion Androutsopoulos, Daniel Martin Katz, and Nikolaos Aletras. 2021. LexGLUE: A benchmark dataset for legal language understanding in English. arXiv preprint arXiv:2110.00976 (2021)."},{"key":"e_1_3_2_1_2_1","volume-title":"Cuad: An expert-annotated nlp dataset for legal contract review. arXiv preprint arXiv:2103.06268","author":"Hendrycks Dan","year":"2021","unstructured":"Dan Hendrycks, Collin Burns, Anya Chen, and Spencer Ball. 2021. Cuad: An expert-annotated nlp dataset for legal contract review. arXiv preprint arXiv:2103.06268 (2021)."},{"key":"e_1_3_2_1_3_1","volume-title":"ContractNLI: A dataset for document-level natural language inference for contracts. arXiv preprint arXiv:2110.01799","author":"Koreeda Yuta","year":"2021","unstructured":"Yuta Koreeda and Christopher D Manning. 2021. ContractNLI: A dataset for document-level natural language inference for contracts. arXiv preprint arXiv:2110.01799 (2021)."},{"key":"e_1_3_2_1_4_1","volume-title":"Learning from multiple expert annotators for enhancing anomaly detection in medical image analysis. arXiv preprint arXiv:2203.10611","author":"Le Khiem H","year":"2022","unstructured":"Khiem H Le, Tuan V Tran, Hieu H Pham, Hieu T Nguyen, Tung T Le, and Ha Q Nguyen. 2022. Learning from multiple expert annotators for enhancing anomaly detection in medical image analysis. arXiv preprint arXiv:2203.10611 (2022)."},{"key":"e_1_3_2_1_5_1","volume-title":"A benchmark for lease contract review. arXiv preprint arXiv:2010.10386","author":"Leivaditi Spyretta","year":"2020","unstructured":"Spyretta Leivaditi, Julien Rossi, and Evangelos Kanoulas. 2020. A benchmark for lease contract review. arXiv preprint arXiv:2010.10386 (2020)."},{"key":"e_1_3_2_1_6_1","volume-title":"2021 ABA PRIVATE TARGET MERGERS ACQUISITIONS DEAL POINTS STUDY. Retrieved","author":"Pearlman Jessica C.","year":"2023","unstructured":"Jessica C. Pearlman. 2021. 2021 ABA PRIVATE TARGET MERGERS ACQUISITIONS DEAL POINTS STUDY. Retrieved May 5, 2023 from https:\/\/www.klgates.com\/2021-ABA-Private-Target-Mergers-Acquisitions-Deal-Points-Study-12-31-2021"},{"key":"e_1_3_2_1_7_1","volume-title":"Two contrasting data annotation paradigms for subjective nlp tasks. arXiv preprint arXiv:2112.07475","author":"R\u00f6ttger Paul","year":"2021","unstructured":"Paul R\u00f6ttger, Bertie Vidgen, Dirk Hovy, and Janet B Pierrehumbert. 2021. Two contrasting data annotation paradigms for subjective nlp tasks. arXiv preprint arXiv:2112.07475 (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"MAUD: An Expert-Annotated Legal NLP Dataset for Merger Agreement Understanding. arXiv preprint arXiv:2301.00876","author":"Wang Steven H","year":"2023","unstructured":"Steven H Wang, Antoine Scardigli, Leonard Tang, Wei Chen, Dimitry Levkin, Anya Chen, Spencer Ball, Thomas Woodside, Oliver Zhang, and Dan Hendrycks. 2023. MAUD: An Expert-Annotated Legal NLP Dataset for Merger Agreement Understanding. arXiv preprint arXiv:2301.00876 (2023)."},{"key":"e_1_3_2_1_9_1","first-page":"219","article-title":"Contracts as systems","author":"Williams Spencer","year":"2020","unstructured":"Spencer Williams. 2020. Contracts as systems. Del. J. Corp. L. 45 (2020), 219.","journal-title":"Del. J. Corp."}],"event":{"name":"ICAIL 2023: Nineteenth International Conference on Artificial Intelligence and Law","location":"Braga Portugal","acronym":"ICAIL 2023","sponsor":["IAAIL Intl Asso for Artifical Intel & Law","UMinho University of Minho","SIGAI ACM Special Interest Group on Artificial Intelligence","AAAI Am Assoc for Artifical Intelligence"]},"container-title":["Proceedings of the Nineteenth International Conference on Artificial Intelligence and Law"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3594536.3595139","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3594536.3595139","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:06Z","timestamp":1750183746000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3594536.3595139"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,19]]},"references-count":9,"alternative-id":["10.1145\/3594536.3595139","10.1145\/3594536"],"URL":"https:\/\/doi.org\/10.1145\/3594536.3595139","relation":{},"subject":[],"published":{"date-parts":[[2023,6,19]]},"assertion":[{"value":"2023-09-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}