{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,15]],"date-time":"2026-04-15T17:51:52Z","timestamp":1776275512031,"version":"3.50.1"},"reference-count":129,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"funder":[{"name":"Thomson Reuters"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2022]]},"DOI":"10.1109\/access.2022.3190408","type":"journal-article","created":{"date-parts":[[2022,7,12]],"date-time":"2022-07-12T19:26:58Z","timestamp":1657654018000},"page":"75835-75858","source":"Crossref","is-referenced-by-count":24,"title":["On the Effectiveness of Pre-Trained Language Models for Legal Natural Language Processing: An Empirical Study"],"prefix":"10.1109","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2553-3108","authenticated-orcid":false,"given":"Dezhao","family":"Song","sequence":"first","affiliation":[{"name":"Thomson Reuters, Eagan, MN, USA"}]},{"given":"Sally","family":"Gao","sequence":"additional","affiliation":[{"name":"Thomson Reuters, New York, NY, USA"}]},{"given":"Baosheng","family":"He","sequence":"additional","affiliation":[{"name":"Meta Platforms Inc., Menlo Park, CA, USA"}]},{"given":"Frank","family":"Schilder","sequence":"additional","affiliation":[{"name":"Thomson Reuters, Eagan, MN, USA"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.1810.04805"},{"key":"ref2","article-title":"RoBERTa: A robustly optimized BERT pretraining approach","volume-title":"arXiv:1907.11692","author":"Liu","year":"2019"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.703"},{"key":"ref4","first-page":"11328","article-title":"PEGASUS: Pre-training with extracted gap-sentences for abstractive summarization","volume-title":"Proc. 37th Int. Conf. Mach. Learn. (ICML)","volume":"119","author":"Zhang"},{"key":"ref5","first-page":"1","article-title":"Big bird: Transformers for longer sequences","volume-title":"Proc. Adv. Neural Inf. Process. Syst., Annu. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Zaheer"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/d19-1371"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1093\/bioinformatics\/btz682"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3950755"},{"key":"ref9","first-page":"1","article-title":"GraphcodeBERT: Pre-training code representations with data flow","volume-title":"Proc. ICLR","author":"Guo"},{"key":"ref10","first-page":"2898","article-title":"LEGAL-BERT: \u2018Preparing the muppets for court,\u2019","volume-title":"Findings Assoc. Comput. Linguistics (EMNLP)","author":"Chalkidis","year":"2020"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-013-9149-8"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2021.101718"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-019-09243-2"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/3462757.3466088"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p19-1636"},{"key":"ref16","first-page":"1235","article-title":"LEDGAR: A large-scale multi-label corpus for text classification of legal provisions in contracts","volume-title":"Proc. 12th Lang. Resour. Eval. Conf. (LREC)","author":"Tuggener"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/d19-1500"},{"key":"ref18","first-page":"2142","article-title":"The JRC-Acquis: A multilingual aligned parallel corpus with 20+ languages","volume-title":"Proc. 5th Int. Conf. Lang. Resour. Eval. (LREC)","author":"Steinberger"},{"key":"ref19","article-title":"BillSum: A corpus for automatic summarization of U.S. legislation","volume-title":"arXiv:1910.00523","author":"Kornilova","year":"2019"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/BigData50022.2020.9378308"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1007\/s12626-022-00105-z"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.nllp-1.3"},{"key":"ref23","article-title":"CAIL2018: A large-scale legal dataset for judgment prediction","volume-title":"arXiv:1807.02478","author":"Xiao","year":"2018"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.nllp-1.13"},{"key":"ref25","first-page":"116","article-title":"Litigation analytics: Extracting and querying motions and orders from U.S. federal courts","volume-title":"Proc. Conf. North Amer. Chapter Assoc. Comput. Linguistics: Hum. Lang. Technol. (NAACL-HLT)","author":"Vacek"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/bf00994018"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/3462757.3466102"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/484"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-79942-7_13"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/w18-5446"},{"key":"ref31","first-page":"3261","article-title":"Superglue: A stickier benchmark for general-purpose language understanding systems","volume-title":"Proc. Adv. Neural Inf. Process. Syst., Annu. Conf. Neural Inf. Process. Syst. (NeurIPS)","author":"Wang"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.419"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.naacl-main.200"},{"key":"ref34","article-title":"CUGE: A Chinese language understanding and generation evaluation benchmark","volume-title":"arXiv:2112.13610","author":"Yao","year":"2021"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3936759"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.gem-1.10"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p19-1424"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1162\/neco.1997.9.8.1735"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/n16-1174"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/n18-1100"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2021.101822"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2021.102798"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3290979"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3097987"},{"key":"ref45","first-page":"3069","article-title":"PD-Sparse: A primal and dual sparse approach to extreme multiclass and multilabel classification","volume-title":"Proc. 33rd Int. Conf. Mach. Learn. (ICML)","volume":"48","author":"Yen"},{"issue":"1","key":"ref46","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests","volume":"45","author":"Breiman","year":"2001","journal-title":"Mach. Learn."},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623651"},{"key":"ref48","first-page":"4671","article-title":"Craftml, an efficient clustering-based random forest for extreme multi-label learning","volume-title":"Proc. 35th Int. Conf. Mach. Learn. (ICML)","volume":"80","author":"Siblini"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1016\/0893-6080(89)90020-8"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21363"},{"key":"ref51","first-page":"404","article-title":"TextRank: Bringing order into text","volume-title":"Proc. Conf. Empirical Methods Natural Lang. Process., EMNLP meeting SIGDAT, Special Interest Group ACL, Held Conjunct. ACL","author":"Mihalcea"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1523"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2016.12.021"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/d14-1179"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/p18-1063"},{"key":"ref56","first-page":"1","article-title":"Towards CNL-based verbalization of computational contracts","volume-title":"Proc. 7th Int. Workshop Controlled Natural Lang. (CNL)","author":"Listenmaa"},{"key":"ref57","first-page":"258","article-title":"Casesummarizer: A system for automated summarization of legal texts","volume-title":"Proc. COLING 26th Int. Conf. Comput. Linguistics, Conf. Syst. Demonstrations","author":"Polsley"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-007-9039-z"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/2746090.2746096"},{"key":"ref60","first-page":"27","article-title":"Legal text summarization by exploration of the thematic structure and argumentative roles","volume-title":"Text Summarization Branches Out","author":"Farzindar","year":"2004"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3357384.3357940"},{"key":"ref62","first-page":"1","article-title":"Global-aware beam search for neural abstractive summarization","volume":"34","author":"Ma","year":"2021","journal-title":"Proc. Adv. Neural Inf. Process. Syst."},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-017-9566-2"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1016\/j.cosrev.2021.100388"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/3411763.3443441"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-10061-6_14"},{"key":"ref67","doi-asserted-by":"publisher","DOI":"10.29007\/16q5"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/3322640.3326711"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-50953-2_20"},{"key":"ref70","article-title":"Legal question answering using ranking SVM and deep convolutional neural network","volume-title":"arXiv:1703.05320","author":"Do","year":"2017"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6519"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33017088"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020\/510"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2021.06.003"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6310"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i05.6502"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v36i10.21316"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.emnlp-main.295"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1016\/j.neucom.2021.06.076"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.411"},{"key":"ref81","article-title":"Machine reading comprehension: The role of contextualized language models and beyond","volume-title":"arXiv:2005.06249","author":"Zhang","year":"2020"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.eacl-main.305"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.3233\/FAIA210326"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1145\/3459637.3481994"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462876"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1007\/BF00877694"},{"key":"ref87","first-page":"73","article-title":"Extending full text search for legal document collections using word embeddings","volume-title":"Proc. JURIX","volume":"294","author":"Landthaler"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.3233\/FAIA210328"},{"key":"ref89","first-page":"125","article-title":"A semi-supervised training method for semantic search of legal facts in Canadian immigration cases","volume-title":"Proc. JURIX","volume":"302","author":"Nejadgholi"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-021-09293-5"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-37256-8_32"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2021.101842"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4471-2099-5_22"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-70145-5_14"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1145\/3322640.3326740"},{"key":"ref96","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_2"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/j.is.2021.101967"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1109\/TII.2021.3131180"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463250"},{"key":"ref100","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-2124"},{"key":"ref101","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-15712-8_27"},{"key":"ref102","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-021-09282-8"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-021-09296-2"},{"key":"ref104","doi-asserted-by":"publisher","DOI":"10.1007\/s10506-018-9238-9"},{"key":"ref105","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.acl-main.466"},{"key":"ref106","first-page":"793","article-title":"Textual predictors of bill survival in congressional committees","volume-title":"Proc. HLT-NAACL","author":"Yano"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1109\/ICIINFS.2017.8300343"},{"key":"ref108","first-page":"1","article-title":"Plain English summarization of contracts","volume-title":"Proc. Natural Legal Lang. Process. Workshop","author":"Manor"},{"key":"ref109","article-title":"Passing a USA national bar exam: A first corpus for experimentation","volume-title":"Proc. 10th Int. Conf. Lang. Resour. Eval. (LREC)","author":"Fawei"},{"key":"ref110","article-title":"Solving bar exam questions with deep neural networks","volume-title":"Proc. 2nd Workshop Automated Semantic Anal. Inf. Legal Texts Co-Located 16th Int. Conf. Artif. Intell. Law (ICAIL)","volume":"2143","author":"Adebayo"},{"key":"ref111","first-page":"1","article-title":"Overview of the FIRE 2020 AILA track: Artificial intelligence for legal assistance","volume-title":"FIRE (Working Notes)","volume":"2826","author":"Bhattacharya","year":"2020"},{"key":"ref112","article-title":"A statutory article retrieval dataset in French","volume-title":"arXiv:2108.11792","author":"Louis","year":"2021"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.coling-main.86"},{"key":"ref114","first-page":"140","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"J. Mach. Learn. Res."},{"key":"ref115","first-page":"2825","article-title":"Scikit-learn: Machine learning in Python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"J. Mach. Learn. Res."},{"key":"ref116","volume-title":"Keras","author":"Chollet","year":"2015"},{"key":"ref117","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i9.16974"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2021.acl-long.501"},{"key":"ref119","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/D14-1162"},{"key":"ref120","first-page":"109","article-title":"Okapi at TREC-3","volume-title":"Proc. 3rd Text Retr. Conf. (TREC)","author":"Robertson"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291008"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1720347115"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.1177\/1077699020932304"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1177\/0956797620963619"},{"key":"ref125","article-title":"Equality before the law: Legal judgment consistency analysis for fairness","volume-title":"arXiv:2103.13868","author":"Wang","year":"2021"},{"key":"ref126","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1339"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-62077-6_14"},{"key":"ref128","first-page":"6565","article-title":"Towards understanding and mitigating social biases in language models","volume-title":"Proc. Int. Conf. Mach. Learn.","volume":"139","author":"Liang"},{"key":"ref129","article-title":"Unintended bias in language model-driven conversational recommendation","volume-title":"arXiv:2201.06224","author":"Shen","year":"2022"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9668973\/09826728.pdf?arnumber=9826728","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,1]],"date-time":"2024-02-01T03:51:24Z","timestamp":1706759484000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9826728\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"references-count":129,"URL":"https:\/\/doi.org\/10.1109\/access.2022.3190408","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022]]}}}