{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T18:52:10Z","timestamp":1781635930375,"version":"3.54.5"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2026,2,9]],"date-time":"2026-02-09T00:00:00Z","timestamp":1770595200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,2,9]],"date-time":"2026-02-09T00:00:00Z","timestamp":1770595200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Lang Resources &amp; Evaluation"],"published-print":{"date-parts":[[2026,3]]},"abstract":"<jats:title>Abstract<\/jats:title>\n                  <jats:p>\n                    This paper introduces\n                    <jats:sc>JurisTCU<\/jats:sc>\n                    , a Brazilian Portuguese dataset for legal information retrieval (LIR). The dataset is freely available (\n                    <jats:ext-link xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" xlink:href=\"https:\/\/huggingface.co\/datasets\/LeandroRibeiro\/JurisTCU\" ext-link-type=\"uri\">https:\/\/huggingface.co\/datasets\/LeandroRibeiro\/JurisTCU<\/jats:ext-link>\n                    ) and consists of 16,045 jurisprudential documents from the Brazilian Federal Court of Accounts, along with 150 queries annotated with relevance judgments. It addresses the scarcity of Portuguese-language LIR datasets with query relevance annotations. The queries are organized into three groups: real user keyword-based queries, synthetic keyword-based queries, and synthetic question-based queries. Relevance judgments were produced through a hybrid approach combining LLM-based scoring with expert domain validation. We used\n                    <jats:sc>JurisTCU<\/jats:sc>\n                    in 14 experiments using lexical search (document expansion methods) and semantic search (BERT-based and OpenAI embeddings). We show that the document expansion methods significantly improve the performance of standard BM25 search on this dataset, with improvements exceeding 45% in P@10, R@10, and nDCG@10 metrics when evaluating short keyword-based queries. Among the embedding models, the OpenAI models produced the best results, with improvements of approximately 70% in P@10, R@10, and nDCG@10 metrics for short keyword-based queries, suggesting that these dense embeddings capture semantic relationships in this domain, surpassing the reliance on lexical terms. Besides offering a dataset for the Portuguese-language IR research community, suitable for evaluating search systems, the results also contribute to enhancing a search system highly relevant to Brazilian citizens.\n                  <\/jats:p>","DOI":"10.1007\/s10579-025-09881-w","type":"journal-article","created":{"date-parts":[[2026,2,9]],"date-time":"2026-02-09T11:38:23Z","timestamp":1770637103000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["JurisTCU: a Brazilian Portuguese information retrieval dataset with query relevance judgments"],"prefix":"10.1007","volume":"60","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4114-2334","authenticated-orcid":false,"given":"Leandro Car\u00edsio","family":"Fernandes","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-3715-9927","authenticated-orcid":false,"given":"Leandro dos Santos","family":"Ribeiro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-8633-1385","authenticated-orcid":false,"given":"Marcos Vin\u00edcius Borela","family":"de Castro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6506-2366","authenticated-orcid":false,"given":"Leonardo Augusto","family":"da Silva Pacheco","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-4553-4783","authenticated-orcid":false,"given":"Edans Fl\u00e1vius","family":"de Oliveira Sandes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,2,9]]},"reference":[{"key":"9881_CR1","doi-asserted-by":"publisher","unstructured":"Abonizio, H., Almeida, T.S., Laitz, T., Junior, R. M., Bon\u00e1s, G. K., Nogueira, R., & Pires, R. (2025). Sabi\u00e1-3 Technical Report . https:\/\/doi.org\/10.48550\/arXiv.2410.12049","DOI":"10.48550\/arXiv.2410.12049"},{"key":"9881_CR2","unstructured":"Baeza-Yates, R., & Ribeiro-Neto, B. Modern Information Retrieval (Addison Wesley, 1999)"},{"key":"9881_CR3","doi-asserted-by":"publisher","unstructured":"Bajaj, P., Campos, D., Craswell, N., Deng, L., Gao, J., Liu, X., Majumder, R., McNamara, A., Mitra, B., Nguyen, T., Rosenberg, M., Song, X., Stoica, A., Tiwary, S., & Wang, T. (2018). MS MARCO: A Human Generated MAchine Reading COmprehension Dataset https:\/\/doi.org\/10.48550\/arXiv.1611.09268","DOI":"10.48550\/arXiv.1611.09268"},{"key":"9881_CR4","doi-asserted-by":"publisher","unstructured":"Bommarito II, M., & Katz, D. M. (2022). GPT Takes the Bar Exam. https:\/\/doi.org\/10.48550\/arXiv.2212.14402","DOI":"10.48550\/arXiv.2212.14402"},{"key":"9881_CR5","doi-asserted-by":"publisher","unstructured":"Bonifacio, L., Jeronymo, V., Abonizio, H. Q., Campiotti, I., Fadaee, M., Lotufo, R., & Nogueira, R.(2022). mMARCO: A Multilingual Version of the MS MARCO Passage Ranking Dataset . https:\/\/doi.org\/10.48550\/arXiv.2108.13897","DOI":"10.48550\/arXiv.2108.13897"},{"key":"9881_CR6","doi-asserted-by":"publisher","unstructured":"Bueno, M., de Oliveira, E. S., Nogueira, R., Lotufo, R. A., & Pereira, J. A. (2024). Quati: A Brazilian Portuguese Information Retrieval Dataset from Native Speakers . https:\/\/doi.org\/10.48550\/arXiv.2404.06976","DOI":"10.48550\/arXiv.2404.06976"},{"key":"9881_CR7","doi-asserted-by":"publisher","unstructured":"Canaverde, B., Pires, T. P., Ribeiro, L. M., & Martins, A. F. T. (2025). LegalBench.PT: A Benchmark for Portuguese Law. https:\/\/doi.org\/10.48550\/arXiv.2502.16357","DOI":"10.48550\/arXiv.2502.16357"},{"key":"9881_CR8","unstructured":"Caselaw Access Project (2024). https:\/\/case.law\/. Accessed: January 6, 2025"},{"key":"9881_CR9","doi-asserted-by":"publisher","unstructured":"Chalkidis, I., Jana, A., Hartung, D., Bommarito, M., Androutsopoulos, I., Katz, D. M., & Aletras, N. (2022). LexGLUE: A Benchmark Dataset for Legal Language Understanding in English. https:\/\/doi.org\/10.48550\/arXiv.2110.00976","DOI":"10.48550\/arXiv.2110.00976"},{"key":"9881_CR10","doi-asserted-by":"publisher","unstructured":"Clarke, C. L. A., & Dietz, L. (2024). LLM-based relevance assessment still can\u2019t replace human relevance assessment . https:\/\/doi.org\/10.48550\/arXiv.2412.17156","DOI":"10.48550\/arXiv.2412.17156"},{"key":"9881_CR11","doi-asserted-by":"publisher","unstructured":"Colombo, P., Pires, T. P., Boudiaf, M., Culver, D., Melo, R., Corro, C., Martins, A. F. T., Esposito, F., Raposo, V. L., Morgado, S., & Desa, M. (2024). SaulLM-7B: A pioneering Large Language Model for Law. https:\/\/doi.org\/10.48550\/arXiv.2403.03883","DOI":"10.48550\/arXiv.2403.03883"},{"key":"9881_CR12","doi-asserted-by":"publisher","unstructured":"Colombo, P., Pires, T., Boudiaf, M., Melo, R., Culver, D., Morgado, S., Malaboeuf, E., Hautreux, G., Charpentier, J., & Desa, M. (2024). SaulLM-54B & SaulLM-141B: Scaling Up Domain Adaptation for the Legal Domain. https:\/\/doi.org\/10.48550\/arXiv.2407.19584","DOI":"10.48550\/arXiv.2407.19584"},{"key":"9881_CR13","doi-asserted-by":"publisher","unstructured":"Cormack, G. V., Palmer, C. R., & Clarke, C. L. A. (1998). Efficient construction of large test collections. Proceedings of the 21st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval . https:\/\/doi.org\/10.1145\/290941.291009","DOI":"10.1145\/290941.291009"},{"key":"9881_CR14","doi-asserted-by":"publisher","unstructured":"dos Santos, J. A., Souza, E., Filho, C. J. A. B., Albuquerque, H. O., Vit\u00f3rio, D., de\u00a0Lucena, D. C. G., Silva, N., & de\u00a0Carvalho, A.HIRS: A Hybrid Information Retrieval System for Legislative Documents, in Progress in Artificial Intelligence, ed. by M.F. Santos, J.\u00a0Machado, P.\u00a0Novais, P.\u00a0Cortez, P.M. Moreira (Springer Nature Switzerland, Cham, 2025), pp. 320\u2013331. https:\/\/doi.org\/10.1007\/978-3-031-73497-7_26","DOI":"10.1007\/978-3-031-73497-7_26"},{"key":"9881_CR15","doi-asserted-by":"publisher","unstructured":"Fei, Z., Shen, X., Zhu, D., Zhou, F., Han, Z., Zhang, S., Chen, K., Shen, Z., & Ge, J. (2023). LawBench: Benchmarking Legal Knowledge of Large Language Models. https:\/\/doi.org\/10.48550\/arXiv.2309.16289","DOI":"10.48550\/arXiv.2309.16289"},{"key":"9881_CR16","doi-asserted-by":"publisher","unstructured":"Formal, T., Piwowarski, B., & Clinchant, S. (2021). SPLADE: Sparse Lexical and Expansion Model for First Stage Ranking . https:\/\/doi.org\/10.48550\/arXiv.2107.05720","DOI":"10.48550\/arXiv.2107.05720"},{"key":"9881_CR17","doi-asserted-by":"publisher","unstructured":"Freitas, P. M., & Gomes, L. M. Does ChatGPT Pass the Brazilian Bar Exam?, in Progress in Artificial Intelligence, ed. by Moniz, N., Vale, Z., Cascalho, J., Silva, C., Sebasti\u00e3o, R. (Springer Nature Switzerland, Cham, 2023), pp. 131\u2013141. https:\/\/doi.org\/10.1007\/978-3-031-49011-8_11","DOI":"10.1007\/978-3-031-49011-8_11"},{"issue":"4","key":"9881_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3486250","volume":"40","author":"J Guo","year":"2022","unstructured":"Guo, J., Cai, Y., Fan, Y., Sun, F., Zhang, R., & Cheng, X. (2022). Semantic models for the first-stage retrieval: A comprehensive review. ACM Transactions on Information Systems, 40(4), 1\u201342. https:\/\/doi.org\/10.1145\/3486250","journal-title":"ACM Transactions on Information Systems"},{"key":"9881_CR19","doi-asserted-by":"publisher","unstructured":"Hou, A. B., Weller, O., Qin, G., Yang, E., Lawrie, D., Holzenberger, N., Blair-Stanek, A., & Durme, B. V. (2024). CLERC: A Dataset for Legal Case Retrieval and Retrieval-Augmented Analysis Generation. https:\/\/doi.org\/10.48550\/arXiv.2406.17186","DOI":"10.48550\/arXiv.2406.17186"},{"key":"9881_CR20","doi-asserted-by":"publisher","unstructured":"Junior, D. d. S., Corval, P. R. d. S., Paes, A., & de Oliveira, D. (2023). Datasets for Portuguese Legal Semantic Textual Similarity: Comparing weak supervision and an annotation process approaches . https:\/\/doi.org\/10.48550\/arXiv.2306.00007","DOI":"10.48550\/arXiv.2306.00007"},{"key":"9881_CR21","doi-asserted-by":"publisher","unstructured":"Karpukhin, V., Oguz, B., Min, S., Lewis, P., Wu, L., Edunov, S., Chen, D., & Yih, W. t. (2020) \"Dense Passage Retrieval for Open-Domain Question Answering\", in Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), ed. by Webber, B., Cohn, T., He, Y., Liu, Y. pp. 6769\u20136781. https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.550","DOI":"10.18653\/v1\/2020.emnlp-main.550"},{"key":"9881_CR22","unstructured":"Labs, E.S. (2023). Evaluating Search Relevance - Part 1. https:\/\/www.elastic.co\/search-labs\/blog\/evaluating-search-relevance-part-1. Accessed: 2025-02-12"},{"key":"9881_CR23","doi-asserted-by":"publisher","unstructured":"Li, H., Chen, Y., Hu, Y., Ai, Q., Chen, J., Yang, X., Yang, J., Wu, Y., Liu, Z., & Liu, Y. (2025). LexRAG: Benchmarking Retrieval-Augmented Generation in Multi-Turn Legal Consultation Conversation. https:\/\/doi.org\/10.48550\/arXiv.2502.20640","DOI":"10.48550\/arXiv.2502.20640"},{"key":"9881_CR24","doi-asserted-by":"publisher","unstructured":"Lin, J., Nogueira, R., & Yates, A. (2021). Pretrained Transformers for Text Ranking: BERT and Beyond. https:\/\/doi.org\/10.48550\/arXiv.2010.06467","DOI":"10.48550\/arXiv.2010.06467"},{"key":"9881_CR25","doi-asserted-by":"crossref","unstructured":"Manning, C. D., Raghavan, P., & Schutze, H. Introduction to information retrieval (Cambridge University Press, 2008)","DOI":"10.1017\/CBO9780511809071"},{"key":"9881_CR26","doi-asserted-by":"publisher","unstructured":"Melo, R., Santos, P. A., & Dias, J. A Semantic Search System for the Supremo Tribunal de Justi\u00e7a, in Progress in Artificial Intelligence, ed. by Moniz, N., Vale, Z., Cascalho, J., Silva, C., Sebasti\u00e3o, R. (Springer Nature Switzerland, Cham, 2023), pp. 142\u2013154. https:\/\/doi.org\/10.1007\/978-3-031-49011-8_12","DOI":"10.1007\/978-3-031-49011-8_12"},{"key":"9881_CR27","doi-asserted-by":"publisher","unstructured":"Niklaus, J., Matoshi, V., Rani, P., Galassi, A., St\u00fcrmer, M., & Chalkidis, I. LEXTREME: A Multi-Lingual and Multi-Task Benchmark for the Legal Domain, in Findings of the Association for Computational Linguistics: EMNLP 2023 (Association for Computational Linguistics, 2023), p. 3016\u20133054. https:\/\/doi.org\/10.18653\/v1\/2023.findings-emnlp.200","DOI":"10.18653\/v1\/2023.findings-emnlp.200"},{"key":"9881_CR28","unstructured":"Nogueira, R., & Lin, J. (2019). From doc2query to docTTTTTquery"},{"key":"9881_CR29","doi-asserted-by":"publisher","unstructured":"Nogueira, R., Yang, W., Lin, J., & Cho, K. (2019). Document Expansion by Query Prediction . https:\/\/doi.org\/10.48550\/arXiv.1904.08375","DOI":"10.48550\/arXiv.1904.08375"},{"key":"9881_CR30","doi-asserted-by":"publisher","unstructured":"Overwijk, A., Xiong, C., & Callan, J. ClueWeb22: 10 Billion Web Documents with Rich Information, in Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval (Association for Computing Machinery, New York, NY, USA, 2022), SIGIR \u201922, p. 3360\u20133362. https:\/\/doi.org\/10.1145\/3477495.3536321","DOI":"10.1145\/3477495.3536321"},{"key":"9881_CR31","doi-asserted-by":"publisher","unstructured":"Pipitone, N., & Alami, G. H. (2024). LegalBench-RAG: A Benchmark for Retrieval-Augmented Generation in the Legal Domain. https:\/\/doi.org\/10.48550\/arXiv.2408.10343","DOI":"10.48550\/arXiv.2408.10343"},{"key":"9881_CR32","doi-asserted-by":"crossref","unstructured":"Reimers, N., & Gurevych, I. Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks, in Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing (Association for Computational Linguistics, 2019). arXiv: 1908.10084","DOI":"10.18653\/v1\/D19-1410"},{"key":"9881_CR33","doi-asserted-by":"publisher","unstructured":"Souza, F., Nogueira, R., & Lotufo, R. (2020). BERTimbau: pretrained BERT models for Brazilian Portuguese, in 9th Brazilian Conference on Intelligent Systems, BRACIS, Rio Grande do Sul, Brazil, October 20-23 (to appear) . https:\/\/doi.org\/10.1007\/978-3-030-61377-8_28","DOI":"10.1007\/978-3-030-61377-8_28"},{"key":"9881_CR34","doi-asserted-by":"publisher","unstructured":"Takehi, R., Voorhees, E. M., & Sakai, T. (2024). LLM-Assisted Relevance Assessments: When Should We Ask LLMs for Help? . https:\/\/doi.org\/10.48550\/arXiv.2411.06877","DOI":"10.48550\/arXiv.2411.06877"},{"key":"9881_CR35","doi-asserted-by":"publisher","unstructured":"Thomas, P., Spielman, S., Craswell, N., & Mitra, B. (2024). Large language models can accurately predict searcher preferences. https:\/\/doi.org\/10.48550\/arXiv.2309.10621","DOI":"10.48550\/arXiv.2309.10621"},{"issue":"1","key":"9881_CR36","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1007\/s10506-017-9195-8","volume":"25","author":"M van Opijnen","year":"2017","unstructured":"van Opijnen, M., & Santos, C. (2017). On the concept of relevance in legal information retrieval. Artificial Intelligence and Law, 25(1), 65\u201387. https:\/\/doi.org\/10.1007\/s10506-017-9195-8","journal-title":"Artificial Intelligence and Law"},{"key":"9881_CR37","doi-asserted-by":"publisher","unstructured":"Vit\u00f3rio, D., Souza, E., Martins, L., da Silva, N. F. F., de Leon, A. C. P., de Carvalho, A. L. I., de Oliveira, F. E., & Andrade,. (2024). Building a relevance feedback corpus for legal information retrieval in the real-case scenario of the Brazilian Chamber of Deputies. Language Resources and Evaluation. https:\/\/doi.org\/10.1007\/s10579-024-09767-3","DOI":"10.1007\/s10579-024-09767-3"},{"issue":"7","key":"9881_CR38","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3648471","volume":"56","author":"J Wang","year":"2024","unstructured":"Wang, J., Huang, J. X., Tu, X., Wang, J., Huang, A. J., Laskar, M. T. R., & Bhuiyan, A. (2024). Utilizing BERT for information retrieval: Survey, applications, resources, and challenges. ACM Computing Surveys, 56(7), 1\u201333. https:\/\/doi.org\/10.1145\/3648471","journal-title":"ACM Computing Surveys"},{"key":"9881_CR39","doi-asserted-by":"publisher","unstructured":"Zhan, J., Mao, J., Liu, Y., Guo, J., Zhang, M., & Ma, S. (2021). \"optimizing dense retrieval model training with hard negatives\" . https:\/\/doi.org\/10.48550\/arXiv.2104.08051","DOI":"10.48550\/arXiv.2104.08051"}],"container-title":["Language Resources and Evaluation"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-025-09881-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10579-025-09881-w","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10579-025-09881-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,16]],"date-time":"2026-06-16T17:56:28Z","timestamp":1781632588000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10579-025-09881-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,2,9]]},"references-count":39,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2026,3]]}},"alternative-id":["9881"],"URL":"https:\/\/doi.org\/10.1007\/s10579-025-09881-w","relation":{},"ISSN":["1574-020X","1574-0218"],"issn-type":[{"value":"1574-020X","type":"print"},{"value":"1574-0218","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,2,9]]},"assertion":[{"value":"11 March 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"20 November 2025","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 February 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"23"}}