{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:12:44Z","timestamp":1755839564894,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591893","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:23Z","timestamp":1689726143000},"page":"2880-2889","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4,"title":["HC3: A Suite of Test Collections for CLIR Evaluation over Informal Text"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7347-7086","authenticated-orcid":false,"given":"Dawn","family":"Lawrie","sequence":"first","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3866-3013","authenticated-orcid":false,"given":"James","family":"Mayfield","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1696-0407","authenticated-orcid":false,"given":"Douglas W.","family":"Oard","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0051-1535","authenticated-orcid":false,"given":"Eugene","family":"Yang","sequence":"additional","affiliation":[{"name":"Johns Hopkins University, Baltimore, MD, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2283-7672","authenticated-orcid":false,"given":"Suraj","family":"Nair","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, France"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6328-7131","authenticated-orcid":false,"given":"Petra","family":"Galu\u0161\u010d\u00e1kov\u00e1","sequence":"additional","affiliation":[{"name":"Universit\u00e9 Grenoble Alpes, Grenoble, France"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"CLEF 2001-Overview of Results. In Workshop of the Cross-Language Evaluation Forum for European Languages. Springer, 9--26","author":"Braschler Martin","year":"2001","unstructured":"Martin Braschler. 2001. CLEF 2001-Overview of Results. In Workshop of the Cross-Language Evaluation Forum for European Languages. Springer, 9--26."},{"key":"e_1_3_2_2_2_1","volume-title":"CLEF 2003-Overview of results. In Workshop of the Cross-Language Evaluation Forum for European Languages. Springer, 44--63","author":"Braschler Martin","year":"2003","unstructured":"Martin Braschler. 2003. CLEF 2003-Overview of results. In Workshop of the Cross-Language Evaluation Forum for European Languages. Springer, 44--63."},{"key":"e_1_3_2_2_3_1","volume-title":"Lin (Eds.)","volume":"33","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. In Advances in Neural Information Processing Systems, H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (Eds.), Vol. 33. Curran Associates, Inc., 1877--1901. https:\/\/proceedings.neurips.cc\/paper\/2020\/file\/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf"},{"key":"e_1_3_2_2_4_1","volume-title":"Bias and the limits of pooling for large collections. Information retrieval","author":"Buckley Chris","year":"2007","unstructured":"Chris Buckley, Darrin Dimmick, Ian Soboroff, and Ellen Voorhees. 2007. Bias and the limits of pooling for large collections. Information retrieval, Vol. 10 (2007), 491--508."},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331354"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331354"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99739-7_33"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/860435.860497"},{"key":"e_1_3_2_2_10_1","volume-title":"BERT: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","author":"Devlin Jacob","year":"2018","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018. BERT: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805 (2018)."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15754-7_2"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914728"},{"volume-title":"TREC: Experiment and Evaluation in Information Retrieval","author":"Harman Donna K.","key":"e_1_3_2_2_13_1","unstructured":"Donna K. Harman. 2005. The TREC Test Collections. In TREC: Experiment and Evaluation in Information Retrieval (Digital Libraries and Electronic Publishing), Ellen M Voorhees and Donna K Harman (Eds.). The MIT Press."},{"key":"e_1_3_2_2_14_1","volume-title":"Towards unsupervised dense information retrieval with contrastive learning. arXiv preprint arXiv:2112.09118","author":"Izacard Gautier","year":"2021","unstructured":"Gautier Izacard, Mathilde Caron, Lucas Hosseini, Sebastian Riedel, Piotr Bojanowski, Armand Joulin, and Edouard Grave. 2021. Towards unsupervised dense information retrieval with contrastive learning. arXiv preprint arXiv:2112.09118 (2021)."},{"key":"e_1_3_2_2_15_1","unstructured":"Noriko Kando Kazuko Kuriyama and Masaharu Yoshioka. 2001. Overview of Japanese and English Imformation Retrieval Tasks (JEIR) at the Second NTCIR Workshop.. In NTCIR."},{"key":"e_1_3_2_2_16_1","unstructured":"Kazuaki Kishida and Makoto P Kato. 2016. Overview of NTCIR-12.. In NTCIR."},{"key":"e_1_3_2_2_17_1","volume-title":"Overview of the TREC 2022 NeuCLIR Track. In The Thirty-first Text REtrieval Conference (TREC 2022) Proceedings.","author":"Lawrie Dawn","year":"2022","unstructured":"Dawn Lawrie, Sean MacAvaney, James Mayfield, Paul McNamee, Douglas W. Oard, Luca Soldanini, and Eugene Yang. 2022a. Overview of the TREC 2022 NeuCLIR Track. In The Thirty-first Text REtrieval Conference (TREC 2022) Proceedings."},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_24"},{"key":"e_1_3_2_2_19_1","volume-title":"Threading electronic mail: A preliminary study. Information processing & management","author":"Lewis David D","year":"1997","unstructured":"David D Lewis and Kimberly A Knowles. 1997. Threading electronic mail: A preliminary study. Information processing & management, Vol. 33, 2 (1997), 209--217."},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"crossref","unstructured":"Jimmy Lin Miles Efron Yulu Wang and Garrick Sherman. 2014. Overview of the TREC-2014 microblog track. In TREC.","DOI":"10.6028\/NIST.SP.500-308.microblog-overview"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.5555\/3103315"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/1842890.1842899"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1838745.1838747"},{"key":"e_1_3_2_2_24_1","volume-title":"Proceedings of the Thirteenth Language Resources and Evaluation Conference. 910--918","author":"McNamee Paul","year":"2022","unstructured":"Paul McNamee and Kevin Duh. 2022. The multilingual microblog translation corpus: Improving and evaluating translation of user-generated text. In Proceedings of the Thirteenth Language Resources and Evaluation Conference. 910--918."},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_26"},{"key":"e_1_3_2_2_27_1","volume-title":"Proceedings of Design of Experimental Search & Information REtrieval Systems (DESIRES).","author":"Nair Suraj","year":"2022","unstructured":"Suraj Nair, Eugene Yang, Dawn Lawrie, James Mayfield, and Douglas W Oard. 2022b. Learning a Sparse Representation Model for Neural CLIR. In Proceedings of Design of Experimental Search & Information REtrieval Systems (DESIRES)."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3536321"},{"volume-title":"Multilingual information retrieval: From research to practice","author":"Peters Carol","key":"e_1_3_2_2_29_1","unstructured":"Carol Peters, Martin Braschler, and Paul Clough. 2012. Multilingual information retrieval: From research to practice. Springer."},{"key":"e_1_3_2_2_30_1","volume-title":"Overview of the TREC 2016 clinical decision support track.. In TREC.","author":"Roberts Kirk","year":"2016","unstructured":"Kirk Roberts, Dina Demner-Fushman, Ellen M Voorhees, and William R Hersh. 2016. Overview of the TREC 2016 clinical decision support track.. In TREC."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557325"},{"key":"e_1_3_2_2_33_1","unstructured":"Lifeng Shang Tetsuya Sakai Zhengdong Lu Hang Li Ryuichiro Higashinaka and Yusuke Miyao. 2012. Overview of the NTCIR-12 Short Text Conversation Task. (2012)."},{"key":"e_1_3_2_2_34_1","volume-title":"Variations in relevance judgments and the measurement of retrieval effectiveness. Information processing & management","author":"Voorhees Ellen M","year":"2000","unstructured":"Ellen M Voorhees. 2000. Variations in relevance judgments and the measurement of retrieval effectiveness. Information processing & management, Vol. 36, 5 (2000), 697--716."},{"key":"e_1_3_2_2_35_1","volume-title":"The evolution of Cranfield. Information Retrieval Evaluation in a Changing World: Lessons Learned from 20 Years of CLEF","author":"Voorhees Ellen M","year":"2019","unstructured":"Ellen M Voorhees. 2019. The evolution of Cranfield. Information Retrieval Evaluation in a Changing World: Lessons Learned from 20 Years of CLEF (2019), 45--69."},{"key":"e_1_3_2_2_36_1","volume-title":"Overview of TREC 2003","author":"Voorhees Ellen M","year":"2003","unstructured":"Ellen M Voorhees and Donna Harman. 2003. Overview of TREC 2003. NIST special publication SP 251 (2003), 1--16."},{"key":"e_1_3_2_2_37_1","unstructured":"William Webber Alistair Moffat and Justin Zobel. 2010. The Effect of Pooling and Evaluation Depth on Metric Stability.. In EVIA@ NTCIR. 7--15."},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.3115\/1117794.1117806"},{"key":"e_1_3_2_2_39_1","volume-title":"Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence.","author":"Xue Zhenzhen","year":"2011","unstructured":"Zhenzhen Xue, Dawei Yin, and Brian D Davison. 2011. Normalizing microtext. In Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531886"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2108.08787"},{"key":"e_1_3_2_2_42_1","volume-title":"Making a MIRACL: Multilingual Information Retrieval Across a Continuum of Languages. arXiv preprint arXiv:2210.09984","author":"Zhang Xinyu","year":"2022","unstructured":"Xinyu Zhang, Nandan Thakur, Odunayo Ogundepo, Ehsan Kamalloo, David Alfonso-Hermelo, Xiaoguang Li, Qun Liu, Mehdi Rezagholizadeh, and Jimmy Lin. 2022. Making a MIRACL: Multilingual Information Retrieval Across a Continuum of Languages. arXiv preprint arXiv:2210.09984 (2022)."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Taipei Taiwan","acronym":"SIGIR '23"},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591893","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591893","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:38:07Z","timestamp":1750178287000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591893"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":43,"alternative-id":["10.1145\/3539618.3591893","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591893","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}