{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:10:15Z","timestamp":1755839415752,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,10,17]],"date-time":"2022-10-17T00:00:00Z","timestamp":1665964800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"EU Horizon 2020 ITN\/ETN","award":["H2020-EU.1.3.1. ID: 860721"],"award-info":[{"award-number":["H2020-EU.1.3.1. ID: 860721"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,10,17]]},"DOI":"10.1145\/3511808.3557714","type":"proceedings-article","created":{"date-parts":[[2022,10,16]],"date-time":"2022-10-16T01:29:57Z","timestamp":1665883797000},"page":"3801-3805","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["TripJudge"],"prefix":"10.1145","author":[{"given":"Sophia","family":"Althammer","sequence":"first","affiliation":[{"name":"Vienna University of Technology, Vienna, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sebastian","family":"Hofst\u00e4tter","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Vienna, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Suzan","family":"Verberne","sequence":"additional","affiliation":[{"name":"Leiden University, Leiden, Netherlands"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[{"name":"Vienna University of Technology, Vienna, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,10,17]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2012.01.004"},{"key":"e_1_3_2_1_2_1","volume-title":"43rd European Conference on IR Research, ECIR 2021.","author":"Althammer Sophia","year":"2021","unstructured":"Sophia Althammer , Sebastian Hofst\u00e4tter , and Allan Hanbury . 2021 . Cross-domain Retrieval in the Legal and Patent Domains: a Reproducibility Study. In Advances in Information Retrieval , 43rd European Conference on IR Research, ECIR 2021. Sophia Althammer, Sebastian Hofst\u00e4tter, and Allan Hanbury. 2021. Cross-domain Retrieval in the Legal and Patent Domains: a Reproducibility Study. In Advances in Information Retrieval, 43rd European Conference on IR Research, ECIR 2021."},{"key":"e_1_3_2_1_3_1","volume-title":"44rd European Conference on IR Research, ECIR 2022.","author":"Althammer Sophia","year":"2022","unstructured":"Sophia Althammer , Sebastian Hofst\u00e4tter , Mete Sertkan , Suzan Verberne , and Allan Hanbury . 2022 . Paragraph Aggregation Retrieval Model (PARM) for Dense Document-to-Document Retrieval. In Advances in Information Retrieval , 44rd European Conference on IR Research, ECIR 2022. Sophia Althammer, Sebastian Hofst\u00e4tter, Mete Sertkan, Suzan Verberne, and Allan Hanbury. 2022. Paragraph Aggregation Retrieval Model (PARM) for Dense Document-to-Document Retrieval. In Advances in Information Retrieval, 44rd European Conference on IR Research, ECIR 2022."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390447"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10286"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-007-9032-x"},{"volume-title":"Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Buckley Chris","key":"e_1_3_2_1_7_1","unstructured":"Chris Buckley and Ellen M. Voorhees . 2004. Retrieval Evaluation with Incomplete Information . In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval ( Sheffield, United Kingdom) (SIGIR '04). Association for Computing Machinery, New York, NY, USA, 25--32. https:\/\/doi.org\/10.1145\/1008992.1009000 10.1145\/1008992.1009000 Chris Buckley and Ellen M. Voorhees. 2004. Retrieval Evaluation with Incomplete Information. In Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (Sheffield, United Kingdom) (SIGIR '04). Association for Computing Machinery, New York, NY, USA, 25--32. https:\/\/doi.org\/10.1145\/1008992.1009000"},{"volume-title":"Overview of the TREC 2012 Web Track. In Text Retrieval Conference (TREC). http:\/\/trec.nist.gov\/pubs\/trec21\/papers\/WEB12","author":"Clarke Charles","key":"e_1_3_2_1_8_1","unstructured":"Charles Clarke , Nick Craswell , and Ellen M. Voorhees . 2012 . Overview of the TREC 2012 Web Track. In Text Retrieval Conference (TREC). http:\/\/trec.nist.gov\/pubs\/trec21\/papers\/WEB12 .overview.pdf Charles Clarke, Nick Craswell, and Ellen M. Voorhees. 2012. Overview of the TREC 2012 Web Track. In Text Retrieval Conference (TREC). http:\/\/trec.nist.gov\/pubs\/trec21\/papers\/WEB12.overview.pdf"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/122860.122861"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"volume-title":"TREC Deep Learning Track: Reusable Test Collections in the Large Data Regime","author":"Craswell Nick","key":"e_1_3_2_1_11_1","unstructured":"Nick Craswell , Bhaskar Mitra , Emine Yilmaz , Daniel Campos , Ellen M. Voorhees , and Ian Soboroff . 2021. TREC Deep Learning Track: Reusable Test Collections in the Large Data Regime . Association for Computing Machinery , New York, NY, USA , 2369--2375. https:\/\/doi.org\/10.1145\/3404835.3463249 10.1145\/3404835.3463249 Nick Craswell, Bhaskar Mitra, Emine Yilmaz, Daniel Campos, Ellen M. Voorhees, and Ian Soboroff. 2021. TREC Deep Learning Track: Reusable Test Collections in the Large Data Regime. Association for Computing Machinery, New York, NY, USA, 2369--2375. https:\/\/doi.org\/10.1145\/3404835.3463249"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866723"},{"key":"e_1_3_2_1_13_1","volume-title":"44th European Conference on IR Research, ECIR 2022.","author":"Sebastian","year":"2022","unstructured":"Sebastian Hofst\"atter, Sophia Althammer , Mete Sertkan , and Allan Hanbury . 2022 a. Establishing Strong Baselines for TripClick Health Retrieval. In Advances in Information Retrieval , 44th European Conference on IR Research, ECIR 2022. Sebastian Hofst\"atter, Sophia Althammer, Mete Sertkan, and Allan Hanbury. 2022a. Establishing Strong Baselines for TripClick Health Retrieval. In Advances in Information Retrieval, 44th European Conference on IR Research, ECIR 2022."},{"key":"e_1_3_2_1_14_1","unstructured":"Sebastian Hofst\"atter Omar Khattab Sophia Althammer Mete Sertkan and Allan Hanbury. 2022b. Introducing Neural Bag of Whole-Words with ColBERTer: Contextualized Late Interactions using Enhanced Reduction. ArXiv. https:\/\/doi.org\/10.48550\/ARXIV.2203.13088    10.48550\/ARXIV.2203.13088\nSebastian Hofst\"atter Omar Khattab Sophia Althammer Mete Sertkan and Allan Hanbury. 2022b. Introducing Neural Bag of Whole-Words with ColBERTer: Contextualized Late Interactions using Enhanced Reduction. ArXiv. https:\/\/doi.org\/10.48550\/ARXIV.2203.13088"},{"key":"e_1_3_2_1_15_1","volume-title":"Proc. of SIGIR.","author":"Sebastian","year":"2021","unstructured":"Sebastian Hofst\"atter, Sheng-Chieh Lin , Jheng-Hong Yang , Jimmy Lin , and Allan Hanbury . 2021 . Efficiently Teaching an Effective Dense Retriever with Balanced Topic Aware Sampling . In Proc. of SIGIR. Sebastian Hofst\"atter, Sheng-Chieh Lin, Jheng-Hong Yang, Jimmy Lin, and Allan Hanbury. 2021. Efficiently Teaching an Effective Dense Retriever with Balanced Topic Aware Sampling. In Proc. of SIGIR."},{"key":"e_1_3_2_1_16_1","volume-title":"Advances in Information Retrieval: 42nd European Conference on IR Research, ECIR 2020, Lisbon, Portugal, April 14--17, 2020, Proceedings, Part II","author":"Sebastian","year":"2020","unstructured":"Sebastian Hofst\"atter, Markus Zlabinger , and Allan Hanbury . 2020 . Neural-IR-Explorer: A Content-Focused Tool to Explore Neural Re-Ranking Results . In Advances in Information Retrieval: 42nd European Conference on IR Research, ECIR 2020, Lisbon, Portugal, April 14--17, 2020, Proceedings, Part II ( Lisbon, Portugal). Springer-Verlag, Berlin, Heidelberg, 459--464. https:\/\/doi.org\/10.1007\/978--3-030--45442--5_58 10.1007\/978--3-030--45442--5_58 Sebastian Hofst\"atter, Markus Zlabinger, and Allan Hanbury. 2020. Neural-IR-Explorer: A Content-Focused Tool to Explore Neural Re-Ranking Results. In Advances in Information Retrieval: 42nd European Conference on IR Research, ECIR 2020, Lisbon, Portugal, April 14--17, 2020, Proceedings, Part II (Lisbon, Portugal). Springer-Verlag, Berlin, Heidelberg, 459--464. https:\/\/doi.org\/10.1007\/978--3-030--45442--5_58"},{"key":"e_1_3_2_1_17_1","volume-title":"Proc. of CIKM.","author":"Sebastian","year":"2020","unstructured":"Sebastian Hofst\"atter, Markus Zlabinger , Mete Sertkan , Michael Schr\u00f6der , and Allan Hanbury . 2020 . Fine-Grained Relevance Annotations for Multi-Task Document Ranking and Question Answering . In Proc. of CIKM. Sebastian Hofst\"atter, Markus Zlabinger, Mete Sertkan, Michael Schr\u00f6der, and Allan Hanbury. 2020. Fine-Grained Relevance Annotations for Multi-Task Document Ranking and Question Answering. In Proc. of CIKM."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076063"},{"key":"e_1_3_2_1_19_1","volume-title":"Comparative analysis of clicks and judgments for IR evaluation. Journal of Visual Communication and Image Representation - JVCIR (01","author":"Kamps Jaap","year":"2009","unstructured":"Jaap Kamps , Marijn Koolen , and Andrew Trotman . 2009. Comparative analysis of clicks and judgments for IR evaluation. Journal of Visual Communication and Image Representation - JVCIR (01 2009 ), 80--87. https:\/\/doi.org\/10.1145\/1507509.1507522 10.1145\/1507509.1507522 Jaap Kamps, Marijn Koolen, and Andrew Trotman. 2009. Comparative analysis of clicks and judgments for IR evaluation. Journal of Visual Communication and Image Representation - JVCIR (01 2009), 80--87. https:\/\/doi.org\/10.1145\/1507509.1507522"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Joao Palotti Guido Zuccon Johannes Bernhardt Allan Hanbury and Lorraine Goeuriot. 2016. Assessors Agreement: A Case Study Across Assessor Type Payment Levels Query Variations and Relevance Dimensions. In Experimental IR Meets Multilinguality Multimodality and Interaction Norbert Fuhr Paulo Quaresma Teresa Goncc alves Birger Larsen Krisztian Balog Craig Macdonald Linda Cappellato and Nicola Ferro (Eds.). Springer International Publishing Cham 40--53.  Joao Palotti Guido Zuccon Johannes Bernhardt Allan Hanbury and Lorraine Goeuriot. 2016. Assessors Agreement: A Case Study Across Assessor Type Payment Levels Query Variations and Relevance Dimensions. In Experimental IR Meets Multilinguality Multimodality and Interaction Norbert Fuhr Paulo Quaresma Teresa Goncc alves Birger Larsen Krisztian Balog Craig Macdonald Linda Cappellato and Nicola Ferro (Eds.). Springer International Publishing Cham 40--53.","DOI":"10.1007\/978-3-319-44564-9_4"},{"volume-title":"TripClick: The Log Files of a Large Health Web Search Engine","author":"Rekabsaz Navid","key":"e_1_3_2_1_22_1","unstructured":"Navid Rekabsaz , Oleg Lesota , Markus Schedl , Jon Brassey , and Carsten Eickhoff . 2021. TripClick: The Log Files of a Large Health Web Search Engine . Association for Computing Machinery , New York, NY, USA , 2507--2513. https:\/\/doi.org\/10.1145\/3404835.3463242 10.1145\/3404835.3463242 Navid Rekabsaz, Oleg Lesota, Markus Schedl, Jon Brassey, and Carsten Eickhoff. 2021. TripClick: The Log Files of a Large Health Web Search Engine. Association for Computing Machinery, New York, NY, USA, 2507--2513. https:\/\/doi.org\/10.1145\/3404835.3463242"},{"key":"e_1_3_2_1_23_1","volume-title":"Overview of the TREC 2017 Precision Medicine Track. The 26th text REtrieval conference: TREC. Text REtrieval Conference","volume":"26","author":"Roberts Kirk","year":"2017","unstructured":"Kirk Roberts , Dina Demner-Fushman , Ellen Voorhees , William Hersh , Steven Bedrick , Alexander Lazar , and Shubham Pant . 2017 . Overview of the TREC 2017 Precision Medicine Track. The 26th text REtrieval conference: TREC. Text REtrieval Conference , Vol. 26 (11 2017). Kirk Roberts, Dina Demner-Fushman, Ellen Voorhees, William Hersh, Steven Bedrick, Alexander Lazar, and Shubham Pant. 2017. Overview of the TREC 2017 Precision Medicine Track. The 26th text REtrieval conference: TREC. Text REtrieval Conference, Vol. 26 (11 2017)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277935"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3082064"},{"key":"e_1_3_2_1_27_1","unstructured":"Nandan Thakur Nils Reimers Andreas R\u00fcckl\u00e9 Abhishek Srivastava and Iryna Gurevych. 2021. BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). https:\/\/openreview.net\/forum?id=wCu6T5xFjeJ  Nandan Thakur Nils Reimers Andreas R\u00fcckl\u00e9 Abhishek Srivastava and Iryna Gurevych. 2021. BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models. In Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2). https:\/\/openreview.net\/forum?id=wCu6T5xFjeJ"},{"volume-title":"Proceedings of the 2017 Conference on Conference Human Information Interaction and Retrieval. 253--256","author":"Verberne Suzan","key":"e_1_3_2_1_28_1","unstructured":"Suzan Verberne , Antal van den Bosch, Sander Wubben, and Emiel Krahmer. 2017. Automatic summarization of domain-specific forum threads: collecting reference data . In Proceedings of the 2017 Conference on Conference Human Information Interaction and Retrieval. 253--256 . Suzan Verberne, Antal van den Bosch, Sander Wubben, and Emiel Krahmer. 2017. Automatic summarization of domain-specific forum threads: collecting reference data. In Proceedings of the 2017 Conference on Conference Human Information Interaction and Retrieval. 253--256."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271766"},{"volume-title":"Proceedings of the 25th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval","author":"Ellen","key":"e_1_3_2_1_31_1","unstructured":"Ellen M. Voorhees and Chris Buckley. 2002. The Effect of Topic Set Size on Retrieval Experiment Error . In Proceedings of the 25th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval ( Tampere, Finland) (SIGIR '02). Association for Computing Machinery, New York, NY, USA, 316--323. https:\/\/doi.org\/10.1145\/564376.564432 10.1145\/564376.564432 Ellen M. Voorhees and Chris Buckley. 2002. The Effect of Topic Set Size on Retrieval Experiment Error. In Proceedings of the 25th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (Tampere, Finland) (SIGIR '02). Association for Computing Machinery, New York, NY, USA, 316--323. https:\/\/doi.org\/10.1145\/564376.564432"},{"key":"e_1_3_2_1_32_1","volume-title":"Voorhees and Kirk Roberts","author":"Ellen","year":"2021","unstructured":"Ellen M. Voorhees and Kirk Roberts . 2021 . On the Quality of the TREC-COVID IR Test Collections. Association for Computing Machinery , New York, NY, USA, 2422--2428. https:\/\/doi.org\/10.1145\/3404835.3463244 10.1145\/3404835.3463244 Ellen M. Voorhees and Kirk Roberts. 2021. On the Quality of the TREC-COVID IR Test Collections. Association for Computing Machinery, New York, NY, USA, 2422--2428. https:\/\/doi.org\/10.1145\/3404835.3463244"},{"key":"e_1_3_2_1_33_1","volume-title":"Can Old TREC Collections Reliably Evaluate Modern Neural Retrieval Models? CoRR","author":"Voorhees Ellen M.","year":"2022","unstructured":"Ellen M. Voorhees , Ian Soboroff , and Jimmy Lin . 2022. Can Old TREC Collections Reliably Evaluate Modern Neural Retrieval Models? CoRR , Vol. abs\/ 2201 .11086 ( 2022 ). showeprint[arXiv]2201.11086 https:\/\/arxiv.org\/abs\/2201.11086 Ellen M. Voorhees, Ian Soboroff, and Jimmy Lin. 2022. Can Old TREC Collections Reliably Evaluate Modern Neural Retrieval Models? CoRR, Vol. abs\/2201.11086 (2022). showeprint[arXiv]2201.11086 https:\/\/arxiv.org\/abs\/2201.11086"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484053"},{"key":"e_1_3_2_1_35_1","volume-title":"CMT in TREC-COVID Round 2: Mitigating the Generalization Gaps from Web to Special Domain Search. arxiv","author":"Xiong Chenyan","year":"2011","unstructured":"Chenyan Xiong , Zhenghao Liu , Si Sun , Zhuyun Dai , Kaitao Zhang , Shi Yu , Zhiyuan Liu , Hoifung Poon , Jianfeng Gao , and Paul Bennett . 2020. CMT in TREC-COVID Round 2: Mitigating the Generalization Gaps from Web to Special Domain Search. arxiv : 2011 .01580 [cs.IR] Chenyan Xiong, Zhenghao Liu, Si Sun, Zhuyun Dai, Kaitao Zhang, Shi Yu, Zhiyuan Liu, Hoifung Poon, Jianfeng Gao, and Paul Bennett. 2020. CMT in TREC-COVID Round 2: Mitigating the Generalization Gaps from Web to Special Domain Search. arxiv: 2011.01580 [cs.IR]"},{"key":"#cr-split#-e_1_3_2_1_36_1.1","unstructured":"Jingtao Zhan Xiaohui Xie Jiaxin Mao Yiqun Liu Min Zhang and Shaoping Ma. 2022. Evaluating Extrapolation Performance of Dense Retrieval. (2022). https:\/\/doi.org\/10.48550\/ARXIV.2204.11447 10.48550\/ARXIV.2204.11447"},{"key":"#cr-split#-e_1_3_2_1_36_1.2","unstructured":"Jingtao Zhan Xiaohui Xie Jiaxin Mao Yiqun Liu Min Zhang and Shaoping Ma. 2022. Evaluating Extrapolation Performance of Dense Retrieval. (2022). https:\/\/doi.org\/10.48550\/ARXIV.2204.11447"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291014"}],"event":{"name":"CIKM '22: The 31st ACM International Conference on Information and Knowledge Management","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Atlanta GA USA","acronym":"CIKM '22"},"container-title":["Proceedings of the 31st ACM International Conference on Information & Knowledge Management"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3511808.3557714","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3511808.3557714","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:48:49Z","timestamp":1750182529000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3511808.3557714"}},"subtitle":["A Relevance Judgement Test Collection for TripClick Health Retrieval"],"short-title":[],"issued":{"date-parts":[[2022,10,17]]},"references-count":38,"alternative-id":["10.1145\/3511808.3557714","10.1145\/3511808"],"URL":"https:\/\/doi.org\/10.1145\/3511808.3557714","relation":{},"subject":[],"published":{"date-parts":[[2022,10,17]]},"assertion":[{"value":"2022-10-17","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}