{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,11]],"date-time":"2026-07-11T02:51:45Z","timestamp":1783738305242,"version":"3.55.0"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783032213204","type":"print"},{"value":"9783032213211","type":"electronic"}],"license":[{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2026,1,1]],"date-time":"2026-01-01T00:00:00Z","timestamp":1767225600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2026]]},"DOI":"10.1007\/978-3-032-21321-1_55","type":"book-chapter","created":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T11:10:43Z","timestamp":1774264243000},"page":"496-510","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":1,"title":["UserSimCRS v2: Simulation-Based Evaluation for Conversational Recommender Systems"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-0565-3210","authenticated-orcid":false,"given":"Nolwenn","family":"Bernard","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2762-721X","authenticated-orcid":false,"given":"Krisztian","family":"Balog","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,3,24]]},"reference":[{"key":"55_CR1","doi-asserted-by":"crossref","unstructured":"Afzali, J., Drzewiecki, A.M., Balog, K., Zhang, S.: UserSimCRS: a user simulation toolkit for evaluating conversational recommender systems. In: Proceedings of the Sixteenth ACM International Conference on Web Search and Data Mining, WSDM 2023, pp. 1160\u20131163 (2023)","DOI":"10.1145\/3539597.3573029"},{"key":"55_CR2","doi-asserted-by":"crossref","unstructured":"Alaofi, M., Thomas, P., Scholer, F., Sanderson, M.: LLMs can be fooled into labelling a document as relevant. In: Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region, SIGIR-AP 2024, pp. 32\u201341 (2024)","DOI":"10.1145\/3673791.3698431"},{"key":"55_CR3","doi-asserted-by":"crossref","unstructured":"Balog, K., Metzler, D., Qin, Z.: Rankers, judges, and assistants: towards understanding the interplay of LLMs in information retrieval evaluation. In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, pp. 3865\u20133875 (2025)","DOI":"10.1145\/3726302.3730348"},{"key":"55_CR4","doi-asserted-by":"crossref","unstructured":"Balog, K., Zhai, C.: User simulation for evaluating information access systems. Found. Trends Inf. Retrieval 18(1-2), 1\u2013261 (2024). ISSN 1554-0669","DOI":"10.1561\/1500000098"},{"key":"55_CR5","doi-asserted-by":"crossref","unstructured":"Bernard, N., Balog, K.: Limitations of current evaluation practices for conversational recommender systems and the potential of user simulation. In: Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region, SIGIR-AP 2025, pp. 261\u2013271 (2025)","DOI":"10.1145\/3767695.3769478"},{"key":"55_CR6","doi-asserted-by":"crossref","unstructured":"Bernard, N., Joko, H., Hasibi, F., Balog, K.: CRS Arena: crowdsourced benchmarking of conversational recommender systems. In: Proceedings of the Eighteenth ACM International Conference on Web Search and Data Mining, WSDM 2025, pp. 1028\u20131031 (2025)","DOI":"10.1145\/3701551.3704120"},{"issue":"2","key":"55_CR7","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3722449.3722460","volume":"58","author":"T Breuer","year":"2025","unstructured":"Breuer, T., et al.: Report on the workshop on simulations for information access (Sim4IA 2024) at SIGIR 2024. SIGIR Forum 58(2), 1\u201314 (2025)","journal-title":"SIGIR Forum"},{"key":"55_CR8","doi-asserted-by":"crossref","unstructured":"Cai, W., Chen, L.: Predicting user intents and satisfaction with dialogue-based conversational recommendations. In: Proceedings of the 28th ACM Conference on User Modeling, Adaptation and Personalization, UMAP 2020, pp. 33\u201342 (2020)","DOI":"10.1145\/3340631.3394856"},{"key":"55_CR9","doi-asserted-by":"crossref","unstructured":"Chen, L., et al.: RecUserSim: a realistic and diverse user simulator for evaluating conversational recommender systems. In: Companion Proceedings of the ACM on Web Conference 2025, WWW 2025, pp. 133\u2013142 (2025)","DOI":"10.1145\/3701716.3715258"},{"key":"55_CR10","doi-asserted-by":"crossref","unstructured":"Chen, Q., et al.: Towards knowledge-based recommender dialog system. In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing, EMNLP-IJCNLP 2019, pp. 1803\u20131813 (2019)","DOI":"10.18653\/v1\/D19-1189"},{"key":"55_CR11","doi-asserted-by":"crossref","unstructured":"Dietz, L., et al.: Principles and guidelines for the use of LLM judges. In: Proceedings of the 2025 International ACM SIGIR Conference on Innovative Concepts and Theories in Information Retrieval, ICTIR 2025, pp. 218\u2013229 (2025)","DOI":"10.1145\/3731120.3744588"},{"key":"55_CR12","doi-asserted-by":"crossref","unstructured":"Faggioli, G., et al.: Perspectives on large language models for relevance judgment. In: Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval, ICTIR 2023 (2023)","DOI":"10.1145\/3578337.3605136"},{"key":"55_CR13","doi-asserted-by":"publisher","first-page":"100","DOI":"10.1016\/j.aiopen.2021.06.002","volume":"2","author":"C Gao","year":"2021","unstructured":"Gao, C., Lei, W., He, X., de Rijke, M., Chua, T.S.: Advances and challenges in conversational recommender systems: a survey. AI Open 2, 100\u2013126 (2021)","journal-title":"AI Open"},{"key":"55_CR14","doi-asserted-by":"crossref","unstructured":"Habib, J., Zhang, S., Balog, K.: IAI MovieBot: a conversational movie recommender system. In: Proceedings of the 29th ACM International Conference on Information & Knowledge Management, CIKM 2020, pp. 3405\u20133408 (2020)","DOI":"10.1145\/3340531.3417433"},{"key":"55_CR15","doi-asserted-by":"crossref","unstructured":"Hayati, S.A., Kang, D., Zhu, Q., Shi, W., Yu, Z.: INSPIRED: toward sociable recommendation dialog systems. In: Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing, EMNLP 2020, pp. 8142\u20138152 (2020)","DOI":"10.18653\/v1\/2020.emnlp-main.654"},{"key":"55_CR16","unstructured":"Huang, C., Qin, P., Deng, Y., Lei, W., Lv, J., Chua, T.S.: Concept \u2013 An evaluation protocol on conversation recommender systems with system-centric and user-centric factors. arXiv cs.CL\/2404.03304 (2024)"},{"issue":"3","key":"55_CR17","doi-asserted-by":"publisher","first-page":"2365","DOI":"10.1007\/s10462-022-10229-x","volume":"56","author":"D Jannach","year":"2023","unstructured":"Jannach, D.: Evaluating conversational recommender systems. Artif. Intell. Rev. 56(3), 2365\u20132400 (2023)","journal-title":"Artif. Intell. Rev."},{"key":"55_CR18","doi-asserted-by":"crossref","unstructured":"Kelly, D.: Methods for evaluating interactive information retrieval systems with users. Found. Trends\u00ae Inf. Retrieval 3(1-2), 1\u2013224 (2009)","DOI":"10.1561\/1500000012"},{"key":"55_CR19","doi-asserted-by":"crossref","unstructured":"Kim, M., et al.: Pearl: a review-driven persona-knowledge grounded conversational recommendation dataset. In: Findings of the Association for Computational Linguistics: ACL 2024, pp. 1105\u20131120 (2024)","DOI":"10.18653\/v1\/2024.findings-acl.65"},{"key":"55_CR20","doi-asserted-by":"crossref","unstructured":"Kim, S., Kim, T., Seo, K., Yeo, J., Lee, D.: Stop playing the guessing game! Target-free user simulation for evaluating conversational recommender systems. arXiv cs.IR\/2411.16160 (2024)","DOI":"10.18653\/v1\/2025.findings-emnlp.1067"},{"key":"55_CR21","doi-asserted-by":"crossref","unstructured":"Lee, S., et al.: ConvLab: multi-domain end-to-end dialog system platform. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, ACL 2019, pp. 64\u201369 (2019)","DOI":"10.18653\/v1\/P19-3011"},{"key":"55_CR22","unstructured":"Li, R., Kahou, S., Schulz, H., Michalski, V., Charlin, L., Pal, C.: Towards deep conversational recommendations. In: Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS 2018, pp. 9748\u20139758 (2018)"},{"key":"55_CR23","doi-asserted-by":"crossref","unstructured":"Lyu, S., Rana, A., Sanner, S., Bouadjenek, M.R.: A workflow analysis of context-driven conversational recommendation. In: Proceedings of the Web Conference 2021, WWW 2021, pp. 866\u2013877 (2021)","DOI":"10.1145\/3442381.3450123"},{"key":"55_CR24","doi-asserted-by":"crossref","unstructured":"Moon, S., Shah, P., Kumar, A., Subba, R.: OpenDialKG: explainable conversational reasoning with attention-based walks over knowledge graphs. In: Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, ACL 2019, pp. 845\u2013854 (2019)","DOI":"10.18653\/v1\/P19-1081"},{"key":"55_CR25","doi-asserted-by":"crossref","unstructured":"Quan, J., et al.: FORCE: a framework of rule-based conversational recommender system. In: Proceedings of the AAAI Conference on Artificial Intelligence, AAAI 2022, pp. 13215\u201313217 (2022)","DOI":"10.1609\/aaai.v36i11.21732"},{"key":"55_CR26","doi-asserted-by":"crossref","unstructured":"Schaer, P., Kreutz, C.K., Balog, K., Breuer, T., Kruff, A.K.: Second SIGIR workshop on simulations for information access (Sim4IA 2025). In: Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2025, pp. 4172\u20134175 (2025)","DOI":"10.1145\/3726302.3730363"},{"key":"#cr-split#-55_CR27.1","doi-asserted-by":"crossref","unstructured":"Schatzmann, J., Thomson, B., Weilhammer, K., Ye, H., Young, S.: Agenda-based user simulation for bootstrapping a POMDP dialogue system. In: Human Language Technologies 2007: The Conference of the North American Chapter of the Association for Computational Linguistics","DOI":"10.3115\/1614108.1614146"},{"key":"#cr-split#-55_CR27.2","unstructured":"Companion Volume, Short Papers, NAACL 2007, pp. 149-152 (2007)"},{"key":"55_CR28","unstructured":"Su, H., Ye, J.: Large language models for automating fine-grained speech act annotation: a critical evaluation of GPT-4o and DeepSeek. Corpus Pragmatics (2005)"},{"key":"55_CR29","unstructured":"Terragni, S., Filipavicius, M., Khau, N., Guedes, B., Manso, A., Mathis, R.: In-context learning user simulators for task-oriented dialog systems. arXiv cs.CL\/2306.00774 (2023)"},{"key":"55_CR30","doi-asserted-by":"crossref","unstructured":"Thomas, P., Spielman, S., Craswell, N., Mitra, B.: Large language models can accurately predict searcher preferences. In: Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2024, pp. 1930\u20131940 (2024)","DOI":"10.1145\/3626772.3657707"},{"key":"55_CR31","doi-asserted-by":"crossref","unstructured":"Ultes, S., et al.: PyDial: a multi-domain statistical dialogue system toolkit. In: Proceedings of ACL 2017, System Demonstrations, ACL 2017, pp. 73\u201378 (2017)","DOI":"10.18653\/v1\/P17-4013"},{"key":"55_CR32","doi-asserted-by":"crossref","unstructured":"Vlachou, M.: Fashion-AlterEval: a dataset for improved evaluation of conversational recommendation systems with alternative relevant items. In: Proceedings of the Nineteenth ACM Conference on Recommender Systems, RecSys 2025, pp. 755\u2013763 (2025)","DOI":"10.1145\/3705328.3748149"},{"key":"55_CR33","doi-asserted-by":"crossref","unstructured":"Wang, P., et al.: Large language models are not fair evaluators. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), ACL 2024, pp. 9440\u20139450 (2024)","DOI":"10.18653\/v1\/2024.acl-long.511"},{"key":"55_CR34","unstructured":"Wang, T.C., Su, S.Y., Chen, Y.N.: BARCOR: towards a unified framework for conversational recommendation systems. arXiv cs.CL\/2203.14257 (2022)"},{"key":"55_CR35","doi-asserted-by":"crossref","unstructured":"Wang, X., Tang, X., Zhao, X., Wang, J., Wen, J.R.: Rethinking the evaluation for conversational recommendation in the era of large language models. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, EMNLP 2023, pp. 10052\u201310065 (2023)","DOI":"10.18653\/v1\/2023.emnlp-main.621"},{"key":"55_CR36","doi-asserted-by":"crossref","unstructured":"Yoon, S., He, Z., Echterhoff, J., McAuley, J.: Evaluating large language models as generative user simulators for conversational recommendation. In: Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers), NAACL 2024, pp. 1490\u20131504 (2024)","DOI":"10.18653\/v1\/2024.naacl-long.83"},{"key":"55_CR37","unstructured":"Zhang, H., Zhao, X., Chen, J., Guo, J.: A literature review on simulation in conversational recommender systems. arXiv cs.HC\/2506.20291 (2025)"},{"key":"55_CR38","doi-asserted-by":"crossref","unstructured":"Zhang, S., Balog, K.: Evaluating conversational recommender systems via user simulation. In: Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining, KDD 2020, pp. 1512\u20131520 (2020)","DOI":"10.1145\/3394486.3403202"},{"key":"55_CR39","doi-asserted-by":"crossref","unstructured":"Zhang, Z., et al.: RecWizard: a toolkit for conversational recommendation with modular, portable models and interactive user interface. In: Proceedings of the 38th Annual AAAI Conference on Artificial Intelligence, AAAI 2024 (2024)","DOI":"10.1609\/aaai.v38i21.30588"},{"key":"55_CR40","doi-asserted-by":"crossref","unstructured":"Zhangwenbo, Z., Yuhan, W.: Act2P: LLM-driven online dialogue act classification for power analysis. In: Findings of the Association for Computational Linguistics: ACL 2025, ACL 2025, pp. 20494\u201320504 (2025)","DOI":"10.18653\/v1\/2025.findings-acl.1052"},{"key":"55_CR41","unstructured":"Zhao, X., et al.: Exploring the impact of personality traits on conversational recommender systems: a simulation with large language models. arXiv cs.CL\/2504.12313 (2025)"},{"key":"55_CR42","unstructured":"Zheng, L., et al.: Judging LLM-as-a-judge with MT-bench and Chatbot Arena. In: Proceedings of the 37th International Conference on Neural Information Processing Systems, NeurIPS 2023 (2023)"},{"key":"55_CR43","doi-asserted-by":"crossref","unstructured":"Zhou, K., et al.: CRSLab: an open-source toolkit for building conversational recommender system. In: Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing: System Demonstrations, ACL-IJCNLP 2021, pp. 185\u2013193 (2021)","DOI":"10.18653\/v1\/2021.acl-demo.22"},{"key":"55_CR44","doi-asserted-by":"crossref","unstructured":"Zhu, L., Huang, X., Sang, J.: A LLM-based controllable, scalable, human-involved user simulator framework for conversational recommender systems. In: Proceedings of the ACM on Web Conference 2025, WWW 2025, pp. 4653\u20134661 (2025)","DOI":"10.1145\/3696410.3714858"},{"key":"55_CR45","doi-asserted-by":"crossref","unstructured":"Zhu, Q., et al.: ConvLab-3: a flexible dialogue system toolkit based on a unified data format. In: Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: System Demonstrations, EMNLP 2023, pp. 106\u2013123 (2023)","DOI":"10.18653\/v1\/2023.emnlp-demo.9"},{"key":"55_CR46","doi-asserted-by":"crossref","unstructured":"Zhu, Q., et al.: ConvLab-2: an open-source toolkit for building, evaluating, and diagnosing dialogue systems. In: Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, ACL 2020, pp. 142\u2013149 (2020)","DOI":"10.18653\/v1\/2020.acl-demos.19"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-032-21321-1_55","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,23]],"date-time":"2026-03-23T23:16:09Z","timestamp":1774307769000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-032-21321-1_55"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026]]},"ISBN":["9783032213204","9783032213211"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-032-21321-1_55","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026]]},"assertion":[{"value":"24 March 2026","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"The authors have no competing interests to declare that are relevant to the content of this article.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Disclosure of Interests"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Delft","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"The Netherlands","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2026","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"29 March 2026","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2 April 2026","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"48","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2026","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2026.eu\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}}]}}