{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:09:38Z","timestamp":1784178578596,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,7,11]],"date-time":"2021-07-11T00:00:00Z","timestamp":1625961600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"National Key R&D Program of China","award":["2020YFB1406704"],"award-info":[{"award-number":["2020YFB1406704"]}]},{"name":"Key Scientific and Technological Innovation Program of Shandong Province","award":["2019JZZY010129"],"award-info":[{"award-number":["2019JZZY010129"]}]},{"name":"Natural Science Foundation of China","award":["61972234, 61902219, 62072279"],"award-info":[{"award-number":["61972234, 61902219, 62072279"]}]},{"name":"Tencent WeChat Rhino-Bird Focused Research Program","award":["JR-WXG-2021411"],"award-info":[{"award-number":["JR-WXG-2021411"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,7,11]]},"DOI":"10.1145\/3404835.3463241","type":"proceedings-article","created":{"date-parts":[[2021,7,12]],"date-time":"2021-07-12T03:08:25Z","timestamp":1626059305000},"page":"2499-2506","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":37,"title":["Simulating User Satisfaction for the Evaluation of Task-oriented Dialogue Systems"],"prefix":"10.1145","author":[{"given":"Weiwei","family":"Sun","sequence":"first","affiliation":[{"name":"Shandong University, Qingdao, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shuo","family":"Zhang","sequence":"additional","affiliation":[{"name":"Bloomberg, London, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Krisztian","family":"Balog","sequence":"additional","affiliation":[{"name":"University of Stavanger, Stavanger, Norway"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhaochun","family":"Ren","sequence":"additional","affiliation":[{"name":"Shandong University, Qingdao, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pengjie","family":"Ren","sequence":"additional","affiliation":[{"name":"Shandong University, Qingdao, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhumin","family":"Chen","sequence":"additional","affiliation":[{"name":"Shandong University, Qingdao, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maarten","family":"de Rijke","sequence":"additional","affiliation":[{"name":"University of Amsterdam & Ahold Delhaize Research, Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Mining Text Data","author":"Aggarwal Charu","unstructured":"Charu Aggarwal and ChengXiang Zhai. 2012. Mining Text Data. In Springer US ."},{"key":"e_1_3_2_1_2_1","volume-title":"I am Your Father: Dealing with Out-of-Domain Requests by Using Movies Subtitles","author":"Ameixa David","unstructured":"David Ameixa, Luisa Coheur, Pedro Fialho, and Paulo Quaresma. 2014. Luke, I am Your Father: Dealing with Out-of-Domain Requests by Using Movies Subtitles .Springer International Publishing."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"crossref","unstructured":"Layla El Asri Jing He and Kaheer Suleman. 2016. A Sequence-to-Sequence Model for User Simulation in Spoken Dialogue Systems. In INTERSPEECH .","DOI":"10.21437\/Interspeech.2016-1175"},{"key":"e_1_3_2_1_4_1","volume-title":"Banchs and Haizhou Li","author":"Rafael","year":"2013","unstructured":"Rafael E. Banchs and Haizhou Li. 2013. IRIS: a Chat-Oriented Dialogue System Based on the Vector Space Model. In ACL ."},{"key":"e_1_3_2_1_5_1","volume-title":"Is the User Enjoying the Conversation? A Case Study on the Impact on the Reward Function. ArXiv","author":"Rojas Barahona Lina M.","year":"2021","unstructured":"Lina M. Rojas Barahona. 2021. Is the User Enjoying the Conversation? A Case Study on the Impact on the Reward Function. ArXiv, Vol. abs\/2101.05004 (2021)."},{"key":"e_1_3_2_1_6_1","volume-title":"SIGDIAL Conference .","author":"Black Alan W","year":"2011","unstructured":"Alan W Black, Susanne Burger, Alistair Conkie, Helen Hastie, Simon Keizer, Oliver Lemon, Nicolas Merigaud, Gabriel Parent, Gabriel Schubiner, B. Thomson, Jason Williams, Kai Yu, Steve Young, and Maxine Esk\u00e9nazi. 2011. Spoken Dialog Challenge 2010: Comparison of Live and Control Test Results. In SIGDIAL Conference ."},{"key":"e_1_3_2_1_7_1","volume-title":"Multi-domain Conversation Quality Evaluation via User Satisfaction Estimation. ArXiv","author":"Bodigutla Praveen Kumar","year":"2019","unstructured":"Praveen Kumar Bodigutla, Lazaros Polymenakos, and Spyros Matsoukas. 2019. Multi-domain Conversation Quality Evaluation via User Satisfaction Estimation. ArXiv, Vol. abs\/1911.08567 (2019)."},{"key":"e_1_3_2_1_8_1","volume-title":"Joint Turn and Dialogue level User Satisfaction Estimation on Multi-Domain Conversations. ArXiv","author":"Bodigutla Praveen Kumar","year":"2020","unstructured":"Praveen Kumar Bodigutla, Aditya Tiwari, Josep Valls-Vargas, Lazaros Polymenakos, and Spyridon Matsoukas. 2020. Joint Turn and Dialogue level User Satisfaction Estimation on Multi-Domain Conversations. ArXiv, Vol. abs\/2010.02495 (2020)."},{"key":"e_1_3_2_1_9_1","volume-title":"Predicting User Intents and Satisfaction with Dialogue-based Conversational Recommendations. ACM UMAP","author":"Cai Wanling","year":"2020","unstructured":"Wanling Cai and Li Chen. 2020. Predicting User Intents and Satisfaction with Dialogue-based Conversational Recommendations. ACM UMAP (2020)."},{"key":"e_1_3_2_1_10_1","volume-title":"Eric Yihong Zhao, and Dawei Yin","author":"Chen Hongshen","year":"2018","unstructured":"Hongshen Chen, Zhaochun Ren, Jiliang Tang, Eric Yihong Zhao, and Dawei Yin. 2018. Hierarchical Variational Memory Network for Dialogue Generation. In WWW. 1653--1662."},{"key":"e_1_3_2_1_11_1","unstructured":"Meng Chen Ruixue Liu Lei Shen Shaozu Yuan Jingyan Zhou Youzheng Wu Xiaodong He and Bowen Zhou. 2020. The JDDC Corpus: A Large-Scale Multi-Turn Chinese Dialogue Dataset for E-commerce Customer Service. In LREC ."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASRU.2005.1566485"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-020-09866-x"},{"key":"e_1_3_2_1_14_1","volume-title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In NAACL-HLT .","author":"Devlin Jacob","year":"2019","unstructured":"Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2019. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding. In NAACL-HLT ."},{"key":"e_1_3_2_1_15_1","volume-title":"User Modeling for Spoken Dialogue System Evaluation. IEEE Workshop on ASRU","author":"Eckert Wieland","year":"1997","unstructured":"Wieland Eckert, Esther Levin, and Roberto Pieraccini. 1997. User Modeling for Spoken Dialogue System Evaluation. IEEE Workshop on ASRU (1997), 80--87."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"Klaus-Peter Engelbrecht Florian G\u00f6dde Felix Hartard Hamed Ketabdar and Sebastian M\u00f6ller. 2009. Modeling User Satisfaction with Hidden Markov Models. In SIGDIAL .","DOI":"10.3115\/1708376.1708402"},{"key":"e_1_3_2_1_17_1","volume-title":"Peter Ku, Sanchit Agarwal, Shuyang Gao, and Dilek Hakkani-Tur.","author":"Eric Mihail","year":"2020","unstructured":"Mihail Eric, Rahul Goel, Shachi Paul, Adarsh Kumar, Abhishek Sethi, Anuj Kumar Goyal, Peter Ku, Sanchit Agarwal, Shuyang Gao, and Dilek Hakkani-Tur. 2020. MultiWOZ 2.1: A Consolidated Multi-Domain Dialogue Dataset with State Corrections and State Tracking Baselines. In LREC ."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"crossref","unstructured":"Kallirroi Georgila James Henderson and Oliver Lemon. 2005. Learning User Simulations for Information State Update Dialogue Systems. In INTERSPEECH .","DOI":"10.21437\/Interspeech.2005-401"},{"key":"e_1_3_2_1_19_1","unstructured":"Sunao Hara Norihide Kitaoka and Kazuya Takeda. 2010. Estimation Method of User Satisfaction Using N-gram-based Dialog History Model for Spoken Dialog System. In LREC ."},{"key":"e_1_3_2_1_20_1","volume-title":"Workshop on Query Understanding .","author":"Hashemi Homa B","year":"2016","unstructured":"Homa B Hashemi, Amir Asiaee, and Reiner Kraft. 2016. Query Intent Detection using Convolutional Neural Networks. In WSDM, Workshop on Query Understanding ."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"crossref","unstructured":"Ryuichiro Higashinaka Yasuhiro Minami Kohji Dohsaka and Toyomi Meguro. 2010. Issues in Predicting User Satisfaction Transitions in Dialogues: Individual Differences Evaluation Criteria and Prediction Models. In IWSDS .","DOI":"10.1007\/978-3-642-16202-2_5"},{"key":"e_1_3_2_1_22_1","volume-title":"A Survey on Conversational Recommender Systems. ArXiv","author":"Jannach Dietmar","year":"2020","unstructured":"Dietmar Jannach, Ahtsham Manzoor, Wanling Cai, and Li'e Chen. 2020. A Survey on Conversational Recommender Systems. ArXiv, Vol. abs\/2004.00646 (2020)."},{"key":"e_1_3_2_1_23_1","volume-title":"Lyu","author":"Jiao Wenxiang","year":"2019","unstructured":"Wenxiang Jiao, Haiqin Yang, Irwin King, and Michael R. Lyu. 2019. HiGRU: Hierarchical Gated Recurrent Units for Utterance-level Emotion Recognition. NAACL-HLT (2019), 397--406."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"crossref","unstructured":"Xisen Jin Wenqiang Lei Zhaochun Ren Hongshen Chen Shangsong Liang Yihong Zhao and Dawei Yin. 2018. Explicit State Tracking with Semi-Supervision for Neural Dialogue Generation. In ACM CIKM. 1403--1412.","DOI":"10.1145\/3269206.3271683"},{"key":"e_1_3_2_1_25_1","volume-title":"B. Thomson, Kai Yu, and S. Young.","author":"Jurc'icek Filip","year":"2011","unstructured":"Filip Jurc'icek, Simon Keizer, Milica Gasic, Franc cois Mairesse, B. Thomson, Kai Yu, and S. Young. 2011. Real User Evaluation of Spoken Dialogue Systems Using Amazon Mechanical Turk. In INTERSPEECH ."},{"key":"e_1_3_2_1_26_1","volume-title":"Pawel Budzianowski, and Milica Gasic.","author":"Kreyssig Florian","year":"2018","unstructured":"Florian Kreyssig, I nigo Casanueva, Pawel Budzianowski, and Milica Gasic. 2018. Neural User Simulation for Corpus-based Policy Optimisation for Spoken Dialogue Systems. ArXiv, Vol. abs\/1805.06966 (2018)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0167-6393(99)00067-9"},{"key":"e_1_3_2_1_28_1","volume-title":"Sequicity: Simplifying task-oriented dialogue systems with single sequence-to-sequence architectures. In ACL. 1437--1447.","author":"Lei Wenqiang","year":"2018","unstructured":"Wenqiang Lei, Xisen Jin, Min-Yen Kan, Zhaochun Ren, Xiangnan He, and Dawei Yin. 2018. Sequicity: Simplifying task-oriented dialogue systems with single sequence-to-sequence architectures. In ACL. 1437--1447."},{"key":"e_1_3_2_1_29_1","unstructured":"Wenqiang Lei Gangyi Zhang Xiangnan He Yisong Miao Xiang Wang Liang Chen and Tat-Seng Chua. 2020. Interactive path reasoning on graph for conversational recommendation. In SIGKDD ."},{"key":"e_1_3_2_1_30_1","volume-title":"Data-Driven Methods for Adaptive Spoken Dialogue Systems","author":"Lemon Oliver","unstructured":"Oliver Lemon and Olivier Pietquin. 2012. Data-Driven Methods for Adaptive Spoken Dialogue Systems. In Springer New York ."},{"key":"e_1_3_2_1_31_1","unstructured":"Jiwei Li Michel Galley Chris Brockett Jianfeng Gao and Bill Dolan. 2016a. A Diversity-Promoting Objective Function for Neural Conversation Models. In NAACL ."},{"key":"e_1_3_2_1_32_1","unstructured":"Jiwei Li Michel Galley Chris Brockett Georgios P. Spithourakis Jianfeng Gao and W. Dolan. 2016b. A Persona-Based Neural Conversation Model. ACL ."},{"key":"e_1_3_2_1_33_1","volume-title":"Hannes Schulz, Vincent Michalski, Laurent Charlin, and Chris Pal.","author":"Li Raymond","year":"2018","unstructured":"Raymond Li, Samira Ebrahimi Kahou, Hannes Schulz, Vincent Michalski, Laurent Charlin, and Chris Pal. 2018. Towards Deep Conversational Recommendations. ArXiv, Vol. abs\/1812.07617 (2018)."},{"key":"e_1_3_2_1_34_1","volume-title":"Time to Transfer: Predicting and Evaluating Machine-Human Chatting Handoff. AAAI","author":"Liu Jiawei","year":"2021","unstructured":"Jiawei Liu, Zhe Gao, Yangyang Kang, Zhuoren Jiang, Guoxiu He, Changlong Sun, Xiaozhong Liu, and Wei Lu. 2021. Time to Transfer: Predicting and Evaluating Machine-Human Chatting Handoff. AAAI (2021)."},{"key":"e_1_3_2_1_35_1","unstructured":"Longxuan Ma Weinan Zhang Runxin Sun and Ting Liu. 2020. A Compare Aggregate Transformer for Understanding Document-grounded Dialogue. In Findings of EMNLP. 1358--1367."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Chuan Meng Pengjie Ren Zhumin Chen Weiwei Sun Z. Ren Zhaopeng Tu and Maarten de Rijke. 2020. DukeNet: A Dual Knowledge Interaction Network for Knowledge-Grounded Conversation. SIGIR .","DOI":"10.1145\/3397271.3401097"},{"key":"e_1_3_2_1_37_1","unstructured":"Nikola Mrkvs i\u0107 Diarmuid \u00d3 S\u00e9aghdha Blaise Thomson Milica Gasic Pei-Hao Su David Vandyke Tsung-Hsien Wen and Steve Young. 2015. Multi-domain Dialog State Tracking using Recurrent Neural Networks. In IJCNLP (Volume 2: Short Papers). 794--799."},{"key":"e_1_3_2_1_38_1","unstructured":"Nikola Mrkvs i\u0107 Diarmuid \u00d3 S\u00e9aghdha Tsung-Hsien Wen Blaise Thomson and Steve Young. 2017. Neural Belief Tracker: Data-Driven Dialogue State Tracking. In ACL. 1777--1788."},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Louisa Pragst Stefan Ultes and Wolfgang Minker. 2016. Recurrent Neural Network Interaction Quality Estimation. In IWSDS .","DOI":"10.1007\/978-981-10-2585-3_31"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"Filip Radlinski Krisztian Balog Bill Byrne and Karthik Krishnamoorthi. 2019. Coached Conversational Preference Elicitation: A Case Study in Understanding Movie Preferences. In SIGdial .","DOI":"10.18653\/v1\/W19-5941"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"crossref","unstructured":"Abhinav Rastogi Xiaoxue Zang Srinivas Sunkara Raghav Gupta and Pranav Khaitan. 2020. Towards Scalable Multi-domain Conversational Agents: The Schema-Guided Dialogue Dataset. In AAAI .","DOI":"10.1609\/aaai.v34i05.6394"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"Jost Schatzmann Blaise Thomson Karl Weilhammer Hui Ye and Steve Young. 2007. Agenda-Based User Simulation for Bootstrapping a POMDP Dialogue System. In HLT-NAACL .","DOI":"10.3115\/1614108.1614146"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2000.859185"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.specom.2015.06.003"},{"key":"e_1_3_2_1_45_1","unstructured":"Alexander Schmitt Stefan Ultes and Wolfgang Minker. 2012. A Parameterized and Annotated Spoken Dialog Corpus of the Carnegie Mellon University Let's Go Bus Information System. In LREC ."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"crossref","unstructured":"Iulian Vlad Serban Alessandro Sordoni Yoshua Bengio Aaron C Courville and Joelle Pineau. 2016. Building End-To-End Dialogue Systems Using Generative Hierarchical Neural Network Models.. In AAAI .","DOI":"10.1609\/aaai.v30i1.9883"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Weiwei Sun Chuan Meng Qi Meng Zhaochun Ren Pengjie Ren Zhumin Chen and Maarten de Rijke. 2021. Conversations Powered by Cross-Lingual Knowledge. In SIGIR .","DOI":"10.1145\/3404835.3462883"},{"key":"e_1_3_2_1_48_1","unstructured":"Stefan Ultes Alexander Schmitt and Wolfgang Minker. 2013. On Quality Ratings for Spoken Dialogue Systems - Experts vs. Users. In HLT-NAACL ."},{"key":"e_1_3_2_1_49_1","unstructured":"Oriol Vinyals and Quoc Le. 2015. A Neural Conversational Model. In ICML ."},{"key":"e_1_3_2_1_50_1","volume-title":"PARADISE: A Framework for Evaluating Spoken Dialogue Agents. ArXiv","author":"Walker Marilyn A.","year":"1997","unstructured":"Marilyn A. Walker, Diane Litman, Candace A. Kamm, and Alicia Abella. 1997. PARADISE: A Framework for Evaluating Spoken Dialogue Agents. ArXiv, Vol. cmp-lg\/9704004 (1997)."},{"key":"e_1_3_2_1_51_1","volume-title":"Milica Gasic, Lina M Rojas Barahona, Pei-Hao Su, Stefan Ultes, and Steve Young.","author":"Wen Tsung-Hsien","year":"2017","unstructured":"Tsung-Hsien Wen, David Vandyke, Nikola Mrkvs i\u0107, Milica Gasic, Lina M Rojas Barahona, Pei-Hao Su, Stefan Ultes, and Steve Young. 2017. A Network-based End-to-End Trainable Task-oriented Dialogue System. In EACL. 438--449."},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"crossref","unstructured":"Jun Xu Haifeng Wang Zheng-Yu Niu Hua Wu Wanxiang Che and Ting Liu. 2020. Conversational graph grounded policy learning for open-domain conversation generation. In ACL. 1835--1845.","DOI":"10.18653\/v1\/2020.acl-main.166"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Zhao Yan Nan Duan Peng Chen M. Zhou Jianshe Zhou and Zhoujun Li. 2017. Building Task-Oriented Dialogue Systems for Online Shopping. In AAAI .","DOI":"10.1609\/aaai.v31i1.11182"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"crossref","unstructured":"Zichao Yang Diyi Yang Chris Dyer Xiaodong He Alex Smola and Eduard Hovy. 2016. Hierarchical Attention Networks for Document Classification. In HLT-NAACL .","DOI":"10.18653\/v1\/N16-1174"},{"key":"e_1_3_2_1_55_1","unstructured":"Kaisheng Yao Geoffrey Zweig Mei-Yuh Hwang Yangyang Shi and Dong Yu. 2013. Recurrent Neural Networks for Language Understanding. In Interspeech ."},{"key":"e_1_3_2_1_56_1","article-title":"POMDP-based Statistical Spoken Dialog Systems","volume":"101","author":"Young Steve","year":"2013","unstructured":"Steve Young, Milica Gavs i\u0107, Blaise Thomson, and Jason D Williams. 2013. POMDP-based Statistical Spoken Dialog Systems: A Review. Proc. IEEE, Vol. 101, 5 (2013).","journal-title":"A Review. Proc. IEEE"},{"key":"e_1_3_2_1_57_1","volume-title":"Evaluating Conversational Recommender Systems via User Simulation. ACM SIGKDD","author":"Zhang Shuo","year":"2020","unstructured":"Shuo Zhang and Krisztian Balog. 2020. Evaluating Conversational Recommender Systems via User Simulation. ACM SIGKDD (2020)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"crossref","unstructured":"Hao Zhou Minlie Huang Tianyang Zhang Xiaoyan Zhu and Bing Liu. 2018. Emotional Chatting Machine: Emotional Conversation Generation with Internal and External Memory. In AAAI .","DOI":"10.1609\/aaai.v32i1.11325"}],"event":{"name":"SIGIR '21: The 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Virtual Event Canada","acronym":"SIGIR '21","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463241","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3404835.3463241","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:44Z","timestamp":1750191464000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3404835.3463241"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,11]]},"references-count":58,"alternative-id":["10.1145\/3404835.3463241","10.1145\/3404835"],"URL":"https:\/\/doi.org\/10.1145\/3404835.3463241","relation":{},"subject":[],"published":{"date-parts":[[2021,7,11]]},"assertion":[{"value":"2021-07-11","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}