{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,17]],"date-time":"2025-11-17T05:55:23Z","timestamp":1763358923439,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,4,30]],"date-time":"2023-04-30T00:00:00Z","timestamp":1682812800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,4,30]]},"DOI":"10.1145\/3543507.3583496","type":"proceedings-article","created":{"date-parts":[[2023,4,26]],"date-time":"2023-04-26T23:30:51Z","timestamp":1682551851000},"page":"3837-3848","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["HybridEval: A Human-AI Collaborative Approach for Evaluating Design Ideas at Scale"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8963-3757","authenticated-orcid":false,"given":"Sepideh","family":"Mesbah","sequence":"first","affiliation":[{"name":"Booking, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7513-6197","authenticated-orcid":false,"given":"Ines","family":"Arous","sequence":"additional","affiliation":[{"name":"University of Fribourg, Switzerland"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0350-0313","authenticated-orcid":false,"given":"Jie","family":"Yang","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3300-2913","authenticated-orcid":false,"given":"Alessandro","family":"Bozzon","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Netherlands"}]}],"member":"320","published-online":{"date-parts":[[2023,4,30]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1115\/1.4038070"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Faez Ahmed and Mark\u00a0D. Fuge. 2017. Capturing Winning Ideas in Online Design Communities. In CSCW. ACM 1675\u20131687.","DOI":"10.1145\/2998181.2998249"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1115\/1.4041856"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Salvatore Andolina Khalil Klouche Diogo Cabral Tuukka Ruotsalo and Giulio Jacucci. 2015. InspirationWall: Supporting Idea Generation Through Automatic Information Exploration. In Creativity & Cognition. ACM 103\u2013106.","DOI":"10.1145\/2757226.2757252"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1115\/1.4054688"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380254"},{"key":"e_1_3_2_1_7_1","volume-title":"WWW. ACM \/ IW3C2","author":"Arous Ines","year":"1916","unstructured":"Ines Arous, Jie Yang, Mourad Khayati, and Philippe Cudr\u00e9-Mauroux. 2021. Peer Grading the Peer Reviews: A Dual-Role Approach for Lightening the Scholarly Paper Review Process. In WWW. ACM \/ IW3C2, 1916\u20131927."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v8i1.7460"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450069"},{"key":"e_1_3_2_1_10_1","volume-title":"Using thematic analysis in psychology. Qualitative research in psychology 3, 2","author":"Braun Virginia","year":"2006","unstructured":"Virginia Braun and Victoria Clarke. 2006. Using thematic analysis in psychology. Qualitative research in psychology 3, 2 (2006), 77\u2013101."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380250"},{"key":"e_1_3_2_1_12_1","volume-title":"Design Thinking. harvard business review","author":"Brown Tim","year":"2008","unstructured":"Tim Brown. 2008. Design Thinking. harvard business review (2008), 1."},{"key":"e_1_3_2_1_13_1","unstructured":"Tim Brown. 2009. Change by design: How design thinking creates new alternatives for business and society. Collins Business."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1115\/1.4029065"},{"key":"e_1_3_2_1_15_1","volume-title":"Google struggles to give away $10 million. Wired Magazine","author":"Buskirk EV","year":"2010","unstructured":"EV Buskirk. 2010. Google struggles to give away $10 million. Wired Magazine (2010)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1948-7177.2011.00024.x"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.intmar.2015.05.003"},{"key":"e_1_3_2_1_18_1","volume-title":"18th Americas Conference on Information Systems 2012, AMCIS 2012. 97\u2013107","author":"Chen Liang","year":"2012","unstructured":"Liang Chen and De Liu. 2012. Comparing strategies for winning expert-rated and crowd-rated crowdsourcing contests: First findings. In 18th Americas Conference on Information Systems 2012, AMCIS 2012. 97\u2013107."},{"key":"e_1_3_2_1_19_1","volume-title":"Grow-and-Clip: Informative-yet-Concise Evidence Distillation for Answer Explanation. arXiv preprint arXiv:2201.05088","author":"Chen Yuyan","year":"2022","unstructured":"Yuyan Chen, Yanghua Xiao, and Bang Liu. 2022. Grow-and-Clip: Informative-yet-Concise Evidence Distillation for Answer Explanation. arXiv preprint arXiv:2201.05088 (2022)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012438"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P18-1065"},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of 14th International CINet Conference on Business Development and Co-creation. 241\u2013256","author":"Efeoglu Arkin","year":"2013","unstructured":"Arkin Efeoglu, Charles M\u00f8ller, Michel S\u00e9ri\u00e9, and Harry Boer. 2013. Design thinking: characteristics and promises. In Proceedings of 14th International CINet Conference on Business Development and Co-creation. 241\u2013256."},{"key":"e_1_3_2_1_23_1","unstructured":"Joe Gerber. 2000. How to Prototype a New Business. https:\/\/www.ideou.com\/blogs\/inspiration\/how-to-prototype-a-new-business. Accessed: 2022-05-31."},{"key":"e_1_3_2_1_24_1","volume-title":"Inspiration choices that matter: the selection of external stimuli during ideation. Design Science 2","author":"Gon\u00e7alves Milene","year":"2016","unstructured":"Milene Gon\u00e7alves, Carlos Cardoso, and Petra Badke-Schaub. 2016. Inspiration choices that matter: the selection of external stimuli during ideation. Design Science 2 (2016)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3502004"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.destud.2019.01.001"},{"key":"e_1_3_2_1_27_1","volume-title":"Crowdsourcing new product ideas under consumer learning. Management science 60, 9","author":"Huang Yan","year":"2014","unstructured":"Yan Huang, Param Vir\u00a0Singh, and Kannan Srinivasan. 2014. Crowdsourcing new product ideas under consumer learning. Management science 60, 9 (2014), 2138\u20132159."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3449168"},{"key":"e_1_3_2_1_29_1","volume-title":"The bag of stars: High-speed idea filtering for open innovation. Available at SSRN 2387180","author":"Klein Mark","year":"2014","unstructured":"Mark Klein and Ana\u00a0Cristina Garcia. 2014. The bag of stars: High-speed idea filtering for open innovation. Available at SSRN 2387180 (2014)."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2015.06.005"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3392850"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300641"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2678025.2701399"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10631"},{"key":"e_1_3_2_1_35_1","volume-title":"Harvard Business School Technology & Operations Mgt. Unit Case612-066","author":"Lakhani R","year":"2012","unstructured":"Karim\u00a0R Lakhani, Anne-Laure Fayard, Natalia Levina, and Stephanie\u00a0Healy Pokrywa. 2012. OpenIDEO. Harvard Business School Technology & Operations Mgt. Unit Case612-066 (2012)."},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512137"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Yuyu Lin Jiahao Guo Yang Chen Cheng Yao and Fangtian Ying. 2020. It Is Your Turn: Collaborative Ideation With a Co-Creative Robot through Sketch. In CHI. ACM 1\u201314.","DOI":"10.1145\/3313831.3376258"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Ryan Louie Andy Coenen Cheng\u00a0Zhi Huang Michael Terry and Carrie\u00a0J. Cai. 2020. Novice-AI Music Co-Creation via AI-Steering Tools for Deep Generative Models. In CHI. ACM 1\u201313.","DOI":"10.1145\/3313831.3376739"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274384"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3170427.3188485"},{"volume-title":"Why is that relevant? Collecting annotator rationales for relevance judgments","author":"McDonnell Tyler","key":"e_1_3_2_1_41_1","unstructured":"Tyler McDonnell, Matthew Lease, Mucahid Kutlu, and Tamer Elsayed. 2016. Why is that relevant? Collecting annotator rationales for relevance judgments. In HCOMP. AAAI Press, USA, 139\u2013148."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.destud.2009.07.002"},{"key":"e_1_3_2_1_43_1","unstructured":"Changhoon Oh Jungwoo Song Jinhan Choi Seonghyeon Kim Sungwoo Lee and Bongwon Suh. 2018. I Lead You Help but Only with Enough Details: Understanding User Experience of Co-Creation with Artificial Intelligence. In CHI. ACM 649."},{"key":"e_1_3_2_1_44_1","volume-title":"The value of crowdsourcing: can users really compete with professionals in generating new product ideas?Journal of product innovation management 29, 2","author":"Poetz K","year":"2012","unstructured":"Marion\u00a0K Poetz and Martin Schreier. 2012. The value of crowdsourcing: can users really compete with professionals in generating new product ideas?Journal of product innovation management 29, 2 (2012), 245\u2013256."},{"volume-title":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201335","author":"Rhys\u00a0Cox Samuel","key":"e_1_3_2_1_45_1","unstructured":"Samuel Rhys\u00a0Cox, Yunlong Wang, Ashraf Abdul, Christian Von Der\u00a0Weth, and Brian Y.\u00a0Lim. 2021. Directed diversity: Leveraging language embedding distances for collective creativity in crowd ideation. In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems. 1\u201335."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313547"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.respol.2016.02.003"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/TBDATA.2018.2877350"},{"key":"e_1_3_2_1_50_1","volume-title":"Metrics for measuring ideation effectiveness. Design studies 24, 2","author":"Shah J","year":"2003","unstructured":"Jami\u00a0J Shah, Steve\u00a0M Smith, and Noe Vargas-Hernandez. 2003. Metrics for measuring ideation effectiveness. Design studies 24, 2 (2003), 111\u2013134."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512040"},{"key":"e_1_3_2_1_52_1","volume-title":"Proceedings of the 2008 conference on empirical methods in natural language processing. 254\u2013263","author":"Snow Rion","year":"2008","unstructured":"Rion Snow, Brendan O\u2019connor, Dan Jurafsky, and Andrew\u00a0Y Ng. 2008. Cheap and fast\u2013but is it good? evaluating non-expert annotations for natural language tasks. In Proceedings of the 2008 conference on empirical methods in natural language processing. 254\u2013263."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2008.929620"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.destud.2012.08.003"},{"key":"e_1_3_2_1_55_1","volume-title":"The multidimensional wisdom of crowds. Advances in neural information processing systems 23","author":"Welinder Peter","year":"2010","unstructured":"Peter Welinder, Steve Branson, Pietro Perona, and Serge Belongie. 2010. The multidimensional wisdom of crowds. Advances in neural information processing systems 23 (2010)."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178876.3186033"},{"key":"e_1_3_2_1_57_1","volume-title":"The conference of the North American chapter of the association for computational linguistics; proceedings of the main conference. The Association for Computational Linguistics, 260\u2013267","author":"Zaidan Omar","year":"2007","unstructured":"Omar Zaidan, Jason Eisner, and Christine Piatko. 2007. Using \u201cannotator rationales\u201d to improve machine learning for text categorization. In Human language technologies 2007: The conference of the North American chapter of the association for computational linguistics; proceedings of the main conference. The Association for Computational Linguistics, 260\u2013267."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v3i1.13231"}],"event":{"name":"WWW '23: The ACM Web Conference 2023","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"],"location":"Austin TX USA","acronym":"WWW '23"},"container-title":["Proceedings of the ACM Web Conference 2023"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583496","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3543507.3583496","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:02Z","timestamp":1750178822000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3543507.3583496"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,4,30]]},"references-count":58,"alternative-id":["10.1145\/3543507.3583496","10.1145\/3543507"],"URL":"https:\/\/doi.org\/10.1145\/3543507.3583496","relation":{},"subject":[],"published":{"date-parts":[[2023,4,30]]},"assertion":[{"value":"2023-04-30","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}