{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,19]],"date-time":"2025-12-19T21:14:24Z","timestamp":1766178864862,"version":"3.48.0"},"publisher-location":"New York, NY, USA","reference-count":28,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,3,25]],"date-time":"2024-03-25T00:00:00Z","timestamp":1711324800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,3,25]]},"DOI":"10.1145\/3649792.3649806","type":"proceedings-article","created":{"date-parts":[[2024,5,3]],"date-time":"2024-05-03T17:57:15Z","timestamp":1714759035000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Quantifying and Leveraging Uncertain and Imprecise Answers in Multiple Choice Questionnaires for Crowdsourcing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6680-2837","authenticated-orcid":false,"given":"Constance","family":"Thierry","sequence":"first","affiliation":[{"name":"DRUID - Univ Rennes, CNRS, IRISA, Univ. Rennes, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1905-815X","authenticated-orcid":false,"given":"G\u00e9ry","family":"Casiez","sequence":"additional","affiliation":[{"name":"Univ. Lille, CNRS, Inria, Centrale Lille, UMR 9189 CRIStAL, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6099-9050","authenticated-orcid":false,"given":"Jean-Christophe","family":"Dubois","sequence":"additional","affiliation":[{"name":"DRUID - Univ Rennes, CNRS, IRISA, Univ. Rennes, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6869-8407","authenticated-orcid":false,"given":"Yolande","family":"Le-Gall","sequence":"additional","affiliation":[{"name":"DRUID - Univ Rennes, CNRS, IRISA, Univ. Rennes, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5201-5875","authenticated-orcid":false,"given":"Sylvain","family":"Malacria","sequence":"additional","affiliation":[{"name":"Univ. Lille, Inria, CNRS, Centrale Lille, UMR 9189 CRIStAL, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2013-7253","authenticated-orcid":false,"given":"Thomas","family":"Pietrzak","sequence":"additional","affiliation":[{"name":"Universit\u00e9 de Lille, France"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0882-0153","authenticated-orcid":false,"given":"Arnaud","family":"Martin","sequence":"additional","affiliation":[{"name":"DRUID - Univ Rennes, CNRS, IRISA, Univ. Rennes, France"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-1333-3433","authenticated-orcid":false,"given":"Pierrick","family":"Uro","sequence":"additional","affiliation":[{"name":"Univ. Lille, France"}]}],"member":"320","published-online":{"date-parts":[[2024,5,3]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10489-018-1209-z"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465318"},{"key":"e_1_3_2_1_3_1","unstructured":"AMT [n.d.]. Amazon Mechanical Turk. https:\/\/docs.aws.amazon.com\/AWSMechTurk\/latest\/ RequesterUI\/CreatingYourBatchofHITs.htmlAccessed: 17\/12\/2020."},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems -","volume":"1","author":"Bachrach Yoram","year":"2012","unstructured":"Yoram Bachrach, Thore Graepel, Gjergji Kasneci, Michal Kosinski, and Jurgen Van\u00a0Gael. 2012. Crowd IQ: Aggregating Opinions to Boost Performance. In Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems - Volume 1 (Valencia, Spain) (AAMAS \u201912). International Foundation for Autonomous Agents and Multiagent Systems, Richland, SC, 535\u2013542."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2012.122"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047276"},{"key":"e_1_3_2_1_7_1","volume-title":"Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied statistics","author":"Dawid Alexander\u00a0Philip","year":"1979","unstructured":"Alexander\u00a0Philip Dawid and Allan\u00a0M Skene. 1979. Maximum likelihood estimation of observer error-rates using the EM algorithm. Applied statistics (1979), 20\u201328."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-44792-4_3"},{"key":"e_1_3_2_1_9_1","series-title":"Series B (methodological)","volume-title":"Maximum likelihood from incomplete data via the EM algorithm. Journal of the royal statistical society","author":"Dempster P","year":"1977","unstructured":"Arthur\u00a0P Dempster, Nan\u00a0M Laird, and Donald\u00a0B Rubin. 1977. Maximum likelihood from incomplete data via the EM algorithm. Journal of the royal statistical society. Series B (methodological) (1977), 1\u201338."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-87605-2_17"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145355"},{"key":"e_1_3_2_1_12_1","unstructured":"Graham Farrell. 2006. A comparison of an innovative web-based assessment tool utilizing confidence measurement to the traditional multiple choice short answer and problem solving questions. https:\/\/hdl.handle.net\/2134\/4587"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aci.2014.07.001"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9205-0"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1080\/01421590031075"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/NAFIPS.2016.7851590"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1561\/1900000044"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-03643-0_4"},{"key":"e_1_3_2_1_20_1","volume-title":"Characterization of experts in crowdsourcing platforms. Belief Functions: Theory and Applications. 9861","author":"Mouloud Amal Ben","year":"2016","unstructured":"Amal Ben Rjab; Mouloud Kharoune; Zoltan Miklos;\u00a0Arnaud Martin. 2016. Characterization of experts in crowdsourcing platforms. Belief Functions: Theory and Applications. 9861 (2016)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v3i1.13225"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2598153.2602248"},{"key":"e_1_3_2_1_23_1","volume-title":"Annotation models for crowdsourced ordinal data. Journal of Machine Learning Research 13","author":"Raykar C","year":"2012","unstructured":"Vikas\u00a0C Raykar and Shipeng Yu. 2012. Annotation models for crowdsourced ordinal data. Journal of Machine Learning Research 13 (2012)."},{"volume-title":"A mathematical theory of evidence. Vol.\u00a042","author":"Shafer Glenn","key":"e_1_3_2_1_24_1","unstructured":"Glenn Shafer. 1976. A mathematical theory of evidence. Vol.\u00a042. Princeton university press."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958865"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICTAI.2019.00112"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.5555\/2997046.2997166"},{"key":"e_1_3_2_1_28_1","volume-title":"Advances in Neural Information Processing Systems, Y.\u00a0Bengio, D.\u00a0Schuurmans, J.\u00a0Lafferty, C.\u00a0Williams, and A.\u00a0Culotta (Eds.). Vol.\u00a022. Curran Associates","author":"Whitehill Jacob","year":"2035","unstructured":"Jacob Whitehill, Ting-fan Wu, Jacob Bergsma, Javier Movellan, and Paul Ruvolo. 2009. Whose Vote Should Count More: Optimal Integration of Labels from Labelers of Unknown Expertise. In Advances in Neural Information Processing Systems, Y.\u00a0Bengio, D.\u00a0Schuurmans, J.\u00a0Lafferty, C.\u00a0Williams, and A.\u00a0Culotta (Eds.). Vol.\u00a022. Curran Associates, Inc., 2035\u20132043. https:\/\/proceedings.neurips.cc\/paper\/2009\/file\/f899139df5e1059396431415e770c6dd-Paper.pdf"}],"event":{"name":"IHM '24: 35th International Francophone Conference on Human-Computer Interaction","acronym":"IHM '24","location":"Paris France"},"container-title":["Proceedings of the 35th Conference on l'Interaction Humain-Machine"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649792.3649806","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3649792.3649806","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,12,19]],"date-time":"2025-12-19T21:11:54Z","timestamp":1766178714000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3649792.3649806"}},"subtitle":["Quantifier et exploiter des r\u00e9ponses incertaines et impr\u00e9cises dans des questionnaires \u00e0 choix multiples pour le crowdsourcing"],"short-title":[],"issued":{"date-parts":[[2024,3,25]]},"references-count":28,"alternative-id":["10.1145\/3649792.3649806","10.1145\/3649792"],"URL":"https:\/\/doi.org\/10.1145\/3649792.3649806","relation":{},"subject":[],"published":{"date-parts":[[2024,3,25]]},"assertion":[{"value":"2024-05-03","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}