{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,4]],"date-time":"2025-07-04T05:56:34Z","timestamp":1751608594025,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":36,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,10,30]],"date-time":"2023-10-30T00:00:00Z","timestamp":1698624000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["DGE-1842487, 2046230"],"award-info":[{"award-number":["DGE-1842487, 2046230"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,10,30]]},"DOI":"10.1145\/3617694.3623254","type":"proceedings-article","created":{"date-parts":[[2023,10,29]],"date-time":"2023-10-29T15:52:11Z","timestamp":1698594731000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Deploying a Robust Active Preference Elicitation Algorithm on MTurk: Experiment Design, Interface, and Evaluation for COVID-19 Patient Prioritization"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0002-6219-7606","authenticated-orcid":false,"given":"Caroline M","family":"Johnston","sequence":"first","affiliation":[{"name":"USC Center for AI in Society, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9506-0940","authenticated-orcid":false,"given":"Patrick","family":"Vossler","sequence":"additional","affiliation":[{"name":"USC Center for AI in Society, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1762-7064","authenticated-orcid":false,"given":"Simon","family":"Blessenohl","sequence":"additional","affiliation":[{"name":"USC Center for AI in Society, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7800-7235","authenticated-orcid":false,"given":"Phebe","family":"Vayanos","sequence":"additional","affiliation":[{"name":"USC Center for AI in Society, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,10,30]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Decision making under uncertainty when preference information is incomplete. Management Science 61, 1 (1","author":"Armbruster Benjamin","year":"2015","unstructured":"Benjamin Armbruster and Erick Delage . 2015. Decision making under uncertainty when preference information is incomplete. Management Science 61, 1 (1 2015 ), 111\u2013128. https:\/\/doi.org\/10.1287\/mnsc.2014.2059 10.1287\/mnsc.2014.2059 Benjamin Armbruster and Erick Delage. 2015. Decision making under uncertainty when preference information is incomplete. Management Science 61, 1 (1 2015), 111\u2013128. https:\/\/doi.org\/10.1287\/mnsc.2014.2059"},{"key":"e_1_3_2_1_2_1","volume-title":"The Moral Machine experiment. Nature 563, 7729 (11","author":"Awad Edmond","year":"2018","unstructured":"Edmond Awad , Sohan Dsouza , Richard Kim , Jonathan Schulz , Joseph Henrich , Azim Shariff , Jean\u00a0Fran\u00e7ois Bonnefon , and Iyad Rahwan . 2018. The Moral Machine experiment. Nature 563, 7729 (11 2018 ), 59\u201364. https:\/\/doi.org\/10.1038\/s41586-018-0637-6 10.1038\/s41586-018-0637-6 Edmond Awad, Sohan Dsouza, Richard Kim, Jonathan Schulz, Joseph Henrich, Azim Shariff, Jean\u00a0Fran\u00e7ois Bonnefon, and Iyad Rahwan. 2018. The Moral Machine experiment. Nature 563, 7729 (11 2018), 59\u201364. https:\/\/doi.org\/10.1038\/s41586-018-0637-6"},{"key":"e_1_3_2_1_3_1","volume-title":"Evaluating online labor markets for experimental research: Amazon.com\u2019s mechanical turk. Political Analysis 20, 3 (7","author":"Berinsky J.","year":"2012","unstructured":"Adam\u00a0 J. Berinsky , Gregory\u00a0 A. Huber , and Gabriel\u00a0 S. Lenz . 2012. Evaluating online labor markets for experimental research: Amazon.com\u2019s mechanical turk. Political Analysis 20, 3 (7 2012 ), 351\u2013368. https:\/\/doi.org\/10.1093\/pan\/mpr057 10.1093\/pan Adam\u00a0J. Berinsky, Gregory\u00a0A. Huber, and Gabriel\u00a0S. Lenz. 2012. Evaluating online labor markets for experimental research: Amazon.com\u2019s mechanical turk. Political Analysis 20, 3 (7 2012), 351\u2013368. https:\/\/doi.org\/10.1093\/pan\/mpr057"},{"key":"e_1_3_2_1_4_1","volume-title":"On the efficiency-fairness trade-off. Management Science 58, 12 (12","author":"Bertsimas Dimitris","year":"2012","unstructured":"Dimitris Bertsimas , Vivek\u00a0 F. Farias , and Nikolaos Trichakis . 2012. On the efficiency-fairness trade-off. Management Science 58, 12 (12 2012 ), 2234\u20132250. https:\/\/doi.org\/10.1287\/mnsc.1120.1549 10.1287\/mnsc.1120.1549 Dimitris Bertsimas, Vivek\u00a0F. Farias, and Nikolaos Trichakis. 2012. On the efficiency-fairness trade-off. Management Science 58, 12 (12 2012), 2234\u20132250. https:\/\/doi.org\/10.1287\/mnsc.1120.1549"},{"key":"e_1_3_2_1_5_1","volume-title":"Learning preferences under noise and loss aversion: An optimization approach. Operations Research 61, 5 (9","author":"Bertsimas Dimitris","year":"2013","unstructured":"Dimitris Bertsimas and Allison O\u2019Hair . 2013. Learning preferences under noise and loss aversion: An optimization approach. Operations Research 61, 5 (9 2013 ), 1190\u20131199. https:\/\/doi.org\/10.1287\/opre.2013.1209 10.1287\/opre.2013.1209 Dimitris Bertsimas and Allison O\u2019Hair. 2013. Learning preferences under noise and loss aversion: An optimization approach. Operations Research 61, 5 (9 2013), 1190\u20131199. https:\/\/doi.org\/10.1287\/opre.2013.1209"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.5555\/777092.777132"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2006.02.003"},{"key":"e_1_3_2_1_8_1","volume-title":"Machine Learning-powered Iterative Combinatorial Auctions. (11","author":"Brero Gianluca","year":"2019","unstructured":"Gianluca Brero , Benjamin Lubin , and Sven Seuken . 2019. Machine Learning-powered Iterative Combinatorial Auctions. (11 2019 ). http:\/\/arxiv.org\/abs\/1911.08042 Gianluca Brero, Benjamin Lubin, and Sven Seuken. 2019. Machine Learning-powered Iterative Combinatorial Auctions. (11 2019). http:\/\/arxiv.org\/abs\/1911.08042"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the Seventeenth National Conference on Artificial Intelligence. 363\u2013369","author":"Chajewska Urszula","year":"2000","unstructured":"Urszula Chajewska , Daphne Koller , and Ronald Parr . 2000 . Making Rational Decisions using Adaptive Utility Elicitation . In Proceedings of the Seventeenth National Conference on Artificial Intelligence. 363\u2013369 . Urszula Chajewska, Daphne Koller, and Ronald Parr. 2000. Making Rational Decisions using Adaptive Utility Elicitation. In Proceedings of the Seventeenth National Conference on Artificial Intelligence. 363\u2013369."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF00994106"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the 11th ACM conference on Electronic commerce. Association for Computing Machinery, 219\u2013228","author":"Braziunas Darius","year":"2010","unstructured":"Darius Braziunas and Craig Boutilier . 2010 . Assessing regret-based preference elicitation with the UTPREF recommendation system. In EC \u201910 : Proceedings of the 11th ACM conference on Electronic commerce. Association for Computing Machinery, 219\u2013228 . Darius Braziunas and Craig Boutilier. 2010. Assessing regret-based preference elicitation with the UTPREF recommendation system. In EC \u201910: Proceedings of the 11th ACM conference on Electronic commerce. Association for Computing Machinery, 219\u2013228."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMsb2005114"},{"key":"e_1_3_2_1_13_1","unstructured":"Mattia Ferraresi. 2020. A coronavirus cautionary tale from Italy: Don\u2019t do what we did. https:\/\/www.bostonglobe.com\/2020\/03\/13\/opinion\/coronavirus-cautionary-tale-italy-dont-do-what-we-did\/  Mattia Ferraresi. 2020. A coronavirus cautionary tale from Italy: Don\u2019t do what we did. https:\/\/www.bostonglobe.com\/2020\/03\/13\/opinion\/coronavirus-cautionary-tale-italy-dont-do-what-we-did\/"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1257\/089533005775196732"},{"key":"e_1_3_2_1_15_1","volume-title":"Adapting a Kidney Exchange Algorithm to Align With Human Values. Artificial Intelligence 283","author":"Freedman Rachel","year":"2020","unstructured":"Rachel Freedman , Jana\u00a0Schaich Borg , Walter Sinnott-Armstrong , John\u00a0 P. Dickerson , and Vincent Conitzer . 2020. Adapting a Kidney Exchange Algorithm to Align With Human Values. Artificial Intelligence 283 ( 2020 ). Rachel Freedman, Jana\u00a0Schaich Borg, Walter Sinnott-Armstrong, John\u00a0P. Dickerson, and Vincent Conitzer. 2020. Adapting a Kidney Exchange Algorithm to Align With Human Values. Artificial Intelligence 283 (2020)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1136\/amiajnl-2011-000089"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1086\/208721"},{"key":"e_1_3_2_1_18_1","volume-title":"Managing mental health challenges faced by healthcare workers during covid-19 pandemic. BMJ 368 (3","author":"Greenberg Neil","year":"2020","unstructured":"Neil Greenberg , Mary Docherty , Sam Gnanapragasam , and Simon Wessely . 2020. Managing mental health challenges faced by healthcare workers during covid-19 pandemic. BMJ 368 (3 2020 ). https:\/\/doi.org\/10.1136\/bmj.m1211 10.1136\/bmj.m1211 Neil Greenberg, Mary Docherty, Sam Gnanapragasam, and Simon Wessely. 2020. Managing mental health challenges faced by healthcare workers during covid-19 pandemic. BMJ 368 (3 2020). https:\/\/doi.org\/10.1136\/bmj.m1211"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Joel Ross Lilly Irani M. Six Silberman Andrew Zaldivar and Bill Tomlinson. 2010. Who are the crowdworkers? Shifting demographics in Mechanical Turk. In CHI EA \u201910: CHI \u201910 Extended Abstracts on Human Factors in Computing Systems. 2863\u20132872.  Joel Ross Lilly Irani M. Six Silberman Andrew Zaldivar and Bill Tomlinson. 2010. Who are the crowdworkers? Shifting demographics in Mechanical Turk. In CHI EA \u201910: CHI \u201910 Extended Abstracts on Human Factors in Computing Systems. 2863\u20132872.","DOI":"10.1145\/1753846.1753873"},{"key":"e_1_3_2_1_20_1","volume-title":"Workshop on Participatory Approaches to Machine Learning","author":"Johnston M.","year":"2020","unstructured":"Caroline\u00a0 M. Johnston , Simon Blessenohl , and Phebe Vayanos . 2020 . Preference Elicitation and Aggregation to Aid with Patient Triage during the COVID-19 Pandemic . Workshop on Participatory Approaches to Machine Learning (2020). Caroline\u00a0M. Johnston, Simon Blessenohl, and Phebe Vayanos. 2020. Preference Elicitation and Aggregation to Aid with Patient Triage during the COVID-19 Pandemic. Workshop on Participatory Approaches to Machine Learning (2020)."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3544548.3580882"},{"key":"e_1_3_2_1_22_1","first-page":"1","article-title":"WeBuildAI: Participatory framework for algorithmic governance","volume":"3","author":"Noothigattu Ritesh","year":"2019","unstructured":"Min\u00a0Kyung Lee, Daniel Kusbit, Anson Kahng, Ji\u00a0Tae Kim, Xinran Yuan, Allissa Chan, Daniel See, Ritesh Noothigattu , Siheon Lee , Alexandros Psomas , and Ariel\u00a0 D. Procaccia . 2019 . WeBuildAI: Participatory framework for algorithmic governance . Proceedings of the ACM on Human-Computer Interaction 3 (2019), 1 \u2013 35 . https:\/\/doi.org\/10.1145\/3359283 10.1145\/3359283 Min\u00a0Kyung Lee, Daniel Kusbit, Anson Kahng, Ji\u00a0Tae Kim, Xinran Yuan, Allissa Chan, Daniel See, Ritesh Noothigattu, Siheon Lee, Alexandros Psomas, and Ariel\u00a0D. Procaccia. 2019. WeBuildAI: Participatory framework for algorithmic governance. Proceedings of the ACM on Human-Computer Interaction 3 (2019), 1\u201335. https:\/\/doi.org\/10.1145\/3359283","journal-title":"Proceedings of the ACM on Human-Computer Interaction"},{"key":"e_1_3_2_1_23_1","volume-title":"Indecision Modeling. In Proceedings of the AAAI Conference on Artificial Intelligence (AAAI). http:\/\/arxiv.org\/abs\/2012","author":"McElfresh C.","year":"2021","unstructured":"Duncan\u00a0 C. McElfresh , Lok Chan , Kenzie Doyle , Walter Sinnott-Armstrong , Vincent Conitzer , Jana\u00a0Schaich Borg , and John\u00a0 P. Dickerson . 2021 . Indecision Modeling. In Proceedings of the AAAI Conference on Artificial Intelligence (AAAI). http:\/\/arxiv.org\/abs\/2012 .08485 Duncan\u00a0C. McElfresh, Lok Chan, Kenzie Doyle, Walter Sinnott-Armstrong, Vincent Conitzer, Jana\u00a0Schaich Borg, and John\u00a0P. Dickerson. 2021. Indecision Modeling. In Proceedings of the AAAI Conference on Artificial Intelligence (AAAI). http:\/\/arxiv.org\/abs\/2012.08485"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence, Vol.\u00a01. 122\u2013128","author":"Pfeiffer Thomas","year":"2012","unstructured":"Thomas Pfeiffer , Xi\u00a0Alice Gao , Andrew Mao , Yiling Chen , and David\u00a0 G. Rand . 2012 . Adaptive polling for information aggregation . In Proceedings of the AAAI Conference on Artificial Intelligence, Vol.\u00a01. 122\u2013128 . Thomas Pfeiffer, Xi\u00a0Alice Gao, Andrew Mao, Yiling Chen, and David\u00a0G. Rand. 2012. Adaptive polling for information aggregation. In Proceedings of the AAAI Conference on Artificial Intelligence, Vol.\u00a01. 122\u2013128."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMp2006141"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.4103\/0972-5229.130573"},{"key":"e_1_3_2_1_27_1","volume-title":"Use of discrete choice experiments to elicit preferences. BMJ Quality in Health Care","author":"Ryan Mandy","year":"2001","unstructured":"Mandy Ryan , Angela Bate , Clifford\u00a0 J. Eastmond , and Anne Ludbrook . 2001. Use of discrete choice experiments to elicit preferences. BMJ Quality in Health Care ( 2001 ), i55\u2013i60. Mandy Ryan, Angela Bate, Clifford\u00a0J. Eastmond, and Anne Ludbrook. 2001. Use of discrete choice experiments to elicit preferences. BMJ Quality in Health Care (2001), i55\u2013i60."},{"volume-title":"Combinatorial Auctions","author":"Sandholm Tuomas","key":"e_1_3_2_1_28_1","unstructured":"Tuomas Sandholm and Craig Boutilier . 2006. Preference Elicitation in Combinatorial Auctions . In Combinatorial Auctions . The MIT Press , Chapter\u00a010, 233\u2013263. Tuomas Sandholm and Craig Boutilier. 2006. Preference Elicitation in Combinatorial Auctions. In Combinatorial Auctions. The MIT Press, Chapter\u00a010, 233\u2013263."},{"key":"e_1_3_2_1_29_1","first-page":"295","article-title":"Ellipsoidal methods for adaptive choice-based conjoint analysis","volume":"67","author":"Saur\u00e9 Denis","year":"2019","unstructured":"Denis Saur\u00e9 and Juan\u00a0Pablo Vielma . 2019 . Ellipsoidal methods for adaptive choice-based conjoint analysis . Operations Research 67 (2019), 295 \u2013 597 . Denis Saur\u00e9 and Juan\u00a0Pablo Vielma. 2019. Ellipsoidal methods for adaptive choice-based conjoint analysis. Operations Research 67 (2019), 295\u2013597.","journal-title":"Operations Research"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3531146.3533110"},{"key":"e_1_3_2_1_31_1","volume-title":"Probabilistic polyhedral methods for adaptive choice-based conjoint analysis: Theory and application. Marketing Science 26, 5 (9","author":"Toubia Olivier","year":"2007","unstructured":"Olivier Toubia , John Hauser , and Rosanna Garcia . 2007. Probabilistic polyhedral methods for adaptive choice-based conjoint analysis: Theory and application. Marketing Science 26, 5 (9 2007 ), 596\u2013610. https:\/\/doi.org\/10.1287\/mksc.1060.0257 10.1287\/mksc.1060.0257 Olivier Toubia, John Hauser, and Rosanna Garcia. 2007. Probabilistic polyhedral methods for adaptive choice-based conjoint analysis: Theory and application. Marketing Science 26, 5 (9 2007), 596\u2013610. https:\/\/doi.org\/10.1287\/mksc.1060.0257"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1509\/jmkr.41.1.116.25082"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/2805249.2805251"},{"key":"e_1_3_2_1_34_1","volume-title":"Robust Active Preference Elicitation. (3","author":"Vayanos Phebe","year":"2020","unstructured":"Phebe Vayanos , Yingxiao Ye , Duncan\u00a0 C. McElfresh , John\u00a0 P. Dickerson , and Eric Rice . 2020. Robust Active Preference Elicitation. (3 2020 ). http:\/\/arxiv.org\/abs\/2003.01899 Phebe Vayanos, Yingxiao Ye, Duncan\u00a0C. McElfresh, John\u00a0P. Dickerson, and Eric Rice. 2020. Robust Active Preference Elicitation. (3 2020). http:\/\/arxiv.org\/abs\/2003.01899"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357236.3395528"},{"key":"e_1_3_2_1_36_1","volume-title":"A Cost-Effective Framework for Preference Elicitation and Aggregation. (5","author":"Zhao Zhibing","year":"2018","unstructured":"Zhibing Zhao , Haoming Li , Junming Wang , Jeffrey Kephart , Nicholas Mattei , Hui Su , and Lirong Xia . 2018. A Cost-Effective Framework for Preference Elicitation and Aggregation. (5 2018 ). http:\/\/arxiv.org\/abs\/1805.05287 Zhibing Zhao, Haoming Li, Junming Wang, Jeffrey Kephart, Nicholas Mattei, Hui Su, and Lirong Xia. 2018. A Cost-Effective Framework for Preference Elicitation and Aggregation. (5 2018). http:\/\/arxiv.org\/abs\/1805.05287"}],"event":{"name":"EAAMO '23: Equity and Access in Algorithms, Mechanisms, and Optimization","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGecom Special Interest Group on Economics and Computation"],"location":"Boston MA USA","acronym":"EAAMO '23"},"container-title":["Equity and Access in Algorithms, Mechanisms, and Optimization"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3617694.3623254","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3617694.3623254","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:36:32Z","timestamp":1750178192000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3617694.3623254"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,10,30]]},"references-count":36,"alternative-id":["10.1145\/3617694.3623254","10.1145\/3617694"],"URL":"https:\/\/doi.org\/10.1145\/3617694.3623254","relation":{},"subject":[],"published":{"date-parts":[[2023,10,30]]},"assertion":[{"value":"2023-10-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}