{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:30:28Z","timestamp":1750221028484,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":38,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,10,1]],"date-time":"2019-10-01T00:00:00Z","timestamp":1569888000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,10]]},"DOI":"10.1145\/3338286.3347546","type":"proceedings-article","created":{"date-parts":[[2019,10,1]],"date-time":"2019-10-01T14:08:52Z","timestamp":1569938932000},"page":"1-10","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Crowdsourcing Images for Global Diversity"],"prefix":"10.1145","author":[{"given":"Pei-Yu Peggy","family":"Chi","sequence":"first","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthew","family":"Long","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Akshay","family":"Gaur","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Abhimanyu","family":"Deora","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anurag","family":"Batra","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Daphne","family":"Luong","sequence":"additional","affiliation":[{"name":"Google Research, Mountain View, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2019,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1868914.1868921"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240772"},{"key":"e_1_3_2_1_3_1","volume-title":"Open Images Extended - Crowdsourced. Retrieved","author":"Google","year":"2018","unstructured":"Google Crowdsource app users. 2018 . Open Images Extended - Crowdsourced. Retrieved December , 2018 from https:\/\/ai.google\/tools\/datasets\/open-images-extended-crowdsourced\/ Google Crowdsource app users. 2018. Open Images Extended - Crowdsourced. Retrieved December, 2018 from https:\/\/ai.google\/tools\/datasets\/open-images-extended-crowdsourced\/"},{"key":"e_1_3_2_1_4_1","volume-title":"Proceedings of the 24th Annual ACM Symposium on User Interface Software and Technology (UIST '11)","author":"Bernstein Michael S.","year":"2047","unstructured":"Michael S. Bernstein , Joel Brandt , Robert C. Miller , and David R. Karger . 2011. Crowds in Two Seconds: Enabling Realtime Crowd-powered Interfaces . In Proceedings of the 24th Annual ACM Symposium on User Interface Software and Technology (UIST '11) . ACM, New York, NY, USA, 33--42. https:\/\/doi.org\/10.1145\/ 2047 196.2047201 10.1145\/2047196.2047201 Michael S. Bernstein, Joel Brandt, Robert C. Miller, and David R. Karger. 2011. Crowds in Two Seconds: Enabling Realtime Crowd-powered Interfaces. In Proceedings of the 24th Annual ACM Symposium on User Interface Software and Technology (UIST '11). ACM, New York, NY, USA, 33--42. https:\/\/doi.org\/10.1145\/2047196.2047201"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.5555\/1699510.1699548"},{"volume-title":"Proceedings of the 24th International Conference on World Wide Web (WWW '15 Companion). ACM","author":"Chang Shuo","key":"e_1_3_2_1_6_1","unstructured":"Shuo Chang , Peng Dai , Jilin Chen , and Ed H. Chi . 2015. Got Many Labels?: Deriving Topic Labels from Multiple Sources for Social Media Posts Using Crowdsourcing and Ensemble Learning . In Proceedings of the 24th International Conference on World Wide Web (WWW '15 Companion). ACM , New York, NY, USA, 397--406. https:\/\/doi.org\/10.1145\/2740908.2745401 10.1145\/2740908.2745401 Shuo Chang, Peng Dai, Jilin Chen, and Ed H. Chi. 2015. Got Many Labels?: Deriving Topic Labels from Multiple Sources for Social Media Posts Using Crowdsourcing and Ensemble Learning. In Proceedings of the 24th International Conference on World Wide Web (WWW '15 Companion). ACM, New York, NY, USA, 397--406. https:\/\/doi.org\/10.1145\/2740908.2745401"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236112.3236176"},{"key":"e_1_3_2_1_8_1","volume-title":"About The Licenses. Retrieved","author":"Commons Creative","year":"2018","unstructured":"Creative Commons . 2018. About The Licenses. Retrieved September 1, 2018 from https:\/\/creativecommons.org\/licenses\/ Creative Commons. 2018. About The Licenses. Retrieved September 1, 2018 from https:\/\/creativecommons.org\/licenses\/"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2145204.2145396"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2556967"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858094"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130916"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2208320"},{"volume-title":"Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18)","author":"Hara Kotaro","key":"e_1_3_2_1_15_1","unstructured":"Kotaro Hara , Abigail Adams , Kristy Milland , Saiph Savage , Chris Callison-Burch , and Jeffrey P. Bigham . 2018. A Data-Driven Analysis of Workers' Earnings on Amazon Mechanical Turk . In Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18) . ACM, New York, NY, USA, Article 449, 14 pages. https:\/\/doi.org\/10.1145\/3173574.3174023 10.1145\/3173574.3174023 Kotaro Hara, Abigail Adams, Kristy Milland, Saiph Savage, Chris Callison-Burch, and Jeffrey P. Bigham. 2018. A Data-Driven Analysis of Workers' Earnings on Amazon Mechanical Turk. In Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18). ACM, New York, NY, USA, Article 449, 14 pages. https:\/\/doi.org\/10.1145\/3173574.3174023"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025917"},{"volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13)","author":"Irani Lilly C.","key":"e_1_3_2_1_17_1","unstructured":"Lilly C. Irani and M. Six Silberman . 2013. Turkopticon: Interrupting Worker Invisibility in Amazon Mechanical Turk . In Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13) . ACM, New York, NY, USA, 611--620. https:\/\/doi.org\/10.1145\/2470654.2470742 10.1145\/2470654.2470742 Lilly C. Irani and M. Six Silberman. 2013. Turkopticon: Interrupting Worker Invisibility in Amazon Mechanical Turk. In Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13). ACM, New York, NY, USA, 611--620. https:\/\/doi.org\/10.1145\/2470654.2470742"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2141512.2141556"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173670"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1518930"},{"key":"e_1_3_2_1_21_1","volume-title":"OpenImages: A public dataset for large-scale multi-label and multi-class image classification. Dataset available from https:\/\/storage.googleapis.com\/openimages\/web\/index.html","author":"Krasin Ivan","year":"2017","unstructured":"Ivan Krasin , Tom Duerig , Neil Alldrin , Vittorio Ferrari , Sami Abu-El-Haija , Alina Kuznetsova , Hassan Rom , Jasper Uijlings , Stefan Popov , Shahab Kamali , Matteo Malloci , Jordi Pont-Tuset , Andreas Veit , Serge Belongie , Victor Gomes , Abhinav Gupta , Chen Sun , Gal Chechik , David Cai , Zheyun Feng , Dhyanesh Narayanan , and Kevin Murphy . 2017. OpenImages: A public dataset for large-scale multi-label and multi-class image classification. Dataset available from https:\/\/storage.googleapis.com\/openimages\/web\/index.html ( 2017 ). Ivan Krasin, Tom Duerig, Neil Alldrin, Vittorio Ferrari, Sami Abu-El-Haija, Alina Kuznetsova, Hassan Rom, Jasper Uijlings, Stefan Popov, Shahab Kamali, Matteo Malloci, Jordi Pont-Tuset, Andreas Veit, Serge Belongie, Victor Gomes, Abhinav Gupta, Chen Sun, Gal Chechik, David Cai, Zheyun Feng, Dhyanesh Narayanan, and Kevin Murphy. 2017. OpenImages: A public dataset for large-scale multi-label and multi-class image classification. Dataset available from https:\/\/storage.googleapis.com\/openimages\/web\/index.html (2017)."},{"key":"e_1_3_2_1_22_1","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13)","author":"Lasecki Walter S.","year":"2033","unstructured":"Walter S. Lasecki , Christopher D. Miller , and Jeffrey P. Bigham . 2013. Warping Time for More Effective Real-time Crowdsourcing . In Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13) . ACM, New York, NY, USA , 2033 --2036. https:\/\/doi.org\/10.1145\/2470654.2466269 10.1145\/2470654.2466269 Walter S. Lasecki, Christopher D. Miller, and Jeffrey P. Bigham. 2013. Warping Time for More Effective Real-time Crowdsourcing. In Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (CHI '13). ACM, New York, NY, USA, 2033--2036. https:\/\/doi.org\/10.1145\/2470654.2466269"},{"volume-title":"Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18)","author":"Mahyar Narges","key":"e_1_3_2_1_23_1","unstructured":"Narges Mahyar , Michael R. James , Michelle M. Ng , Reginald A. Wu , and Steven P. Dow . 2018. CommunityCrit: Inviting the Public to Improve and Evaluate Urban Design Ideas Through Micro-Activities . In Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18) . ACM, New York, NY, USA, Article 195, 14 pages. https:\/\/doi.org\/10.1145\/3173574.3173769 10.1145\/3173574.3173769 Narges Mahyar, Michael R.James, Michelle M. Ng, Reginald A. Wu, and Steven P. Dow. 2018. CommunityCrit: Inviting the Public to Improve and Evaluate Urban Design Ideas Through Micro-Activities. In Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems (CHI '18). ACM, New York, NY, USA, Article 195, 14 pages. https:\/\/doi.org\/10.1145\/3173574.3173769"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1978942.1979366"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531663"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858539"},{"key":"e_1_3_2_1_27_1","volume-title":"The 90-9-1 Rule for Participation Inequality in Social Media and Online Communities. Retrieved","author":"Nielsen Jakob","year":"2018","unstructured":"Jakob Nielsen . 2006. The 90-9-1 Rule for Participation Inequality in Social Media and Online Communities. Retrieved September 1, 2018 from https:\/\/www.nngroup.com\/articles\/participation-inequality\/ Jakob Nielsen. 2006. The 90-9-1 Rule for Participation Inequality in Social Media and Online Communities. Retrieved September 1, 2018 from https:\/\/www.nngroup.com\/articles\/participation-inequality\/"},{"key":"e_1_3_2_1_28_1","volume-title":"We don't need no bounding-boxes: Training object class detectors using only human verification. CoRR abs\/1602.08405","author":"Papadopoulos Dim P.","year":"2016","unstructured":"Dim P. Papadopoulos , Jasper R. R. Uijlings , Frank Keller , and Vittorio Ferrari . 2016. We don't need no bounding-boxes: Training object class detectors using only human verification. CoRR abs\/1602.08405 ( 2016 ). arXiv:1602.08405 http:\/\/arxiv.org\/abs\/1602.08405 Dim P. Papadopoulos, Jasper R. R. Uijlings, Frank Keller, and Vittorio Ferrari. 2016. We don't need no bounding-boxes: Training object class detectors using only human verification. CoRR abs\/1602.08405 (2016). arXiv:1602.08405 http:\/\/arxiv.org\/abs\/1602.08405"},{"key":"e_1_3_2_1_29_1","volume-title":"Extreme clicking for efficient object annotation. CoRR abs\/1708.02750","author":"Papadopoulos Dim P.","year":"2017","unstructured":"Dim P. Papadopoulos , Jasper R. R. Uijlings , Frank Keller , and Vittorio Ferrari . 2017. Extreme clicking for efficient object annotation. CoRR abs\/1708.02750 ( 2017 ). arXiv:1708.02750 http:\/\/arxiv.org\/abs\/1708.02750 Dim P. Papadopoulos, Jasper R. R. Uijlings, Frank Keller, and Vittorio Ferrari. 2017. Extreme clicking for efficient object annotation. CoRR abs\/1708.02750 (2017). arXiv:1708.02750 http:\/\/arxiv.org\/abs\/1708.02750"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-007-0090-8"},{"volume-title":"NIPS 2017 workshop: Machine Learning for the Developing World.","author":"Shankar Shreya","key":"e_1_3_2_1_31_1","unstructured":"Shreya Shankar , Yoni Halpern , Eric Breck , James Atwood , Jimbo Wilson , and D. Sculley . 2017. No Classification without Representation: Assessing Geodiversity Issues in Open Data Sets for the Developing World . In NIPS 2017 workshop: Machine Learning for the Developing World. Shreya Shankar, Yoni Halpern, Eric Breck, James Atwood, Jimbo Wilson, and D. Sculley. 2017. No Classification without Representation: Assessing Geodiversity Issues in Open Data Sets for the Developing World. In NIPS 2017 workshop: Machine Learning for the Developing World."},{"key":"e_1_3_2_1_32_1","volume-title":"Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP '10)","author":"Stewart Osamuyimen","year":"1837","unstructured":"Osamuyimen Stewart , David Lubensky , and Juan M. Huerta . 2010. Crowdsourcing Participation Inequality: A SCOUT Model for the Enterprise Domain . In Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP '10) . ACM, New York, NY, USA, 30--33. https:\/\/doi.org\/10.1145\/ 1837 885.1837895 10.1145\/1837885.1837895 Osamuyimen Stewart, David Lubensky, and Juan M. Huerta. 2010. Crowdsourcing Participation Inequality: A SCOUT Model for the Enterprise Domain. In Proceedings of the ACM SIGKDD Workshop on Human Computation (HCOMP '10). ACM, New York, NY, USA, 30--33. https:\/\/doi.org\/10.1145\/1837885.1837895"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2037373.2037438"},{"volume-title":"Proceedings of the 30th Annual ACM Symposium on User Interface Software and Technology (UIST'17)","author":"Vaish Rajan","key":"e_1_3_2_1_34_1","unstructured":"Rajan Vaish , Snehalkumar (Neil) S. Gaikwad , Geza Kovacs , Andreas Veit , Ranjay Krishna , Imanol Arrieta Ibarra , Camelia Simoiu , Michael Wilber , Serge Belongie , Sharad Goel , James Davis , and Michael S. Bernstein . 2017. Crowd Research: Open and Scalable University Laboratories . In Proceedings of the 30th Annual ACM Symposium on User Interface Software and Technology (UIST'17) . ACM, New York, NY, USA, 829--843. https:\/\/doi.org\/10.1145\/3126594.3126648 10.1145\/3126594.3126648 Rajan Vaish, Snehalkumar (Neil) S. Gaikwad, Geza Kovacs, Andreas Veit, Ranjay Krishna, Imanol Arrieta Ibarra, Camelia Simoiu, Michael Wilber, Serge Belongie, Sharad Goel, James Davis, and Michael S. Bernstein. 2017. Crowd Research: Open and Scalable University Laboratories. In Proceedings of the 30th Annual ACM Symposium on User Interface Software and Technology (UIST'17). ACM, New York, NY, USA, 829--843. https:\/\/doi.org\/10.1145\/3126594.3126648"},{"volume-title":"Proceedings of the 32Nd Annual ACM Conference on Human Factors in Computing Systems (CHI '14)","author":"Vaish Rajan","key":"e_1_3_2_1_35_1","unstructured":"Rajan Vaish , Keith Wyngarden , Jingshu Chen , Brandon Cheung , and Michael S. Bernstein . 2014. Twitch Crowdsourcing: Crowd Contributions in Short Bursts of Time . In Proceedings of the 32Nd Annual ACM Conference on Human Factors in Computing Systems (CHI '14) . ACM, New York, NY, USA, 3645--3654. https:\/\/doi.org\/10.1145\/2556288.2556996 10.1145\/2556288.2556996 Rajan Vaish, Keith Wyngarden, Jingshu Chen, Brandon Cheung, and Michael S. Bernstein. 2014. Twitch Crowdsourcing: Crowd Contributions in Short Bursts of Time. In Proceedings of the 32Nd Annual ACM Conference on Human Factors in Computing Systems (CHI '14). ACM, New York, NY, USA, 3645--3654. https:\/\/doi.org\/10.1145\/2556288.2556996"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2531602.2531659"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2014.2316116"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"}],"event":{"name":"MobileHCI '19: 21st International Conference on Human-Computer Interaction with Mobile Devices and Services","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Taipei Taiwan","acronym":"MobileHCI '19"},"container-title":["Proceedings of the 21st International Conference on Human-Computer Interaction with Mobile Devices and Services"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3338286.3347546","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3338286.3347546","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T00:43:21Z","timestamp":1750207401000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3338286.3347546"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,10]]},"references-count":38,"alternative-id":["10.1145\/3338286.3347546","10.1145\/3338286"],"URL":"https:\/\/doi.org\/10.1145\/3338286.3347546","relation":{},"subject":[],"published":{"date-parts":[[2019,10]]},"assertion":[{"value":"2019-10-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}