{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,24]],"date-time":"2026-09-24T23:49:31Z","timestamp":1790293771610,"version":"4.1.0"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2018,5,12]],"date-time":"2018-05-12T00:00:00Z","timestamp":1526083200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61690201)"],"award-info":[{"award-number":["61690201)"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2019,4]]},"DOI":"10.1007\/s10664-018-9618-5","type":"journal-article","created":{"date-parts":[[2018,5,12]],"date-time":"2018-05-12T13:35:41Z","timestamp":1526132141000},"page":"537-561","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":42,"title":["Successes, challenges, and rethinking \u2013 an industrial investigation on crowdsourced mobile application testing"],"prefix":"10.1007","volume":"24","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3258-689X","authenticated-orcid":false,"given":"Ruizhi","family":"Gao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yabin","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yang","family":"Feng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhenyu","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"W.","family":"Eric Wong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2018,5,12]]},"reference":[{"key":"9618_CR1","unstructured":"Adaptive Vehicle Make (2018) http:\/\/www.darpa.mil\/news-events\/2014-02-05"},{"issue":"2","key":"9618_CR2","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1109\/MIC.2013.20","volume":"17","author":"M Allahbakhsh","year":"2013","unstructured":"Allahbakhsh M, Benatallah B, Ignjatovic A, Motahari-Nezhad H, Bertino E, Dustdar S (2013) Quality control in crowdsourcing systems: issues and directions. IEEE Internet Comput 17(2):76\u201381","journal-title":"IEEE Internet Comput"},{"key":"9618_CR3","unstructured":"Amazon Mechanical Turk (2018) https:\/\/www.mturk.com\/mturk\/welcome"},{"key":"9618_CR4","doi-asserted-by":"publisher","first-page":"40","DOI":"10.1016\/j.jss.2015.03.043","volume":"105","author":"A Bruun","year":"2015","unstructured":"Bruun A, Stage J (2015) New approaches to usability evaluation in software development: barefoot and crowdsourcing. J Syst Softw 105:40\u201353","journal-title":"J Syst Softw"},{"key":"9618_CR5","unstructured":"Capgemini (2017\u20132018) World Quality Report for Mobile Testing"},{"key":"9618_CR6","doi-asserted-by":"crossref","unstructured":"Z. Chen and B. Luo (2014) \u201cQuasi-crowdsourcing testing for educational projects,\u201d in Proceedings of international conference on software engineering, pp. 272.275, Hyderabad, India, Mary","DOI":"10.1145\/2591062.2591153"},{"key":"9618_CR7","doi-asserted-by":"crossref","unstructured":"J. Cheng, J. Teevan, M. S. Bernstein (2015) \u201cMeasuring crowdsourcing effort with error-time curves,\u201d in Proceedings of ACM conference on human factors in computing systems, pp. 1365\u20131374, Seoul, Korea","DOI":"10.1145\/2702123.2702145"},{"key":"9618_CR8","unstructured":"CloudMusic (2018) https:\/\/play.google.com\/store\/apps\/details?id=com.netease.cloudmusic"},{"key":"9618_CR9","unstructured":"CrowdMed (2018) https:\/\/www.crowdmed.com \/"},{"key":"9618_CR10","unstructured":"Crowdsourcing.org (2013) \u201cUsing crowdsourcing for software testing\u201d"},{"key":"9618_CR11","unstructured":"Lucas Dargis (2013) \u201cIs UTest a Scam\u201d http:\/\/lucasdargis.com\/is-utest-a-scam\/"},{"key":"9618_CR12","doi-asserted-by":"crossref","unstructured":"E. Dolstra, R. Vliegendhart, and J. Pouwelse (2013) \u201cCrowdsourcing GUI tests,\u201d In Proceedings of the IEEE International Conference on Software Testing, Verification and Validation, pages 332\u2013341, Luxembourg","DOI":"10.1109\/ICST.2013.44"},{"key":"9618_CR13","doi-asserted-by":"crossref","unstructured":"Y. Feng, Z. Chen, J. A. Jone, C. Fang, and B. Xu (2015) \u201cTest report prioritization to assist Crowdsourced testing,\u201d in Proceedings of joint meeting on foundations of software engineering, pp. 225\u2013236, Bergamo, Italy","DOI":"10.1145\/2786805.2786862"},{"key":"9618_CR14","doi-asserted-by":"crossref","unstructured":"M. Goldman (2011) \u201cRole-based interfaces for collaborative software development,\u201d in Proceedings of the 24th annual ACM symposium adjunct on user Interface software and technology, pp. 23\u201326, Charlotte, USA","DOI":"10.1145\/2046396.2046410"},{"key":"9618_CR15","doi-asserted-by":"crossref","unstructured":"M. Goldman, G. Little, and R. C. Miller (2011) \u201cReal-time collaborative coding in a web IDE,\u201d in Proceedings of the 24th annual ACM symposium on user interface software and technology, pp. 155\u2013164, Santa Barbara, USA","DOI":"10.1145\/2047196.2047215"},{"key":"9618_CR16","doi-asserted-by":"crossref","unstructured":"M. Gomez, R. Rouvoy, B. Adams, and L. Seinturier (2016) \u201cReproducing context-sensitive crashes of mobile apps using Crowdsourced monitoring,\u201d in Proceedings of the international conference on mobile software engineering and systems, pp. 88\u201399, Austin, Texas","DOI":"10.1145\/2897073.2897088"},{"key":"9618_CR17","doi-asserted-by":"crossref","unstructured":"F. Guaiani and H. Muccini (2016) \u201cCrowd and laboratory testing, can they co-exist? An exploratory study,\u201d in Proceedings of the second international workshop on CrowdSourcing in software engineering, pp. 32\u201337, Florence, Italy","DOI":"10.1109\/CSI-SE.2015.14"},{"issue":"5","key":"9618_CR18","doi-asserted-by":"publisher","first-page":"1320","DOI":"10.1037\/0021-9010.92.5.1320","volume":"92","author":"T Haerem","year":"2007","unstructured":"Haerem T, Rau D (2007) The influence of degree of expertise and objective task complexity on perceived task complexity and performance. J Appl Psychol 92(5):1320\u20131331","journal-title":"J Appl Psychol"},{"key":"9618_CR19","doi-asserted-by":"crossref","unstructured":"M. Harman, Y. Jia, W. B. Langdon, J. Petke, I. H. Moghadam, S. Yoo, and F. Wu (2014) \u201cGenetic improvement for adaptive software engineering,\u201d in Proceedings of the international symposium on software engineering for Adaptiveand self-managing systems, pp. 1\u20134, Austin, USA","DOI":"10.1145\/2593929.2600116"},{"issue":"2","key":"9618_CR20","doi-asserted-by":"crossref","first-page":"193","DOI":"10.1111\/j.2517-6161.1953.tb00135.x","volume":"15","author":"H Hotelling","year":"1953","unstructured":"Hotelling H (1953) New light on the correlation coefficient and its transforms. J R Stat Soc 15(2):193\u2013232","journal-title":"J R Stat Soc"},{"key":"9618_CR21","unstructured":"J. Howe (2016) \u201cThe rise of crowdsourcing,\u201d Wired Magazine"},{"key":"9618_CR22","doi-asserted-by":"crossref","unstructured":"Y.-C. Huang, C.-I. Wang, and J. Hsu (2013) \u201cLeveraging the crowd for creating wireframe-based exploration of mobile design pattern gallery,\u201d in Proceedings of the companion publication of the 2013 international conference on intelligent user interfaces, pp. 17\u201320, Santa Monica, USA","DOI":"10.1145\/2451176.2451182"},{"key":"9618_CR23","unstructured":"JustForFun (2018) http:\/\/apk.hiapk.com\/appinfo\/com.xp.tugele"},{"issue":"1","key":"9618_CR24","doi-asserted-by":"publisher","first-page":"74","DOI":"10.1109\/MS.2016.12","volume":"33","author":"TD Latoza","year":"2016","unstructured":"Latoza TD, Van der Hoek A (2016) Crowdsourcing in software engineering: models, motivations, and challenges. IEEE Softw 33(1):74\u201380","journal-title":"IEEE Softw"},{"key":"9618_CR25","doi-asserted-by":"crossref","unstructured":"N. Leicht, N. Knop, I. Blohm, C. M\u00fcller-Bloch, and J. M. Leimeister (2016) \u201cWhen is crowdsourcing advantageous? The case of Crowdsourced software testing,\u201d in Proceedings of European conference on information systems, pp. 1\u201317, Istanbul, Turkey","DOI":"10.2139\/ssrn.3159166"},{"issue":"2","key":"9618_CR26","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1109\/MS.2017.37","volume":"34","author":"N Leicht","year":"2017","unstructured":"Leicht N, Blohm I, Leimeister JM (2017) Leveraging the power of the crowd for software testing. IEEE Softw 34(2):62\u201369","journal-title":"IEEE Softw"},{"key":"9618_CR27","doi-asserted-by":"crossref","unstructured":"D. Liu, M. Lease, R. Kuipers, and R. Bia (2012) \u201cCrowdsourcing for usability testing,\u201d in Proceedings of the American Society for Information Science and Technology, vol. 49, no. 1, pp. 1\u201310","DOI":"10.1002\/meet.14504901100"},{"issue":"6","key":"9618_CR28","doi-asserted-by":"publisher","first-page":"986","DOI":"10.1016\/j.infsof.2012.12.004","volume":"55","author":"MV Mantyla","year":"2013","unstructured":"Mantyla MV, Itkonen J (2013) More testers - the effect of crowd size and time restriction in software testing. Inf Softw Technol 55(6):986\u20131003","journal-title":"Inf Softw Technol"},{"key":"9618_CR29","unstructured":"K. Mao, L. Capra, M. Harman, and Y. Jia (2015) \u201cA survey of the use of crowdsourcing in software engineering,\u201d Research Note, University College London"},{"issue":"3","key":"9618_CR30","doi-asserted-by":"publisher","first-page":"530","DOI":"10.1109\/TMM.2016.2619901","volume":"19","author":"R Mok","year":"2017","unstructured":"Mok R, Chang R, Li W (2017) Detecting low-quality workers in QoE Crowdtesting: a worker behavior-based approach. IEEE Transactions on Multimedia 19(3):530\u2013543","journal-title":"IEEE Transactions on Multimedia"},{"key":"9618_CR31","doi-asserted-by":"crossref","unstructured":"D. Mujumdar, M. Kallenbach, B. Liu, and B. Hartmann (2011) \u201cCrowdsourcing suggestions to programming problems for dynamic web development languages,\u201d in Proceedings of the 2011 annual conference extended abstracts on human factors in computing systems, pp. 1525\u20131530, Vancouver, Canada","DOI":"10.1145\/1979742.1979802"},{"key":"9618_CR32","unstructured":"MyCrowd (2018) https:\/\/mycrowd.com \/"},{"key":"9618_CR33","doi-asserted-by":"crossref","unstructured":"M. Nebeling, M. Speicher, and M. C. Norrie (2013) \u201cCrowdStudy: General Toolkit for Crowdsourced Evaluation of Web Interfaces,\u201d in Proceedings of the 5th ACM SIGCHI Symposium on Engineering Interactive Computing Systems, pp 255\u2013264, London, UK","DOI":"10.1145\/2494603.2480303"},{"key":"9618_CR34","unstructured":"OpenSignal, Android Fragmentation Visualized (2015)"},{"key":"9618_CR35","doi-asserted-by":"crossref","unstructured":"F. Pastore, L. Mariani, and G. Fraser (2013) \u201cCrowdoracles: can the crowd solve the Oracle problem?\u201d In Proceedings of the IEEE International Conference on Software Testing, Verification and Validation, pages 342\u2013351, Luxembourg","DOI":"10.1109\/ICST.2013.13"},{"key":"9618_CR36","unstructured":"iShopping (2018) https:\/\/play.google.com\/store\/apps\/details?id=com.taobao.shopstreet"},{"key":"9618_CR37","unstructured":"Uber (2018) https:\/\/www.uber.com \/"},{"key":"9618_CR38","unstructured":"UBook (2018) http:\/\/www.pgyer.com\/qe34"},{"key":"9618_CR39","unstructured":"UserTesting (2018) https:\/\/www.usertesting.com \/"},{"key":"9618_CR40","unstructured":"UTest (2018) https:\/\/www.utest.com \/"},{"key":"9618_CR41","unstructured":"Waze (2018) https:\/\/www.waze.com \/"},{"key":"9618_CR42","unstructured":"H. Xue (2013) \u201cUsing redundancy to improve security and testing,\u201d Ph.D. dissertation, University of Illinois at Urbana-Champaign"},{"key":"9618_CR43","doi-asserted-by":"crossref","unstructured":"M. Yan, H. Sun, and X. Liu (2014) \u201ciTest: testing software with mobile crowdsourcing,\u201d in Proceedings of the 1st International Workshop on Crowd-based Software Development Methods and Technologies, pp. 19\u201324, Hong Kong","DOI":"10.1145\/2666539.2666569"},{"key":"9618_CR44","doi-asserted-by":"crossref","unstructured":"M. Yuen, I. King, and K. Leung (2011) \u201cA survey of crowdsourcing systems,\u201d in Proceedings of IEEE International Conference on Privacy, Security, Risk, and Trust, and IEEE Conference on Social Computing, pp. 766\u2013773, Boston, USA","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"issue":"12","key":"9618_CR45","doi-asserted-by":"publisher","first-page":"3190","DOI":"10.1109\/TPDS.2013.2297112","volume":"25","author":"X Zhang","year":"2014","unstructured":"Zhang X, Yang Z, Zhou Z, Cai H, Chen L, Li X (2014) Free market of crowdsourcing: incentive mechanism Design for Mobile Sensing. IEEE Trans Prallel Dist Syst 25(12):3190\u20133200","journal-title":"IEEE Trans Prallel Dist Syst"},{"key":"9618_CR46","doi-asserted-by":"publisher","first-page":"375","DOI":"10.1007\/s11573-014-0721-9","volume":"84","author":"S Zogaj","year":"2014","unstructured":"Zogaj S, Bretschneider U, Leimeister JM (2014) Managing Crowdsourced software testing: a case study based insight on the challenges of a crowdsourcing intermediary. J Bus Econ 84:375\u2013405","journal-title":"J Bus Econ"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-018-9618-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-018-9618-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-018-9618-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,6]],"date-time":"2024-07-06T18:34:39Z","timestamp":1720290879000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-018-9618-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,5,12]]},"references-count":46,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2019,4]]}},"alternative-id":["9618"],"URL":"https:\/\/doi.org\/10.1007\/s10664-018-9618-5","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,5,12]]},"assertion":[{"value":"12 May 2018","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}