{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,24]],"date-time":"2026-07-24T13:31:58Z","timestamp":1784899918453,"version":"3.55.0"},"reference-count":31,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2018,12,1]],"date-time":"2018-12-01T00:00:00Z","timestamp":1543622400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2018,12,1]],"date-time":"2018-12-01T00:00:00Z","timestamp":1543622400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"}],"content-domain":{"domain":["clinicalkey.com","clinicalkey.com.au","clinicalkey.es","clinicalkey.fr","clinicalkey.jp","journalofsurgicalresearch.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Surgical Research"],"published-print":{"date-parts":[[2018,12]]},"DOI":"10.1016\/j.jss.2018.06.032","type":"journal-article","created":{"date-parts":[[2018,7,18]],"date-time":"2018-07-18T11:21:53Z","timestamp":1531912913000},"page":"351-364","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":19,"special_numbering":"C","title":["Systematic Review: Online Crowdsourcing to Assess Perceptual Speech Outcomes"],"prefix":"10.1016","volume":"232","author":[{"given":"Anne M.","family":"Sescleifer","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Caitlin A.","family":"Francoisse","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexander Y.","family":"Lin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.jss.2018.06.032_bib1","author":"American Cleft Palate-Craniofacial Association"},{"key":"10.1016\/j.jss.2018.06.032_bib2","doi-asserted-by":"crossref","first-page":"1057","DOI":"10.1111\/jpc.13318","article-title":"The demand for speech pathology services for children: do we need more or just different?","volume":"52","author":"Reilly","year":"2016","journal-title":"J Paediatr Child Health"},{"key":"10.1016\/j.jss.2018.06.032_bib3","doi-asserted-by":"crossref","first-page":"e57410","DOI":"10.1371\/journal.pone.0057410","article-title":"Evaluating Amazon's Mechanical Turk as a tool for experimental behavioral research","volume":"8","author":"Crump","year":"2013","journal-title":"PLoS One"},{"key":"10.1016\/j.jss.2018.06.032_bib4","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1177\/1745691610393980","article-title":"Amazon's mechanical Turk: a new source of inexpensive, yet high-quality, data?","volume":"6","author":"Buhrmester","year":"2011","journal-title":"Perspect Psychol Sci"},{"key":"10.1016\/j.jss.2018.06.032_bib5","doi-asserted-by":"crossref","first-page":"271","DOI":"10.1097\/SLE.0000000000000293","article-title":"The role of crowdsourcing in assessing surgical skills","volume":"26","author":"Katz","year":"2016","journal-title":"Surg Laparosc Endosc Percutan Tech"},{"key":"10.1016\/j.jss.2018.06.032_bib6","doi-asserted-by":"crossref","first-page":"e281","DOI":"10.2196\/jmir.4582","article-title":"Assessing pictograph recognition: a comparison of crowdsourcing and traditional survey approaches","volume":"17","author":"Kuang","year":"2015","journal-title":"J Med Internet Res"},{"key":"10.1016\/j.jss.2018.06.032_bib7","doi-asserted-by":"crossref","first-page":"e167","DOI":"10.2196\/jmir.2338","article-title":"Crowdsourcing malaria parasite quantification: an online game for analyzing images of infected thick blood smears","volume":"14","author":"Luengo-Oroz","year":"2012","journal-title":"J Med Internet Res"},{"key":"10.1016\/j.jss.2018.06.032_bib8","doi-asserted-by":"crossref","first-page":"824","DOI":"10.1148\/radiol.11110938","article-title":"Distributed human intelligence for colonic polyp classification in computer-aided detection for CT colonography","volume":"262","author":"Nguyen","year":"2012","journal-title":"Radiology"},{"key":"10.1016\/j.jss.2018.06.032_bib9","doi-asserted-by":"crossref","first-page":"S6","DOI":"10.1186\/1746-1596-9-S1-S6","article-title":"Preliminary results from a crowdsourcing experiment in immunohistochemistry","volume":"9","author":"Della Mea","year":"2014","journal-title":"Diagn Pathol"},{"key":"10.1016\/j.jss.2018.06.032_bib10","series-title":"Crowdsourcing for Speech Processing: Applications to Data Collection, Transcription and Assessment","author":"Esk\u00e9nazi","year":"2013"},{"key":"10.1016\/j.jss.2018.06.032_bib11","series-title":"INTERSPEECH-2013","first-page":"2147","article-title":"Assessing the utility of judgments of children's speechproduction made by untrained listeners in uncontrolled listeningenvironments","author":"Munson","year":"2013"},{"key":"10.1016\/j.jss.2018.06.032_bib12","doi-asserted-by":"crossref","first-page":"104","DOI":"10.3109\/02699206.2016.1174306","article-title":"Finding the experts in the crowd: validity and reliability of crowdsourced measures of children's gradient speech contrasts","volume":"31","author":"Harel","year":"2017","journal-title":"Clin Linguist Phon"},{"key":"10.1016\/j.jss.2018.06.032_bib13","series-title":"16th Annual Conference of the International Speech Communication Association September 6\u201310, 2015","article-title":"Validating and Optimizing a Crowdsourced Method for Gradient Measures of Child Speech","author":"McAllister Byun","year":"2015"},{"key":"10.1016\/j.jss.2018.06.032_bib14","doi-asserted-by":"crossref","first-page":"70","DOI":"10.1016\/j.jcomdis.2014.11.003","article-title":"Online crowdsourcing for efficient rating of speech: a validation study","volume":"53","author":"McAllister Byun","year":"2015","journal-title":"J Commun Disord"},{"key":"10.1016\/j.jss.2018.06.032_bib15","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1016\/j.jcomdis.2016.07.001","article-title":"Deriving gradient measures of child speech from crowdsourced ratings","volume":"64","author":"McAllister Byun","year":"2016","journal-title":"J Commun Disord"},{"key":"10.1016\/j.jss.2018.06.032_bib16","doi-asserted-by":"crossref","first-page":"233","DOI":"10.1044\/2015_AJSLP-15-0059","article-title":"Use of crowdsourcing to assess the ecological validity of perceptual-training paradigms in dysarthria","volume":"25","author":"Lansford","year":"2016","journal-title":"Am J Speech Lang Pathol"},{"key":"10.1016\/j.jss.2018.06.032_bib17","series-title":"CHI\u201916 Proceedings of the 2016 CHI Conference on Human Factors in ComputingSystems, CHIConference on Human Factors in Computing Systems, ACMNew York, NY. 4464\u20134476","article-title":"Speeching: Mobile Crowdsourced Speech Assessment to Support Self-Monitoring and Management for People With Parkinson's","author":"McNaney","year":"2016"},{"key":"10.1016\/j.jss.2018.06.032_bib18","first-page":"2313","article-title":"Methods for pronunciation assessment in computer aided language learning","volume":"73","author":"Peabody","year":"2012","journal-title":"Dissertation Abstr Int Section B: The Sci Eng"},{"key":"10.1016\/j.jss.2018.06.032_bib19","series-title":"2012International Conference on Speech Database and Assessments, 9\u201312 December 2012","article-title":"Deriving Perceptual Gradation of L2 English Mispronunciations Using Crowdsourcing and the WorkerRank Algorithm","author":"Wang","year":"2012"},{"key":"10.1016\/j.jss.2018.06.032_bib20","author":"Bureau of Labor Statistics"},{"key":"10.1016\/j.jss.2018.06.032_bib21","doi-asserted-by":"crossref","first-page":"644.e1","DOI":"10.1016\/j.ajog.2016.06.033","article-title":"Crowdsourcing: a valid alternative to expert evaluation of robotic surgery skills","volume":"215","author":"Polin","year":"2016","journal-title":"Am J obstetrics Gynecol"},{"key":"10.1016\/j.jss.2018.06.032_bib22","doi-asserted-by":"crossref","first-page":"65","DOI":"10.1016\/j.jss.2013.09.024","article-title":"Crowd-Sourced Assessment of Technical Skills: a novel method to evaluate surgical performance","volume":"187","author":"Chen","year":"2014","journal-title":"J Surg Res"},{"key":"10.1016\/j.jss.2018.06.032_bib23","doi-asserted-by":"crossref","first-page":"864","DOI":"10.1097\/PRS.0000000000002545","article-title":"Crowdsourcing as a novel method to evaluate aesthetic outcomes of treatment for unilateral cleft lip","volume":"138","author":"Tse","year":"2016","journal-title":"Plast Reconstr Surg"},{"key":"10.1016\/j.jss.2018.06.032_bib24","series-title":"Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence (AAAI-17) February 4\u20139 at Hilton San Francisco, San Francisco, CA","article-title":"Pairwise HITS: Quality Estimation From Pairwise Comparisons in Creator-Evaluator Crowdsourcing Process","author":"Sunahase","year":"2017"},{"key":"10.1016\/j.jss.2018.06.032_bib25","doi-asserted-by":"crossref","first-page":"488","DOI":"10.1016\/0007-1226(88)90005-7","article-title":"Speech results of cleft palate surgery: two methods of assessment","volume":"41","author":"Bagnall","year":"1988","journal-title":"Br J Plast Surg"},{"key":"10.1016\/j.jss.2018.06.032_bib26","doi-asserted-by":"crossref","first-page":"288","DOI":"10.1159\/000066153","article-title":"Blinded speech evaluation following pharyngeal flap surgery by speech pathologists and lay people in children with cleft palate","volume":"54","author":"Tonz","year":"2002","journal-title":"Folia Phoniatr Logop"},{"key":"10.1016\/j.jss.2018.06.032_bib27","doi-asserted-by":"crossref","first-page":"656","DOI":"10.1080\/13682820802295203","article-title":"Untrained listeners' ratings of speech disorders in a group with cleft palate: a comparison with speech and language pathologists' ratings","volume":"44","author":"Brunnegard","year":"2009","journal-title":"Int J Lang Commun Disord"},{"key":"10.1016\/j.jss.2018.06.032_bib28","doi-asserted-by":"crossref","first-page":"653","DOI":"10.1089\/tmj.2013.0295","article-title":"Telehealth technology applications in speech-language pathology","volume":"20","author":"Keck","year":"2014","journal-title":"Telemed J E Health"},{"key":"10.1016\/j.jss.2018.06.032_bib29","doi-asserted-by":"crossref","first-page":"367","DOI":"10.1177\/1357633X15583215","article-title":"A systematic review of the use of telehealth in speech, language and hearing sciences","volume":"21","author":"Molini-Avejonas","year":"2015","journal-title":"J Telemed Telecare"},{"key":"10.1016\/j.jss.2018.06.032_bib30","doi-asserted-by":"crossref","first-page":"227","DOI":"10.17955\/tvr.112.3.m.704","article-title":"Expanding use of telepractice in speech-language pathology and audiology","volume":"112","author":"Edwards","year":"2012","journal-title":"Volta Rev"},{"key":"10.1016\/j.jss.2018.06.032_bib31","series-title":"Motor Speech Disorders: Substrates, Differential Diagnosis, and Management","author":"Duffy","year":"2013"}],"container-title":["Journal of Surgical Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0022480418304372?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0022480418304372?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,9,20]],"date-time":"2025-09-20T14:45:57Z","timestamp":1758379557000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0022480418304372"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,12]]},"references-count":31,"alternative-id":["S0022480418304372"],"URL":"https:\/\/doi.org\/10.1016\/j.jss.2018.06.032","relation":{},"ISSN":["0022-4804"],"issn-type":[{"value":"0022-4804","type":"print"}],"subject":[],"published":{"date-parts":[[2018,12]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Systematic Review: Online Crowdsourcing to Assess Perceptual Speech Outcomes","name":"articletitle","label":"Article Title"},{"value":"Journal of Surgical Research","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jss.2018.06.032","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2018 Elsevier Inc. All rights reserved.","name":"copyright","label":"Copyright"}]}}