{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,14]],"date-time":"2026-02-14T02:46:58Z","timestamp":1771037218920,"version":"3.50.1"},"reference-count":72,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","license":[{"start":{"date-parts":[[2013,8,1]],"date-time":"2013-08-01T00:00:00Z","timestamp":1375315200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Multimedia"],"published-print":{"date-parts":[[2013,8]]},"DOI":"10.1109\/tmm.2013.2241043","type":"journal-article","created":{"date-parts":[[2013,1,18]],"date-time":"2013-01-18T19:05:37Z","timestamp":1358535937000},"page":"1121-1137","source":"Crossref","is-referenced-by-count":64,"title":["Crowdsourcing Multimedia QoE Evaluation: A Trusted Framework"],"prefix":"10.1109","volume":"15","author":[{"given":"Chen-Chi","family":"Wu","sequence":"first","affiliation":[]},{"given":"Kuan-Ta","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Yu-Chun","family":"Chang","sequence":"additional","affiliation":[]},{"given":"Chin-Laung","family":"Lei","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2012.2190924"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1145\/2072298.2072418"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2010.5543189"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/MMUL.2004.1261114"},{"key":"ref38","year":"2008","journal-title":"Subjective video quality assessment methods for multimedia applications"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2005.846785"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/2155555.2155560"},{"key":"ref31","first-page":"176","article-title":"The rise of crowdsourcing","volume":"14","author":"howe","year":"2006","journal-title":"Wired Mag"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/ISM.2011.87"},{"key":"ref37","year":"2001","journal-title":"Perceptual Evaluation of Speech Quality (PESQ) An Objective Method for End-to-End Speech Quality Assessment of Narrow-Band Telephone Networks and Speech Codecs"},{"key":"ref36","year":"2008","journal-title":"Objective Perceptual Multimedia Video Quality Measurement in the Presence of a Full Reference"},{"key":"ref35","year":"2005","journal-title":"The E-Model a computational model for use in transmission planning"},{"key":"ref34","year":"1996","journal-title":"Methods for subjective determination of transmission quality"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2011.5946971"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1016\/0022-2496(77)90033-5"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1198\/016214501750332668"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600172"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.86.2.420"},{"key":"ref27","first-page":"1359","article-title":"Photoslap: A multi-player online game for semantic annotation","author":"ho","year":"2007","journal-title":"Proc 22nd Conf Artificial Intelligence"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPRW.2008.4562953"},{"key":"ref65","doi-asserted-by":"crossref","DOI":"10.1117\/12.805313","article-title":"Web-based psychometric evaluation of image quality","volume":"7242","author":"sprow","year":"2009","journal-title":"Proc SPIE"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/1807342.1807376"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2006.876279"},{"key":"ref67","author":"suppes","year":"2006","journal-title":"Foundations of Measurement Geometrical Threshold and Probabilistic Representations"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/1459359.1459395"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1145\/290747.290755"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/j.cose.2008.11.002"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/1924421.1924442"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1002\/j.1538-7305.1963.tb00955.x"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1145\/1806565.1806582"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1002\/j.1538-7305.1960.tb03959.x"},{"key":"ref23","year":"2010","journal-title":"Quality of Experience (QoE) Requirements for Real-Time Communication Services"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.2307\/2087442"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"178","DOI":"10.1117\/12.952716","article-title":"The information theoretical significance of spatial and temporal masking in video signals","volume":"1077","author":"girod","year":"1989","journal-title":"Proc SPIE"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1037\/14396-000"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1109\/ISBMSB.2009.5133799"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/ICIP.2011.6116320"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1967.10482901"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1126\/science.1177418"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1117\/12.872387"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2621.1978.tb09732.x"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.2307\/3147054"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1117\/12.526648"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.2307\/2986348"},{"key":"ref10","doi-asserted-by":"crossref","DOI":"10.1145\/1151659.1159959","article-title":"Quantifying Skype user satisfaction","author":"chen","year":"2006","journal-title":"Proc ACM SIGCOMM 2006"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/1566374.1566393"},{"key":"ref11","doi-asserted-by":"crossref","DOI":"10.1155\/2009\/797159","article-title":"Identifying MMORPG bots: A traffic analysis approach","volume":"2009","author":"chen","year":"2009","journal-title":"EURASIP J Adv Signal Process"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/INFCOM.2009.5061978"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/1631272.1631339"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1145\/1943552.1943563"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1121\/1.2385043"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1117\/12.876733"},{"key":"ref17","author":"david","year":"1988","journal-title":"The Method of Paired Comparisons"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/GLOCOM.2003.1258975"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1111\/1467-9876.00125"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1016\/S0140-6736(86)90837-8"},{"key":"ref3","author":"birnbaum","year":"2000","journal-title":"Psychological Experiments on the Internet"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.2307\/2334029"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/78.765133"},{"key":"ref49","article-title":"Are all games equally cloud-gaming-friendly? An electromyographic approach","author":"lee","year":"2012","journal-title":"Proc IEEE\/ACM NetGames 2012"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"3053","DOI":"10.21437\/Interspeech.2011-764","article-title":"Crowdsourcing preference tests, and how to detect cheating","author":"buchholz","year":"2011","journal-title":"Proc Interspeech 2011"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1109\/CQR.2010.5619923"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/WI-IAT.2009.23"},{"key":"ref45","article-title":"Evaluating the perceived quality of infrastructure-less VoIP","author":"lan","year":"2011","journal-title":"Proc IEEE Workshop Streaming and Media Communications"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2011.2157333"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1145\/2072298.2071994"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.2307\/2332613"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177732186"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1111\/j.1475-3995.2004.00442.x"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"}],"container-title":["IEEE Transactions on Multimedia"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/6046\/6558949\/06415278.pdf?arnumber=6415278","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,2,6]],"date-time":"2022-02-06T07:42:41Z","timestamp":1644133361000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/6415278\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,8]]},"references-count":72,"journal-issue":{"issue":"5"},"URL":"https:\/\/doi.org\/10.1109\/tmm.2013.2241043","relation":{},"ISSN":["1520-9210","1941-0077"],"issn-type":[{"value":"1520-9210","type":"print"},{"value":"1941-0077","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,8]]}}}