{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,22]],"date-time":"2026-04-22T20:20:23Z","timestamp":1776889223497,"version":"3.51.2"},"reference-count":65,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2014,2,1]],"date-time":"2014-02-01T00:00:00Z","timestamp":1391212800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Multimedia"],"published-print":{"date-parts":[[2014,2]]},"DOI":"10.1109\/tmm.2013.2291663","type":"journal-article","created":{"date-parts":[[2014,1,24]],"date-time":"2014-01-24T19:15:40Z","timestamp":1390590940000},"page":"541-558","source":"Crossref","is-referenced-by-count":227,"title":["Best Practices for QoE Crowdtesting: QoE Assessment With Crowdsourcing"],"prefix":"10.1109","volume":"16","author":[{"given":"Tobias","family":"Hossfeld","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christian","family":"Keimel","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Matthias","family":"Hirth","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bruno","family":"Gardlo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Julian","family":"Habigt","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Klaus","family":"Diepold","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Phuoc","family":"Tran-Gia","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref39","first-page":"11","article-title":"How crowdsourcable is your task","author":"eickhoff","year":"2011","journal-title":"Proc Workshop Crowdsourcing for Search and Data Mining (CSDM) at the 4th ACM Int Conf Web Search and Data Mining (WSDM)"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"ref33","article-title":"Honesty in an online labor market","author":"suri","year":"2011","journal-title":"Human Computation Papers from the 2011 AAAI Workshop (WS-11?11)"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1109\/IMIS.2011.89"},{"key":"ref31","first-page":"103","article-title":"The memory effect and its implications on Web QoE modeling","author":"hosfeld","year":"2011","journal-title":"2011 23rd International Teletraffic Congress (ITC) ITC"},{"key":"ref30","article-title":"The impact of test duration on user fatigue and reliability of subjective quality ratings","author":"schatz","year":"2012","journal-title":"J Audio Eng Soc (JAES)"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"ref35","article-title":"Programmatic gold: Targeted and scalable quality assurance in crowdsourcing","author":"oleson","year":"2011","journal-title":"Procs of HCOMP"},{"key":"ref34","first-page":"21","article-title":"Ensuring quality in crowdsourced search relevance evaluation: The effects of training question distribution","author":"le","year":"2010","journal-title":"Proceedings of the SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010)"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2011.5946971"},{"key":"ref62","author":"gwet","year":"2012","journal-title":"Handbook of Inter-rater Reliability the Definitive Guide to Measuring the Extent of Agreement among Multiple Raters"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/2442576.2442591"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1080\/19312450709336664"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2012.2190924"},{"key":"ref64","article-title":"Impact of screening technique on crowdsourcing qoe assessments","author":"gardlo","year":"2012","journal-title":"Proc 22nd Int Conf Radioelektronika 2012 Special Session on Quality in Multimedia Syst"},{"key":"ref27","article-title":"Initial delay vs. Interruptions: Between the devil and the deep blue sea","author":"ho\ufffdfeld","year":"2012","journal-title":"QoMEX 2012"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.3115\/1564131.1564137"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2012.6263866"},{"key":"ref2","year":"2013","journal-title":"Recaptcha"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551512437638"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1109\/ISM.2011.87"},{"key":"ref22","year":"2009","journal-title":"I R Assembly ITU-R BT 500?12 Methodology for the Subjective Assessment of the Quality of Television Pictures"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/PCS.2012.6213338"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/1631272.1631339"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1109\/PV.2012.6229729"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2241043"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/MNET.2010.5430141"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348400"},{"key":"ref51","author":"owl","year":"2012","journal-title":"Java market share"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2011.6065690"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753688"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1145\/1480506.1480508"},{"key":"ref56","article-title":"Subjective assessment of H.264\/AVC video sequences transmitted over a noisy channel","author":"simone","year":"2009","journal-title":"Proceedings of QoMEX 2009"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2010.5496296"},{"key":"ref54","first-page":"4","article-title":"Crowdsourcing for affective annotation of video: Development of a viewer-reported boredom corpus","author":"soleymani","year":"2010","journal-title":"Proc SIGIR 2010 Workshop Crowdsourcing for Search Eval (CSE 2010)"},{"key":"ref53","author":"assembly","year":"1997","journal-title":"ITU-R BS 1116-1 Methods for the Subjective Assessment of Small Impairments in Audio Systems Including Multichannel Sound Systems"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2012.6263885"},{"key":"ref10","year":"2013","journal-title":"Microworkers"},{"key":"ref11","year":"2013","journal-title":"Facebook"},{"key":"ref40","doi-asserted-by":"crossref","first-page":"1669","DOI":"10.1145\/1979742.1979826","article-title":"Shepherding the crowd: managing and providing feedback to crowd workers","author":"dow","year":"2011","journal-title":"Proc 2011 Annu ACM Conf Extended Abstracts on Human Factors in Comput Syst"},{"key":"ref12","year":"2012","journal-title":"Qualinet White Paper on Definitions of Quality of Experience (2012)"},{"key":"ref13","author":"ho\ufffdfeld","year":"2013","journal-title":"Data Traffic Monitoring and Analysis From Measurement Classification and Anomaly Detection to Quality of Experience"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1109\/MCOM.2012.6178831"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/MCOM.2012.6178833"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/MCOM.2012.6178832"},{"key":"ref17","author":"schatz","year":"2013","journal-title":"Data Traffic Monitoring and Analysis From Measurement Classification and Anomaly Detection to Quality of Experience"},{"key":"ref18","year":"2009","journal-title":"ITU-R BT 500 Methodology for the Subjective Assessment of the Quality for Television Pictures"},{"key":"ref19","year":"2008","journal-title":"ITU-T P 910 Subjective Video Quality Assessment Methods for Multimedia Applications"},{"key":"ref4","article-title":"Modeling of crowdsourcing platforms and granularity of work organization in future internet","author":"ho\ufffdfeld","year":"2011","journal-title":"Proc 14th Int Teletraffic Congress (ITC)"},{"key":"ref3","year":"2013","journal-title":"InnoCentive"},{"key":"ref6","year":"2013","journal-title":"CrowdSource"},{"key":"ref5","year":"2013","journal-title":"Crowdflower"},{"key":"ref8","year":"2013","journal-title":"TaskRabbit"},{"key":"ref7","year":"2013","journal-title":"Microtask"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2006.196"},{"key":"ref9","year":"2013","journal-title":"Amazon Mechanical Turk"},{"key":"ref46","first-page":"17","article-title":"Crowdsourcing research opportunities: lessons from natural language processing","author":"sabou","year":"2012","journal-title":"Proc 11th ACM Int Conf Knowl Manag Knowl Technol"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-011-9181-9"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/1958824.1958865"},{"key":"ref47","first-page":"15","article-title":"You're hired! an examination of crowdsourcing incentive models in human resource tasks","author":"harris","year":"2011","journal-title":"Proc WSDM Workshop Crowdsourcing for Search and Data Mining (CSDM)"},{"key":"ref42","article-title":"Decision-theoretic control of crowd-sourced workflows","author":"dai","year":"2010","journal-title":"Proc 22nd Conf Artif Intell AAAI"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1145\/1600150.1600159"},{"key":"ref44","article-title":"Analyzing costs and accuracy of validation mechanisms for crowdsourcing platforms","author":"hirth","year":"2012","journal-title":"Math Comput Model"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/1460563.1460572"}],"container-title":["IEEE Transactions on Multimedia"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6046\/6712943\/06671455.pdf?arnumber=6671455","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:31:53Z","timestamp":1642005113000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/6671455\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,2]]},"references-count":65,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tmm.2013.2291663","relation":{},"ISSN":["1520-9210","1941-0077"],"issn-type":[{"value":"1520-9210","type":"print"},{"value":"1941-0077","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014,2]]}}}