{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,12]],"date-time":"2025-07-12T22:45:05Z","timestamp":1752360305972,"version":"3.40.3"},"publisher-location":"Cham","reference-count":31,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319026800"},{"type":"electronic","value":"9783319026817"}],"license":[{"start":{"date-parts":[[2014,1,1]],"date-time":"2014-01-01T00:00:00Z","timestamp":1388534400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014]]},"DOI":"10.1007\/978-3-319-02681-7_21","type":"book-chapter","created":{"date-parts":[[2014,3,4]],"date-time":"2014-03-04T06:36:00Z","timestamp":1393914960000},"page":"315-327","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15,"title":["Crowdsourcing in QoE Evaluation"],"prefix":"10.1007","author":[{"given":"Tobias","family":"Ho\u00dffeld","sequence":"first","affiliation":[]},{"given":"Christian","family":"Keimel","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2014,3,5]]},"reference":[{"key":"21_CR1","doi-asserted-by":"publisher","unstructured":"Alonso O, Rose DE, Stewart B (2008) Crowdsourcing for relevance evaluation. ACM SIGIR Forum 42(2):9\u201315. doi:10.1145\/1480506.1480508. http:\/\/doi.acm.org\/10.1145\/1480506.1480508","DOI":"10.1145\/1480506.1480508"},{"key":"21_CR2","unstructured":"Amazon Mechanical Turk (2013). http:\/\/mturk.com"},{"key":"21_CR3","doi-asserted-by":"crossref","unstructured":"Barkowsky M, Li J, Han T, Youn S, Ok J, Lee C, Hedberg C, Ananth IV, Wang K, Brunnstr\u00f6m K et al (2013) Towards standardized 3dtv qoe assessment: cross-lab study on display technology and viewing environment parameters. In: IS&T\/SPIE electronic imaging, International Society for Optics and Photonics, pp 864, 809\u2013864, 809","DOI":"10.1117\/12.2004050"},{"key":"21_CR4","doi-asserted-by":"crossref","unstructured":"Catellier A, Pinson M, Ingram W, Webster A (2012) Impact of mobile devices and usage location on perceived multimedia quality. In: 2012 fourth international workshop on quality of multimedia experience (QoMEX), IEEE, pp 39\u201344","DOI":"10.1109\/QoMEX.2012.6263834"},{"issue":"2","key":"21_CR5","doi-asserted-by":"publisher","first-page":"28","DOI":"10.1109\/MNET.2010.5430141","volume":"24","author":"KT Chen","year":"2010","unstructured":"Chen KT, Chang CJ, Wu CC, Chang YC, Lei CL (2010) Quadrant of euphoria: a crowdsourcing platform for QoE assessment. Network, IEEE 24(2):28\u201335. doi:10.1109\/MNET.2010.5430141","journal-title":"Network, IEEE"},{"key":"21_CR6","doi-asserted-by":"publisher","unstructured":"De Simone F, Naccari M, Tagliasacchi M, Dufaux F, Tubaro S, Ebrahimi T (2009) Subjective assessment of H.264\/AVC video sequences transmitted over a noisy channel. In: Proceedings of the first international workshop on quality of multimedia experience (QoMEX 2009), pp 204\u2013209. doi:10.1109\/QOMEX.2009.5246952","DOI":"10.1109\/QOMEX.2009.5246952"},{"key":"21_CR7","doi-asserted-by":"publisher","unstructured":"Downs JS, Holbrook MB, Sheng S, Cranor LF (2010) Are your participants gaming the system? Screening mechanical turk workers. In: Proceedings of the SIGCHI conference on human factors in computing systems, CHI \u201910, ACM, New York, NY, USA, pp 2399\u20132402 doi:10.1145\/1753326.1753688. http:\/\/doi.acm.org\/10.1145\/1753326.1753688","DOI":"10.1145\/1753326.1753688"},{"key":"21_CR8","doi-asserted-by":"crossref","unstructured":"Eickhoff C, Harris CG, de Vries AP, Srinivasan P (2012) Quality through flow and immersion: gamifying crowdsourced relevance assessments. In: Proceedings of ACM SIGIR conference on research and development in information retrieval 2012. ACM","DOI":"10.1145\/2348283.2348400"},{"issue":"2","key":"21_CR9","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/s10791-011-9181-9","volume":"16","author":"C Eickhoff","year":"2012","unstructured":"Eickhoff C, de Vries A (2012) Increasing cheat robustness of crowdsourcing tasks. Inf Retrieval 16(2):121\u2013137. doi:10.1007\/s10791-011-9181-9","journal-title":"Inf Retrieval"},{"key":"21_CR10","unstructured":"Facebook (2013). http:\/\/www.facebook.com"},{"key":"21_CR11","unstructured":"Gardlo B, Ries M, Ho\u00dffeld T (2012) Impact of screening technique on crowdsourcing QoE assessments. In: 22nd international conference radioelektronika 2012, special session on quality in multimedia systems. Brno, Czech Republic"},{"key":"21_CR12","doi-asserted-by":"publisher","unstructured":"Hirth M, Ho\u00dffeld T, Tran-Gia P (2011) Anatomy of a crowdsourcing platform\u2014using the example of Microworkers.com. In: Workshop on future internet and next generation networks (FINGNet). Seoul, Korea. doi:10.1109\/IMIS.2011.89","DOI":"10.1109\/IMIS.2011.89"},{"key":"21_CR13","unstructured":"Ho\u00dffeld T, Keimel C, Hirth M, Gardlo B, Habigt J, Diepold K, Tran-Gia P (2013) CrowdTesting: a novel methodology for subjective user studies and QoE evaluation. Technical report 486, University of W\u00fcrzburg"},{"key":"21_CR14","doi-asserted-by":"publisher","unstructured":"Ho\u00dffeld T, Seufert M, Hirth M, Zinner T, Tran-Gia P, Schatz R (2011) Quantification of youtube qoe via crowdsourcing. In: 2011 IEEE international symposium on multimedia (ISM), pp 494\u2013499. doi:10.1109\/ISM.2011.87","DOI":"10.1109\/ISM.2011.87"},{"key":"21_CR15","doi-asserted-by":"crossref","unstructured":"Hsueh PY, Melville P, Sindhwani V (2009) Data quality from crowdsourcing: a study of annotation selection criteria. In: Proceedings of the NAACL HLT 2009 workshop on active learning for natural language processing, HLT \u201909, Association for Computational Linguistics, Stroudsburg, PA, USA, pp 27\u201335 http:\/\/dl.acm.org\/citation.cfm?id=1564131.1564137","DOI":"10.3115\/1564131.1564137"},{"key":"21_CR16","unstructured":"ITU-R BT.500-13 (2012) Methodology for the subjective assessment of the quality of television pictures. International Telecommunication Union, Geneva, Switzerland"},{"key":"21_CR17","doi-asserted-by":"publisher","unstructured":"Keimel C, Habigt J, Diepold K (2012) Challenges in crowd-based video quality assessment. In: 2012 fourth international workshop on quality of multimedia experience (QoMEX), pp 13\u201318. doi:10.1109\/QoMEX.2012.6263866","DOI":"10.1109\/QoMEX.2012.6263866"},{"key":"21_CR18","doi-asserted-by":"publisher","unstructured":"Keimel C, Habigt J, Horch C, Diepold K (2012) Qualitycrowd\u2014a framework for crowd-based quality evaluation. In: Picture coding symposium (PCS), pp 245\u2013248. doi:10.1109\/PCS.2012.6213338","DOI":"10.1109\/PCS.2012.6213338"},{"key":"21_CR19","doi-asserted-by":"publisher","unstructured":"Keimel C, Habigt J, Horch C, Diepold K (2012) Video quality evaluation in the cloud. In: Packet video workshop (PV), 2012 19th, international, pp 155\u2013160. doi:10.1109\/PV.2012.6229729","DOI":"10.1109\/PV.2012.6229729"},{"key":"21_CR20","doi-asserted-by":"crossref","unstructured":"Kittur A, Chi E, Suh B (2008) Crowdsourcing user studies with mechanical turk. In: Proceedings of the twenty-sixth annual SIGCHI conference on human factors in computing systems. ACM, pp 453\u2013456","DOI":"10.1145\/1357054.1357127"},{"key":"21_CR21","doi-asserted-by":"publisher","unstructured":"Kittur A, Chi EH, Suh B (2008) Crowdsourcing user studies with mechanical turk. In: Proceedings of the SIGCHI conference on human factors in computing systems, CHI \u201908, ACM, New York, NY, USA, pp 453\u2013456. doi:10.1145\/1357054.1357127. http:\/\/doi.acm.org\/10.1145\/1357054.1357127","DOI":"10.1145\/1357054.1357127"},{"key":"21_CR22","unstructured":"Microworkers (2013). http:\/\/microworkers.com"},{"key":"21_CR23","doi-asserted-by":"publisher","unstructured":"Pinson MH, Janowski L, Pepion R, Huynh-Thu Q, Schmidmer C, Corriveau P, Younkin A, Le Callet P, Barkowsky M, Ingram W (2011) The influence of subjects and environment on audiovisual subjective tests: an international study. Selected Topics in Signal Processing, IEEE Journal of, 6(6):640\u2013651. doi:10.1109\/JSTSP.2012.2215306","DOI":"10.1109\/JSTSP.2012.2215306"},{"key":"21_CR24","doi-asserted-by":"publisher","unstructured":"Ross J, Irani L, Silberman M, Zaldivar A, Tomlinson B (2010) Who are the crowdworkers? Shifting demographics in mechanical turk. In: Proceedings of the 28th of the international conference extended abstracts on human factors in computing systems, ACM, pp 2863\u20132872. doi:10.1145\/1753846.1753873","DOI":"10.1145\/1753846.1753873"},{"key":"21_CR25","doi-asserted-by":"crossref","unstructured":"Sabou M, Bontcheva K, Scharl A (2012) Crowdsourcing research opportunities: lessons from natural language processing. In: Proceedings of the 12th international conference on knowledge management and knowledge technologies, ACM, p 17","DOI":"10.1145\/2362456.2362479"},{"key":"21_CR26","doi-asserted-by":"publisher","unstructured":"Shaw AD, Horton JJ, Chen DL (2011) Designing incentives for inexpert human raters. In: Proceedings of the ACM 2011 conference on computer supported cooperative work, CSCW \u201911, ACM, New York, NY, USA, pp 275\u2013284. doi:10.1145\/1958824.1958865\n                  http:\/\/doi.acm.org\/10.1145\/1958824.1958865","DOI":"10.1145\/1958824.1958865"},{"key":"21_CR27","unstructured":"Suri S, Goldstein D, Mason W (2011) Honesty in an online labor market. In: Human computation: papers from the 2011 AAAI, Workshop (WS-11-11)"},{"key":"21_CR28","unstructured":"Technische Universit\u00e4t M\u00fcnchen, Institute for Data Processing: Qualitycrowd (2013). http:\/\/www.ldv.ei.tum.de\/videolab"},{"key":"21_CR29","doi-asserted-by":"crossref","unstructured":"Varela M, M\u00e4ki T, Skorin-Kapov L, Ho\u00dffeld T (2013) Increasing payments in crowdsourcing: don\u2019t look a gift horse in the mouth. In: 4th international workshop on perceptual quality of systems (PQS 2013). Vienna, Austria","DOI":"10.21437\/PQS.2013-4"},{"key":"21_CR30","doi-asserted-by":"crossref","unstructured":"Von Ahn L, Dabbish L (2004) Labeling images with a computer game. In: Proceedings of the SIGCHI conference on human factors in computing systems, ACM, pp 319\u2013326","DOI":"10.1145\/985692.985733"},{"issue":"4","key":"21_CR31","doi-asserted-by":"publisher","first-page":"600","DOI":"10.1109\/TIP.2003.819861","volume":"13","author":"Z Wang","year":"2004","unstructured":"Wang Z, Bovik A, Sheikh H, Simoncelli E (2004) Image quality assessment: from error visibility to structural similarity. IEEE Trans Image Process 13(4):600\u2013612. doi:10.1109\/TIP.2003.819861","journal-title":"IEEE Trans Image Process"}],"container-title":["T-Labs Series in Telecommunication Services","Quality of Experience"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-02681-7_21","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,2,19]],"date-time":"2023-02-19T22:55:41Z","timestamp":1676847341000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-319-02681-7_21"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014]]},"ISBN":["9783319026800","9783319026817"],"references-count":31,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-02681-7_21","relation":{},"ISSN":["2192-2810","2192-2829"],"issn-type":[{"type":"print","value":"2192-2810"},{"type":"electronic","value":"2192-2829"}],"subject":[],"published":{"date-parts":[[2014]]},"assertion":[{"value":"5 March 2014","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}}]}}