{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,19]],"date-time":"2025-11-19T06:33:35Z","timestamp":1763534015676,"version":"3.41.0"},"publisher-location":"Cham","reference-count":112,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783319664347"},{"type":"electronic","value":"9783319664354"}],"license":[{"start":{"date-parts":[[2017,1,1]],"date-time":"2017-01-01T00:00:00Z","timestamp":1483228800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2017]]},"DOI":"10.1007\/978-3-319-66435-4_7","type":"book-chapter","created":{"date-parts":[[2017,9,27]],"date-time":"2017-09-27T11:51:34Z","timestamp":1506513094000},"page":"154-190","source":"Crossref","is-referenced-by-count":20,"title":["Crowdsourcing Quality of Experience Experiments"],"prefix":"10.1007","author":[{"given":"Sebastian","family":"Egger-Lampl","sequence":"first","affiliation":[]},{"given":"Judith","family":"Redi","sequence":"additional","affiliation":[]},{"given":"Tobias","family":"Ho\u00dffeld","sequence":"additional","affiliation":[]},{"given":"Matthias","family":"Hirth","sequence":"additional","affiliation":[]},{"given":"Sebastian","family":"M\u00f6ller","sequence":"additional","affiliation":[]},{"given":"Babak","family":"Naderi","sequence":"additional","affiliation":[]},{"given":"Christian","family":"Keimel","sequence":"additional","affiliation":[]},{"given":"Dietmar","family":"Saupe","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,9,28]]},"reference":[{"key":"7_CR1","doi-asserted-by":"crossref","unstructured":"Alonso, O., Rose, D.E., Stewart, B.: Crowdsourcing for relevance evaluation. In: ACM SigIR Forum, vol. 42, pp. 9\u201315. ACM (2008)","DOI":"10.1145\/1480506.1480508"},{"key":"7_CR2","doi-asserted-by":"crossref","unstructured":"Becker, M., Borchert, K., Hirth, M., Mewes, H., Hotho, A., Tran-Gia, P.: Microtrails: comparing hypotheses about task selection on a crowd sourcing platform. In: International Conference on Knowledge Technologies and Data-driven Business (I-KNOW), Graz, Austria, October 2015","DOI":"10.1145\/2809563.2809608"},{"key":"7_CR3","doi-asserted-by":"crossref","unstructured":"Bhatti, N., Bouch, A., Kuchinsky, A.: Integrating User-Perceived quality into web server design. In: 9th International World Wide Web Conference, pp. 1\u201316 (2000)","DOI":"10.1016\/S1389-1286(00)00087-6"},{"key":"7_CR4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-84172-9","volume-title":"A Comparison of Probabilistic Unfolding Theories for Paired Comparisons Data","author":"P Bossuyt","year":"1990","unstructured":"Bossuyt, P.: A Comparison of Probabilistic Unfolding Theories for Paired Comparisons Data. Springer, Heidelberg (1990). doi: 10.1007\/978-3-642-84172-9"},{"key":"7_CR5","doi-asserted-by":"crossref","unstructured":"Bouch, A., Kuchinsky, A., Bhatti, N.: Quality is in the eye of the beholder: meeting users\u2019 requirements for internet quality of service. In: CHI 2000: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 297\u2013304. ACM, New York (2000)","DOI":"10.1145\/332040.332447"},{"issue":"3\/4","key":"7_CR6","doi-asserted-by":"crossref","first-page":"324","DOI":"10.2307\/2334029","volume":"39","author":"RA Bradley","year":"1952","unstructured":"Bradley, R.A., Terry, M.E.: Rank analysis of incomplete block designs: I. The method of paired comparisons. Biometrika 39(3\/4), 324\u2013345 (1952)","journal-title":"Biometrika"},{"key":"7_CR7","unstructured":"Callet, P.L., M\u00f6ller, S., Perkis, A. (eds.): Qualinet white paper on definitions of Quality of Experience (2012)"},{"key":"7_CR8","doi-asserted-by":"crossref","unstructured":"Chen, K.T., Chang, C.J., Wu, C.C., Chang, Y.C., Lei, C.L.: Quadrant of euphoria: a crowdsourcing platform for QoE assessment. Network 24(2) (2010)","DOI":"10.1109\/MNET.2010.5430141"},{"key":"7_CR9","doi-asserted-by":"crossref","unstructured":"Chen, K.T., Wu, C.C., Chang, Y.C., Lei, C.L.: A crowdsourceable QoE evaluation framework for multimedia content. In: Proceedings of the 17th ACM international conference on Multimedia, MM 2009, pp. 491\u2013500. ACM (2009)","DOI":"10.1145\/1631272.1631339"},{"key":"7_CR10","doi-asserted-by":"crossref","unstructured":"Cooke, M., Barker, J., Lecumberri, G., Wasilewski, K.: Crowdsourcing in Speech Perception. Crowdsourcing for Speech Processing: Applications to Data Collection, Transcription and Assessment, pp. 137\u2013172 (2013)","DOI":"10.1002\/9781118541241.ch6"},{"issue":"1","key":"7_CR11","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/S0165-1684(99)00018-3","volume":"77","author":"P Corriveau","year":"1999","unstructured":"Corriveau, P., Gojmerac, C., Hughes, B., Stelmach, L.: All subjective scales are not created equal: the effects of context on different scales. Sig. Process. 77(1), 1\u20139 (1999)","journal-title":"Sig. Process."},{"key":"7_CR12","doi-asserted-by":"crossref","unstructured":"Strohmeier, D., Jumisko-Pyykk\u00f6, S., Raake, A.: Toward task-dependent evaluation of Web-QoE: free exploration vs. who ate what? In: Globecom Workshops, pp. 1309\u20131313. IEEE (2012)","DOI":"10.1109\/GLOCOMW.2012.6477771"},{"key":"7_CR13","series-title":"Griffin\u2019s statistical monographs","volume-title":"The Method of Paired Comparisons","author":"HA David","year":"1988","unstructured":"David, H.A.: The Method of Paired Comparisons. Griffin\u2019s statistical monographs, vol. 41, 2nd edn. Charles Griffin & Company Limited, London (1988)","edition":"2"},{"key":"7_CR14","doi-asserted-by":"crossref","unstructured":"Downs, J.S., Holbrook, M.B., Sheng, S., Cranor, L.F.: Are your participants gaming the system?: screening mechanical turk workers. In: Proceedings of the SIGCHI Conference on Human Factors in Computing Systems, pp. 2399\u20132402. ACM (2010)","DOI":"10.1145\/1753326.1753688"},{"key":"7_CR15","doi-asserted-by":"crossref","unstructured":"Egger, S., Gardlo, B., Seufert, M., Schatz, R.: The impact of adaptation strategies on perceived quality of HTTP adaptive streaming. In: Proceedings of the 2014 Workshop on Design, Quality and Deployment of Adaptive Video Streaming, pp. 31\u201336. ACM (2014)","DOI":"10.1145\/2676652.2676658"},{"key":"7_CR16","doi-asserted-by":"crossref","unstructured":"Egger, S., Reichl, P., Hosfeld, T., Schatz, R.: time is bandwidth? narrowing the gap between subjective time perception and quality of experience. In: 2012 IEEE International Conference on Communications (ICC), pp. 1325\u20131330. IEEE (2012)","DOI":"10.1109\/ICC.2012.6363769"},{"key":"7_CR17","unstructured":"Egger, S., Schatz, R.: Interactive content for subjective studies on web browsing QoE: A Kepler derivative. In: ETSI STQ Workshop on Selected Items on Telecommunication Quality Matters, pp. 27\u201328 (2012)"},{"key":"7_CR18","doi-asserted-by":"crossref","unstructured":"Eichhorn, A., Ni, P., Eg, R.: Randomised pair comparison: an economic and robust method for audiovisual quality assessment. In: Proceedings of the 20th International Workshop on Network and Operating Systems Support for Digital Audio and Video, pp. 63\u201368. ACM (2010)","DOI":"10.1145\/1806565.1806582"},{"key":"7_CR19","volume-title":"Psychometric Scaling: A Toolkit for Imaging Systems Development","author":"PG Engeldrum","year":"2000","unstructured":"Engeldrum, P.G.: Psychometric Scaling: A Toolkit for Imaging Systems Development. Imcotek Press, Winchester (2000)"},{"key":"7_CR20","unstructured":"ETSI: Speech Processing, Transmission and Quality Aspects (STQ); Reference webpage for subjective testing. ETSI Standard TR 103 256, October 2014"},{"key":"7_CR21","unstructured":"Fliegel, K.: Qualinet multimedia databases v5. 5 (2014)"},{"key":"7_CR22","doi-asserted-by":"crossref","unstructured":"Freitas, P.G., Redi, J.A., Farias, M.C., Silva, A.F.: Video quality ruler: a new experimental methodology for assessing video quality. In: 2015 Seventh International Workshop on Quality of Multimedia Experience (QoMEX), pp. 1\u20136. IEEE (2015)","DOI":"10.1109\/QoMEX.2015.7148137"},{"key":"7_CR23","unstructured":"Gardlo, B.: Quality of experience evaluation methodology via crowdsourcing. Ph.D. thesis, University of Zilina (2012)"},{"key":"7_CR24","doi-asserted-by":"crossref","unstructured":"Gardlo, B., Egger, S., Hossfeld, T.: Do scale-design and training matter for video QoE assessments through crowdsourcing? In: Proceedings of the Fourth International Workshop on Crowdsourcing for Multimedia, pp. 15\u201320. ACM (2015)","DOI":"10.1145\/2810188.2810193"},{"key":"7_CR25","doi-asserted-by":"crossref","unstructured":"Gardlo, B., Egger, S., Seufert, M., Schatz, R.: Crowdsourcing 2.0: enhancing execution speed and reliability of web-based QoE testing. In: International Conference on Communications, Sydney, AU, June 2014","DOI":"10.1109\/ICC.2014.6883463"},{"key":"7_CR26","doi-asserted-by":"crossref","unstructured":"Guse, D., Egger, S., Raake, A., M\u00f6ller, S.: Web-QoE under real-world distractions: two test cases. In: 2014 Sixth International Workshop on Quality of Multimedia Experience (QoMEX), pp. 220\u2013225. IEEE (2014)","DOI":"10.1109\/QoMEX.2014.6982322"},{"key":"7_CR27","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1007\/3-540-48109-5_5","volume-title":"Interactive Distributed Multimedia Systems and Telecommunication Services","author":"D Hands","year":"1999","unstructured":"Hands, D., Wilkins, M.: A study of the impact of network loss and burst size on video streaming quality and acceptability. In: Diaz, M., Owezarski, P., S\u00e9nac, P. (eds.) IDMS 1999. LNCS, vol. 1718, pp. 45\u201357. Springer, Heidelberg (1999). doi: 10.1007\/3-540-48109-5_5"},{"key":"7_CR28","doi-asserted-by":"crossref","unstructured":"Hanhart, P., Korshunov, P., Ebrahimi, T.: Crowd-based quality assessment of multiview video plus depth coding. In: IEEE International Conference on Image Processing, ICIP 2014. Paris France, April 2014","DOI":"10.1109\/ICIP.2014.7025149"},{"key":"7_CR29","doi-asserted-by":"crossref","unstructured":"Hanhart, P., Korshunov, P., Ebrahimi, T.: Crowdsourcing evaluation of high dynamic range image compression. In: SPIE Optical Engineering + Applications. International Society for Optics and Photonics, San Diego, CA, USA, August 2014","DOI":"10.1117\/12.2065560"},{"key":"7_CR30","doi-asserted-by":"crossref","unstructured":"Hanhart, P., Krasula, L., Le Callet, P., Ebrahimi, T.: How to benchmark objective quality metrics from paired comparison data? In: Quality of Multimedia Experience (QoMEX), pp. 1\u20136. IEEE (2016)","DOI":"10.1109\/QoMEX.2016.7498960"},{"key":"7_CR31","unstructured":"Harris, C.: You\u2019re hired! an examination of crowdsourcing incentive models in human resource tasks. In: WSDM Workshop on Crowdsourcing for Search and Data Mining (CSDM), pp. 15\u201318 (2011)"},{"key":"7_CR32","doi-asserted-by":"crossref","unstructured":"Hirth, M., Ho\u00dffeld, T., Tran-Gia, P.: Anatomy of a crowdsourcing platform - using the example of Microworkers.com . In: Workshop on Future Internet and Next Generation Networks (FINGNet), Seoul, Korea, June 2011","DOI":"10.1109\/IMIS.2011.89"},{"issue":"2","key":"7_CR33","doi-asserted-by":"crossref","first-page":"541","DOI":"10.1109\/TMM.2013.2291663","volume":"16","author":"T Hossfeld","year":"2014","unstructured":"Hossfeld, T., Keimel, C., Hirth, M., Gardlo, B., Habigt, J., Diepold, K., Tran-Gia, P.: Best practices for QoE crowdtesting: QoE assessment with crowdsourcing. Trans. Multimedia 16(2), 541\u2013558 (2014)","journal-title":"Trans. Multimedia"},{"key":"7_CR34","doi-asserted-by":"crossref","unstructured":"Ho\u00dffeld, T., Seufert, M., Hirth, M., Zinner, T., Tran-Gia, P., Schatz, R.: Quantification of YouTube QoE via crowdsourcing. In: Symposium on Multimedia, Dana Point, USA, December 2011","DOI":"10.1109\/ISM.2011.87"},{"issue":"1","key":"7_CR35","first-page":"8","volume":"1","author":"T Hossfeld","year":"2014","unstructured":"Hossfeld, T.: On training the crowd for subjective quality studies. VQEG eLetter 1(1), 8 (2014)","journal-title":"VQEG eLetter"},{"key":"7_CR36","doi-asserted-by":"crossref","unstructured":"Ho\u00dffeld, T., Egger, S., Schatz, R., Fiedler, M., Masuch, K., Lorentzen, C.: Initial delay vs. interruptions: between the devil and the deep blue sea. In: QoMEX 2012, Yarra Valley, Australia, July 2012","DOI":"10.1109\/QoMEX.2012.6263849"},{"key":"7_CR37","doi-asserted-by":"crossref","unstructured":"Ho\u00dffeld, T., Hirth, M., Korshunov, P., Hanhart, P., Gardlo, B., Keimel, C., Timmerer, C.: Survey of web-based crowdsourcing frameworks for subjective quality assessment. In: 2014 IEEE 16th International Workshop on Multimedia Signal Processing (MMSP), pp. 1\u20136. IEEE (2014)","DOI":"10.1109\/MMSP.2014.6958831"},{"key":"7_CR38","unstructured":"Ho\u00dffeld, T., Hirth, M., Redi, J., Mazza, F., Korshunov, P., Naderi, B., Seufert, M., Gardlo, B., Egger, S., Keimel, C.: Best practices and recommendations for crowdsourced QoE - lessons learned from the qualinet task force \u201cCrowdsourcing\u201d October 2014. https:\/\/hal.archives-ouvertes.fr\/hal-01078761 , lessons learned from the Qualinet Task Force \u201cCrowdsourcing\u201d COST Action IC1003 European Network on Quality of Experience in Multimedia Systems and Services (QUALINET)"},{"key":"7_CR39","unstructured":"Ho\u00dffeld, T., Schatz, R., Biedermann, S., Platzer, A., Egger, S., Fiedler, M.: The memory effect and its implications on web QoE modeling. In: 23rd International Teletraffic Congress (ITC 2011), San Francisco, CA, USA (2011)"},{"key":"7_CR40","doi-asserted-by":"crossref","unstructured":"Ho\u00dffeld, T., Schatz, R., Seufert, M., Hirth, M., Zinner, T., Tran-Gia, P.: Quantification of YouTube QoE via Crowdsourcing. In: IEEE International Workshop on Multimedia Quality of Experience - Modeling, Evaluation, and Directions (MQoE 2011), Dana Point, CA, USA, December 2011","DOI":"10.1109\/ISM.2011.87"},{"key":"7_CR41","doi-asserted-by":"crossref","unstructured":"Hsueh, P.Y., Melville, P., Sindhwani, V.: Data quality from crowdsourcing: a study of annotation selection criteria. In: Proceedings of the NAACL HLT 2009 Workshop on Active Learning for Natural Language Processing, pp. 27\u201335. Association for Computational Linguistics (2009)","DOI":"10.3115\/1564131.1564137"},{"issue":"1","key":"7_CR42","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1109\/TBC.2010.2086750","volume":"57","author":"Q Huynh-Thu","year":"2011","unstructured":"Huynh-Thu, Q., Garcia, M.N., Speranza, F., Corriveau, P., Raake, A.: Study of rating scales for subjective quality assessment of high-definition video. IEEE Trans. Broadcast. 57(1), 1\u201314 (2011)","journal-title":"IEEE Trans. Broadcast."},{"key":"7_CR43","unstructured":"International Telecommunication Union: Methods for Subjective Determination of Transmission Quality. ITU-T Recommendation P.800, August 1996"},{"key":"7_CR44","unstructured":"International Telecommunication Union: Interactive test methods for audiovisual communications. ITU-T Recommendation P.920, May 2000"},{"key":"7_CR45","unstructured":"International Telecommunication Union: Vocabulary and effects of transmission parameters on customer opinion of transmission quality, amendment 2. ITU-T Recommendation P.10\/G.100 (2006)"},{"key":"7_CR46","unstructured":"International Telecommunication Union: ITU-T recommendation e.800. Quality of Telecommunication Services: Concepts, models, objectives and dependability planning. terms and definitions related to the quality of telecommunication services. ITU-T Recommendation E.800, September 2008"},{"key":"7_CR47","unstructured":"International Telecommunication Union: Subjective video quality assessment methods for multimedia applications. ITU-T Recommendation P.910, April 2008"},{"key":"7_CR48","unstructured":"International Telecommunication Union: Methodology for the Subjective Assessment of the Quality of Television Pictures. ITU-R Recommendation BT.500-12, March 2009"},{"key":"7_CR49","doi-asserted-by":"crossref","unstructured":"International Telecommunication Union: Subjective Testing Methodology for web browsing. ITU-T Recommendation P.1501 (2013)","DOI":"10.18356\/6c417aca-en"},{"key":"7_CR50","unstructured":"International Telecommunication Union: Subjective Methods for the Assessment of stereoscopic 3DTV Systems. ITU-R Recommendation BT.2021, July 2015"},{"key":"7_CR51","doi-asserted-by":"crossref","unstructured":"Janowski, L., Pinson, M.: Subject bias: introducing a theoretical user model. In: 2014 Sixth International Workshop on Quality of Multimedia Experience (QoMEX), pp. 251\u2013256. IEEE (2014)","DOI":"10.1109\/QoMEX.2014.6982327"},{"issue":"11","key":"7_CR52","doi-asserted-by":"crossref","first-page":"1166","DOI":"10.5594\/J04083","volume":"95","author":"BL Jones","year":"1986","unstructured":"Jones, B.L., McManus, P.R.: Graphic scaling of qualitative terms. SMPTE J. 95(11), 1166\u20131171 (1986)","journal-title":"SMPTE J."},{"key":"7_CR53","doi-asserted-by":"crossref","unstructured":"Jumisko-Pyykk\u00f6, S., Hannuksela, M.M.: Does context matter in quality evaluation of mobile television?. In: Proceedings of the 10th International Conference on Human Computer Interaction with Mobile Devices and Services, pp. 63\u201372. ACM (2008)","DOI":"10.1145\/1409240.1409248"},{"key":"7_CR54","doi-asserted-by":"crossref","DOI":"10.1201\/9780203910825","volume-title":"Handbook of Image Quality: Characterization and Prediction","author":"B Keelan","year":"2002","unstructured":"Keelan, B.: Handbook of Image Quality: Characterization and Prediction. CRC Press, Boca Raton (2002)"},{"key":"7_CR55","doi-asserted-by":"crossref","unstructured":"Keelan, B.W., Urabe, H.: ISO 20462: a psychophysical image quality measurement standard. In: Electronic Imaging 2004, pp. 181\u2013189. International Society for Optics and Photonics (2003)","DOI":"10.1117\/12.532064"},{"key":"7_CR56","doi-asserted-by":"crossref","unstructured":"Keimel, C., Habigt, J., Diepold, K.: Challenges in crowd-based video quality assessment. In: Forth International Workshop on Quality of Multimedia Experience (QoMEX 2012), Yarra Valey, Australia, July 2012","DOI":"10.1109\/QoMEX.2012.6263866"},{"key":"7_CR57","doi-asserted-by":"crossref","unstructured":"Keimel, C., Habigt, J., Horch, C., Diepold, K.: QualityCrowd - a framework for crowd-based quality evaluation. In: Picture Coding Symposium, Krakow, PL, May 2012","DOI":"10.1109\/PCS.2012.6213338"},{"key":"7_CR58","doi-asserted-by":"crossref","unstructured":"Keimel, C., Habigt, J., Horch, C., Diepold, K.: Video quality evaluation in the cloud. In: Packet Video Workshop, Munich, DE, May 2012","DOI":"10.1109\/PV.2012.6229729"},{"key":"7_CR59","doi-asserted-by":"crossref","unstructured":"Korshunov, P., Cai, S., Ebrahimi, T.: Crowdsourcing approach for evaluation of privacy filters in video surveillance. In: 1st International ACM workshop on Crowdsourcing for Multimedia (CrowdMM 2012). ACM, Nara, October 2012","DOI":"10.1145\/2390803.2390817"},{"key":"7_CR60","unstructured":"Korshunov, P., Nemoto, H., Skodras, A., Ebrahimi, T.: The effect of HDR images on privacy: crowdsourcing evaluation. In: SPIE Photonics Europe 2014, Optics, Photonics and Digital Technologies for Multimedia Applications, Brussels, Belgium, April 2014"},{"key":"7_CR61","unstructured":"Kraft, S., Z\u00f6lzer, U.: BeaqleJS: HTML5 and JavaScript based framework for the subjective evaluation of audio quality. In: Linux Audio Conference, Karlsruhe, DE, May 2014"},{"key":"7_CR62","unstructured":"Kubey, R., Csikszentmihalyi, M.: Television and the Quality of Life: How Viewing Shapes Everyday Experience. A Volume in the Communication Series. L. Erlbaum Associates (1990). http:\/\/books.google.at\/books?id=zk_Zg5fJSVwC"},{"issue":"4","key":"7_CR63","doi-asserted-by":"crossref","first-page":"58","DOI":"10.1109\/MCOM.2012.6178834","volume":"50","author":"K Laghari","year":"2012","unstructured":"Laghari, K., Crespi, N., Connelly, K.: Toward total quality of experience: a QoE model in a communication ecosystem. IEEE Commun. Mag. 50(4), 58\u201365 (2012)","journal-title":"IEEE Commun. Mag."},{"key":"7_CR64","doi-asserted-by":"crossref","unstructured":"Lebreton, P.R., M\u00e4ki, T., Skodras, E., Hupont, I., Hirth, M.: Bridging the gap between eye tracking and crowdsourcing. In: Human Vision and Electronic Imaging XX, San Francisco, California, USA, 9\u201312 February 2015, p. 93940W (2015)","DOI":"10.1117\/12.2076745"},{"issue":"5","key":"7_CR65","doi-asserted-by":"crossref","first-page":"882","DOI":"10.1109\/TMM.2011.2157333","volume":"13","author":"JS Lee","year":"2011","unstructured":"Lee, J.S., De Simone, F., Ebrahimi, T.: Subjective quality evaluation via paired comparison: application to scalable video coding. IEEE Trans. Multimedia 13(5), 882\u2013893 (2011)","journal-title":"IEEE Trans. Multimedia"},{"key":"7_CR66","doi-asserted-by":"crossref","unstructured":"Li, J., Barkowsky, M., Le Callet, P.: Boosting paired comparison methodology in measuring visual discomfort of 3DTV: performances of three different designs. In: Proceeding SPIE Electronic Imaging-Stereoscopic Displays and Applications XXIV (2013)","DOI":"10.1117\/12.2002075"},{"key":"7_CR67","doi-asserted-by":"crossref","unstructured":"Little, G., Chilton, L., Goldman, M., Miller, R.: TurKit: tools for iterative tasks on mechanical turk. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 29\u201330. ACM (2009)","DOI":"10.1145\/1600150.1600159"},{"key":"7_CR68","doi-asserted-by":"crossref","unstructured":"Mayo, C., Aubanel, V., Cooke, M.: Effect of prosodic changes on speech intelligibility. In: Interspeech. Citeseer (2012)","DOI":"10.21437\/Interspeech.2012-467"},{"key":"7_CR69","doi-asserted-by":"crossref","unstructured":"Mazza, F., Da Silva, M.P., Le Callet, P.: Would you hire me? Selfie portrait images perception in a recruitment context. In: IS&T\/SPIE Electronic Imaging, p. 90140X. International Society for Optics and Photonics (2014)","DOI":"10.1117\/12.2042411"},{"key":"7_CR70","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-11548-6","volume-title":"Quality Engineering - Qualit\u00e4t kommunikationstechnischer Systeme","author":"S M\u00f6ller","year":"2010","unstructured":"M\u00f6ller, S.: Quality Engineering - Qualit\u00e4t kommunikationstechnischer Systeme. Springer, Heidelberg (2010). doi: 10.1007\/978-3-642-11548-6"},{"key":"7_CR71","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1016\/S0167-6393(01)00043-7","volume":"38","author":"S M\u00f6ller","year":"2002","unstructured":"M\u00f6ller, S., Raake, A.: Telephone speech quality prediction: towards network planning and monitoring models for modern network scenarios. Speech Commun. 38, 47\u201375 (2002). http:\/\/portal.acm.org\/citation.cfm?id=638078.638082 , ACM ID: 638082","journal-title":"Speech Commun."},{"issue":"1","key":"7_CR72","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1007\/BF02313422","volume":"16","author":"F Mosteller","year":"1951","unstructured":"Mosteller, F.: Remarks on the method of paired comparisons: I. The least squares solution assuming equal standard deviations and equal correlations. Psychometrika 16(1), 3\u20139 (1951)","journal-title":"Psychometrika"},{"key":"7_CR73","unstructured":"Naderi, B., Polzehl, T., Beyer, A., Pilz, T., M\u00f6ller, S.: Crowdee: mobile crowdsourcing micro-task platform - for celebrating the diversity of languages. In: Proceeding of 15th Annual Conference of the International Speech Communication Assocation (Interspeech 2014) (2014)"},{"key":"7_CR74","doi-asserted-by":"crossref","unstructured":"Naderi, B., Polzehl, T., Wechsung, I., K\u00f6ster, F., M\u00f6ller, S.: Effect of trapping questions on the reliability of speech quality judgments in a crowdsourcing paradigm. In: 16th Annual Conference of the International Speech Communication Assocation (Interspeech 2015), ISCA, pp. 2799\u20132803 (2015)","DOI":"10.21437\/Interspeech.2015-589"},{"issue":"4","key":"7_CR75","doi-asserted-by":"crossref","first-page":"287","DOI":"10.1109\/JIOT.2014.2387771","volume":"2","author":"Y Ouyang","year":"2015","unstructured":"Ouyang, Y., Yan, T., Wang, G.: CrowdMi: scalable and diagnosable mobile voice quality assessment through wireless analytics. IEEE Internet Things J. 2(4), 287\u2013294 (2015)","journal-title":"IEEE Internet Things J."},{"key":"7_CR76","doi-asserted-by":"crossref","first-page":"41","DOI":"10.2307\/1251430","volume":"49","author":"A Parasuraman","year":"1985","unstructured":"Parasuraman, A., Zeithaml, V.A., Berry, L.L.: A conceptual model of service quality and its implications for future research. J. Market. 49, 41\u201350 (1985)","journal-title":"J. Market."},{"key":"7_CR77","unstructured":"Pitrey, Y., Engelke, U., Barkowsky, M., P\u00e9pion, R., Le Callet, P.: Aligning subjective tests using a low cost common set. In: Euro ITV, IRCCyN-Contribution (2011)"},{"key":"7_CR78","doi-asserted-by":"crossref","unstructured":"Polzehl, T., Naderi, B., K\u00f6ster, F., M\u00f6ller, S.: Robustness in speech quality assessment and temporal training expiry in mobile crowdsourcing environments. In: 16th Annual Conference of the International Speech Communication Assocation (Interspeech 2015), ISCA, pp. 2794\u20132798 (2015)","DOI":"10.21437\/Interspeech.2015-588"},{"key":"7_CR79","doi-asserted-by":"crossref","DOI":"10.1002\/9780470033005","volume-title":"Speech Quality of VoIP: Assessment and Prediction","author":"A Raake","year":"2006","unstructured":"Raake, A.: Speech Quality of VoIP: Assessment and Prediction. Wiley, New York (2006)"},{"key":"7_CR80","doi-asserted-by":"crossref","unstructured":"Rainer, B., Waltl, M., Timmerer, C.: A web based subjective evaluation platform. In: Workshop on Quality of Multimedia Experience, Klagenfurth, AT, July 2013","DOI":"10.1109\/QoMEX.2013.6603196"},{"key":"7_CR81","doi-asserted-by":"crossref","unstructured":"Redi, J., Ho\u00dffeld, T., Korshunov, P., Mazza, F., Povoa, I., Keimel, C.: Crowdsourcing-based multimedia subjective evaluations: a case study on image recognizability and aesthetic appeal. In: Workshop on Crowdsourcing for Multimedia, Barcelona, ES, October 2013","DOI":"10.1145\/2506364.2506368"},{"key":"7_CR82","doi-asserted-by":"crossref","unstructured":"Redi, J., Liu, H., Alers, H., Zunino, R., Heynderickx, I.: Comparing subjective image quality measurement methods for the creation of public databases. In: IS&T\/SPIE Electronic Imaging, p. 752903. International Society for Optics and Photonics (2010)","DOI":"10.1117\/12.839195"},{"key":"7_CR83","doi-asserted-by":"crossref","unstructured":"Redi, J., Povoa, I.: Crowdsourcing for rating image aesthetic appeal: Better a paid or a volunteer crowd? In: 3rd International ACM workshop on Crowdsourcing for Multimedia (CrowdMM 2014), Orlando, FL, USA, November 2014","DOI":"10.1145\/2660114.2660118"},{"key":"7_CR84","doi-asserted-by":"crossref","unstructured":"Redi, J., Siahaan, E., Korshunov, P., Habigt, J., Hossfeld, T.: When the crowd challenges the lab: lessons learnt from subjective studies on image aesthetic appeal. In: Proceedings of the Fourth International Workshop on Crowdsourcing for Multimedia, pp. 33\u201338. ACM (2015)","DOI":"10.1145\/2810188.2810194"},{"issue":"3\u20134","key":"7_CR85","doi-asserted-by":"crossref","first-page":"189","DOI":"10.1007\/s12243-009-0144-8","volume":"65","author":"P Reichl","year":"2010","unstructured":"Reichl, P.: From charging for Quality of Aervice to charging for Quality of Experience. Annales des T\u00e9l\u00e9communications 65(3\u20134), 189\u2013199 (2010)","journal-title":"Annales des T\u00e9l\u00e9communications"},{"key":"7_CR86","doi-asserted-by":"crossref","unstructured":"Rerabek, M., Yuan, L., Krasula, L., Korshunov, P., Fliegel, K., Ebrahimi, T.: Evaluation of privacy in high dynamic range video sequences. In: SPIE Optical Engineering + Applications. International Society for Optics and Photonics, San Diego, CA, USA, August 2014","DOI":"10.1117\/12.2065559"},{"key":"7_CR87","doi-asserted-by":"crossref","unstructured":"Ribeiro, F., Florencio, D., Nascimento, V.: Crowdsourcing subjective image quality evaluation. In: Image Processing. Brussels, BE, September 2011","DOI":"10.1109\/ICIP.2011.6116320"},{"key":"7_CR88","doi-asserted-by":"crossref","unstructured":"Ribeiro, F., Florencio, D., Zhang, C., Seltzer, M.: CrowdMOS: an approach for crowdsourcing mean opinion score studies. In: International Conference on Acoustics, Speech and Signal Processing. Prague, CZ, May 2011","DOI":"10.1109\/ICASSP.2011.5946971"},{"issue":"1","key":"7_CR89","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1117\/1.1335529","volume":"10","author":"H Ridder de","year":"2001","unstructured":"de Ridder, H.: Cognitive issues in image quality measurement. J. Electron. Imaging 10(1), 47\u201355 (2001)","journal-title":"J. Electron. Imaging"},{"issue":"453","key":"7_CR90","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1198\/016214501750332668","volume":"96","author":"PE Rossi","year":"2001","unstructured":"Rossi, P.E., Gilula, Z., Allenby, G.M.: Overcoming scale usage heterogeneity. J. Am. Stat. Assoc. 96(453), 20\u201331 (2001). http:\/\/www.tandfonline.com\/doi\/abs\/10.1198\/016214501750332668","journal-title":"J. Am. Stat. Assoc."},{"key":"7_CR91","doi-asserted-by":"crossref","unstructured":"Rubino, G.: Quantifying the quality of audio and video transmissions over the internet: the PSQA approach. In: Design and Operations of Communication Networks: A Review of Wired and Wireless Modelling and Management Challenges. Imperial College Press (2005)","DOI":"10.1142\/9781860948947_0014"},{"key":"7_CR92","doi-asserted-by":"crossref","unstructured":"Sackl, A., Schatz, R.: Evaluating the impact of expectations on end-user quality perception. In: Proceedings of International Workshop Perceptual Quality of Systems (PQS), pp. 122\u2013128 (2013)","DOI":"10.21437\/PQS.2013-22"},{"key":"7_CR93","doi-asserted-by":"crossref","unstructured":"Sanchez-Iborra, R., JPC Rodrigues, J., Cano, M.D., Moreno-Urrea, S.: QoE measurements and analysis for VoIP services. Emerging Research on Networked Multimedia Communication Systems, p. 285 (2015)","DOI":"10.4018\/978-1-4666-8850-6.ch009"},{"key":"7_CR94","doi-asserted-by":"crossref","unstructured":"Schatz, R., Egger, S.: Vienna surfing - assessing mobile broadband quality in the field. In: Taft, N., Wetherall, D. (eds.) Proceedings of the 1st ACM SIGCOMM Workshop on Measurements Up the STack (W-MUST). ACM (2011)","DOI":"10.1145\/2018602.2018608"},{"key":"7_CR95","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"219","DOI":"10.1007\/978-3-642-36784-7_10","volume-title":"Data Traffic Monitoring and Analysis","author":"R Schatz","year":"2013","unstructured":"Schatz, R., Ho\u00dffeld, T., Janowski, L., Egger, S.: From packets to people: quality of experience as a new measurement challenge. In: Biersack, E., Callegari, C., Matijasevic, M. (eds.) Data Traffic Monitoring and Analysis. LNCS, vol. 7754, pp. 219\u2013263. Springer, Heidelberg (2013). doi: 10.1007\/978-3-642-36784-7_10"},{"key":"7_CR96","unstructured":"Schnitzer, S., Rensing, C., Schmidt, S., Borchert, K., Hirth, M., Tran-Gia, P.: Demands on task recommendation in crowdsourcing platforms - the workers perspective. In: CrowdRec Workshop, Vienna, Austria (9 2015)"},{"key":"7_CR97","doi-asserted-by":"crossref","unstructured":"Shaikh, J., Fiedler, M., Paul, P., Egger, S., Guyard, F.: Back to normal? Impact of temporally increasing network disturbances on QoE. In: 2013 IEEE Globecom Workshops (GC Workshops), pp. 1186\u20131191. IEEE (2013)","DOI":"10.1109\/GLOCOMW.2013.6825154"},{"key":"7_CR98","unstructured":"Silverstein, D.A., Farrell, J.E.: Quantifying perceptual image quality. In: PICS, vol. 98, pp. 242\u2013246 (1998)"},{"key":"7_CR99","doi-asserted-by":"crossref","DOI":"10.1002\/9780470034057","volume-title":"QoS and QoE management in UMTS cellular systems","author":"D Soldani","year":"2006","unstructured":"Soldani, D., Li, M., Cuny, R.: QoS and QoE management in UMTS cellular systems. Wiley, West Sussex (2006)"},{"issue":"4","key":"7_CR100","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1037\/h0070288","volume":"34","author":"LL Thurstone","year":"1927","unstructured":"Thurstone, L.L.: A law of comparative judgment. Psychol. Rev. 34(4), 273 (1927)","journal-title":"Psychol. Rev."},{"key":"7_CR101","unstructured":"Tsukida, K., Gupta, M.R.: How to analyze paired comparison data. Technical report, DTIC Document (2011)"},{"key":"7_CR102","doi-asserted-by":"crossref","unstructured":"Varela, M., M\u00e4ki, T., Skorin-Kapov, L., Ho\u00dffeld, T.: Increasing payments in crowdsourcing: don\u2019t look a gift horse in the mouth. In: 4th International Workshop on Perceptual Quality of Systems (PQS 2013), Vienna, Austria (2013)","DOI":"10.21437\/PQS.2013-4"},{"key":"7_CR103","doi-asserted-by":"crossref","unstructured":"Varela, M., M\u00e4ki, T., Skorin-Kapov, L., Ho\u00dffeld, T.: Towards an understanding of visual appeal in website design. In: 2013 Fifth International Workshop on Quality of Multimedia Experience (QoMEX), pp. 70\u201375. IEEE (2013)","DOI":"10.1109\/QoMEX.2013.6603213"},{"key":"7_CR104","doi-asserted-by":"crossref","unstructured":"Varela, M., Skorin-Kapov, L., M\u00e4ki, T., Ho\u00dffeld, T.: QoE in the web: a dance of design and performance. In: 2015 Seventh International Workshop on Quality of Multimedia Experience (QoMEX), pp. 1\u20137. IEEE (2015)","DOI":"10.1109\/QoMEX.2015.7148084"},{"key":"7_CR105","unstructured":"Virtanen, M., Gleiss, N., Goldstein, M.: On the use of evaluative category scales in telecommunications. In: Human Factors in Telecommunications (1995)"},{"issue":"1","key":"7_CR106","doi-asserted-by":"crossref","first-page":"29","DOI":"10.3758\/BF03195547","volume":"36","author":"F Wickelmaier","year":"2004","unstructured":"Wickelmaier, F., Schmid, C.: A matlab function to estimate choice model parameters from paired-comparison data. Behav. Res.h Methods Instrum. Comput. 36(1), 29\u201340 (2004)","journal-title":"Behav. Res.h Methods Instrum. Comput."},{"issue":"3","key":"7_CR107","doi-asserted-by":"crossref","first-page":"660","DOI":"10.1109\/TBC.2008.2000733","volume":"54","author":"S Winkler","year":"2008","unstructured":"Winkler, S., Mohandas, P.: The evolution of video quality measurement: from PSNR to hybrid metrics. IEEE Trans. Broadcast. 54(3), 660\u2013668 (2008). http:\/\/ieeexplore.ieee.org\/lpdocs\/epic03\/wrapper.htm?arnumber=4550731","journal-title":"IEEE Trans. Broadcast."},{"key":"7_CR108","unstructured":"Wolters, M.K., Isaac, K.B., Renals, S.: Evaluating speech synthesis intelligibility using Amazon mechanical turk. In: 7th Speech Synthesis Workshop (2010)"},{"issue":"5","key":"7_CR109","doi-asserted-by":"crossref","first-page":"1121","DOI":"10.1109\/TMM.2013.2241043","volume":"15","author":"CC Wu","year":"2013","unstructured":"Wu, C.C., Chen, K.T., Chang, Y.C., Lei, C.L.: Crowdsourcing multimedia QoE evaluation: a trusted framework. IEEE Trans. Multimedia 15(5), 1121\u20131137 (2013)","journal-title":"IEEE Trans. Multimedia"},{"issue":"3","key":"7_CR110","doi-asserted-by":"crossref","first-page":"844","DOI":"10.1109\/TMM.2012.2190924","volume":"14","author":"Q Xu","year":"2012","unstructured":"Xu, Q., Huang, Q., Jiang, T., Yan, B., Lin, W., Yao, Y.: HodgeRank on random graphs for subjective video quality assessment. Trans. Multimedia 14(3), 844\u2013857 (2012)","journal-title":"Trans. Multimedia"},{"key":"7_CR111","doi-asserted-by":"crossref","unstructured":"Yu-Chuan, Y., Chu, C.Y., Yeh, S.L., Chu, H.H., Huang, P.: Lab experiment vs. crowdsourcing: a comparative user study on Skype call quality. In Proceedings of the 9th Asian Internet Engineering Conference, pp. 65\u201372 (2013)","DOI":"10.1145\/2534142.2534151"},{"key":"7_CR112","doi-asserted-by":"crossref","unstructured":"Zinner, T., Hirth, M., Fischer, V., Hohlfeld, O.: Erwin - enabling the reproducible investigation of waiting times for arbitrary workflows. In: 8th International Conference on Quality of Multimedia Experiene (QoMEX), Lisbon, Portugal, June 2016","DOI":"10.1109\/QoMEX.2016.7498938"}],"container-title":["Lecture Notes in Computer Science","Evaluation in the Crowd. Crowdsourcing and Human-Centered Experiments"],"original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-319-66435-4_7","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,25]],"date-time":"2025-06-25T22:31:50Z","timestamp":1750890710000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-319-66435-4_7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017]]},"ISBN":["9783319664347","9783319664354"],"references-count":112,"URL":"https:\/\/doi.org\/10.1007\/978-3-319-66435-4_7","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2017]]}}}