{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,11,28]],"date-time":"2025-11-28T12:29:18Z","timestamp":1764332958039,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":40,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,10,15]],"date-time":"2020-10-15T00:00:00Z","timestamp":1602720000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100004180","name":"Nippon Telegraph and Telephone","doi-asserted-by":"publisher","award":["20193641"],"award-info":[{"award-number":["20193641"]}],"id":[{"id":"10.13039\/501100004180","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,10,16]]},"DOI":"10.1145\/3423328.3423499","type":"proceedings-article","created":{"date-parts":[[2020,10,15]],"date-time":"2020-10-15T23:26:01Z","timestamp":1602804361000},"page":"19-27","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":3,"title":["Improving the Efficiency of QoE Crowdtesting"],"prefix":"10.1145","author":[{"given":"Ricky K. P.","family":"Mok","sequence":"first","affiliation":[{"name":"University of California, San Diego, La Jolla, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ginga","family":"Kawaguti","sequence":"additional","affiliation":[{"name":"NTT Corporation, Tokyo, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jun","family":"Okamoto","sequence":"additional","affiliation":[{"name":"NTT Corporation, Tokyo, Japan"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,10,15]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"[n.d.]. Clickworker. https:\/\/www.clickworker.com.  [n.d.]. Clickworker. https:\/\/www.clickworker.com."},{"key":"e_1_3_2_1_2_1","unstructured":"[n.d.]. Meteor. https:\/\/www.meteor.com.  [n.d.]. Meteor. https:\/\/www.meteor.com."},{"key":"e_1_3_2_1_3_1","unstructured":"[n.d.]. Microworkers. https:\/\/microworkers.com\/.  [n.d.]. Microworkers. https:\/\/microworkers.com\/."},{"key":"e_1_3_2_1_4_1","unstructured":"Amazon. [n.d.]. Mechanical Turk. https:\/\/www.mturk.com.  Amazon. [n.d.]. Mechanical Turk. https:\/\/www.mturk.com."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/GLOCOM.2014.7036964"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/IUCS.2010.5666762"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1631272.1631339"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2181037.2181040"},{"volume-title":"Proc. Crowdsourcing and HumanCentred Experiments","series-title":"Dagstuhl Seminar 15481","author":"Egger-Lampl Sebastian","key":"e_1_3_2_1_10_1"},{"volume-title":"de Vries","year":"2012","author":"Eickhoff Carsten","key":"e_1_3_2_1_11_1"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741639"},{"volume-title":"Mandryk","year":"2011","author":"Flatla David R.","key":"e_1_3_2_1_13_1"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICC.2014.6883463"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Dennis Guse Henrique R. Orefice Gabriel Reimers and Oliver Hohlfeld. 2019. TheFragebogen: A Web Browser-based Questionnaire Framework for Scientific Research. In QoMEX.  Dennis Guse Henrique R. Orefice Gabriel Reimers and Oliver Hohlfeld. 2019. TheFragebogen: A Web Browser-based Questionnaire Framework for Scientific Research. In QoMEX.","DOI":"10.1109\/QoMEX.2019.8743231"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2015.03.036"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/CCE.2014.6916756"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2291663"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1109\/PV.2012.6229729"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/PCS.2012.6213338"},{"volume-title":"Proc. ICWSM.","year":"2013","author":"Lee Tak Yeon","key":"e_1_3_2_1_21_1"},{"key":"e_1_3_2_1_22_1","unstructured":"M-Lab. [n.d.]. NDT (Network Diagnostic Tool). https:\/\/www.measurementlab. net\/tests\/ndt\/.  M-Lab. [n.d.]. NDT (Network Diagnostic Tool). https:\/\/www.measurementlab. net\/tests\/ndt\/."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1109\/BigData.2017.8258476"},{"volume-title":"Klaus Opwis, and Alexandre N. Tuch.","year":"2013","author":"Mekler Elisa D.","key":"e_1_3_2_1_24_1"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2016.2619901"},{"volume-title":"Proc. ACM SIGCOMM poster session.","author":"Mok Ricky","key":"e_1_3_2_1_26_1"},{"key":"e_1_3_2_1_27_1","unstructured":"ITU-T P.913. 2016. Methods for the subjective assessment of video quality audio quality and audiovisual quality of Internet video and distribution quality television in any environment. ITU-T Recommendation P.913.  ITU-T P.913. 2016. Methods for the subjective assessment of video quality audio quality and audiovisual quality of Internet video and distribution quality television in any environment. ITU-T Recommendation P.913."},{"volume-title":"Pinson and Stephen Wolf","year":"2003","author":"Margaret","key":"e_1_3_2_1_28_1"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2013.6603196"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2011.5946971"},{"key":"e_1_3_2_1_31_1","first-page":"63","article-title":"The Impact of Test Duration on User Fatigue and Reliability of Subjective Quality Ratings","volume":"60","author":"Schatz Raimund","year":"2012","journal-title":"Journal of the AES"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/COMST.2014.2360940"},{"key":"e_1_3_2_1_33_1","unstructured":"Telecommunication Technology Committee. 2018. TTC JJ-201.01 A Method for Speech Quality Assessment for IP Telephony.  Telecommunication Technology Committee. 2018. TTC JJ-201.01 A Method for Speech Quality Assessment for IP Telephony."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2999572.2999590"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1080\/10447318.2017.1287458"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2241043"},{"key":"e_1_3_2_1_37_1","unstructured":"Qianqian Xu Tingting Jiang Yuan Yao Qingming Huang Bowei Yan and Weisi Lin. 2011. Random Partial Paired Comparison for Subjective Video Quality Assessment via HodgeRank. In ACM MM.  Qianqian Xu Tingting Jiang Yuan Yao Qingming Huang Bowei Yan and Weisi Lin. 2011. Random Partial Paired Comparison for Subjective Video Quality Assessment via HodgeRank. In ACM MM."},{"key":"e_1_3_2_1_38_1","unstructured":"Qianqian Xu Ming Yan Chendi Huang Jiechao Xiong Qingming Huang and Yuan Yao. 2017. Exploring Outliers in Crowdsourced Ranking for QoE. In ACM MM.  Qianqian Xu Ming Yan Chendi Huang Jiechao Xiong Qingming Huang and Yuan Yao. 2017. Exploring Outliers in Crowdsourced Ranking for QoE. In ACM MM."},{"volume-title":"Chang","year":"2017","author":"Zhang Lingyan","key":"e_1_3_2_1_39_1"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1108\/OIR-08-2014-0188"}],"event":{"name":"MM '20: The 28th ACM International Conference on Multimedia","sponsor":["SIGMM ACM Special Interest Group on Multimedia"],"location":"Seattle WA USA","acronym":"MM '20"},"container-title":["Proceedings of the 1st Workshop on Quality of Experience (QoE) in Visual Multimedia Applications"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3423328.3423499","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3423328.3423499","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:24:57Z","timestamp":1750195497000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3423328.3423499"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,15]]},"references-count":40,"alternative-id":["10.1145\/3423328.3423499","10.1145\/3423328"],"URL":"https:\/\/doi.org\/10.1145\/3423328.3423499","relation":{},"subject":[],"published":{"date-parts":[[2020,10,15]]},"assertion":[{"value":"2020-10-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}