{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,22]],"date-time":"2025-12-22T14:53:34Z","timestamp":1766415214490,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2020,10,12]],"date-time":"2020-10-12T00:00:00Z","timestamp":1602460800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2020,10,12]]},"DOI":"10.1145\/3394171.3413749","type":"proceedings-article","created":{"date-parts":[[2020,10,12]],"date-time":"2020-10-12T13:10:44Z","timestamp":1602508244000},"page":"852-861","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Describing Subjective Experiment Consistency by p-Value P--P Plot"],"prefix":"10.1145","author":[{"given":"Jakub","family":"Nawala","sequence":"first","affiliation":[{"name":"AGH University of Science and Technology, Krak\u00f3w, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Lucjan","family":"Janowski","sequence":"additional","affiliation":[{"name":"AGH University of Science and Technology, Krak\u00f3w, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bogdan","family":"Cmiel","sequence":"additional","affiliation":[{"name":"AGH University of Science and Technology, Krak\u00f3w, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Krzysztof","family":"Rusek","sequence":"additional","affiliation":[{"name":"AGH University of Science and Technology, Krak\u00f3w, Poland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2020,10,12]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"crossref","unstructured":"A. Agresti. 2002. Categorical Data Analysis (2th ed.). Wiley.  A. Agresti. 2002. Categorical Data Analysis (2th ed.). Wiley.","DOI":"10.1002\/0471249688"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240508.3240664"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1117\/1.JEI.27.5.053013"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.landurbplan.2012.01.005"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"crossref","unstructured":"B. Efron and R. J. Tibshirani. 1993. An Introduction to the Bootstrap. Chapman and Hall.  B. Efron and R. J. Tibshirani. 1993. An Introduction to the Bootstrap. Chapman and Hall.","DOI":"10.1007\/978-1-4899-4541-9"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/MNET.2010.5430142"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"crossref","unstructured":"Stephen Fleming and Nathaniel D. Daw. 2017. Self-Evaluation of DecisionMaking: A General Bayesian Framework for Metacognitive Computation. Psychological Review 124 (Jan 2017) 91--114. https:\/\/doi.org\/10.1037\/rev0000045  Stephen Fleming and Nathaniel D. Daw. 2017. Self-Evaluation of DecisionMaking: A General Bayesian Framework for Metacognitive Computation. Psychological Review 124 (Jan 2017) 91--114. https:\/\/doi.org\/10.1037\/rev0000045","DOI":"10.1037\/rev0000045"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1117\/1.JEI.27.5.053020"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2017.7965671"},{"key":"e_1_3_2_2_10_1","unstructured":"Tobias Ho\u00dffeld Poul E. Heegaard Martin Varela and Sebastian M\u00f6ller. 2016. Formal Definition of QoE Metrics. (2016) 1--23. https:\/\/doi.org\/10.1007\/s41233- 016-0002--1 arXiv:1607.00321  Tobias Ho\u00dffeld Poul E. Heegaard Martin Varela and Sebastian M\u00f6ller. 2016. Formal Definition of QoE Metrics. (2016) 1--23. https:\/\/doi.org\/10.1007\/s41233- 016-0002--1 arXiv:1607.00321"},{"key":"e_1_3_2_2_11_1","unstructured":"Tobias Hossfeld Poul E. Heegaard Martin Varela and Lea Skorin-Kapov. 2018. Confidence Interval Estimators for MOS Values. (2018). arXiv:1806.01126 http: \/\/arxiv.org\/abs\/1806.01126  Tobias Hossfeld Poul E. Heegaard Martin Varela and Lea Skorin-Kapov. 2018. Confidence Interval Estimators for MOS Values. (2018). arXiv:1806.01126 http: \/\/arxiv.org\/abs\/1806.01126"},{"volume-title":"4th International Workshop on Quality of Experience Management (QoE Management 2020), featured by IEEE Conference on Network Softwarization (IEEE NetSoft 2020","year":"2020","author":"Ho\u00dffeld Tobias","key":"e_1_3_2_2_12_1"},{"key":"e_1_3_2_2_13_1","first-page":"131","article-title":"SOS: The MOS is not enough! 2011 3rd International Workshop on Quality of Multimedia Experience","volume":"2011","author":"Ho\u00dffeld Tobias","year":"2011","journal-title":"QoMEX"},{"key":"e_1_3_2_2_14_1","unstructured":"ITU-T Study Group 12. 1998. ITU-T Coded-Speech Database. http:\/\/handle.itu. int\/11.1002\/1000\/4415  ITU-T Study Group 12. 1998. ITU-T Coded-Speech Database. http:\/\/handle.itu. int\/11.1002\/1000\/4415"},{"key":"e_1_3_2_2_15_1","unstructured":"Lucjan Janowski Jakub Nawa?a Werner Robitza Zhi Li Luk\u00e1 Krasula and Krzysztof Rusek. 2019. Notation for Subject Answer Analysis. arXiv:1903.05940 [cs.MM]  Lucjan Janowski Jakub Nawa?a Werner Robitza Zhi Li Luk\u00e1 Krasula and Krzysztof Rusek. 2019. Notation for Subject Answer Analysis. arXiv:1903.05940 [cs.MM]"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/QOMEX.2009.5246979"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2014.6982327"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2015.2484963"},{"key":"e_1_3_2_2_19_1","unstructured":"Lucjan Janowski Krzysztof Rusek Jakub Nawaa and Zhi Li. 2019. Generalized Score Distribution. arXiv:1909.04369 [stat.ME]  Lucjan Janowski Krzysztof Rusek Jakub Nawaa and Zhi Li. 2019. Generalized Score Distribution. arXiv:1909.04369 [stat.ME]"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2016.2638681"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2018.8463400"},{"volume-title":"Data Compression Conference Proceedings Part F127767 (2017","year":"2017","author":"Li Zhi","key":"e_1_3_2_2_22_1"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1088\/0004-637X\/757\/2\/135"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.concog.2011.09.021"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0043158"},{"key":"e_1_3_2_2_26_1","unstructured":"Lucjan Janowski Bogdan ?miel and Krzysztof Rusek. 2020. subjective-exp-consistency-check GitHub Repository. https:\/\/github.com\/Qub3k\/ subjective-exp-consistency-check  Lucjan Janowski Bogdan ?miel and Krzysztof Rusek. 2020. subjective-exp-consistency-check GitHub Repository. https:\/\/github.com\/Qub3k\/ subjective-exp-consistency-check"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"crossref","unstructured":"Tormod N\u00e6s Per B. Brockhoff and Oliver Tomic. 2010. Statistics for Sensory and Consumer Science. John Wiley & Sons Ltd.  Tormod N\u00e6s Per B. Brockhoff and Oliver Tomic. 2010. Statistics for Sensory and Consumer Science. John Wiley & Sons Ltd.","DOI":"10.1002\/9780470669181"},{"key":"e_1_3_2_2_31_1","unstructured":"Margaret H. Pinson and Lucjan Janowski. 2014. AGH\/NTIA: A Video Quality Subjective Test with Repeated Sequences. Technical Report NTIA Technical Memorandum 14--505. U.S. Department of Commerce National Telecommunications and Information Administration Institute for Telecommunication Sciences.  Margaret H. Pinson and Lucjan Janowski. 2014. AGH\/NTIA: A Video Quality Subjective Test with Repeated Sequences. Technical Report NTIA Technical Memorandum 14--505. U.S. Department of Commerce National Telecommunications and Information Administration Institute for Telecommunication Sciences."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2012.2215306"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3304109.3306231"},{"key":"e_1_3_2_2_34_1","unstructured":"David Robinson. 2014. How to interpret a p-value histogram. http:\/\/ varianceexplained.org\/statistics\/interpreting-pvalue-histogram\/  David Robinson. 2014. How to interpret a p-value histogram. http:\/\/ varianceexplained.org\/statistics\/interpreting-pvalue-histogram\/"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.foodqual.2007.06.008"},{"key":"e_1_3_2_2_36_1","unstructured":"Jonathan Rosenblatt. 2013. A Practitioner's Guide to Multiple Testing Error Rates. arXiv:1304.4920 [stat.ME]  Jonathan Rosenblatt. 2013. A Practitioner's Guide to Multiple Testing Error Rates. arXiv:1304.4920 [stat.ME]"},{"key":"e_1_3_2_2_37_1","unstructured":"T. Schweder and E. Spj\u00f8tvoll. 1982. Plots of P-Values to Evaluate Many Tests Simultaneously. Biometrika 69 3 (12 1982) 493--502. https:\/\/doi.org\/10.1093\/biomet\/ 69.3.493 arXiv:https:\/\/academic.oup.com\/biomet\/article-pdf\/69\/3\/493\/589074\/69- 3--493.pdf  T. Schweder and E. Spj\u00f8tvoll. 1982. Plots of P-Values to Evaluate Many Tests Simultaneously. Biometrika 69 3 (12 1982) 493--502. https:\/\/doi.org\/10.1093\/biomet\/ 69.3.493 arXiv:https:\/\/academic.oup.com\/biomet\/article-pdf\/69\/3\/493\/589074\/69- 3--493.pdf"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2019.8743296"},{"volume-title":"2012 IEEE Consumer Communications and Networking Conference (CCNC). 127--131","author":"Singh K. D.","key":"e_1_3_2_2_39_1"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2017.2652064"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.lwt.2005.09.014"},{"volume-title":"Applications of Digital Image Processing XLI, Andrew G","year":"2081","author":"Wang Haiqiang","key":"e_1_3_2_2_42_1"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v059.i10"},{"key":"e_1_3_2_2_44_1","first-page":"1","article-title":"Probability Plotting Methods for the Analysis of Data","volume":"55","author":"Wilk M. B.","year":"1968","journal-title":"Biometrika"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/2733373.2806269"},{"key":"e_1_3_2_2_46_1","first-page":"6","article-title":"On some biases encountered in modern listening tests","volume":"56","author":"Rumsey Francis","year":"2008","journal-title":"Journal of the Audio Engineering Society"}],"event":{"name":"MM '20: The 28th ACM International Conference on Multimedia","sponsor":["SIGMM ACM Special Interest Group on Multimedia"],"location":"Seattle WA USA","acronym":"MM '20"},"container-title":["Proceedings of the 28th ACM International Conference on Multimedia"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3394171.3413749","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3394171.3413749","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T22:01:16Z","timestamp":1750197676000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3394171.3413749"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,10,12]]},"references-count":43,"alternative-id":["10.1145\/3394171.3413749","10.1145\/3394171"],"URL":"https:\/\/doi.org\/10.1145\/3394171.3413749","relation":{},"subject":[],"published":{"date-parts":[[2020,10,12]]},"assertion":[{"value":"2020-10-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}