{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,23]],"date-time":"2025-09-23T13:43:08Z","timestamp":1758634988655,"version":"3.37.3"},"reference-count":50,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2016,7,1]],"date-time":"2016-07-01T00:00:00Z","timestamp":1467331200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"funder":[{"DOI":"10.13039\/501100003246","name":"Netherlands Organisation for Scientific Research (NWO)","doi-asserted-by":"publisher","award":["639.021.230"],"award-info":[{"award-number":["639.021.230"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Multimedia"],"published-print":{"date-parts":[[2016,7]]},"DOI":"10.1109\/tmm.2016.2559942","type":"journal-article","created":{"date-parts":[[2016,4,28]],"date-time":"2016-04-28T16:19:09Z","timestamp":1461860349000},"page":"1338-1350","source":"Crossref","is-referenced-by-count":27,"title":["A Reliable Methodology to Collect Ground Truth Data of Image Aesthetic Appeal"],"prefix":"10.1109","volume":"18","author":[{"given":"Ernestasia","family":"Siahaan","sequence":"first","affiliation":[]},{"given":"Alan","family":"Hanjalic","sequence":"additional","affiliation":[]},{"given":"Judith","family":"Redi","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"doi-asserted-by":"publisher","key":"ref39","DOI":"10.1007\/s11336-008-9101-0"},{"doi-asserted-by":"publisher","key":"ref38","DOI":"10.1109\/TBC.2010.2067710"},{"key":"ref33","first-page":"35","article-title":"Perceptual image quality: Concept and measurement","volume":"47","author":"roufs","year":"1992","journal-title":"Philips J Res"},{"doi-asserted-by":"publisher","key":"ref32","DOI":"10.1145\/2506364.2506368"},{"doi-asserted-by":"publisher","key":"ref31","DOI":"10.1117\/12.911894"},{"doi-asserted-by":"publisher","key":"ref30","DOI":"10.1117\/12.2006353"},{"doi-asserted-by":"publisher","key":"ref37","DOI":"10.1145\/127719.122752"},{"doi-asserted-by":"publisher","key":"ref36","DOI":"10.1109\/QoMEX.2014.6982326"},{"doi-asserted-by":"publisher","key":"ref35","DOI":"10.1007\/978-3-642-22819-3_12"},{"doi-asserted-by":"publisher","key":"ref34","DOI":"10.1109\/TMM.2013.2261481"},{"doi-asserted-by":"publisher","key":"ref28","DOI":"10.1109\/JSTSP.2012.2215306"},{"doi-asserted-by":"publisher","key":"ref27","DOI":"10.1109\/CVPR.2012.6247954"},{"doi-asserted-by":"publisher","key":"ref29","DOI":"10.1117\/12.839195"},{"year":"2012","journal-title":"Methodology for the Subjective Assessment of the Quality of Television Pictures","key":"ref2"},{"year":"2008","journal-title":"Subjective video quality assessment methods for multimedia applications","key":"ref1"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"564","DOI":"10.1109\/TMM.2013.2292590","article-title":"On designing paired comparison experiments for subjective multimedia quality assessment","volume":"16","author":"jong-seok","year":"2014","journal-title":"IEEE Trans Multimedia"},{"doi-asserted-by":"publisher","key":"ref22","DOI":"10.1201\/9780203910825"},{"doi-asserted-by":"publisher","key":"ref21","DOI":"10.1109\/MSP.2011.941851"},{"doi-asserted-by":"publisher","key":"ref24","DOI":"10.1109\/JSTSP.2009.2015077"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1117\/12.532064","article-title":"ISO 20462: A psychophysical image quality measurement standard","volume":"5294","author":"keelan","year":"0","journal-title":"Proc SPIE"},{"doi-asserted-by":"publisher","key":"ref26","DOI":"10.1111\/j.1467-8659.2012.03188.x"},{"doi-asserted-by":"publisher","key":"ref25","DOI":"10.1145\/2072298.2072320"},{"doi-asserted-by":"publisher","key":"ref50","DOI":"10.1109\/QoMEX.2014.6982300"},{"doi-asserted-by":"publisher","key":"ref10","DOI":"10.1145\/1348246.1348248"},{"doi-asserted-by":"publisher","key":"ref11","DOI":"10.1117\/1.1335529"},{"year":"2009","author":"stout","article-title":"A lightweight evolutionary art package for Java","key":"ref40"},{"year":"2000","author":"engeldrum","journal-title":"Psychometric Scaling","key":"ref12"},{"key":"ref13","doi-asserted-by":"crossref","DOI":"10.4324\/9780203774458","author":"gescheider","year":"2013","journal-title":"Psychophysics The Fundamental"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"23","DOI":"10.20982\/tqmp.08.1.p023","article-title":"Computing inter-rater reliability for observational data: An overview and tutorial","volume":"8","author":"hallgren","year":"2012","journal-title":"Tuts Quantitative Methods Psychology"},{"year":"2014","author":"hossfeld","article-title":"Best practices and recommendations for crowdsourced QoE—Lessons learned from the qualinet task force crowdsourcing","key":"ref15"},{"doi-asserted-by":"publisher","key":"ref16","DOI":"10.1109\/TMM.2013.2291663"},{"key":"ref17","first-page":"131","article-title":"SOS: The MOS is not enough!","author":"hossfeld","year":"0","journal-title":"Proceedings of the International Workshop on Quality of Multimedia Experience"},{"doi-asserted-by":"publisher","key":"ref18","DOI":"10.1109\/TBC.2010.2086750"},{"doi-asserted-by":"publisher","key":"ref19","DOI":"10.1109\/QoMEX.2014.6982327"},{"doi-asserted-by":"publisher","key":"ref4","DOI":"10.1145\/1873951.1873990"},{"doi-asserted-by":"publisher","key":"ref3","DOI":"10.1007\/978-3-642-53926-8_3"},{"doi-asserted-by":"publisher","key":"ref6","DOI":"10.1109\/QOMEX.2009.5246977"},{"year":"2013","author":"brunnstr\u00f6m","journal-title":"Qualinet White Paper on Definitions of Quality of Experience","key":"ref5"},{"doi-asserted-by":"publisher","key":"ref8","DOI":"10.1145\/2502081.2502102"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"1121","DOI":"10.1109\/TMM.2013.2241043","article-title":"Crowdsourcing multimedia QoE evaluation: A trusted framework","volume":"15","author":"chen-chi","year":"2013","journal-title":"IEEE Trans Multimedia"},{"doi-asserted-by":"publisher","key":"ref49","DOI":"10.1109\/TMM.2013.2283468"},{"key":"ref9","first-page":"288","article-title":"Studying aesthetics in photographic images using a computational approach","author":"datta","year":"2006","journal-title":"Proc 9th Eur Conf Comput Vis"},{"doi-asserted-by":"publisher","key":"ref46","DOI":"10.1167\/10.2.18"},{"doi-asserted-by":"publisher","key":"ref45","DOI":"10.1117\/12.201231"},{"doi-asserted-by":"publisher","key":"ref48","DOI":"10.1109\/TMM.2013.2282128"},{"doi-asserted-by":"publisher","key":"ref47","DOI":"10.1109\/QOMEX.2009.5246961"},{"doi-asserted-by":"publisher","key":"ref42","DOI":"10.1037\/h0070288"},{"doi-asserted-by":"publisher","key":"ref41","DOI":"10.1109\/TMM.2013.2269899"},{"doi-asserted-by":"publisher","key":"ref44","DOI":"10.1109\/QOMEX.2010.5517948"},{"key":"ref43","first-page":"342","article-title":"Impact of implicit and explicit affective labeling on a recommender system's performance","author":"tkal?i?","year":"0","journal-title":"Proc UMAP 2011 Workshops"}],"container-title":["IEEE Transactions on Multimedia"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6046\/7492217\/07460955.pdf?arnumber=7460955","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:11:00Z","timestamp":1642003860000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/7460955\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,7]]},"references-count":50,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tmm.2016.2559942","relation":{},"ISSN":["1520-9210","1941-0077"],"issn-type":[{"type":"print","value":"1520-9210"},{"type":"electronic","value":"1941-0077"}],"subject":[],"published":{"date-parts":[[2016,7]]}}}