{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,29]],"date-time":"2026-09-29T11:16:07Z","timestamp":1790680567420,"version":"4.1.0"},"reference-count":50,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"7","license":[{"start":{"date-parts":[[2016,7,1]],"date-time":"2016-07-01T00:00:00Z","timestamp":1467331200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"funder":[{"DOI":"10.13039\/501100003246","name":"Netherlands Organisation for Scientific Research (NWO)","doi-asserted-by":"publisher","award":["639.021.230"],"award-info":[{"award-number":["639.021.230"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Multimedia"],"published-print":{"date-parts":[[2016,7]]},"DOI":"10.1109\/tmm.2016.2559942","type":"journal-article","created":{"date-parts":[[2016,4,28]],"date-time":"2016-04-28T16:19:09Z","timestamp":1461860349000},"page":"1338-1350","source":"Crossref","is-referenced-by-count":29,"title":["A Reliable Methodology to Collect Ground Truth Data of Image Aesthetic Appeal"],"prefix":"10.1109","volume":"18","author":[{"given":"Ernestasia","family":"Siahaan","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alan","family":"Hanjalic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Judith","family":"Redi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1007\/s11336-008-9101-0"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1109\/TBC.2010.2067710"},{"key":"ref33","first-page":"35","article-title":"Perceptual image quality: Concept and measurement","volume":"47","author":"roufs","year":"1992","journal-title":"Philips J Res"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1145\/2506364.2506368"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1117\/12.911894"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1117\/12.2006353"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1145\/127719.122752"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2014.6982326"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-22819-3_12"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2261481"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2012.2215306"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2012.6247954"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1117\/12.839195"},{"key":"ref2","year":"2012","journal-title":"Methodology for the Subjective Assessment of the Quality of Television Pictures"},{"key":"ref1","year":"2008","journal-title":"Subjective video quality assessment methods for multimedia applications"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"564","DOI":"10.1109\/TMM.2013.2292590","article-title":"On designing paired comparison experiments for subjective multimedia quality assessment","volume":"16","author":"jong-seok","year":"2014","journal-title":"IEEE Trans Multimedia"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1201\/9780203910825"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2011.941851"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/JSTSP.2009.2015077"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1117\/12.532064","article-title":"ISO 20462: A psychophysical image quality measurement standard","volume":"5294","author":"keelan","year":"0","journal-title":"Proc SPIE"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-8659.2012.03188.x"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2072298.2072320"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2014.6982300"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/1348246.1348248"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1117\/1.1335529"},{"key":"ref40","article-title":"A lightweight evolutionary art package for Java","author":"stout","year":"2009"},{"key":"ref12","author":"engeldrum","year":"2000","journal-title":"Psychometric Scaling"},{"key":"ref13","doi-asserted-by":"crossref","DOI":"10.4324\/9780203774458","author":"gescheider","year":"2013","journal-title":"Psychophysics The Fundamental"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"23","DOI":"10.20982\/tqmp.08.1.p023","article-title":"Computing inter-rater reliability for observational data: An overview and tutorial","volume":"8","author":"hallgren","year":"2012","journal-title":"Tuts Quantitative Methods Psychology"},{"key":"ref15","article-title":"Best practices and recommendations for crowdsourced QoE—Lessons learned from the qualinet task force crowdsourcing","author":"hossfeld","year":"2014"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2291663"},{"key":"ref17","first-page":"131","article-title":"SOS: The MOS is not enough!","author":"hossfeld","year":"0","journal-title":"Proceedings of the International Workshop on Quality of Multimedia Experience"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TBC.2010.2086750"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1109\/QoMEX.2014.6982327"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1145\/1873951.1873990"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-53926-8_3"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/QOMEX.2009.5246977"},{"key":"ref5","author":"brunnstr\u00f6m","year":"2013","journal-title":"Qualinet White Paper on Definitions of Quality of Experience"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/2502081.2502102"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"1121","DOI":"10.1109\/TMM.2013.2241043","article-title":"Crowdsourcing multimedia QoE evaluation: A trusted framework","volume":"15","author":"chen-chi","year":"2013","journal-title":"IEEE Trans Multimedia"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2283468"},{"key":"ref9","first-page":"288","article-title":"Studying aesthetics in photographic images using a computational approach","author":"datta","year":"2006","journal-title":"Proc 9th Eur Conf Comput Vis"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1167\/10.2.18"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1117\/12.201231"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2282128"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/QOMEX.2009.5246961"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1037\/h0070288"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2013.2269899"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/QOMEX.2010.5517948"},{"key":"ref43","first-page":"342","article-title":"Impact of implicit and explicit affective labeling on a recommender system's performance","author":"tkal?i?","year":"0","journal-title":"Proc UMAP 2011 Workshops"}],"container-title":["IEEE Transactions on Multimedia"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6046\/7492217\/07460955.pdf?arnumber=7460955","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T16:11:00Z","timestamp":1642003860000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/7460955\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,7]]},"references-count":50,"journal-issue":{"issue":"7"},"URL":"https:\/\/doi.org\/10.1109\/tmm.2016.2559942","relation":{},"ISSN":["1520-9210","1941-0077"],"issn-type":[{"value":"1520-9210","type":"print"},{"value":"1941-0077","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,7]]}}}