{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,29]],"date-time":"2026-03-29T07:51:57Z","timestamp":1774770717923,"version":"3.50.1"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2021,1,6]],"date-time":"2021-01-06T00:00:00Z","timestamp":1609891200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,6]],"date-time":"2021-01-06T00:00:00Z","timestamp":1609891200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Behav Res"],"published-print":{"date-parts":[[2021,8]]},"DOI":"10.3758\/s13428-020-01499-w","type":"journal-article","created":{"date-parts":[[2021,1,7]],"date-time":"2021-01-07T01:25:45Z","timestamp":1609982745000},"page":"1609-1647","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4,"title":["Improving reverse correlation analysis of faces: Diagnostics of order effects, runs, rater agreement, and image pairs"],"prefix":"10.3758","volume":"53","author":[{"given":"Michael","family":"Kevane","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Birgit","family":"Koopmann-Holm","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2021,1,6]]},"reference":[{"issue":"6B","key":"1499_CR1","doi-asserted-by":"publisher","first-page":"1751","DOI":"10.1121\/1.1912577","volume":"49","author":"A Ahumada Jr","year":"1971","unstructured":"Ahumada Jr, A., & Lovell, J. (1971). Stimulus features in signal detection. The Journal of the Acoustical Society of America, 49(6B), 1751\u20131756.","journal-title":"The Journal of the Acoustical Society of America"},{"issue":"2","key":"1499_CR2","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1080\/14640746608400019","volume":"18","author":"AD Baddeley","year":"1966","unstructured":"Baddeley, A. D. (1966). The capacity for generating information by randomization. The Quarterly Journal of Experimental Psychology, 18(2), 119\u2013129.","journal-title":"The Quarterly Journal of Experimental Psychology"},{"issue":"1","key":"1499_CR3","doi-asserted-by":"publisher","first-page":"127","DOI":"10.2307\/1419124","volume":"73","author":"P Bakan","year":"1960","unstructured":"Bakan, P. (1960). Response-tendencies in attempts to generate random binaryseries. The American Journal of Psychology, 73(1), 127\u2013131.","journal-title":"The American Journal of Psychology"},{"issue":"6","key":"1499_CR4","first-page":"1656","volume":"40","author":"M Bar-Hillel","year":"2014","unstructured":"Bar-Hillel, M., Peer, E., & Acquisti, A. (2014). \u201cHeads or tails?\u201d\u2014A reachability bias in binary choice. Journal of Experimental Psychology:Learning, Memory, and Cognition, 40(6), 1656\u20131663.","journal-title":"Journal of Experimental Psychology:Learning, Memory, and Cognition"},{"issue":"5","key":"1499_CR5","doi-asserted-by":"publisher","first-page":"2059","DOI":"10.3758\/s13428-019-01232-2","volume":"51","author":"L Brinkman","year":"2019","unstructured":"Brinkman, L., Goffin, S., van de Schoot, R., van Haren, N. E., Dotsch, R., &Aarts, H. (2019). Quantifying the informational value of classification images. Behavior Research Methods, 51(5), 2059\u20132073.","journal-title":"Behavior Research Methods"},{"issue":"1","key":"1499_CR6","doi-asserted-by":"publisher","first-page":"333","DOI":"10.1080\/10463283.2017.1381469","volume":"28","author":"L Brinkman","year":"2017","unstructured":"Brinkman, L., Todorov, A., & Dotsch, R. (2017). Visualising mental representations: A primer on noise-based reverse correlation in social psychology. European Review of Social Psychology, 28(1), 333\u2013361.","journal-title":"European Review of Social Psychology"},{"issue":"5","key":"1499_CR7","doi-asserted-by":"publisher","first-page":"481","DOI":"10.1521\/soco.2018.36.5.481","volume":"36","author":"JA Brooks","year":"2018","unstructured":"Brooks, J. A., Stolier, R. M., & Freeman, J. B. (2018). Stereotypes bias visual prototypes for sex and emotion categories. Social Cognition, 36(5), 481\u2013493.","journal-title":"Social Cognition"},{"issue":"1","key":"1499_CR8","doi-asserted-by":"publisher","first-page":"92","DOI":"10.1177\/0956797616674999","volume":"28","author":"JL Brown-Iannuzzi","year":"2017","unstructured":"Brown-Iannuzzi, J. L., Dotsch, R., Cooley, E., & Payne, B. K. (2017). The relationship between mental representations of welfare recipients and attitudes toward welfare. Psychological Science, 28(1), 92\u2013103.","journal-title":"Psychological Science"},{"key":"1499_CR9","unstructured":"Chandran, A., Venezia, S. A., Mattan, B. D., Cloutier, J., & Kubota, J. T. (2018). Understanding status and race perception using reverse correlation image classification. Retrieved from osf.io\/ewb2k."},{"issue":"9","key":"1499_CR10","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1167\/5.9.1","volume":"5","author":"A Chauvin","year":"2005","unstructured":"Chauvin, A., Worsley, K. J., Schyns, P. G., Arguin, M., & Gosselin, F. (2005). Accurate statistical tests for smooth classification images. Journal of Vision, 5(9), 1\u20131.","journal-title":"Journal of Vision"},{"issue":"1","key":"1499_CR11","doi-asserted-by":"publisher","first-page":"338","DOI":"10.2466\/07.09.17.PR0.110.1.338-350","volume":"110","author":"M Ch\u00f3liz","year":"2012","unstructured":"Ch\u00f3liz, M., & Fern\u00e1ndez-Abascal, E. G. (2012). Recognition of emotional facial expressions: the role of facial and contextual information in the accuracy of recognition. Psychological Reports, 110(1), 338\u2013350.","journal-title":"Psychological Reports"},{"key":"1499_CR12","doi-asserted-by":"crossref","unstructured":"Cone, J., Brown-Iannuzzi, J. L., Lei, R., & Dotsch, R. (2020). Type I Error Is Inflated in the Two-Phase Reverse Correlation Procedure. Social Psychological and Personality Science, 1948550620938616.","DOI":"10.1177\/1948550620938616"},{"issue":"2","key":"1499_CR13","doi-asserted-by":"publisher","first-page":"322","DOI":"10.1037\/0033-2909.88.2.322","volume":"88","author":"AJ Conger","year":"1980","unstructured":"Conger, A. J. (1980). Integration and generalization of kappas for multipleraters. Psychological Bulletin, 88(2), 322\u2013328.","journal-title":"Psychological Bulletin"},{"key":"1499_CR14","unstructured":"R Core Team. (2014). R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria. URL http:\/\/www.R-project.org\/."},{"key":"1499_CR15","unstructured":"Dotsch, R. (2016). rcicr: Reverse-correlation image-classification toolbox. R package version 0.3.4.1."},{"issue":"5","key":"1499_CR16","doi-asserted-by":"publisher","first-page":"562","DOI":"10.1177\/1948550611430272","volume":"3","author":"R Dotsch","year":"2012","unstructured":"Dotsch, R., & Todorov, A. (2012). Reverse correlating social face perception. Social Psychological and Personality Science, 3(5), 562\u2013571.","journal-title":"Social Psychological and Personality Science"},{"key":"1499_CR17","doi-asserted-by":"crossref","unstructured":"Ekman, P., & Friesen, W. V. (1978). Manual for the Facial Action Coding System. Palo Alto: Consulting Psychologists Press.","DOI":"10.1037\/t27734-000"},{"key":"1499_CR18","unstructured":"Fleiss, J. L., Levin, B., & Paik, M. C. (2013). Statistical methods for rates and proportions: John Wiley & Sons."},{"issue":"5","key":"1499_CR19","doi-asserted-by":"publisher","first-page":"892","DOI":"10.1093\/poq\/nfn059","volume":"72","author":"M Galesic","year":"2008","unstructured":"Galesic, M., Tourangeau, R., Couper, M. P., & Conrad, F. G. (2008). Eye-tracking data: New insights on response order effects and other cognitive shortcuts in survey responding. Public Opinion Quarterly, 72(5), 892\u2013913.","journal-title":"Public Opinion Quarterly"},{"issue":"1","key":"1499_CR20","doi-asserted-by":"publisher","first-page":"201","DOI":"10.1080\/00221309.1940.10544078","volume":"23","author":"LD Goodfellow","year":"1940","unstructured":"Goodfellow, L. D. (1940). The human element in probability. The Journal of General Psychology, 23(1), 201\u2013205.","journal-title":"The Journal of General Psychology"},{"key":"1499_CR21","doi-asserted-by":"publisher","first-page":"386","DOI":"10.3389\/fpsyg.2013.00386","volume":"4","author":"R Imhoff","year":"2013","unstructured":"Imhoff, R., Woelki, J., Hanke, S., & Dotsch, R. (2013). Warmth and competence in your face! Visual encoding of stereotype content. Frontiers in Psychology, 4, 386.","journal-title":"Frontiers in Psychology"},{"key":"1499_CR22","doi-asserted-by":"publisher","first-page":"269","DOI":"10.1146\/annurev-psych-010416-044242","volume":"68","author":"RE Jack","year":"2017","unstructured":"Jack, R. E., & Schyns, P. G. (2017). Toward a social psychophysics of face communication. Annual Review of Psychology, 68, 269\u2013297.","journal-title":"Annual Review of Psychology"},{"issue":"6","key":"1499_CR23","doi-asserted-by":"publisher","first-page":"e0198745","DOI":"10.1371\/journal.pone.0198745","volume":"13","author":"JC Jackson","year":"2018","unstructured":"Jackson, J. C., Hester, N., & Gray, K. (2018). The faces of God in America: Revealing religious diversity across people and politics. PloS one, 13(6), e0198745.","journal-title":"PloS one"},{"issue":"3","key":"1499_CR24","doi-asserted-by":"publisher","first-page":"422","DOI":"10.1016\/j.cognition.2011.07.008","volume":"121","author":"JC Karremans","year":"2011","unstructured":"Karremans, J. C., Dotsch, R., & Corneille, O. (2011). Romantic relationship status biases memory of faces of attractive opposite-sex others: Evidence from a reverse-correlation paradigm. Cognition, 121(3), 422\u2013426.","journal-title":"Cognition"},{"key":"1499_CR25","doi-asserted-by":"publisher","first-page":"315","DOI":"10.1016\/j.appet.2017.05.004","volume":"116","author":"AL Kerrihard","year":"2017","unstructured":"Kerrihard, A. L., Khair, M. B., Blumberg, R., Feldman, C. H., & Wunderlich, S. M. (2017). The effects of acclimation to the United States and other demographic factors on responses to salt levels in foods: An examination utilizing face reader technology. Appetite, 116, 315\u2013322.","journal-title":"Appetite"},{"issue":"6","key":"1499_CR26","doi-asserted-by":"publisher","first-page":"1092","DOI":"10.1037\/a0037684","volume":"107","author":"B Koopmann-Holm","year":"2014","unstructured":"Koopmann-Holm, B., & Tsai, J. L. (2014). Focusing on the negative: Cultural differences in expressions of sympathy. Journal of Personality and Social Psychology, 107(6), 1092\u20131115.","journal-title":"Journal of Personality and Social Psychology"},{"key":"1499_CR27","doi-asserted-by":"crossref","unstructured":"Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. Biometrics, 159\u2013174.","DOI":"10.2307\/2529310"},{"issue":"4","key":"1499_CR28","doi-asserted-by":"publisher","first-page":"227","DOI":"10.1037\/npe0000028","volume":"7","author":"P Lewinski","year":"2014","unstructured":"Lewinski, P., den Uyl, T. M., & Butler, C. (2014). Automated facial coding: Validation of basic emotions and FACS AUs in FaceReader. Journal of Neuroscience, Psychology, and Economics, 7(4), 227\u2013236.","journal-title":"Journal of Neuroscience, Psychology, and Economics"},{"issue":"3","key":"1499_CR29","first-page":"392","volume":"13","author":"LL Lopes","year":"1987","unstructured":"Lopes, L. L., & Oden, G. C. (1987). Distinguishing between random and non random events. Journal of Experimental Psychology: Learning, Memory, and Cognition, 13(3), 392\u2013400.","journal-title":"Journal of Experimental Psychology: Learning, Memory, and Cognition"},{"key":"1499_CR30","doi-asserted-by":"crossref","unstructured":"Lundqvist, D., & Litton, J. (1998). The averaged Karolinska directed emotional faces. Stockholm: Karolinska Institute, Department of Clinical Neuroscience, Section Psychology.","DOI":"10.1037\/t27732-000"},{"issue":"4","key":"1499_CR31","doi-asserted-by":"publisher","first-page":"1122","DOI":"10.3758\/s13428-014-0532-5","volume":"47","author":"DS Ma","year":"2015","unstructured":"Ma, D. S., Correll, J., & Wittenbrink, B. (2015). The Chicago face database: A free stimulus set of faces and norming data. Behavior Research Methods, 47(4), 1122\u20131135.","journal-title":"Behavior Research Methods"},{"issue":"5","key":"1499_CR32","doi-asserted-by":"publisher","first-page":"914","DOI":"10.1093\/poq\/nfn050","volume":"72","author":"N Malhotra","year":"2008","unstructured":"Malhotra, N. (2008). Completion time and response order effects in web surveys. Public Opinion Quarterly, 72(5), 914\u2013934.","journal-title":"Public Opinion Quarterly"},{"issue":"2","key":"1499_CR33","doi-asserted-by":"publisher","first-page":"209","DOI":"10.1207\/s15516709cog2802_4","volume":"28","author":"MC Mangini","year":"2004","unstructured":"Mangini, M. C., & Biederman, I. (2004). Making the ineffable explicit: Estimating the information employed for face classifications. Cognitive Science, 28(2), 209\u2013226.","journal-title":"Cognitive Science"},{"issue":"1","key":"1499_CR34","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1167\/2.1.6","volume":"2","author":"RF Murray","year":"2002","unstructured":"Murray, R. F., Bennett, P. J., & Sekuler, A. B. (2002). Optimal methods for calculating classification images: Weighted sums. Journal of Vision, 2(1), 6\u20136.","journal-title":"Journal of Vision"},{"key":"1499_CR35","unstructured":"Noldus. (2019). FaceReader: Tool for automatic analysis of facial expression: Version 8.0. Wageningen, the Netherlands: Noldus Information Technology B.V."},{"issue":"32","key":"1499_CR36","doi-asserted-by":"publisher","first-page":"11087","DOI":"10.1073\/pnas.0805664105","volume":"105","author":"NN Oosterhof","year":"2008","unstructured":"Oosterhof, N. N., & Todorov, A. (2008). The functional basis of face evaluation. Proceedings of the National Academy of Sciences, 105(32), 11087\u201311092.","journal-title":"Proceedings of the National Academy of Sciences"},{"issue":"1","key":"1499_CR37","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1007\/s11002-018-9448-7","volume":"29","author":"LJ Paas","year":"2018","unstructured":"Paas, L. J., & Morren, M. (2018). Please do not answer if you are reading this:respondent attention in online panels. Marketing Letters, 29(1), 13\u201321. https:\/\/doi.org\/10.1007\/s11002-018-9448-7.","journal-title":"Marketing Letters"},{"issue":"2","key":"1499_CR38","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1016\/j.mehy.2005.02.019","volume":"65","author":"N Persaud","year":"2005","unstructured":"Persaud, N. (2005). Humans can consciously generate random number sequences: A possible test for artificial intelligence. Medical Hypotheses, 65(2), 211\u2013214.","journal-title":"Medical Hypotheses"},{"issue":"9","key":"1499_CR39","doi-asserted-by":"publisher","first-page":"1103","DOI":"10.1002\/sim.4780090917","volume":"9","author":"KL Posner","year":"1990","unstructured":"Posner, K. L., Sampson, P. D., Caplan, R. A., Ward, R. J., & Cheney, F. W. (1990). Measuring interrater reliability among multiple raters: an example of methods for nominal data. Statistics in Medicine, 9(9), 1103\u20131115.","journal-title":"Statistics in Medicine"},{"key":"1499_CR40","doi-asserted-by":"publisher","first-page":"1161","DOI":"10.1037\/h0077714","volume":"39","author":"JA Russell","year":"1980","unstructured":"Russell, J. A. (1980). A circumplex model of affect. Journal of Personality and Social Psychology, 39, 1161\u20131178.","journal-title":"Journal of Personality and Social Psychology"},{"key":"1499_CR41","doi-asserted-by":"publisher","first-page":"1383","DOI":"10.3758\/s13428-019-01295-1","volume":"52","author":"M Schmitz","year":"2019","unstructured":"Schmitz, M., Rougier, M., & Yzerbyt, V. (2019). Comment on \u201cQuantifyingthe informational value of classification images\u201d: A miscomputation of the InfoVal metric. Behavior Research Methods, 52, 1383\u20131386.","journal-title":"Behavior Research Methods"},{"key":"1499_CR42","doi-asserted-by":"publisher","first-page":"1800","DOI":"10.3758\/s13428-020-01367-7","volume":"58","author":"M Schmitz","year":"2020","unstructured":"Schmitz, M., Rougier, M., Yzerbyt, V., Brinkman, L., & Dotsch, R. (2020). Erratum to: Comment on \u201cQuantifying the informational value of classification images\u201d: Miscomputation of InfoVal metric was a minor issue and is now corrected. Behavior Research Methods, 58, 1800\u20131801.","journal-title":"Behavior Research Methods"},{"issue":"1","key":"1499_CR43","doi-asserted-by":"publisher","first-page":"116","DOI":"10.1509\/jmkr.45.1.116","volume":"45","author":"SD Swain","year":"2008","unstructured":"Swain, S. D., Weathers, D., & Niedrich, R. W. (2008). Assessing three sources of misresponse to reversed Likert items. Journal of Marketing Research, 45(1), 116\u2013131. doi: https:\/\/doi.org\/10.1509\/jmkr.45.1.116","journal-title":"Journal of Marketing Research"},{"issue":"2","key":"1499_CR44","doi-asserted-by":"publisher","first-page":"288","DOI":"10.1037\/0022-3514.90.2.288","volume":"90","author":"JL Tsai","year":"2006","unstructured":"Tsai, J. L., Knutson, B., & Fung, H. H. (2006). Cultural variation in affect valuation. Journal of Personality and Social Psychology, 90(2), 288\u2013307.","journal-title":"Journal of Personality and Social Psychology"},{"issue":"3","key":"1499_CR45","doi-asserted-by":"publisher","first-page":"477","DOI":"10.1007\/s11336-009-9116-1","volume":"74","author":"S Vanbelle","year":"2009","unstructured":"Vanbelle, S., & Albert, A. (2009). Agreement between two independent groups of raters. Psychometrika, 74(3), 477\u2013491.","journal-title":"Psychometrika"},{"issue":"1","key":"1499_CR46","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1037\/h0032060","volume":"77","author":"WA Wagenaar","year":"1972","unstructured":"Wagenaar, W. A. (1972). Generation of random sequences by human subjects: A critical survey of literature. Psychological Bulletin, 77(1), 65\u201372.","journal-title":"Psychological Bulletin"},{"issue":"2","key":"1499_CR47","doi-asserted-by":"publisher","first-page":"147","DOI":"10.1214\/aoms\/1177731909","volume":"11","author":"A Wald","year":"1940","unstructured":"Wald, A., & Wolfowitz, J. (1940). On a test whether two samples are from the same population. The Annals of Mathematical Statistics, 11(2), 147\u2013162.","journal-title":"The Annals of Mathematical Statistics"},{"key":"1499_CR48","doi-asserted-by":"crossref","unstructured":"Williams, G. W. (1976). Comparing the joint agreement of several raters with another rater. Biometrics, 619\u2013627.","DOI":"10.2307\/2529750"},{"issue":"2","key":"1499_CR49","doi-asserted-by":"publisher","first-page":"503","DOI":"10.1177\/0956797613510717","volume":"25","author":"AI Young","year":"2014","unstructured":"Young, A. I., Ratner, K. G., & Fazio, R. H. (2014). Political attitudes bias the mental representation of a presidential candidate\u2019s face. Psychological Science, 25(2), 503\u2013510.","journal-title":"Psychological Science"},{"issue":"2","key":"1499_CR50","first-page":"127","volume":"8","author":"C Zhang","year":"2014","unstructured":"Zhang, C., & Conrad, F. (2014). Speeding in web surveys: The tendency toanswer very fast and its association with straightlining. Survey Research Methods, 8(2), 127\u2013135.","journal-title":"Survey Research Methods"}],"container-title":["Behavior Research Methods"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.3758\/s13428-020-01499-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.3758\/s13428-020-01499-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.3758\/s13428-020-01499-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,6]],"date-time":"2022-01-06T01:03:08Z","timestamp":1641430988000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.3758\/s13428-020-01499-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,1,6]]},"references-count":50,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,8]]}},"alternative-id":["1499"],"URL":"https:\/\/doi.org\/10.3758\/s13428-020-01499-w","relation":{},"ISSN":["1554-3528"],"issn-type":[{"value":"1554-3528","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,1,6]]},"assertion":[{"value":"4 October 2020","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"6 January 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}