{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T22:57:10Z","timestamp":1793141830010,"version":"4.5.0"},"reference-count":152,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2017,9,11]],"date-time":"2017-09-11T00:00:00Z","timestamp":1505088000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Scientometrics"],"published-print":{"date-parts":[[2017,11]]},"DOI":"10.1007\/s11192-017-2516-6","type":"journal-article","created":{"date-parts":[[2017,9,11]],"date-time":"2017-09-11T16:48:36Z","timestamp":1505148516000},"page":"1059-1092","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":31,"title":["Inter-rater reliability and validity of peer reviews in an interdisciplinary field"],"prefix":"10.1007","volume":"113","author":[{"given":"Jens","family":"Jirschitzka","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aileen","family":"Oeberst","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Richard","family":"G\u00f6llner","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ulrike","family":"Cress","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,9,11]]},"reference":[{"issue":"1","key":"2516_CR1","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1017\/S0140525X00065687","volume":"14","author":"KM Adams","year":"1991","unstructured":"Adams, K. M. (1991). Peer review: An unflattering picture. Behavioral and Brain Sciences, 14(1), 135\u2013136.","journal-title":"Behavioral and Brain Sciences"},{"key":"2516_CR2","unstructured":"Akerlof, G. A. (2003). Writing the \u201cThe Market for \u2018Lemons\u2019\u201d: A personal and interpretive essay. https:\/\/www.nobelprize.org\/nobel_prizes\/economic-sciences\/laureates\/2001\/akerlof-article.html . Accessed 4 Sept 2017."},{"issue":"3","key":"2516_CR3","doi-asserted-by":"crossref","first-page":"159","DOI":"10.3152\/147154403781776645","volume":"12","author":"DW Aksnes","year":"2003","unstructured":"Aksnes, D. W. (2003). Characteristics of highly cited papers. Research Evaluation, 12(3), 159\u2013170.","journal-title":"Research Evaluation"},{"key":"2516_CR4","doi-asserted-by":"crossref","first-page":"d2304","DOI":"10.1136\/bmj.d2304","volume":"343","author":"DG Altman","year":"2011","unstructured":"Altman, D. G., & Bland, J. M. (2011). How to obtain the P value from a confidence interval. BMJ, 343, d2304.","journal-title":"BMJ"},{"key":"2516_CR5","unstructured":"Anderson, K. (2012). The problems with calling comments \u201cPost-Publication Peer-Review\u201d [Web log message]. Retrieved from http:\/\/scholarlykitchen.sspnet.org\/2012\/03\/26\/the-problems-with-calling-comments-post-publication-peer-review ."},{"key":"2516_CR6","unstructured":"Asparouhov, T., & Muth\u00e9n, B. (2010). Bayesian analysis of latent variable models using Mplus. http:\/\/www.statmodel.com\/download\/BayesAdvantages18.pdf . Accessed 30 Mar 2017."},{"issue":"5","key":"2516_CR7","doi-asserted-by":"crossref","first-page":"e61401","DOI":"10.1371\/journal.pone.0061401","volume":"8","author":"C Baethge","year":"2013","unstructured":"Baethge, C., Franklin, J., & Mertens, S. (2013). Substantial agreement of referee recommendations at a general medical journal\u2014A peer review evaluation at Deutsches \u00c4rzteblatt International. PLoS ONE, 8(5), e61401.","journal-title":"PLoS ONE"},{"issue":"10","key":"2516_CR8","doi-asserted-by":"crossref","first-page":"654","DOI":"10.1056\/NEJM198503073121023","volume":"312","author":"JC Bailar","year":"1985","unstructured":"Bailar, J. C., & Patterson, K. (1985). Journal peer review\u2014The need for a research agenda. The New England Journal of Medicine, 312(10), 654\u2013657.","journal-title":"The New England Journal of Medicine"},{"issue":"1","key":"2516_CR9","doi-asserted-by":"crossref","first-page":"166","DOI":"10.1016\/j.ijforecast.2010.03.003","volume":"27","author":"WGG Benda","year":"2011","unstructured":"Benda, W. G. G., & Engels, T. C. E. (2011). The predictive validity of peer review: A selective review of the judgmental forecasting qualities of peers, and implications for innovation in science. International Journal of Forecasting, 27(1), 166\u2013182.","journal-title":"International Journal of Forecasting"},{"issue":"5","key":"2516_CR10","doi-asserted-by":"crossref","first-page":"1219","DOI":"10.2307\/256856","volume":"38","author":"JM Beyer","year":"1995","unstructured":"Beyer, J. M., Chanove, R. G., & Fox, W. B. (1995). Review process and the fates of manuscripts submitted to AMJ. Academy of Management Journal, 38(5), 1219\u20131260.","journal-title":"Academy of Management Journal"},{"issue":"5","key":"2516_CR11","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1111\/j.1467-9280.2006.01715.x","volume":"17","author":"JL Blackburn","year":"2006","unstructured":"Blackburn, J. L., & Hakel, M. D. (2006). An examination of sources of peer-review bias. Psychological Science, 17(5), 378\u2013382.","journal-title":"Psychological Science"},{"issue":"2","key":"2516_CR12","doi-asserted-by":"crossref","first-page":"297","DOI":"10.1007\/s11192-005-0214-2","volume":"63","author":"L Bornmann","year":"2005","unstructured":"Bornmann, L., & Daniel, H.-D. (2005). Selection of research fellowship recipients by committee peer review. Reliability, fairness and predictive validity of Board of Trustees\u2019 decisions. Scientomentrics, 63(2), 297\u2013320.","journal-title":"Scientomentrics"},{"issue":"11","key":"2516_CR13","doi-asserted-by":"crossref","first-page":"1841","DOI":"10.1002\/asi.20901","volume":"59","author":"L Bornmann","year":"2008","unstructured":"Bornmann, L., & Daniel, H.-D. (2008a). Selecting manuscripts for a high-impact journal through peer review: A citation analysis of communications that were accepted by Angewandte Chemie International Edition, or rejected but published elsewhere. Journal of the American Society for Information Science and Technology, 59(11), 1841\u20131852.","journal-title":"Journal of the American Society for Information Science and Technology"},{"issue":"38","key":"2516_CR14","doi-asserted-by":"crossref","first-page":"7173","DOI":"10.1002\/anie.200800513","volume":"47","author":"L Bornmann","year":"2008","unstructured":"Bornmann, L., & Daniel, H.-D. (2008b). The effectiveness of the peer review process: Inter-referee agreement and predictive validity of manuscript refereeing at Angewandte Chemie. Angewandte Chemie-International Edition, 47(38), 7173\u20137178.","journal-title":"Angewandte Chemie-International Edition"},{"issue":"1","key":"2516_CR15","doi-asserted-by":"crossref","first-page":"45","DOI":"10.1108\/00220410810844150","volume":"64","author":"L Bornmann","year":"2008","unstructured":"Bornmann, L., & Daniel, H.-D. (2008c). What do citations counts measure? A review of studies on citing behavior. Journal of Documentation, 64(1), 45\u201380.","journal-title":"Journal of Documentation"},{"issue":"12","key":"2516_CR16","doi-asserted-by":"crossref","first-page":"e14331","DOI":"10.1371\/journal.pone.0014331","volume":"5","author":"L Bornmann","year":"2010","unstructured":"Bornmann, L., Mutz, R., & Daniel, H.-D. (2010). A reliability-generalization study of journal peer reviews: A multilevel meta-analysis of inter-rater reliability and its determinants. PLoS ONE, 5(12), e14331.","journal-title":"PLoS ONE"},{"key":"2516_CR17","volume-title":"Forschungsmethoden und evaluation f\u00fcr Human- und Sozialwissenschaftler [Research methods and evaluation for human and social scientists]","author":"J Bortz","year":"2006","unstructured":"Bortz, J., & D\u00f6ring, N. (2006). Forschungsmethoden und evaluation f\u00fcr Human- und Sozialwissenschaftler [Research methods and evaluation for human and social scientists] (4th ed.). Heidelberg, DE: Springer.","edition":"4"},{"key":"2516_CR18","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4757-3456-0","volume-title":"Generalizability theory","author":"RL Brennan","year":"2001","unstructured":"Brennan, R. L. (2001). Generalizability theory. New York, NY: Springer."},{"key":"2516_CR19","volume-title":"Confirmatory factor analysis for applied research","author":"TA Brown","year":"2015","unstructured":"Brown, T. A. (2015). Confirmatory factor analysis for applied research (2nd ed.). New York, NY: Guilford Press.","edition":"2"},{"issue":"4","key":"2516_CR20","doi-asserted-by":"crossref","first-page":"373","DOI":"10.1111\/j.1744-6570.1963.tb01283.x","volume":"16","author":"EI Burdock","year":"1963","unstructured":"Burdock, E. I., Fleiss, J. L., & Hardesty, A. S. (1963). A new view of inter-observer agreement. Personnel Psychology, 16(4), 373\u2013384.","journal-title":"Personnel Psychology"},{"issue":"1","key":"2516_CR21","doi-asserted-by":"crossref","first-page":"e40","DOI":"10.1371\/journal.pmed.0040040","volume":"4","author":"ML Callaham","year":"2007","unstructured":"Callaham, M. L., & Tercier, J. (2007). The relationship of previous training and experience of journal peer reviewers to subsequent review quality. PLoS Medicine, 4(1), e40.","journal-title":"PLoS Medicine"},{"issue":"3","key":"2516_CR22","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1177\/1075547098019003002","volume":"19","author":"JM Campanario","year":"1998","unstructured":"Campanario, J. M. (1998). Peer review for journals as it stands today\u2014Part 1. Science Communication, 19(3), 181\u2013211.","journal-title":"Science Communication"},{"issue":"2","key":"2516_CR23","doi-asserted-by":"crossref","first-page":"81","DOI":"10.1037\/h0046016","volume":"56","author":"DT Campbell","year":"1959","unstructured":"Campbell, D. T., & Fiske, D. W. (1959). Convergent and discriminant validation by the multitrait-multimethod matrix. Psychological Bulletin, 56(2), 81\u2013105.","journal-title":"Psychological Bulletin"},{"issue":"3","key":"2516_CR24","doi-asserted-by":"crossref","first-page":"705","DOI":"10.1111\/j.1744-6570.1993.tb00896.x","volume":"46","author":"MA Campion","year":"1993","unstructured":"Campion, M. A. (1993). Article review checklist: A criterion checklist for reviewing research articles in applied psychology. Personnel Psychology, 46(3), 705\u2013718.","journal-title":"Personnel Psychology"},{"issue":"2","key":"2516_CR25","doi-asserted-by":"crossref","first-page":"245","DOI":"10.1207\/s15327906mbr0102_10","volume":"1","author":"RB Cattell","year":"1966","unstructured":"Cattell, R. B. (1966). The scree test for the number of factors. Multivariate Behavioral Research, 1(2), 245\u2013276.","journal-title":"Multivariate Behavioral Research"},{"key":"2516_CR26","first-page":"1","volume":"67","author":"RB Cattell","year":"1967","unstructured":"Cattell, R. B., & Jaspers, J. (1967). A general plasmode (No. 30-10-5-2) for factor analytic exercises and research. Multivariate Behavioral Research Monographs, 67, 1\u2013212.","journal-title":"Multivariate Behavioral Research Monographs"},{"issue":"3","key":"2516_CR27","first-page":"262","volume":"5","author":"JM Chase","year":"1970","unstructured":"Chase, J. M. (1970). Normative criteria for scientific publication. American Sociologist, 5(3), 262\u2013265.","journal-title":"American Sociologist"},{"issue":"2","key":"2516_CR28","doi-asserted-by":"crossref","first-page":"305","DOI":"10.3758\/BF03204787","volume":"28","author":"RM Church","year":"1996","unstructured":"Church, R. M., Crystal, J. D., & Collyer, C. E. (1996). Correction of errors in scientific research. Behavior Research Methods, Instruments, & Computers, 28(2), 305\u2013310.","journal-title":"Behavior Research Methods, Instruments, & Computers"},{"issue":"1","key":"2516_CR29","doi-asserted-by":"crossref","first-page":"119","DOI":"10.1017\/S0140525X00065675","volume":"14","author":"DV Cicchetti","year":"1991","unstructured":"Cicchetti, D. V. (1991). The reliability of peer review for manuscript and grant submissions: A cross-disciplinary investigation. Behavioral and Brain Sciences, 14(1), 119\u2013135.","journal-title":"Behavioral and Brain Sciences"},{"issue":"4","key":"2516_CR30","doi-asserted-by":"crossref","first-page":"284","DOI":"10.1037\/1040-3590.6.4.284","volume":"6","author":"DV Cicchetti","year":"1994","unstructured":"Cicchetti, D. V. (1994). Guidelines, criteria, and rules of thumb for evaluating normed and standardized assessment instruments in psychology. Psychological Assessment, 6(4), 284\u2013290.","journal-title":"Psychological Assessment"},{"issue":"4","key":"2516_CR31","first-page":"373","volume":"49","author":"DV Cicchetti","year":"1976","unstructured":"Cicchetti, D. V., & Conn, H. O. (1976). A statistical analysis of reviewer agreement and bias in evaluating medical abstracts. Yale Journal of Biology and Medicine, 49(4), 373\u2013383.","journal-title":"Yale Journal of Biology and Medicine"},{"issue":"1","key":"2516_CR32","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J. (1960). A coefficient of agreement for nominal scales. Educational and Psychological Measurement, 20(1), 37\u201346.","journal-title":"Educational and Psychological Measurement"},{"key":"2516_CR33","volume-title":"Statistical power analysis for the behavioral sciences","author":"J Cohen","year":"1988","unstructured":"Cohen, J. (1988). Statistical power analysis for the behavioral sciences (2nd ed.). Hillsdale, NJ: Erlbaum.","edition":"2"},{"issue":"10","key":"2516_CR34","doi-asserted-by":"crossref","first-page":"1425","DOI":"10.1177\/0146167205275614","volume":"31","author":"JC Cohrs","year":"2005","unstructured":"Cohrs, J. C., Moschner, B., Maes, J., & Kielmann, S. (2005). The motivational bases of right-wing authoritarianism and social dominance orientation: Relations to values and attitudes in the aftermath of September 11, 2001. Personality and Social Psychology Bulletin, 31(10), 1425\u20131434.","journal-title":"Personality and Social Psychology Bulletin"},{"issue":"4523","key":"2516_CR35","doi-asserted-by":"crossref","first-page":"881","DOI":"10.1126\/science.7302566","volume":"214","author":"S Cole","year":"1981","unstructured":"Cole, S., Cole, J. R., & Simon, G. A. (1981). Chance and consensus in peer review. Science, 214(4523), 881\u2013886.","journal-title":"Science"},{"issue":"892","key":"2516_CR36","first-page":"39","volume":"62","author":"JW Cornforth","year":"1974","unstructured":"Cornforth, J. W. (1974). Referees. New Scientist, 62(892), 39.","journal-title":"New Scientist"},{"issue":"12","key":"2516_CR37","doi-asserted-by":"crossref","first-page":"1505","DOI":"10.1016\/j.ijnurstu.2011.06.004","volume":"48","author":"M Crowe","year":"2011","unstructured":"Crowe, M., & Sheppard, L. (2011a). A general critical appraisal tool: An evaluation of construct validity. International Journal of Nursing Studies, 48(12), 1505\u20131516.","journal-title":"International Journal of Nursing Studies"},{"issue":"1","key":"2516_CR38","doi-asserted-by":"crossref","first-page":"79","DOI":"10.1016\/j.jclinepi.2010.02.008","volume":"64","author":"M Crowe","year":"2011","unstructured":"Crowe, M., & Sheppard, L. (2011b). A review of critical appraisal tools show they lack rigor: Alternative tool structure is proposed. Journal of Clinical Epidemiology, 64(1), 79\u201389.","journal-title":"Journal of Clinical Epidemiology"},{"issue":"2","key":"2516_CR39","doi-asserted-by":"crossref","first-page":"1547","DOI":"10.1007\/s11192-013-1089-2","volume":"98","author":"JCF Winter de","year":"2014","unstructured":"de Winter, J. C. F., Zadpoor, A. A., & Dodou, D. (2014). The expansion of Google Scholar versus Web of science: A longitudinal study. Scientometrics, 98(2), 1547\u20131565.","journal-title":"Scientometrics"},{"key":"2516_CR40","unstructured":"DeCoursey, T. (2006). The pros and cons of open peer review. Nature. Retrieved from http:\/\/www.nature.com\/nature\/peerreview\/debate\/nature04991.html ."},{"issue":"1","key":"2516_CR41","doi-asserted-by":"crossref","first-page":"67","DOI":"10.2307\/1403259","volume":"54","author":"A Donner","year":"1986","unstructured":"Donner, A. (1986). A review of inference procedures for the intraclass correlation coefficient in the one-way random effects model. International Statistical Review, 54(1), 67\u201382.","journal-title":"International Statistical Review"},{"issue":"6","key":"2516_CR42","doi-asserted-by":"crossref","first-page":"358","DOI":"10.1037\/h0036316","volume":"81","author":"CD Dziuban","year":"1974","unstructured":"Dziuban, C. D., & Shirkey, E. C. (1974). When is a correlation matrix appropriate for factor analysis? Some decision rules. Psychological Bulletin, 81(6), 358\u2013361.","journal-title":"Psychological Bulletin"},{"issue":"2","key":"2516_CR43","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1007\/BF02294377","volume":"65","author":"M Eid","year":"2000","unstructured":"Eid, M. (2000). A multitrait-multimethod model with minimal assumptions. Psychometrika, 65(2), 241\u2013261.","journal-title":"Psychometrika"},{"issue":"1","key":"2516_CR44","doi-asserted-by":"crossref","first-page":"38","DOI":"10.1037\/1082-989X.8.1.38","volume":"8","author":"M Eid","year":"2003","unstructured":"Eid, M., Lischetzke, T., Nussbeck, F. W., & Trierweiler, L. I. (2003). Separating trait effects from trait-specific method effects in multitrait-multimethod models: A multiple-indicator CT-C(M-1) model. Psychological Methods, 8(1), 38\u201360.","journal-title":"Psychological Methods"},{"issue":"5","key":"2516_CR45","doi-asserted-by":"crossref","first-page":"713","DOI":"10.1177\/0013164401615001","volume":"61","author":"CK Enders","year":"2001","unstructured":"Enders, C. K. (2001). The performance of the full information maximum likelihood estimator in multiple regression models with missing data. Educational and Psychological Measurement, 61(5), 713\u2013740.","journal-title":"Educational and Psychological Measurement"},{"key":"2516_CR46","volume-title":"Applied missing data analysis","author":"CK Enders","year":"2010","unstructured":"Enders, C. K. (2010). Applied missing data analysis. New York, NY: Guilford Press."},{"issue":"6","key":"2516_CR47","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1016\/0895-4356(90)90158-L","volume":"43","author":"AR Feinstein","year":"1990","unstructured":"Feinstein, A. R., & Cicchetti, D. V. (1990). High agreement but low kappa: I. The problems of two paradoxes. Journal of Clinical Epidemiology, 43(6), 543\u2013549.","journal-title":"Journal of Clinical Epidemiology"},{"key":"2516_CR48","volume-title":"Discovering statistics using SPSS","author":"A Field","year":"2009","unstructured":"Field, A. (2009). Discovering statistics using SPSS (3rd ed.). Thousand Oaks, CA: Sage.","edition":"3"},{"key":"2516_CR49","volume-title":"Statistical methods for research workers","author":"RA Fisher","year":"1934","unstructured":"Fisher, R. A. (1934). Statistical methods for research workers (5th ed.). Edinburgh: Oliver and Boyd.","edition":"5"},{"issue":"5","key":"2516_CR50","doi-asserted-by":"crossref","first-page":"591","DOI":"10.1037\/0003-066X.45.5.591","volume":"45","author":"DW Fiske","year":"1990","unstructured":"Fiske, D. W., & Fogg, L. (1990). But the reviewers are making different criticisms of my paper! Diversity and uniqueness in reviewer comments. American Psychologist, 45(5), 591\u2013598.","journal-title":"American Psychologist"},{"issue":"5","key":"2516_CR51","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"JL Fleiss","year":"1971","unstructured":"Fleiss, J. L. (1971). Measuring nominal scale agreement among many raters. Psychological Bulletin, 76(5), 378\u2013382.","journal-title":"Psychological Bulletin"},{"key":"2516_CR52","doi-asserted-by":"crossref","DOI":"10.1201\/b16018","volume-title":"Bayesian data analysis","author":"A Gelman","year":"2013","unstructured":"Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian data analysis (3rd ed.). Boca Raton, FL: Chapman and Hall\/CRC.","edition":"3"},{"issue":"4","key":"2516_CR53","doi-asserted-by":"crossref","first-page":"457","DOI":"10.1214\/ss\/1177011136","volume":"7","author":"A Gelman","year":"1992","unstructured":"Gelman, A., & Rubin, D. B. (1992). Inference from iterative simulation using multiple sequences. Statistical Science, 7(4), 457\u2013472.","journal-title":"Statistical Science"},{"issue":"2","key":"2516_CR54","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1111\/j.1744-6570.1997.tb00914.x","volume":"50","author":"SW Gilliland","year":"1997","unstructured":"Gilliland, S. W., & Cortina, J. M. (1997). Reviewer and editor decision making in the journal review process. Personnel Psychology, 50(2), 427\u2013452.","journal-title":"Personnel Psychology"},{"issue":"10","key":"2516_CR55","doi-asserted-by":"crossref","first-page":"920","DOI":"10.1037\/0003-066X.33.10.920","volume":"33","author":"SD Gottfredson","year":"1978","unstructured":"Gottfredson, S. D. (1978). Evaluating psychological research reports: Dimensions, reliability, and correlates of quality judgments. American Psychologist, 33(10), 920\u2013934.","journal-title":"American Psychologist"},{"key":"2516_CR56","doi-asserted-by":"crossref","first-page":"c6424","DOI":"10.1136\/bmj.c6424","volume":"341","author":"T Groves","year":"2010","unstructured":"Groves, T. (2010). Is open peer the fairest system? Yes. BMJ, 341, c6424.","journal-title":"BMJ"},{"key":"2516_CR57","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1348\/000711006X126600","volume":"61","author":"KL Gwet","year":"2008","unstructured":"Gwet, K. L. (2008). Computing inter-rater reliability and its variance in the presence of high agreement. British Journal of Mathematical and Statistical Psychology, 61, 29\u201348.","journal-title":"British Journal of Mathematical and Statistical Psychology"},{"key":"2516_CR58","volume-title":"The definitive guide to measuring the extent of agreement among raters","author":"KL Gwet","year":"2014","unstructured":"Gwet, K. L. (2014). The definitive guide to measuring the extent of agreement among raters (4th ed.). Gaithersburg, MD: Advanced Analytics.","edition":"4"},{"issue":"11","key":"2516_CR59","doi-asserted-by":"crossref","first-page":"e111958","DOI":"10.1371\/journal.pone.0111958","volume":"9","author":"I Halatchliyski","year":"2014","unstructured":"Halatchliyski, I., & Cress, U. (2014). How structure shapes dynamics: Knowledge development in Wikipedia\u2014A network multilevel modeling approach. PLoS ONE, 9(11), e111958.","journal-title":"PLoS ONE"},{"issue":"7","key":"2516_CR60","doi-asserted-by":"crossref","first-page":"335","DOI":"10.2307\/2026523","volume":"82","author":"J Hardwig","year":"1985","unstructured":"Hardwig, J. (1985). Epistemic dependence. The Journal of Philosophy, 82(7), 335\u2013349.","journal-title":"The Journal of Philosophy"},{"issue":"5","key":"2516_CR61","doi-asserted-by":"crossref","first-page":"357","DOI":"10.1016\/j.envsci.2004.06.005","volume":"7","author":"C Harrison","year":"2004","unstructured":"Harrison, C. (2004). Peer review, politics and pluralism. Environmental Science & Policy, 7(5), 357\u2013368.","journal-title":"Environmental Science & Policy"},{"issue":"2","key":"2516_CR62","first-page":"152","volume":"14","author":"M Hassebrauck","year":"1983","unstructured":"Hassebrauck, M. (1983). Die Beurteilung der physischen Attraktivit\u00e4t: Konsens unter Urteilern? [Judging physical attractiveness: Consensus among judges?]. Zeitschrift f\u00fcr Sozialpsychologie, 14(2), 152\u2013161.","journal-title":"Zeitschrift f\u00fcr Sozialpsychologie"},{"key":"2516_CR63","first-page":"29","volume-title":"Physische Attraktivit\u00e4t [Physical attractiveness]","author":"M Hassebrauck","year":"1993","unstructured":"Hassebrauck, M. (1993). Die Beurteilung der physischen Attraktivit\u00e4t [The assessment of physical attractiveness]. In M. Hassebrauck & R. Niketta (Eds.), Physische Attraktivit\u00e4t [Physical attractiveness] (1st ed., pp. 29\u201359). G\u00f6ttingen, DE: Hogrefe.","edition":"1"},{"issue":"2","key":"2516_CR64","doi-asserted-by":"crossref","first-page":"191","DOI":"10.1177\/1094428104263675","volume":"7","author":"JC Hayton","year":"2004","unstructured":"Hayton, J. C., Allen, D. G., & Scarpello, V. (2004). Factor retention decisions in exploratory factor analysis: A tutorial on parallel analysis. Organizational Research Methods, 7(2), 191\u2013205.","journal-title":"Organizational Research Methods"},{"issue":"1","key":"2516_CR65","first-page":"73","volume":"3","author":"S Hemlin","year":"1990","unstructured":"Hemlin, S., & Montgomery, H. (1990). Scientists\u2019 conceptions of scientific quality: An interview study. Science Studies, 3(1), 73\u201381.","journal-title":"Science Studies"},{"issue":"2","key":"2516_CR66","doi-asserted-by":"crossref","first-page":"173","DOI":"10.1177\/0162243905283639","volume":"31","author":"S Hemlin","year":"2006","unstructured":"Hemlin, S., & Rasmussen, S. B. (2006). The shift in academic quality control. Science, Technology and Human Values, 31(2), 173\u2013198.","journal-title":"Science, Technology and Human Values"},{"key":"2516_CR67","volume-title":"\u201cSpieglein, Spieglein an der Wand \u2026\u201d: Geschlecht, Alter und physische Attraktivi\u00e4t [\u201cMirror, mirror on the wall\u2026\u201d: Sex, age, and physical attractiveness]","author":"R Henss","year":"1992","unstructured":"Henss, R. (1992). \u201cSpieglein, Spieglein an der Wand \u2026\u201d: Geschlecht, Alter und physische Attraktivi\u00e4t [\u201cMirror, mirror on the wall\u2026\u201d: Sex, age, and physical attractiveness]. Weinheim, DE: PVU."},{"issue":"2","key":"2516_CR68","doi-asserted-by":"crossref","first-page":"175","DOI":"10.2752\/089279305785594180","volume":"18","author":"HA Herzog","year":"2005","unstructured":"Herzog, H. A., Podberscek, A. L., & Docherty, A. (2005). The reliability of peer review in anthrozoology. Anthrozoos, 18(2), 175\u2013182.","journal-title":"Anthrozoos"},{"key":"2516_CR69","doi-asserted-by":"crossref","DOI":"10.1017\/CBO9780511973420","volume-title":"Negative binomial regression","author":"JM Hilbe","year":"2011","unstructured":"Hilbe, J. M. (2011). Negative binomial regression (2nd ed.). Cambridge: Cambridge University Press.","edition":"2"},{"issue":"2","key":"2516_CR70","first-page":"65","volume":"6","author":"S Holm","year":"1979","unstructured":"Holm, S. (1979). A simple sequentially rejective multiple test procedure. Scandinavian Journal of Statistics, 6(2), 65\u201370.","journal-title":"Scandinavian Journal of Statistics"},{"issue":"2","key":"2516_CR71","first-page":"199","volume":"32","author":"J H\u00f6nekopp","year":"2006","unstructured":"H\u00f6nekopp, J. (2006). Once more: Is beauty in the eye of the beholder? Relative contributions of private and shared taste to judgments of facial attractiveness. Journal of Experimental Psychology, 32(2), 199\u2013209.","journal-title":"Journal of Experimental Psychology"},{"issue":"1","key":"2516_CR72","doi-asserted-by":"crossref","first-page":"72","DOI":"10.1037\/1082-989X.11.1.72","volume":"11","author":"J H\u00f6nekopp","year":"2006","unstructured":"H\u00f6nekopp, J., Becker, B. J., & Oswald, F. L. (2006). The meaning and suitability of various effect sizes for structured Rater x Ratee designs. Psychological Methods, 11(1), 72\u201386.","journal-title":"Psychological Methods"},{"issue":"2","key":"2516_CR73","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1007\/BF02289447","volume":"30","author":"JL Horn","year":"1965","unstructured":"Horn, J. L. (1965). A rationale and test for the number of factors in factor analysis. Psychometrika, 30(2), 179\u2013185.","journal-title":"Psychometrika"},{"key":"2516_CR74","doi-asserted-by":"crossref","unstructured":"Houry, D., Green, S., & Callaham, M. (2012). Does mentoring new peer reviewers improve review quality? A randomized trial. BMC Medical Education, 12.","DOI":"10.1186\/1472-6920-12-83"},{"key":"2516_CR75","doi-asserted-by":"crossref","first-page":"110","DOI":"10.1192\/bjp.173.2.110","volume":"173","author":"L Howard","year":"1998","unstructured":"Howard, L., & Wilkinson, G. (1998). Peer review and editorial decision-making. British Journal of Psychiatry, 173, 110\u2013113.","journal-title":"British Journal of Psychiatry"},{"key":"2516_CR76","doi-asserted-by":"crossref","DOI":"10.4135\/9780857028075","volume-title":"The multivariate social scientist","author":"GD Hutcheson","year":"1999","unstructured":"Hutcheson, G. D., & Sofroniou, N. (1999). The multivariate social scientist. Thousand Oaks, CA: Sage."},{"key":"2516_CR77","unstructured":"IBM Corp. (2011). IBM SPSS Statistics for windows (version 20.0) [computer software]. Armonk, NY: IBM Corp."},{"issue":"3","key":"2516_CR78","doi-asserted-by":"crossref","first-page":"279","DOI":"10.1111\/1467-985X.00278","volume":"166","author":"UW Jayasinghe","year":"2003","unstructured":"Jayasinghe, U. W., Marsh, H. W., & Bond, N. (2003). A multilevel cross-classified modelling approach to peer review of grant proposals: The effects of assessor and researcher attributes on assessor ratings. Journal of the Royal Statistical Society A, 166(3), 279\u2013300.","journal-title":"Journal of the Royal Statistical Society A"},{"issue":"3","key":"2516_CR79","doi-asserted-by":"crossref","first-page":"591","DOI":"10.1007\/s11192-006-0171-4","volume":"69","author":"UW Jayasinghe","year":"2006","unstructured":"Jayasinghe, U. W., Marsh, H. W., & Bond, N. (2006). A new reader trial approach to peer review in funding research grants: An Australian experiment. Scientometrics, 69(3), 591\u2013606.","journal-title":"Scientometrics"},{"issue":"4","key":"2516_CR80","doi-asserted-by":"crossref","first-page":"401","DOI":"10.1007\/BF02291817","volume":"35","author":"HF Kaiser","year":"1970","unstructured":"Kaiser, H. F. (1970). A second generation Little Jiffy. Psychometrika, 35(4), 401\u2013415.","journal-title":"Psychometrika"},{"issue":"1","key":"2516_CR81","doi-asserted-by":"crossref","first-page":"111","DOI":"10.1177\/001316447403400115","volume":"34","author":"HF Kaiser","year":"1974","unstructured":"Kaiser, H. F., & Rice, J. (1974). Little Jiffy, Mark IV. Educational and Psychological Measurement, 34(1), 111\u2013117.","journal-title":"Educational and Psychological Measurement"},{"key":"2516_CR82","first-page":"407","volume-title":"The Oxford handbook of quantitative methods","author":"D Kaplan","year":"2013","unstructured":"Kaplan, D., & Depaoli, S. (2013). Bayesian statistical methods. In T. D. Little (Ed.), The Oxford handbook of quantitative methods (Vol. 1, pp. 407\u2013437). New York, NY: Oxford University Press."},{"issue":"4","key":"2516_CR83","doi-asserted-by":"crossref","first-page":"380","DOI":"10.1001\/archpedi.1996.02170290046007","volume":"150","author":"KJ Kemper","year":"1996","unstructured":"Kemper, K. J., McCarthy, P. L., & Cicchetti, D. V. (1996). Improving participation and interrater agreement in scoring ambulatory pediatric association abstracts: How well have we succeeded? Archives of Pediatrics and Adolescent Medicine, 150(4), 380\u2013383.","journal-title":"Archives of Pediatrics and Adolescent Medicine"},{"key":"2516_CR84","doi-asserted-by":"crossref","first-page":"c6425","DOI":"10.1136\/bmj.c6425","volume":"341","author":"K Khan","year":"2010","unstructured":"Khan, K. (2010). Is open peer review the fairest system? No. BMJ, 341, c6425.","journal-title":"BMJ"},{"issue":"2","key":"2516_CR85","doi-asserted-by":"crossref","first-page":"121","DOI":"10.1093\/swr\/21.2.121","volume":"21","author":"SA Kirk","year":"1997","unstructured":"Kirk, S. A., & Franke, T. M. (1997). Agreeing to disagree: A study of the reliability of manuscript reviews. Social Work Research, 21(2), 121\u2013126.","journal-title":"Social Work Research"},{"issue":"1","key":"2516_CR86","doi-asserted-by":"crossref","first-page":"5","DOI":"10.2307\/2026796","volume":"87","author":"P Kitcher","year":"1990","unstructured":"Kitcher, P. (1990). The division of cognitive labor. The Journal of Philosophy, 87(1), 5\u201322.","journal-title":"The Journal of Philosophy"},{"issue":"6","key":"2516_CR87","doi-asserted-by":"crossref","first-page":"820","DOI":"10.1177\/030631201031006002","volume":"31","author":"L Langfeldt","year":"2001","unstructured":"Langfeldt, L. (2001). The decision-making constraints and processes of grant peer review, and their effects on the review outcome. Social Studies of Science, 31(6), 820\u2013841.","journal-title":"Social Studies of Science"},{"issue":"1","key":"2516_CR88","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1002\/asi.22784","volume":"64","author":"CJ Lee","year":"2013","unstructured":"Lee, C. J., Sugimoto, C. R., Zhang, G., & Cronin, B. (2013). Bias in peer review. Journal of the American Society for Information Science and Technology, 64(1), 2\u201317.","journal-title":"Journal of the American Society for Information Science and Technology"},{"key":"2516_CR89","doi-asserted-by":"crossref","first-page":"434","DOI":"10.1126\/science.aaa0185","volume":"348","author":"D Li","year":"2015","unstructured":"Li, D., & Agha, L. (2015). Big names or big ideas: Do peer-review panels select the best science proposals? Science, 348, 434\u2013438.","journal-title":"Science"},{"issue":"1\u20132","key":"2516_CR90","doi-asserted-by":"crossref","first-page":"75","DOI":"10.1007\/BF02020243","volume":"14","author":"D Lindsey","year":"1988","unstructured":"Lindsey, D. (1988). Assessing precision in the manuscript review process: A little better than a dice roll. Scientometrics, 14(1\u20132), 75\u201382.","journal-title":"Scientometrics"},{"issue":"3\u20134","key":"2516_CR91","doi-asserted-by":"crossref","first-page":"189","DOI":"10.1007\/BF02017198","volume":"15","author":"D Lindsey","year":"1989","unstructured":"Lindsey, D. (1989). Using citation counts as a measure of quality in science measuring what\u2019s measurable rather than what\u2019s valid. Scientometrics, 15(3\u20134), 189\u2013203.","journal-title":"Scientometrics"},{"issue":"7656","key":"2516_CR92","doi-asserted-by":"crossref","first-page":"9","DOI":"10.1038\/546009a","volume":"546","author":"B List","year":"2017","unstructured":"List, B. (2017). Crowd-based peer review can be good and fast. Nature, 546(7656), 9.","journal-title":"Nature"},{"issue":"11","key":"2516_CR93","doi-asserted-by":"crossref","first-page":"2098","DOI":"10.1037\/0022-3514.37.11.2098","volume":"37","author":"CG Lord","year":"1979","unstructured":"Lord, C. G., Ross, L., & Lepper, M. R. (1979). Biased assimilation and attitude polarization: The effects of prior theories on subsequently considered evidence. Journal of Personality and Social Psychology, 37(11), 2098\u20132109.","journal-title":"Journal of Personality and Social Psychology"},{"issue":"2","key":"2516_CR94","doi-asserted-by":"crossref","first-page":"399","DOI":"10.1017\/S0140525X00030776","volume":"16","author":"RD Luce","year":"1993","unstructured":"Luce, R. D. (1993). Reliability is neither to be expected nor desired in peer review. Behavioral and Brain Sciences, 16(2), 399\u2013400.","journal-title":"Behavioral and Brain Sciences"},{"issue":"6","key":"2516_CR95","doi-asserted-by":"crossref","first-page":"872","DOI":"10.1037\/0022-0663.73.6.872","volume":"73","author":"HW Marsh","year":"1981","unstructured":"Marsh, H. W., & Ball, S. (1981). Interjudgmental reliability of reviews for the Journal of Educational Psychology. Journal of Educational Psychology, 73(6), 872\u2013880.","journal-title":"Journal of Educational Psychology"},{"issue":"2","key":"2516_CR96","doi-asserted-by":"crossref","first-page":"151","DOI":"10.1080\/00220973.1989.10806503","volume":"57","author":"HW Marsh","year":"1989","unstructured":"Marsh, H. W., & Ball, S. (1989). The peer review process used to evaluate manuscripts submitted to academic journals: Interjudgmental reliability. The Journal of Experimental Education, 57(2), 151\u2013169.","journal-title":"The Journal of Experimental Education"},{"issue":"1","key":"2516_CR97","doi-asserted-by":"crossref","first-page":"33","DOI":"10.1080\/00050060600823275","volume":"42","author":"HW Marsh","year":"2007","unstructured":"Marsh, H. W., Bond, N. W., & Jayasinghe, U. W. (2007). Peer review process: Assessments by applicant-nominated referees are biased, inflated, unreliable and invalid. Australian Psychologist, 42(1), 33\u201338.","journal-title":"Australian Psychologist"},{"issue":"3","key":"2516_CR98","doi-asserted-by":"crossref","first-page":"160","DOI":"10.1037\/0003-066X.63.3.160","volume":"63","author":"HW Marsh","year":"2008","unstructured":"Marsh, H. W., Jayasinghe, U. W., & Bond, N. W. (2008). Improving the peer-review process for grant applications: Reliability, validity, bias, and generalizability. American Psychologist, 63(3), 160\u2013168.","journal-title":"American Psychologist"},{"issue":"1","key":"2516_CR99","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1037\/1082-989X.1.1.30","volume":"1","author":"KO McGraw","year":"1996","unstructured":"McGraw, K. O., & Wong, S. P. (1996). Forming inferences about some intraclass correlation coefficients. Psychological Methods, 1(1), 30\u201346.","journal-title":"Psychological Methods"},{"issue":"9","key":"2516_CR100","doi-asserted-by":"crossref","first-page":"741","DOI":"10.1037\/0003-066X.50.9.741","volume":"50","author":"S Messick","year":"1995","unstructured":"Messick, S. (1995). Validity of psychological assessment: Validation of inferences from persons\u2019 responses and performances as scientific inquiry into score meaning. American Psychologist, 50(9), 741\u2013749.","journal-title":"American Psychologist"},{"key":"2516_CR101","doi-asserted-by":"crossref","unstructured":"Montgomery, A. A., Graham, A., Evans, P. H., & Fahey, T. (2002). Inter-rater agreement in the scoring of abstracts submitted to a primary care research conference. BMC Health Services Research, 2.","DOI":"10.1186\/1472-6963-2-8"},{"key":"2516_CR102","unstructured":"Muth\u00e9n, B. (2010). Bayesian analysis in Mplus: A brief introduction [manuscript]. http:\/\/www.statmodel.com\/download\/IntroBayesVersion%203.pdf . Accessed March 30 2017."},{"key":"2516_CR103","unstructured":"Muth\u00e9n, B., & Asparouhov, T. (2011). Bayesian SEM: A more flexible representation of substantive theory [manuscript]. http:\/\/www.statmodel.com\/download\/BSEMv4REVISED . Accessed March 30 2017."},{"key":"2516_CR104","volume-title":"Mplus user\u2019s guide","author":"LK Muth\u00e9n","year":"2012","unstructured":"Muth\u00e9n, L. K., & Muth\u00e9n, B. O. (2012). Mplus user\u2019s guide (7th ed.). Los Angeles, CA: Muth\u00e9n & Muth\u00e9n.","edition":"7"},{"issue":"10","key":"2516_CR105","doi-asserted-by":"crossref","first-page":"e48509","DOI":"10.1371\/journal.pone.0048509","volume":"7","author":"R Mutz","year":"2012","unstructured":"Mutz, R., Bornmann, L., & Daniel, H.-D. (2012). Heterogeneity of inter-rater reliabilities of grant peer reviews and its determinants: A general estimating equations approach. PLoS ONE, 7(10), e48509.","journal-title":"PLoS ONE"},{"issue":"3","key":"2516_CR106","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1016\/0049-089X(91)90010-Z","volume":"20","author":"RM O\u2019Brien","year":"1991","unstructured":"O\u2019Brien, R. M. (1991). The reliability of composites of referee assessments of manuscripts. Social Science Research, 20(3), 319\u2013328.","journal-title":"Social Science Research"},{"issue":"3","key":"2516_CR107","doi-asserted-by":"crossref","first-page":"436","DOI":"10.1177\/1094428112438699","volume":"15","author":"TA O\u2019Neill","year":"2012","unstructured":"O\u2019Neill, T. A., Goffin, R. D., & Gellatly, I. R. (2012). The use of random coefficient modeling for understanding and predicting job performance ratings: An application with field data. Organizational Research Methods, 15(3), 436\u2013462.","journal-title":"Organizational Research Methods"},{"issue":"3","key":"2516_CR108","doi-asserted-by":"crossref","first-page":"339","DOI":"10.1016\/S0008-6363(02)00712-5","volume":"56","author":"T Opthof","year":"2002","unstructured":"Opthof, T., Coronel, R., & Janse, M. J. (2002). The significance of the peer review process against the background of bias: Priority ratings of reviewers and editors and the prediction of citation, the role of geographical bias. Cardiovascular Research, 56(3), 339\u2013346.","journal-title":"Cardiovascular Research"},{"issue":"1","key":"2516_CR109","doi-asserted-by":"crossref","first-page":"91","DOI":"10.1016\/0895-4356(91)90205-N","volume":"44","author":"AD Oxman","year":"1991","unstructured":"Oxman, A. D., Guyatt, G. H., Singer, J., Goldsmith, C. H., Hutchison, B. G., et al. (1991). Agreement among reviewers of review articles. Journal of Clinical Epidemiology, 44(1), 91\u201398.","journal-title":"Journal of Clinical Epidemiology"},{"issue":"2","key":"2516_CR110","doi-asserted-by":"crossref","first-page":"188","DOI":"10.1177\/0146167299025002005","volume":"25","author":"RE Petty","year":"1999","unstructured":"Petty, R. E., Fleming, M. A., & Fabrigar, L. R.\u00a0(1999). The review process at PSPB: Correlates of interreviewer agreement and manuscript acceptance. Personality and Social Psychology Bulletin, 25(2), 188\u2013203.","journal-title":"Personality and Social Psychology Bulletin"},{"issue":"3642","key":"2516_CR111","first-page":"347","volume":"146","author":"JR Platt","year":"1964","unstructured":"Platt, J. R. (1964). Strong inference:\u00a0Certain systematic methods of scientific thinking may produce much more rapid progress than others. Science, New Series, 146(3642), 347\u2013353.","journal-title":"Science, New Series"},{"key":"2516_CR112","doi-asserted-by":"crossref","first-page":"333","DOI":"10.1016\/S0049-237X(08)71204-7","volume":"52","author":"KR Popper","year":"1968","unstructured":"Popper, K. R. (1968). Epistemology without a knowing subject. Studies in Logic and the Foundations of Mathematics, 52, 333\u2013373.","journal-title":"Studies in Logic and the Foundations of Mathematics"},{"issue":"1","key":"2516_CR113","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1037\/0021-9010.71.1.29","volume":"71","author":"ED Pulakos","year":"1986","unstructured":"Pulakos, E. D., Schmitt, N., & Ostroff, C. (1986). A warning about the use of a standard deviation across dimensions within ratees to measure halo. Journal of Applied Psychology, 71(1), 29\u201332.","journal-title":"Journal of Applied Psychology"},{"key":"2516_CR114","unstructured":"Putka, D. J. (2002). The variance architecture approach to the study of constructs in organizational contexts (Doctoral dissertation, Ohio University). http:\/\/etd.ohiolink.edu\/ . Accessed March 30 2017."},{"issue":"3","key":"2516_CR115","doi-asserted-by":"crossref","first-page":"503","DOI":"10.1177\/1094428110362107","volume":"14","author":"DJ Putka","year":"2011","unstructured":"Putka, D. J., Lance, C. E., Le, H., & McCloy, R. A. (2011). A cautionary note on modeling multitrait\u2013multirater data arising from ill-structured measurement designs. Organizational Research Methods, 14(3), 503\u2013529.","journal-title":"Organizational Research Methods"},{"issue":"5","key":"2516_CR116","doi-asserted-by":"crossref","first-page":"959","DOI":"10.1037\/0021-9010.93.5.959","volume":"93","author":"DJ Putka","year":"2008","unstructured":"Putka, D. J., Le, H., McCloy, R. A., & Diaz, T. (2008). Ill-structured measurement designs in organizational research: Implications for estimating interrater reliability. Journal of Applied Psychology, 93(5), 959\u2013981.","journal-title":"Journal of Applied Psychology"},{"issue":"3","key":"2516_CR117","doi-asserted-by":"crossref","first-page":"169","DOI":"10.1093\/rev\/2.3.169","volume":"2","author":"L Qiu","year":"1992","unstructured":"Qiu, L. (1992). A study of interdisciplinary research collaboration. Research Evaluation, 2(3), 169\u2013175.","journal-title":"Research Evaluation"},{"key":"2516_CR138","unstructured":"R Core Team. (2016). R: A language and environment for statistical computing (Version 3.3.1) [computer software]. Vienna, AT: R Foundation for Statistical Computing. Retrieved from http:\/\/www.R-project.org ."},{"issue":"7","key":"2516_CR118","doi-asserted-by":"crossref","first-page":"1041","DOI":"10.1002\/sim.3523","volume":"28","author":"CF Ramasundarahettige","year":"2009","unstructured":"Ramasundarahettige, C. F., Donner, A., & Zou, G. Y. (2009). Confidence interval construction for a difference between two dependent intraclass correlation coefficients. Statistics in Medicine, 28(7), 1041\u20131053.","journal-title":"Statistics in Medicine"},{"key":"2516_CR119","doi-asserted-by":"crossref","DOI":"10.4324\/9780203841624","volume-title":"Introduction to psychometric theory","author":"T Raykov","year":"2011","unstructured":"Raykov, T., & Marcoulides, G. A. (2011). Introduction to psychometric theory. New York, NY: Routledge."},{"key":"2516_CR120","unstructured":"Revelle, W. (2016). Psych: Procedures for personality and psychological research (Version 1.6.9) [computer software]. Evanston, IL: Northwestern University. http:\/\/cran.r-project.org\/web\/packages\/psych\/ . Accessed March 30 2017."},{"issue":"3","key":"2516_CR121","doi-asserted-by":"crossref","first-page":"354","DOI":"10.1037\/a0029315","volume":"17","author":"M Rhemtulla","year":"2012","unstructured":"Rhemtulla, M., Brosseau-Liard, P. \u00c9., & Savalei, V. (2012). When can categorical variables be treated as continuous? A comparison of robust continuous and categorical SEM estimation methods under suboptimal conditions. Psychological Methods, 17(3), 354\u2013373.","journal-title":"Psychological Methods"},{"key":"2516_CR122","volume-title":"Resonanz - Eine Soziologie der Weltbeziehung [Resonance\u2014A sociology of the relationship to the world]","author":"H Rosa","year":"2016","unstructured":"Rosa, H. (2016). Resonanz - Eine Soziologie der Weltbeziehung [Resonance\u2014A sociology of the relationship to the world]. Berlin, DE: Suhrkamp."},{"issue":"3","key":"2516_CR123","doi-asserted-by":"crossref","first-page":"581","DOI":"10.1093\/biomet\/63.3.581","volume":"63","author":"DB Rubin","year":"1976","unstructured":"Rubin, D. B. (1976). Inference and missing data. Biometrika, 63(3), 581\u2013592.","journal-title":"Biometrika"},{"issue":"5","key":"2516_CR124","doi-asserted-by":"crossref","first-page":"255","DOI":"10.1007\/BF02600092","volume":"8","author":"HR Rubin","year":"1993","unstructured":"Rubin, H. R., Redelmeier, D. A., Wu, A. W., & Steinberg, E. P. (1993). How reliable is peer review of scientific abstracts? Looking back at the 1991 annual meeting of the Society of General Internal Medicine. Journal of General Internal Medicine, 8(5), 255\u2013258.","journal-title":"Journal of General Internal Medicine"},{"issue":"2","key":"2516_CR125","doi-asserted-by":"crossref","first-page":"243","DOI":"10.1007\/s11336-009-9135-y","volume":"75","author":"A Satorra","year":"2010","unstructured":"Satorra, A., & Bentler, P. M. (2010). Ensuring positiveness of the scaled Chi square test statistic. Psychometrika, 75(2), 243\u2013248.","journal-title":"Psychometrika"},{"issue":"10","key":"2516_CR126","doi-asserted-by":"crossref","first-page":"935","DOI":"10.1037\/h0078544","volume":"33","author":"S Scarr","year":"1978","unstructured":"Scarr, S., & Weber, B. L. R. (1978). The reliability of reviews for the American Psychologist. American Psychologist, 33(10), 935.","journal-title":"American Psychologist"},{"issue":"2","key":"2516_CR127","doi-asserted-by":"crossref","first-page":"461","DOI":"10.1214\/aos\/1176344136","volume":"6","author":"G Schwarz","year":"1978","unstructured":"Schwarz, G. (1978). Estimating the dimension of a model. The Annals of Statistics, 6(2), 461\u2013464.","journal-title":"The Annals of Statistics"},{"issue":"9","key":"2516_CR128","doi-asserted-by":"crossref","first-page":"698","DOI":"10.1037\/h0037631","volume":"29","author":"WA Scott","year":"1974","unstructured":"Scott, W. A. (1974). Interreferee agreement on some characteristics of manuscripts submitted to Journal of Personality and Social Psychology. American Psychologist, 29(9), 698\u2013702.","journal-title":"American Psychologist"},{"key":"2516_CR129","doi-asserted-by":"crossref","DOI":"10.1002\/9780470316856","volume-title":"Variance components","author":"SR Searle","year":"1992","unstructured":"Searle, S. R., Casella, G., & McCulloch, C. E. (1992). Variance components. New York, NY: Wiley."},{"issue":"4","key":"2516_CR130","doi-asserted-by":"crossref","first-page":"350","DOI":"10.1080\/00220973.1993.10806595","volume":"61","author":"RC Serlin","year":"1993","unstructured":"Serlin, R. C. (1993). Confidence intervals and the scientific method: A case for Holm on the range. Journal of Experimental Education, 61(4), 350\u2013360.","journal-title":"Journal of Experimental Education"},{"issue":"1","key":"2516_CR131","doi-asserted-by":"crossref","first-page":"561","DOI":"10.1146\/annurev.ps.46.020195.003021","volume":"46","author":"JP Shaffer","year":"1995","unstructured":"Shaffer, J. P. (1995). Multiple hypothesis testing. Annual Review of Psychology, 46(1), 561\u2013584.","journal-title":"Annual Review of Psychology"},{"issue":"2","key":"2516_CR132","doi-asserted-by":"crossref","first-page":"420","DOI":"10.1037\/0033-2909.86.2.420","volume":"86","author":"PE Shrout","year":"1979","unstructured":"Shrout, P. E., & Fleiss, J. L. (1979). Intraclass correlations: Uses in assessing rater reliability. Psychological Bulletin, 86(2), 420\u2013428.","journal-title":"Psychological Bulletin"},{"key":"2516_CR133","unstructured":"Smith, R. (2003). The future of peer review. http:\/\/pdfs.semanticscholar.org\/7c06\/8fcda6956132db6732e6c353ffe5fe6b6f62.pdf?_ga=1.116839174.1674370711.1490806067 . Accessed March 29 2017."},{"issue":"4","key":"2516_CR134","doi-asserted-by":"crossref","first-page":"583","DOI":"10.1111\/1467-9868.00353","volume":"64","author":"DJ Spiegelhalter","year":"2002","unstructured":"Spiegelhalter, D. J., Best, N. G., Carlin, B. P., & van der Linde, A. (2002). Bayesian measures of model complexity and fit. Journal of the Royal Statistical Society: Series B, 64(4), 583\u2013639.","journal-title":"Journal of the Royal Statistical Society: Series B"},{"issue":"7651","key":"2516_CR135","doi-asserted-by":"crossref","first-page":"411","DOI":"10.1038\/544411a","volume":"544","author":"P Stephan","year":"2017","unstructured":"Stephan, P., Veugelers, R., & Wang, J. (2017). Reviewers are blinkered by bibliometrics. Nature, 544(7651), 411\u2013412.","journal-title":"Nature"},{"key":"2516_CR136","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1146\/annurev.clinpsy.032408.153639","volume":"5","author":"ME Strauss","year":"2009","unstructured":"Strauss, M. E., & Smith, G. T. (2009). Construct validity: Advances in theory and methodology. Annual Review of Clinical Psychology, 5, 1\u201325.","journal-title":"Annual Review of Clinical Psychology"},{"issue":"3","key":"2516_CR137","doi-asserted-by":"crossref","first-page":"1195","DOI":"10.1007\/s11192-016-1889-2","volume":"107","author":"I Tahamtan","year":"2016","unstructured":"Tahamtan, I., Afshar, A. S., & Ahamdzadeh, K. (2016). Factors affecting number of citations: A comprehensive review of the literature. Scientometrics, 107(3), 1195\u20131225.","journal-title":"Scientometrics"},{"issue":"1","key":"2516_CR139","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1037\/h0071663","volume":"4","author":"EL Thorndike","year":"1920","unstructured":"Thorndike, E. L. (1920). A constant error in psychological ratings. Journal of Applied Psychology, 4(1), 25\u201329.","journal-title":"Journal of Applied Psychology"},{"key":"2516_CR140","doi-asserted-by":"crossref","unstructured":"Uebersax, J. S. (1982\u20131983). A design-independent method for measuring the reliability of psychiatric diagnosis. Journal of Psychiatric Research, 17(4), 335\u2013342.","DOI":"10.1016\/0022-3956(82)90039-5"},{"issue":"7","key":"2516_CR141","doi-asserted-by":"crossref","first-page":"1282","DOI":"10.1002\/asi.22636","volume":"63","author":"HP van Dalen","year":"2012","unstructured":"van Dalen, H. P., & Henkens, K. (2012). Intended and unintended consequences of a publish-or-perish culture: A worldwide survey. Journal of the American Society for Information Science and Technology, 63(7), 1282\u20131293.","journal-title":"Journal of the American Society for Information Science and Technology"},{"issue":"3","key":"2516_CR142","doi-asserted-by":"crossref","first-page":"842","DOI":"10.1111\/cdev.12169","volume":"85","author":"R Schoot Van de","year":"2014","unstructured":"Van de Schoot, R., Kaplan, D., Denissen, J., Asendorpf, J. B., Neyer, F. J., & van Aken, M. A. G. (2014). A gentle introduction to Bayesian analysis: Applications to developmental research. Child Development, 85(3), 842\u2013860.","journal-title":"Child Development"},{"issue":"7569","key":"2516_CR143","doi-asserted-by":"crossref","first-page":"306","DOI":"10.1038\/525306a","volume":"525","author":"R Noorden van","year":"2015","unstructured":"van Noorden, R. (2015). Interdisciplinary research by the numbers: An analysis reveals the extent and impact of research that bridges disciplines. Nature, 525(7569), 306\u2013307.","journal-title":"Nature"},{"issue":"1","key":"2516_CR144","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1192\/bjp.176.1.47","volume":"176","author":"E Walsh","year":"2000","unstructured":"Walsh, E., Rooney, M., Appleby, L., & Wilkinson, G. (2000). Open peer review: A randomised controlled trial. The British Journal Of Psychiatry, 176(1), 47\u201351.","journal-title":"The British Journal Of Psychiatry"},{"issue":"4","key":"2516_CR145","doi-asserted-by":"crossref","first-page":"817","DOI":"10.2307\/1912934","volume":"48","author":"H White","year":"1980","unstructured":"White, H. (1980). A heteroskedasticity-consistent covariance matrix estimator and a direct test for heteroskedasticity. Econometrica, 48(4), 817\u2013838.","journal-title":"Econometrica"},{"issue":"1","key":"2516_CR146","doi-asserted-by":"crossref","first-page":"73","DOI":"10.1016\/0273-2297(83)90009-6","volume":"3","author":"GJ Whitehurst","year":"1983","unstructured":"Whitehurst, G. J. (1983). Interrater agreement for reviews for Developmental Review. Developmental Review, 3(1), 73\u201378.","journal-title":"Developmental Review"},{"key":"2516_CR147","unstructured":"Wirtz, M., & Caspar, F. (2002). Beurteiler\u00fcbereinstimmung und Beurteilerreliabilit\u00e4t: Methoden zur Bestimmung und Verbesserung der Zuverl\u00e4ssigkeit von Einsch\u00e4tzungen mitttels Kategoriensystemen und Ratingskalen [Inter-rater agreement and inter-rater reliability: Methods on analysis and improvement of the reliability of assessments by categorical systems and rating scales]. G\u00f6ttingen, DE: Hogrefe."},{"issue":"1","key":"2516_CR148","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1177\/0165551504041673","volume":"30","author":"M Wood","year":"2004","unstructured":"Wood, M., Roberts, M., & Howell, B. (2004). The reliability of peer reviews of papers on information systems. Journal of Information Science, 30(1), 2\u201311.","journal-title":"Journal of Information Science"},{"key":"2516_CR149","volume-title":"Multivariate exploratory data analysis: A perspective on exploratory factor analysis","author":"A Yates","year":"1987","unstructured":"Yates, A. (1987). Multivariate exploratory data analysis: A perspective on exploratory factor analysis. Albany, NY: State University of New York Press."},{"issue":"1","key":"2516_CR150","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1026\/0012-1924.51.1.1","volume":"51","author":"S Yousfi","year":"2005","unstructured":"Yousfi, S. (2005). Mythen und Paradoxien der klassischen Testtheorie (I): Testl\u00e4nge und G\u00fctekriterien [Myths and paradoxes of classical test theory (I): About test length, reliability, and validity]. Diagnostica, 51(1), 1\u201311.","journal-title":"Diagnostica"},{"issue":"1","key":"2516_CR151","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1111\/0081-1750.00078","volume":"30","author":"K-H Yuan","year":"2000","unstructured":"Yuan, K.-H., & Bentler, P. M. (2000). Three likelihood-based methods for mean and covariance structure analysis with nonnormal missing data. Sociological Methodology, 30(1), 165\u2013200.","journal-title":"Sociological Methodology"},{"issue":"2","key":"2516_CR152","doi-asserted-by":"crossref","first-page":"390","DOI":"10.1177\/0149206313501200","volume":"41","author":"MJ Zyphur","year":"2015","unstructured":"Zyphur, M. J., & Oswald, F. L. (2015). Bayesian estimation and inference: A user\u2019s guide. Journal of Management, 41(2), 390\u2013420.","journal-title":"Journal of Management"}],"container-title":["Scientometrics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11192-017-2516-6\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-017-2516-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-017-2516-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,10,17]],"date-time":"2020-10-17T11:46:45Z","timestamp":1602935205000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11192-017-2516-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,9,11]]},"references-count":152,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2017,11]]}},"alternative-id":["2516"],"URL":"https:\/\/doi.org\/10.1007\/s11192-017-2516-6","relation":{},"ISSN":["0138-9130","1588-2861"],"issn-type":[{"value":"0138-9130","type":"print"},{"value":"1588-2861","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,9,11]]}}}