{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,30]],"date-time":"2025-10-30T01:54:37Z","timestamp":1761789277203,"version":"3.37.3"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2016,3,23]],"date-time":"2016-03-23T00:00:00Z","timestamp":1458691200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Scientometrics"],"published-print":{"date-parts":[[2016,9]]},"DOI":"10.1007\/s11192-016-1929-y","type":"journal-article","created":{"date-parts":[[2016,3,23]],"date-time":"2016-03-23T11:23:18Z","timestamp":1458732198000},"page":"1651-1671","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":54,"title":["Do they agree? Bibliometric evaluation versus informed peer review in the Italian research assessment exercise"],"prefix":"10.1007","volume":"108","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-0293-482X","authenticated-orcid":false,"given":"Alberto","family":"Baccini","sequence":"first","affiliation":[]},{"given":"Giuseppe","family":"De Nicolao","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2016,3,23]]},"reference":[{"key":"1929_CR1","unstructured":"AA.VV. (2013). I voti all\u2019universit\u00e0. La Valutazione della qualit\u00e0 della ricerca in Italia. MIlano: Corriere della Sera."},{"key":"1929_CR2","doi-asserted-by":"publisher","DOI":"10.1002\/asi.23323","author":"G Abramo","year":"2015","unstructured":"Abramo, G., & D\u2019Angelo, C. A. (2015). The VQR, Italy\u2019s second national research assessment: Methodological failures and ranking distortions. Journal of the Association for Information Science and Technology.,. doi: 10.1002\/asi.23323 .","journal-title":"Journal of the Association for Information Science and Technology."},{"key":"1929_CR3","doi-asserted-by":"publisher","first-page":"33","DOI":"10.3152\/147154404781776563","volume":"13","author":"DW Aksnes","year":"2004","unstructured":"Aksnes, D. W., & Taxt, R. E. (2004). Peer reviews and bibliometric indicators: A comparative study at a Norwegian University. Research Evaluation, 13, 33\u201341. doi: 10.3152\/147154404781776563 .","journal-title":"Research Evaluation"},{"issue":"6","key":"1929_CR4","doi-asserted-by":"publisher","first-page":"e5910","DOI":"10.1371\/journal.pone.0005910","volume":"4","author":"L Allen","year":"2009","unstructured":"Allen, L., Jones, C., Dolby, K., Lynn, D., & Walport, M. (2009). Looking for landmarks: The role of expert review and bibliometric analysis in evaluating scientific publication outputs. PLoS ONE, 4(6), e5910. doi: 10.1371\/journal.pone.0005910 .","journal-title":"PLoS ONE"},{"key":"1929_CR5","volume-title":"Practical statistics for medical research","author":"DG Altman","year":"1991","unstructured":"Altman, D. G. (1991). Practical statistics for medical research. London: Chapman and Hall."},{"issue":"3","key":"1929_CR6","doi-asserted-by":"publisher","first-page":"242","DOI":"10.1093\/reseval\/rvv008","volume":"24","author":"A Ancaiani","year":"2015","unstructured":"Ancaiani, A., Anfossi, A. F., Barbara, A., Benedetto, S., Blasi, B., Carletti, V., et al. (2015). Evaluating scientific research in Italy: The 2004\u201310 research evaluation exercise. Research Evaluation, 24(3), 242\u2013255. doi: 10.1093\/reseval\/rvv008 .","journal-title":"Research Evaluation"},{"key":"1929_CR7","unstructured":"ANVUR. (2013). Rapporto finale. Valutazione della qualit\u00e0 della ricerca 2004-2010 (VQR 2004\u20132010). Roma. http:\/\/www.anvur.org\/rapporto\/ ."},{"issue":"3","key":"1929_CR8","first-page":"32","volume":"3","author":"A Baccini","year":"2014","unstructured":"Baccini, A. (2014a). La VQR di Area 13: una riflessione di sintesi. Statistica & Societ\u00e0, 3(3), 32\u201337.","journal-title":"Statistica & Societ\u00e0"},{"key":"1929_CR9","unstructured":"Baccini, A. (2014b). Lo strano caso delle concordanze della VQR. http:\/\/www.roars.it\/online\/lo-strano-caso-delle-concordanze-della-vqr\/ . www.roars.it ."},{"key":"1929_CR10","unstructured":"Baccini, A. (2016). Napol\u00e9on et l\u2019\u00e9valuation bibliom\u00e9trique de la recherche. Consid\u00e9rations sur la r\u00e9forme de l\u2019universit\u00e9 et sur l\u2019action de l\u2019agence national d\u2019\u00e9valuation en Italie. Canadian Journal of Information and Library Science-Revue Canadienne des Sciences de l\u2019Information et de Bibliotheconomie."},{"issue":"5","key":"1929_CR11","doi-asserted-by":"publisher","first-page":"715","DOI":"10.1093\/annonc\/mdg203","volume":"14","author":"T Berghmans","year":"2003","unstructured":"Berghmans, T., Meert, A. P., Mascaux, C., Paesmans, M., Lafitte, J. J., & Sculier, J. P. (2003). Citation indexes do not reflect methodological quality in lung cancer randomised trials. Annals of Oncology, 14(5), 715\u2013721. doi: 10.1093\/annonc\/mdg203 .","journal-title":"Annals of Oncology"},{"key":"1929_CR12","doi-asserted-by":"crossref","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2013a). Bibliometric evaluation vs. informed peer review: Evidence from Italy. Department of Economics DEMB. University of Modena and Reggio Emilia, Department of Economics Marco Biagi.","DOI":"10.2139\/ssrn.2363226"},{"key":"1929_CR13","doi-asserted-by":"crossref","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2013b). Bibliometric evaluation vs. informed peer review: Evidence from Italy. ReCent WP. Center for Economic Research, University of Modena and Reggio Emilia, Dept. of Economics Marco Biagi.","DOI":"10.2139\/ssrn.2363226"},{"key":"1929_CR14","doi-asserted-by":"crossref","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2013c). Bibliometric evaluation vs. informed peer review: Evidence from Italy. IZA Discussion paper. Institute for the Study of Labour (IZA), Bonn.","DOI":"10.2139\/ssrn.2363226"},{"key":"1929_CR15","doi-asserted-by":"crossref","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2013d). Bibliometric evaluation vs. informed peer review: Evidence from Italy. CEPR Discussion papers.","DOI":"10.2139\/ssrn.2363226"},{"key":"1929_CR16","doi-asserted-by":"crossref","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2013e). Bibliometric evaluation vs. informed peer review: Evidence from Italy. CSEF working papers. Naples: Centre for Studies in Economics and Finance (CSEF).","DOI":"10.2139\/ssrn.2363226"},{"key":"1929_CR17","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2014). Assessing Italian research quality: A comparison between bibliometric evaluation and informed peer review. In V. C. s. P. Portal (Ed.). www.voxeu.org . http:\/\/www.voxeu.org\/article\/research-quality-assessment-tools-lessons-italy . CEPR (Centre for Economic Policy Research)."},{"issue":"2","key":"1929_CR18","doi-asserted-by":"publisher","first-page":"451","DOI":"10.1016\/j.respol.2014.08.004","volume":"44","author":"G Bertocchi","year":"2015","unstructured":"Bertocchi, G., Gambardella, A., Jappelli, T., Nappi, C. A., & Peracchi, F. (2015). Bibliometric evaluation vs. informed peer review: Evidence from Italy. Research Policy, 44(2), 451\u2013466. doi: 10.1016\/j.respol.2014.08.004 .","journal-title":"Research Policy"},{"key":"1929_CR19","unstructured":"Cicero, T., Malgarini, M., Nappi, C. A., & Peracchi, F. (2013). Bibliometric and peer review methods for research evaluation: a methodological appraisement (in Italian). MPRA (Munich Personal REPEc Archive). Munich."},{"issue":"1","key":"1929_CR20","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J. (1960). A coefficient of agreement for nominal scales. Educational and Psychological Measurement, 20(1), 37\u201346. doi: 10.1177\/001316446002000104 .","journal-title":"Educational and Psychological Measurement"},{"issue":"4","key":"1929_CR21","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1037\/h0026256","volume":"70","author":"J Cohen","year":"1968","unstructured":"Cohen, J. (1968). Weighted kappa: Nominal scale agreement with provision for scaled disagreement or partial credit. Psychological Bulletin, 70(4), 213\u2013220. doi: 10.1037\/h0026256 .","journal-title":"Psychological Bulletin"},{"key":"1929_CR22","unstructured":"De Nicolao, G. (2014). VQR da buttare? Persino ANVUR cestina i voti usati per l\u2019assegnazione FFO 2013. http:\/\/www.roars.it\/online\/vqr-da-buttare-persino-anvur-cestina-i-voti-usati-per-lassegnazione-ffo-2013\/ ."},{"key":"1929_CR23","doi-asserted-by":"crossref","DOI":"10.1002\/0471445428","volume-title":"Statistical methods for rates and proportions","author":"JL Fleiss","year":"2003","unstructured":"Fleiss, J. L., Levin, B., & Myunghee, C. P. (2003). Statistical methods for rates and proportions. Hoboken, NJ: Wiley."},{"key":"1929_CR24","volume-title":"SPSS for windows step by step: A simple guide and reference","author":"D George","year":"2003","unstructured":"George, D., & Mallery, P. (2003). SPSS for windows step by step: A simple guide and reference (4th ed.). Boston: Allys & Bacon.","edition":"4"},{"key":"1929_CR25","unstructured":"HEFCE. (2015). The metric tide: Correlation analysis of REF2014 scores and metrics (Supplementary Report II to the Independent Review of the Role of Metrics in Research Assessment and Management)."},{"key":"1929_CR26","doi-asserted-by":"crossref","first-page":"136","DOI":"10.1002\/asi.4630340207","volume":"34","author":"MED Koenig","year":"1983","unstructured":"Koenig, M. E. D. (1983). Bibliometric indicators versus expert opinion in assessing research performance. Journal of the American Society for Information Science, 34, 136\u2013145.","journal-title":"Journal of the American Society for Information Science"},{"issue":"1","key":"1929_CR27","doi-asserted-by":"crossref","first-page":"159","DOI":"10.2307\/2529310","volume":"33","author":"JR Landis","year":"1977","unstructured":"Landis, J. R., & Koch, G. G. (1977). The measurement of observer agreement for categorical data. Biometrics, 33(1), 159\u2013174.","journal-title":"Biometrics"},{"issue":"2","key":"1929_CR28","first-page":"309","volume":"31","author":"FS Lee","year":"2007","unstructured":"Lee, F. S. (2007). The research assessment exercise, the state and the dominance of mainstream economics in British universities. Cambridge Journal of Economics, 31(2), 309\u2013325.","journal-title":"Cambridge Journal of Economics"},{"issue":"2","key":"1929_CR29","doi-asserted-by":"publisher","first-page":"160","DOI":"10.1641\/B580210","volume":"58","author":"BG Lovegrove","year":"2008","unstructured":"Lovegrove, B. G., & Johnson, S. D. (2008). Assessment of research performance in biology: How well do peer review and bibliometry correlate? BioScience, 58(2), 160\u2013164. doi: 10.1641\/B580210 .","journal-title":"BioScience"},{"key":"1929_CR30","first-page":"51","volume-title":"Reconceptualising evaluation in higher education the practice turn","author":"I McNay","year":"2011","unstructured":"McNay, I. (2011). Research assessment: Work in progress, or \u2018la lutta continua\u2019. In M. Saunders, P. Trowler, & V. Bamber (Eds.), Reconceptualising evaluation in higher education the practice turn (pp. 51\u201357). New York: McGRaw Hill."},{"issue":"3","key":"1929_CR31","doi-asserted-by":"publisher","first-page":"2165","DOI":"10.1007\/s11192-014-1512-3","volume":"102","author":"O Mryglod","year":"2015","unstructured":"Mryglod, O., Kenna, R., Holovatch, Y., & Berche, B. (2015). Predicting results of the research excellence framework using departmental h-index. Scientometrics, 102(3), 2165\u20132180. doi: 10.1007\/s11192-014-1512-3 .","journal-title":"Scientometrics"},{"key":"1929_CR32","unstructured":"RAE. (2005). RAE 2008. Guidance to panels. London: HEFCE. http:\/\/www.rae.ac.uk\/pubs\/2005\/01\/rae0105.pdf ."},{"issue":"1","key":"1929_CR33","doi-asserted-by":"crossref","first-page":"95","DOI":"10.1016\/S0048-7333(98)00026-2","volume":"27","author":"EJ Rinia","year":"1998","unstructured":"Rinia, E. J., van Leeuwen, T. N., van Vuren, H. G., & van Raan, A. F. J. (1998). Comparative analysis of a set of bibliometric indicators and central peer review criteria: Evaluation of condensed matter physics in the Netherlands. Reseach Policy, 27(1), 95\u2013107.","journal-title":"Reseach Policy"},{"key":"1929_CR34","doi-asserted-by":"crossref","DOI":"10.1201\/9781420036268","volume-title":"Handbook of parametric and nonparametric statistical procedures","author":"DJ Sheskin","year":"2003","unstructured":"Sheskin, D. J. (2003). Handbook of parametric and nonparametric statistical procedures. London: Chapman & Hall."},{"issue":"8","key":"1929_CR35","doi-asserted-by":"publisher","first-page":"1185","DOI":"10.1002\/sim.1970","volume":"24","author":"DJ Spiegelhalter","year":"2005","unstructured":"Spiegelhalter, D. J. (2005). Funnel plots for comparing institutional performance. Statistics in Medicine, 24(8), 1185\u20131202. doi: 10.1002\/sim.1970 .","journal-title":"Statistics in Medicine"},{"key":"1929_CR36","doi-asserted-by":"crossref","first-page":"29","DOI":"10.4135\/9781412995627.d5","volume-title":"Best practices in quantitative methods","author":"SE Stemler","year":"2008","unstructured":"Stemler, S. E., & Tsai, J. (2008). Best practices in interrater reliability three common approaches. In J. Osborne (Ed.), Best practices in quantitative methods (pp. 29\u201349). Thousand Oaks: Sage."},{"issue":"3\u20134","key":"1929_CR37","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1007\/s10742-011-0077-3","volume":"11","author":"S Sun","year":"2011","unstructured":"Sun, S. (2011). Meta-analysis of Cohen\u2019s kappa. Health Services and Outcomes Research Methodology, 11(3\u20134), 145\u2013163. doi: 10.1007\/s10742-011-0077-3 .","journal-title":"Health Services and Outcomes Research Methodology"},{"issue":"3","key":"1929_CR38","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1556\/Scient.67.2006.3.10","volume":"67","author":"AFJ Raan van","year":"2006","unstructured":"van Raan, A. F. J. (2006). Comparison of the Hirsch-index with standard bibliometric indicators and with peer judgment for 147 chemistry research groups. Scientometrics, 67(3), 491\u2013502. doi: 10.1556\/Scient.67.2006.3.10 .","journal-title":"Scientometrics"},{"key":"1929_CR39","unstructured":"Wouters, P., Thelwall, M., Kousha, K., Waltman, L., de Rijcke, S., Rushforth, A., et al. (2015). The metric tide: Literature review (Supplementary Report I to the Independent Review of the Role of Metrics in Research Assessment and Management). HEFCE."}],"container-title":["Scientometrics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-016-1929-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s11192-016-1929-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s11192-016-1929-y","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,16]],"date-time":"2022-06-16T01:32:20Z","timestamp":1655343140000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s11192-016-1929-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,3,23]]},"references-count":39,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2016,9]]}},"alternative-id":["1929"],"URL":"https:\/\/doi.org\/10.1007\/s11192-016-1929-y","relation":{},"ISSN":["0138-9130","1588-2861"],"issn-type":[{"type":"print","value":"0138-9130"},{"type":"electronic","value":"1588-2861"}],"subject":[],"published":{"date-parts":[[2016,3,23]]}}}