{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,24]],"date-time":"2025-12-24T12:40:30Z","timestamp":1766580030641},"reference-count":42,"publisher":"Elsevier BV","issue":"5","license":[{"start":{"date-parts":[[2003,9,1]],"date-time":"2003-09-01T00:00:00Z","timestamp":1062374400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Computer Standards &amp; Interfaces"],"published-print":{"date-parts":[[2003,9]]},"DOI":"10.1016\/s0920-5489(03)00018-7","type":"journal-article","created":{"date-parts":[[2003,4,7]],"date-time":"2003-04-07T17:19:33Z","timestamp":1049735973000},"page":"477-499","source":"Crossref","is-referenced-by-count":15,"title":["Evaluating interrater agreement in SPICE-based assessments"],"prefix":"10.1016","volume":"25","author":[{"given":"Ho-Won","family":"Jung","sequence":"first","affiliation":[]}],"member":"78","reference":[{"key":"10.1016\/S0920-5489(03)00018-7_BIB1","series-title":"Categorical Data Analysis","author":"Agresti","year":"1990"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB2","series-title":"An Introduction to Categorical Data Analysis","author":"Agresti","year":"1996"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB3","doi-asserted-by":"crossref","first-page":"375","DOI":"10.2307\/3001775","article-title":"Test for linear trend in proportions and frequencies","volume":"11","author":"Armitage","year":"1955","journal-title":"Biometrics"},{"issue":"5","key":"10.1016\/S0920-5489(03)00018-7_BIB4","doi-asserted-by":"crossref","first-page":"423","DOI":"10.1016\/0895-4356(93)90018-V","article-title":"Bias, prevalence and Kappa","volume":"46","author":"Byrt","year":"1993","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB5","doi-asserted-by":"crossref","first-page":"113","DOI":"10.2307\/3314658","article-title":"Refining binomial confidence intervals","volume":"14","author":"Casella","year":"1987","journal-title":"Canadian Journal of Statistics"},{"issue":"6","key":"10.1016\/S0920-5489(03)00018-7_BIB6","doi-asserted-by":"crossref","first-page":"551","DOI":"10.1016\/0895-4356(90)90159-M","article-title":"High agreement but low Kappa: II. The problems of two paradoxes","volume":"43","author":"Cicchetti","year":"1990","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB7","doi-asserted-by":"crossref","first-page":"417","DOI":"10.2307\/3001616","article-title":"Some methods for strengthening the common tests","volume":"10","author":"Cochran","year":"1954","journal-title":"Biometrics"},{"issue":"1","key":"10.1016\/S0920-5489(03)00018-7_BIB8","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","article-title":"A coefficient of agreement for nominal scales","volume":"XX","author":"Cohen","year":"1960","journal-title":"Educational and Psychological Measurement"},{"issue":"4","key":"10.1016\/S0920-5489(03)00018-7_BIB9","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1037\/h0026256","article-title":"Weighted Kappa: nominal scale agreement with provision for scaled disagreement or partial credit","volume":"70","author":"Cohen","year":"1968","journal-title":"Psychological Bulletin"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB10","series-title":"Proceedings of the Second IEEE International Software Engineering Standards Symposium","first-page":"84","article-title":"SPICE: an empiricist's perspective","author":"El-Emam","year":"1995"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB11","series-title":"Proceedings of the Fourth International Conference on the Software Process","first-page":"149","article-title":"Interrater agreement in SPICE based assessments: some preliminary results","author":"El-Emam","year":"1996"},{"issue":"4","key":"10.1016\/S0920-5489(03)00018-7_BIB12","doi-asserted-by":"crossref","first-page":"291","DOI":"10.1002\/(SICI)1099-1670(199612)2:4<291::AID-SPIP54>3.0.CO;2-F","article-title":"Assessor agreement in rating SPICE processes","volume":"2","author":"El-Emam","year":"1996","journal-title":"Software Process Improvement and Practice"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB13","series-title":"Proceedings of the Third International Symposium on Software Engineering Standards","first-page":"69","article-title":"Modeling the reliability of SPICE based assessments","author":"El-Emam","year":"1997"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB14","series-title":"Proceedings of the 5th International Symposium on Software Metrics","first-page":"72","article-title":"The internal consistency of the ISO\/IEC 15504 software process capability scale","author":"El-Emam","year":"1998"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB15","series-title":"SPICE: The Theory and Practice of Software Process Improvement and Capability Determination","first-page":"357","article-title":"Interrater agreement in assessment ratings","author":"El-Emam","year":"1998"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB16","series-title":"Proceedings of the 5th International Symposium on Software Metrics","first-page":"38","article-title":"Cost implications of interrater agreement for software process assessment","author":"El-Emam","year":"1998"},{"issue":"2","key":"10.1016\/S0920-5489(03)00018-7_BIB17","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1023\/A:1009820201126","article-title":"Benchmarking Kappa: interrater agreement in software process assessments","volume":"4","author":"El-Emam","year":"1999","journal-title":"Empirical Software Engineering: An International Journal"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB18","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1016\/S0065-2458(00)80008-X","article-title":"An empirical review of software process assessments","volume":"53","author":"El-Emam","year":"2000","journal-title":"Advances in Computers"},{"issue":"1","key":"10.1016\/S0920-5489(03)00018-7_BIB19","doi-asserted-by":"crossref","first-page":"23","DOI":"10.1016\/S0164-1212(01)00046-2","article-title":"An evaluation of the ISO\/IEC 15504 assessment model","volume":"59","author":"El-Emam","year":"2001","journal-title":"Journal of Systems and Software"},{"issue":"6","key":"10.1016\/S0920-5489(03)00018-7_BIB20","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1016\/0895-4356(90)90158-L","article-title":"High agreement but low Kappa: I. The problems of two paradoxes","volume":"43","author":"Feinstein","year":"1990","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB21","series-title":"Statistical Methods for Rates and Proportions","author":"Fleiss","year":"1981"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB22","doi-asserted-by":"crossref","first-page":"613","DOI":"10.1177\/001316447303300309","article-title":"The equivalence of weighted kappa and the interclass correlation coefficient as measured of reliability","volume":"33","author":"Fleiss","year":"1973","journal-title":"Educational and Psychological Measurement"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB23","series-title":"Applied Categorical Data Analysis","author":"Freeman","year":"1987"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB24","series-title":"Proceedings of the Fourth International Software Metrics Symposium","first-page":"2","article-title":"Evaluating the interrater agreement of process capability ratings","author":"Fusaro","year":"1997"},{"issue":"2","key":"10.1016\/S0920-5489(03)00018-7_BIB25","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1023\/A:1008036316053","article-title":"The internal consistencies of the 1987 SEI maturity questionnaire and the SPICE capability dimension","volume":"3","author":"Fusaro","year":"1998","journal-title":"Empirical Software Engineering: An International Journal"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB26","doi-asserted-by":"crossref","first-page":"797","DOI":"10.1002\/(SICI)1097-0258(19980430)17:8<797::AID-SIM776>3.0.CO;2-G","article-title":"Kappa-like indices of observer agreement viewed from a latent class perspective","volume":"17","author":"Guggenmoos-Holzmann","year":"1998","journal-title":"Statistics in Medicine"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB27","unstructured":"ISO\/IEC JTC1 Directive, Procedures for the Technical Work of ISO\/IEC JTC1 (ISO, Geneva, Switzerland, 1999), http:\/\/www.jtc1.org\/directives\/toc.htm."},{"key":"10.1016\/S0920-5489(03)00018-7_BIB28","unstructured":"ISO\/IEC TR2 15504, Information Technology\u2014Software Process Assessment: Part 1\u20139, ISO, Geneva, Switzerland, 1998 (Part 5 was published in 1999)."},{"issue":"5","key":"10.1016\/S0920-5489(03)00018-7_BIB29","doi-asserted-by":"crossref","first-page":"205","DOI":"10.1002\/spip.146","article-title":"Findings from Phase 2 of the SPICE Trials","volume":"6","author":"Jung","year":"2001","journal-title":"Software Process Improvement and Practice: International Journal"},{"issue":"1","key":"10.1016\/S0920-5489(03)00018-7_BIB30","doi-asserted-by":"crossref","first-page":"43","DOI":"10.1016\/S0164-1212(01)00047-4","article-title":"The Relationship between ISO\/IEC 15504 process capability levels, ISO 9001 certification, and organizational size: an empirical study","volume":"59","author":"Jung","year":"2001","journal-title":"Journal of Systems and Software"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB31","series-title":"Proceedings of SPICE2002","first-page":"105","article-title":"An evaluation of SPICE rating scale with regard to the internal consistency of capability measurement","author":"Jung","year":"2002"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB32","unstructured":"H.-W. Jung, Evaluating the internal consistency of SPICE process capability indictors, submitted for publication, 2002."},{"key":"10.1016\/S0920-5489(03)00018-7_BIB33","series-title":"A Guideline for KSPICE Assessment Procedure","year":"2002"},{"issue":"4","key":"10.1016\/S0920-5489(03)00018-7_BIB34","doi-asserted-by":"crossref","first-page":"431","DOI":"10.1016\/0895-4356(95)00571-4","article-title":"Behavior and interpretation of the statistic: resolution of the two paradoxes","volume":"49","author":"Lantz","year":"1996","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB35","series-title":"Proceedings of the 2nd Asia-Pacific Conference on Quality Software (APAQS2001)","first-page":"341","article-title":"Analysis of interrater agreement In ISO\/IEC 15504-based software process assessment","author":"Lee","year":"2001"},{"issue":"12","key":"10.1016\/S0920-5489(03)00018-7_BIB36","doi-asserted-by":"crossref","first-page":"811","DOI":"10.1016\/S0950-5849(98)00101-3","article-title":"Combining empirical results in software engineering","volume":"40","author":"Pickard","year":"1998","journal-title":"Information and Software Technology"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB37","series-title":"Engineering Statistics","author":"Montgomery","year":"1998"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB38","series-title":"Standard CMMI Appraisal Method for Process Improvement (SCAMPI), Version 1.1: Method Definition Document (CMU\/SEI-HB-2001)","year":"2000"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB39","series-title":"Assessment Requirements for CMMI (ARC), Version 1.0 (CMU\/SEI-2000-TR-011)","year":"2000"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB40","doi-asserted-by":"crossref","unstructured":"J.-M. Simon, K. El-Emam, S.-R. Jacquet, F. Babet, The Reliability of ISO\/IEC PDTR 15504 assessments, International Software Engineering Research Network, Technical Report ISERN-97-28, 1997.","DOI":"10.1002\/(SICI)1099-1670(199709)3:3<177::AID-SPIP80>3.0.CO;2-P"},{"key":"10.1016\/S0920-5489(03)00018-7_BIB41","unstructured":"StatXact-4 for Windows, Software for Exact Nonparametric Inference, Cytel Software, Cambridge, MA, 1999."},{"key":"10.1016\/S0920-5489(03)00018-7_BIB42","unstructured":"J.S. Uebersax, Statistical Methods for Rater Agreement, http:\/\/ourworld.compuserve.com\/homepages\/jsuebersax\/agree.htm, 2001."}],"container-title":["Computer Standards &amp; Interfaces"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0920548903000187?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0920548903000187?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,3,24]],"date-time":"2019-03-24T17:57:46Z","timestamp":1553450266000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0920548903000187"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2003,9]]},"references-count":42,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2003,9]]}},"alternative-id":["S0920548903000187"],"URL":"https:\/\/doi.org\/10.1016\/s0920-5489(03)00018-7","relation":{},"ISSN":["0920-5489"],"issn-type":[{"value":"0920-5489","type":"print"}],"subject":[],"published":{"date-parts":[[2003,9]]}}}