{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,30]],"date-time":"2026-04-30T03:34:18Z","timestamp":1777520058639,"version":"3.51.4"},"reference-count":48,"publisher":"Elsevier BV","issue":"1","license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"}],"content-domain":{"domain":["clinicalkey.com","clinicalkey.com.au","clinicalkey.es","clinicalkey.fr","clinicalkey.jp","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Surgical Education"],"published-print":{"date-parts":[[2020,1]]},"DOI":"10.1016\/j.jsurg.2019.07.007","type":"journal-article","created":{"date-parts":[[2019,8,20]],"date-time":"2019-08-20T17:38:53Z","timestamp":1566322733000},"page":"189-201","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":17,"title":["Reliability of Observational Assessment Methods for Outcome-based Assessment of Surgical Skill: Systematic Review and Meta-analyses"],"prefix":"10.1016","volume":"77","author":[{"given":"Marleen","family":"Groenier","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Leonie","family":"Brummer","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Brendan P.","family":"Bunting","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Anthony G.","family":"Gallagher","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"key":"10.1016\/j.jsurg.2019.07.007_bib0001","series-title":"Response to the general medical council determination on the Bristol case: senate paper 5","author":"Giddings","year":"1998"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0002","series-title":"To Err is Human","author":"Kohn","year":"2000"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0003","series-title":"Graduate Medical Education that meets the Nation's Health Needs","article-title":"Committee on the Governance and Financing of Graduate Medical Education","year":"2014"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0004","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1111\/jedm.12000","article-title":"Validating the interpretations and uses of test scores","volume":"50","author":"Kane","year":"2013","journal-title":"J Educ Meas"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0005","doi-asserted-by":"crossref","first-page":"887","DOI":"10.1136\/bmj.318.7188.887","article-title":"Assessing operative skill. Needs to become more objective","volume":"318","author":"Darzi","year":"1999","journal-title":"BMJ"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0006","doi-asserted-by":"crossref","first-page":"972","DOI":"10.1002\/bjs.7115","article-title":"Objective assessment of technical surgical skills","volume":"97","author":"Van Hove","year":"2010","journal-title":"Br J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0007","doi-asserted-by":"crossref","first-page":"1025","DOI":"10.1097\/SLA.0b013e318284f658","article-title":"Prospective, randomized assessment of transfer of training (ToT) and transfer effectiveness ratio (TER) of virtual reality simulation training for laparoscopic skill acquisition","volume":"257","author":"Gallagher","year":"2013","journal-title":"Ann Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0008","doi-asserted-by":"crossref","first-page":"274","DOI":"10.1016\/S0002-9610(99)00016-1","article-title":"Assessment of innate ability and skills for endoscopic manipulations by the advanced dundee endoscopic psychomotor tester: predictive and concurrent validity","volume":"177","author":"MacMillan","year":"1999","journal-title":"Am J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0009","doi-asserted-by":"crossref","first-page":"650","DOI":"10.1111\/medu.12220","article-title":"Tools for the direct observation and assessment of psychomotor skills in medical trainees: a systematic review","volume":"47","author":"Jelovsek","year":"2013","journal-title":"Med Educ"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0010","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1016\/j.amjsurg.2010.10.020","article-title":"Observational tools for assessment of procedural skills: a systematic review","volume":"202","author":"Ahmed","year":"2011","journal-title":"Am J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0011","doi-asserted-by":"crossref","first-page":"2664","DOI":"10.1056\/NEJMra054785","article-title":"Teaching surgical skill \u2013 changes in the wind","volume":"355","author":"Reznick","year":"2006","journal-title":"NEJM"},{"issue":"sup1","key":"10.1016\/j.jsurg.2019.07.007_bib0012","doi-asserted-by":"crossref","first-page":"S17","DOI":"10.1080\/10401334.2013.842916","article-title":"Assessment of clinical skills with standardized patients: state of the art revisited","volume":"25","author":"Swanson","year":"2013","journal-title":"Teach Learn Med"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0013","series-title":"Health Measurement Scales: A Practical Guide to Their Development and Use","author":"Streiner","year":"2003"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0014","doi-asserted-by":"crossref","first-page":"236","DOI":"10.1016\/j.injury.2010.11.042","article-title":"What makes a measurement instrument valid and reliable?","volume":"42","author":"Scholtes","year":"2011","journal-title":"Injury"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0015","doi-asserted-by":"crossref","first-page":"23","DOI":"10.20982\/tqmp.08.1.p023","article-title":"Computing inter-rater reliability for observational data: an overview and tutorial","volume":"8","author":"Hallgren","year":"2012","journal-title":"Tutor Quant Methods Psychol"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0016","doi-asserted-by":"crossref","first-page":"1869","DOI":"10.1097\/PRS.0b013e31816b19bc","article-title":"Assessing surgical skill using bench station models","volume":"121","author":"Kennedy","year":"2008","journal-title":"Plast Reconstr Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0017","doi-asserted-by":"crossref","first-page":"284","DOI":"10.1037\/1040-3590.6.4.284","article-title":"Guidelines, criteria, and rules of thumb for evaluating normed and standardized assessment instruments in psychology","volume":"6","author":"Cicchetti","year":"1994","journal-title":"Psychol Assess"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0018","doi-asserted-by":"crossref","first-page":"307","DOI":"10.1016\/S0140-6736(86)90837-8","article-title":"Statistical methods for assessing agreement between two methods of clinical measurement","volume":"327","author":"Bland","year":"1986","journal-title":"Lancet"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0019","doi-asserted-by":"crossref","first-page":"568","DOI":"10.1111\/j.1445-2197.2012.06236.x","article-title":"Objective structured assessment of technical skills and checklist scales reliability compared for high stakes assessments","volume":"84","author":"Gallagher","year":"2014","journal-title":"ANZ J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0020","series-title":"Standards for Educational and Psychological Testing","article-title":"NCME (American Educational Research Association, American Psychological Association, & National Council on Measurement in Education)","year":"1999"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0021","series-title":"Individual Differences and Personality","author":"Cooper","year":"2002"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0022","series-title":"Practical Statistics for Medical Research","author":"Altman","year":"1990"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0023","doi-asserted-by":"crossref","first-page":"1149","DOI":"10.1007\/s10459-015-9593-1","article-title":"Constructing a validity argument for the objective structured assessment of technical skills (OSATS): a systematic review of validity evidence","volume":"20","author":"Hatala","year":"2015","journal-title":"Adv Heal Sci Educ"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0024","doi-asserted-by":"crossref","first-page":"661","DOI":"10.1016\/j.ijnurstu.2011.01.016","article-title":"Guidelines for reporting reliability and agreement studies (GRRAS) were proposed","volume":"48","author":"Kottner","year":"2011","journal-title":"Int J Nurs Stud"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0025","doi-asserted-by":"crossref","first-page":"874","DOI":"10.7326\/0003-4819-140-11-200406010-00008","article-title":"Effects of training in direct observation of medical residents\u2019 clinical competence","volume":"140","author":"Holmboe","year":"2004","journal-title":"Ann Intern Med"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0026","doi-asserted-by":"crossref","first-page":"325","DOI":"10.1007\/s10459-012-9372-1","article-title":"Seeing the same thing differently: mechanisms that contribute to assessor differences in directly-observed performance assessments","volume":"18","author":"Yeates","year":"2013","journal-title":"Adv Heal Sci Educ"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0027","doi-asserted-by":"crossref","first-page":"74","DOI":"10.1007\/s11606-008-0842-3","article-title":"Effect of rater training on reliability and accuracy of mini-CEX scores: a randomized, controlled trial","volume":"24","author":"Cook","year":"2009","journal-title":"J Gen Intern Med"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0028","doi-asserted-by":"crossref","first-page":"1856","DOI":"10.1016\/j.ajog.2005.07.057","article-title":"A new curriculum for hysteroscopy training as demonstrated by an objective structured assessment of technical skills (OSATS)","volume":"193","author":"VanBlaricom","year":"2005","journal-title":"Am J Obstet Gynecol"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0029","doi-asserted-by":"crossref","first-page":"544.e1","DOI":"10.1016\/j.ajog.2007.07.027","article-title":"A curriculum for Burch colposuspension and diagnostic cystoscopy evaluated by an objective structured assessment of technical skills","volume":"197","author":"Fialkow","year":"2007","journal-title":"Am J Obstet Gynecol"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0030","doi-asserted-by":"crossref","first-page":"746","DOI":"10.1136\/bmj.292.6522.746","article-title":"Confidence intervals rather than P values: estimation rather than hypothesis testing","volume":"292","author":"Gardner","year":"1986","journal-title":"BMJ (Clin Res Ed)"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0031","doi-asserted-by":"crossref","first-page":"1158","DOI":"10.1093\/ije\/dyn204","article-title":"Commentary: heterogeneity in meta-analysis should be expected and appropriately quantified","volume":"37","author":"Higgins","year":"2008","journal-title":"Int J Epidemiol"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0032","doi-asserted-by":"crossref","first-page":"145","DOI":"10.1007\/s10742-011-0077-3","article-title":"Meta-analysis of Cohen's kappa","volume":"11","author":"Sun","year":"2011","journal-title":"Heal Serv Outcomes Res Methodol"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0033","doi-asserted-by":"crossref","first-page":"420","DOI":"10.1037\/0033-2909.86.2.420","article-title":"Intraclass correlations: uses in assessing rater reliability","volume":"86","author":"Shrout","year":"1979","journal-title":"Psychol Bull"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0034","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1037\/1082-989X.1.1.30","article-title":"Forming inferences about some intraclass correlations coefficients","volume":"1","author":"McGraw","year":"1996","journal-title":"Psychol Methods"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0035","doi-asserted-by":"crossref","first-page":"1","DOI":"10.18637\/jss.v036.i03","article-title":"Conducting meta-analyses in R with the metafor Package","volume":"36","author":"Viechtbauer","year":"2010","journal-title":"J Stat Softw"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0036","doi-asserted-by":"crossref","first-page":"458","DOI":"10.1097\/00000658-200210000-00008","article-title":"Virtual reality training improves operating room performance: results of a randomized, double-blinded study","volume":"236","author":"Seymour","year":"2002","journal-title":"Ann Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0037","doi-asserted-by":"crossref","first-page":"238","DOI":"10.1016\/j.amjsurg.2005.07.031","article-title":"Development of assessing generic and specific technical skills in laparoscopic surgery","volume":"191","author":"Sarker","year":"2006","journal-title":"Am J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0038","doi-asserted-by":"crossref","first-page":"797","DOI":"10.1016\/j.amjsurg.2006.06.050","article-title":"Proficiency-based virtual reality training significantly reduces the error rate for residents during their first 10 laparoscopic cholecystectomies","volume":"193","author":"Ahlberg","year":"2007","journal-title":"Am J Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0039","doi-asserted-by":"crossref","first-page":"2402","DOI":"10.1002\/lary.20678","article-title":"Pilot testing of an assessment tool for competency in mastoidectomy","volume":"119","author":"Laeeq","year":"2009","journal-title":"Laryngoscope"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0040","doi-asserted-by":"crossref","first-page":"431","DOI":"10.1002\/lary.24838","article-title":"Mastoidectomy performance assessment of virtual simulation training using final-product analysis","volume":"125","author":"Andersen","year":"2015","journal-title":"Laryngoscope"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0041","doi-asserted-by":"crossref","first-page":"399","DOI":"10.1097\/AAP.0000000000000126","article-title":"Evaluation of a task-specific checklist and global rating scale for ultrasound-guided regional anesthesia","volume":"39","author":"Wong","year":"2014","journal-title":"Reg Anesth Pain Med"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0042","doi-asserted-by":"crossref","first-page":"1639","DOI":"10.1016\/j.arthro.2015.04.092","article-title":"The Bankart performance metrics combined with a shoulder model simulator create a precise and accurate training tool for measuring surgeon skill","volume":"31","author":"Angelo","year":"2015","journal-title":"Arthroscopy"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0043","doi-asserted-by":"crossref","first-page":"1655","DOI":"10.1016\/j.arthro.2015.05.006","article-title":"The Bankart performance metrics combined with a cadaveric shoulder create a precise and accurate assessment tool for measuring surgeon skill","volume":"31","author":"Angelo","year":"2015","journal-title":"Arthroscopy"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0044","doi-asserted-by":"crossref","first-page":"46","DOI":"10.1016\/j.jss.2015.06.057","article-title":"Rapid assessment of technical competency: the 8-min suture test","volume":"200","author":"Day","year":"2015","journal-title":"J Surg Res"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0045","doi-asserted-by":"crossref","first-page":"202","DOI":"10.1016\/j.otohns.2009.11.023","article-title":"From virtual reality to the operating room: the endoscopic sinus surgery simulator experiment","volume":"142","author":"Fried","year":"2010","journal-title":"Otolaryngol- Head Neck Surg"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0046","doi-asserted-by":"crossref","first-page":"627","DOI":"10.1111\/codi.12887","article-title":"Simulated training in colonoscopic stenting of colonic strictures: Validation of a cadaver model","volume":"17","author":"Iordache","year":"2015","journal-title":"Color Dis"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0047","doi-asserted-by":"crossref","first-page":"457","DOI":"10.1007\/s10459-011-9322-3","article-title":"Comparing the use of global rating scale with checklists for the assessment of central venous catheterization skills using simulation","volume":"17","author":"Ma","year":"2012","journal-title":"Adv Heal Sci Educ"},{"key":"10.1016\/j.jsurg.2019.07.007_bib0048","doi-asserted-by":"crossref","first-page":"e187","DOI":"10.2106\/JBJS.M.00340","article-title":"Using the arthroscopic surgery skill evaluation tool as a pass-fail examination","volume":"95","author":"Koehler","year":"2013","journal-title":"J Bone Joint Surg Am"}],"container-title":["Journal of Surgical Education"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S193172041930251X?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S193172041930251X?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,30]],"date-time":"2025-10-30T12:50:30Z","timestamp":1761828630000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S193172041930251X"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,1]]},"references-count":48,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2020,1]]}},"alternative-id":["S193172041930251X"],"URL":"https:\/\/doi.org\/10.1016\/j.jsurg.2019.07.007","relation":{},"ISSN":["1931-7204"],"issn-type":[{"value":"1931-7204","type":"print"}],"subject":[],"published":{"date-parts":[[2020,1]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Reliability of Observational Assessment Methods for Outcome-based Assessment of Surgical Skill: Systematic Review and Meta-analyses","name":"articletitle","label":"Article Title"},{"value":"Journal of Surgical Education","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.jsurg.2019.07.007","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2019 Association of Program Directors in Surgery. Published by Elsevier Inc. All rights reserved.","name":"copyright","label":"Copyright"}]}}