{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,21]],"date-time":"2026-04-21T03:00:43Z","timestamp":1776740443956,"version":"3.51.2"},"reference-count":87,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2020,7,1]],"date-time":"2020-07-01T00:00:00Z","timestamp":1593561600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100008289","name":"James Madison University","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100008289","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Learning Technol."],"published-print":{"date-parts":[[2020,7,1]]},"DOI":"10.1109\/tlt.2020.2971495","type":"journal-article","created":{"date-parts":[[2020,2,4]],"date-time":"2020-02-04T21:00:20Z","timestamp":1580850020000},"page":"580-592","source":"Crossref","is-referenced-by-count":4,"title":["The Effects of Dispersion and Reciprocity on Assessment Fidelity in Peer-Review Systems: A Simulation Study"],"prefix":"10.1109","volume":"13","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6464-6028","authenticated-orcid":false,"given":"Dmytro","family":"Babik","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4691-4599","authenticated-orcid":false,"given":"Scott P.","family":"Stevens","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Andrew","family":"Waters","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8493-0023","authenticated-orcid":false,"given":"David","family":"Tinapple","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1086\/268949"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.89.1.158"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.87.1.72"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1016\/j.paid.2008.01.010"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1098\/rstb.2008.0275"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1037\/a0018174"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.1111\/deci.12049"},{"key":"ref39","first-page":"1","article-title":"Probing the landscape: Toward a systematic taxonomy of online peer assessment systems in education","author":"babik","year":"2016","journal-title":"Proc Workshop Comp -Supported Peer Rev Educ 9th Int Conf Educ Data Mining"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1016\/j.geb.2005.03.001"},{"key":"ref38","article-title":"The pedagogical anatomy of peer assessment: Dissecting a peerScholar assignment","volume":"7","author":"joordens","year":"2009","journal-title":"J Syst Cybern Informat"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1007\/s11192-017-2244-y"},{"key":"ref79","author":"kendall","year":"1990","journal-title":"Rank Correlation Methods"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/2538862.2538900"},{"key":"ref32","first-page":"29","article-title":"CritViz: Web-based software supporting peer critique in large creative classrooms","volume":"15","author":"tinapple","year":"2013","journal-title":"Bulletin of the IEEE Technical Committee on Learning Technology"},{"key":"ref31","article-title":"Calibrated peer review: A writing and critical-thinking instructional tool","volume":"54","author":"russell","year":"2004","journal-title":"Teaching Tips Innov Undergraduate Sci Instruct"},{"key":"ref30","author":"benkler","year":"2006","journal-title":"The Wealth of Networks How Social Production Transforms Markets and Freedom"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1016\/j.compedu.2005.02.004"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/s10796-015-9586-x"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.1145\/1176617.1176742"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/1878450.1878470"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2008.77"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2012.167"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1287\/isre.2016.0622"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1137\/S003614450342480"},{"key":"ref28","doi-asserted-by":"crossref","DOI":"10.1093\/oso\/9780199290727.001.0001","author":"chesbrough","year":"2006","journal-title":"Open Innovation Researching a New Paradigm"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2008.07.002"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/1526993.1527001"},{"key":"ref65","article-title":"How does ranking rate? A comparison of ranking and rating tasks","author":"krosnick","year":"2003","journal-title":"Proc Conf Papers Amer Assoc Public Opinion Res"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.85.3.555"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.5437\/08956308X5504085"},{"key":"ref67","first-page":"5","article-title":"An exploratory study of reliability of ranking vs. rating in peer assessment","volume":"11","author":"song","year":"2017","journal-title":"Int J Educ Pedagog Sci"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.3758\/s13423-016-1039-0"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1016\/0191-8869(86)90014-0"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.2307\/25148625"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.2307\/258353"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1111\/j.1527-2001.2011.01178.x"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/1531793.1531816"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1016\/j.dss.2012.02.006"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.3389\/fncom.2012.00001"},{"key":"ref23","article-title":"A case for ordinal peer evaluation in MOOCs","author":"shah","year":"2013","journal-title":"NIPS Workshop on Data Driven Education"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.2307\/41703458"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10296"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1016\/j.artint.2018.11.001"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/2724660.2724672"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1287\/isre.14.1.87.14765"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2008.03.058"},{"key":"ref57","author":"shannon","year":"1964","journal-title":"The Mathematical Theory of Communication"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1037\/h0043158"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1002\/hipo.20480"},{"key":"ref54","author":"arrow","year":"1951","journal-title":"Social Choice and Individual Values"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1037\/h0037514"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.7892\/boris.37106"},{"key":"ref10","first-page":"675","article-title":"Aggregating partial rankings with applications to peer grading in massive online open courses","author":"caragiannis","year":"2015","journal-title":"Proc 1st Int Conf Autonomous Agents Multiagent Syst"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/2623330.2623654"},{"key":"ref40","article-title":"A systematic review of educational online peer assessment systems: Charting the landscape","author":"babik","year":"0","journal-title":"TBA"},{"key":"ref12","first-page":"389","article-title":"Improving the peer assessment experience on MOOC platforms","author":"staubitz","year":"2016","journal-title":"Proc ACM SIGMOD Conf"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1016\/j.asw.2014.11.001"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1080\/03075079912331379935"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.3102\/00346543068003249"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.0808450106"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1080\/01443410500345172"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1287\/orsc.2.1.125"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1080\/00405840802577569"},{"key":"ref84","author":"becker","year":"2003","journal-title":"History of the Stanford-Binet Intelligence Scales Content and Psychometrics"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1037\/0022-0663.98.4.891"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.2307\/1412107"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/1325555.1325571"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1080\/0022250X.2010.532258"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1177\/000271624222000196"},{"key":"ref3","author":"bandura","year":"1962","journal-title":"Social Learning Through Imitation"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1016\/j.stueduc.2006.01.005"},{"key":"ref5","author":"simon","year":"1969","journal-title":"The Sciences of the Artificial"},{"key":"ref85","doi-asserted-by":"publisher","DOI":"10.1002\/9781118660584.ese1325"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1109\/TPC.2008.2001254"},{"key":"ref86","author":"kaufman","year":"2006","journal-title":"Assessing Adolescent and Adult Intelligence"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1057\/palgrave.ejis.3000397"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1016\/j.jslw.2008.06.002"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1037\/10020-000"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1145\/1531674.1531692"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1002\/smj.4250171110"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/2505057"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1002\/smj.4250171106"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1016\/S0268-4012(01)00007-X"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/BF02289146"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1103\/PhysRevLett.93.268701"},{"key":"ref44","article-title":"Tuned models of peer assessment in MOOCs","author":"piech","year":"2013","journal-title":"Proc 6th Int Conf Data Mining"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1016\/j.socnet.2009.02.002"}],"container-title":["IEEE Transactions on Learning Technologies"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/4620076\/9199389\/08982044.pdf?arnumber=8982044","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,31]],"date-time":"2024-07-31T01:51:52Z","timestamp":1722390712000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8982044\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,7,1]]},"references-count":87,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/tlt.2020.2971495","relation":{},"ISSN":["1939-1382","2372-0050"],"issn-type":[{"value":"1939-1382","type":"electronic"},{"value":"2372-0050","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,7,1]]}}}