{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,28]],"date-time":"2025-10-28T15:02:06Z","timestamp":1761663726774,"version":"3.37.3"},"reference-count":67,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[2018,6,1]],"date-time":"2018-06-01T00:00:00Z","timestamp":1527811200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2018,6,1]],"date-time":"2018-06-01T00:00:00Z","timestamp":1527811200000},"content-version":"am","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"},{"start":{"date-parts":[[2018,6,1]],"date-time":"2018-06-01T00:00:00Z","timestamp":1527811200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2018,6,1]],"date-time":"2018-06-01T00:00:00Z","timestamp":1527811200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/100000001","name":"NSF","doi-asserted-by":"publisher","award":["1534471"],"award-info":[{"award-number":["1534471"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Comput. Soc. Syst."],"published-print":{"date-parts":[[2018,6]]},"DOI":"10.1109\/tcss.2018.2798624","type":"journal-article","created":{"date-parts":[[2018,3,5]],"date-time":"2018-03-05T22:08:47Z","timestamp":1520287727000},"page":"358-370","source":"Crossref","is-referenced-by-count":2,"title":["FlexCCT: A Methodological Framework and Software for Ratings Analysis and Wisdom of the Crowd Applications"],"prefix":"10.1109","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7226-2750","authenticated-orcid":false,"given":"Stephen L.","family":"France","sequence":"first","affiliation":[]},{"given":"Mahyar","family":"Sharif Vaghefi","sequence":"additional","affiliation":[]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0579-7752","authenticated-orcid":false,"given":"William H.","family":"Batchelder","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1177\/1525822X13520280"},{"journal-title":"ANTHROPAC","year":"1992","author":"borgatti","key":"ref38"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/TAC.1974.1100705"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/BF02294361"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1984.10478083"},{"key":"ref30","first-page":"1137","article-title":"A study of cross-validation and bootstrap for accuracy estimation and model selection","volume":"14","author":"kohavi","year":"1995","journal-title":"Proc Int Joint Conf AI"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-011-0138-0"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmp.2012.06.002"},{"key":"ref35","doi-asserted-by":"publisher","DOI":"10.2307\/2334280"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1176344136"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1007\/BF02289694"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-12079-4_15"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1007\/BF02290164"},{"key":"ref63","first-page":"1","article-title":"Consensus ratings: Reconceptualizing additive bias","author":"france","year":"2015","journal-title":"Proc KDD ASSESS Workshop"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/212094.212114"},{"key":"ref64","first-page":"1","article-title":"Semi-supervised learning literature survey","author":"zhu","year":"2005"},{"journal-title":"Bootstrap A Statistical Method","year":"2008","author":"singh","key":"ref27"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2016.10.015"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1214\/12-BA703"},{"journal-title":"The Elements of Statistical Learning Data Mining Inference and Prediction","year":"2009","author":"hastie","key":"ref29"},{"key":"ref67","first-page":"1","article-title":"Stan: A probabilistic programming language","volume":"20","author":"carpenter","year":"2016","journal-title":"J Statist Softw"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2014.06.044"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1109\/TCSS.2016.2563460"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1177\/0013164488484007"},{"key":"ref22","doi-asserted-by":"crossref","DOI":"10.1201\/9780429258589","author":"altman","year":"1990","journal-title":"Practical Statistics for Medical Research"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1177\/00131640121971239"},{"journal-title":"Statistical Power Analysis for the Behavioral Sciences","year":"1988","author":"cohen","key":"ref24"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.2307\/2529310"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4899-4541-9"},{"key":"ref25","first-page":"231","author":"rosenthal","year":"1994","journal-title":"Parametric Measures of Effect Size"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2929.2005.02237.x"},{"key":"ref51","first-page":"56","article-title":"Practical applications of correspondence analysis to categorical data in market research","volume":"5","author":"kennedy","year":"1996","journal-title":"J Target Measure Anal Marketing"},{"key":"ref59","first-page":"369","article-title":"Peer grading in a course on algorithms and data structures: Machine learning algorithms do not improve over simple baselines","author":"sajjadi","year":"2016","journal-title":"Proc 3rd ACM Conf Learn Scale (L S)"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1109\/TSMCC.2010.2053532"},{"key":"ref57","first-page":"3","article-title":"The state of educational data mining in 2009: A review and future visions","volume":"1","author":"baker","year":"2009","journal-title":"Educational Data Mining"},{"key":"ref56","first-page":"382","article-title":"Bayesian model averaging: A tutorial","volume":"14","author":"hoeting","year":"1999","journal-title":"Statist Sci"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1214\/09-SS054"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1177\/0049124104268644"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1037\/a0027127"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1177\/0049124103262065"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1177\/0022022106295441"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1007\/s11336-013-9382-9"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1007\/s00357-006-0014-2"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0152803"},{"journal-title":"An Introduction to MATLAB for Behavioral Researchers","year":"2013","author":"madan","key":"ref13"},{"key":"ref14","doi-asserted-by":"crossref","DOI":"10.4324\/9780203112106","author":"rosenbaum","year":"2014","journal-title":"MATLAB for Behavioral Scientists"},{"journal-title":"Probabilistic models for some intelligence and attainment tests","year":"1993","author":"rasch","key":"ref15"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1177\/014662168400800210"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1111\/j.1745-3984.1993.tb00426.x"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmp.2014.06.001"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-83943-6_15"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1080\/0022250X.1997.9990193"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1002\/9781119170174.epcn506"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1525\/aa.1986.88.2.02a00020"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1177\/0013164414527448"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1016\/j.ins.2014.06.015"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1037\/h0063675"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1177\/0081175014529767"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2923.2008.03172.x"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.2307\/2094981"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-2929.2004.02012.x"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.87.3.564"},{"journal-title":"Mathematical Techniques in Multisensor Data Fusion","year":"2004","author":"hall","key":"ref42"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1198\/jasa.2009.0111"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/BF02294151"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1201\/CHSTSOBESCI"}],"container-title":["IEEE Transactions on Computational Social Systems"],"original-title":[],"link":[{"URL":"https:\/\/ieeexplore.ieee.org\/ielaam\/6570650\/8365861\/8306488-aam.pdf","content-type":"application\/pdf","content-version":"am","intended-application":"syndication"},{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6570650\/8365861\/08306488.pdf?arnumber=8306488","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,4,8]],"date-time":"2022-04-08T18:54:21Z","timestamp":1649444061000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8306488\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,6]]},"references-count":67,"journal-issue":{"issue":"2"},"URL":"https:\/\/doi.org\/10.1109\/tcss.2018.2798624","relation":{},"ISSN":["2329-924X"],"issn-type":[{"type":"electronic","value":"2329-924X"}],"subject":[],"published":{"date-parts":[[2018,6]]}}}