{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,8]],"date-time":"2026-02-08T04:38:47Z","timestamp":1770525527061,"version":"3.49.0"},"reference-count":35,"publisher":"IEEE","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018,10]]},"DOI":"10.1109\/snams.2018.8554744","type":"proceedings-article","created":{"date-parts":[[2018,12,4]],"date-time":"2018-12-04T01:03:09Z","timestamp":1543885389000},"page":"80-87","source":"Crossref","is-referenced-by-count":10,"title":["Inter-Rater Agreement for Social Computing Studies"],"prefix":"10.1109","author":[{"given":"Joni O.","family":"Salminen","sequence":"first","affiliation":[]},{"given":"Hind A.","family":"Al-Merekhi","sequence":"additional","affiliation":[]},{"given":"Partha","family":"Dey","sequence":"additional","affiliation":[]},{"given":"Bernard J.","family":"Jansen","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref33","first-page":"543","volume":"43","author":"feinstein","year":"1990","journal-title":"High agreement but low kappa I the problems of two paradoxes"},{"key":"ref32","author":"girard","year":"2016","journal-title":"mReliability Reliability analysis in MATLAB"},{"key":"ref31","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1016\/0895-4356(90)90158-L","volume":"43","author":"feinstein","year":"1990","journal-title":"Journal of Clinical Epidemiology"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1348\/000711006X126600","volume":"61","author":"gwet","year":"2008","journal-title":"British Journal of Mathematical and Statistical Psychology"},{"key":"ref35","first-page":"1","volume":"14","author":"howe","year":"2006","journal-title":"The Rise of Crowdsourcing"},{"key":"ref34","first-page":"53","author":"saltelli","year":"2008","journal-title":"Experimental Designs"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753504"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1109\/BigDataService.2016.24"},{"key":"ref12","first-page":"280","volume":"58","author":"alvaro","year":"2015","journal-title":"Crowdsourcing twitter annotations to identify first-hand experiences of prescription drug use"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/1999676.1999689"},{"key":"ref14","first-page":"95","volume":"30","author":"di eugenio","year":"2004","journal-title":"The kappa statistic A second look"},{"key":"ref15","volume":"34","author":"artstein","year":"2008","journal-title":"Inter-coder agreement for computational linguistics"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/2872427.2883062"},{"key":"ref17","first-page":"590","article-title":"A comparative study of demographic attribute inference in twitter","volume":"15","author":"chen","year":"2015","journal-title":"Ninth International AAAI Conference on Web and Social Media"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1145\/2964284.2967261"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/2851581.2890377"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1086\/266577"},{"key":"ref4","author":"krippendorff","year":"0","journal-title":"Reliability"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1037\/h0031619"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"101","DOI":"10.1145\/2756406.2757741","article-title":"Debugging a crowdsourced task with low inter-rater agreement","author":"alonso","year":"2015","journal-title":"Proceedings of the 15th ACM\/IEEE-CS Joint Conference on Digital Libraries ser JCDL '15"},{"key":"ref6","volume":"41","year":"0","journal-title":"The unified and holistic method gamma for inter-annotator agreement measure and alignment"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1177\/001316447003000105"},{"key":"ref5","author":"joyce","year":"2013","journal-title":"Picking the best intercoder reliability statistic for your digital activism content analysis"},{"key":"ref8","year":"0","journal-title":"Using semantic similarity to reduce wrong labels in distant supervision for relation extraction"},{"key":"ref7","volume":"37","year":"0","journal-title":"What determines inter-coder agreement in manual annotations? a meta-analytic investigation"},{"key":"ref2","first-page":"319","volume":"34","author":"reidsma","year":"2008","journal-title":"Reliability measurement without limits"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"969","DOI":"10.1016\/j.ipm.2018.06.007","volume":"54","author":"hsu","year":"2018","journal-title":"Information Processing and Management"},{"key":"ref1","doi-asserted-by":"crossref","first-page":"490","DOI":"10.1016\/j.ipm.2018.03.006","volume":"54","author":"neto","year":"2018","journal-title":"Information Processing and Management"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/2700171.2791023"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1109\/PASSAT\/SocialCom.2011.91"},{"key":"ref21","first-page":"16","volume":"3","author":"orellana-rodriguez","year":"2017","journal-title":"Spreading ones tweets How can journalists gain attention for their tweeted news?"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/1926180.1926199"},{"key":"ref23","author":"hu","year":"2014","journal-title":"What We Instagram A First Analysis of Instagram Photo Content and User Types"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1086\/266520"}],"event":{"name":"2018 Fifth International Conference on Social Networks Analysis, Management and Security (SNAMS)","location":"Valencia","start":{"date-parts":[[2018,10,15]]},"end":{"date-parts":[[2018,10,18]]}},"container-title":["2018 Fifth International Conference on Social Networks Analysis, Management and Security (SNAMS)"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/8536820\/8554420\/08554744.pdf?arnumber=8554744","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,27]],"date-time":"2022-01-27T04:24:19Z","timestamp":1643257459000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/8554744\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,10]]},"references-count":35,"URL":"https:\/\/doi.org\/10.1109\/snams.2018.8554744","relation":{},"subject":[],"published":{"date-parts":[[2018,10]]}}}