{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T00:49:16Z","timestamp":1775263756535,"version":"3.50.1"},"reference-count":36,"publisher":"Springer Science and Business Media LLC","issue":"S12","license":[{"start":{"date-parts":[[2013,9,1]],"date-time":"2013-09-01T00:00:00Z","timestamp":1377993600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/2.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["BMC Bioinformatics"],"published-print":{"date-parts":[[2013,9]]},"DOI":"10.1186\/1471-2105-14-s12-s5","type":"journal-article","created":{"date-parts":[[2019,12,11]],"date-time":"2019-12-11T02:00:57Z","timestamp":1576029657000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Learning by aggregating experts and filtering novices: a solution to crowdsourcing problems in bioinformatics"],"prefix":"10.1186","volume":"14","author":[{"given":"Ping","family":"Zhang","sequence":"first","affiliation":[]},{"given":"Weidan","family":"Cao","sequence":"additional","affiliation":[]},{"given":"Zoran","family":"Obradovic","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2013,9,24]]},"reference":[{"key":"6049_CR1","first-page":"88","volume-title":"Proc Human Computation Workshop","author":"JJ Hyun","year":"2011","unstructured":"Hyun JJ, Lease M: Improving Consensus Accuracy via Z-score and Weighted Voting. Proc Human Computation Workshop. 2011, 88-90."},{"key":"6049_CR2","first-page":"419","volume-title":"Proc AAAI Conference on Artificial Intelligence","author":"S Chen","year":"2010","unstructured":"Chen S, Zhang J, Chen G, Zhang C: What if the irresponsible teachers are dominating. Proc AAAI Conference on Artificial Intelligence. 2010, 419-424."},{"key":"6049_CR3","volume-title":"Proc Conference on Learning Theory","author":"O Dekel","year":"2009","unstructured":"Dekel O, Shamir O: Vox populi: Collecting high-quality labels from a crowd. Proc Conference on Learning Theory. 2009"},{"key":"6049_CR4","doi-asserted-by":"publisher","first-page":"254","DOI":"10.3115\/1613715.1613751","volume-title":"Proc Conference on Empirical Methods on Natural Language Processing","author":"R Snow","year":"2008","unstructured":"Snow R, O'Connor B, Jurafsky D, Ng AY: Cheap and fast - but is it good? Evaluating non-expert annotations for natural language tasks. Proc Conference on Empirical Methods on Natural Language Processing. 2008, 254-263."},{"key":"6049_CR5","doi-asserted-by":"crossref","first-page":"889","DOI":"10.1145\/1553374.1553488","volume-title":"Proc International Conference on Machine Learning","author":"VC Raykar","year":"2009","unstructured":"Raykar VC, Yu S, Zhao LH, Jerebko AK, Florin C, Valadez GH, Bogoni L, Moy L: Supervised learning from multiple experts: whom to trust when everyone lies a bit. Proc International Conference on Machine Learning. 2009, 889-896."},{"key":"6049_CR6","first-page":"2035","volume-title":"Advances in Neural Information Processing Systems","author":"J Whitehill","year":"2009","unstructured":"Whitehill J, Ruvolo P, Wu T, Bergsma J, Movellan J: Whose vote should count more: optimal integration of labels from labelers of unknown expertise. Advances in Neural Information Processing Systems. 2009, 2035-2043."},{"key":"6049_CR7","first-page":"2424","volume-title":"Advances in Neural Information Processing Systems","author":"P Welinder","year":"2010","unstructured":"Welinder P, Branson S, Belongie S, Perona P: The multidimensional wisdom of crowds. Advances in Neural Information Processing Systems. 2010, 2424-2432."},{"key":"6049_CR8","first-page":"49","volume-title":"Proc IEEE Conference on Bioinformatics and Biomedicine","author":"P Zhang","year":"2010","unstructured":"Zhang P, Obradovic Z: Unsupervised integration of multiple protein disorder predictors. Proc IEEE Conference on Bioinformatics and Biomedicine. 2010, 49-52."},{"key":"6049_CR9","first-page":"932","volume-title":"Proc International Conference on Artificial Intelligence and Statistics","author":"Y Yan","year":"2010","unstructured":"Yan Y, Rosales R, Fung G, Schmidt MW, Valadez GH, Bogoni L, Moy L, Dy JG: Modeling annotator expertise: Learning when everybody knows a bit of something. Proc International Conference on Artificial Intelligence and Statistics. 2010, 932-939."},{"key":"6049_CR10","doi-asserted-by":"publisher","first-page":"553","DOI":"10.1007\/978-3-642-23808-6_36","volume-title":"Proc European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases","author":"P Zhang","year":"2011","unstructured":"Zhang P, Obradovic Z: Learning from Inconsistent and Unreliable Annotators by a Gaussian Mixture Model and Bayesian Information Criteria. Proc European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases. 2011, 553-568."},{"key":"6049_CR11","doi-asserted-by":"crossref","first-page":"465","DOI":"10.1145\/1935826.1935896","volume-title":"Proc ACM International Conference on Web Search and Data Mining","author":"G Kasneci","year":"2011","unstructured":"Kasneci G, Gael JV, Stern DH, Graepel T: CoBayes: bayesian knowledge corroboration with assessors of unknown areas of expertise. Proc ACM International Conference on Web Search and Data Mining. 2011, 465-474."},{"key":"6049_CR12","first-page":"635","volume-title":"Proc IEEE International Conference on Data Mining","author":"VS Sheng","year":"2011","unstructured":"Sheng VS: Simple Multiple Noisy Label Utilization Strategies. Proc IEEE International Conference on Data Mining. 2011, 635-644."},{"key":"6049_CR13","first-page":"73","volume-title":"Proc AAAI Conference on Artificial Intelligence","author":"H Kajino","year":"2012","unstructured":"Kajino H, Tsuboi Y, Kashima H: A Convex Formulation for Learning from Crowds. Proc AAAI Conference on Artificial Intelligence. 2012, 73-79."},{"key":"6049_CR14","first-page":"107","volume-title":"Proc of the Human Computation Workshop","author":"H Kajino","year":"2012","unstructured":"Kajino H, Tsuboi Y, Sato I, Kashima H: Learning from Crowds and Experts. Proc of the Human Computation Workshop. 2012, 107-113."},{"key":"6049_CR15","volume-title":"Advances in Neural Information Processing Systems","author":"D Zhou","year":"2012","unstructured":"Zhou D, Platt JC, Basu S, Mao Y: Learning from the Wisdom of Crowds by Minimax Entropy. Advances in Neural Information Processing Systems. 2012"},{"key":"6049_CR16","volume-title":"Advances in Neural Information Processing Systems","author":"Q Liu","year":"2012","unstructured":"Liu Q, Peng J, Ihler A: Variational Inference for Crowdsourcing. Advances in Neural Information Processing Systems. 2012"},{"key":"6049_CR17","doi-asserted-by":"publisher","first-page":"173","DOI":"10.1007\/978-3-642-35527-1_15","volume-title":"Proc Advanced Data Mining and Applications","author":"C Wolley","year":"2012","unstructured":"Wolley C, Quafafou M: Learning from Multiple Naive Annotators. Proc Advanced Data Mining and Applications. 2012, 173-185."},{"key":"6049_CR18","doi-asserted-by":"publisher","first-page":"595","DOI":"10.1007\/978-3-642-37453-1_49","volume-title":"Proc Pacific-Asia Conference on Knowledge Discovery and Data Mining","author":"H Xiao","year":"2013","unstructured":"Xiao H, Xiao H, Eckert C: Learning from Multiple Observers with Unknown Expertise. Proc Pacific-Asia Conference on Knowledge Discovery and Data Mining. 2013, 595-606."},{"issue":"5","key":"6049_CR19","doi-asserted-by":"publisher","first-page":"e1000391","DOI":"10.1371\/journal.pcbi.1000391","volume":"5","author":"A Rzhetsky","year":"2009","unstructured":"Rzhetsky A, Shatkay H, Wilbur WJ: How to get the most out of your curation effort. PLoS Comput. Biol. 2009, 5 (5): e1000391-10.1371\/journal.pcbi.1000391.","journal-title":"PLoS Comput. Biol"},{"issue":"S3","key":"6049_CR20","doi-asserted-by":"publisher","first-page":"S5","DOI":"10.1186\/1471-2105-12-S3-S5","volume":"12","author":"WJ Wilbur","year":"2011","unstructured":"Wilbur WJ, Kim W: Improving a gold standard: treating human relevance judgments of MEDLINE document pairs. BMC Bioinformatics. 2011, 12 (S3): S5-","journal-title":"BMC Bioinformatics"},{"key":"6049_CR21","doi-asserted-by":"publisher","first-page":"633","DOI":"10.1142\/S0218213009000330","volume":"18","author":"SR Cholleti","year":"2009","unstructured":"Cholleti SR, Goldman SA, Blum A, Politte DG, Don S, Smith K, Prior F: Veritas: combining expert opinions without labeled data. International Journal on Artificial Intelligence Tools. 2009, 18: 633-651. 10.1142\/S0218213009000330.","journal-title":"International Journal on Artificial Intelligence Tools"},{"issue":"5","key":"6049_CR22","doi-asserted-by":"publisher","first-page":"e37245","DOI":"10.1371\/journal.pone.0037245","volume":"7","author":"S Mavandadi","year":"2012","unstructured":"Mavandadi S, Dimitrov S, Feng S, Yu F, Sikora U, Yaglidere O, Padmanabhan S, Nielsen K, Ozcan A: Distributed medical image analysis and diagnosis through crowd-sourced games: a malaria case study. PLoS ONE. 2012, 7 (5): e37245-10.1371\/journal.pone.0037245.","journal-title":"PLoS ONE"},{"issue":"1","key":"6049_CR23","doi-asserted-by":"publisher","first-page":"25-34","DOI":"10.1145\/2408736.2408741","volume":"14","author":"XS Zhou","year":"2012","unstructured":"Zhou XS, Zhan Y, Raykar VC, Hermosillo G, Bogoni L, Peng Z: Mining anatomical, physiological and pathological information from medical images. ACM SIGKDD Explorations. 2012, 14 (1): 25-34-","journal-title":"ACM SIGKDD Explorations"},{"key":"6049_CR24","volume-title":"Proc SPIE Medical Imaging: Computer-Aided Diagnosis","author":"L Raghupathi","year":"2013","unstructured":"Raghupathi L, Devarakota PR, Wolf M: Learning-based image preprocessing for robust computer-aided detection. Proc SPIE Medical Imaging: Computer-Aided Diagnosis. 2013"},{"key":"6049_CR25","first-page":"921","volume-title":"Proc AMIA Annu Symp","author":"H Valizadegan","year":"2012","unstructured":"Valizadegan H, Nguyen Q, Hauskrecht M: Learning Medical Diagnosis Models from Multiple Experts. Proc AMIA Annu Symp. 2012, 921-930."},{"issue":"11","key":"6049_CR26","doi-asserted-by":"publisher","first-page":"1344","DOI":"10.1093\/bioinformatics\/btn195","volume":"24","author":"T Ishida","year":"2008","unstructured":"Ishida T, Kinoshita K: Prediction of disordered regions in proteins based on the meta approach. Bioinformatics. 2008, 24 (11): 1344-1348. 10.1093\/bioinformatics\/btn195.","journal-title":"Bioinformatics"},{"issue":"2","key":"6049_CR27","doi-asserted-by":"publisher","first-page":"e4433","DOI":"10.1371\/journal.pone.0004433","volume":"4","author":"A Schlessinger","year":"2009","unstructured":"Schlessinger A, Punta M, Yachdav G, Kajan L, Rost B: Improved disorder prediction by combination of orthogonal approaches. PLoS ONE. 2009, 4 (2): e4433-10.1371\/journal.pone.0004433.","journal-title":"PLoS ONE"},{"issue":"4","key":"6049_CR28","doi-asserted-by":"publisher","first-page":"996","DOI":"10.1016\/j.bbapap.2010.01.011","volume":"1804","author":"B Xue","year":"2010","unstructured":"Xue B, Dunbrack RL, Williams RW, Dunker AK, Uversky VN: PONDR-FIT: a meta-predictor of intrinsically disordered amino acids. Biochim Biophys Acta. 2010, 1804 (4): 996-1010. 10.1016\/j.bbapap.2010.01.011.","journal-title":"Biochim Biophys Acta"},{"issue":"18","key":"6049_CR29","doi-asserted-by":"publisher","first-page":"i489","DOI":"10.1093\/bioinformatics\/btq373","volume":"26","author":"MJ Mizianty","year":"2010","unstructured":"Mizianty MJ, Stach W, Chen K, Kedarisetti KD, Disfani FM, Kurgan L: Improved sequence-based prediction of disordered regions with multilayer fusion of multiple information sources. Bioinformatics. 2010, 26 (18): i489-i496. 10.1093\/bioinformatics\/btq373.","journal-title":"Bioinformatics"},{"key":"6049_CR30","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1186\/1471-2105-13-111","volume":"13","author":"LP Kozlowski","year":"2012","unstructured":"Kozlowski LP, Bujnicki JM: MetaDisorder: a meta-server for the prediction of intrinsic disorder in proteins. BMC Bioinformatics. 2012, 13: 111-10.1186\/1471-2105-13-111.","journal-title":"BMC Bioinformatics"},{"key":"6049_CR31","volume-title":"Journal of Biomolecular Structure and Dynamics","author":"X Fan","year":"2013","unstructured":"Fan X, Kurgan L: Accurate prediction of disorder in protein chains with a comprehensive and empirically designed consensus. Journal of Biomolecular Structure and Dynamics. 2013"},{"issue":"S1","key":"6049_CR32","doi-asserted-by":"publisher","first-page":"S12","DOI":"10.1186\/1477-5956-9-S1-S12","volume":"9","author":"P Zhang","year":"2011","unstructured":"Zhang P, Obradovic Z: Unsupervised Integration of Multiple Protein Disorder Predictors: The Method and Evaluation on CASP7, CASP8 and CASP9 Data. Proteome Science. 2011, 9 (S1): S12-","journal-title":"Proteome Science"},{"key":"6049_CR33","first-page":"1","volume-title":"Bioinformatics and Biomedicine (BIBM), 2012 IEEE International Conference on: 4-7 October 2012","author":"P Zhang","year":"2012","unstructured":"Zhang P, Obradovic Z: Integration of multiple annotators by aggregating experts and filtering novices. Bioinformatics and Biomedicine (BIBM), 2012 IEEE International Conference on: 4-7 October 2012. 2012, 1-6. 10.1109\/BIBM.2012.6392657."},{"key":"6049_CR34","unstructured":"CASP9 Experiment. [http:\/\/predictioncenter.org\/casp9\/]"},{"issue":"1","key":"6049_CR35","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1142\/S0219720005000886","volume":"3","author":"K Peng","year":"2005","unstructured":"Peng K, Vucetic S, Radivojac P, Brown CJ, Dunker AK, Obradovic Z: Optimizing long intrinsic disorder predictors with protein evolutionary information. J Bioinform Comput Biol. 2005, 3 (1): 35-60. 10.1142\/S0219720005000886.","journal-title":"J Bioinform Comput Biol"},{"issue":"S10","key":"6049_CR36","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1002\/prot.23161","volume":"79","author":"B Monastyrskyy","year":"2011","unstructured":"Monastyrskyy B, Fidelis K, Moult J, Tramontano A, Kryshtafovych A: Evaluation of disorder predictions in CASP9. Proteins. 2011, 79 (S10): 107-118. 10.1002\/prot.23161.","journal-title":"Proteins"}],"container-title":["BMC Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1186\/1471-2105-14-S12-S5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1186\/1471-2105-14-S12-S5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/1471-2105-14-S12-S5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,23]],"date-time":"2023-09-23T21:14:13Z","timestamp":1695503653000},"score":1,"resource":{"primary":{"URL":"https:\/\/bmcbioinformatics.biomedcentral.com\/articles\/10.1186\/1471-2105-14-S12-S5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,9]]},"references-count":36,"journal-issue":{"issue":"S12","published-print":{"date-parts":[[2013,9]]}},"alternative-id":["6049"],"URL":"https:\/\/doi.org\/10.1186\/1471-2105-14-s12-s5","relation":{},"ISSN":["1471-2105"],"issn-type":[{"value":"1471-2105","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,9]]},"assertion":[{"value":"24 September 2013","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"S5"}}