{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,2,21]],"date-time":"2025-02-21T07:31:29Z","timestamp":1740123089183,"version":"3.37.3"},"reference-count":48,"publisher":"Springer Science and Business Media LLC","issue":"8-10","license":[{"start":{"date-parts":[[2018,5,11]],"date-time":"2018-05-11T00:00:00Z","timestamp":1525996800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2018,5,11]],"date-time":"2018-05-11T00:00:00Z","timestamp":1525996800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["LP150100671"],"award-info":[{"award-number":["LP150100671"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]},{"name":"ARC Future Fellowship","award":["FT130100746"],"award-info":[{"award-number":["FT130100746"]}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2018,9]]},"DOI":"10.1007\/s10994-018-5716-2","type":"journal-article","created":{"date-parts":[[2018,5,11]],"date-time":"2018-05-11T15:17:37Z","timestamp":1526051857000},"page":"1333-1361","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Stagewise learning for noisy k-ary preferences"],"prefix":"10.1007","volume":"107","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-7950-4900","authenticated-orcid":false,"given":"Yuangang","family":"Pan","sequence":"first","affiliation":[]},{"given":"Bo","family":"Han","sequence":"additional","affiliation":[]},{"given":"Ivor W.","family":"Tsang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2018,5,11]]},"reference":[{"key":"5716_CR1","volume-title":"Pattern recognition and machine learning","author":"CM Bishop","year":"2006","unstructured":"Bishop, C. M. (2006). Pattern recognition and machine learning. Berlin: Springer."},{"key":"5716_CR2","first-page":"324","volume":"39","author":"RA Bradley","year":"1952","unstructured":"Bradley, R. A., & Terry, M. (1952). Rank analysis of incomplete block designs: The method of paired comparisons. Biometrika, 39, 324\u2013345.","journal-title":"Biometrika"},{"key":"5716_CR3","doi-asserted-by":"crossref","unstructured":"Chen, X., Bennett, P., Collins-Thompson, K., & Horvitz, E. (2013). Pairwise ranking aggregation in a crowdsourced setting. In WWW.","DOI":"10.1145\/2433396.2433420"},{"key":"5716_CR4","doi-asserted-by":"crossref","unstructured":"De\u00a0Alfaro, L., & Shavlovsky, M. (2014). Crowdgrader: A tool for crowdsourcing the evaluation of homework assignments. In ACM technical symposium on computer science education.","DOI":"10.1145\/2538862.2538900"},{"key":"5716_CR5","doi-asserted-by":"crossref","unstructured":"Deng, J., Dong, W., Socher, R., Li, L.J., Li, K., & Fei-Fei, L. (2009). Imagenet: A large-scale hierarchical image database. In CVPR (pp. 248\u2013255). IEEE.","DOI":"10.1109\/CVPR.2009.5206848"},{"key":"5716_CR6","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1016\/j.eswa.2015.12.005","volume":"49","author":"MS Desarkar","year":"2016","unstructured":"Desarkar, M. S., Sarkar, S., & Mitra, P. (2016). Preference relations based unsupervised rank aggregation for metasearch. Expert Systems with Applications, 49, 86\u201398.","journal-title":"Expert Systems with Applications"},{"key":"5716_CR7","doi-asserted-by":"crossref","unstructured":"Dwork, C., Kumar, R., Naor, M., & Sivakumar, D. (2001). Rank aggregation methods for the web. In WWW (pp. 613\u2013622). ACM.","DOI":"10.1145\/371920.372165"},{"key":"5716_CR8","volume-title":"The rating of chessplayers, past and present","author":"AE Elo","year":"1978","unstructured":"Elo, A. E. (1978). The rating of chessplayers, past and present. Nagoya: Arco Pub."},{"key":"5716_CR9","doi-asserted-by":"crossref","first-page":"359","DOI":"10.1111\/j.2517-6161.1986.tb01420.x","volume":"48","author":"MA Fligner","year":"1986","unstructured":"Fligner, M. A., & Verducci, J. S. (1986). Distance based ranking models. Journal of the Royal Statistical Society Series B (Methodological), 48, 359\u2013369.","journal-title":"Journal of the Royal Statistical Society Series B (Methodological)"},{"issue":"3","key":"5716_CR10","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1111\/1467-9876.00159","volume":"48","author":"ME Glickman","year":"1999","unstructured":"Glickman, M. E. (1999). Parameter estimation in large dynamic paired comparison experiments. Journal of the Royal Statistical Society: Series C (Applied Statistics), 48(3), 377\u2013394.","journal-title":"Journal of the Royal Statistical Society: Series C (Applied Statistics)"},{"key":"5716_CR11","doi-asserted-by":"crossref","unstructured":"Guiver, J., & Snelson, E. (2009). Bayesian inference for Plackett-Luce ranking models. In ICML. ACM.","DOI":"10.1145\/1553374.1553423"},{"key":"5716_CR12","unstructured":"Herbrich, R., Minka, T., & Graepel, T. (2007). Trueskill$$^{{\\rm TM}}$$: A Bayesian skill rating system. In NIPS (pp. 569\u2013576)."},{"key":"5716_CR13","unstructured":"Jaini, P., Chen, Z., Carbajal, P., Law, E., Middleton, L., Regan, K., Schaekermann, M., Trimponias, G., Tung, J., & Poupart, P. (2016). Online Bayesian transfer learning for sequential data modeling. In ICLR"},{"key":"5716_CR14","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Koolen, M., & Milic-Frayling, N. (2011). Crowdsourcing for book search evaluation: Impact of hit design on comparative system ranking. In SIGIR.","DOI":"10.1145\/2009916.2009947"},{"issue":"1","key":"5716_CR15","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1093\/bib\/bbv021","volume":"17","author":"R Khare","year":"2015","unstructured":"Khare, R., Good, B. M., Leaman, R., Su, A. I., & Lu, Z. (2015). Crowdsourcing in biomedicine: Challenges and opportunities. Briefings in Bioinformatics, 17(1), 23\u201332.","journal-title":"Briefings in Bioinformatics"},{"issue":"193","key":"5716_CR16","first-page":"1","volume":"17","author":"A Khetan","year":"2016","unstructured":"Khetan, A., & Oh, S. (2016). Data-driven rank breaking for efficient rank aggregation. Journal of Machine Learning Research, 17(193), 1\u201354.","journal-title":"Journal of Machine Learning Research"},{"issue":"4","key":"5716_CR17","doi-asserted-by":"publisher","first-page":"340","DOI":"10.1093\/ejo\/cji042","volume":"27","author":"H Knight","year":"2005","unstructured":"Knight, H., & Keith, O. (2005). Ranking facial attractiveness. The European Journal of Orthodontics, 27(4), 340\u2013348.","journal-title":"The European Journal of Orthodontics"},{"issue":"6","key":"5716_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/2505057","volume":"20","author":"C Kulkarni","year":"2013","unstructured":"Kulkarni, C., Wei, K. P., Le, H., Chia, D., Papadopoulos, K., Cheng, J., et al. (2013). Peer and self assessment in massive online classes. ACM Transactions on Computer-Human Interaction, 20(6), 1\u201331.","journal-title":"ACM Transactions on Computer-Human Interaction"},{"key":"5716_CR19","doi-asserted-by":"publisher","DOI":"10.1093\/acprof:oso\/9780198273479.001.0001","volume-title":"Electoral systems and party systems: A study of twenty-seven democracies, 1945\u20131990","author":"A Lijphart","year":"1994","unstructured":"Lijphart, A. (1994). Electoral systems and party systems: A study of twenty-seven democracies, 1945\u20131990. Oxford: Oxford University Press."},{"key":"5716_CR20","unstructured":"Liu, Q., Peng, J., & Ihler, A. T. (2012). Variational inference for crowdsourcing. In NIPS (pp. 692\u2013700)."},{"key":"5716_CR21","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1561\/1500000016","volume":"3","author":"TY Liu","year":"2009","unstructured":"Liu, T. Y. (2009). Learning to rank for information retrieval. Foundations and Trends\u00ae in Information Retrieval, 3, 225\u2013331.","journal-title":"Foundations and Trends\u00ae in Information Retrieval"},{"key":"5716_CR22","doi-asserted-by":"publisher","first-page":"322","DOI":"10.1016\/j.knosys.2015.05.019","volume":"85","author":"O Luaces","year":"2015","unstructured":"Luaces, O., D\u00edez, J., Alonso-Betanzos, A., Troncoso, A., & Bahamonde, A. (2015). A factorization approach to evaluate open-response assignments in moocs using preference learning on peer assessments. Knowledge-Based Systems, 85, 322\u2013328.","journal-title":"Knowledge-Based Systems"},{"key":"5716_CR23","volume-title":"Individual choice behavior: A theoretical analysis","author":"RD Luce","year":"1959","unstructured":"Luce, R. D. (1959). Individual choice behavior: A theoretical analysis. New York: Wiley."},{"key":"5716_CR24","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1093\/biomet\/44.1-2.114","volume":"44","author":"C Mallows","year":"1957","unstructured":"Mallows, C. (1957). Non-null ranking models. Biometrika, 44, 114\u2013130.","journal-title":"Biometrika"},{"key":"5716_CR25","doi-asserted-by":"publisher","first-page":"325","DOI":"10.1007\/BF02294299","volume":"64","author":"A Maydeu-Olivares","year":"1999","unstructured":"Maydeu-Olivares, A. (1999). Thurstonian modeling of ranking data via mean and covariance structure analysis. Psychometrika, 64, 325\u2013340.","journal-title":"Psychometrika"},{"key":"5716_CR26","doi-asserted-by":"publisher","first-page":"442","DOI":"10.1007\/s11336-016-9530-0","volume":"82","author":"C Mollica","year":"2016","unstructured":"Mollica, C., & Tardella, L. (2016). Bayesian Plackett-Luce mixture models for partially ranked data. Psychometrika, 82, 442\u2013458.","journal-title":"Psychometrika"},{"issue":"1","key":"5716_CR27","doi-asserted-by":"publisher","first-page":"266","DOI":"10.1287\/opre.2016.1534","volume":"65","author":"S Negahban","year":"2016","unstructured":"Negahban, S., Oh, S., & Shah, D. (2016). Rank centrality: Ranking from pairwise comparisons. Operations Research, 65(1), 266\u2013287.","journal-title":"Operations Research"},{"key":"5716_CR28","unstructured":"Ok, J., Oh, S., Shin, J., & Yi, Y. (2016). Optimality of belief propagation for crowdsourced classification. In ICML (pp. 535\u2013544)."},{"key":"5716_CR29","doi-asserted-by":"publisher","first-page":"193","DOI":"10.2307\/2346567","volume":"24","author":"R Plackett","year":"1975","unstructured":"Plackett, R. (1975). The analysis of permutations. Applied Statistics, 24, 193\u2013202.","journal-title":"Applied Statistics"},{"key":"5716_CR30","doi-asserted-by":"crossref","unstructured":"Raman, K., & Joachims, T. (2014). Methods for ordinal peer grading. In KDD.","DOI":"10.1145\/2623330.2623654"},{"key":"5716_CR31","unstructured":"Rashwan, A., Zhao, H., & Poupart, P. (2016). Online and distributed Bayesian moment matching for parameter learning in sum-product networks. In AISTATS (pp. 1469\u20131477)."},{"key":"5716_CR32","unstructured":"Richard, B. (2013). Cheap solutions: Managing a co-producing crowd of strangers to solve your problems. Contemporary perspectives on technical innovation, management and policy (pp. 261\u2013287)."},{"issue":"2","key":"5716_CR33","doi-asserted-by":"publisher","first-page":"313","DOI":"10.1006\/jeth.1999.2541","volume":"87","author":"DG Saari","year":"1999","unstructured":"Saari, D. G. (1999). Explaining all three-alternative voting outcomes. Journal of Economic Theory, 87(2), 313\u2013355.","journal-title":"Journal of Economic Theory"},{"key":"5716_CR34","unstructured":"Shah, N., Balakrishnan, S., Bradley, J., Parekh, A., Ramchandran, K., & Wainwright, M. (2015). Estimation from pairwise comparisons: Sharp minimax bounds with topology dependence. In AISTATS."},{"key":"5716_CR35","unstructured":"Shah, N., Bradley, J., Parekh, A., Wainwright, M., & Ramchandran, K. (2013). A case for ordinal peer-evaluation in moocs. In NIPS Workshop on Data Driven Education."},{"key":"5716_CR36","unstructured":"Soufiani, H. A., Chen, W., Parkes, D. C., & Xia, L. (2013). Generalized method-of-moments for rank aggregation. In NIPS (pp. 2706\u20132714)."},{"key":"5716_CR37","unstructured":"Soufiani, H. A., Parkes, D. C., & Xia, L. (2014). Computing parametric ranking models via rank-breaking. In ICML (pp. 360\u2013368)."},{"key":"5716_CR38","doi-asserted-by":"publisher","first-page":"384","DOI":"10.1037\/h0065439","volume":"21","author":"LL Thurstone","year":"1927","unstructured":"Thurstone, L. L. (1927). The method of paired comparisons for social values. Journal of Abnormal and Social Psychology, 21, 384.","journal-title":"Journal of Abnormal and Social Psychology"},{"issue":"18","key":"5716_CR39","doi-asserted-by":"publisher","first-page":"2673","DOI":"10.1016\/j.ins.2005.11.010","volume":"176","author":"E Tsiporkova","year":"2006","unstructured":"Tsiporkova, E., & Boeva, V. (2006). Multi-step ranking of alternatives in a multi-criteria and multi-expert decision making environment. Information Sciences, 176(18), 2673\u20132697.","journal-title":"Information Sciences"},{"issue":"2","key":"5716_CR40","doi-asserted-by":"publisher","first-page":"633","DOI":"10.1534\/genetics.112.139337","volume":"191","author":"TL Turner","year":"2012","unstructured":"Turner, T. L., & Miller, P. M. (2012). Investigating natural variation in drosophila courtship song by the evolve and resequence approach. Genetics, 191(2), 633\u2013642.","journal-title":"Genetics"},{"key":"5716_CR41","unstructured":"Vitelli, V., S\u00f8rensen, \u00d8., Frigessi, A., & Arjas, E. (2014). Probabilistic preference learning with the mallows rank model. \n                    arXiv:1405.7945\n                    \n                  ."},{"key":"5716_CR42","doi-asserted-by":"crossref","unstructured":"Volkovs, M., & Zemel, R. (2012). A flexible generative model for preference aggregation. In WWW.","DOI":"10.1145\/2187836.2187902"},{"key":"5716_CR43","unstructured":"Vuurens, J., de\u00a0Vries, A. P., & Eickhoff, C. (2011). How much spam can you take? An analysis of crowdsourcing results to increase accuracy. In SIGIR Workshop on CIR."},{"key":"5716_CR44","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1561\/2200000001","volume":"1","author":"MJ Wainwright","year":"2008","unstructured":"Wainwright, M. J., & Jordan, M. I. (2008). Graphical models, exponential families, and variational inference. Foundations and Trends\u00ae in Machine Learning, 1, 1\u2013305.","journal-title":"Foundations and Trends\u00ae in Machine Learning"},{"key":"5716_CR45","first-page":"267","volume":"12","author":"R Weng","year":"2011","unstructured":"Weng, R., & Lin, C. J. (2011). A Bayesian approximation method for online ranking. Journal of Machine Learning Research, 12, 267\u2013300.","journal-title":"Journal of Machine Learning Research"},{"issue":"3","key":"5716_CR46","doi-asserted-by":"publisher","first-page":"1087","DOI":"10.1214\/aos\/1176347257","volume":"17","author":"M Woodroofe","year":"1989","unstructured":"Woodroofe, M., et al. (1989). Very weak expansions for sequentially designed experiments: Linear models. The Annals of Statistics, 17(3), 1087\u20131102.","journal-title":"The Annals of Statistics"},{"key":"5716_CR47","unstructured":"Yan, L., Dodier, R. H., Mozer, M., & Wolniewicz, R. H. (2003). Optimizing classifier performance via an approximation to the Wilcoxon\u2013Mann\u2013Whitney statistic. In ICML."},{"key":"5716_CR48","doi-asserted-by":"publisher","first-page":"50","DOI":"10.1007\/978-3-319-60486-2_5","volume-title":"Advances in the human side of service engineering","author":"C Zagel","year":"2018","unstructured":"Zagel, C., Piazza, A., Petrov, Y., & Bodendorf, F. (2018). Sciencomat: A gamified research platform for evaluating visual attractiveness. In L. E. Freund & W. Cellary (Eds.), Advances in the human side of service engineering (pp. 50\u201360). Berlin: Springer."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10994-018-5716-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-018-5716-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-018-5716-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,5,17]],"date-time":"2020-05-17T08:10:07Z","timestamp":1589703007000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10994-018-5716-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,5,11]]},"references-count":48,"journal-issue":{"issue":"8-10","published-print":{"date-parts":[[2018,9]]}},"alternative-id":["5716"],"URL":"https:\/\/doi.org\/10.1007\/s10994-018-5716-2","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"type":"print","value":"0885-6125"},{"type":"electronic","value":"1573-0565"}],"subject":[],"published":{"date-parts":[[2018,5,11]]},"assertion":[{"value":"1 October 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"2 May 2018","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 May 2018","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}