{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,14]],"date-time":"2026-01-14T20:47:42Z","timestamp":1768423662510,"version":"3.49.0"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2017,10,25]],"date-time":"2017-10-25T00:00:00Z","timestamp":1508889600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2017,10,25]],"date-time":"2017-10-25T00:00:00Z","timestamp":1508889600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["No. LP150100671"],"award-info":[{"award-number":["No. LP150100671"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100000923","name":"Australian Research Council","doi-asserted-by":"publisher","award":["No. FT130100746"],"award-info":[{"award-number":["No. FT130100746"]}],"id":[{"id":"10.13039\/501100000923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2018,4]]},"DOI":"10.1007\/s10994-017-5674-0","type":"journal-article","created":{"date-parts":[[2017,10,25]],"date-time":"2017-10-25T21:11:35Z","timestamp":1508965895000},"page":"675-702","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["Robust Plackett\u2013Luce model for k-ary crowdsourced preferences"],"prefix":"10.1007","volume":"107","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6338-0958","authenticated-orcid":false,"given":"Bo","family":"Han","sequence":"first","affiliation":[]},{"given":"Yuangang","family":"Pan","sequence":"additional","affiliation":[]},{"given":"Ivor W.","family":"Tsang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,10,25]]},"reference":[{"key":"5674_CR1","doi-asserted-by":"crossref","unstructured":"Alfaro, L., & Shavlovsky, M. (2014). Crowdgrader: A tool for crowdsourcing the evaluation of homework assignments. In ACM technical symposium on computer science education (pp. 415\u2013420). ACM.","DOI":"10.1145\/2538862.2538900"},{"key":"5674_CR2","volume-title":"Pattern recognition and machine learning","author":"CM Bishop","year":"2006","unstructured":"Bishop, C. M. (2006). Pattern recognition and machine learning. Berlin: Springer."},{"key":"5674_CR3","first-page":"324","volume":"39","author":"R Bradley","year":"1952","unstructured":"Bradley, R., & Terry, M. (1952). Rank analysis of incomplete block designs: I. The method of paired comparisons. Biometrika, 39, 324\u2013345.","journal-title":"Biometrika"},{"key":"5674_CR4","doi-asserted-by":"crossref","unstructured":"Chen, X., Bennett, P. N., Collins-Thompson, K., & Horvitz, E. (2013). Pairwise ranking aggregation in a crowdsourced setting. In WSDM (pp. 193\u2013202). ACM.","DOI":"10.1145\/2433396.2433420"},{"issue":"4","key":"5674_CR5","doi-asserted-by":"publisher","first-page":"666","DOI":"10.1109\/TPAMI.2015.2439285","volume":"38","author":"J Deng","year":"2016","unstructured":"Deng, J., Krause, J., Stark, M., & Fei-Fei, L. (2016). Leveraging the wisdom of the crowd for fine-grained recognition. IEEE Transactions on Pattern Analysis and Machine Intelligence, 38(4), 666\u2013676.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"5674_CR6","doi-asserted-by":"crossref","unstructured":"Diaconis, P. (1988). Group representations in probability and statistics. Lecture Notes-Monograph Series (Vol. 11, pp. i\u2013192). Hayward, CA: Institute of Mathematical Statistics.","DOI":"10.1214\/lnms\/1215467407"},{"key":"5674_CR7","doi-asserted-by":"crossref","unstructured":"Dwork, C., Kumar, R., Naor, M., & Sivakumar, D. (2001). Rank aggregation methods for the web. In WWW (pp. 613\u2013622). ACM.","DOI":"10.1145\/371920.372165"},{"issue":"4","key":"5674_CR8","doi-asserted-by":"publisher","first-page":"482","DOI":"10.1187\/cbe.10-03-0017","volume":"9","author":"S Freeman","year":"2010","unstructured":"Freeman, S., & Parks, J. W. (2010). How accurate is peer grading? CBE-Life Sciences Education, 9(4), 482\u2013488.","journal-title":"CBE-Life Sciences Education"},{"key":"5674_CR9","doi-asserted-by":"crossref","unstructured":"Guiver, J., & Snelson, E. (2009). Bayesian inference for Plackett\u2013Luce ranking models. In ICML (pp. 377\u2013384). ACM.","DOI":"10.1145\/1553374.1553423"},{"key":"5674_CR10","doi-asserted-by":"crossref","unstructured":"Kazai, G., Kamps, J., Koolen, M., & Milic-Frayling, N. (2011). Crowdsourcing for book search evaluation: Impact of hit design on comparative system ranking. In SIGIR conference on research and development in information retrieval (pp. 205\u2013214). ACM.","DOI":"10.1145\/2009916.2009947"},{"key":"5674_CR11","unstructured":"Kendall, M. G. (1948). Rank correlation methods. London: Charles Griffin & Company Limited."},{"issue":"4","key":"5674_CR12","doi-asserted-by":"publisher","first-page":"573","DOI":"10.1093\/bioinformatics\/btr709","volume":"28","author":"R Kolde","year":"2012","unstructured":"Kolde, R., Laur, S., Adler, P., & Vilo, J. (2012). Robust rank aggregation for gene list integration and meta-analysis. Bioinformatics, 28(4), 573\u2013580.","journal-title":"Bioinformatics"},{"key":"5674_CR13","doi-asserted-by":"crossref","unstructured":"Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K., Kravitz, J., Chen, S., Kalantidis, Y., Li, L. J., Shamma, D. A., Bernstein, M. S., & Fei-Fei, L. (2017). Visual genome: Connecting language and vision using crowdsourced dense image annotations. International Journal of Computer Vision, 123, 1\u201342.","DOI":"10.1007\/s11263-016-0981-7"},{"key":"5674_CR14","unstructured":"Kulkarni, C., Wei, K., Le, H., Chia, D., Papadopoulos, K., Cheng, J., et al. (2015). Peer and self assessment in massive online classes. In Design thinking research (pp. 131\u2013168). Berlin: Springer."},{"issue":"3","key":"5674_CR15","doi-asserted-by":"publisher","first-page":"225","DOI":"10.1561\/1500000016","volume":"3","author":"TY Liu","year":"2009","unstructured":"Liu, T. Y. (2009). Learning to rank for information retrieval. Foundations and Trends Information Retrieval, 3(3), 225\u2013331.","journal-title":"Foundations and Trends Information Retrieval"},{"key":"5674_CR16","unstructured":"Lu, T., & Boutilier, C. (2011). Learning Mallows models with pairwise preferences. In ICML (pp. 145\u2013152)."},{"issue":"1","key":"5674_CR17","first-page":"3783","volume":"15","author":"T Lu","year":"2014","unstructured":"Lu, T., & Boutilier, C. (2014). Effective sampling and learning for Mallows models with pairwise-preference data. Journal of Machine Learning Research, 15(1), 3783\u20133829.","journal-title":"Journal of Machine Learning Research"},{"key":"5674_CR18","doi-asserted-by":"publisher","first-page":"322","DOI":"10.1016\/j.knosys.2015.05.019","volume":"85","author":"O Luaces","year":"2015","unstructured":"Luaces, O., D\u00edez, J., Alonso-Betanzos, A., Troncoso, A., & Bahamonde, A. (2015). A factorization approach to evaluate open-response assignments in MOOCs using preference learning on peer assessments. Knowledge-Based Systems, 85, 322\u2013328.","journal-title":"Knowledge-Based Systems"},{"key":"5674_CR19","volume-title":"Individual choice behavior: A theoretical analysis","author":"RD Luce","year":"1959","unstructured":"Luce, R. D. (1959). Individual choice behavior: A theoretical analysis. New York: Wiley."},{"key":"5674_CR20","doi-asserted-by":"crossref","unstructured":"Lv, Y., Moon, T., Kolari, P., Zheng, Z., Wang, X., & Chang, Y. (2011). Learning to model relatedness for news recommendation. In WWW (pp. 57\u201366). ACM.","DOI":"10.1145\/1963405.1963417"},{"issue":"1\/2","key":"5674_CR21","doi-asserted-by":"publisher","first-page":"114","DOI":"10.2307\/2333244","volume":"44","author":"CL Mallows","year":"1957","unstructured":"Mallows, C. L. (1957). Non-null ranking models. I. Biometrika, 44(1\/2), 114\u2013130.","journal-title":"Biometrika"},{"issue":"3","key":"5674_CR22","doi-asserted-by":"publisher","first-page":"325","DOI":"10.1007\/BF02294299","volume":"64","author":"A Maydeu-Olivares","year":"1999","unstructured":"Maydeu-Olivares, A. (1999). Thurstonian modeling of ranking data via mean and covariance structure analysis. Psychometrika, 64(3), 325\u2013340.","journal-title":"Psychometrika"},{"key":"5674_CR23","doi-asserted-by":"publisher","unstructured":"Mollica, C., & Tardella, L. (2017). Bayesian Plackett\u2013Luce mixture models for partially ranked data. Psychometrika, 82(2):442\u2013458. doi: 10.1007\/s11336-016-9530-0 .","DOI":"10.1007\/s11336-016-9530-0"},{"key":"5674_CR24","unstructured":"Ok, J., Oh, S., Shin, J., & Yi, Y. (2016). Optimality of belief propagation for crowdsourced classification. In ICML (pp. 535\u2013544)."},{"key":"5674_CR25","doi-asserted-by":"crossref","unstructured":"Plackett, R. L. (1975). The analysis of permutations. Applied Statistics, 24, 193\u2013202.","DOI":"10.2307\/2346567"},{"key":"5674_CR26","doi-asserted-by":"crossref","unstructured":"Prpi\u0107, J., Melton, J., Taeihagh, A., & Anderson, T. (2015). MOOCs and crowdsourcing: Massive courses and massive resources.","DOI":"10.5210\/fm.v20i12.6143"},{"key":"5674_CR27","doi-asserted-by":"crossref","unstructured":"Raman, K., & Joachims, T. (2014). Methods for ordinal peer grading. In KDD (pp. 1037\u20131046). ACM.","DOI":"10.1145\/2623330.2623654"},{"key":"5674_CR28","doi-asserted-by":"crossref","unstructured":"Schalekamp, F., & Zuylen, A. (2009). Rank aggregation: Together we\u2019re strong. In Workshop on algorithm engineering and experiments (pp. 38\u201351). Society for Industrial and Applied Mathematics.","DOI":"10.1137\/1.9781611972894.4"},{"key":"5674_CR29","unstructured":"Shah, N., Bradley, J., Parekh, A., Wainwright, M., & Ramchandran, K. (2013). A case for ordinal peer-evaluation in MOOCs. In NIPS workshop on data driven education."},{"key":"5674_CR30","doi-asserted-by":"crossref","unstructured":"Sheng, V., Provost, F., & Ipeirotis, P. (2008). Get another label? Improving data quality and data mining using multiple, noisy labelers. In KDD (pp. 614\u2013622). ACM.","DOI":"10.1145\/1401890.1401965"},{"key":"5674_CR31","unstructured":"Snow, R., O\u2019Connor, B., Jurafsky, D., & Ng, A. Y. (2008). Cheap and fast\u2014but is it good? Evaluating non-expert annotations for natural language tasks. In EMNLP (pp. 254\u2013263). Association for Computational Linguistics."},{"issue":"4","key":"5674_CR32","doi-asserted-by":"publisher","first-page":"384","DOI":"10.1037\/h0065439","volume":"21","author":"L Thurstone","year":"1927","unstructured":"Thurstone, L. (1927). The method of paired comparisons for social values. Journal of Abnormal and Social Psychology, 21(4), 384\u2013400.","journal-title":"Journal of Abnormal and Social Psychology"},{"key":"5674_CR33","doi-asserted-by":"crossref","unstructured":"Venanzi, M., Guiver, J., Kazai, G., Kohli, P., & Shokouhi, M. (2014). Community-based Bayesian aggregation models for crowdsourcing. In WWW (pp. 155\u2013164). ACM.","DOI":"10.1145\/2566486.2567989"},{"key":"5674_CR34","doi-asserted-by":"crossref","unstructured":"Volkovs, M., Larochelle, H., & Zemel, R. (2012). Learning to rank by aggregating expert preferences. In CIKM (pp. 843\u2013851). ACM.","DOI":"10.1145\/2396761.2396868"},{"key":"5674_CR35","unstructured":"Vuurens, J., Vries, A., & Eickhoff, C. (2011). How much spam can you take? An analysis of crowdsourcing results to increase accuracy. In SIGIR workshop on crowdsourcing for information retrieval (pp. 21\u201326)."},{"key":"5674_CR36","unstructured":"Wang, Y. S., Matsueda, R., & Erosheva, E. A. (2015). A variational EM method for mixed membership models with multivariate rank data: An analysis of public policy preferences. arXiv preprint arXiv:1512.08731"},{"issue":"Jan","key":"5674_CR37","first-page":"267","volume":"12","author":"RC Weng","year":"2011","unstructured":"Weng, R. C., & Lin, C. J. (2011). A Bayesian approximation method for online ranking. Journal of Machine Learning Research, 12(Jan), 267\u2013300.","journal-title":"Journal of Machine Learning Research"},{"key":"5674_CR38","unstructured":"Yan, L., Dodier, R., Mozer, M., & Wolniewicz, R. (2003). Optimizing classifier performance via an approximation to the Wilcoxon\u2013Mann\u2013Whitney statistic. In ICML (pp. 848\u2013855)."},{"key":"5674_CR39","unstructured":"Zhao, Z., Piech, P., & Xia, L. (2016). Learning mixtures of Plackett\u2013Luce models. In ICML (pp. 2906\u20132914)."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10994-017-5674-0\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5674-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-017-5674-0.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,8,27]],"date-time":"2023-08-27T15:59:38Z","timestamp":1693151978000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10994-017-5674-0"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,10,25]]},"references-count":39,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2018,4]]}},"alternative-id":["5674"],"URL":"https:\/\/doi.org\/10.1007\/s10994-017-5674-0","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,10,25]]},"assertion":[{"value":"31 March 2017","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 September 2017","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 October 2017","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}