{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,16]],"date-time":"2025-10-16T06:59:43Z","timestamp":1760597983474,"version":"3.37.3"},"reference-count":46,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2020,9,21]],"date-time":"2020-09-21T00:00:00Z","timestamp":1600646400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,9,21]],"date-time":"2020-09-21T00:00:00Z","timestamp":1600646400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"name":"National Key Research and Development Plan of China","award":["2019YFB2102100"],"award-info":[{"award-number":["2019YFB2102100"]}]},{"DOI":"10.13039\/501100006469","name":"FDCT","doi-asserted-by":"crossref","award":["SKL-IOTSC-2018-2020","SKL-IOTSC-2018-2020"],"award-info":[{"award-number":["SKL-IOTSC-2018-2020","SKL-IOTSC-2018-2020"]}],"id":[{"id":"10.13039\/501100006469","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100006469","name":"FDCT","doi-asserted-by":"crossref","award":["SKL-IOTSC-2018-2020"],"award-info":[{"award-number":["SKL-IOTSC-2018-2020"]}],"id":[{"id":"10.13039\/501100006469","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100004733","name":"University of Macau","doi-asserted-by":"crossref","award":["MYRG2019-00119-FST"],"award-info":[{"award-number":["MYRG2019-00119-FST"]}],"id":[{"id":"10.13039\/501100004733","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["The VLDB Journal"],"published-print":{"date-parts":[[2021,3]]},"DOI":"10.1007\/s00778-020-00631-8","type":"journal-article","created":{"date-parts":[[2020,9,21]],"date-time":"2020-09-21T04:12:01Z","timestamp":1600661521000},"page":"189-213","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Crowdsourced top-k queries by pairwise preference judgments with confidence and budget control"],"prefix":"10.1007","volume":"30","author":[{"given":"Yan","family":"Li","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ngai Meng","family":"Kou","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Leong Hou","family":"U","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhiguo","family":"Gong","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2020,9,21]]},"reference":[{"key":"631_CR1","doi-asserted-by":"crossref","unstructured":"Amsterdamer, Y., Grossman, Y., Milo, T., Senellart, P.: Crowd mining. In: SIGMOD (2013)","DOI":"10.1145\/2463676.2465318"},{"issue":"3\/4","key":"631_CR2","doi-asserted-by":"publisher","first-page":"324","DOI":"10.2307\/2334029","volume":"39","author":"RA Bradley","year":"1952","unstructured":"Bradley, R.A., Terry, M.E.: Rank analysis of incomplete block designs: I The method of paired comparisons. Biometrika 39(3\/4), 324\u2013345 (1952)","journal-title":"Biometrika"},{"key":"631_CR3","unstructured":"Busa-Fekete, R., Sz\u00f6r\u00e9nyi, B., Cheng, W., Weng, P., H\u00fcllermeier, E.: Top-k selection based on adaptive sampling of noisy preferences. In: ICML (2013)"},{"key":"631_CR4","doi-asserted-by":"crossref","unstructured":"Chen, X., Bennett, P.N., Collins-Thompson, K., Horvitz, E.: Pairwise ranking aggregation in a crowdsourced setting. In: WSDM (2013)","DOI":"10.1145\/2433396.2433420"},{"key":"631_CR5","doi-asserted-by":"crossref","unstructured":"Chu, W., Ghahramani, Z.: Preference learning with Gaussian processes. In: ICML (2005)","DOI":"10.1145\/1102351.1102369"},{"issue":"1","key":"631_CR6","first-page":"41","volume":"28","author":"E Ciceri","year":"2016","unstructured":"Ciceri, E., Fraternali, P., Martinenghi, D., Tagliasacchi, M.: Crowdsourcing for top-k query processing over uncertain data. TKDE 28(1), 41\u201353 (2016)","journal-title":"TKDE"},{"key":"631_CR7","doi-asserted-by":"crossref","unstructured":"Davidson, S.B., Khanna, S., Milo, T., Roy, S.: Using the crowd for top-k and group-by queries. In: ICDT (2013)","DOI":"10.1145\/2448496.2448524"},{"issue":"4","key":"631_CR8","doi-asserted-by":"publisher","first-page":"35:1","DOI":"10.1145\/2684066","volume":"39","author":"SB Davidson","year":"2014","unstructured":"Davidson, S.B., Khanna, S., Milo, T., Roy, S.: Top-k and clustering with noisy comparisons. TODS 39(4), 35:1\u201335:39 (2014)","journal-title":"TODS"},{"key":"631_CR9","doi-asserted-by":"crossref","unstructured":"de\u00a0Alfaro, L., Polychronopoulos, V., Polyzotis, N.: Efficient techniques for crowdsourced top-k lists. In: IJCAI (2017)","DOI":"10.24963\/ijcai.2017\/670"},{"key":"631_CR10","doi-asserted-by":"crossref","unstructured":"Diaconis, P., Graham, R.L.: Spearman\u2019s footrule as a measure of disarray. JRSS: Series B pp. 262\u2013268 (1977)","DOI":"10.1111\/j.2517-6161.1977.tb01624.x"},{"key":"631_CR11","doi-asserted-by":"crossref","unstructured":"Dong, J., Yang, K., Shi, Y.: Ranking from crowdsourced pairwise comparisons via smoothed matrix manifold optimization. In: ICDM workshops (2017)","DOI":"10.1109\/ICDMW.2017.130"},{"key":"631_CR12","doi-asserted-by":"crossref","unstructured":"Dushkin, E., Milo, T.: Top-k sorting under partial order information. In: SIGMOD (2018)","DOI":"10.1145\/3183713.3199672"},{"key":"631_CR13","doi-asserted-by":"crossref","unstructured":"Dwork, C., Kumar, R., Naor, M., Sivakumar, D.: Rank aggregation methods for the web. In: WWW (2001)","DOI":"10.1145\/371920.372165"},{"key":"631_CR14","doi-asserted-by":"publisher","DOI":"10.1201\/9780429246593","volume-title":"An Introduction to the Bootstrap","author":"B Efron","year":"1994","unstructured":"Efron, B., Tibshirani, R.J.: An Introduction to the Bootstrap. CRC Press, Boca Raton (1994)"},{"key":"631_CR15","volume-title":"Sequential Estimation","author":"M Ghosh","year":"2011","unstructured":"Ghosh, M., Mukhopadhyay, N., Sen, P.K.: Sequential Estimation, 1st edn. Wiley, Hoboken (2011)","edition":"1"},{"issue":"2","key":"631_CR16","doi-asserted-by":"publisher","first-page":"133","DOI":"10.1023\/A:1011419012209","volume":"4","author":"KY Goldberg","year":"2001","unstructured":"Goldberg, K.Y., Roeder, T., Gupta, D., Perkins, C.: Eigentaste: a constant time collaborative filtering algorithm. Inf. Retr. 4(2), 133\u2013151 (2001)","journal-title":"Inf. Retr."},{"issue":"3","key":"631_CR17","doi-asserted-by":"publisher","first-page":"e80","DOI":"10.2196\/jmir.3962","volume":"17","author":"A Gottlieb","year":"2015","unstructured":"Gottlieb, A., Hoehndorf, R., Dumontier, M., Altman, R.B.: Ranking adverse drug reactions with crowdsourcing. J. Med. Internet Res. 17(3), e80 (2015)","journal-title":"J. Med. Internet Res."},{"key":"631_CR18","doi-asserted-by":"crossref","unstructured":"Guo, S., Parameswaran, A.G., Garcia-Molina, H.: So who won?. In: SIGMOD, Dynamic Max Discovery with the Crowd (2012)","DOI":"10.1145\/2213836.2213880"},{"issue":"7","key":"631_CR19","first-page":"321","volume":"4","author":"CAR Hoare","year":"1961","unstructured":"Hoare, C.A.R.: Algorithm 65: Find. Commun. ACM 4(7), 321\u2013322 (1961)","journal-title":"Commun. ACM"},{"key":"631_CR20","volume-title":"Probability and Statistical Inference","author":"R Hogg","year":"2013","unstructured":"Hogg, R., Tanis, E., Zimmerman, D.: Probability and Statistical Inference, 9th edn. Pearson, London (2013)","edition":"9"},{"issue":"4","key":"631_CR21","first-page":"577","volume":"88","author":"JG Kemeny","year":"1959","unstructured":"Kemeny, J.G.: Mathematics without numbers. Daedalus 88(4), 577\u2013591 (1959)","journal-title":"Daedalus"},{"key":"631_CR22","unstructured":"Khan, A.R., Garc\u00eda-Molina, H.: Hybrid strategies for finding the max with the crowd. Tech. Rep. 1090, Stanford InfoLab, Stanford University (2014)"},{"key":"631_CR23","doi-asserted-by":"crossref","unstructured":"Kou, N.M., Li, Y., Wang, H., U, L.H., Gong, Z.: Crowdsourced top-k queries by confidence-aware pairwise judgments. In: SIGMOD (2017)","DOI":"10.1145\/3035918.3035953"},{"key":"631_CR24","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1016\/S0065-2458(08)60467-2","volume":"23","author":"S Lakshmivarahan","year":"1984","unstructured":"Lakshmivarahan, S., Dhall, S.K., Miller, L.L.: Parallel sorting algorithms. Adv. Comput. 23, 295\u2013354 (1984)","journal-title":"Adv. Comput."},{"key":"631_CR25","doi-asserted-by":"publisher","first-page":"98","DOI":"10.1016\/j.ins.2017.03.010","volume":"399","author":"J Lee","year":"2017","unstructured":"Lee, J., Lee, D., Hwang, S.: CrowdK: answering top-k queries with crowdsourcing. Inf. Sci. 399, 98\u2013120 (2017)","journal-title":"Inf. Sci."},{"key":"631_CR26","doi-asserted-by":"crossref","unstructured":"Li, K., Zhang, X., Li, G.: A rating-ranking method for crowdsourced top-k computation. In: SIGMOD (2018)","DOI":"10.1145\/3183713.3183762"},{"issue":"9","key":"631_CR27","first-page":"2296","volume":"28","author":"G Li","year":"2016","unstructured":"Li, G., Wang, J., Zheng, Y., Franklin, M.J.: Crowdsourced data management: a survey. TKDE 28(9), 2296\u20132319 (2016)","journal-title":"TKDE"},{"issue":"12","key":"631_CR28","first-page":"1909","volume":"10","author":"Y Li","year":"2017","unstructured":"Li, Y., Kou, N.M., Wang, H., U, L.H., Gong, Z.: A confidence-aware top-k query processing toolkit on crowdsourcing. PVLDB 10(12), 1909\u20131912 (2017)","journal-title":"PVLDB"},{"issue":"10","key":"631_CR29","first-page":"2290","volume":"29","author":"X Lin","year":"2017","unstructured":"Lin, X., Xu, J., Hu, H., Fan, Z.: Reducing uncertainty of probabilistic top-k ranking via pairwise crowdsourcing. TKDE 29(10), 2290\u20132303 (2017)","journal-title":"TKDE"},{"key":"631_CR30","volume-title":"Individual Choice Behavior: A Theoretical Analysis","author":"RD Luce","year":"1959","unstructured":"Luce, R.D.: Individual Choice Behavior: A Theoretical Analysis. Wiley, Hoboken (1959)"},{"issue":"1","key":"631_CR31","first-page":"13","volume":"5","author":"A Marcus","year":"2011","unstructured":"Marcus, A., Wu, E., Karger, D.R., Madden, S., Miller, R.C.: Human-powered sorts and joins. PVLDB 5(1), 13\u201324 (2011)","journal-title":"PVLDB"},{"key":"631_CR32","doi-asserted-by":"crossref","unstructured":"Matsui, T., Baba, Y., Kamishima, T., Kashima, H.: Crowdordering. In: PAKDD (2014)","DOI":"10.1007\/978-3-319-06605-9_28"},{"key":"631_CR33","doi-asserted-by":"crossref","unstructured":"Mohajer, S., Suh, C., Elmahdy, A.: Active learning for top-k rank aggregation from noisy comparisons. In: ICML (2017)","DOI":"10.1109\/ALLERTON.2016.7852326"},{"key":"631_CR34","unstructured":"Page, L., Brin, S., Motwani, R., Winograd, T.: The PageRank citation ranking: bringing order to the web. Tech. Rep. 422, Stanford InfoLab, Stanford University (1999)"},{"key":"631_CR35","unstructured":"Polychronopoulos, V., de\u00a0Alfaro, L., Davis, J., Garcia-Molina, H., Polyzotis, N.: Human-powered top-k lists. In: WebDB (2013)"},{"key":"631_CR36","doi-asserted-by":"crossref","unstructured":"Rajpal, S., Parameswaran, A.: Holistic crowd-powered sorting via AID: Optimizing for accuracies, inconsistencies, and difficulties. In: CIKM (2018)","DOI":"10.1145\/3269206.3269279"},{"issue":"1","key":"631_CR37","first-page":"2","volume":"13","author":"J Snyder","year":"2005","unstructured":"Snyder, J.: Estimating the distribution of voter preferences using partially aggregated voting data. Polit. Methodol. 13(1), 2\u20135 (2005)","journal-title":"Polit. Methodol."},{"issue":"3","key":"631_CR38","doi-asserted-by":"publisher","first-page":"243","DOI":"10.1214\/aoms\/1177731088","volume":"16","author":"C Stein","year":"1945","unstructured":"Stein, C.: A two-sample test for a linear hypothesis whose power is independent of the variance. Ann. Math. Stat. 16(3), 243\u2013258 (1945)","journal-title":"Ann. Math. Stat."},{"key":"631_CR39","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1037\/h0070288","volume":"34","author":"LL Thurstone","year":"1927","unstructured":"Thurstone, L.L.: A law of comparative judgement. Psychol. Rev. 34, 273\u2013286 (1927)","journal-title":"Psychol. Rev."},{"key":"631_CR40","doi-asserted-by":"crossref","unstructured":"Venetis, P., Garcia-Molina, H., Huang, K., Polyzotis, N.: Max algorithms in crowdsourcing environments. In: WWW (2012)","DOI":"10.1145\/2187836.2187969"},{"issue":"5","key":"631_CR41","doi-asserted-by":"publisher","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees, E.M.: Variations in relevance judgments and the measurement of retrieval effectiveness. Inf. Process. Manag. 36(5), 697\u2013716 (2000)","journal-title":"Inf. Process. Manag."},{"key":"631_CR42","doi-asserted-by":"crossref","unstructured":"Wang, J., Li, G., Kraska, T., Franklin, M.J., Feng, J.: Leveraging transitive relations for crowdsourced joins. In: SIGMOD (2013)","DOI":"10.1145\/2463676.2465280"},{"key":"631_CR43","doi-asserted-by":"crossref","unstructured":"Xu, Q., Xiong, J., Sun, X., Yang, Z., Cao, X., Huang, Q., Yao, Y.: A margin-based MLE for crowdsourced partial ranking. In: Multimedia (2018)","DOI":"10.1145\/3240508.3240597"},{"key":"631_CR44","unstructured":"Ye, P., Doermann, D.: Combining preference and absolute judgements in a crowd-sourced setting. In: Machine Learning Meets Crowdsourcing (2013)"},{"issue":"8","key":"631_CR45","first-page":"612","volume":"9","author":"X Zhang","year":"2016","unstructured":"Zhang, X., Li, G., Feng, J.: Crowdsourced top-k algorithms: an experimental evaluation. PVLDB 9(8), 612\u2013623 (2016)","journal-title":"PVLDB"},{"key":"631_CR46","doi-asserted-by":"crossref","unstructured":"Ziegler, C.N., McNee, S.M., Konstan, J.A., Lausen, G.: Improving recommendation lists through topic diversification. In: WWW (2005)","DOI":"10.1145\/1060745.1060754"}],"container-title":["The VLDB Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-020-00631-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s00778-020-00631-8\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s00778-020-00631-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,9,21]],"date-time":"2021-09-21T00:10:39Z","timestamp":1632183039000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s00778-020-00631-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,9,21]]},"references-count":46,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2021,3]]}},"alternative-id":["631"],"URL":"https:\/\/doi.org\/10.1007\/s00778-020-00631-8","relation":{},"ISSN":["1066-8888","0949-877X"],"issn-type":[{"type":"print","value":"1066-8888"},{"type":"electronic","value":"0949-877X"}],"subject":[],"published":{"date-parts":[[2020,9,21]]},"assertion":[{"value":"15 November 2019","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 June 2020","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"26 August 2020","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 September 2020","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}