{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,20]],"date-time":"2026-10-20T22:55:53Z","timestamp":1792536953818,"version":"4.5.0"},"reference-count":73,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T00:00:00Z","timestamp":1611705600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T00:00:00Z","timestamp":1611705600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Inf Retrieval J"],"published-print":{"date-parts":[[2021,4]]},"DOI":"10.1007\/s10791-020-09386-w","type":"journal-article","created":{"date-parts":[[2021,1,27]],"date-time":"2021-01-27T06:02:52Z","timestamp":1611727372000},"page":"85-113","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":40,"title":["Evaluation metrics for measuring bias in search engine results"],"prefix":"10.1007","volume":"24","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9782-5751","authenticated-orcid":false,"given":"Gizem","family":"Gezici","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aldo","family":"Lipani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yucel","family":"Saygin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,1,27]]},"reference":[{"key":"9386_CR1","unstructured":"(2018). Internetlivestats. http:\/\/www.internetlivestats.com\/. Retrieved 2018-10-06."},{"key":"9386_CR2","unstructured":"(2018). Procon.org, procon.org - pros and cons of controversial issues. https:\/\/www.procon.org\/. Retrieved 2018-07-31."},{"key":"9386_CR3","unstructured":"(2018). Search engine statistics 2018. https:\/\/www.smartinsights.com\/search-engine-marketing\/search-engine-statistics\/. Retrieved 2018-10-06."},{"key":"9386_CR4","unstructured":"99Firms (2019). Search engine statistics. https:\/\/99firms.com\/blog\/search-engine-statistics\/#gref. Retrieved 2019-09-06."},{"key":"9386_CR5","doi-asserted-by":"crossref","unstructured":"Aktolga, E., & Allan, J., (2013). Sentiment diversification with different biases. Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval (pp. 593\u2013602), ACM.","DOI":"10.1145\/2484028.2484060"},{"key":"9386_CR6","doi-asserted-by":"crossref","unstructured":"Alam, M. A., & Downey, D. (2014). Analyzing the content emphasis of web search engines. In Proceedings of the 37th international ACM SIGIR conference on Research and development in information retrieval (pp. 1083\u20131086), ACM.","DOI":"10.1145\/2600428.2609515"},{"key":"9386_CR7","doi-asserted-by":"publisher","first-page":"1053","DOI":"10.1016\/j.ipm.2012.01.004","volume":"48","author":"O Alonso","year":"2012","unstructured":"Alonso, O., & Mizzaro, S. (2012). Using crowdsourcing for trec relevance assessment. Information Processing & Management, 48, 1053\u20131066.","journal-title":"Information Processing & Management"},{"key":"9386_CR8","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1145\/1480506.1480508","volume":"42","author":"O Alonso","year":"2008","unstructured":"Alonso, O., Rose, D. E., & Stewart, B. (2008). Crowdsourcing for relevance evaluation. SIGIR Forum, 42, 9\u201315.","journal-title":"SIGIR Forum"},{"key":"9386_CR9","doi-asserted-by":"crossref","unstructured":"Baeza-Yates, R. (2016). Data and algorithmic bias in the web. Proceedings of the 8th ACM Conference on Web Science (pp. 1\u20131), ACM.","DOI":"10.1145\/2908131.2908135"},{"key":"9386_CR10","doi-asserted-by":"publisher","first-page":"1130","DOI":"10.1126\/science.aaa1160","volume":"348","author":"E Bakshy","year":"2015","unstructured":"Bakshy, E., Messing, S., & Adamic, L. A. (2015). Exposure to ideologically diverse news and opinion on facebook. Science, 348, 1130\u20131132.","journal-title":"Science"},{"key":"9386_CR11","doi-asserted-by":"publisher","first-page":"1014","DOI":"10.1037\/0022-3514.81.6.1014","volume":"81","author":"JA Bargh","year":"2001","unstructured":"Bargh, J. A., Gollwitzer, P. M., Lee-Chai, A., Barndollar, K., & Tr\u00f6tschel, R. (2001). The automated will: Nonconscious activation and pursuit of behavioral goals. Journal of Personality and Social Psychology, 81, 1014.","journal-title":"Journal of Personality and Social Psychology"},{"key":"9386_CR12","doi-asserted-by":"crossref","unstructured":"Beutel, A., Chen, J., Doshi, T., Qian, H., Wei, L., Wu, Y., Heldt, L., Zhao, Z., Hong, L., & Chi, E. H. et\u00a0al. (2019). Fairness in recommendation ranking through pairwise comparisons. arXiv:1903.00780.","DOI":"10.1145\/3292500.3330745"},{"key":"9386_CR13","doi-asserted-by":"publisher","first-page":"250","DOI":"10.1093\/poq\/nfw007","volume":"80","author":"C Budak","year":"2016","unstructured":"Budak, C., Goel, S., & Rao, J. M. (2016). Fair and balanced? Quantifying media bias through crowdsourced content analysis. Public Opinion Quarterly, 80, 250\u2013271.","journal-title":"Public Opinion Quarterly"},{"key":"9386_CR14","doi-asserted-by":"crossref","unstructured":"Chelaru, S., Altingovde, I. S. & Siersdorfer, S. (2012). Analyzing the polarity of opinionated queries. In European Conference on Information Retrieval (pp. 463\u2013467), Springer.","DOI":"10.1007\/978-3-642-28997-2_42"},{"key":"9386_CR15","first-page":"6","volume":"8","author":"S Chelaru","year":"2013","unstructured":"Chelaru, S., Altingovde, I. S., Siersdorfer, S., & Nejdl, W. (2013). Analyzing, detecting, and exploiting sentiment in web queries. ACM Transactions on the Web (TWEB), 8, 6.","journal-title":"ACM Transactions on the Web (TWEB)"},{"key":"9386_CR16","unstructured":"Chen, X. & Yang, C. Z. (2006). Position paper: A study of web search engine bias and its assessment. IW3C2 WWW."},{"key":"9386_CR17","doi-asserted-by":"crossref","unstructured":"Chen, L., Ma, R., Hann\u00e1k, A. & Wilson, C. (2018). Investigating the impact of gender on rank in resume search engines. In Proceedings of the 2018 chi conference on human factors in computing systems (pp. 1\u201314).","DOI":"10.1145\/3173574.3174225"},{"key":"9386_CR18","doi-asserted-by":"crossref","unstructured":"Culpepper, J. S., Diaz, F. & Smucker, M. D. (2018). Research frontiers in information retrieval: Report from the third strategic workshop on information retrieval in lorne (swirl 2018). ACM SIGIR Forum, vol.\u00a052, pp. 46\u201347, ACM New York, NY, USA.","DOI":"10.1145\/3274784.3274788"},{"key":"9386_CR19","doi-asserted-by":"crossref","unstructured":"Demartini, G. & Siersdorfer, S. (2010). Dear search engine: what\u2019s your opinion about...?: Sentiment analysis for semantic enrichment of web search results. In Proceedings of the 3rd International Semantic Search Workshop, (P.\u00a04), ACM.","DOI":"10.1145\/1863879.1863883"},{"key":"9386_CR20","unstructured":"Diakopoulos, N., Trielli, D., Stark, J. & Mussenden, S. (2018). I vote for\u2014how search informs our choice of candidate. Digital Dominance: The Power of Google, Amazon, Facebook, and Apple, M. Moore and D. Tambini (Eds.), 22."},{"key":"9386_CR21","doi-asserted-by":"crossref","unstructured":"Diaz, A. (2008). Through the google goggles: Sociopolitical bias in search engine design. Web search, (pp. 11\u201334), Springer.","DOI":"10.1007\/978-3-540-75829-7_2"},{"key":"9386_CR22","doi-asserted-by":"crossref","unstructured":"Dutton, W. H., Reisdorf, B., Dubois, E. & Blank, G. (2017). Search and politics: The uses and impacts of search in Britain, France, Germany, Italy, Poland, Spain, and the United States.","DOI":"10.2139\/ssrn.2960697"},{"key":"9386_CR23","volume-title":"Cultures of the internet: the internet in Britain: Oxford Internet Survey 2013 Report","author":"WH Dutton","year":"2013","unstructured":"Dutton, W. H., Blank, G., & Groselj, D. (2013). Cultures of the internet: the internet in Britain: Oxford Internet Survey 2013 Report. Oxford: Oxford Internet Institute."},{"key":"9386_CR24","unstructured":"Elisa\u00a0Shearer, K. E. M. (2018). News use across social media platforms 2018. https:\/\/www.journalism.org\/2018\/09\/10\/news-use-across-social-media-platforms-2018\/."},{"key":"9386_CR25","unstructured":"Epstein, R. & Robertson, R.E. (2017). A method for detecting bias in search rankings, with evidence of systematic bias related to the 2016 presidential election. Technical Report White Paper no. WP-17-02."},{"key":"9386_CR26","doi-asserted-by":"publisher","first-page":"E4512","DOI":"10.1073\/pnas.1419828112","volume":"112","author":"R Epstein","year":"2015","unstructured":"Epstein, R., & Robertson, R. E. (2015). The search engine manipulation effect (seme) and its possible impact on the outcomes of elections. Proceedings of the National Academy of Sciences, 112, E4512\u2013E4521.","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"9386_CR27","first-page":"42","volume":"1","author":"R Epstein","year":"2017","unstructured":"Epstein, R., Robertson, R. E., Lazer, D., & Wilson, C. (2017). Suppressing the search engine manipulation effect (seme). Proceedings of the ACM: Human\u2013Computer Interaction, 1, 42.","journal-title":"Proceedings of the ACM: Human\u2013Computer Interaction"},{"key":"9386_CR28","doi-asserted-by":"crossref","unstructured":"Fang, Y., Si, L., Somasundaram, N. & Yu, Z. (2012). Mining contrastive opinions on political texts using cross-perspective topic model. In Proceedings of the fifth ACM international conference on Web search and data mining (pp. 63\u201372), ACM.","DOI":"10.1145\/2124295.2124306"},{"key":"9386_CR29","unstructured":"Finin, T., Murnane, W., Karandikar, A., Keller, N., Martineau, J. & Dredze, M. (2010). Annotating named entities in twitter data with crowdsourcing. In: Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon\u2019s Mechanical Turk, pp. 80\u201388, Association for Computational Linguistics."},{"key":"9386_CR30","doi-asserted-by":"crossref","unstructured":"Gao, R. & Shah, C. (2019). How fair can we go: Detecting the boundaries of fairness optimization in information retrieval. In: Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval (pp. 229\u2013236).","DOI":"10.1145\/3341981.3344215"},{"key":"9386_CR31","doi-asserted-by":"publisher","first-page":"102138","DOI":"10.1016\/j.ipm.2019.102138","volume":"57","author":"R Gao","year":"2020","unstructured":"Gao, R., & Shah, C. (2020). Toward creating a fairer ranking in search engine results. Information Processing & Management, 57, 102138.","journal-title":"Information Processing & Management"},{"key":"9386_CR32","doi-asserted-by":"publisher","first-page":"35","DOI":"10.3982\/ECTA7195","volume":"78","author":"M Gentzkow","year":"2010","unstructured":"Gentzkow, M., & Shapiro, J. M. (2010). What drives media slant? Evidence from us daily newspapers. Econometrica, 78, 35\u201371.","journal-title":"Econometrica"},{"key":"9386_CR33","doi-asserted-by":"crossref","unstructured":"Geyik, S. C., Ambler, S. & Kenthapadi, K. (2019). Fairness-aware ranking in search & recommendation systems with application to linkedin talent search. In: Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (pp. 2221\u20132231).","DOI":"10.1145\/3292500.3330691"},{"key":"9386_CR34","unstructured":"Ginger, G., & David, S. (2018). Google responds to trump, says no political motive in search results. https:\/\/www.reuters.com\/article\/us-usa-trump-tech-alphabet\/google-responds-to-trump-says-no-political-motive-in-search-results-idUSKCN1LD1QP. Retrieved 2018-10-06."},{"key":"9386_CR35","doi-asserted-by":"crossref","unstructured":"Goldman, E. (2008). Search engine bias and the demise of search engine utopianism. Web Search (pp. 121\u2013133), Springer.","DOI":"10.1007\/978-3-540-75829-7_8"},{"key":"9386_CR36","unstructured":"Grimes, D. R. (2016). Impartial journalism is laudable. but false balance is dangerous. https:\/\/www.theguardian.com\/science\/blog\/2016\/nov\/08\/impartial-journalism-is-laudable-but-false-balance-is-dangerous. Retrieved 2019-08-15."},{"key":"9386_CR37","doi-asserted-by":"crossref","unstructured":"Hu, D., Jiang, S., E.\u00a0Robertson, R. & Wilson, C. (2019). Auditing the partisanship of google search snippets. The World Wide Web Conference (pp. 693\u2013704).","DOI":"10.1145\/3308558.3313654"},{"key":"9386_CR38","volume-title":"The personal news cycle: How americans choose to get their news","author":"AP Institute","year":"2014","unstructured":"Institute, A. P. (2014). The personal news cycle: How americans choose to get their news. Reston: American Press Institute."},{"key":"9386_CR39","unstructured":"Kallus, N., & Zhou, A. (2019). The fairness of risk scores beyond classification: Bipartite ranking and the xauc metric. arXiv:1902.05826."},{"key":"9386_CR40","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1007\/s11263-016-0981-7","volume":"123","author":"R Krishna","year":"2017","unstructured":"Krishna, R., Zhu, Y., Groth, O., Johnson, J., Hata, K., Kravitz, J., et al. (2017). Visual genome: Connecting language and vision using crowdsourced dense image annotations. International Journal of Computer Vision, 123, 32\u201373.","journal-title":"International Journal of Computer Vision"},{"key":"9386_CR41","doi-asserted-by":"crossref","unstructured":"Kulshrestha, J., Eslami, M., Messias, J., Zafar, M. B., Ghosh, S., Gummadi, K. P. & Karahalios, K. (2017). Quantifying search bias: Investigating sources of bias for political searches in social media. In Proceedings of the 2017 ACM conference on computer supported cooperative work and social computing (pp. 417\u2013432), ACM.","DOI":"10.1145\/2998181.2998321"},{"issue":"1\u20132","key":"9386_CR42","first-page":"188","volume":"22","author":"J Kulshrestha","year":"2018","unstructured":"Kulshrestha, J., Eslami, M., Messias, J., Zafar, M. B., Ghosh, S., Gummadi, K. P., et al. (2018). Search bias quantification: Investigating political bias in social media and web search. Information Retrieval Journal, 22(1\u20132), 188\u2013227.","journal-title":"Information Retrieval Journal"},{"key":"9386_CR43","doi-asserted-by":"crossref","unstructured":"Lahoti, P., Garimella, K., & Gionis, A. (2018). Joint non-negative matrix factorization for learning ideological leaning on twitter. In Proceedings of the Eleventh ACM International Conference on Web Search and Data Mining (pp. 351\u2013359).","DOI":"10.1145\/3159652.3159669"},{"key":"9386_CR44","unstructured":"Lawson, N., Eustice, K., Perkowitz, M. & Yetisgen-Yildiz, M. (2010). Annotating large email datasets for named entity recognition with mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on creating speech and language data with Amazon\u2019s Mechanical Turk (pp. 71\u201379), Association for Computational Linguistics."},{"key":"9386_CR45","unstructured":"Mellebeek, B., Benavent, F., Grivolla, J., Codina, J., Costa-Jussa, M. R. & Banchs, R. (2010). Opinion mining of spanish customer comments with non-expert annotations on mechanical turk. In Proceedings of the NAACL HLT 2010 workshop on Creating speech and language data with Amazon\u2019s mechanical turk (pp. 114\u2013121), Association for Computational Linguistics."},{"key":"9386_CR46","doi-asserted-by":"publisher","first-page":"141","DOI":"10.1016\/S0306-4573(01)00020-6","volume":"38","author":"A Mowshowitz","year":"2002","unstructured":"Mowshowitz, A., & Kawaguchi, A. (2002a). Assessing bias in search engines. Information Processing & Management, 38, 141\u2013156.","journal-title":"Information Processing & Management"},{"key":"9386_CR47","doi-asserted-by":"publisher","first-page":"56","DOI":"10.1145\/567498.567527","volume":"45","author":"A Mowshowitz","year":"2002","unstructured":"Mowshowitz, A., & Kawaguchi, A. (2002b). Bias on the web. Communications of the ACM, 45, 56\u201360.","journal-title":"Communications of the ACM"},{"key":"9386_CR48","doi-asserted-by":"publisher","first-page":"1193","DOI":"10.1016\/j.ipm.2004.05.005","volume":"41","author":"A Mowshowitz","year":"2005","unstructured":"Mowshowitz, A., & Kawaguchi, A. (2005). Measuring search engine bias. Information Processing & Management, 41, 1193\u20131205.","journal-title":"Information Processing & Management"},{"key":"9386_CR49","doi-asserted-by":"publisher","first-page":"1031","DOI":"10.1257\/0002828054825619","volume":"95","author":"S Mullainathan","year":"2005","unstructured":"Mullainathan, S., & Shleifer, A. (2005). The market for news. American Economic Review, 95, 1031\u20131053.","journal-title":"American Economic Review"},{"key":"9386_CR50","unstructured":"Newman, N., Fletcher, R., Kalogeropoulos, A. & Nielsen, R. (2019). Reuters institute digital news report 2019, vol. 2019. Reuters Institute for the Study of Journalism."},{"key":"9386_CR51","unstructured":"Newman, N., Fletcher, R., Kalogeropoulos, A., Levy, D.A.L. & Nielsen, R. (2018). Reuters institute digital news report 2018, vol. 2018. Reuters Institute for the Study of Journalism."},{"key":"9386_CR52","doi-asserted-by":"publisher","DOI":"10.2307\/j.ctt1pwt9w5","volume-title":"Algorithms of Oppression: How search engines reinforce racism","author":"SU Noble","year":"2018","unstructured":"Noble, S. U. (2018). Algorithms of Oppression: How search engines reinforce racism. New York: NYU Press."},{"key":"9386_CR53","doi-asserted-by":"crossref","unstructured":"Otterbacher, J., Bates, J. & Clough, P. (2017). Competent men and warm women: Gender stereotypes and backlash in image search results. In Proceedings of the 2017 chi conference on human factors in computing systems (pp. 6620\u20136631).","DOI":"10.1145\/3025453.3025727"},{"key":"9386_CR54","doi-asserted-by":"crossref","unstructured":"Otterbacher, J., Checco, A., Demartini, G. & Clough, P. (2018). Investigating user perception of gender bias in image search: The role of sexism. In The 41st International ACM SIGIR conference on research & development in information retrieval (pp. 933\u2013936).","DOI":"10.1145\/3209978.3210094"},{"key":"9386_CR55","doi-asserted-by":"publisher","first-page":"801","DOI":"10.1111\/j.1083-6101.2007.00351.x","volume":"12","author":"B Pan","year":"2007","unstructured":"Pan, B., Hembrooke, H., Joachims, T., Lorigo, L., Gay, G., & Granka, L. (2007). In google we trust: Users\u2019 decisions on rank, position, and relevance. Journal of Computer-Mediated Communication, 12, 801\u2013823.","journal-title":"Journal of Computer-Mediated Communication"},{"key":"9386_CR56","doi-asserted-by":"crossref","unstructured":"R\u00e4biger, S., Gezici, G., Sayg\u0131n, Y. & Spiliopoulou, M. (2018). Predicting worker disagreement for more effective crowd labeling. In 2018 IEEE 5th International Conference on Data Science and Advanced Analytics (DSAA), (pp. 179\u2013188), IEEE.","DOI":"10.1109\/DSAA.2018.00028"},{"key":"9386_CR57","doi-asserted-by":"crossref","unstructured":"Raji, I. D. & Buolamwini, J. (2019). Actionable auditing: Investigating the impact of publicly naming biased performance results of commercial ai products. In Proceedings of the 2019 AAAI\/ACM Conference on AI, Ethics, and Society (pp. 429\u2013435).","DOI":"10.1145\/3306618.3314244"},{"key":"9386_CR58","doi-asserted-by":"crossref","unstructured":"Robertson, R. E., Lazer, D. & Wilson, C. (2018b). Auditing the personalization and composition of politically-related search engine results pages. In Proceedings of the 2018 World Wide Web Conference (pp. 955\u2013965).","DOI":"10.1145\/3178876.3186143"},{"key":"9386_CR59","doi-asserted-by":"publisher","first-page":"148","DOI":"10.1145\/3274417","volume":"2","author":"RE Robertson","year":"2018","unstructured":"Robertson, R. E., Jiang, S., Joseph, K., Friedland, L., Lazer, D., & Wilson, C. (2018a). Auditing partisan audience bias within google search. Proceedings of the ACM on Human\u2013Computer Interaction, 2, 148.","journal-title":"Proceedings of the ACM on Human\u2013Computer Interaction"},{"key":"9386_CR60","doi-asserted-by":"crossref","unstructured":"Saez-Trumper, D., Castillo, C. & Lalmas, M. (2013). Social media news communities: gatekeeping, coverage, and statement bias. In Proceedings of the 22nd ACM international conference on Information & Knowledge Management (pp. 1679\u20131684), ACM.","DOI":"10.1145\/2505515.2505623"},{"key":"9386_CR61","unstructured":"Sandvig, C., Hamilton, K., Karahalios, K. & Langbort, C. (2014). Auditing algorithms: Research methods for detecting discrimination on internet platforms. Data and discrimination: Converting critical concerns into productive inquiry, 22."},{"key":"9386_CR62","doi-asserted-by":"crossref","unstructured":"Sapiezynski, P., Zeng, W., E\u00a0Robertson, R., Mislove, A. & Wilson, C. (2019). Quantifying the impact of user attentionon fair group representation in ranked lists. In Companion Proceedings of The 2019 World Wide Web Conference (pp. 553\u2013562).","DOI":"10.1145\/3308560.3317595"},{"key":"9386_CR63","unstructured":"Sarcona, C. (2019). Organic search click through rates: The numbers never lie. https:\/\/www.zerolimitweb.com\/organic-vs-ppc-2019-ctr-results-best-practices\/. Retrieved 2019-09-06."},{"key":"9386_CR64","unstructured":"Stokes, P. (2019). False media balance. https:\/\/www.newphilosopher.com\/articles\/false-media-balance\/. Retrieved 2019-09-15."},{"key":"9386_CR65","unstructured":"Su, H., Deng, J., & Fei-Fei, L. (2012). Crowdsourcing annotations for visual object detection. In Workshops at the Twenty-Sixth AAAI Conference on Artificial,. Intelligence."},{"key":"9386_CR66","unstructured":"Tavani, H. (2012). Search engines and ethics."},{"key":"9386_CR67","doi-asserted-by":"crossref","first-page":"505","DOI":"10.1609\/icwsm.v13i01.3248","volume":"13","author":"N Vincent","year":"2019","unstructured":"Vincent, N., Johnson, I., Sheehan, P., & Hecht, B. (2019). Measuring the importance of user-generated content to search engines. Proceedings of the International AAAI Conference on Web and Social Media, 13, 505\u2013516.","journal-title":"Proceedings of the International AAAI Conference on Web and Social Media"},{"key":"9386_CR68","doi-asserted-by":"publisher","first-page":"184","DOI":"10.1007\/s11263-012-0564-1","volume":"101","author":"C Vondrick","year":"2013","unstructured":"Vondrick, C., Patterson, D., & Ramanan, D. (2013). Efficiently scaling up crowdsourced video annotation. International Journal of Computer Vision, 101, 184\u2013204.","journal-title":"International Journal of Computer Vision"},{"key":"9386_CR69","doi-asserted-by":"crossref","unstructured":"White, R. (2013). Beliefs and biases in web search. In Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval (pp. 3\u201312), ACM.","DOI":"10.1145\/2484028.2484053"},{"key":"9386_CR70","doi-asserted-by":"crossref","unstructured":"Yang, K. & Stoyanovich, J. (2017). Measuring fairness in ranked outputs. In Proceedings of the 29th International Conference on Scientific and Statistical Database Management (p.\u00a022), ACM.","DOI":"10.1145\/3085504.3085526"},{"key":"9386_CR71","doi-asserted-by":"crossref","unstructured":"Yigit-Sert, S., Altingovde, I.S. & Ulusoy, \u00d6. (2016). Towards detecting media bias by utilizing user comments.","DOI":"10.1145\/2908131.2908186"},{"key":"9386_CR72","doi-asserted-by":"crossref","unstructured":"Yuen, M. C., King, I. & Leung, K. S. (2011). A survey of crowdsourcing systems. In 2011 IEEE Third International Conference on Privacy, Security, Risk and Trust and 2011 IEEE Third International Conference on Social Computing (pp. 766\u2013773), IEEE.","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"key":"9386_CR73","doi-asserted-by":"crossref","unstructured":"Zehlike, M., Bonchi, F., Castillo, C., Hajian, S., Megahed, M. & Baeza-Yates, R. (2017). Fa* ir: A fair top-k ranking algorithm. In Proceedings of the 2017 ACM on Conference on Information and Knowledge Management (pp. 1569\u20131578), ACM.","DOI":"10.1145\/3132847.3132938"}],"container-title":["Information Retrieval Journal"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-020-09386-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10791-020-09386-w\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10791-020-09386-w.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,2]],"date-time":"2024-01-02T14:06:54Z","timestamp":1704204414000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10791-020-09386-w"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,1,27]]},"references-count":73,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2021,4]]}},"alternative-id":["9386"],"URL":"https:\/\/doi.org\/10.1007\/s10791-020-09386-w","relation":{},"ISSN":["1386-4564","1573-7659"],"issn-type":[{"value":"1386-4564","type":"print"},{"value":"1573-7659","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,1,27]]},"assertion":[{"value":"4 February 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 December 2020","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 January 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Compliance with ethical standards"}},{"value":"Author Emine Yilmaz previously worked as a research consultant for Microsoft Research and she is currently a research consultant for Amazon Research.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Ethical standard"}},{"value":"This content has been made available to all.","name":"free","label":"Free to read"}]}}