{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,7]],"date-time":"2026-04-07T12:15:51Z","timestamp":1775564151834,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":64,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,22]],"date-time":"2025-04-22T00:00:00Z","timestamp":1745280000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,28]]},"DOI":"10.1145\/3696410.3714738","type":"proceedings-article","created":{"date-parts":[[2025,5,5]],"date-time":"2025-05-05T16:42:02Z","timestamp":1746463322000},"page":"3702-3715","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Reducing Symbiosis Bias through Better A\/B Tests of Recommendation Algorithms"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-8101-8939","authenticated-orcid":false,"given":"Jennifer","family":"Brennan","sequence":"first","affiliation":[{"name":"Google Research, Kirkland, WA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-3024-0235","authenticated-orcid":false,"given":"Yahu","family":"Cong","sequence":"additional","affiliation":[{"name":"University of California, Berkeley, Berkeley, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1832-675X","authenticated-orcid":false,"given":"Yiwei","family":"Yu","sequence":"additional","affiliation":[{"name":"University of California, Berkeley, Berkeley, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-7261-1425","authenticated-orcid":false,"given":"Lina","family":"Lin","sequence":"additional","affiliation":[{"name":"Google, San Bruno, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4005-6429","authenticated-orcid":false,"given":"Yajun","family":"Peng","sequence":"additional","affiliation":[{"name":"Google, San Bruno, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0009-0005-9427-6511","authenticated-orcid":false,"given":"Changping","family":"Meng","sequence":"additional","affiliation":[{"name":"Google, Mountain View, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4513-8575","authenticated-orcid":false,"given":"Ningren","family":"Han","sequence":"additional","affiliation":[{"name":"Google, Mountain View, CA, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3729-9547","authenticated-orcid":false,"given":"Jean","family":"Pouget-Abadie","sequence":"additional","affiliation":[{"name":"Google Research, New York, NY, USA"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0896-8628","authenticated-orcid":false,"given":"David M.","family":"Holtz","sequence":"additional","affiliation":[{"name":"Haas School of Business, University of California, Berkeley, Berkeley, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2025,4,22]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Algorithmic pricing and consumer sensitivity to price variability. Available at SSRN 4435831","author":"Aparicio Diego","year":"2023","unstructured":"Diego Aparicio, Dean Eckles, and Madhav Kumar. 2023. Algorithmic pricing and consumer sensitivity to price variability. Available at SSRN 4435831 (2023)."},{"key":"e_1_3_2_1_2_1","volume-title":"Multiple randomization designs. arXiv preprint arXiv:2112.13495","author":"Bajari Patrick","year":"2021","unstructured":"Patrick Bajari, Brian Burdick, Guido W Imbens, Lorenzo Masoero, James McQueen, Thomas Richardson, and Ido M Rosen. 2021. Multiple randomization designs. arXiv preprint arXiv:2112.13495 (2021)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1177\/0081175018782569"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/asy036"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2022.4583"},{"key":"e_1_3_2_1_6_1","article-title":"Counterfactual reasoning and learning systems: The example of computational advertising","volume":"14","author":"Bottou L\u00e9on","year":"2013","unstructured":"L\u00e9on Bottou, Jonas Peters, Joaquin Qui nonero-Candela, Denis X Charles, D Max Chickering, Elon Portugaly, Dipankar Ray, Patrice Simard, and Ed Snelson. 2013. Counterfactual reasoning and learning systems: The example of computational advertising. Journal of Machine Learning Research, Vol. 14, 11 (2013).","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_7_1","first-page":"37962","article-title":"Cluster randomized designs for one-sided bipartite experiments","volume":"35","author":"Brennan Jennifer","year":"2022","unstructured":"Jennifer Brennan, Vahab Mirrokni, and Jean Pouget-Abadie. 2022. Cluster randomized designs for one-sided bipartite experiments. Advances in Neural Information Processing Systems, Vol. 35 (2022), 37962--37974.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_8_1","volume-title":"Optimizing randomized and deterministic saturation designs under interference. arXiv preprint arXiv:2203.09682","author":"Cai Chencheng","year":"2022","unstructured":"Chencheng Cai, Jean Pouget-Abadie, and Edoardo M Airoldi. 2022. Optimizing randomized and deterministic saturation designs under interference. arXiv preprint arXiv:2203.09682 (2022)."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240370"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/3564284"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3474601"},{"key":"e_1_3_2_1_12_1","volume-title":"Combining Rollout Designs and Clustering for Causal Inference under Low-order Interference. arXiv preprint arXiv:2405.05119","author":"Cortez-Rodriguez Mayleen","year":"2024","unstructured":"Mayleen Cortez-Rodriguez, Matthew Eichhorn, and Christina Lee Yu. 2024. Combining Rollout Designs and Clustering for Causal Inference under Low-order Interference. arXiv preprint arXiv:2405.05119 (2024)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1515\/jci-2015-0021"},{"key":"e_1_3_2_1_14_1","first-page":"535","article-title":"Markovian interference in experiments","volume":"35","author":"Farias Vivek","year":"2022","unstructured":"Vivek Farias, Andrew Li, Tianyi Peng, and Andrew Zheng. 2022. Markovian interference in experiments. Advances in Neural Information Processing Systems, Vol. 35 (2022), 535--549.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2020.1768100"},{"key":"e_1_3_2_1_16_1","first-page":"15054","article-title":"Adaptive experimental design with temporal interference: A maximum likelihood approach","volume":"33","author":"Glynn Peter W","year":"2020","unstructured":"Peter W Glynn, Ramesh Johari, and Mohammad Rasouli. 2020. Adaptive experimental design with temporal interference: A maximum likelihood approach. Advances in Neural Information Processing Systems, Vol. 33 (2020), 15054--15064.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_17_1","volume-title":"A bias correction approach for interference in ranking experiments. Marketing Science","author":"Goli Ali","year":"2023","unstructured":"Ali Goli, Anja Lambrecht, and Hema Yoganarasimhan. 2023. A bias correction approach for interference in ranking experiments. Marketing Science (2023)."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741081"},{"key":"e_1_3_2_1_19_1","volume-title":"KDD Workshop on Multi-Armed Bandits and Reinforcement Learning: Advancing Decision Making in E-Commerce and Beyond","author":"Guo Hongbo","year":"2023","unstructured":"Hongbo Guo, Ruben Naeff, Alex Nikulkov, and Zheqing Zhu. 2023. Evaluating online bandit exploration in large-scale recommender system. KDD Workshop on Multi-Armed Bandits and Reinforcement Learning: Advancing Decision Making in E-Commerce and Beyond (2023)."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401434"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1214\/23-EJS2111"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3391403.3399532"},{"key":"e_1_3_2_1_23_1","volume-title":"Reducing interference bias in online marketplace experiments using cluster randomization: Evidence from a pricing meta-experiment on airbnb. Management Science","author":"Holtz David","year":"2024","unstructured":"David Holtz, Felipe Lobel, Ruben Lobel, Inessa Liskovich, and Sinan Aral. 2024. Reducing interference bias in online marketplace experiments using cluster randomization: Evidence from a pricing meta-experiment on airbnb. Management Science (2024)."},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1952.10483446"},{"key":"e_1_3_2_1_25_1","volume-title":"Switchback experiments under geometric mixing. arXiv preprint arXiv:2209.00197","author":"Hu Yuchen","year":"2022","unstructured":"Yuchen Hu and Stefan Wager. 2022. Switchback experiments under geometric mixing. arXiv preprint arXiv:2209.00197 (2022)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401230"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3636341.3636358"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3569930"},{"key":"e_1_3_2_1_29_1","volume-title":"Causal Inference in Social Platforms Under Approximate Interference Networks. arXiv preprint arXiv:2408.04441","author":"Jiang Yiming","year":"2024","unstructured":"Yiming Jiang, Lu Deng, Yong Wang, and He Wang. 2024. Causal Inference in Social Platforms Under Approximate Interference Networks. arXiv preprint arXiv:2408.04441 (2024)."},{"key":"e_1_3_2_1_30_1","volume-title":"Causal Inference under Network Interference Using a Mixture of Randomized Experiments. arXiv preprint arXiv:2309.00141","author":"Jiang Yiming","year":"2023","unstructured":"Yiming Jiang and He Wang. 2023. Causal Inference under Network Interference Using a Mixture of Randomized Experiments. arXiv preprint arXiv:2309.00141 (2023)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2021.4247"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3383313.3412260"},{"key":"e_1_3_2_1_33_1","volume-title":"Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery","author":"Kohavi Ron","year":"2009","unstructured":"Ron Kohavi, Roger Longbotham, Dan Sommerfield, and Randal M Henne. 2009. Controlled experiments on the web: survey and practical guide. Data mining and knowledge discovery, Vol. 18 (2009), 140--181."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1214\/22-AOS2224"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3640457.3688046"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2959100.2959141"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1515\/spp-2013-0002"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"crossref","unstructured":"Paul Musgrave Cuize Han and Parth Gupta. 2023. Measuring service-level learning effects in search via query-randomized experiments. (2023).","DOI":"10.1145\/3539618.3592020"},{"key":"e_1_3_2_1_39_1","first-page":"6466","article-title":"A\/b testing for recommender systems in a two-sided marketplace","volume":"34","author":"Nandy Preetam","year":"2021","unstructured":"Preetam Nandy, Divya Venugopalan, Chun Lo, and Shaunak Chatterjee. 2021. A\/b testing for recommender systems in a two-sided marketplace. Advances in Neural Information Processing Systems, Vol. 34 (2021), 6466--6477.","journal-title":"Advances in Neural Information Processing Systems"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.2307\/2983637"},{"key":"e_1_3_2_1_41_1","volume-title":"Science","volume":"377","author":"Rajkumar Karthik","year":"2022","unstructured":"Karthik Rajkumar, Guillaume Saint-Jacques, Iavor Bojinov, Erik Brynjolfsson, and Sinan Aral. 2022. A causal test of the strength of weak ties. Science, Vol. 377, 6612 (2022), 1304--1310."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","first-page":"591","DOI":"10.2307\/2287653","article-title":"Randomization analysis of experimental data: The Fisher randomization test comment","volume":"75","author":"Rubin Donald B","year":"1980","unstructured":"Donald B Rubin. 1980. Randomization analysis of experimental data: The Fisher randomization test comment. Journal of the American statistical association, Vol. 75, 371 (1980), 591--593.","journal-title":"Journal of the American statistical association"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"Donald B Rubin. 2008. For objective causal inference design trumps analysis. (2008).","DOI":"10.1214\/08-AOAS187"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401114"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3098192"},{"key":"e_1_3_2_1_46_1","volume-title":"international conference on machine learning. PMLR, 1670--1679","author":"Schnabel Tobias","year":"2016","unstructured":"Tobias Schnabel, Adith Swaminathan, Ashudeep Singh, Navin Chandak, and Thorsten Joachims. 2016. Recommendations as treatments: Debiasing learning and evaluation. In international conference on machine learning. PMLR, 1670--1679."},{"key":"e_1_3_2_1_47_1","volume-title":"International Conference on Artificial Intelligence and Statistics. PMLR, 19--27","author":"Shankar Shiv","year":"2024","unstructured":"Shiv Shankar, Ritwik Sinha, Yash Chandak, Saayan Mitra, and Madalina Fiterau. 2024b. A\/B testing under Interference with Partial Network Information. In International Conference on Artificial Intelligence and Statistics. PMLR, 19--27."},{"key":"e_1_3_2_1_48_1","volume-title":"Forty-first International Conference on Machine Learning.","author":"Shankar Shiv","year":"2024","unstructured":"Shiv Shankar, Ritwik Sinha, and Madalina Fiterau. 2024a. On Online Experimentation without Device Identifiers. In Forty-first International Conference on Machine Learning."},{"key":"e_1_3_2_1_49_1","volume-title":"Tackling Interference Induced by Data Training Loops in A\/B Tests: A Weighted Training Approach. arXiv preprint arXiv:2310.17496","author":"Nian Si.","year":"2023","unstructured":"Nian Si. 2023. Tackling Interference Induced by Data Training Loops in A\/B Tests: A Weighted Training Approach. arXiv preprint arXiv:2310.17496 (2023)."},{"key":"e_1_3_2_1_50_1","volume-title":"Deconvolving feedback loops in recommender systems. Advances in neural information processing systems","author":"Sinha Ayan","year":"2016","unstructured":"Ayan Sinha, David F Gleich, and Karthik Ramani. 2016. Deconvolving feedback loops in recommender systems. Advances in neural information processing systems, Vol. 29 (2016)."},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835895"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3616855.3635833"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308560.3317303"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3604915.3610246"},{"key":"e_1_3_2_1_55_1","volume-title":"International conference on machine learning. PMLR, 1489--1497","author":"Toulis Panos","year":"2013","unstructured":"Panos Toulis and Edward Kao. 2013. Estimation of causal peer influence effects. In International conference on machine learning. PMLR, 1489--1497."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2487575.2487695"},{"key":"e_1_3_2_1_57_1","volume-title":"Causal clustering: design of cluster experiments under network interference. arXiv preprint arXiv:2310.14983","author":"Viviano Davide","year":"2023","unstructured":"Davide Viviano, Lihua Lei, Guido Imbens, Brian Karrer, Okke Schrijvers, and Liang Shi. 2023. Causal clustering: design of cluster experiments under network interference. arXiv preprint arXiv:2310.14983 (2023)."},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3534678.3539461"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240323.3240355"},{"key":"e_1_3_2_1_60_1","volume-title":"Estimating Treatment Effects under Recommender Interference: A Structured Neural Networks Approach. arXiv preprint arXiv:2406.14380","author":"Zhan Ruohan","year":"2024","unstructured":"Ruohan Zhan, Shichao Han, Yuchen Hu, and Zhenling Jiang. 2024. Estimating Treatment Effects under Recommender Interference: A Structured Neural Networks Approach. arXiv preprint arXiv:2406.14380 (2024)."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3449788"},{"key":"e_1_3_2_1_62_1","volume-title":"Seller-side experiments under interference induced by feedback loops in two-sided platforms. arXiv preprint arXiv:2401.15811","author":"Zhu Zhihua","year":"2024","unstructured":"Zhihua Zhu, Zheng Cai, Liang Zheng, and Nian Si. 2024. Seller-side experiments under interference induced by feedback loops in two-sided platforms. arXiv preprint arXiv:2401.15811 (2024)."},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3437963.3441820"},{"key":"e_1_3_2_1_64_1","volume-title":"Bipartite causal inference with interference. Statistical science: a review journal of the Institute of Mathematical Statistics","author":"Zigler Corwin M","year":"2021","unstructured":"Corwin M Zigler and Georgia Papadogeorgou. 2021. Bipartite causal inference with interference. Statistical science: a review journal of the Institute of Mathematical Statistics, Vol. 36, 1 (2021), 109."}],"event":{"name":"WWW '25: The ACM Web Conference 2025","location":"Sydney NSW Australia","acronym":"WWW '25","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the ACM on Web Conference 2025"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696410.3714738","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3696410.3714738","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:18:41Z","timestamp":1750295921000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3696410.3714738"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,22]]},"references-count":64,"alternative-id":["10.1145\/3696410.3714738","10.1145\/3696410"],"URL":"https:\/\/doi.org\/10.1145\/3696410.3714738","relation":{},"subject":[],"published":{"date-parts":[[2025,4,22]]},"assertion":[{"value":"2025-04-22","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}