{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,11]],"date-time":"2026-03-11T05:40:30Z","timestamp":1773207630501,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":55,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,8,14]],"date-time":"2022-08-14T00:00:00Z","timestamp":1660435200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,8,14]]},"DOI":"10.1145\/3534678.3539059","type":"proceedings-article","created":{"date-parts":[[2022,8,12]],"date-time":"2022-08-12T19:06:41Z","timestamp":1660331201000},"page":"3513-3523","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["Looper: An End-to-End ML Platform for Product Decisions"],"prefix":"10.1145","author":[{"given":"Igor L.","family":"Markov","sequence":"first","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Hanson","family":"Wang","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Nitya S.","family":"Kasturi","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Shaun","family":"Singh","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Mia R.","family":"Garrard","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Yin","family":"Huang","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Sze Wai Celeste","family":"Yuen","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Sarah","family":"Tran","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Zehui","family":"Wang","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Igor","family":"Glotov","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Tanvi","family":"Gupta","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Peng","family":"Chen","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Boshuang","family":"Huang","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Xiaowen","family":"Xie","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Michael","family":"Belkin","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Sal","family":"Uryasev","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Sam","family":"Howie","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Eytan","family":"Bakshy","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]},{"given":"Norm","family":"Zhou","sequence":"additional","affiliation":[{"name":"Meta, Menlo Park, CA, USA"}]}],"member":"320","published-online":{"date-parts":[[2022,8,14]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Mart\u00edn Abadi et al. 2016. TensorFlow: A System for Large-Scale Machine Learning. In OSDI. 265--283."},{"key":"e_1_3_2_1_2_1","unstructured":"Alekh Agarwal et al. 2016. Making contextual decisions with low technical debt. arXiv:1606.03966 (2016)."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2009.52"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"crossref","unstructured":"Saleema Amershi et al . 2019. Software engineering for machine learning: A case study. In ICSE-SEIP. 291--300.","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465272"},{"key":"e_1_3_2_1_6_1","unstructured":"Anonymous. 2021. ETL vs ELT: Must Know Differences. https:\/\/www.guru99.com\/etl-vs-elt.html"},{"key":"e_1_3_2_1_7_1","unstructured":"Pavlos Athanasios Apostolopoulos et al. 2021. Personalization for Web-based Services using Offline Reinforcement Learning. arXiv:2102.05612 (2021)."},{"key":"e_1_3_2_1_8_1","volume-title":"A Simple but Tough-to-Beat Baseline for Sentence Embeddings. ICLR","author":"Arora Sanjeev","year":"2017","unstructured":"Sanjeev Arora, Yingyu Liang, and Tengyu Ma. 2017. A Simple but Tough-to-Beat Baseline for Sentence Embeddings. ICLR (2017)."},{"key":"e_1_3_2_1_9_1","volume-title":"NeurIPS 2018 Systems for ML Workshop.","author":"Eytan","unstructured":"Eytan Bakshy et al. 2018. AE: A domain-agnostic platform for adaptive experimentation. NeurIPS 2018 Systems for ML Workshop."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"crossref","unstructured":"Eytan Bakshy Dean Eckles and Michael S Bernstein. 2014. Designing and deploying online field experiments. In WWW' 14. 283--292.","DOI":"10.1145\/2566486.2567967"},{"key":"e_1_3_2_1_11_1","volume-title":"Andrew Gordon Wilson, and Eytan Bakshy","author":"Balandat Maximilian","year":"2020","unstructured":"Maximilian Balandat, Brian Karrer, Daniel R. Jiang, Samuel Daulton, Benjamin Letham, Andrew Gordon Wilson, and Eytan Bakshy. 2020. BoTorch: A Framework for Efficient Monte-Carlo Bayesian Optimization. In NeurIPS 33."},{"key":"e_1_3_2_1_12_1","volume-title":"Zhiyuan Cheng, Hubert Pham, and John Anderson.","author":"Beutel Alex","year":"2017","unstructured":"Alex Beutel, Ed H Chi, Zhiyuan Cheng, Hubert Pham, and John Anderson. 2017. Beyond globally optimal: Focused learning for improved recommendations. In WWW' 17. 203--212."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Eric Breck Shanqing Cai Eric Nielsen Michael Salib and D. Sculley. 2017. The ML Test Score: A Rubric for ML Production Readiness and Technical Debt Reduction. In IEEE Big Data.","DOI":"10.1109\/BigData.2017.8258038"},{"key":"e_1_3_2_1_14_1","unstructured":"Lee Byron. 2015. GraphQL: A data query language. https:\/\/engineering.fb.com\/2015\/09\/14\/core-data\/graphql-a-data-query-language"},{"key":"e_1_3_2_1_15_1","volume-title":"SmartChoices: Hybridizing programming and machine learning. arXiv:1810.00619","author":"Carbune Victor","year":"2018","unstructured":"Victor Carbune, Thierry Coppey, Alexander Daryin, Thomas Deselaers, Nikhil Sarda, and Jay Yagnik. 2018. SmartChoices: Hybridizing programming and machine learning. arXiv:1810.00619 (2018)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"crossref","unstructured":"P. Covington J. Adams and E. Sargin. 2016. Deep neural networks for Youtube recommendations. In RecSys.","DOI":"10.1145\/2959100.2959190"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/1341531.1341545"},{"key":"e_1_3_2_1_18_1","volume-title":"Opinion: The 3 Post-COVID Trends Empowering People and Shaping the Future. https:\/\/adage.com\/article\/opinion\/opinion-3-post-covid-trends-empowering-people-and-shaping-future\/2342861","author":"D'Arcy Marc","year":"2021","unstructured":"Marc D'Arcy. 2021. Opinion: The 3 Post-COVID Trends Empowering People and Shaping the Future. https:\/\/adage.com\/article\/opinion\/opinion-3-post-covid-trends-empowering-people-and-shaping-future\/2342861"},{"key":"e_1_3_2_1_19_1","unstructured":"Samuel Daulton et al. 2019. Thompson sampling for contextual bandit problems with auxiliary safety constraints. arXiv:1911.00638 (2019)."},{"key":"e_1_3_2_1_20_1","unstructured":"Samuel Daulton Maximilian Balandat and Eytan Bakshy. 2021. Parallel Bayesian Optimization of Multiple Noisy Objectives with Expected Hypervolume Improvement In NeurIPS 34. arXiv:2006.05078."},{"key":"e_1_3_2_1_21_1","unstructured":"Ben Dickson. 2021. Why machine learning strategies fail. https:\/\/venturebeat.com\/2021\/02\/25\/why-machine-learning-strategies-fail\/"},{"key":"e_1_3_2_1_22_1","unstructured":"Jeffrey Dunn. 2016. Introducing FBLearner Flow: Facebook's AI backbone. https:\/\/engineering.fb.com\/2016\/05\/09\/core-data\/introducing-fblearner-flow-facebook-s-ai-backbone"},{"key":"e_1_3_2_1_23_1","volume-title":"High-dimensional contextual policy search with unknown context rewards using Bayesian optimization. NeurIPS 33","author":"Feng Qing","year":"2020","unstructured":"Qing Feng, Benjamin Letham, Hongzi Mao, and Eytan Bakshy. 2020. High-dimensional contextual policy search with unknown context rewards using Bayesian optimization. NeurIPS 33 (2020)."},{"key":"e_1_3_2_1_24_1","volume-title":"Horizon: Facebook's Open Source Applied Reinforcement Learning Platform. arXiv:1811.00260","author":"Jason Gauci","year":"2018","unstructured":"Jason Gauci et al. 2018. Horizon: Facebook's Open Source Applied Reinforcement Learning Platform. arXiv:1811.00260 (2018)."},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"crossref","unstructured":"Udit Gupta et al. 2020. The Architectural Implications of Facebook's DNN-based Personalized Recommendation. HPCA (2020) 488--501. arXiv:1906.03109","DOI":"10.1109\/HPCA47549.2020.00047"},{"key":"e_1_3_2_1_26_1","volume-title":"HPCA","author":"Kim","year":"2018","unstructured":"Kim Hazelwood et al. 2018. Applied machine learning at Facebook: A datacenter infrastructure perspective. In HPCA 2018. IEEE, 620--629."},{"key":"e_1_3_2_1_27_1","volume-title":"Meet Michelangelo: Uber's Machine Learning Platform. https:\/\/eng.uber.com\/michelangelo-machine-learning-platform\/","author":"Hermann J.","unstructured":"J. Hermann and M. Del Balso. 2017. Meet Michelangelo: Uber's Machine Learning Platform. https:\/\/eng.uber.com\/michelangelo-machine-learning-platform\/"},{"key":"e_1_3_2_1_28_1","volume-title":"Controlled experiments on the Web: survey and practical guide. Data mining and knowledge discovery 18, 1","author":"Kohavi Ron","year":"2009","unstructured":"Ron Kohavi, Roger Longbotham, Dan Sommerfield, and Randal M Henne. 2009. Controlled experiments on the Web: survey and practical guide. Data mining and knowledge discovery 18, 1 (2009), 140--181."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","unstructured":"Tim Kraska et al. 2017. The Case for Learned Index Structures. CoRR (2017). arXiv:1712.01208","DOI":"10.1145\/3183713.3196909"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1804597116"},{"key":"e_1_3_2_1_31_1","unstructured":"Adam Daniel Laud. 2004. Theory and application of reward shaping in reinforcement learning. UIUC."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"L. Li W. Chu J. Langford and R. E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In WWW. 661--670.","DOI":"10.1145\/1772690.1772758"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"Shen Li et al. 2020. PyTorch Distributed: Experiences on Accelerating Data Parallel Training. In VLDB Vol. 13(12).","DOI":"10.14778\/3415478.3415530"},{"key":"e_1_3_2_1_34_1","unstructured":"Hongzi Mao et al. 2020. Real-world video adaptation with reinforcement learning. arXiv:2008.12858 (2020)."},{"key":"e_1_3_2_1_35_1","unstructured":"L. J. Miranda. 2021. Towards data-centric machine learning: a short review. (2021). https:\/\/ljvmiranda921.github.io\/notebook\/2021\/07\/30\/data-centric-ml\/"},{"key":"e_1_3_2_1_36_1","unstructured":"P. Molino Y. Dudin and S. S. Miryala. 2019. Ludwig: a type-based declarative deep learning toolbox. arxiv:1909.07930 (2019)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"P. Molino and C. R\u00e9. 2021. Declarative Machine Learning Systems. ACM Queue 19 (2021). Issue 3.","DOI":"10.1145\/3475965.3479315"},{"key":"e_1_3_2_1_38_1","unstructured":"Maxim Naumov et al. 2019. Deep Learning Recommendation Model for Personalization and Recommendation Systems. CoRR abs\/1906.00091 (2019)."},{"key":"e_1_3_2_1_39_1","volume-title":"Orr et al","author":"Laurel","year":"2021","unstructured":"Laurel J. Orr et al. 2021. Managing ML Pipelines: Feature Stores and the Coming Wave of Embedding Ecosystems. CoRR (2021). arXiv:2108.05053"},{"key":"e_1_3_2_1_40_1","volume-title":"Challenges in deploying machine learning: a survey of case studies. arXiv:2011.09926","author":"Paleyes Andrei","year":"2020","unstructured":"Andrei Paleyes, Raoul-Gabriel Urma, and Neil D Lawrence. 2020. Challenges in deploying machine learning: a survey of case studies. arXiv:2011.09926 (2020)."},{"key":"e_1_3_2_1_41_1","volume-title":"Overton: A data system for monitoring and improving machine-learned products. arXiv:1909.05372","author":"Christopher R\u00e9","year":"2019","unstructured":"Christopher R\u00e9 et al. 2019. Overton: A data system for monitoring and improving machine-learned products. arXiv:1909.05372 (2019)."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","unstructured":"P. Rodr\u00edguez M. Bautista J. Gonz\u00e0lez and S. Escalera. 2018. Beyond One-hot Encoding: lower dimensional target embedding. arXiv:1806.10805 (2018).","DOI":"10.1016\/j.imavis.2018.04.004"},{"key":"e_1_3_2_1_43_1","unstructured":"Gautam Roy. 2016. How we built Facebook Lite for every Android phone and network. https:\/\/engineering.fb.com\/2016\/03\/09\/android\/how-we-built-facebook-lite-for-every-android-phone-and-network"},{"key":"e_1_3_2_1_44_1","unstructured":"Ram Sagar. 2021. Andrew Ng Urges ML Community To Be More Data-Centric. https:\/\/analyticsindiamag.com\/big-data-to-good-data-andrew-ng-urges-ml-community-to-be-more-data-centric-and-less-model-centric\/"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"crossref","unstructured":"Nithya Sambasivan et al. 2021. \"Everyone wants to do the model work not the data work\": Data Cascades in High-Stakes AI. SIGCHI ACM (2021).","DOI":"10.1145\/3411764.3445518"},{"key":"e_1_3_2_1_46_1","first-page":"2503","article-title":"Hidden technical debt in machine learning systems","volume":"28","author":"David Sculley","year":"2015","unstructured":"David Sculley et al. 2015. Hidden technical debt in machine learning systems. NIPS 28 (2015), 2503--2511.","journal-title":"NIPS"},{"key":"e_1_3_2_1_47_1","volume-title":"USENIX Conf. Operational Machine Learning (OpML). USENIX, 15--17","author":"Jonathan","unstructured":"Jonathan Soifer et al. 2019. Deep Learning Inference Service at Microsoft. In USENIX Conf. Operational Machine Learning (OpML). USENIX, 15--17. https:\/\/www.usenix.org\/conference\/opml19\/presentation\/soifer"},{"key":"e_1_3_2_1_48_1","unstructured":"Gregory J. Stein. 2019. Proxy metrics are everywhere in machine learning. http:\/\/cachestocaches.com\/2019\/1\/proxy-metrics-are-everywhere-machine-lea"},{"key":"e_1_3_2_1_49_1","first-page":"16","article-title":"MODELDB: Opportunities and Challenges in Managing Machine Learning Models","volume":"41","author":"Vartak M.","year":"2018","unstructured":"M. Vartak and S. Madden. 2018. MODELDB: Opportunities and Challenges in Managing Machine Learning Models. IEEE Data Eng. Bull. 41, 4 (2018), 16--25.","journal-title":"IEEE Data Eng. Bull."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.2017.1319839"},{"key":"e_1_3_2_1_51_1","volume-title":"Predictive Precompute with Recurrent Neural Networks. arXiv:1912.06779","author":"Wang Hanson","year":"2019","unstructured":"Hanson Wang, Zehui Wang, and Yuanyuan Ma. 2019. Predictive Precompute with Recurrent Neural Networks. arXiv:1912.06779 (2019)."},{"key":"e_1_3_2_1_52_1","volume-title":"Sustainable AI: Environmental Implications, Challenges and Opportunities. arXiv:2111.00364 [cs.LG]","author":"Wu Carole-Jean","year":"2021","unstructured":"Carole-Jean Wu et al. 2021. Sustainable AI: Environmental Implications, Challenges and Opportunities. arXiv:2111.00364 [cs.LG]"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"crossref","unstructured":"Ya Xu et al. 2015. From infrastructure to culture: A\/B testing challenges in large scale social networks. In KDD. 2227--2236.","DOI":"10.1145\/2783258.2788602"},{"key":"e_1_3_2_1_54_1","volume-title":"Evaluation of explore-exploit policies in multi-result ranking systems. arXiv:1504.07662","author":"Yankov Dragomir","year":"2015","unstructured":"Dragomir Yankov, Pavel Berkhin, and Lihong Li. 2015. Evaluation of explore-exploit policies in multi-result ranking systems. arXiv:1504.07662 (2015)."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"crossref","unstructured":"Zhe Zhao et al. 2019. Recommending what video to watch next: a multitask ranking system. In RecSys ?19. 43--51.","DOI":"10.1145\/3298689.3346997"}],"event":{"name":"KDD '22: The 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","location":"Washington DC USA","acronym":"KDD '22","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data"]},"container-title":["Proceedings of the 28th ACM SIGKDD Conference on Knowledge Discovery and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539059","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3534678.3539059","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T18:09:50Z","timestamp":1750183790000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3534678.3539059"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,8,14]]},"references-count":55,"alternative-id":["10.1145\/3534678.3539059","10.1145\/3534678"],"URL":"https:\/\/doi.org\/10.1145\/3534678.3539059","relation":{},"subject":[],"published":{"date-parts":[[2022,8,14]]},"assertion":[{"value":"2022-08-14","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}