{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,17]],"date-time":"2026-06-17T04:56:14Z","timestamp":1781672174610,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":68,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,6,9]],"date-time":"2021-06-09T00:00:00Z","timestamp":1623196800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,6,9]]},"DOI":"10.1145\/3448016.3457552","type":"proceedings-article","created":{"date-parts":[[2021,6,18]],"date-time":"2021-06-18T17:22:30Z","timestamp":1624036950000},"page":"2503-2516","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":45,"title":["Real-time Data Infrastructure at Uber"],"prefix":"10.1145","author":[{"given":"Yupeng","family":"Fu","sequence":"first","affiliation":[{"name":"Uber, Inc., San Francisco, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chinmay","family":"Soman","sequence":"additional","affiliation":[{"name":"Uber, Inc., San Francisco, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,6,18]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Amazon Kinesis. https:\/\/aws.amazon.com\/kinesis\/  Amazon Kinesis. https:\/\/aws.amazon.com\/kinesis\/"},{"key":"e_1_3_2_2_2_1","unstructured":"Amazon S3. https:\/\/aws.amazon.com\/s3\/  Amazon S3. https:\/\/aws.amazon.com\/s3\/"},{"key":"e_1_3_2_2_3_1","unstructured":"Apache ActiveMQ. http:\/\/activemq.apache.org\/  Apache ActiveMQ. http:\/\/activemq.apache.org\/"},{"key":"e_1_3_2_2_4_1","unstructured":"Apache Apex. https:\/\/apex.apache.org\/  Apache Apex. https:\/\/apex.apache.org\/"},{"key":"e_1_3_2_2_5_1","unstructured":"Apache Beam. https:\/\/beam.apache.org\/  Apache Beam. https:\/\/beam.apache.org\/"},{"key":"e_1_3_2_2_6_1","unstructured":"Apache Calcite. https:\/\/calcite.apache.org\/  Apache Calcite. https:\/\/calcite.apache.org\/"},{"key":"e_1_3_2_2_7_1","unstructured":"Apache Drill. https:\/\/drill.apache.org\/  Apache Drill. https:\/\/drill.apache.org\/"},{"key":"e_1_3_2_2_8_1","unstructured":"Apache HDFS. https:\/\/hadoop.apache.org\/  Apache HDFS. https:\/\/hadoop.apache.org\/"},{"key":"e_1_3_2_2_9_1","unstructured":"Apache Hive. https:\/\/hive.apache.org\/  Apache Hive. https:\/\/hive.apache.org\/"},{"key":"e_1_3_2_2_10_1","unstructured":"Apache Impala. https:\/\/impala.apache.org\/  Apache Impala. https:\/\/impala.apache.org\/"},{"key":"e_1_3_2_2_11_1","unstructured":"Apache Kylin. http:\/\/kylin.apache.org\/  Apache Kylin. http:\/\/kylin.apache.org\/"},{"key":"e_1_3_2_2_12_1","unstructured":"Apache Pulsar. https:\/\/pulsar.apache.org\/  Apache Pulsar. https:\/\/pulsar.apache.org\/"},{"key":"e_1_3_2_2_13_1","unstructured":"Apache Samza. http:\/\/samza.apache.org\/  Apache Samza. http:\/\/samza.apache.org\/"},{"key":"e_1_3_2_2_14_1","unstructured":"Apache Storm. https:\/\/storm.apache.org\/  Apache Storm. https:\/\/storm.apache.org\/"},{"key":"e_1_3_2_2_15_1","unstructured":"Benchmarking Apache Kafka Apache Pulsar and RabbitMQ: Which is the Fastest? https:\/\/www.confluent.io\/blog\/kafka-fastest-messaging-system\/  Benchmarking Apache Kafka Apache Pulsar and RabbitMQ: Which is the Fastest? https:\/\/www.confluent.io\/blog\/kafka-fastest-messaging-system\/"},{"key":"e_1_3_2_2_16_1","unstructured":"Clickhouse. https:\/\/clickhouse.tech\/  Clickhouse. https:\/\/clickhouse.tech\/"},{"key":"e_1_3_2_2_17_1","unstructured":"Elasticsearch. https:\/\/www.elastic.co\/elasticsearch\/  Elasticsearch. https:\/\/www.elastic.co\/elasticsearch\/"},{"key":"e_1_3_2_2_18_1","unstructured":"Google Cloud Storage. https:\/\/cloud.google.com\/storage  Google Cloud Storage. https:\/\/cloud.google.com\/storage"},{"key":"e_1_3_2_2_19_1","unstructured":"IBM Websphere MQ. https:\/\/www.ibm.com\/products\/mq  IBM Websphere MQ. https:\/\/www.ibm.com\/products\/mq"},{"key":"e_1_3_2_2_20_1","unstructured":"Monorepo. https:\/\/en.wikipedia.org\/wiki\/Monorepo  Monorepo. https:\/\/en.wikipedia.org\/wiki\/Monorepo"},{"key":"e_1_3_2_2_21_1","unstructured":"MySQL. https:\/\/www.mysql.com\/  MySQL. https:\/\/www.mysql.com\/"},{"key":"e_1_3_2_2_22_1","unstructured":"Oracle Enterprise Messaging Service. https:\/\/www.oracle.com\/technetwork\/ topics\/oracleenterprisemessagingservicepre-129348.pdf  Oracle Enterprise Messaging Service. https:\/\/www.oracle.com\/technetwork\/ topics\/oracleenterprisemessagingservicepre-129348.pdf"},{"key":"e_1_3_2_2_23_1","unstructured":"RabbitMQ. http:\/\/www.rabbitmq.com\/  RabbitMQ. http:\/\/www.rabbitmq.com\/"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"crossref","unstructured":"Tyler Akidau Robert Bradshaw Craig Chambers Slava Chernyak Rafael J Fern\u00e1ndez-Moctezuma Reuven Lax Sam McVeety Daniel Mills Frances Perry Eric Schmidt etal 2015. The dataflow model: a practical approach to balancing correctness latency and cost in massive-scale unbounded out-of-order data processing. (2015).  Tyler Akidau Robert Bradshaw Craig Chambers Slava Chernyak Rafael J Fern\u00e1ndez-Moctezuma Reuven Lax Sam McVeety Daniel Mills Frances Perry Eric Schmidt et al. 2015. The dataflow model: a practical approach to balancing correctness latency and cost in massive-scale unbounded out-of-order data processing. (2015).","DOI":"10.14778\/2824032.2824076"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465272"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2742797"},{"key":"e_1_3_2_2_27_1","volume-title":"Apache flink: Stream and batch processing in a single engine. Bulletin of the IEEE Computer Society Technical Committee on Data Engineering 36, 4","author":"Carbone Paris","year":"2015","unstructured":"Paris Carbone , Asterios Katsifodimos , Stephan Ewen , Volker Markl , Seif Haridi , and Kostas Tzoumas . 2015. Apache flink: Stream and batch processing in a single engine. Bulletin of the IEEE Computer Society Technical Committee on Data Engineering 36, 4 ( 2015 ). Paris Carbone, Asterios Katsifodimos, Stephan Ewen, Volker Markl, Seif Haridi, and Kostas Tzoumas. 2015. Apache flink: Stream and batch processing in a single engine. Bulletin of the IEEE Computer Society Technical Committee on Data Engineering 36, 4 (2015)."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.14778\/2735496.2735503"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.14778\/3352063.3352121"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2882903.2904441"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2940716.2940798"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2491245"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1327452.1327492"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094114.2094126"},{"key":"e_1_3_2_2_35_1","volume-title":"Driver surge pricing. arXiv preprint arXiv:1905.07544","author":"Garg Nikhil","year":"2019","unstructured":"Nikhil Garg and Hamid Nazerzadeh . 2019. Driver surge pricing. arXiv preprint arXiv:1905.07544 ( 2019 ). Nikhil Garg and Hamid Nazerzadeh. 2019. Driver surge pricing. arXiv preprint arXiv:1905.07544 (2019)."},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/564585.564601"},{"key":"e_1_3_2_2_37_1","unstructured":"Naveen Cherukuri Haohui Mai Bill Liu. Introducing AthenaX Uber Engineering's Open Source Streaming Analytics Platform. https:\/\/eng.uber.com\/athenax\/  Naveen Cherukuri Haohui Mai Bill Liu. Introducing AthenaX Uber Engineering's Open Source Streaming Analytics Platform. https:\/\/eng.uber.com\/athenax\/"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415535"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183713.3190661"},{"key":"e_1_3_2_2_40_1","volume-title":"Del Balso Jeremy Hermann. Meet Michelangelo: Uber's Machine Learning Platform. https:\/\/eng.uber.com\/michelangelo-machine-learning-platform\/","author":"Mike","unstructured":"Mike Del Balso Jeremy Hermann. Meet Michelangelo: Uber's Machine Learning Platform. https:\/\/eng.uber.com\/michelangelo-machine-learning-platform\/ Mike Del Balso Jeremy Hermann. Meet Michelangelo: Uber's Machine Learning Platform. https:\/\/eng.uber.com\/michelangelo-machine-learning-platform\/"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415550"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/IC2EW.2016.56"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2011.5767867"},{"key":"e_1_3_2_2_44_1","unstructured":"Alex Kira. 2019. Managing Uber's Data Workflows at Scale. https:\/\/eng.uber.com\/managing-data-workflows-at-scale\/  Alex Kira. 2019. Managing Uber's Data Workflows at Scale. https:\/\/eng.uber.com\/managing-data-workflows-at-scale\/"},{"key":"e_1_3_2_2_45_1","first-page":"1","article-title":"Kafka: A distributed messaging system for log processing","volume":"11","author":"Kreps Jay","year":"2011","unstructured":"Jay Kreps , Neha Narkhede , Jun Rao , 2011 . Kafka: A distributed messaging system for log processing . In Proceedings of the NetDB , Vol. 11. 1 -- 7 . Jay Kreps, Neha Narkhede, Jun Rao, et al. 2011. Kafka: A distributed messaging system for log processing. In Proceedings of the NetDB, Vol. 11. 1--7.","journal-title":"Proceedings of the NetDB"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723372.2742788"},{"key":"e_1_3_2_2_47_1","unstructured":"Roman Leventov. Comparison of the Open Source OLAP Systems for Big Data: Click- House Druid and Pinot. https:\/\/leventov.medium.com\/comparison-of-the-opensource-olap-systems-for-big-data-clickhouse-druid-and-pinot-8e042a5ed1c7  Roman Leventov. Comparison of the Open Source OLAP Systems for Big Data: Click- House Druid and Pinot. https:\/\/leventov.medium.com\/comparison-of-the-opensource-olap-systems-for-big-data-clickhouse-druid-and-pinot-8e042a5ed1c7"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3035918.3035959"},{"key":"e_1_3_2_2_49_1","volume-title":"Big Data: Principles and best practices of scalable real-time data systems.","author":"Marz Nathan","year":"2015","unstructured":"Nathan Marz and James Warren . 2015 . Big Data: Principles and best practices of scalable real-time data systems. New York; Manning Publications Co. Nathan Marz and James Warren. 2015. Big Data: Principles and best practices of scalable real-time data systems. New York; Manning Publications Co."},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415568"},{"key":"e_1_3_2_2_51_1","unstructured":"Mayank Bansal Min Cai. Peloton: Uber's unified resource scheduler for diverse cluster workloads. https:\/\/eng.uber.com\/resource-scheduler-cluster-managementpeloton\/  Mayank Bansal Min Cai. Peloton: Uber's unified resource scheduler for diverse cluster workloads. https:\/\/eng.uber.com\/resource-scheduler-cluster-managementpeloton\/"},{"key":"e_1_3_2_2_52_1","unstructured":"Roshan Naik. Moving from Lambda and Kappa Architectures to Kappa+ at Uber. https:\/\/sf-2019.flink-forward.org\/conference-program#moving-fromlambda-and-kappa--architectures-to-kappa--at-uber  Roshan Naik. Moving from Lambda and Kappa Architectures to Kappa+ at Uber. https:\/\/sf-2019.flink-forward.org\/conference-program#moving-fromlambda-and-kappa--architectures-to-kappa--at-uber"},{"key":"e_1_3_2_2_53_1","unstructured":"Chetan Narain. Introducing UberEATS Restaurant Manager. https:\/\/www.uber.com\/newsroom\/eats-restaurant-manager\/  Chetan Narain. Introducing UberEATS Restaurant Manager. https:\/\/www.uber.com\/newsroom\/eats-restaurant-manager\/"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415547"},{"key":"e_1_3_2_2_55_1","volume-title":"Technology Conference on Performance Evaluation and Benchmarking. Springer, 97--112","author":"Psaroudakis Iraklis","year":"2014","unstructured":"Iraklis Psaroudakis , Florian Wolf , Norman May , Thomas Neumann , Alexander B\u00f6hm , Anastasia Ailamaki , and Kai-Uwe Sattler . 2014 . Scaling up mixed workloads: a battle of data freshness, flexibility, and scheduling . In Technology Conference on Performance Evaluation and Benchmarking. Springer, 97--112 . Iraklis Psaroudakis, Florian Wolf, Norman May, Thomas Neumann, Alexander B\u00f6hm, Anastasia Ailamaki, and Kai-Uwe Sattler. 2014. Scaling up mixed workloads: a battle of data freshness, flexibility, and scheduling. In Technology Conference on Performance Evaluation and Benchmarking. Springer, 97--112."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3328905.3338224"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2019.00196"},{"key":"e_1_3_2_2_58_1","unstructured":"Jeff Shute Mircea Oancea Stephan Ellner Ben Handy Eric Rollins Bart Samwel Radek Vingralek Chad Whipkey Xin Chen Beat Jegerlehner etal 2012. F1-the fault-tolerant distributed rdbms supporting google's ad business. (2012).  Jeff Shute Mircea Oancea Stephan Ellner Ben Handy Eric Rollins Bart Samwel Radek Vingralek Chad Whipkey Xin Chen Beat Jegerlehner et al. 2012. F1-the fault-tolerant distributed rdbms supporting google's ad business. (2012)."},{"key":"e_1_3_2_2_59_1","unstructured":"Chinmay Soman. uReplicator: Uber Engineering's Robust Apache Kafka Replicator. https:\/\/eng.uber.com\/ureplicator-apache-kafka-replicator\/  Chinmay Soman. uReplicator: Uber Engineering's Robust Apache Kafka Replicator. https:\/\/eng.uber.com\/ureplicator-apache-kafka-replicator\/"},{"key":"e_1_3_2_2_60_1","unstructured":"Sandeep Karmakar Sriharsha Chintalapani. No Code Workflow Orchestrator for Building Batch Streaming Pipelines at Scale.  Sandeep Karmakar Sriharsha Chintalapani. No Code Workflow Orchestrator for Building Batch Streaming Pipelines at Scale."},{"key":"e_1_3_2_2_61_1","unstructured":"Pradeep Venkata. Real-time Security Insights: Apache Pinot at Confluera. https:\/\/medium.com\/confluera-engineering\/real-time-security-insightsapache- pinot-at-confluera-a6e5f401ff02  Pradeep Venkata. Real-time Security Insights: Apache Pinot at Confluera. https:\/\/medium.com\/confluera-engineering\/real-time-security-insightsapache- pinot-at-confluera-a6e5f401ff02"},{"key":"e_1_3_2_2_62_1","unstructured":"Haibo Wang. 2020. Engineering SQL Support on Apache Pinot at Uber. https:\/\/eng.uber.com\/engineering-sql-support-on-apache-pinot\/  Haibo Wang. 2020. Engineering SQL Support on Apache Pinot at Uber. https:\/\/eng.uber.com\/engineering-sql-support-on-apache-pinot\/"},{"key":"e_1_3_2_2_63_1","unstructured":"Ning Xia. Building Reliable Reprocessing and Dead Letter Queues with Apache Kafka. https:\/\/eng.uber.com\/reliable-reprocessing\/  Ning Xia. Building Reliable Reprocessing and Dead Letter Queues with Apache Kafka. https:\/\/eng.uber.com\/reliable-reprocessing\/"},{"key":"e_1_3_2_2_64_1","unstructured":"Ankur Bansal Xiaobing Li. Introducing Chaperone: How Uber Engineering Audits Apache Kafka End-to-End. https:\/\/eng.uber.com\/chaperone-audit-kafkamessages\/  Ankur Bansal Xiaobing Li. Introducing Chaperone: How Uber Engineering Audits Apache Kafka End-to-End. https:\/\/eng.uber.com\/chaperone-audit-kafkamessages\/"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/2588555.2595631"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.14778\/3415478.3415553"},{"key":"e_1_3_2_2_67_1","unstructured":"Ting Chen Chinmay Soman Yupeng Fu Girish Baliga. Operating Apache Pinot @ Uber Scale. https:\/\/eng.uber.com\/operating-apache-pinot\/  Ting Chen Chinmay Soman Yupeng Fu Girish Baliga. Operating Apache Pinot @ Uber Scale. https:\/\/eng.uber.com\/operating-apache-pinot\/"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522737"}],"event":{"name":"SIGMOD\/PODS '21: International Conference on Management of Data","location":"Virtual Event China","acronym":"SIGMOD\/PODS '21","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 2021 International Conference on Management of Data"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3448016.3457552","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3448016.3457552","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T21:25:04Z","timestamp":1750195504000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3448016.3457552"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6,9]]},"references-count":68,"alternative-id":["10.1145\/3448016.3457552","10.1145\/3448016"],"URL":"https:\/\/doi.org\/10.1145\/3448016.3457552","relation":{},"subject":[],"published":{"date-parts":[[2021,6,9]]},"assertion":[{"value":"2021-06-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}