{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,10]],"date-time":"2026-06-10T07:53:33Z","timestamp":1781078013152,"version":"3.54.1"},"publisher-location":"New York, NY, USA","reference-count":46,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,6,9]],"date-time":"2022-06-09T00:00:00Z","timestamp":1654732800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,6,9]]},"DOI":"10.1145\/3519935.3519973","type":"proceedings-article","created":{"date-parts":[[2022,6,10]],"date-time":"2022-06-10T15:29:32Z","timestamp":1654874972000},"page":"818-831","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":23,"title":["Reproducibility in learning"],"prefix":"10.1145","author":[{"given":"Russell","family":"Impagliazzo","sequence":"first","affiliation":[{"name":"University of California at San Diego, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rex","family":"Lei","sequence":"additional","affiliation":[{"name":"University of California at San Diego, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Toniann","family":"Pitassi","sequence":"additional","affiliation":[{"name":"Columbia University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jessica","family":"Sorrell","sequence":"additional","affiliation":[{"name":"University of California at San Diego, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,6,10]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3158146"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1038\/533452a"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611973068.129"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/3434289"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2676726.2677000"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2897518.2897566"},{"key":"e_1_3_2_1_7_1","volume-title":"Raise standards for preclinical cancer research. Nature (London), 483, 7391","author":"Glenn Begley C.","year":"2012","unstructured":"C. Glenn Begley and Lee M Ellis . 2012. Raise standards for preclinical cancer research. Nature (London), 483, 7391 ( 2012 ), 531\u2013533. issn:0028-0836 C. Glenn Begley and Lee M Ellis. 2012. Raise standards for preclinical cancer research. Nature (London), 483, 7391 (2012), 531\u2013533. issn:0028-0836"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2422436.2422450"},{"key":"e_1_3_2_1_9_1","unstructured":"O. Bousquet and A. Elisseeff. 2002. Stability and generalization. J. Mach. Learn. Res. 2 (2002).  O. Bousquet and A. Elisseeff. 2002. Stability and generalization. J. Mach. Learn. Res. 2 (2002)."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the Twenty-First Annual Conference on Neural Information Processing Systems (NIPS).","author":"Bradley J.","unstructured":"J. Bradley and R. Schapire . 2007. FilterBoost: Regression and Classification on Large Datasets . In Proceedings of the Twenty-First Annual Conference on Neural Information Processing Systems (NIPS). J. Bradley and R. Schapire. 2007. FilterBoost: Regression and Classification on Large Datasets. In Proceedings of the Twenty-First Annual Conference on Neural Information Processing Systems (NIPS)."},{"key":"e_1_3_2_1_11_1","volume-title":"Conference on Learning Theory, COLT 2020","author":"Bun Mark","year":"2020","unstructured":"Mark Bun , Marco Leandro Carmosino , and Jessica Sorrell . 2020 . Efficient, Noise-Tolerant, and Private Learning via Boosting . In Conference on Learning Theory, COLT 2020 , 9-12 July 2020, Virtual Event [Graz, Austria], Jacob D. Abernethy and Shivani Agarwal (Eds.) (Proceedings of Machine Learning Research). PMLR. http:\/\/proceedings.mlr.press\/v125\/bun20a.html Mark Bun, Marco Leandro Carmosino, and Jessica Sorrell. 2020. Efficient, Noise-Tolerant, and Private Learning via Boosting. In Conference on Learning Theory, COLT 2020, 9-12 July 2020, Virtual Event [Graz, Austria], Jacob D. Abernethy and Shivani Agarwal (Eds.) (Proceedings of Machine Learning Research). PMLR. http:\/\/proceedings.mlr.press\/v125\/bun20a.html"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/FOCS46700.2020.00044"},{"key":"e_1_3_2_1_13_1","volume-title":"Differentially Private Release and Learning of Threshold Functions. 2015 IEEE 56th Annual Symposium on Foundations of Computer Science, 634\u2013649","author":"Bun Mark","unstructured":"Mark Bun , Kobbi Nissim , Uri Stemmer , and Salil P. Vadhan . 2015 . Differentially Private Release and Learning of Threshold Functions. 2015 IEEE 56th Annual Symposium on Foundations of Computer Science, 634\u2013649 . Mark Bun, Kobbi Nissim, Uri Stemmer, and Salil P. Vadhan. 2015. Differentially Private Release and Learning of Threshold Functions. 2015 IEEE 56th Annual Symposium on Foundations of Computer Science, 634\u2013649."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"L. Devroye and T. J. Wagner. 1979. Distribution-free inequalities for the deleted and holdout error estimates. IEEE Trans. Inform. Theory 25 (1979).  L. Devroye and T. J. Wagner. 1979. Distribution-free inequalities for the deleted and holdout error estimates. IEEE Trans. Inform. Theory 25 (1979).","DOI":"10.1109\/TIT.1979.1056032"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"crossref","unstructured":"L. Devroye and T. J. Wagner. 1979. Distribution-free performance bounds for potential function fules. IEEE Trans. Inform. Theory 25 (1979).  L. Devroye and T. J. Wagner. 1979. Distribution-free performance bounds for potential function fules. IEEE Trans. Inform. Theory 25 (1979).","DOI":"10.1109\/TIT.1979.1056087"},{"key":"e_1_3_2_1_16_1","volume-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems -","volume":"2","author":"Dwork Cynthia","year":"2015","unstructured":"Cynthia Dwork , Vitaly Feldman , Moritz Hardt , Toniann Pitassi , Omer Reingold , and Aaron Roth . 2015 . Generalization in Adaptive Data Analysis and Holdout Reuse . In Proceedings of the 28th International Conference on Neural Information Processing Systems - Volume 2 (NIPS\u201915). MIT Press, Cambridge, MA, USA. 23502358. Cynthia Dwork, Vitaly Feldman, Moritz Hardt, Toniann Pitassi, Omer Reingold, and Aaron Roth. 2015. Generalization in Adaptive Data Analysis and Holdout Reuse. In Proceedings of the 28th International Conference on Neural Information Processing Systems - Volume 2 (NIPS\u201915). MIT Press, Cambridge, MA, USA. 23502358."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2746539.2746580"},{"key":"e_1_3_2_1_18_1","volume-title":"A General Characterization of the Statistical Query Complexity. CoRR, abs\/1608.02198","author":"Feldman V.","year":"2016","unstructured":"V. Feldman . 2016. A General Characterization of the Statistical Query Complexity. CoRR, abs\/1608.02198 ( 2016 ). V. Feldman. 2016. A General Characterization of the Statistical Query Complexity. CoRR, abs\/1608.02198 (2016)."},{"key":"e_1_3_2_1_19_1","volume-title":"Proceedings of The 27th Conference on Learning Theory, COLT","author":"Feldman Vitaly","year":"2014","unstructured":"Vitaly Feldman and David Xiao . 2014. Sample Complexity Bounds on Differentially Private Learning via Communication Complexity . In Proceedings of The 27th Conference on Learning Theory, COLT 2014 , Barcelona, Spain, June 13-15, 2014, Maria-Florina Balcan, Vitaly Feldman, and Csaba Szepesv\u00e1ri (Eds.) (JMLR Workshop and Conference Proceedings , Vol. 35). JMLR.org, 1000\u2013 1019 . http:\/\/proceedings.mlr.press\/v35\/feldman14b.html Vitaly Feldman and David Xiao. 2014. Sample Complexity Bounds on Differentially Private Learning via Communication Complexity. In Proceedings of The 27th Conference on Learning Theory, COLT 2014, Barcelona, Spain, June 13-15, 2014, Maria-Florina Balcan, Vitaly Feldman, and Csaba Szepesv\u00e1ri (Eds.) (JMLR Workshop and Conference Proceedings, Vol. 35). JMLR.org, 1000\u20131019. http:\/\/proceedings.mlr.press\/v35\/feldman14b.html"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2603088.2603097"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2429069.2429113"},{"key":"e_1_3_2_1_22_1","volume-title":"47th International Colloquium on Automata, Languages, and Programming, ICALP","author":"Gaboardi Marco","year":"2020","unstructured":"Marco Gaboardi , Kobbi Nissim , and David Purser . 2020. The Complexity of Verifying Loop-Free Programs as Differentially Private . In 47th International Colloquium on Automata, Languages, and Programming, ICALP 2020 , July 8-11, 2020, Saarbr\u00fccken, Germany (Virtual Conference), Artur Czumaj, Anuj Dawar, and Emanuela Merelli (Eds.) (LIPIcs , Vol. 168). Schloss Dagstuhl - Leibniz-Zentrum f\u00fcr Informatik, 129:1\u2013129: 17 . Marco Gaboardi, Kobbi Nissim, and David Purser. 2020. The Complexity of Verifying Loop-Free Programs as Differentially Private. In 47th International Colloquium on Automata, Languages, and Programming, ICALP 2020, July 8-11, 2020, Saarbr\u00fccken, Germany (Virtual Conference), Artur Czumaj, Anuj Dawar, and Emanuela Merelli (Eds.) (LIPIcs, Vol. 168). Schloss Dagstuhl - Leibniz-Zentrum f\u00fcr Informatik, 129:1\u2013129:17."},{"key":"e_1_3_2_1_23_1","volume-title":"Proc. 35th Annual Conference on Neural Information Processing Systems (NeurIPS)","author":"Ghazi Badih","year":"2021","unstructured":"Badih Ghazi , Ravi Kumar , and Pasin Manurangsi . 2021 . User-Level Differentially Private Learning via Correlated Sampling . In Proc. 35th Annual Conference on Neural Information Processing Systems (NeurIPS) Badih Ghazi, Ravi Kumar, and Pasin Manurangsi. 2021. User-Level Differentially Private Learning via Correlated Sampling. In Proc. 35th Annual Conference on Neural Information Processing Systems (NeurIPS)"},{"key":"e_1_3_2_1_24_1","volume-title":"Gilbert and Audra McMillan","author":"Anna","year":"2018","unstructured":"Anna C. Gilbert and Audra McMillan . 2018 . Property Testing for Differential Privacy. CoRR , abs\/1806.06427 (2018), arXiv:1806.06427. arxiv:1806.06427 Anna C. Gilbert and Audra McMillan. 2018. Property Testing for Differential Privacy. CoRR, abs\/1806.06427 (2018), arXiv:1806.06427. arxiv:1806.06427"},{"key":"e_1_3_2_1_25_1","first-page":"12","article-title":"Multi-pseudodeterministic algorithms","volume":"26","author":"Goldreich Oded","year":"2019","unstructured":"Oded Goldreich . 2019 . Multi-pseudodeterministic algorithms . Electron. Colloquium Comput. Complex. , 26 (2019), 12 . https:\/\/eccc.weizmann.ac.il\/report\/2019\/012 Oded Goldreich. 2019. Multi-pseudodeterministic algorithms. Electron. Colloquium Comput. Complex., 26 (2019), 12. https:\/\/eccc.weizmann.ac.il\/report\/2019\/012","journal-title":"Electron. Colloquium Comput. Complex."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611975482.38"},{"key":"e_1_3_2_1_27_1","volume-title":"Deep Reinforcement Learning that Matters. arxiv:1709.06560 cite arxiv:1709.06560Comment: Accepted to the Thirthy-Second AAAI Conference On Artificial Intelligence (AAAI)","author":"Henderson Peter","year":"2018","unstructured":"Peter Henderson , Riashat Islam , Philip Bachman , Joelle Pineau , Doina Precup , and David Meger . 2017. Deep Reinforcement Learning that Matters. arxiv:1709.06560 cite arxiv:1709.06560Comment: Accepted to the Thirthy-Second AAAI Conference On Artificial Intelligence (AAAI) , 2018 Peter Henderson, Riashat Islam, Philip Bachman, Joelle Pineau, Doina Precup, and David Meger. 2017. Deep Reinforcement Learning that Matters. arxiv:1709.06560 cite arxiv:1709.06560Comment: Accepted to the Thirthy-Second AAAI Conference On Artificial Intelligence (AAAI), 2018"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Russell Impagliazzo Rex Lei and Jessica Sorrell. 2021. Reproducibility in Learning. In Theory and Practice of Differential Privacy.  Russell Impagliazzo Rex Lei and Jessica Sorrell. 2021. Reproducibility in Learning. In Theory and Practice of Differential Privacy.","DOI":"10.1145\/3519935.3519973"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_1_30_1","volume-title":"Reproducibility of Benchmarked Deep Reinforcement Learning Tasks for Continuous Control. CoRR, abs\/1708.04133","author":"Islam Riashat","year":"2017","unstructured":"Riashat Islam , Peter Henderson , Maziar Gomrokchi , and Doina Precup . 2017. Reproducibility of Benchmarked Deep Reinforcement Learning Tasks for Continuous Control. CoRR, abs\/1708.04133 ( 2017 ), arxiv:1708.04133. arxiv:1708.04133 Riashat Islam, Peter Henderson, Maziar Gomrokchi, and Doina Precup. 2017. Reproducibility of Benchmarked Deep Reinforcement Learning Tasks for Continuous Control. CoRR, abs\/1708.04133 (2017), arxiv:1708.04133. arxiv:1708.04133"},{"key":"e_1_3_2_1_31_1","volume-title":"Privately Learning Thresholds: Closing the Exponential Gap. In Conference on Learning Theory, COLT 2020","volume":"2285","author":"Kaplan Haim","year":"2020","unstructured":"Haim Kaplan , Katrina Ligett , Yishay Mansour , Moni Naor , and Uri Stemmer . 2020 . Privately Learning Thresholds: Closing the Exponential Gap. In Conference on Learning Theory, COLT 2020 , 9-12 July 2020, Virtual Event [Graz, Austria], Jacob D. Abernethy and Shivani Agarwal (Eds.) (Proceedings of Machine Learning Research , Vol. 125). PMLR, 2263\u2013 2285 . http:\/\/proceedings.mlr.press\/v125\/kaplan20a.html Haim Kaplan, Katrina Ligett, Yishay Mansour, Moni Naor, and Uri Stemmer. 2020. Privately Learning Thresholds: Closing the Exponential Gap. In Conference on Learning Theory, COLT 2020, 9-12 July 2020, Virtual Event [Graz, Austria], Jacob D. Abernethy and Shivani Agarwal (Eds.) (Proceedings of Machine Learning Research, Vol. 125). PMLR, 2263\u20132285. http:\/\/proceedings.mlr.press\/v125\/kaplan20a.html"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/293347.293351"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"crossref","unstructured":"M. J. Kearns and D. Ron. 1999. Algorithmic stability and sanity-check bounds for leave-one-out cross-validation. Neural Comput. 11 (1999).  M. J. Kearns and D. Ron. 1999. Algorithmic stability and sanity-check bounds for leave-one-out cross-validation. Neural Comput. 11 (1999).","DOI":"10.1162\/089976699300016304"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2347736.2347757"},{"key":"e_1_3_2_1_35_1","volume-title":"Advances in Neural Information Processing Systems","author":"Lucic Mario","year":"2018","unstructured":"Mario Lucic , Karol Kurach , Marcin Michalski , Sylvain Gelly , and Olivier Bousquet . 2018. Are GANs Created Equal? A Large-Scale Study . In Advances in Neural Information Processing Systems , S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett (Eds.). 31, Curran Associates, Inc. . https:\/\/proceedings.neurips.cc\/paper\/ 2018 \/file\/e46de7e1bcaaced9a54f1e9d0d2f800d-Paper.pdf Mario Lucic, Karol Kurach, Marcin Michalski, Sylvain Gelly, and Olivier Bousquet. 2018. Are GANs Created Equal? A Large-Scale Study. In Advances in Neural Information Processing Systems, S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett (Eds.). 31, Curran Associates, Inc.. https:\/\/proceedings.neurips.cc\/paper\/2018\/file\/e46de7e1bcaaced9a54f1e9d0d2f800d-Paper.pdf"},{"key":"e_1_3_2_1_36_1","unstructured":"Nima Mousavi. [n. d.]. How tight is Chernoff Bound? Unpublished manuscript  Nima Mousavi. [n. d.]. How tight is Chernoff Bound? Unpublished manuscript"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2741948.2741978"},{"key":"e_1_3_2_1_38_1","unstructured":"Joelle Pineau Philippe Vincent-Lamarre Koustuv Sinha Vincent Larivi\u00e8re Alina Beygelzimer Florence d\u2019Alch\u00e9 Buc Emily Fox and Hugo Larochelle. 2020. Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program). arXiv preprint arXiv:2003.12206.  Joelle Pineau Philippe Vincent-Lamarre Koustuv Sinha Vincent Larivi\u00e8re Alina Beygelzimer Florence d\u2019Alch\u00e9 Buc Emily Fox and Hugo Larochelle. 2020. Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program). arXiv preprint arXiv:2003.12206."},{"key":"e_1_3_2_1_39_1","volume-title":"Proceeding of the 15th ACM SIGPLAN ICFP 2010","author":"Reed Jason","year":"2010","unstructured":"Jason Reed and Benjamin C. Pierce . 2010. Distance makes the types grow stronger: a calculus for differential privacy . In Proceeding of the 15th ACM SIGPLAN ICFP 2010 , Baltimore, Maryland, USA , September 27-29, 2010 , Paul Hudak and Stephanie Weirich (Eds.). ACM, 157\u2013168. Jason Reed and Benjamin C. Pierce. 2010. Distance makes the types grow stronger: a calculus for differential privacy. In Proceeding of the 15th ACM SIGPLAN ICFP 2010, Baltimore, Maryland, USA, September 27-29, 2010, Paul Hudak and Stephanie Weirich (Eds.). ACM, 157\u2013168."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1162\/153244304773936072"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1023\/A:1013633619373"},{"key":"e_1_3_2_1_42_1","unstructured":"S. Shalev-Schwartz O. Shamir N. Srebro and K. Sridharan. 2010. Learnability stability and uniform convergence. J. Mach. Learn. Res. 11 (2010).  S. Shalev-Schwartz O. Shamir N. Srebro and K. Sridharan. 2010. Learnability stability and uniform convergence. J. Mach. Learn. Res. 11 (2010)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/800057.808710"},{"key":"e_1_3_2_1_44_1","volume-title":"Chervonenkis","author":"Vapnik V. N.","year":"1971","unstructured":"V. N. Vapnik and A. Ya . Chervonenkis . 1971 . On the Uniform Convergence of Relative Frequencies of Events to their Probabilities. Theory of Probability and its Applications , 16 (1971). V. N. Vapnik and A. Ya. Chervonenkis. 1971. On the Uniform Convergence of Relative Frequencies of Events to their Probabilities. Theory of Probability and its Applications, 16 (1971)."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1561\/2200000008"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3009837.3009884"}],"event":{"name":"STOC '22: 54th Annual ACM SIGACT Symposium on Theory of Computing","location":"Rome Italy","acronym":"STOC '22","sponsor":["SIGACT ACM Special Interest Group on Algorithms and Computation Theory"]},"container-title":["Proceedings of the 54th Annual ACM SIGACT Symposium on Theory of Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3519935.3519973","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3519935.3519973","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T17:49:38Z","timestamp":1750268978000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3519935.3519973"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6,9]]},"references-count":46,"alternative-id":["10.1145\/3519935.3519973","10.1145\/3519935"],"URL":"https:\/\/doi.org\/10.1145\/3519935.3519973","relation":{},"subject":[],"published":{"date-parts":[[2022,6,9]]},"assertion":[{"value":"2022-06-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}