{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,9,8]],"date-time":"2025-09-08T05:56:30Z","timestamp":1757310990757,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,7,18]],"date-time":"2023-07-18T00:00:00Z","timestamp":1689638400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,7,19]]},"DOI":"10.1145\/3539618.3591865","type":"proceedings-article","created":{"date-parts":[[2023,7,19]],"date-time":"2023-07-19T00:22:59Z","timestamp":1689726179000},"page":"3415-3420","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":6,"title":["How Well do Offline Metrics Predict Online Performance of Product Ranking Models?"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2565-5831","authenticated-orcid":false,"given":"Xiaojie","family":"Wang","sequence":"first","affiliation":[{"name":"Amazon.com, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8784-4171","authenticated-orcid":false,"given":"Ruoyuan","family":"Gao","sequence":"additional","affiliation":[{"name":"Amazon.com, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-5542-6191","authenticated-orcid":false,"given":"Anoop","family":"Jain","sequence":"additional","affiliation":[{"name":"Amazon.com, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0009-9730-808X","authenticated-orcid":false,"given":"Graham","family":"Edge","sequence":"additional","affiliation":[{"name":"Amazon.com, Seattle, WA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-2156-0128","authenticated-orcid":false,"given":"Sachin","family":"Ahuja","sequence":"additional","affiliation":[{"name":"Amazon.com, Palo Alto, CA, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2023,7,18]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/3308558.3313697"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3209986"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277902"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390347"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210027"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557123"},{"key":"e_1_3_2_2_7_1","volume-title":"Proceedings of SIGIR eCom","volume":"18","author":"Brenner Eliot P","year":"2018","unstructured":"Eliot P Brenner, Jun Zhao, Aliasgar Kutiyanawala, and Z Yan. 2018. End-to-end neural ranking for ecommerce product search. Proceedings of SIGIR eCom, Vol. 18 (2018)."},{"key":"e_1_3_2_2_8_1","unstructured":"C E Buckley and Ellen M Voorhees. 2005. Retrieval system evaluation. (2005)."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380122"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094078"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02294-4"},{"key":"e_1_3_2_2_14_1","volume-title":"Criteria for measures of association. American Sociological Review","author":"Costner Herbert L","year":"1965","unstructured":"Herbert L Costner. 1965. Criteria for measures of association. American Sociological Review (1965), 341--353."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3488560.3498420"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3511808.3557624"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.24648"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3394486.3403336"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462917"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331651.3331655"},{"key":"e_1_3_2_2_21_1","volume-title":"Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems (TOIS)","author":"J\u00e4rvelin Kalervo","year":"2002","unstructured":"Kalervo J\u00e4rvelin and Jaana Kek\u00e4l\u00e4inen. 2002. Cumulated gain-based evaluation of IR techniques. ACM Transactions on Information Systems (TOIS) (2002), 422--446."},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/775047.775067"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3097983.3097992"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080838"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/30.1-2.81"},{"key":"e_1_3_2_2_26_1","volume-title":"Did we get it right? Predicting query performance in e-commerce search. arXiv preprint arXiv:1808.00239","author":"Kumar Rohan","year":"2018","unstructured":"Rohan Kumar, Mohit Kumar, Neil Shah, and Christos Faloutsos. 2018. Did we get it right? Predicting query performance in e-commerce search. arXiv preprint arXiv:1808.00239 (2018)."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3460231.3473899"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911507"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3052768"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3531924"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_2_33_1","volume-title":"Two-sided confidence intervals for the single proportion: comparison of seven methods. Statistics in medicine","author":"Newcombe Robert G","year":"1998","unstructured":"Robert G Newcombe. 1998. Two-sided confidence intervals for the single proportion: comparison of seven methods. Statistics in medicine, Vol. 17, 8 (1998), 857--872."},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/1458082.1458092"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187904"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484031"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331215"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148176"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3038912.3052568"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911497"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159732"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-93040-4_47"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2017.2729559"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1927.10502953"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3209993"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.aiopen.2021.02.003"},{"key":"e_1_3_2_2_50_1","volume-title":"KDD Workshop on Industrial Recommendation Systems.","author":"Zuo Zhen","year":"2020","unstructured":"Zhen Zuo, Lixi Wang, Michinari Momma, Wenbo Wang, Yikai Ni, Jianfeng Lin, and Yi Sun. 2020. A flexible large-scale similar product identification system in e-commerce. In KDD Workshop on Industrial Recommendation Systems."}],"event":{"name":"SIGIR '23: The 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Taipei Taiwan","acronym":"SIGIR '23"},"container-title":["Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591865","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3539618.3591865","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:37:59Z","timestamp":1750178279000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3539618.3591865"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,18]]},"references-count":50,"alternative-id":["10.1145\/3539618.3591865","10.1145\/3539618"],"URL":"https:\/\/doi.org\/10.1145\/3539618.3591865","relation":{},"subject":[],"published":{"date-parts":[[2023,7,18]]},"assertion":[{"value":"2023-07-18","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}