{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:12:32Z","timestamp":1750219952765,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,11,7]],"date-time":"2022-11-07T00:00:00Z","timestamp":1667779200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,11,7]]},"DOI":"10.1145\/3540250.3558941","type":"proceedings-article","created":{"date-parts":[[2022,11,9]],"date-time":"2022-11-09T20:46:22Z","timestamp":1668026782000},"page":"1291-1301","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["Improving ML-based information retrieval software with user-driven functional testing and defect class analysis"],"prefix":"10.1145","author":[{"given":"Junjie","family":"Zhu","sequence":"first","affiliation":[{"name":"Apple, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Teng","family":"Long","sequence":"additional","affiliation":[{"name":"Apple, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Wei","family":"Wang","sequence":"additional","affiliation":[{"name":"Apple, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Atif","family":"Memon","sequence":"additional","affiliation":[{"name":"Apple, USA"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,11,9]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP.2019.00042"},{"key":"e_1_3_2_1_2_1","unstructured":"Houssem Ben Braiek and Foutse Khomh. 2018. On testing machine learning programs. arXiv preprint arXiv:1812.02257. \t\t\t\t  Houssem Ben Braiek and Foutse Khomh. 2018. On testing machine learning programs. arXiv preprint arXiv:1812.02257."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.5555\/205361.205364"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00042"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409768"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3409759"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1327452.1327492"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1561\/2000000039"},{"key":"e_1_3_2_1_9_1","volume-title":"Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916)","author":"Derczynski Leon","year":"2016","unstructured":"Leon Derczynski . 2016 . Complementarity, F-score, and NLP Evaluation . In Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916) . European Language Resources Association (ELRA), Portoro\u017e, Slovenia. 261\u2013266. Leon Derczynski. 2016. Complementarity, F-score, and NLP Evaluation. In Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC\u201916). European Language Resources Association (ELRA), Portoro\u017e, Slovenia. 261\u2013266."},{"key":"e_1_3_2_1_10_1","volume-title":"Proceedings of the NeurIPS 2020 Competition and Demonstration Track, Hugo Jair Escalante and Katja Hofmann (Eds.) (Proceedings of Machine Learning Research","volume":"169","author":"Desmond Michael","year":"2021","unstructured":"Michael Desmond , Evelyn Duesterwald , Kristina Brimijoin , Michelle Brachman , and Qian Pan . 2021 . Semi-Automated Data Labeling . In Proceedings of the NeurIPS 2020 Competition and Demonstration Track, Hugo Jair Escalante and Katja Hofmann (Eds.) (Proceedings of Machine Learning Research , Vol. 133). PMLR, 156\u2013 169 . https:\/\/proceedings.mlr.press\/v133\/desmond21a.html Michael Desmond, Evelyn Duesterwald, Kristina Brimijoin, Michelle Brachman, and Qian Pan. 2021. Semi-Automated Data Labeling. In Proceedings of the NeurIPS 2020 Competition and Demonstration Track, Hugo Jair Escalante and Katja Hofmann (Eds.) (Proceedings of Machine Learning Research, Vol. 133). PMLR, 156\u2013169. https:\/\/proceedings.mlr.press\/v133\/desmond21a.html"},{"key":"e_1_3_2_1_11_1","volume-title":"Deep Learning & Software Engineering: State of Research and Future Directions. CoRR, abs\/2009.08525","author":"Devanbu Prem","year":"2020","unstructured":"Prem Devanbu , Matthew B. Dwyer , Sebastian G. Elbaum , Michael Lowry , Kevin Moran , Denys Poshyvanyk , Baishakhi Ray , Rishabh Singh , and Xiangyu Zhang . 2020. Deep Learning & Software Engineering: State of Research and Future Directions. CoRR, abs\/2009.08525 ( 2020 ), https:\/\/doi.org\/10.48550\/ARXIV.2009.08525 10.48550\/ARXIV.2009.08525 Prem Devanbu, Matthew B. Dwyer, Sebastian G. Elbaum, Michael Lowry, Kevin Moran, Denys Poshyvanyk, Baishakhi Ray, Rishabh Singh, and Xiangyu Zhang. 2020. Deep Learning & Software Engineering: State of Research and Future Directions. CoRR, abs\/2009.08525 (2020), https:\/\/doi.org\/10.48550\/ARXIV.2009.08525"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2013.6693111"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390392"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3395363.3397366"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-84858-7"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1963405.1963423"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468565"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.33.2.25"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3428270"},{"volume-title":"Mining of massive data sets","author":"Leskovec Jure","key":"e_1_3_2_1_21_1","unstructured":"Jure Leskovec , Anand Rajaraman , and Jeffrey David Ullman . 2020. Mining of massive data sets . Cambridge university press . Jure Leskovec, Anand Rajaraman, and Jeffrey David Ullman. 2020. Mining of massive data sets. Cambridge university press."},{"key":"e_1_3_2_1_22_1","volume-title":"Characterizing and Detecting Mismatch in Machine-Learning-Enabled Systems. CoRR, abs\/2103.14101","author":"Lewis Grace A.","year":"2021","unstructured":"Grace A. Lewis , Stephany Bellomo , and Ipek Ozkaya . 2021. Characterizing and Detecting Mismatch in Machine-Learning-Enabled Systems. CoRR, abs\/2103.14101 ( 2021 ), https:\/\/doi.org\/10.48550\/arXiv.2103.14101 10.48550\/arXiv.2103.14101 Grace A. Lewis, Stephany Bellomo, and Ipek Ozkaya. 2021. Characterizing and Detecting Mismatch in Machine-Learning-Enabled Systems. CoRR, abs\/2103.14101 (2021), https:\/\/doi.org\/10.48550\/arXiv.2103.14101"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2020.106368"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest.2019.00010"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSTW.2018.00061"},{"volume-title":"The art of software testing","author":"Myers Glenford J","key":"e_1_3_2_1_26_1","unstructured":"Glenford J Myers , Corey Sandler , and Tom Badgett . 2011. The art of software testing . John Wiley & Sons . Glenford J Myers, Corey Sandler, and Tom Badgett. 2011. The art of software testing. John Wiley & Sons."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1017\/S0269888907001282"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1075\/li.30.1.03nad"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415242"},{"volume-title":"Conversion optimization: The art and science of converting prospects to customers. \" O\u2019Reilly Media","author":"Saleh Khalid","key":"e_1_3_2_1_30_1","unstructured":"Khalid Saleh and Ayat Shukairy . 2010. Conversion optimization: The art and science of converting prospects to customers. \" O\u2019Reilly Media , Inc .\". Khalid Saleh and Ayat Shukairy. 2010. Conversion optimization: The art and science of converting prospects to customers. \" O\u2019Reilly Media, Inc.\"."},{"key":"e_1_3_2_1_31_1","volume-title":"Engineering Reliable Deep Learning Systems. CoRR, abs\/1910.12582","author":"Santhanam P.","year":"2019","unstructured":"P. Santhanam , Eitan Farchi , and Victor Pankratius . 2019. Engineering Reliable Deep Learning Systems. CoRR, abs\/1910.12582 ( 2019 ), https:\/\/doi.org\/10.48550\/ARXIV.1910.12582 10.48550\/ARXIV.1910.12582 P. Santhanam, Eitan Farchi, and Victor Pankratius. 2019. Engineering Reliable Deep Learning Systems. CoRR, abs\/1910.12582 (2019), https:\/\/doi.org\/10.48550\/ARXIV.1910.12582"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/215206.215351"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348364"},{"volume-title":"The most powerful way to turn clicks into customers","author":"Siroker Dan","key":"e_1_3_2_1_34_1","unstructured":"Dan Siroker and Pete Koomen . 2013. A\/ B testing : The most powerful way to turn clicks into customers . John Wiley & Sons . Dan Siroker and Pete Koomen. 2013. A\/B testing: The most powerful way to turn clicks into customers. John Wiley & Sons."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3412841.3442027"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/1146238.1146240"},{"volume-title":"Query Understanding for Search Engines","author":"Wang Xuanhui","key":"e_1_3_2_1_37_1","unstructured":"Xuanhui Wang . 2020. Query Segmentation and Tagging . In Query Understanding for Search Engines . Springer , 43\u201367. Xuanhui Wang. 2020. Query Segmentation and Tagging. In Query Understanding for Search Engines. Springer, 43\u201367."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/IWESEP49350.2019.00017"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1093\/comjnl\/25.4.465"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1189748.1189752"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2019.2962027"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/SANER50967.2021.00081"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-Companion.2019.00131"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2015.2478001"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE-SEIP52600.2021.00049"}],"event":{"name":"ESEC\/FSE '22: 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering","NUS NUS"],"location":"Singapore Singapore","acronym":"ESEC\/FSE '22"},"container-title":["Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3540250.3558941","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3540250.3558941","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:04Z","timestamp":1750182544000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3540250.3558941"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,11,7]]},"references-count":45,"alternative-id":["10.1145\/3540250.3558941","10.1145\/3540250"],"URL":"https:\/\/doi.org\/10.1145\/3540250.3558941","relation":{},"subject":[],"published":{"date-parts":[[2022,11,7]]},"assertion":[{"value":"2022-11-09","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}