{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,27]],"date-time":"2026-02-27T01:47:45Z","timestamp":1772156865069,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":73,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,3,22]],"date-time":"2022-03-22T00:00:00Z","timestamp":1647907200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2022,3,22]]},"DOI":"10.1145\/3490099.3511134","type":"proceedings-article","created":{"date-parts":[[2022,3,21]],"date-time":"2022-03-21T23:24:13Z","timestamp":1647905053000},"page":"566-583","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Trade-offs in Sampling and Search for Early-stage Interactive Text Classification"],"prefix":"10.1145","author":[{"given":"Zachary","family":"Levonian","sequence":"first","affiliation":[{"name":"GroupLens, University of Minnesota, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Chia-Jung","family":"Lee","sequence":"additional","affiliation":[{"name":"Amazon, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Vanessa","family":"Murdock","sequence":"additional","affiliation":[{"name":"Amazon, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"F. Maxwell","family":"Harper","sequence":"additional","affiliation":[{"name":"Amazon, United States"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2022,3,22]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1622176.1622222"},{"key":"e_1_3_2_1_2_1","volume-title":"Effective End-User Interaction with Machine Learning. In Twenty-Fifth AAAI Conference on Artificial Intelligence. https:\/\/www.aaai.org\/ocs\/index.php\/AAAI\/AAAI11\/paper\/view\/3736","author":"Amershi Saleema","year":"2011","unstructured":"Saleema Amershi , James Fogarty , Ashish Kapoor , and Desney Tan . 2011 . Effective End-User Interaction with Machine Learning. In Twenty-Fifth AAAI Conference on Artificial Intelligence. https:\/\/www.aaai.org\/ocs\/index.php\/AAAI\/AAAI11\/paper\/view\/3736 Saleema Amershi, James Fogarty, Ashish Kapoor, and Desney Tan. 2011. Effective End-User Interaction with Machine Learning. In Twenty-Fifth AAAI Conference on Artificial Intelligence. https:\/\/www.aaai.org\/ocs\/index.php\/AAAI\/AAAI11\/paper\/view\/3736"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646031"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835859"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1964897.1964906"},{"key":"e_1_3_2_1_6_1","volume-title":"Few-Shot Text Classification with Pre-Trained Word Embeddings and a Human in the Loop. arXiv:1804.02063 [cs] (April","author":"Bailey Katherine","year":"2018","unstructured":"Katherine Bailey and Sunny Chopra . 2018. Few-Shot Text Classification with Pre-Trained Word Embeddings and a Human in the Loop. arXiv:1804.02063 [cs] (April 2018 ). http:\/\/arxiv.org\/abs\/1804.02063 arXiv:1804.02063. Katherine Bailey and Sunny Chopra. 2018. Few-Shot Text Classification with Pre-Trained Word Embeddings and a Human in the Loop. arXiv:1804.02063 [cs] (April 2018). http:\/\/arxiv.org\/abs\/1804.02063 arXiv:1804.02063."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1553374.1553381"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.5555\/3157382.3157472"},{"key":"e_1_3_2_1_9_1","volume-title":"DeepSI: Interactive Deep Learning for Semantic Interaction. In 26th International Conference on Intelligent User Interfaces. ACM, College Station TX USA, 197\u2013207","author":"Bian Yali","year":"2021","unstructured":"Yali Bian and Chris North . 2021 . DeepSI: Interactive Deep Learning for Semantic Interaction. In 26th International Conference on Intelligent User Interfaces. ACM, College Station TX USA, 197\u2013207 . https:\/\/doi.org\/10.1145\/3397481.3450670 10.1145\/3397481.3450670 Yali Bian and Chris North. 2021. DeepSI: Interactive Deep Learning for Semantic Interaction. In 26th International Conference on Intelligent User Interfaces. ACM, College Station TX USA, 197\u2013207. https:\/\/doi.org\/10.1145\/3397481.3450670"},{"key":"e_1_3_2_1_10_1","volume-title":"Evaluation of Interactive Machine Learning Systems. arXiv:1801.07964 [cs] (Jan","author":"Boukhelifa Nadia","year":"2018","unstructured":"Nadia Boukhelifa , Anastasia Bezerianos , and Evelyne Lutton . 2018. Evaluation of Interactive Machine Learning Systems. arXiv:1801.07964 [cs] (Jan . 2018 ). http:\/\/arxiv.org\/abs\/1801.07964 arXiv:1801.07964. Nadia Boukhelifa, Anastasia Bezerianos, and Evelyne Lutton. 2018. Evaluation of Interactive Machine Learning Systems. arXiv:1801.07964 [cs] (Jan. 2018). http:\/\/arxiv.org\/abs\/1801.07964 arXiv:1801.07964."},{"key":"e_1_3_2_1_11_1","volume-title":"Is cross-validation valid for small-sample microarray classification?Bioinformatics 20, 3 (Feb","author":"Braga-Neto M.","year":"2004","unstructured":"Ulisses\u00a0 M. Braga-Neto and Edward\u00a0 R. Dougherty . 2004. Is cross-validation valid for small-sample microarray classification?Bioinformatics 20, 3 (Feb . 2004 ), 374\u2013380. https:\/\/doi.org\/10.1093\/bioinformatics\/btg419 Publisher : Oxford Academic . 10.1093\/bioinformatics Ulisses\u00a0M. Braga-Neto and Edward\u00a0R. Dougherty. 2004. Is cross-validation valid for small-sample microarray classification?Bioinformatics 20, 3 (Feb. 2004), 374\u2013380. https:\/\/doi.org\/10.1093\/bioinformatics\/btg419 Publisher: Oxford Academic."},{"key":"e_1_3_2_1_12_1","volume-title":"Language Models are Few-Shot Learners. arXiv:2005.14165 [cs] (July","author":"Brown B.","year":"2020","unstructured":"Tom\u00a0 B. Brown , Benjamin Mann , Nick Ryder , Melanie Subbiah , Jared Kaplan , Prafulla Dhariwal , Arvind Neelakantan , Pranav Shyam , Girish Sastry , Amanda Askell , Sandhini Agarwal , Ariel Herbert-Voss , Gretchen Krueger , Tom Henighan , Rewon Child , Aditya Ramesh , Daniel\u00a0 M. Ziegler , Jeffrey Wu , Clemens Winter , Christopher Hesse , Mark Chen , Eric Sigler , Mateusz Litwin , Scott Gray , Benjamin Chess , Jack Clark , Christopher Berner , Sam McCandlish , Alec Radford , Ilya Sutskever , and Dario Amodei . 2020. Language Models are Few-Shot Learners. arXiv:2005.14165 [cs] (July 2020 ). http:\/\/arxiv.org\/abs\/2005.14165 arXiv:2005.14165. Tom\u00a0B. Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel\u00a0M. Ziegler, Jeffrey Wu, Clemens Winter, Christopher Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020. Language Models are Few-Shot Learners. arXiv:2005.14165 [cs] (July 2020). http:\/\/arxiv.org\/abs\/2005.14165 arXiv:2005.14165."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"e_1_3_2_1_14_1","volume-title":"Ranked batch-mode active learning. Information Sciences 379 (Feb","author":"Cardoso Thiago","year":"2017","unstructured":"Thiago N.\u00a0C. Cardoso , Rodrigo\u00a0 M. Silva , S\u00e9rgio Canuto , Mirella\u00a0 M. Moro , and Marcos\u00a0 A. Gon\u00e7alves . 2017. Ranked batch-mode active learning. Information Sciences 379 (Feb . 2017 ), 313\u2013337. https:\/\/doi.org\/10.1016\/j.ins.2016.10.037 10.1016\/j.ins.2016.10.037 Thiago N.\u00a0C. Cardoso, Rodrigo\u00a0M. Silva, S\u00e9rgio Canuto, Mirella\u00a0M. Moro, and Marcos\u00a0A. Gon\u00e7alves. 2017. Ranked batch-mode active learning. Information Sciences 379 (Feb. 2017), 313\u2013337. https:\/\/doi.org\/10.1016\/j.ins.2016.10.037"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2856767.2856783"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3185515"},{"key":"e_1_3_2_1_17_1","first-page":"1","article-title":"SMART: An Open Source Data Labeling Platform for Supervised Learning","volume":"20","author":"Chew Rob","year":"2019","unstructured":"Rob Chew , Michael Wenger , Caroline Kery , Jason Nance , Keith Richards , Emily Hadley , and Peter Baumgartner . 2019 . SMART: An Open Source Data Labeling Platform for Supervised Learning . Journal of Machine Learning Research 20 , 82 (2019), 1 \u2013 5 . http:\/\/jmlr.org\/papers\/v20\/18-859.html Rob Chew, Michael Wenger, Caroline Kery, Jason Nance, Keith Richards, Emily Hadley, and Peter Baumgartner. 2019. SMART: An Open Source Data Labeling Platform for Supervised Learning. Journal of Machine Learning Research 20, 82 (2019), 1\u20135. http:\/\/jmlr.org\/papers\/v20\/18-859.html","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450698"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377524"},{"key":"e_1_3_2_1_20_1","first-page":"6","article-title":"Model Selection Techniques","volume":"35","author":"Ding Jie","year":"2018","unstructured":"Jie Ding , Vahid Tarokh , and Yuhong Yang . 2018 . Model Selection Techniques : An Overview. IEEE Signal Process. Mag. 35 , 6 (Nov. 2018), 16\u201334. https:\/\/doi.org\/10.1109\/MSP.2018.2867638 arXiv:1810.09583. 10.1109\/MSP.2018.2867638 Jie Ding, Vahid Tarokh, and Yuhong Yang. 2018. Model Selection Techniques: An Overview. IEEE Signal Process. Mag. 35, 6 (Nov. 2018), 16\u201334. https:\/\/doi.org\/10.1109\/MSP.2018.2867638 arXiv:1810.09583.","journal-title":"An Overview. IEEE Signal Process. Mag."},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3185517"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1983.10477973"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611976236.14"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/1882471.1882479"},{"key":"e_1_3_2_1_26_1","volume-title":"UPAL: Unbiased Pool Based Active Learning. In Artificial Intelligence and Statistics. PMLR, 422\u2013431","author":"Ganti Ravi","year":"2012","unstructured":"Ravi Ganti and Alexander Gray . 2012 . UPAL: Unbiased Pool Based Active Learning. In Artificial Intelligence and Statistics. PMLR, 422\u2013431 . http:\/\/proceedings.mlr.press\/v22\/ganti12.html ISSN: 1938-7228. Ravi Ganti and Alexander Gray. 2012. UPAL: Unbiased Pool Based Active Learning. In Artificial Intelligence and Statistics. PMLR, 422\u2013431. http:\/\/proceedings.mlr.press\/v22\/ganti12.html ISSN: 1938-7228."},{"key":"e_1_3_2_1_27_1","volume-title":"Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence(UAI\u201913)","author":"Ganti Ravi","year":"2013","unstructured":"Ravi Ganti and Alexander\u00a0 G. Gray . 2013 . Building bridges: viewing active learning from the multi-armed bandit lens . In Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence(UAI\u201913) . AUAI Press, Arlington, Virginia, USA, 232\u2013241. https:\/\/arxiv.org\/abs\/1309.6830 Ravi Ganti and Alexander\u00a0G. Gray. 2013. Building bridges: viewing active learning from the multi-armed bandit lens. In Proceedings of the Twenty-Ninth Conference on Uncertainty in Artificial Intelligence(UAI\u201913). AUAI Press, Arlington, Virginia, USA, 232\u2013241. https:\/\/arxiv.org\/abs\/1309.6830"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415219"},{"key":"e_1_3_2_1_29_1","volume-title":"Small-sample precision of ROC-related estimates. Bioinformatics 26, 6 (March","author":"Hanczar Blaise","year":"2010","unstructured":"Blaise Hanczar , Jianping Hua , Chao Sima , John Weinstein , Michael Bittner , and Edward\u00a0 R. Dougherty . 2010. Small-sample precision of ROC-related estimates. Bioinformatics 26, 6 (March 2010 ), 822\u2013830. https:\/\/doi.org\/10.1093\/bioinformatics\/btq037 Publisher : Oxford Academic . 10.1093\/bioinformatics Blaise Hanczar, Jianping Hua, Chao Sima, John Weinstein, Michael Bittner, and Edward\u00a0R. Dougherty. 2010. Small-sample precision of ROC-related estimates. Bioinformatics 26, 6 (March 2010), 822\u2013830. https:\/\/doi.org\/10.1093\/bioinformatics\/btq037 Publisher: Oxford Academic."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-020-2649-2"},{"key":"e_1_3_2_1_31_1","first-page":"1","article-title":"User-driven sampling strategies in image exploitation","volume":"15","author":"Harvey Neal","year":"2014","unstructured":"Neal Harvey and Reid Porter . 2014 . User-driven sampling strategies in image exploitation . Information Visualization 15 , 1 (Nov. 2014), 64\u201374. https:\/\/doi.org\/10.1177\/1473871614557659 10.1177\/1473871614557659 Neal Harvey and Reid Porter. 2014. User-driven sampling strategies in image exploitation. Information Visualization 15, 1 (Nov. 2014), 64\u201374. https:\/\/doi.org\/10.1177\/1473871614557659","journal-title":"Information Visualization"},{"key":"e_1_3_2_1_32_1","volume-title":"Empirically. arXiv:1712.00409 [cs, stat] (Dec.","author":"Hestness Joel","year":"2017","unstructured":"Joel Hestness , Sharan Narang , Newsha Ardalani , Gregory Diamos , Heewoo Jun , Hassan Kianinejad , Md\u00a0 Mostofa\u00a0Ali Patwary , Yang Yang , and Yanqi Zhou . 2017. Deep Learning Scaling is Predictable , Empirically. arXiv:1712.00409 [cs, stat] (Dec. 2017 ). http:\/\/arxiv.org\/abs\/1712.00409 arXiv:1712.00409. Joel Hestness, Sharan Narang, Newsha Ardalani, Gregory Diamos, Heewoo Jun, Hassan Kianinejad, Md\u00a0Mostofa\u00a0Ali Patwary, Yang Yang, and Yanqi Zhou. 2017. Deep Learning Scaling is Predictable, Empirically. arXiv:1712.00409 [cs, stat] (Dec. 2017). http:\/\/arxiv.org\/abs\/1712.00409 arXiv:1712.00409."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2598153.2598189"},{"key":"#cr-split#-e_1_3_2_1_34_1.1","unstructured":"Matthew Honnibal Ines Montani Sofie Van\u00a0Landeghem and Adriane Boyd. 2020. spaCy: Industrial-strength Natural Language Processing in Python. (2020). https:\/\/doi.org\/10.5281\/zenodo.1212303 10.5281\/zenodo.1212303"},{"key":"#cr-split#-e_1_3_2_1_34_1.2","unstructured":"Matthew Honnibal Ines Montani Sofie Van\u00a0Landeghem and Adriane Boyd. 2020. spaCy: Industrial-strength Natural Language Processing in Python. (2020). https:\/\/doi.org\/10.5281\/zenodo.1212303"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2007.55"},{"key":"e_1_3_2_1_36_1","volume-title":"Advances in Neural Information Processing Systems, H.\u00a0Wallach, H.\u00a0Larochelle, A.\u00a0Beygelzimer, F.\u00a0d\\textquotesingle Alch\u00e9-Buc, E.\u00a0Fox, and R.\u00a0Garnett (Eds.). Vol.\u00a032. Curran Associates","author":"Jiang Shali","year":"2019","unstructured":"Shali Jiang , Roman Garnett , and Benjamin Moseley . 2019. Cost Effective Active Search . In Advances in Neural Information Processing Systems, H.\u00a0Wallach, H.\u00a0Larochelle, A.\u00a0Beygelzimer, F.\u00a0d\\textquotesingle Alch\u00e9-Buc, E.\u00a0Fox, and R.\u00a0Garnett (Eds.). Vol.\u00a032. Curran Associates , Inc ., 4880\u20134889. https:\/\/proceedings.neurips.cc\/paper\/ 2019 \/file\/df0e09d6f25a15a815563df9827f48fa-Paper.pdf Shali Jiang, Roman Garnett, and Benjamin Moseley. 2019. Cost Effective Active Search. In Advances in Neural Information Processing Systems, H.\u00a0Wallach, H.\u00a0Larochelle, A.\u00a0Beygelzimer, F.\u00a0d\\textquotesingle Alch\u00e9-Buc, E.\u00a0Fox, and R.\u00a0Garnett (Eds.). Vol.\u00a032. Curran Associates, Inc., 4880\u20134889. https:\/\/proceedings.neurips.cc\/paper\/2019\/file\/df0e09d6f25a15a815563df9827f48fa-Paper.pdf"},{"key":"e_1_3_2_1_37_1","volume-title":"Limitations of Assessing Active Learning Performance at Runtime. arXiv:1901.10338 [cs, stat] (Jan","author":"Kottke Daniel","year":"2019","unstructured":"Daniel Kottke , Jim Schellinger , Denis Huseljic , and Bernhard Sick . 2019. Limitations of Assessing Active Learning Performance at Runtime. arXiv:1901.10338 [cs, stat] (Jan . 2019 ). http:\/\/arxiv.org\/abs\/1901.10338 arXiv:1901.10338. Daniel Kottke, Jim Schellinger, Denis Huseljic, and Bernhard Sick. 2019. Limitations of Assessing Active Learning Performance at Runtime. arXiv:1901.10338 [cs, stat] (Jan. 2019). http:\/\/arxiv.org\/abs\/1901.10338 arXiv:1901.10338."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557238"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609618"},{"key":"e_1_3_2_1_40_1","volume-title":"Proceedings of the 32Nd Annual ACM Symposium on User Interface Software and Technology(UIST \u201919)","author":"Jia-Jun Li Toby","year":"2019","unstructured":"Toby Jia-Jun Li , Marissa Radensky , Justin Jia , Kirielle Singarajah , Tom\u00a0 M. Mitchell , and Brad\u00a0 A. Myers . 2019 . PUMICE: A Multi-Modal Agent That Learns Concepts and Conditionals from Natural Language and Demonstrations . In Proceedings of the 32Nd Annual ACM Symposium on User Interface Software and Technology(UIST \u201919) . ACM, New York, NY, USA, 577\u2013589. https:\/\/doi.org\/10.1145\/3332165.3347899 event-place: New Orleans, LA, USA. 10.1145\/3332165.3347899 Toby Jia-Jun Li, Marissa Radensky, Justin Jia, Kirielle Singarajah, Tom\u00a0M. Mitchell, and Brad\u00a0A. Myers. 2019. PUMICE: A Multi-Modal Agent That Learns Concepts and Conditionals from Natural Language and Demonstrations. In Proceedings of the 32Nd Annual ACM Symposium on User Interface Software and Technology(UIST \u201919). ACM, New York, NY, USA, 577\u2013589. https:\/\/doi.org\/10.1145\/3332165.3347899 event-place: New Orleans, LA, USA."},{"key":"e_1_3_2_1_41_1","unstructured":"Christopher\u00a0H. Lin Mausam and Daniel\u00a0S. Weld. 2018. Active Learning with Unbalanced Classes and Example-Generation Queries. In Sixth AAAI Conference on Human Computation and Crowdsourcing. https:\/\/www.aaai.org\/ocs\/index.php\/HCOMP\/HCOMP18\/paper\/view\/17927  Christopher\u00a0H. Lin Mausam and Daniel\u00a0S. Weld. 2018. Active Learning with Unbalanced Classes and Example-Generation Queries. In Sixth AAAI Conference on Human Computation and Crowdsourcing. https:\/\/www.aaai.org\/ocs\/index.php\/HCOMP\/HCOMP18\/paper\/view\/17927"},{"key":"e_1_3_2_1_42_1","unstructured":"Martin Lindvall and Jesper Molin. 2019. Designing for the Long Tail of Machine Learning. In arXiv:2001.07455 [cs]. http:\/\/arxiv.org\/abs\/2001.07455 arXiv:2001.07455.  Martin Lindvall and Jesper Molin. 2019. Designing for the Long Tail of Machine Learning. In arXiv:2001.07455 [cs]. http:\/\/arxiv.org\/abs\/2001.07455 arXiv:2001.07455."},{"key":"e_1_3_2_1_43_1","volume-title":"Detecting and Correcting for Label Shift with Black Box Predictors. arXiv:1802.03916 [cs, stat] (July","author":"Lipton C.","year":"2018","unstructured":"Zachary\u00a0 C. Lipton , Yu-Xiang Wang , and Alex Smola . 2018. Detecting and Correcting for Label Shift with Black Box Predictors. arXiv:1802.03916 [cs, stat] (July 2018 ). http:\/\/arxiv.org\/abs\/1802.03916 arXiv:1802.03916. Zachary\u00a0C. Lipton, Yu-Xiang Wang, and Alex Smola. 2018. Detecting and Correcting for Label Shift with Black Box Predictors. arXiv:1802.03916 [cs, stat] (July 2018). http:\/\/arxiv.org\/abs\/1802.03916 arXiv:1802.03916."},{"key":"e_1_3_2_1_44_1","volume-title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs] (July","author":"Liu Yinhan","year":"2019","unstructured":"Yinhan Liu , Myle Ott , Naman Goyal , Jingfei Du , Mandar Joshi , Danqi Chen , Omer Levy , Mike Lewis , Luke Zettlemoyer , and Veselin Stoyanov . 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs] (July 2019 ). http:\/\/arxiv.org\/abs\/1907.11692 arXiv:1907.11692. Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arXiv:1907.11692 [cs] (July 2019). http:\/\/arxiv.org\/abs\/1907.11692 arXiv:1907.11692."},{"key":"e_1_3_2_1_45_1","volume-title":"Practical Obstacles to Deploying Active Learning. arXiv:1807.04801 [cs, stat] (July","author":"Lowell David","year":"2018","unstructured":"David Lowell , Zachary\u00a0 C. Lipton , and Byron\u00a0 C. Wallace . 2018. Practical Obstacles to Deploying Active Learning. arXiv:1807.04801 [cs, stat] (July 2018 ). http:\/\/arxiv.org\/abs\/1807.04801 arXiv:1807.04801. David Lowell, Zachary\u00a0C. Lipton, and Byron\u00a0C. Wallace. 2018. Practical Obstacles to Deploying Active Learning. arXiv:1807.04801 [cs, stat] (July 2018). http:\/\/arxiv.org\/abs\/1807.04801 arXiv:1807.04801."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557263"},{"key":"e_1_3_2_1_47_1","volume-title":"ECIS 2019 proceedings. 27th European Conference on Information Systems (ECIS), Stockholm & Uppsala","author":"Meza\u00a0Mart\u00ednez Miguel\u00a0Angel","year":"2019","unstructured":"Miguel\u00a0Angel Meza\u00a0Mart\u00ednez , Mario Nadj , and Alexander Maedche . 2019 . Towards an Integrative Theoretical Framework of Interactive Machine Learning Systems . In ECIS 2019 proceedings. 27th European Conference on Information Systems (ECIS), Stockholm & Uppsala , Sweden , June 8-14, 2019. Research Papers. AISeL, Stockholm, Sweden., Paper: 172. https:\/\/aisel.aisnet.org\/ecis2019_rp\/172\/ Miguel\u00a0Angel Meza\u00a0Mart\u00ednez, Mario Nadj, and Alexander Maedche. 2019. Towards an Integrative Theoretical Framework of Interactive Machine Learning Systems. In ECIS 2019 proceedings. 27th European Conference on Information Systems (ECIS), Stockholm & Uppsala, Sweden, June 8-14, 2019. Research Papers. AISeL, Stockholm, Sweden., Paper: 172. https:\/\/aisel.aisnet.org\/ecis2019_rp\/172\/"},{"key":"e_1_3_2_1_48_1","volume-title":"Human-in-the-Loop Machine Learning: Active learning and annotation for human-centered AI","author":"Monarch Robert","unstructured":"Robert Monarch . 2021. Human-in-the-Loop Machine Learning: Active learning and annotation for human-centered AI . Manning Publications . Robert Monarch. 2021. Human-in-the-Loop Machine Learning: Active learning and annotation for human-centered AI. Manning Publications."},{"key":"e_1_3_2_1_49_1","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)","author":"Nadeem Moin","year":"1865","unstructured":"Moin Nadeem , Anna Bethke , and Siva Reddy . 2021. StereoSet: Measuring stereotypical bias in pretrained language models . In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers) . Association for Computational Linguistics , Online , 5356\u20135371. https:\/\/doi.org\/10. 1865 3\/v1\/2021.acl-long.416 10.18653\/v1 Moin Nadeem, Anna Bethke, and Siva Reddy. 2021. StereoSet: Measuring stereotypical bias in pretrained language models. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers). Association for Computational Linguistics, Online, 5356\u20135371. https:\/\/doi.org\/10.18653\/v1\/2021.acl-long.416"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3357236.3395454"},{"key":"e_1_3_2_1_51_1","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)","author":"Ni Jianmo","year":"1865","unstructured":"Jianmo Ni , Jiacheng Li , and Julian McAuley . 2019. Justifying Recommendations using Distantly-Labeled Reviews and Fine-Grained Aspects . In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP) . Association for Computational Linguistics , Hong Kong , China, 188\u2013197. https:\/\/doi.org\/10. 1865 3\/v1\/D19-1018 10.18653\/v1 Jianmo Ni, Jiacheng Li, and Julian McAuley. 2019. Justifying Recommendations using Distantly-Labeled Reviews and Fine-Grained Aspects. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP). Association for Computational Linguistics, Hong Kong, China, 188\u2013197. https:\/\/doi.org\/10.18653\/v1\/D19-1018"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450637"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.5555\/1953048.2078195"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000019"},{"key":"e_1_3_2_1_55_1","first-page":"109","article-title":"Okapi at TREC-3","volume":"109","author":"Robertson E","year":"1995","unstructured":"Stephen\u00a0 E Robertson , Steve Walker , Susan Jones , Micheline\u00a0 M Hancock-Beaulieu , Mike Gatford , and others. 1995 . Okapi at TREC-3 . Nist Special Publication Sp 109 (1995), 109 . Publisher: National Institute of Standards & Technology. Stephen\u00a0E Robertson, Steve Walker, Susan Jones, Micheline\u00a0M Hancock-Beaulieu, Mike Gatford, and others. 1995. Okapi at TREC-3. Nist Special Publication Sp 109 (1995), 109. Publisher: National Institute of Standards & Technology.","journal-title":"Nist Special Publication Sp"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2914709"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858328"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.2200\/S00429ED1V01Y201207AIM018"},{"key":"e_1_3_2_1_59_1","volume-title":"Baseline Needs More Love: On Simple Word-Embedding-Based Models and Associated Pooling Mechanisms. arXiv:1805.09843 [cs] (May","author":"Shen Dinghan","year":"2018","unstructured":"Dinghan Shen , Guoyin Wang , Wenlin Wang , Martin\u00a0Renqiang Min , Qinliang Su , Yizhe Zhang , Chunyuan Li , Ricardo Henao , and Lawrence Carin . 2018. Baseline Needs More Love: On Simple Word-Embedding-Based Models and Associated Pooling Mechanisms. arXiv:1805.09843 [cs] (May 2018 ). http:\/\/arxiv.org\/abs\/1805.09843 arXiv:1805.09843. Dinghan Shen, Guoyin Wang, Wenlin Wang, Martin\u00a0Renqiang Min, Qinliang Su, Yizhe Zhang, Chunyuan Li, Ricardo Henao, and Lawrence Carin. 2018. Baseline Needs More Love: On Simple Word-Embedding-Based Models and Associated Pooling Mechanisms. arXiv:1805.09843 [cs] (May 2018). http:\/\/arxiv.org\/abs\/1805.09843 arXiv:1805.09843."},{"key":"e_1_3_2_1_60_1","volume-title":"Machine Teaching: A New Paradigm for Building Machine Learning Systems. arXiv:1707.06742 [cs, stat] (Aug.","author":"Simard Y.","year":"2017","unstructured":"Patrice\u00a0 Y. Simard , Saleema Amershi , David\u00a0 M. Chickering , Alicia\u00a0Edelman Pelton , Soroush Ghorashi , Christopher Meek , Gonzalo Ramos , Jina Suh , Johan Verwey , Mo Wang , and John Wernsing . 2017 . Machine Teaching: A New Paradigm for Building Machine Learning Systems. arXiv:1707.06742 [cs, stat] (Aug. 2017). http:\/\/arxiv.org\/abs\/1707.06742 arXiv:1707.06742. Patrice\u00a0Y. Simard, Saleema Amershi, David\u00a0M. Chickering, Alicia\u00a0Edelman Pelton, Soroush Ghorashi, Christopher Meek, Gonzalo Ramos, Jina Suh, Johan Verwey, Mo Wang, and John Wernsing. 2017. Machine Teaching: A New Paradigm for Building Machine Learning Systems. arXiv:1707.06742 [cs, stat] (Aug. 2017). http:\/\/arxiv.org\/abs\/1707.06742 arXiv:1707.06742."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025171.3025208"},{"key":"e_1_3_2_1_62_1","volume-title":"Proceedings of the Sixth International Conference on Language Resources and Evaluation (LREC\u201908)","author":"Tomanek Katrin","year":"2008","unstructured":"Katrin Tomanek and Udo Hahn . 2008 . Approximating Learning Curves for Active-Learning-Driven Annotation . In Proceedings of the Sixth International Conference on Language Resources and Evaluation (LREC\u201908) . European Language Resources Association (ELRA), Marrakech, Morocco. http:\/\/www.lrec-conf.org\/proceedings\/lrec 2008\/pdf\/335_paper.pdf Katrin Tomanek and Udo Hahn. 2008. Approximating Learning Curves for Active-Learning-Driven Annotation. In Proceedings of the Sixth International Conference on Language Resources and Evaluation (LREC\u201908). European Language Resources Association (ELRA), Marrakech, Morocco. http:\/\/www.lrec-conf.org\/proceedings\/lrec2008\/pdf\/335_paper.pdf"},{"key":"e_1_3_2_1_63_1","volume-title":"Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces. In 26th International Conference on Intelligent User Interfaces(IUI \u201921)","author":"van\u00a0der Stappen Almar","year":"2021","unstructured":"Almar van\u00a0der Stappen and Mathias Funk . 2021 . Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces. In 26th International Conference on Intelligent User Interfaces(IUI \u201921) . Association for Computing Machinery, New York, NY, USA, 514\u2013519. https:\/\/doi.org\/10.1145\/3397481.3450668 10.1145\/3397481.3450668 Almar van\u00a0der Stappen and Mathias Funk. 2021. Towards Guidelines for Designing Human-in-the-Loop Machine Training Interfaces. In 26th International Conference on Intelligent User Interfaces(IUI \u201921). Association for Computing Machinery, New York, NY, USA, 514\u2013519. https:\/\/doi.org\/10.1145\/3397481.3450668"},{"key":"e_1_3_2_1_64_1","volume-title":"Human-Computer Interaction - INTERACT 2019(Lecture Notes in Computer Science)","author":"Wall Emily","unstructured":"Emily Wall , Soroush Ghorashi , and Gonzalo Ramos . 2019. Using Expert Patterns in Assisted Interactive Machine Learning: A Study in Machine Teaching . In Human-Computer Interaction - INTERACT 2019(Lecture Notes in Computer Science) , David Lamas, Fernando Loizides, Lennart Nacke, Helen Petrie, Marco Winckler, and Panayiotis Zaphiris (Eds.). Springer International Publishing , Cham , 578\u2013599. https:\/\/doi.org\/10.1007\/978-3-030-29387-1_34 10.1007\/978-3-030-29387-1_34 Emily Wall, Soroush Ghorashi, and Gonzalo Ramos. 2019. Using Expert Patterns in Assisted Interactive Machine Learning: A Study in Machine Teaching. In Human-Computer Interaction - INTERACT 2019(Lecture Notes in Computer Science), David Lamas, Fernando Loizides, Lennart Nacke, Helen Petrie, Marco Winckler, and Panayiotis Zaphiris (Eds.). Springer International Publishing, Cham, 578\u2013599. https:\/\/doi.org\/10.1007\/978-3-030-29387-1_34"},{"key":"e_1_3_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/2110363.2110464"},{"key":"e_1_3_2_1_66_1","volume-title":"Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). Association for Computational Linguistics","author":"Wang Sida","year":"2012","unstructured":"Sida Wang and Christopher Manning . 2012 . Baselines and Bigrams: Simple, Good Sentiment and Topic Classification . In Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). Association for Computational Linguistics , Jeju Island, Korea, 90\u201394. https:\/\/www.aclweb.org\/anthology\/P12- 2018 Sida Wang and Christopher Manning. 2012. Baselines and Bigrams: Simple, Good Sentiment and Topic Classification. In Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers). Association for Computational Linguistics, Jeju Island, Korea, 90\u201394. https:\/\/www.aclweb.org\/anthology\/P12-2018"},{"key":"e_1_3_2_1_67_1","volume-title":"Clinical Word Sense Disambiguation with Interactive Search and Classification. AMIA Annu Symp Proc 2016 (Feb. 2017)","author":"Wang Yue","year":"2017","unstructured":"Yue Wang , Kai Zheng , Hua Xu , and Qiaozhu Mei . 2017 . Clinical Word Sense Disambiguation with Interactive Search and Classification. AMIA Annu Symp Proc 2016 (Feb. 2017) , 2062\u20132071. https:\/\/www.ncbi.nlm.nih.gov\/pmc\/articles\/PMC5333264\/ Yue Wang, Kai Zheng, Hua Xu, and Qiaozhu Mei. 2017. Clinical Word Sense Disambiguation with Interactive Search and Classification. AMIA Annu Symp Proc 2016 (Feb. 2017), 2062\u20132071. https:\/\/www.ncbi.nlm.nih.gov\/pmc\/articles\/PMC5333264\/"},{"key":"e_1_3_2_1_68_1","volume-title":"HuggingFace\u2019s Transformers: State-of-the-art Natural Language Processing. arXiv:1910.03771 [cs] (July","author":"Wolf Thomas","year":"2020","unstructured":"Thomas Wolf , Lysandre Debut , Victor Sanh , Julien Chaumond , Clement Delangue , Anthony Moi , Pierric Cistac , Tim Rault , R\u00e9mi Louf , Morgan Funtowicz , Joe Davison , Sam Shleifer , Patrick von Platen , Clara Ma , Yacine Jernite , Julien Plu , Canwen Xu , Teven\u00a0Le Scao , Sylvain Gugger , Mariama Drame , Quentin Lhoest , and Alexander\u00a0 M. Rush . 2020. HuggingFace\u2019s Transformers: State-of-the-art Natural Language Processing. arXiv:1910.03771 [cs] (July 2020 ). http:\/\/arxiv.org\/abs\/1910.03771 arXiv:1910.03771. Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony Moi, Pierric Cistac, Tim Rault, R\u00e9mi Louf, Morgan Funtowicz, Joe Davison, Sam Shleifer, Patrick von Platen, Clara Ma, Yacine Jernite, Julien Plu, Canwen Xu, Teven\u00a0Le Scao, Sylvain Gugger, Mariama Drame, Quentin Lhoest, and Alexander\u00a0M. Rush. 2020. HuggingFace\u2019s Transformers: State-of-the-art Natural Language Processing. arXiv:1910.03771 [cs] (July 2020). http:\/\/arxiv.org\/abs\/1910.03771 arXiv:1910.03771."},{"key":"e_1_3_2_1_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/359784.360332"},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080721"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"e_1_3_2_1_72_1","unstructured":"Michelle Yuan Hsuan-Tien Lin and Jordan Boyd-Graber. 2020. Cold-start Active Learning through Self-supervised Language Modeling. In arXiv:2010.09535 [cs]. http:\/\/arxiv.org\/abs\/2010.09535 arXiv:2010.09535.  Michelle Yuan Hsuan-Tien Lin and Jordan Boyd-Graber. 2020. Cold-start Active Learning through Self-supervised Language Modeling. In arXiv:2010.09535 [cs]. http:\/\/arxiv.org\/abs\/2010.09535 arXiv:2010.09535."}],"event":{"name":"IUI '22: 27th International Conference on Intelligent User Interfaces","location":"Helsinki Finland","acronym":"IUI '22","sponsor":["SIGAI ACM Special Interest Group on Artificial Intelligence","SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["27th International Conference on Intelligent User Interfaces"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3490099.3511134","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3490099.3511134","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T19:31:24Z","timestamp":1750188684000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3490099.3511134"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,3,22]]},"references-count":73,"alternative-id":["10.1145\/3490099.3511134","10.1145\/3490099"],"URL":"https:\/\/doi.org\/10.1145\/3490099.3511134","relation":{},"subject":[],"published":{"date-parts":[[2022,3,22]]},"assertion":[{"value":"2022-03-22","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}