{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,8]],"date-time":"2026-02-08T08:38:24Z","timestamp":1770539904003,"version":"3.49.0"},"publisher-location":"New York, NY, USA","reference-count":87,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,4,27]],"date-time":"2022-04-27T00:00:00Z","timestamp":1651017600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Shanghai Sailing Program","award":["21YF1402900"],"award-info":[{"award-number":["21YF1402900"]}]},{"name":"Shanghai Science and Technology Commission","award":["21ZR1403300"],"award-info":[{"award-number":["21ZR1403300"]}]},{"name":"Shanghai Municipal Science and Technology Major Project","award":["2018SHZDZX01"],"award-info":[{"award-number":["2018SHZDZX01"]}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022,4,27]]},"DOI":"10.1145\/3491102.3517612","type":"proceedings-article","created":{"date-parts":[[2022,4,28]],"date-time":"2022-04-28T16:33:27Z","timestamp":1651163607000},"page":"1-22","source":"Crossref","is-referenced-by-count":25,"title":["OneLabeler: A Flexible System for Building Data Labeling Tools"],"prefix":"10.1145","author":[{"given":"Yu","family":"Zhang","sequence":"first","affiliation":[{"name":"Department of Computer Science, University of Oxford, United Kingdom"}]},{"given":"Yun","family":"Wang","sequence":"additional","affiliation":[{"name":"Microsoft Research Asia, China"}]},{"given":"Haidong","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Research Asia, China"}]},{"given":"Bin","family":"Zhu","sequence":"additional","affiliation":[{"name":"Microsoft Research Asia, China"}]},{"given":"Siming","family":"Chen","sequence":"additional","affiliation":[{"name":"School of Data Science, Fudan University, China"}]},{"given":"Dongmei","family":"Zhang","sequence":"additional","affiliation":[{"name":"Microsoft Research Asia, China"}]}],"member":"320","published-online":{"date-parts":[[2022,4,28]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"Amazon. 2005. Amazon Mechanical Turk. https:\/\/www.mturk.com\/  Amazon. 2005. Amazon Mechanical Turk. https:\/\/www.mturk.com\/"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240508.3241916"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/MS.2008.130"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.13973"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2017.2744818"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1007\/s00371-018-1500-3"},{"key":"e_1_3_2_2_7_1","volume-title":"Proceedings of the Conference on Computational Natural Language Learning. Association for Computational Linguistics, 39\u201347","author":"Bloodgood Michael","unstructured":"Michael Bloodgood and K. Vijay-Shanker . 2009. A Method for Stopping Active Learning Based on Stabilizing Predictions and the Need for User-Adjustable Stopping . In Proceedings of the Conference on Computational Natural Language Learning. Association for Computational Linguistics, 39\u201347 . Michael Bloodgood and K. Vijay-Shanker. 2009. A Method for Stopping Active Learning Based on Stabilizing Predictions and the Need for User-Adjustable Stopping. In Proceedings of the Conference on Computational Natural Language Learning. Association for Computational Linguistics, 39\u201347."},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2011.185"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/34.969114"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-15561-1_32"},{"key":"e_1_3_2_2_11_1","volume-title":"Proceedings of the International Conference on Machine Learning. AAAI Press, 59\u201366","author":"Brinker Klaus","year":"2003","unstructured":"Klaus Brinker . 2003 . Incorporating Diversity in Active Learning with Support Vector Machines . In Proceedings of the International Conference on Machine Learning. AAAI Press, 59\u201366 . Klaus Brinker. 2003. Incorporating Diversity in Active Learning with Support Vector Machines. In Proceedings of the International Conference on Machine Learning. AAAI Press, 59\u201366."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1613\/jair.606"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557253"},{"key":"e_1_3_2_2_14_1","volume-title":"Proceedings of ICML Workshop on Combining Learning Strategies to Reduce Label Cost. 5\u00a0pages.","author":"Cakmak Maya","year":"2011","unstructured":"Maya Cakmak and Andrea\u00a0 L. Thomaz . 2011 . Mixed-Initiative Active Learning . In Proceedings of ICML Workshop on Combining Learning Strategies to Reduce Label Cost. 5\u00a0pages. Maya Cakmak and Andrea\u00a0L. Thomaz. 2011. Mixed-Initiative Active Learning. In Proceedings of ICML Workshop on Combining Learning Strategies to Reduce Label Cost. 5\u00a0pages."},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE48307.2020.00177"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2675133.2675214"},{"key":"e_1_3_2_2_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300460"},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1240624.1240684"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2019.2934785"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCG.2010.66"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557011"},{"key":"e_1_3_2_2_23_1","unstructured":"Dheeru Dua and Casey Graff. 2017. UCI Machine Learning Repository. http:\/\/archive.ics.uci.edu\/ml  Dheeru Dua and Casey Graff. 2017. UCI Machine Learning Repository. http:\/\/archive.ics.uci.edu\/ml"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3343031.3350535"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.findings-emnlp.181"},{"key":"e_1_3_2_2_26_1","unstructured":"Explosion. 2017. Prodigy. https:\/\/prodi.gy\/  Explosion. 2017. Prodigy. https:\/\/prodi.gy\/"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/604045.604056"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3242587.3242596"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.13667"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.13676"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2012.6400492"},{"key":"e_1_3_2_2_32_1","volume-title":"Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, Vol.\u00a02. IEEE, 302\u2013309","author":"H.","unstructured":"Steven\u00a0C. H. Hoi and Michael\u00a0R. Lyu. 2005. A Semi-Supervised Active Learning Framework for Image Retrieval . In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, Vol.\u00a02. IEEE, 302\u2013309 . Steven\u00a0C.H. Hoi and Michael\u00a0R. Lyu. 2005. A Semi-Supervised Active Learning Framework for Image Retrieval. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, Vol.\u00a02. IEEE, 302\u2013309."},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/1459359.1459379"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0019-9958(60)90757-9"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1089733.1089734"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132169"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556288.2557238"},{"key":"e_1_3_2_2_38_1","unstructured":"Labelbox Inc.2018. Labelbox. https:\/\/labelbox.com\/  Labelbox Inc.2018. Labelbox. https:\/\/labelbox.com\/"},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173610"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1111\/cgf.13971"},{"key":"e_1_3_2_2_41_1","volume-title":"Lewis and Jason Catlett","author":"D.","year":"1994","unstructured":"David\u00a0 D. Lewis and Jason Catlett . 1994 . Heterogeneous Uncertainty Sampling for Supervised Learning. In Proceedings of the International Conference on Machine Learning. Morgan Kaufmann Publishers Inc ., 148\u2013156. David\u00a0D. Lewis and Jason Catlett. 1994. Heterogeneous Uncertainty Sampling for Supervised Learning. In Proceedings of the International Conference on Machine Learning. Morgan Kaufmann Publishers Inc., 148\u2013156."},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2016.2535242"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2016.2614227"},{"key":"e_1_3_2_2_44_1","unstructured":"Tzuta Lin. 2015. LabelImg. https:\/\/github.com\/tzutalin\/labelImg  Tzuta Lin. 2015. LabelImg. https:\/\/github.com\/tzutalin\/labelImg"},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1631272.1631420"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2018.2865018"},{"key":"e_1_3_2_2_47_1","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, 142\u2013150","author":"Maas L.","year":"2011","unstructured":"Andrew\u00a0 L. Maas , Raymond\u00a0 E. Daly , Peter\u00a0 T. Pham , Dan Huang , Andrew\u00a0 Y. Ng , and Christopher Potts . 2011 . Learning Word Vectors for Sentiment Analysis . In Proceedings of the Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, 142\u2013150 . Andrew\u00a0L. Maas, Raymond\u00a0E. Daly, Peter\u00a0T. Pham, Dan Huang, Andrew\u00a0Y. Ng, and Christopher Potts. 2011. Learning Word Vectors for Sentiment Analysis. In Proceedings of the Annual Meeting of the Association for Computational Linguistics: Human Language Technologies. Association for Computational Linguistics, 142\u2013150."},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858435"},{"key":"e_1_3_2_2_49_1","unstructured":"Microsoft. 2017. VoTT: Visual Object Tagging Tool. https:\/\/github.com\/microsoft\/VoTT  Microsoft. 2017. VoTT: Visual Object Tagging Tool. https:\/\/github.com\/microsoft\/VoTT"},{"key":"e_1_3_2_2_50_1","unstructured":"Hiroki Nakayama Takahiro Kubo Junya Kamura Yasufumi Taniguchi and Xu Liang. 2018. doccano: Text Annotation Tool for Human. https:\/\/github.com\/doccano\/doccano  Hiroki Nakayama Takahiro Kubo Junya Kamura Yasufumi Taniguchi and Xu Liang. 2018. doccano: Text Annotation Tool for Human. https:\/\/github.com\/doccano\/doccano"},{"key":"e_1_3_2_2_51_1","volume-title":"Confident Learning: Estimating Uncertainty in Dataset Labels. Journal of Artificial Intelligence Research 70 (may","author":"Northcutt Curtis","year":"2021","unstructured":"Curtis Northcutt , Lu Jiang , and Isaac Chuang . 2021 . Confident Learning: Estimating Uncertainty in Dataset Labels. Journal of Artificial Intelligence Research 70 (may 2021), 1373\u20131411. Curtis Northcutt, Lu Jiang, and Isaac Chuang. 2021. Confident Learning: Estimating Uncertainty in Dataset Labels. Journal of Artificial Intelligence Research 70 (may 2021), 1373\u20131411."},{"key":"e_1_3_2_2_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450685"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2331979"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2015.7178964"},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.14778\/3157794.3157797"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2014.2346291"},{"key":"e_1_3_2_2_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445591"},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298824"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-007-0090-8"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICPR.2004.1334462"},{"key":"e_1_3_2_2_62_1","volume-title":"Proceedings of the Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 1467\u20131478","author":"Settles Burr","year":"2011","unstructured":"Burr Settles . 2011 . Closing the Loop: Fast, Interactive Semi-supervised Annotation with Queries on Features and Instances . In Proceedings of the Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 1467\u20131478 . Burr Settles. 2011. Closing the Loop: Fast, Interactive Semi-supervised Annotation with Queries on Features and Instances. In Proceedings of the Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics, 1467\u20131478."},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3323873.3325056"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.intcom.2007.02.002"},{"key":"e_1_3_2_2_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/2501643.2501651"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2007.383282"},{"key":"e_1_3_2_2_67_1","volume-title":"Label Studio: Data labeling software. https:\/\/github.com\/heartexlabs\/label-studio","author":"Tkachenko Maxim","year":"2020","unstructured":"Maxim Tkachenko , Mikhail Malyuk , Nikita Shevchenko , Andrey Holmanyuk , and Nikolai Liubimov . 2020 . Label Studio: Data labeling software. https:\/\/github.com\/heartexlabs\/label-studio Maxim Tkachenko, Mikhail Malyuk, Nikita Shevchenko, Andrey Holmanyuk, and Nikolai Liubimov. 2020. Label Studio: Data labeling software. https:\/\/github.com\/heartexlabs\/label-studio"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985733"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1145\/1378704.1378719"},{"key":"e_1_3_2_2_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/1124772.1124782"},{"key":"e_1_3_2_2_71_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.1160379"},{"key":"e_1_3_2_2_72_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2018.2840724"},{"key":"e_1_3_2_2_73_1","doi-asserted-by":"publisher","DOI":"10.1145\/1899412.1899414"},{"key":"e_1_3_2_2_74_1","volume-title":"IEEE Conference on Computer Vision and Pattern Recognition. IEEE","author":"Wu Zhirong","year":"2015","unstructured":"Zhirong Wu , Shuran Song , Aditya Khosla , Fisher Yu , Linguang Zhang , Xiaoou Tang , and Jianxiong Xiao . 2015 . 3D ShapeNets: A Deep Representation for Volumetric Shapes . In IEEE Conference on Computer Vision and Pattern Recognition. IEEE , 1912\u20131920. Zhirong Wu, Shuran Song, Aditya Khosla, Fisher Yu, Linguang Zhang, Xiaoou Tang, and Jianxiong Xiao. 2015. 3D ShapeNets: A Deep Representation for Volumetric Shapes. In IEEE Conference on Computer Vision and Pattern Recognition. IEEE, 1912\u20131920."},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST47406.2019.8986943"},{"key":"e_1_3_2_2_76_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-71496-5_24"},{"key":"e_1_3_2_2_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/1459359.1459423"},{"key":"e_1_3_2_2_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/2964284.2984059"},{"key":"e_1_3_2_2_79_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2016.2598497"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1109\/VAST.2014.7042476"},{"key":"e_1_3_2_2_81_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2020.3030383"},{"key":"e_1_3_2_2_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3240508.3240540"},{"key":"e_1_3_2_2_83_1","volume-title":"\u00a0H. Tung","author":"Zhang Tianye","year":"2021","unstructured":"Tianye Zhang , Haozhe Feng , Wei Chen , Zexian Chen , Wenting Zheng , Xiao-Nan Luo , Wenqi Huang , and Anthony K . \u00a0H. Tung . 2021 . ChartNavigator: An Interactive Pattern Identification and Annotation Framework for Charts. IEEE Transactions on Knowledge and Data Engineering ( 2021), 1\u20131. Tianye Zhang, Haozhe Feng, Wei Chen, Zexian Chen, Wenting Zheng, Xiao-Nan Luo, Wenqi Huang, and Anthony K.\u00a0H. Tung. 2021. ChartNavigator: An Interactive Pattern Identification and Annotation Framework for Charts. IEEE Transactions on Knowledge and Data Engineering (2021), 1\u20131."},{"key":"e_1_3_2_2_84_1","article-title":"Stopping Criterion for Active Learning with Model Stability","volume":"9","author":"Zhang Yexun","year":"2017","unstructured":"Yexun Zhang , Wenbin Cai , Wenquan Wang , and Ya Zhang . 2017 . Stopping Criterion for Active Learning with Model Stability . ACM Transactions on Intelligent Systems and Technology 9 , 2, Article 19 (Oct. 2017), 26\u00a0pages. Yexun Zhang, Wenbin Cai, Wenquan Wang, and Ya Zhang. 2017. Stopping Criterion for Active Learning with Model Stability. ACM Transactions on Intelligent Systems and Technology 9, 2, Article 19 (Oct. 2017), 26\u00a0pages.","journal-title":"ACM Transactions on Intelligent Systems and Technology"},{"key":"e_1_3_2_2_85_1","doi-asserted-by":"publisher","DOI":"10.1145\/3412848"},{"key":"e_1_3_2_2_86_1","unstructured":"Yu Zhang Martijn Tennekes Tim de Jong Lyana Curier Bob Coecke and Min Chen. 2021. Using Simulation to Aid the Design and Optimization of Intelligent User Interfaces for Quality Assurance Processes in Machine Learning. arXiv:2104.01129\u00a0[cs.HC]  Yu Zhang Martijn Tennekes Tim de Jong Lyana Curier Bob Coecke and Min Chen. 2021. Using Simulation to Aid the Design and Optimization of Intelligent User Interfaces for Quality Assurance Processes in Machine Learning. arXiv:2104.01129\u00a0[cs.HC]"},{"key":"e_1_3_2_2_87_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753783.1753784"},{"key":"e_1_3_2_2_88_1","volume-title":"Proceedings of the International Conference on Machine Learning. AAAI Press, 920\u2013927","author":"Zhu Xingquan","year":"2003","unstructured":"Xingquan Zhu , Xindong Wu , and Qijun Chen . 2003 . Eliminating Class Noise in Large Datasets . In Proceedings of the International Conference on Machine Learning. AAAI Press, 920\u2013927 . Xingquan Zhu, Xindong Wu, and Qijun Chen. 2003. Eliminating Class Noise in Large Datasets. In Proceedings of the International Conference on Machine Learning. AAAI Press, 920\u2013927."}],"event":{"name":"CHI '22: CHI Conference on Human Factors in Computing Systems","location":"New Orleans LA USA","acronym":"CHI '22","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3491102.3517612","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3491102.3517612","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:16:59Z","timestamp":1750191419000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3491102.3517612"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,4,27]]},"references-count":87,"alternative-id":["10.1145\/3491102.3517612","10.1145\/3491102"],"URL":"https:\/\/doi.org\/10.1145\/3491102.3517612","relation":{},"subject":[],"published":{"date-parts":[[2022,4,27]]}}}