{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,29]],"date-time":"2026-09-29T20:25:18Z","timestamp":1790713518187,"version":"4.1.0"},"publisher-location":"Cham","reference-count":47,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030884932","type":"print"},{"value":"9783030884949","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-88494-9_3","type":"book-chapter","created":{"date-parts":[[2021,10,5]],"date-time":"2021-10-05T13:23:18Z","timestamp":1633440198000},"page":"42-61","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":26,"title":["Into the Unknown: Active Monitoring of\u00a0Neural Networks"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9525-0333","authenticated-orcid":false,"given":"Anna","family":"Lukina","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3658-1065","authenticated-orcid":false,"given":"Christian","family":"Schilling","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2985-7724","authenticated-orcid":false,"given":"Thomas A.","family":"Henzinger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,10,6]]},"reference":[{"key":"3_CR1","doi-asserted-by":"publisher","unstructured":"Bendale, A., Boult, T.E.: Towards open world recognition. In: CVPR, pp. 1893\u20131902. IEEE Computer Society (2015). https:\/\/doi.org\/10.1109\/CVPR.2015.7298799","DOI":"10.1109\/CVPR.2015.7298799"},{"key":"3_CR2","doi-asserted-by":"publisher","unstructured":"Bendale, A., Boult, T.E.: Towards open set deep networks. In: CVPR, pp. 1563\u20131572. IEEE Computer Society (2016). https:\/\/doi.org\/10.1109\/CVPR.2016.173","DOI":"10.1109\/CVPR.2016.173"},{"key":"3_CR3","unstructured":"Bendre, N., Terashima-Mar\u00edn, H., Najafirad, P.: Learning from few samples: a survey. CoRR abs\/2007.15484 (2020). https:\/\/arxiv.org\/abs\/2007.15484"},{"key":"3_CR4","unstructured":"Chen, Y., Cheng, C., Yan, J., Yan, R.: Monitoring object detection abnormalities via data-label and post-algorithm abstractions. CoRR abs\/2103.15456 (2021). https:\/\/arxiv.org\/abs\/2103.15456"},{"key":"3_CR5","doi-asserted-by":"publisher","unstructured":"Cheng, C., N\u00fchrenberg, G., Yasuoka, H.: Runtime monitoring neuron activation patterns. In: DATE, pp. 300\u2013303. IEEE (2019). https:\/\/doi.org\/10.23919\/DATE.2019.8714971","DOI":"10.23919\/DATE.2019.8714971"},{"key":"3_CR6","doi-asserted-by":"publisher","unstructured":"Cohen, G., Afshar, S., Tapson, J., van Schaik, A.: EMNIST: extending MNIST to handwritten letters. In: IJCNN, pp. 2921\u20132926. IEEE (2017). https:\/\/doi.org\/10.1109\/IJCNN.2017.7966217","DOI":"10.1109\/IJCNN.2017.7966217"},{"issue":"2","key":"3_CR7","doi-asserted-by":"publisher","first-page":"201","DOI":"10.1007\/BF00993277","volume":"15","author":"DA Cohn","year":"1994","unstructured":"Cohn, D.A., Atlas, L.E., Ladner, R.E.: Improving generalization with active learning. Mach. Learn. 15(2), 201\u2013221 (1994). https:\/\/doi.org\/10.1007\/BF00993277","journal-title":"Mach. Learn."},{"key":"3_CR8","doi-asserted-by":"publisher","unstructured":"Das, S., Wong, W., Dietterich, T.G., Fern, A., Emmott, A.: Incorporating expert feedback into active anomaly discovery. In: ICDM, pp. 853\u2013858. IEEE Computer Society (2016). https:\/\/doi.org\/10.1109\/ICDM.2016.0102","DOI":"10.1109\/ICDM.2016.0102"},{"key":"3_CR9","unstructured":"Gal, Y., Ghahramani, Z.: Dropout as a Bayesian approximation: representing model uncertainty in deep learning. In: ICML. JMLR Workshop and Conference Proceedings, vol. 48, pp. 1050\u20131059. JMLR.org (2016). http:\/\/proceedings.mlr.press\/v48\/gal16.html"},{"key":"3_CR10","unstructured":"Geifman, Y., El-Yaniv, R.: Selective classification for deep neural networks. In: NeurIPS, pp. 4878\u20134887 (2017). http:\/\/papers.nips.cc\/paper\/7073-selective-classification-for-deep-neural-networks"},{"key":"3_CR11","unstructured":"Guerriero, S., Caputo, B., Mensink, T.: DeepNCM: deep nearest class mean classifiers. In: ICLR. OpenReview.net (2018). https:\/\/openreview.net\/forum?id=rkPLZ4JPM"},{"key":"3_CR12","unstructured":"Guo, C., Pleiss, G., Sun, Y., Weinberger, K.Q.: On calibration of modern neural networks. In: ICML. PMLR, vol. 70, pp. 1321\u20131330. PMLR (2017). http:\/\/proceedings.mlr.press\/v70\/guo17a.html"},{"key":"3_CR13","doi-asserted-by":"publisher","unstructured":"Gupta, A., Carlone, L.: Online monitoring for neural network based monocular pedestrian pose estimation. In: ITSC, pp. 1\u20138. IEEE (2020). https:\/\/doi.org\/10.1109\/ITSC45102.2020.9294609","DOI":"10.1109\/ITSC45102.2020.9294609"},{"key":"3_CR14","unstructured":"Hendrycks, D., Gimpel, K.: A baseline for detecting misclassified and out-of-distribution examples in neural networks. In: ICLR. OpenReview.net (2017). https:\/\/openreview.net\/forum?id=Hkg4TI9xl"},{"key":"3_CR15","doi-asserted-by":"publisher","unstructured":"Henzinger, T.A., Lukina, A., Schilling, C.: Outside the box: abstraction-based monitoring of neural networks. In: ECAI. Frontiers in Artificial Intelligence and Applications, vol. 325, pp. 2433\u20132440. IOS Press (2020). https:\/\/doi.org\/10.3233\/FAIA200375","DOI":"10.3233\/FAIA200375"},{"key":"3_CR16","unstructured":"Ibrahim, S.H., Nassar, M.: Hack the box: fooling deep learning abstraction-based monitors. CoRR abs\/2107.04764 (2021). https:\/\/arxiv.org\/abs\/2107.04764"},{"key":"3_CR17","series-title":"Springer Series in Statistics","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-1904-8","volume-title":"Principal Component Analysis","author":"IT Jolliffe","year":"1986","unstructured":"Jolliffe, I.T.: Principal Component Analysis. Springer Series in Statistics, Springer, Heidelberg (1986). https:\/\/doi.org\/10.1007\/978-1-4757-1904-8"},{"key":"3_CR18","unstructured":"Knorr, E.M., Ng, R.T.: A unified notion of outliers: properties and computation. In: KDD, pp. 219\u2013222. AAAI Press (1997). http:\/\/www.aaai.org\/Library\/KDD\/1997\/kdd97-044.php"},{"key":"3_CR19","unstructured":"Krizhevsky, A.: Learning multiple layers of features from tiny images. University of Toronto, Technical report (2009)"},{"issue":"11","key":"3_CR20","doi-asserted-by":"publisher","first-page":"2278","DOI":"10.1109\/5.726791","volume":"86","author":"Y LeCun","year":"1998","unstructured":"LeCun, Y., Bottou, L., Bengio, Y., Haffner, P.: Gradient-based learning applied to document recognition. Proc. IEEE 86(11), 2278\u20132324 (1998)","journal-title":"Proc. IEEE"},{"key":"3_CR21","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1016\/j.neucom.2016.12.038","volume":"234","author":"W Liu","year":"2017","unstructured":"Liu, W., Wang, Z., Liu, X., Zeng, N., Liu, Y., Alsaadi, F.E.: A survey of deep neural network architectures and their applications. Neurocomputing 234, 11\u201326 (2017). https:\/\/doi.org\/10.1016\/j.neucom.2016.12.038","journal-title":"Neurocomputing"},{"issue":"2","key":"3_CR22","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1109\/TIT.1982.1056489","volume":"28","author":"SP Lloyd","year":"1982","unstructured":"Lloyd, S.P.: Least squares quantization in PCM. Trans. Inf. Theory 28(2), 129\u2013136 (1982). https:\/\/doi.org\/10.1109\/TIT.1982.1056489","journal-title":"Trans. Inf. Theory"},{"key":"3_CR23","unstructured":"Lu, J., Gong, P., Ye, J., Zhang, C.: Learning from very few samples: a survey. CoRR abs\/2009.02653 (2020). https:\/\/arxiv.org\/abs\/2009.02653"},{"key":"3_CR24","doi-asserted-by":"publisher","unstructured":"Mancini, M., Karaoguz, H., Ricci, E., Jensfelt, P., Caputo, B.: Knowledge is never enough: towards web aided deep open world recognition. In: ICRA, pp. 9537\u20139543. IEEE (2019). https:\/\/doi.org\/10.1109\/ICRA.2019.8793803","DOI":"10.1109\/ICRA.2019.8793803"},{"key":"3_CR25","unstructured":"Mandelbaum, A., Weinshall, D.: Distance-based confidence score for neural network classifiers. CoRR abs\/1709.09844 (2017). http:\/\/arxiv.org\/abs\/1709.09844"},{"key":"3_CR26","doi-asserted-by":"crossref","unstructured":"McCloskey, M., Cohen, N.J.: Catastrophic interference in connectionist networks: the sequential learning problem. In: Psychology of Learning and Motivation, vol. 24, pp. 109\u2013165. Elsevier (1989). http:\/\/www.sciencedirect.com\/science\/article\/pii\/S0079742108605368","DOI":"10.1016\/S0079-7421(08)60536-8"},{"issue":"11","key":"3_CR27","doi-asserted-by":"publisher","first-page":"2624","DOI":"10.1109\/TPAMI.2013.83","volume":"35","author":"T Mensink","year":"2013","unstructured":"Mensink, T., Verbeek, J.J., Perronnin, F., Csurka, G.: Distance-based image classification: generalizing to new classes at near-zero cost. IEEE Trans. Pattern Anal. Mach. Intell. 35(11), 2624\u20132637 (2013). https:\/\/doi.org\/10.1109\/TPAMI.2013.83","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"10","key":"3_CR28","doi-asserted-by":"publisher","first-page":"1345","DOI":"10.1109\/TKDE.2009.191","volume":"22","author":"SJ Pan","year":"2010","unstructured":"Pan, S.J., Yang, Q.: A survey on transfer learning. IEEE Trans. Knowl. Data Eng. 22(10), 1345\u20131359 (2010). https:\/\/doi.org\/10.1109\/TKDE.2009.191","journal-title":"IEEE Trans. Knowl. Data Eng."},{"key":"3_CR29","doi-asserted-by":"publisher","unstructured":"Parisi, G.I., Kemker, R., Part, J.L., Kanan, C., Wermter, S.: Continual lifelong learning with neural networks: A review. Neural Networks 113, 54\u201371 (2019). https:\/\/doi.org\/10.1016\/j.neunet.2019.01.012","DOI":"10.1016\/j.neunet.2019.01.012"},{"key":"3_CR30","doi-asserted-by":"publisher","first-page":"215","DOI":"10.1016\/j.sigpro.2013.12.026","volume":"99","author":"MAF Pimentel","year":"2014","unstructured":"Pimentel, M.A.F., Clifton, D.A., Clifton, L.A., Tarassenko, L.: A review of novelty detection. Signal Process. 99, 215\u2013249 (2014). https:\/\/doi.org\/10.1016\/j.sigpro.2013.12.026","journal-title":"Signal Process."},{"key":"3_CR31","doi-asserted-by":"publisher","first-page":"20067","DOI":"10.1109\/ACCESS.2021.3055015","volume":"9","author":"QM Rahman","year":"2021","unstructured":"Rahman, Q.M., Corke, P., Dayoub, F.: Run-time monitoring of machine learning for robotic perception: a survey of emerging trends. IEEE Access 9, 20067\u201320075 (2021). https:\/\/doi.org\/10.1109\/ACCESS.2021.3055015","journal-title":"IEEE Access"},{"key":"3_CR32","doi-asserted-by":"publisher","unstructured":"Rebuffi, S., Kolesnikov, A., Sperl, G., Lampert, C.H.: iCaRL: incremental classifier and representation learning. In: CVPR, pp. 5533\u20135542. IEEE Computer Society (2017). https:\/\/doi.org\/10.1109\/CVPR.2017.587","DOI":"10.1109\/CVPR.2017.587"},{"key":"3_CR33","doi-asserted-by":"crossref","unstructured":"Redko, I., Morvant, E., Habrard, A., Sebban, M., Bennani, Y.: Advances in Domain Adaptation Theory. Elsevier (2019)","DOI":"10.1016\/B978-1-78548-236-6.50008-8"},{"key":"3_CR34","doi-asserted-by":"publisher","unstructured":"Royer, A., Lampert, C.H.: Classifier adaptation at prediction time. In: CVPR, pp. 1401\u20131409. IEEE Computer Society (2015). https:\/\/doi.org\/10.1109\/CVPR.2015.7298746","DOI":"10.1109\/CVPR.2015.7298746"},{"key":"3_CR35","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"583","DOI":"10.1007\/BFb0020217","volume-title":"Artificial Neural Networks \u2014 ICANN\u201997","author":"B Sch\u00f6lkopf","year":"1997","unstructured":"Sch\u00f6lkopf, B., Smola, A., M\u00fcller, K.-R.: Kernel principal component analysis. In: Gerstner, W., Germond, A., Hasler, M., Nicoud, J.-D. (eds.) ICANN 1997. LNCS, vol. 1327, pp. 583\u2013588. Springer, Heidelberg (1997). https:\/\/doi.org\/10.1007\/BFb0020217"},{"key":"3_CR36","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"226","DOI":"10.1007\/978-3-319-66709-6_19","volume-title":"Pattern Recognition","author":"A Schultheiss","year":"2017","unstructured":"Schultheiss, A., K\u00e4ding, C., Freytag, A., Denzler, J.: Finding the unknown: novelty detection with extreme value signatures of deep neural activations. In: Roth, V., Vetter, T. (eds.) GCPR 2017. LNCS, vol. 10496, pp. 226\u2013238. Springer, Cham (2017). https:\/\/doi.org\/10.1007\/978-3-319-66709-6_19"},{"key":"3_CR37","doi-asserted-by":"publisher","unstructured":"Settles, B.: Active Learning. Synthesis Lectures on Artificial Intelligence and Machine Learning. Morgan & Claypool Publishers (2012). https:\/\/doi.org\/10.2200\/S00429ED1V01Y201207AIM018","DOI":"10.2200\/S00429ED1V01Y201207AIM018"},{"key":"3_CR38","doi-asserted-by":"publisher","unstructured":"Stallkamp, J., Schlipsing, M., Salmen, J., Igel, C.: The German traffic sign recognition benchmark: a multi-class classification competition. In: IJCNN, pp. 1453\u20131460. IEEE (2011). https:\/\/doi.org\/10.1109\/IJCNN.2011.6033395","DOI":"10.1109\/IJCNN.2011.6033395"},{"issue":"4","key":"3_CR39","doi-asserted-by":"publisher","first-page":"970","DOI":"10.1007\/s11263-019-01232-x","volume":"128","author":"R Sun","year":"2020","unstructured":"Sun, R., Lampert, C.H.: KS(conf): a light-weight test if a multiclass classifier operates outside of its specifications. Int. J. Comput. Vis. 128(4), 970\u2013995 (2020). https:\/\/doi.org\/10.1007\/s11263-019-01232-x","journal-title":"Int. J. Comput. Vis."},{"key":"3_CR40","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"270","DOI":"10.1007\/978-3-030-01424-7_27","volume-title":"Artificial Neural Networks and Machine Learning \u2013 ICANN 2018","author":"C Tan","year":"2018","unstructured":"Tan, C., Sun, F., Kong, T., Zhang, W., Yang, C., Liu, C.: A survey on deep transfer learning. In: K\u016frkov\u00e1, V., Manolopoulos, Y., Hammer, B., Iliadis, L., Maglogiannis, I. (eds.) ICANN 2018. LNCS, vol. 11141, pp. 270\u2013279. Springer, Cham (2018). https:\/\/doi.org\/10.1007\/978-3-030-01424-7_27"},{"key":"3_CR41","doi-asserted-by":"publisher","unstructured":"Tobin, J., Fong, R., Ray, A., Schneider, J., Zaremba, W., Abbeel, P.: Domain randomization for transferring deep neural networks from simulation to the real world. In: IROS, pp. 23\u201330. IEEE (2017). https:\/\/doi.org\/10.1109\/IROS.2017.8202133","DOI":"10.1109\/IROS.2017.8202133"},{"key":"3_CR42","unstructured":"Wagstaff, K.L., Lu, S.: Efficient active learning for new domains. In: Workshop on Real World Experiment Design and Active Learning (2020)"},{"key":"3_CR43","unstructured":"Wu, C., Falcone, Y., Bensalem, S.: Customizable reference runtime monitoring of neural networks using resolution boxes. CoRR abs\/2104.14435 (2021). https:\/\/arxiv.org\/abs\/2104.14435"},{"key":"3_CR44","unstructured":"Xiao, H., Rasul, K., Vollgraf, R.: Fashion-MNIST: a novel image dataset for benchmarking machine learning algorithms. CoRR abs\/1708.07747 (2017). http:\/\/arxiv.org\/abs\/1708.07747"},{"key":"3_CR45","doi-asserted-by":"publisher","unstructured":"Zhang, P., Wang, J., Farhadi, A., Hebert, M., Parikh, D.: Predicting failures of vision systems. In: CVPR, pp. 3566\u20133573. IEEE Computer Society (2014). https:\/\/doi.org\/10.1109\/CVPR.2014.456","DOI":"10.1109\/CVPR.2014.456"},{"issue":"10","key":"3_CR46","doi-asserted-by":"publisher","first-page":"1943","DOI":"10.1109\/TPAMI.2015.2502579","volume":"38","author":"X Zhang","year":"2016","unstructured":"Zhang, X., Zou, J., He, K., Sun, J.: Accelerating very deep convolutional networks for classification and detection. IEEE Trans. Pattern Anal. Mach. Intell. 38(10), 1943\u20131955 (2016). https:\/\/doi.org\/10.1109\/TPAMI.2015.2502579","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"key":"3_CR47","unstructured":"Zhao, P., Hoi, S.C.H.: OTL: a framework of online transfer learning. In: ICML, pp. 1231\u20131238. Omnipress (2010). https:\/\/icml.cc\/Conferences\/2010\/papers\/219.pdf"}],"container-title":["Lecture Notes in Computer Science","Runtime Verification"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-88494-9_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,10,5]],"date-time":"2021-10-05T14:03:26Z","timestamp":1633442606000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-88494-9_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030884932","9783030884949"],"references-count":47,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-88494-9_3","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"6 October 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"RV","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"International Conference on Runtime Verification","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 October 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 October 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"21","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"rv2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/uva-mcps-lab.github.io\/RV21\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"40","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"11","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"7","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"28% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.5","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}