{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,26]],"date-time":"2025-03-26T18:29:20Z","timestamp":1743013760532,"version":"3.40.3"},"publisher-location":"Cham","reference-count":22,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030473570"},{"type":"electronic","value":"9783030473587"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-47358-7_10","type":"book-chapter","created":{"date-parts":[[2020,5,5]],"date-time":"2020-05-05T20:03:35Z","timestamp":1588709015000},"page":"105-117","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Empirical Confidence Models for Supervised Machine Learning"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4689-6143","authenticated-orcid":false,"given":"Margarita P.","family":"Castro","sequence":"first","affiliation":[]},{"given":"Meinolf","family":"Sellmann","sequence":"additional","affiliation":[]},{"given":"Zhaoyuan","family":"Yang","sequence":"additional","affiliation":[]},{"given":"Nurali","family":"Virani","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,5,6]]},"reference":[{"key":"10_CR1","unstructured":"Amodei, D., Olah, C., Steinhardt, J., Christiano, P., Schulman, J., Man\u00e9, D.: Concrete problems in ai safety. arXiv preprint arXiv:1606.06565 (2016)"},{"issue":"2","key":"10_CR2","doi-asserted-by":"publisher","first-page":"391","DOI":"10.1007\/s10115-012-0600-z","volume":"35","author":"S Bhattacharyya","year":"2013","unstructured":"Bhattacharyya, S.: Confidence in predictions from random tree ensembles. Knowl. Inf. Syst. 35(2), 391\u2013410 (2013)","journal-title":"Knowl. Inf. Syst."},{"issue":"3","key":"10_CR3","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1007\/s10489-007-0084-9","volume":"29","author":"Z Bosni\u0107","year":"2008","unstructured":"Bosni\u0107, Z., Kononenko, I.: Estimation of individual prediction reliability using the local sensitivity analysis. Appl. Intell. 29(3), 187\u2013203 (2008)","journal-title":"Appl. Intell."},{"issue":"1","key":"10_CR4","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1017\/S0269888909990154","volume":"25","author":"Z Bosni\u0107","year":"2010","unstructured":"Bosni\u0107, Z., Kononenko, I.: Automatic selection of reliability estimates for individual regression predictions. Knowl. Eng. Rev. 25(1), 27\u201347 (2010)","journal-title":"Knowl. Eng. Rev."},{"issue":"6","key":"10_CR5","first-page":"929","volume":"29","author":"Z Bosni\u0107","year":"2010","unstructured":"Bosni\u0107, Z., Kononenko, I.: Correction of regression predictions using the secondary learner on the sensitivity analysis outputs. Comput. Inform. 29(6), 929\u2013946 (2010)","journal-title":"Comput. Inform."},{"key":"10_CR6","unstructured":"Christiano, P.F., Leike, J., Brown, T., Martic, M., Legg, S., Amodei, D.: Deep reinforcement learning from human preferences. In: Advances in Neural Information Processing Systems, pp. 4299\u20134307 (2017)"},{"issue":"1","key":"10_CR7","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1037\/xge0000033","volume":"144","author":"BJ Dietvorst","year":"2015","unstructured":"Dietvorst, B.J., Simmons, J.P., Massey, C.: Algorithm aversion: people erroneously avoid algorithms after seeing them err. J. Exp. Psychol. Gen. 144(1), 114\u2013126 (2015)","journal-title":"J. Exp. Psychol. Gen."},{"key":"10_CR8","unstructured":"Asuncion, A., Newman, D.: UCI Machine Learning Repository. University of California, School of Information and Computer Science, Irvine, CA (2007)"},{"issue":"3","key":"10_CR9","doi-asserted-by":"publisher","first-page":"307","DOI":"10.1109\/TSE.2013.59","volume":"40","author":"A Groce","year":"2014","unstructured":"Groce, A., et al.: You are the only possible oracle: effective test selection for end users of interactive machine learning systems. IEEE Trans. Software Eng. 40(3), 307\u2013323 (2014)","journal-title":"IEEE Trans. Software Eng."},{"key":"10_CR10","unstructured":"Heskes, T.: Practical confidence and prediction intervals. In: Advances in Neural Information Processing Systems, pp. 176\u2013182 (1997)"},{"key":"10_CR11","unstructured":"Jiang, H., Kim, B., Guan, M.Y., Gupta, M.: To trust or not to trust a classifier. In: 32nd International Conference on Neural Information Processing Systems, pp. 5546\u20135557 (2018)"},{"issue":"3","key":"10_CR12","first-page":"18","volume":"2","author":"A Liaw","year":"2002","unstructured":"Liaw, A., Wiener, M., et al.: Classification and regression by randomforest. R News 2(3), 18\u201322 (2002)","journal-title":"R News"},{"key":"10_CR13","unstructured":"Lipton, Z.C., Wang, Y.X., Smola, A.: Detecting and correcting for label shift with black box predictors. arXiv preprint arXiv:1802.03916 (2018)"},{"key":"10_CR14","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1016\/j.ins.2016.09.034","volume":"374","author":"EY Matsumoto","year":"2016","unstructured":"Matsumoto, E.Y., Del-Moral-Hernandez, E.: Improving regression predictions using individual point reliability estimates based on critical error scenarios. Inf. Sci. 374, 65\u201384 (2016)","journal-title":"Inf. Sci."},{"key":"10_CR15","doi-asserted-by":"crossref","unstructured":"Nushi, B., Kamar, E., Horvitz, E.: Towards accountable AI: hybrid human-machine analyses for characterizing system failure. In: Sixth AAAI Conference on Human Computation and Crowdsourcing (2018)","DOI":"10.1609\/hcomp.v6i1.13337"},{"key":"10_CR16","doi-asserted-by":"publisher","first-page":"815","DOI":"10.1613\/jair.3198","volume":"40","author":"H Papadopoulos","year":"2011","unstructured":"Papadopoulos, H., Vovk, V., Gammerman, A.: Regression conformal prediction with nearest neighbours. J. Artif. Intell. Res. 40, 815\u2013840 (2011)","journal-title":"J. Artif. Intell. Res."},{"key":"10_CR17","doi-asserted-by":"crossref","unstructured":"Papernot, N., McDaniel, P., Jha, S., Fredrikson, M., Celik, Z.B., Swami, A.: The limitations of deep learning in adversarial settings. In: 2016 IEEE European Symposium on Security and Privacy (EuroS&P), pp. 372\u2013387. IEEE (2016)","DOI":"10.1109\/EuroSP.2016.36"},{"issue":"4","key":"10_CR18","doi-asserted-by":"publisher","first-page":"790","DOI":"10.1007\/s10489-014-0632-z","volume":"42","author":"D Pevec","year":"2014","unstructured":"Pevec, D., Kononenko, I.: Prediction intervals in supervised learning for model evaluation and discrimination. Appl. Intell. 42(4), 790\u2013804 (2014). https:\/\/doi.org\/10.1007\/s10489-014-0632-z","journal-title":"Appl. Intell."},{"issue":"6","key":"10_CR19","doi-asserted-by":"publisher","first-page":"691","DOI":"10.1002\/for.2464","volume":"36","author":"A Prahl","year":"2017","unstructured":"Prahl, A., Van Swol, L.: Understanding algorithm aversion: when is advice from automation discounted? J. Forecast. 36(6), 691\u2013702 (2017)","journal-title":"J. Forecast."},{"key":"10_CR20","first-page":"371","volume":"9","author":"G Shafer","year":"2008","unstructured":"Shafer, G., Vovk, V.: A tutorial on conformal prediction. J. Mach. Learn. Res. 9, 371\u2013421 (2008)","journal-title":"J. Mach. Learn. Res."},{"issue":"3","key":"10_CR21","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1023\/A:1018628609742","volume":"9","author":"JA Suykens","year":"1999","unstructured":"Suykens, J.A., Vandewalle, J.: Least squares support vector machine classifiers. Neural Process. Lett. 9(3), 293\u2013300 (1999)","journal-title":"Neural Process. Lett."},{"key":"10_CR22","doi-asserted-by":"crossref","unstructured":"Yeomans, M., Shah, A., Mullainathan, S., Kleinberg, J.: Making sense of recommendations. J. Behav. Decis. Making 32(4), 403\u2013414 (2019)","DOI":"10.1002\/bdm.2118"}],"container-title":["Lecture Notes in Computer Science","Advances in Artificial Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-47358-7_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T15:05:47Z","timestamp":1710255947000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-47358-7_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030473570","9783030473587"],"references-count":22,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-47358-7_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"6 May 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"Canadian AI","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Canadian Conference on Artificial Intelligence","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Ottawa, ON","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Canada","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"13 May 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"15 May 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"33","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"canadianai2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/www.caiac.ca\/en\/conferences\/canadianai-2020\/home","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"175","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"31","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"24","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"18% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference was held virtually due to the Corona pandemic. In addition, the proceedings include 4 contributions from the Graduate Student Symposium.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}