{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,26]],"date-time":"2026-02-26T15:24:46Z","timestamp":1772119486835,"version":"3.50.1"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2024,3,4]],"date-time":"2024-03-04T00:00:00Z","timestamp":1709510400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2024,3,4]],"date-time":"2024-03-04T00:00:00Z","timestamp":1709510400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Knowl Inf Syst"],"published-print":{"date-parts":[[2024,6]]},"DOI":"10.1007\/s10115-024-02073-y","type":"journal-article","created":{"date-parts":[[2024,3,4]],"date-time":"2024-03-04T14:02:27Z","timestamp":1709560947000},"page":"3497-3521","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Explaining deep multi-class time series classifiers"],"prefix":"10.1007","volume":"66","author":[{"given":"Ramesh","family":"Doddaiah","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Prathyush S.","family":"Parvatharaju","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Elke","family":"Rundensteiner","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Thomas","family":"Hartvigsen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2024,3,4]]},"reference":[{"key":"2073_CR1","doi-asserted-by":"crossref","unstructured":"Dixon MF, London J (2020) Financial forecasting with $$\\alpha $$-RNNs: a time series modeling approach. Front Appl Math Stat","DOI":"10.3389\/fams.2020.551138"},{"issue":"5","key":"2073_CR2","doi-asserted-by":"publisher","first-page":"527","DOI":"10.1016\/j.cels.2017.10.001","volume":"5","author":"BD Fulcher","year":"2017","unstructured":"Fulcher BD, Jones NS (2017) hctsa: A computational framework for automated time-series phenotyping using massive feature extraction. Cell Syst 5(5):527\u20135313","journal-title":"Cell Syst"},{"key":"2073_CR3","doi-asserted-by":"crossref","unstructured":"Che Z, Purushotham S, Cho K, Sontag DA, Liu Y (2018) Recurrent neural networks for multivariate time series with missing values. Sci Rep 8","DOI":"10.1038\/s41598-018-24271-9"},{"key":"2073_CR4","doi-asserted-by":"crossref","unstructured":"Li G, Choi B, Xu J, Bhowmick SS, Chun K-P, Wong GL (2021) Shapenet: A shapelet-neural network approach for multivariate time series classification. In: AAAI","DOI":"10.1609\/aaai.v35i9.17018"},{"key":"2073_CR5","unstructured":"Tonekaboni S, Joshi S, Duvenaud D, Goldenberg A (2020) What went wrong and when? Instance-wise feature importance for time-series models. Adv Neural Inf Process Syst"},{"key":"2073_CR6","doi-asserted-by":"crossref","unstructured":"Schlegel U, Vo DL, Keim DA, Seebacher D (2021) Ts-mule: Local interpretable model-agnostic explanations for time series forecast models. In: PKDD\/ECML Workshops","DOI":"10.1007\/978-3-030-93736-2_1"},{"key":"2073_CR7","doi-asserted-by":"crossref","unstructured":"Parvatharaju PS, Doddaiah R, Hartvigsen T, Rundensteiner EA (2021) Learning saliency maps to explain deep time series classifiers. In: Proceedings of the 30th ACM international conference on information & knowledge management","DOI":"10.1145\/3459637.3482446"},{"key":"2073_CR8","unstructured":"Crabbe J, van\u00a0der Schaar M (2021) Explaining time series predictions with dynamic masks. In: ICML"},{"key":"2073_CR9","doi-asserted-by":"crossref","unstructured":"Bento J, Saleiro P, Cruz AF, Figueiredo MAT, Bizarro P (2020) Timeshap: explaining recurrent models through sequence perturbations. In: Proceedings of the 27th ACM SIGKDD conference on knowledge discovery & data mining","DOI":"10.1145\/3447548.3467166"},{"key":"2073_CR10","doi-asserted-by":"crossref","unstructured":"Guidotti R, Monreale A, Spinnato F, Pedreschi D, Giannotti F (2020) Explaining any time series classifier. In: International conference on cognitive machine intelligence","DOI":"10.1109\/CogMI50398.2020.00029"},{"key":"2073_CR11","doi-asserted-by":"crossref","unstructured":"Guillem\u00e9 M, Masson V, Roz\u00e9 L, Termier A (2019) Agnostic local explanation for time series classification. In: ICTAI, pp 432\u2013439","DOI":"10.1109\/ICTAI.2019.00067"},{"key":"2073_CR12","doi-asserted-by":"crossref","unstructured":"Ribeiro MT, Singh S, Guestrin C (2016) \u201cWhy should I trust you?\u201d: explaining the predictions of any classifier. CoRR arXiv:1602.04938","DOI":"10.1145\/2939672.2939778"},{"key":"2073_CR13","unstructured":"Lundberg SM, Lee S-I (2017) A unified approach to interpreting model predictions. In: NIPS"},{"key":"2073_CR14","unstructured":"Ismail AA, Gunady M, Bravo HC, Feizi S (2020) Benchmarking deep learning interpretability in time series predictions. In: Proceedings of the 34th international conference on neural information processing systems. NIPS\u201920. Curran Associates Inc., Red Hook"},{"key":"2073_CR15","doi-asserted-by":"crossref","unstructured":"Shimoda W, Yanai K (2016) Distinct class-specific saliency maps for weakly supervised semantic segmentation. In: ECCV","DOI":"10.1007\/978-3-319-46493-0_14"},{"key":"2073_CR16","doi-asserted-by":"crossref","unstructured":"Kaur H, Nori H, Jenkins S, Caruana R, Wallach H, Wortman\u00a0Vaughan J (2020) Interpreting interpretability: Understanding data scientists\u2019 use of interpretability tools for machine learning. In: Proceedings of the 2020 CHI conference on human factors in computing systems","DOI":"10.1145\/3313831.3376219"},{"key":"2073_CR17","doi-asserted-by":"crossref","unstructured":"Ribeiro A, Ribeiro M, Paix\u00e3o G, Oliveira D, Gomes P, Canazart A, Ferreira P, Andersson C, Macfarlane P (2020) Automatic diagnosis of the 12-lead ECG using a deep neural network. Nat Commun 11(1)","DOI":"10.1038\/s41467-020-15432-4"},{"key":"2073_CR18","unstructured":"Mujkanovic F, Dosko\u010d V, Schirneck M, Sch\u00e4fer P, Friedrich T (2020) timeXplain\u2014A framework for explaining the predictions of time series classifiers. arXiv e-prints 2007.07606 [cs.LG]"},{"key":"2073_CR19","doi-asserted-by":"crossref","unstructured":"Fong R, Vedaldi A (2017) Interpretable explanations of black boxes by meaningful perturbation. In: ICCV, pp 3449\u20133457","DOI":"10.1109\/ICCV.2017.371"},{"key":"2073_CR20","unstructured":"Kumar IE, Venkatasubramanian S, Scheidegger CE, Friedler SA (2020) Problems with shapley-value-based explanations as feature importance measures. In: ICML"},{"key":"2073_CR21","doi-asserted-by":"crossref","unstructured":"Fong R, Patrick M, Vedaldi A (2019) Understanding deep networks via extremal perturbations and smooth masks. In: ICCV, pp. 2950\u20132958","DOI":"10.1109\/ICCV.2019.00304"},{"key":"2073_CR22","doi-asserted-by":"crossref","unstructured":"Doddaiah R, Parvatharaju P, Rundensteiner E, Hartvigsen T (2022) Class-specific explainability for deep time series classifiers. In: 2022 IEEE International conference on data mining (ICDM)","DOI":"10.1109\/ICDM54844.2022.00020"},{"key":"2073_CR23","doi-asserted-by":"crossref","unstructured":"Kelany OE, Aly S, Ismail MA (2020) Deep learning model for financial time series prediction. In: 2020 14th International conference on innovations in information technology (IIT)","DOI":"10.1109\/IIT50501.2020.9299063"},{"key":"2073_CR24","doi-asserted-by":"publisher","first-page":"242","DOI":"10.1016\/j.procs.2017.09.066","volume":"114","author":"N Mehdiyev","year":"2017","unstructured":"Mehdiyev N, Lahann J, Emrich A, Enke D, Fettke P, Loos P (2017) Time series classification using deep learning for process planning: a case from the process industry. Proc Comput Sci 114:242\u2013249","journal-title":"Proc Comput Sci"},{"key":"2073_CR25","doi-asserted-by":"publisher","first-page":"75213","DOI":"10.1109\/ACCESS.2019.2920969","volume":"7","author":"Y Zhang","year":"2019","unstructured":"Zhang Y, Zhang Z, Zhang Y, Bao J, Zhang Y, Deng H (2019) Human activity recognition based on motion sensor using u-net. IEEE Access 7:75213\u201375226","journal-title":"IEEE Access"},{"key":"2073_CR26","doi-asserted-by":"crossref","unstructured":"Schlegel U, Arnout H, El-Assady M, Oelke D, Keim DA (2019) Towards a rigorous evaluation of xai methods on time series. In: 2019 IEEE\/CVF International conference on computer vision workshop (ICCVW), pp 4197\u20134201","DOI":"10.1109\/ICCVW.2019.00516"},{"key":"2073_CR27","unstructured":"Zhang H, Ciss\u00e9 M, Dauphin YN, Lopez-Paz D (2018) mixup: Beyond empirical risk minimization. In: 6th International conference on learning representations, ICLR 2018, Vancouver, BC, Canada, April 30\u2013May 3, 2018, conference track proceedings. OpenReview.net"},{"key":"2073_CR28","doi-asserted-by":"crossref","unstructured":"Satopaa V, Albrecht JR, Irwin DE, Raghavan B (2011) Finding a \u201ckneedle\u201d in a haystack: Detecting knee points in system behavior. In: 2011 31st international conference on distributed computing systems workshops","DOI":"10.1109\/ICDCSW.2011.20"},{"key":"2073_CR29","unstructured":"Schaul T, Quan J, Antonoglou I, Silver D (2016) Prioritized experience replay. CoRR arXiv:1511.05952"},{"key":"2073_CR30","unstructured":"Mnih V, Kavukcuoglu K, Silver D, Graves A, Antonoglou I, Wierstra D, Riedmiller MA (2013) Playing Atari with deep reinforcement learning. arXiv arXiv:1312.5602"},{"key":"2073_CR31","doi-asserted-by":"crossref","unstructured":"Gisler C, Ridi A, Zufferey D, Khaled OA, Hennebert J (2013) Appliance consumption signature database and recognition test protocols. In: 2013 8th International workshop on systems, signal processing and their applications (WoSSPA), pp 336\u2013341","DOI":"10.1109\/WoSSPA.2013.6602387"},{"key":"2073_CR32","unstructured":"Chen Y, Keogh E, Hu B, Begum N, Bagnall A, Mueen A, Batista G (2015) The UCR time series classification archive. www.cs.ucr.edu\/~eamonn\/time_series_data\/"},{"key":"2073_CR33","unstructured":"Roverso D (2000) Multivariate temporal classification by windowed wavelet decomposition and recurrent neural networks"},{"key":"2073_CR34","doi-asserted-by":"publisher","first-page":"711","DOI":"10.1016\/j.rse.2008.11.007","volume":"113","author":"AM Baldridge","year":"2009","unstructured":"Baldridge AM, Hook SJ, Grove CI, Rivera G (2009) The aster spectral library version 2.0. Remote Sens Environ 113:711\u2013715. https:\/\/doi.org\/10.1016\/j.rse.2008.11.007","journal-title":"Remote Sens Environ"},{"issue":"23","key":"2073_CR35","doi-asserted-by":"publisher","first-page":"215","DOI":"10.1161\/01.CIR.101.23.e215","volume":"101","author":"AL Goldberger","year":"2000","unstructured":"Goldberger AL, Amaral LAN, Glass L, Hausdorff JM, Ivanov PC, Mark RG, Mietus JE, Moody GB, Peng C-K, Stanley HE (2000) Physiobank, physiotoolkit, and physionet: components of a new research resource for complex physiologic signals. Circulation 101(23):215\u201320","journal-title":"Circulation"},{"issue":"2","key":"2073_CR36","doi-asserted-by":"publisher","first-page":"195","DOI":"10.1016\/S0308-8146(96)00289-0","volume":"59","author":"O Al-Jowder","year":"1997","unstructured":"Al-Jowder O, Kemsley EK, Wilson RH (1997) Mid-infrared spectroscopy and authenticity problems in selected meats. Food Chem 59(2):195\u2013201. https:\/\/doi.org\/10.1016\/S0308-8146(96)00289-0","journal-title":"Food Chem"},{"key":"2073_CR37","doi-asserted-by":"publisher","first-page":"917","DOI":"10.1007\/s10618-019-00619-1","volume":"33","author":"HI Fawaz","year":"2018","unstructured":"Fawaz HI, Forestier G, Weber J, Idoumghar L, Muller P-A (2018) Deep learning for time series classification: a review. Data Min Knowl Disc 33:917\u2013963","journal-title":"Data Min Knowl Disc"},{"key":"2073_CR38","doi-asserted-by":"crossref","unstructured":"Wang Z, Yan W, Oates T (2017) Time series classification from scratch with deep neural networks: a strong baseline. IJCNN","DOI":"10.1109\/IJCNN.2017.7966039"},{"key":"2073_CR39","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1007\/978-94-009-3677-5_7","volume-title":"Econometrics of planning and efficiency","author":"A Charnes","year":"1988","unstructured":"Charnes A, Golany B, Keane M, Rousseau J (1988) Extremal principle solutions of games in characteristic function form: core, Chebychev and Shapley value generalizations. Econometrics of planning and efficiency. Springer, Dordrecht, pp 123\u2013133"},{"key":"2073_CR40","unstructured":"Petsiuk V, Das A, Saenko K (2018) Rise: randomized input sampling for explanation of black-box models. CoRR arXiv:1806.07421"},{"key":"2073_CR41","unstructured":"Kingma DP, Ba J (2015) Adam: a method for stochastic optimization. CoRR arXiv:1412.6980"},{"key":"2073_CR42","unstructured":"Biewald L (2020) Experiment tracking with weights and biases. Software available from wandb.com. https:\/\/www.wandb.com\/"},{"key":"2073_CR43","unstructured":"Hama N, Mase M, Owen AB (2022) Deletion and insertion tests in regression models"}],"container-title":["Knowledge and Information Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02073-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10115-024-02073-y\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10115-024-02073-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,29]],"date-time":"2024-05-29T00:12:38Z","timestamp":1716941558000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10115-024-02073-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,4]]},"references-count":43,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2024,6]]}},"alternative-id":["2073"],"URL":"https:\/\/doi.org\/10.1007\/s10115-024-02073-y","relation":{"has-preprint":[{"id-type":"doi","id":"10.21203\/rs.3.rs-2531572\/v1","asserted-by":"object"}]},"ISSN":["0219-1377","0219-3116"],"issn-type":[{"value":"0219-1377","type":"print"},{"value":"0219-3116","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,3,4]]},"assertion":[{"value":"31 January 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 November 2023","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 February 2024","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 March 2024","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}]}}