{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,18]],"date-time":"2026-05-18T10:48:34Z","timestamp":1779101314496,"version":"3.51.4"},"reference-count":43,"publisher":"Springer Science and Business Media LLC","issue":"6","license":[{"start":{"date-parts":[[2016,3,1]],"date-time":"2016-03-01T00:00:00Z","timestamp":1456790400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Bus Inf Syst Eng"],"published-print":{"date-parts":[[2016,12]]},"DOI":"10.1007\/s12599-016-0427-3","type":"journal-article","created":{"date-parts":[[2016,3,1]],"date-time":"2016-03-01T15:37:38Z","timestamp":1456846658000},"page":"397-414","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["Evaluation Methods in Process-Aware Information Systems Research with a Perspective on Human Orientation"],"prefix":"10.1007","volume":"58","author":[{"given":"Simone","family":"Kriglstein","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Maria","family":"Leitner","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Sonja","family":"Kabicher-Fuchs","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Stefanie","family":"Rinderle-Ma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2016,3,1]]},"reference":[{"key":"427_CR1","unstructured":"Auer-Srnka KJ, Koeszegi S (2007) From words to numbers: how to transform qualitative data into meaningful quantitative results. Schmalenbach, Bus Rev 59"},{"key":"427_CR2","doi-asserted-by":"crossref","first-page":"104","DOI":"10.1007\/BF03177550","volume":"1","author":"H Cooper","year":"1988","unstructured":"Cooper H (1988) Organizing knowledge syntheses: a taxonomy of literature reviews. Knowl Technol Policy 1:104\u2013126","journal-title":"Knowl Technol Policy"},{"key":"427_CR3","volume-title":"Understanding your users: a practical guide to user requirements methods, tools, and techniques","author":"C Courage","year":"2004","unstructured":"Courage C, Baxter K (2004) Understanding your users: a practical guide to user requirements methods, tools, and techniques. Morgan Kaufmann, San Francisco"},{"issue":"1","key":"427_CR4","doi-asserted-by":"crossref","first-page":"38","DOI":"10.12968\/bjon.2008.17.1.28059","volume":"17","author":"P Cronin","year":"2008","unstructured":"Cronin P, Ryan F, Coughlan M (2008) Undertaking a literature review: a step-by-step approach. Br J Nurs 17(1):38\u201343","journal-title":"Br J Nurs"},{"key":"427_CR5","doi-asserted-by":"crossref","DOI":"10.1002\/0471741442","volume-title":"Process-aware information systems: bridging people and software through process technology","author":"M Dumas","year":"2005","unstructured":"Dumas M, van der Aalst WMP, ter Hofstede AH (2005) Process-aware information systems: bridging people and software through process technology. Wiley, Hoboken"},{"key":"427_CR6","unstructured":"Effinger P, Seiz S, Jogsch N (2011) Evaluating single features in usability tests for modeling tools. In: Proceedings of the 3rd Workshop \u201dMethodische Entwicklung von Modellierungswerkzeugen\u201d at Informatik 2011, GI-Edition (LNI)"},{"key":"427_CR7","doi-asserted-by":"crossref","first-page":"469","DOI":"10.1007\/11678564_44","volume-title":"Business process management workshops","author":"P Fettke","year":"2006","unstructured":"Fettke P, Loos P, Zwicker J (2006) Business process reference models: survey and classification. In: Bussler CJ, Haller A (eds) Business process management workshops. LNCS, Springer, Heidelberg, pp 469\u2013483"},{"key":"427_CR8","unstructured":"Gediga G, Hamborg KC (2001) Evaluation of software systems. Encycl Comput Sci Technol 45"},{"issue":"8","key":"427_CR9","doi-asserted-by":"crossref","first-page":"491","DOI":"10.1016\/S0950-5849(02)00049-6","volume":"44","author":"R Glass","year":"2002","unstructured":"Glass R, Vessey I, Ramesh V (2002) Research in software engineering: an analysis of the literature. Inf Softw Technol 44(8):491\u2013506","journal-title":"Inf Softw Technol"},{"issue":"1","key":"427_CR10","doi-asserted-by":"crossref","first-page":"75","DOI":"10.2307\/25148625","volume":"28","author":"AR Hevner","year":"2004","unstructured":"Hevner AR, March ST, Park J, Ram S (2004) Design science in information systems research. MIS Q 28(1):75\u2013105","journal-title":"MIS Q"},{"issue":"4","key":"427_CR11","doi-asserted-by":"crossref","first-page":"619","DOI":"10.1108\/14637151011065946","volume":"16","author":"C Houy","year":"2010","unstructured":"Houy C, Fettke P, Loos P (2010) Empirical research in business process management \u2013 analysis of an emerging field of research. Bus Process Manag J 16(4):619\u2013661","journal-title":"Bus Process Manag J"},{"issue":"1","key":"427_CR12","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1016\/S0378-7206(01)00128-8","volume":"40","author":"Z Irani","year":"2002","unstructured":"Irani Z (2002) Information systems evaluation: navigating through the problem domain. Inf Manag 40(1):11\u201324","journal-title":"Inf Manag"},{"key":"427_CR13","doi-asserted-by":"crossref","unstructured":"Kabicher S, Kriglstein S, Rinderle-Ma S (2011) Visual change tracking for business process models. In: Proceedings of the 30th international conference on conceptual modeling, Springer, Heidelberg, ER\u201911, pp 504\u2013513","DOI":"10.1007\/978-3-642-24606-7_41"},{"key":"427_CR14","doi-asserted-by":"crossref","first-page":"678","DOI":"10.1007\/978-3-642-31095-9_44","volume-title":"Proceedings 24th international conference on advanced information systems engineering (CAiSE)","author":"S Kabicher-Fuchs","year":"2012","unstructured":"Kabicher-Fuchs S, Rinderle-Ma S (2012) Work experience in PAIS: concepts, measurements and potentials. Proceedings 24th international conference on advanced information systems engineering (CAiSE). Springer, LNCS, Heidelberg, pp 678\u2013694"},{"key":"427_CR15","unstructured":"Kabicher-Fuchs S, Rinderle-Ma S, Recker J, Indulska M, Charoy F, Christiaanse R, Dunkl R, Grambow G, Kolb J, Leopold H, Mendling J (2012) Human-centric process-aware information systems (HC-PAIS). Research Report arXiv:1211.4986 [cs.HC]"},{"key":"427_CR16","doi-asserted-by":"crossref","unstructured":"Kabicher-Fuchs S, Mangler J, Rinderle-Ma S (2013) Experience breeding in process-aware information systems. In: Int\u2019l Conf. on Advanced information systems engineering, Springer, Heidelberg, pp 594\u2013609","DOI":"10.1007\/978-3-642-38709-8_38"},{"key":"427_CR17","unstructured":"Kitchenham B (2004) Procedures for performing systematic reviews. Joint technical report, Department of Computer Science, Keele University and Empirical Software Engineering, National ICT Australia Ltd"},{"key":"427_CR18","unstructured":"Kitchenham B, Charters S (2007) Guidelines for performing systematic literature reviews in software engineering. EBSE Technical Report EBSE-2007-01 Version 2.3, School of Computer Science and Mathematics, Keele University and Department of Computer Science, University of Durham"},{"key":"427_CR19","doi-asserted-by":"crossref","first-page":"265","DOI":"10.1023\/A:1008764508237","volume":"9","author":"M Klein","year":"2000","unstructured":"Klein M, Dellarocas C, Bernstein A (2000) Introduction to the special issue on adaptive workflow systems. Comput Support Coop Work 9:265\u2013267","journal-title":"Comput Support Coop Work"},{"key":"427_CR20","doi-asserted-by":"crossref","unstructured":"Kriglstein S, Pohl M, Suchy N, G\u00e4rtner J, Gschwandtner T, Miksch S (2014) Experiences and challenges with evaluation methods in practice: A case study. In: Proceedings of the fifth workshop on beyond time and errors: novel evaluation methods for visualization, ACM, BELIV \u201914, pp 118\u2013125","DOI":"10.1145\/2669557.2669571"},{"key":"427_CR21","doi-asserted-by":"crossref","unstructured":"La Rosa M, Dumas M, ter Hofstede AH, Mendling J, Gottschalk F (2007) Beyond control-flow: extending business process configuration to resources and objects. Unpublished","DOI":"10.1007\/978-3-540-87877-3_16"},{"key":"427_CR22","doi-asserted-by":"publisher","first-page":"273","DOI":"10.1016\/j.infsof.2013.12.004","volume":"56","author":"M Leitner","year":"2014","unstructured":"Leitner M, Rinderle-Ma S (2014) A systematic review on security in process-aware information systems constitution, challenges, and future directions. Inf Softw Technol 56:273\u2013293. doi: 10.1016\/j.infsof.2013.12.004","journal-title":"Inf Softw Technol"},{"key":"427_CR23","doi-asserted-by":"crossref","unstructured":"Leitner M, Miller M, Rinderle-Ma S (2013) An analysis and evaluation of security aspects in the business process model and notation. Proceedings of the 8th international conference on availability. reliability and security (ARES), IEEE, pp 262\u2013267","DOI":"10.1109\/ARES.2013.34"},{"issue":"1","key":"427_CR24","doi-asserted-by":"crossref","first-page":"39","DOI":"10.1016\/j.datak.2006.04.007","volume":"61","author":"R Lenz","year":"2007","unstructured":"Lenz R, Reichert M (2007) IT support for healthcare processes \u2013 premises, challenges, perspectives. Data Knowl Eng 61(1):39\u201358","journal-title":"Data Knowl Eng"},{"key":"427_CR25","volume-title":"Qualitative inhaltsanalyse","author":"P Mayring","year":"2003","unstructured":"Mayring P (2003) Qualitative inhaltsanalyse. Beltz, Weinheim"},{"key":"427_CR26","doi-asserted-by":"crossref","unstructured":"Mendling J, Strembeck M (2008) Influence factors of understanding business process models. In: Abramowicz W, Fensel D (eds) Business information systems, no.\u00a07 in lecture notes in business information processing, Springer, Heidelberg, pp 142\u2013153","DOI":"10.1007\/978-3-540-79396-0_13"},{"key":"427_CR27","doi-asserted-by":"crossref","unstructured":"Mendling J, Reijers HA, Cardoso J (2007) What makes process models understandable? In: Alonso G, Dadam P, Rosemann M (eds) Business process management, no. 4714 in lecture notes in computer science, Springer, Heidelberg, pp 48\u201363","DOI":"10.1007\/978-3-540-75183-0_4"},{"key":"427_CR28","unstructured":"Moody DL (2003) Measuring the quality of data models: an empirical evaluation of the use of quality metrics in practice. In: Proceedings of the 11th European conference on information systems, pp 1337\u20131352"},{"key":"427_CR29","doi-asserted-by":"crossref","unstructured":"Pohl M (2012) Methodologies for the analysis of usage patterns in information visualization. In: Proceedings of the BELIV\u201912 workshop: beyond time and errors \u2013 novel evaluation methods for visualization","DOI":"10.1145\/2442576.2442593"},{"issue":"12","key":"427_CR30","doi-asserted-by":"crossref","first-page":"165","DOI":"10.1016\/S0164-1212(03)00015-3","volume":"70","author":"V Ramesh","year":"2004","unstructured":"Ramesh V, Glass RL, Vessey I (2004) Research in computer science: an empirical study. J Syst Softw 70(12):165\u2013176","journal-title":"J Syst Softw"},{"key":"427_CR31","unstructured":"Rozinat A, de\u00a0Medeiros AA, G\u00fcnther C, Weijters A, van\u00a0der Aalst W (2007) Towards an evaluation framework for process mining algorithms. BPM Center Report BPM-07-06, BPMcenter.org"},{"key":"427_CR32","doi-asserted-by":"crossref","first-page":"78","DOI":"10.1109\/MIC.2012.114","volume":"5","author":"S Schulte","year":"2012","unstructured":"Schulte S, Schuller D, Steinmetz R, Abels S (2012) Plug-and-play virtual factories. IEEE Internet Comput 5:78\u201382","journal-title":"IEEE Internet Comput"},{"issue":"3","key":"427_CR33","doi-asserted-by":"crossref","first-page":"251","DOI":"10.1046\/j.1365-2575.2003.00142.x","volume":"13","author":"V Serafeimidis","year":"2003","unstructured":"Serafeimidis V, Smithson S (2003) Information systems evaluation as an organizational institution \u2013 experience from a case study. Inf Syst J 13(3):251\u2013274","journal-title":"Inf Syst J"},{"key":"427_CR34","doi-asserted-by":"crossref","unstructured":"Shneiderman B, Plaisant C (2006) Strategies for evaluating information visualization tools: multi-dimensional in-depth long-term case studies. In: Proceedings of the BELIV\u201906 Workshop on Beyond Time and Errors: Novel Evaluation Methods for Information Visualization, ACM, pp 1\u20137","DOI":"10.1145\/1168149.1168158"},{"issue":"3","key":"427_CR35","first-page":"276","volume":"15","author":"X Song","year":"2012","unstructured":"Song X, Letch N (2012) Research on IT\/IS evaluation: a 25 year review. Electron J Inf Syst Eval (EJISE) 15(3):276\u2013287","journal-title":"Electron J Inf Syst Eval (EJISE)"},{"key":"427_CR36","doi-asserted-by":"crossref","DOI":"10.4135\/9781412991841","volume-title":"Focus groups: theory and practice","author":"DW Stewart","year":"2007","unstructured":"Stewart DW, Shamdasani PN, Rook DW (2007) Focus groups: theory and practice. Sage, Thousand Oaks"},{"key":"427_CR37","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-642-19345-3","volume-title":"Process mining: discovery, conformance and enhancement of business processes","author":"WMP Aalst van der","year":"2011","unstructured":"van der Aalst WMP (2011) Process mining: discovery, conformance and enhancement of business processes. Springer, Heidelberg"},{"key":"427_CR38","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/978-3-642-32885-5_1","volume-title":"Business process management","author":"WMP Aalst van der","year":"2012","unstructured":"van der Aalst WMP (2012) A decade of business process management conferences: personal reflections on a developing discipline. In: Barros A, Gal A, Kindler E (eds) Business process management. LNCS, Springer, Heidelberg, pp 1\u201316"},{"key":"427_CR39","doi-asserted-by":"crossref","unstructured":"van der Aalst WMP (2013) Business process management \u2013 a comprehensive survey. ISRN Softw Eng, article ID 507984","DOI":"10.1155\/2013\/507984"},{"key":"427_CR40","first-page":"80","volume-title":"Proceedings of the 3rd international conference on business process management (bpm)","author":"I Vanderfeesten","year":"2005","unstructured":"Vanderfeesten I, Reijers HA (2005) A human-oriented tuning of workflow management systems. Proceedings of the 3rd international conference on business process management (bpm). Springer, Heidelberg, pp 80\u201395"},{"issue":"3","key":"427_CR41","doi-asserted-by":"crossref","first-page":"261","DOI":"10.1017\/S0269888908001379","volume":"23","author":"L Velsen van","year":"2008","unstructured":"van Velsen L, van der Geest T, Klaassen R, Steehouder M (2008) User-centered evaluation of adaptive and adaptable systems: a literature review. Knowl Eng Rev 23(3):261\u2013281","journal-title":"Knowl Eng Rev"},{"issue":"4","key":"427_CR42","doi-asserted-by":"crossref","first-page":"280","DOI":"10.1007\/s11576-007-0064-z","volume":"49","author":"T Wilde","year":"2007","unstructured":"Wilde T, Hess T (2007) Forschungsmethoden der Wirtschaftsinformatik. WIRTSCHAFTSINFORMATIK 49(4):280\u2013287","journal-title":"WIRTSCHAFTSINFORMATIK"},{"key":"427_CR43","volume-title":"Case study research: design and methods","author":"RK Yin","year":"2003","unstructured":"Yin RK (2003) Case study research: design and methods. Sage, Thousand Oaks"}],"container-title":["Business & Information Systems Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12599-016-0427-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s12599-016-0427-3\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12599-016-0427-3.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s12599-016-0427-3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,9,5]],"date-time":"2019-09-05T04:30:20Z","timestamp":1567657820000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s12599-016-0427-3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,3,1]]},"references-count":43,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2016,12]]}},"alternative-id":["427"],"URL":"https:\/\/doi.org\/10.1007\/s12599-016-0427-3","relation":{},"ISSN":["2363-7005","1867-0202"],"issn-type":[{"value":"2363-7005","type":"print"},{"value":"1867-0202","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,3,1]]}}}