{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,1]],"date-time":"2026-04-01T07:53:40Z","timestamp":1775030020668,"version":"3.50.1"},"publisher-location":"Berlin, Heidelberg","reference-count":41,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783662454886","type":"print"},{"value":"9783662454893","type":"electronic"}],"license":[{"start":{"date-parts":[[2014,1,1]],"date-time":"2014-01-01T00:00:00Z","timestamp":1388534400000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2014]]},"DOI":"10.1007\/978-3-662-45489-3_6","type":"book-chapter","created":{"date-parts":[[2014,11,3]],"date-time":"2014-11-03T10:42:42Z","timestamp":1415011362000},"page":"156-241","source":"Crossref","is-referenced-by-count":7,"title":["Markov Reward Models and Markov Decision Processes in Discrete and Continuous Time: Performance Evaluation and Optimization"],"prefix":"10.1007","author":[{"given":"Alexander","family":"Gouberman","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markus","family":"Siegle","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"6_CR1","unstructured":"Altman, E.: Constrained Markov Decision Processes. Chapman & Hall (1999)"},{"key":"6_CR2","series-title":"International Series in Operations Research & Management Science","doi-asserted-by":"publisher","first-page":"489","DOI":"10.1007\/978-1-4615-0805-2_16","volume-title":"Handbook of Markov Decision Processes","author":"E. Altman","year":"2002","unstructured":"Altman, E.: Applications of Markov Decision Processes in Communication Networks. In: Feinberg, E.A., Shwartz, A. (eds.) Handbook of Markov Decision Processes. International Series in Operations Research & Management Science, vol.\u00a040, pp. 489\u2013536. Springer, US (2002)"},{"issue":"6","key":"6_CR3","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1109\/TSE.2003.1205180","volume":"29","author":"C. Baier","year":"2003","unstructured":"Baier, C., Haverkort, B., Hermanns, H., Katoen, J.-P.: Model-Checking Algorithms for Continuous-Time Markov Chains. IEEE Transactions on Software Engineering\u00a029(6), 524\u2013541 (2003)","journal-title":"IEEE Transactions on Software Engineering"},{"key":"6_CR4","doi-asserted-by":"crossref","unstructured":"B\u00e4uerle, N., Rieder, U.: Markov Decision Processes with Applications to Finance. Springer, Heidelberg (2011)","DOI":"10.1007\/978-3-642-18324-9"},{"key":"6_CR5","volume-title":"Dynamic Programming","author":"R. Bellman","year":"1957","unstructured":"Bellman, R.: Dynamic Programming. Princeton University Press, Princeton (1957)"},{"key":"6_CR6","doi-asserted-by":"publisher","first-page":"813","DOI":"10.1109\/43.766730","volume":"18","author":"L. Benini","year":"1998","unstructured":"Benini, L., Bogliolo, A., Paleologo, G.A., De Micheli, G.: Policy Optimization for Dynamic Power Management. IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems\u00a018, 813\u2013833 (1998)","journal-title":"IEEE Transactions on Computer-Aided Design of Integrated Circuits and Systems"},{"key":"6_CR7","unstructured":"Bertsekas, D.: Dynamic Programming and Optimal Control, 3rd edn., vol.\u00a0I. Athena Scientific (1995) (revised in 2005)"},{"key":"6_CR8","unstructured":"Bertsekas, D.: Dynamic Programming and Optimal Control, 4th edn., vol.\u00a0II. Athena Scientific (1995) (revised in 2012)"},{"issue":"3","key":"6_CR9","doi-asserted-by":"publisher","first-page":"580","DOI":"10.1287\/moor.16.3.580","volume":"16","author":"D. Bertsekas","year":"1991","unstructured":"Bertsekas, D., Tsitsiklis, J.: An analysis of stochastic shortest path problems. Mathematics of Operations Research\u00a016(3), 580\u2013595 (1991)","journal-title":"Mathematics of Operations Research"},{"key":"6_CR10","doi-asserted-by":"crossref","unstructured":"Bertsekas, D., Tsitsiklis, J.: Neuro-Dynamic Programming, 1st edn. Athena Scientific (1996)","DOI":"10.1109\/CDC.1995.478953"},{"key":"6_CR11","unstructured":"Beynier, A., Mouaddib, A.I.: Decentralized Markov decision processes for handling temporal and resource constraints in a multiple robot system. In: Proceedings of the 7th International Symposium on Distributed Autonomous Robotic System, DARS (2004)"},{"key":"6_CR12","doi-asserted-by":"crossref","unstructured":"Bolch, G., Greiner, S., de Meer, H., Trivedi, K.S.: Queueing Networks and Markov Chains - Modelling and Performance Evaluation with Computer Science Applications, 2nd edn. Wiley (2006)","DOI":"10.1002\/0471791571"},{"key":"6_CR13","unstructured":"Cassandra, A.R.: A survey of POMDP applications. In: Working Notes of AAAI 1998 Fall Symposium on Planning with Partially Observable Markov Decision Processes, pp. 17\u201324 (1998)"},{"key":"6_CR14","unstructured":"Diz, F.J., Palacios, M.A., Arias, M.: MDPs in medicine: opportunities and challenges. In: Decision Making in Partially Observable, Uncertain Worlds: Exploring Insights from Multiple Communities, IJCAI Workshop (2011)"},{"issue":"9","key":"6_CR15","doi-asserted-by":"publisher","first-page":"619","DOI":"10.1145\/364063.364082","volume":"11","author":"B.L. Fox","year":"1968","unstructured":"Fox, B.L., Landi, D.M.: An algorithm for identifying the ergodic subchains and transient states of a stochastic matrix. Communications of the ACM\u00a011(9), 619\u2013621 (1968)","journal-title":"Communications of the ACM"},{"key":"6_CR16","doi-asserted-by":"crossref","unstructured":"Gouberman, A., Siegle, M.: On Lifetime Optimization of Boolean Parallel Systems with Erlang Repair Distributions. In: Operations Research Proceedings 2010 - Selected Papers of the Annual International Conference of the German Operations Research Society, pp. 187\u2013192. Springer (January 2011)","DOI":"10.1007\/978-3-642-20009-0_30"},{"key":"6_CR17","doi-asserted-by":"crossref","unstructured":"Guo, X., Hernandez-Lerma, O.: Continuous-Time Markov Decision Processes - Theory and Applications. Springer (2009)","DOI":"10.1007\/978-3-642-02547-1"},{"issue":"3","key":"6_CR18","doi-asserted-by":"publisher","first-page":"381","DOI":"10.1017\/S0269964807000034","volume":"21","author":"B. Heidergott","year":"2007","unstructured":"Heidergott, B., Hordijk, A., Van Uitert, M.: Series Expansions For Finite-State Markov Chains. Probability in the Engineering and Informational Sciences\u00a021(3), 381\u2013400 (2007)","journal-title":"Probability in the Engineering and Informational Sciences"},{"key":"6_CR19","doi-asserted-by":"crossref","unstructured":"Hou, Z., Filar, J.A., Chen, A. (eds.): Markov Processes and Controlled Markov Chains. Springer (2002)","DOI":"10.1007\/978-1-4613-0265-0"},{"key":"6_CR20","volume-title":"Dynamic Programming and Markov Processes","author":"R.A. Howard","year":"1960","unstructured":"Howard, R.A.: Dynamic Programming and Markov Processes. John Wiley & Sons, New York (1960)"},{"key":"6_CR21","unstructured":"Hu, Q., Yue, W.: Markov Decision Processes with their Applications. Springer (2008)"},{"key":"6_CR22","first-page":"247","volume-title":"Applied Semi-Markov Processes","author":"J. Janssen","year":"2006","unstructured":"Janssen, J., Manca, R.: Markov and Semi-Markov Reward Processes. In: Applied Semi-Markov Processes, pp. 247\u2013293. Springer, US (2006)"},{"key":"6_CR23","unstructured":"Janssen, J., Manca, R.: Semi-Markov Risk Models for Finance, Insurance and Reliability. Springer (2007)"},{"key":"6_CR24","first-page":"87","volume":"36","author":"A. Jensen","year":"1953","unstructured":"Jensen, A.: Markoff chains as an aid in the study of Markoff processes. Skandinavisk Aktuarietidskrift\u00a036, 87\u201391 (1953)","journal-title":"Skandinavisk Aktuarietidskrift"},{"issue":"1-3","key":"6_CR25","doi-asserted-by":"publisher","first-page":"291","DOI":"10.1007\/BF01158935","volume":"13","author":"S. Stidham Jr.","year":"1993","unstructured":"Stidham Jr., S., Weber, R.: A survey of Markov decision models for control of networks of queues. Queueing Systems\u00a013(1-3), 291\u2013314 (1993)","journal-title":"Queueing Systems"},{"issue":"4","key":"6_CR26","doi-asserted-by":"publisher","first-page":"403","DOI":"10.1561\/2200000003","volume":"1","author":"S. Mahadevan","year":"2009","unstructured":"Mahadevan, S.: Learning Representation and Control in Markov Decision Processes: New Frontiers. Foundations and Trends in Machine Learning\u00a01(4), 403\u2013565 (2009)","journal-title":"Foundations and Trends in Machine Learning"},{"key":"6_CR27","first-page":"2169","volume":"8","author":"S. Mahadevan","year":"2007","unstructured":"Mahadevan, S., Maggioni, M.: Proto-value Functions: A Laplacian Framework for Learning Representation and Control in Markov Decision Processes. Journal of Machine Learning Research\u00a08, 2169\u20132231 (2007)","journal-title":"Journal of Machine Learning Research"},{"key":"6_CR28","doi-asserted-by":"crossref","unstructured":"Mausam, Kolobov, A.: Planning with Markov Decision Processes: An AI Perspective. Synthesis Lectures on Artificial Intelligence and Machine Learning. Morgan & Claypool Publishers (2012)","DOI":"10.2200\/S00426ED1V01Y201206AIM017"},{"key":"6_CR29","series-title":"Lecture Notes in Artificial Intelligence","doi-asserted-by":"publisher","first-page":"590","DOI":"10.1007\/978-3-540-69052-8_62","volume-title":"New Frontiers in Applied Artificial Intelligence","author":"S. Momtazi","year":"2008","unstructured":"Momtazi, S., Kafi, S., Beigy, H.: Solving Stochastic Path Problem: Particle Swarm Optimization Approach. In: Nguyen, N.T., Borzemski, L., Grzech, A., Ali, M. (eds.) IEA\/AIE 2008. LNCS (LNAI), vol.\u00a05027, pp. 590\u2013600. Springer, Heidelberg (2008)"},{"key":"6_CR30","unstructured":"Obal, W.D., Sanders, W.H.: State-space support for path-based reward variables. In: Proceedings of the Third IEEE International Performance and Dependability Symposium on International Performance and Dependability Symposium, IPDS 1998, pp. 233\u2013251. Elsevier Science Publishers B. V. (1999)"},{"key":"6_CR31","unstructured":"Ott, J.T.: A Markov Decision Model for a Surveillance Application and Risk-Sensitive Markov Decision Processes. PhD thesis, Karlsruhe Institute of Technology (2010)"},{"key":"6_CR32","doi-asserted-by":"crossref","unstructured":"Powell, W.B.: Approximate Dynamic Programming - Solving the Curses of Dimensionality. Wiley (2007)","DOI":"10.1002\/9780470182963"},{"key":"6_CR33","doi-asserted-by":"crossref","unstructured":"Puterman, M.L.: Markov Decision Processes - Discrete Stochastic Dynamic Programming. John Wiley & Sons INC. (1994)","DOI":"10.1002\/9780470316887"},{"key":"6_CR34","doi-asserted-by":"crossref","unstructured":"Qiu, Q., Pedram, M.: Dynamic power management based on continuous-time Markov decision processes. In: Proceedings of the 36th Annual ACM\/IEEE Design Automation Conference, DAC 1999, pp. 555\u2013561. ACM (1999)","DOI":"10.1145\/309847.309997"},{"key":"6_CR35","doi-asserted-by":"publisher","first-page":"215","DOI":"10.1007\/978-3-7091-9123-1_10","volume":"4","author":"W.H. Sanders","year":"1991","unstructured":"Sanders, W.H., Meyer, J.F.: A Unified Approach for Specifying Measures of Performance, Dependability, and Performability. Dependable Computing for Critical Applications\u00a04, 215\u2013238 (1991)","journal-title":"Dependable Computing for Critical Applications"},{"key":"6_CR36","doi-asserted-by":"crossref","unstructured":"Schaefer, A.J., Bailey, M.D., Shechter, S.M., Roberts, M.S.: Modeling medical treatment using Markov decision processes. In: Brandeau, M.L., Sainfort, F., Pierskalla, W.P. (eds.) Operations Research and Health Care. International Series in Operations Research & Management Science, vol.\u00a070, pp. 593\u2013612. Kluwer Academic Publishers (2005)","DOI":"10.1007\/1-4020-8066-2_23"},{"key":"6_CR37","doi-asserted-by":"crossref","unstructured":"Sutton, R.S., Barto, A.G.: Reinforcement Learning: An Introduction. A Bradford Book. MIT Press (March 1998)","DOI":"10.1109\/TNN.1998.712192"},{"key":"6_CR38","doi-asserted-by":"crossref","unstructured":"Trivedi, K.S., Malhotra, M.: Reliability and Performability Techniques and Tools: A Survey. In: Messung, Modellierung und Bewertung von Rechen- und Kommunikationssystemen. Informatik aktuell, pp. 27\u201348. Springer, Heidelberg (1993)","DOI":"10.1007\/978-3-642-78495-8_3"},{"issue":"3","key":"6_CR39","doi-asserted-by":"publisher","first-page":"319","DOI":"10.1016\/j.orl.2006.06.005","volume":"35","author":"J.N. Tsitsiklis","year":"2007","unstructured":"Tsitsiklis, J.N.: NP-Hardness of checking the unichain condition in average cost MDPs. Operations Research Letters\u00a035(3), 319\u2013323 (2007)","journal-title":"Operations Research Letters"},{"issue":"11","key":"6_CR40","doi-asserted-by":"publisher","first-page":"1073","DOI":"10.1057\/jors.1993.181","volume":"44","author":"D.J. White","year":"1993","unstructured":"White, D.J.: A Survey of Applications of Markov Decision Processes. The Journal of the Operational Research Society\u00a044(11), 1073\u20131096 (1993)","journal-title":"The Journal of the Operational Research Society"},{"issue":"2","key":"6_CR41","doi-asserted-by":"publisher","first-page":"223","DOI":"10.1287\/opre.30.2.223","volume":"30","author":"R.W. Wolff","year":"1982","unstructured":"Wolff, R.W.: Poisson Arrivals See Time Averages. Operations Research\u00a030(2), 223\u2013231 (1982)","journal-title":"Operations Research"}],"container-title":["Lecture Notes in Computer Science","Stochastic Model Checking. Rigorous Dependability Analysis Using Model Checking Techniques for Stochastic Systems"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-662-45489-3_6","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,5,6]],"date-time":"2025-05-06T05:46:51Z","timestamp":1746510411000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-662-45489-3_6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014]]},"ISBN":["9783662454886","9783662454893"],"references-count":41,"URL":"https:\/\/doi.org\/10.1007\/978-3-662-45489-3_6","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014]]}}}