{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,13]],"date-time":"2026-05-13T23:53:43Z","timestamp":1778716423313,"version":"3.51.4"},"reference-count":55,"publisher":"Springer Science and Business Media LLC","issue":"1-2","license":[{"start":{"date-parts":[[2018,9,5]],"date-time":"2018-09-05T00:00:00Z","timestamp":1536105600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2018,9,5]],"date-time":"2018-09-05T00:00:00Z","timestamp":1536105600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/100006754","name":"Army Research Laboratory","doi-asserted-by":"publisher","award":["MIPR0010347614"],"award-info":[{"award-number":["MIPR0010347614"]}],"id":[{"id":"10.13039\/100006754","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000183","name":"Army Research Office","doi-asserted-by":"publisher","award":["MIPR0010504121"],"award-info":[{"award-number":["MIPR0010504121"]}],"id":[{"id":"10.13039\/100000183","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Ann Oper Res"],"published-print":{"date-parts":[[2019,9]]},"DOI":"10.1007\/s10479-018-3011-z","type":"journal-article","created":{"date-parts":[[2018,9,5]],"date-time":"2018-09-05T16:12:28Z","timestamp":1536163948000},"page":"189-210","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Modeling uncertainty of expert elicitation for use in risk-based optimization"],"prefix":"10.1007","volume":"280","author":[{"given":"Michael D.","family":"Teter","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Johannes O.","family":"Royset","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alexandra M.","family":"Newman","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2018,9,5]]},"reference":[{"key":"3011_CR1","unstructured":"Arigoni, A. (2016). Optimization techniques in coal markets: A global cost minimization and a multi-stage procurement strategy. Dissertation, Colorado School of Mines."},{"key":"3011_CR2","unstructured":"Arlot, S., & Celisse, A. (2010). A survey of cross-validation procedures for model selection. Statistics Surveys, 4, 40\u201379. https:\/\/projecteuclid.org\/download\/pdfview_1\/euclid.ssu\/1268143839 ."},{"key":"3011_CR3","unstructured":"Assistant Secretary of the Army (Financial Management and Comptroller). (1994). Planning, programing, budgeting, and execution system. Headquarters, Department of the Army."},{"issue":"6","key":"3011_CR4","doi-asserted-by":"publisher","first-page":"530","DOI":"10.1287\/inte.1060.0252","volume":"36","author":"GG Brown","year":"2006","unstructured":"Brown, G. G., Carlyle, M., Salmer\u00f3n, J., & Wood, K. (2006). Defending critical infrastructure. Interfaces, 36(6), 530\u2013544.","journal-title":"Interfaces"},{"issue":"6","key":"3011_CR5","doi-asserted-by":"publisher","first-page":"415","DOI":"10.1287\/inte.1040.0107","volume":"34","author":"GG Brown","year":"2004","unstructured":"Brown, G. G., Dell, R. F., & Newman, A. M. (2004). Optimizing military capital planning. Interfaces, 34(6), 415\u2013425.","journal-title":"Interfaces"},{"key":"3011_CR6","unstructured":"Buttrey, S, Royset, J. O., & Wets, R. J. B. (2014). episplineDensity: Density estimation with soft information by exponential epi-splines. http:\/\/CRAN.R-project.org\/package=episplineDensity , R package version 0.0-1."},{"key":"3011_CR7","doi-asserted-by":"crossref","unstructured":"Casella, G., Robert, C. P., & Wells, M. T. (2004). Generalized accept-reject sampling schemes. Lecture Notes-Monograph Series, pp. 342\u2013347.","DOI":"10.1214\/lnms\/1196285403"},{"issue":"1","key":"3011_CR8","doi-asserted-by":"publisher","first-page":"32","DOI":"10.1108\/13673270610650085","volume":"10","author":"KC Desouza","year":"2006","unstructured":"Desouza, K. C., & Awazu, Y. (2006). Knowledge management at SMEs: Five peculiarities. Journal of Knowledge Management, 10(1), 32\u201343.","journal-title":"Journal of Knowledge Management"},{"key":"3011_CR9","unstructured":"Deveans, T., Jacobs, P., Gaver, D., & Teter, M. (2014). Analysis of the capability portfolio review (cpr). Tech. rep., DTIC Document."},{"issue":"2","key":"3011_CR10","doi-asserted-by":"publisher","first-page":"207","DOI":"10.1287\/ijoc.6.2.207","volume":"6","author":"AS Drud","year":"1994","unstructured":"Drud, A. S. (1994). CONOPT\u2014A large-scale GRG code. ORSA Journal on Computing, 6(2), 207\u2013216.","journal-title":"ORSA Journal on Computing"},{"key":"3011_CR11","volume-title":"Statistics and data analysis: From elementary to intermediate","author":"DD Dunlop","year":"2000","unstructured":"Dunlop, D. D., & Tamhane, A. C. (2000). Statistics and data analysis: From elementary to intermediate. New York: Prentice Hall."},{"key":"3011_CR12","unstructured":"Duong, T. (2015). ks: Kernel smoothing. http:\/\/CRAN.R-project.org\/package=ks , R package version 1.10.0."},{"issue":"3","key":"3011_CR13","doi-asserted-by":"publisher","first-page":"589","DOI":"10.1093\/biomet\/68.3.589","volume":"68","author":"B Efron","year":"1981","unstructured":"Efron, B. (1981). Nonparametric estimates of standard error: The jackknife, the bootstrap and other methods. Biometrika, 68(3), 589\u2013599.","journal-title":"Biometrika"},{"issue":"1","key":"3011_CR14","doi-asserted-by":"publisher","first-page":"293","DOI":"10.1007\/s10479-012-1261-8","volume":"222","author":"L Evers","year":"2014","unstructured":"Evers, L., Dollevoet, T., Barros, A. I., & Monsuur, H. (2014). Robust UAV mission planning. Annals of Operations Research, 222(1), 293\u2013315.","journal-title":"Annals of Operations Research"},{"issue":"1","key":"3011_CR15","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1287\/deca.1060.0062","volume":"3","author":"PL Ewing Jr","year":"2006","unstructured":"Ewing, P. L, Jr., Tarantino, W., & Parnell, G. S. (2006). Use of decision analysis in the Army Base Realignment And Closure (BRAC) 2005 military value analysis. Decision Analysis, 3(1), 33\u201349.","journal-title":"Decision Analysis"},{"issue":"3","key":"3011_CR16","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1007\/s12667-015-0146-8","volume":"6","author":"Y Feng","year":"2015","unstructured":"Feng, Y., Rios, I., Ryan, S. M., Sp\u00fcrkel, K., Watson, J. P., Wets, R. J. B., et al. (2015). Toward scalable stochastic unit commitment. Part 1: Load scenario generation. Energy Systems, 6(3), 309\u2013329. https:\/\/doi.org\/10.1007\/s12667-015-0146-8 .","journal-title":"Energy Systems"},{"key":"3011_CR17","unstructured":"GAMS. (2015). GAMS distribution 24.4.5. Washington, DC."},{"issue":"470","key":"3011_CR18","doi-asserted-by":"publisher","first-page":"680","DOI":"10.1198\/016214505000000105","volume":"100","author":"PH Garthwaite","year":"2005","unstructured":"Garthwaite, P. H., Kadane, J. B., & O\u2019Hagan, A. (2005). Statistical methods for eliciting probability distributions. Journal of the American Statistical Association, 100(470), 680\u2013701.","journal-title":"Journal of the American Statistical Association"},{"issue":"430","key":"3011_CR19","doi-asserted-by":"publisher","first-page":"598","DOI":"10.1080\/01621459.1995.10476552","volume":"90","author":"AE Gelfand","year":"1995","unstructured":"Gelfand, A. E., Mallick, B. K., & Dey, D. K. (1995). Modeling expert opinion arising as a partial probabilistic specification. Journal of the American Statistical Association, 90(430), 598\u2013604.","journal-title":"Journal of the American Statistical Association"},{"key":"3011_CR20","doi-asserted-by":"crossref","unstructured":"Genz, A., Bretz, F. (2009). Computation of multivariate normal and t probabilities. Lecture Notes in Statistics, Vol. 195.","DOI":"10.1007\/978-3-642-01689-9"},{"issue":"5","key":"3011_CR21","doi-asserted-by":"publisher","first-page":"646","DOI":"10.1093\/poq\/nfl033","volume":"70","author":"RM Groves","year":"2006","unstructured":"Groves, R. M. (2006). Nonresponse rates and nonresponse bias in household surveys. Public Opinion Quarterly, 70(5), 646\u2013675.","journal-title":"Public Opinion Quarterly"},{"key":"3011_CR22","unstructured":"IBM. (2015). ILOG CPLEX, IBM\u2014International Business Machines Corporation, Incline Village, NV."},{"issue":"4","key":"3011_CR23","doi-asserted-by":"publisher","first-page":"267","DOI":"10.1080\/00137910903338545","volume":"54","author":"A Ko\u00e7","year":"2009","unstructured":"Ko\u00e7, A., Morton, D. P., Popova, E., Hess, S. M., Kee, E., & Richards, D. (2009). Prioritizing project selection. The Engineering Economist, 54(4), 267\u2013297.","journal-title":"The Engineering Economist"},{"key":"3011_CR24","unstructured":"Kohavi, R. (1995). A study of cross-validation and bootstrap for accuracy estimation and model selection. In International Joint Conference on Artificial Intelligence (IJCAI) (Vol. 14, pp. 1137\u20131145). https:\/\/www.researchgate.net\/profile\/Ron_Kohavi\/publication\/2352264_A_Study_of_Cross-Validation_and_Bootstrap_for_Accuracy_Estimation_and_Model_Selection\/links\/02e7e51bcc14c5e91c000000.pdf ."},{"issue":"3","key":"3011_CR25","doi-asserted-by":"crossref","first-page":"287","DOI":"10.1080\/07350015.1988.10509663","volume":"6","author":"RJ Little","year":"1988","unstructured":"Little, R. J. (1988). Missing-data adjustments in large surveys. Journal of Business & Economic Statistics, 6(3), 287\u2013296.","journal-title":"Journal of Business & Economic Statistics"},{"key":"3011_CR26","unstructured":"Marks, C., Smead, K., & Alt, J. (2013). Enhancing subject matter expert elicitation techniques. Tech. Rep. TRAC-M-TR-13-048, TRADOC Analysis Center - Monterey, 700 Dyer Road Monterey, California 93943."},{"key":"3011_CR27","unstructured":"Moynihan, R. A. (2005). Investment analysis using the Portfolio Analysis Machine (PALMA) tool. Tech. rep., Technical Report MITRE Case."},{"issue":"2","key":"3011_CR28","doi-asserted-by":"publisher","first-page":"427","DOI":"10.1093\/biomet\/asm031","volume":"94","author":"JE Oakley","year":"2007","unstructured":"Oakley, J. E., & O\u2019Hagan, A. (2007). Uncertainty in prior elicitations: A nonparametric approach. Biometrika, 94(2), 427\u2013441.","journal-title":"Biometrika"},{"key":"3011_CR29","unstructured":"Odierno, R. T. (2015). Force 2025 and beyond: How the Army is preparing for its future. Defense One."},{"issue":"1","key":"3011_CR30","doi-asserted-by":"crossref","first-page":"21","DOI":"10.1111\/1467-9884.00114","volume":"47","author":"A O\u2019Hagan","year":"1998","unstructured":"O\u2019Hagan, A. (1998). Eliciting expert beliefs in substantial practical applications. Journal of the Royal Statistical Society: Series D (The Statistician), 47(1), 21\u201335.","journal-title":"Journal of the Royal Statistical Society: Series D (The Statistician)"},{"key":"3011_CR31","doi-asserted-by":"publisher","DOI":"10.1002\/0470033312","volume-title":"Uncertain judgements: Eliciting experts\u2019 probabilities","author":"A O\u2019Hagan","year":"2006","unstructured":"O\u2019Hagan, A., Buck, C. E., Daneshkhah, A., Eiser, J. R., Garthwaite, P. H., Jenkinson, D. J., et al. (2006). Uncertain judgements: Eliciting experts\u2019 probabilities. Hoboken: Wiley."},{"issue":"1","key":"3011_CR32","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1080\/10618600.2012.707343","volume":"23","author":"D Papp","year":"2014","unstructured":"Papp, D., & Alizadeh, F. (2014). Shape-constrained estimation using nonnegative splines. Journal of Computational and Graphical Statistics, 23(1), 211\u2013231.","journal-title":"Journal of Computational and Graphical Statistics"},{"key":"3011_CR33","first-page":"1087","volume":"8","author":"D Pfeffermann","year":"1998","unstructured":"Pfeffermann, D., Krieger, A. M., & Rinott, Y. (1998). Parametric distributions of complex survey data under informative probability sampling. Statistica Sinica, 8, 1087\u20131114.","journal-title":"Statistica Sinica"},{"key":"3011_CR34","unstructured":"R Core Team. (2015). R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria. https:\/\/www.R-project.org\/ ."},{"issue":"2","key":"3011_CR35","doi-asserted-by":"publisher","first-page":"267","DOI":"10.1007\/s10287-015-0230-5","volume":"12","author":"I Rios","year":"2015","unstructured":"Rios, I., Wets, R. J. B., & Woodruff, D. L. (2015). Multi-period forecasting and scenario generation with limited data. Computational Management Science, 12(2), 267\u2013295.","journal-title":"Computational Management Science"},{"issue":"2","key":"3011_CR36","doi-asserted-by":"publisher","first-page":"04015003","DOI":"10.1061\/AJRUA6.0000816","volume":"1","author":"RT Rockafellar","year":"2015","unstructured":"Rockafellar, R. T., & Royset, J. O. (2015). Engineering decisions under risk averseness. ASCE-ASME Journal of Risk and Uncertainty in Engineering Systems, Part A: Civil Engineering, 1(2), 04015003.","journal-title":"ASCE-ASME Journal of Risk and Uncertainty in Engineering Systems, Part A: Civil Engineering"},{"issue":"1","key":"3011_CR37","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/s10479-016-2129-0","volume":"262","author":"RT Rockafellar","year":"2018","unstructured":"Rockafellar, R. T., & Royset, J. O. (2018). Superquantile\/CVaR risk measures: Second-order theory. Annals of Operations Research, 262(1), 3\u201328.","journal-title":"Annals of Operations Research"},{"key":"3011_CR38","doi-asserted-by":"publisher","first-page":"21","DOI":"10.21314\/JOR.2000.038","volume":"2","author":"RT Rockafellar","year":"2000","unstructured":"Rockafellar, R. T., & Uryasev, S. (2000). Optimization of conditional value-at-risk. Journal of Risk, 2, 21\u201342.","journal-title":"Journal of Risk"},{"issue":"7","key":"3011_CR39","doi-asserted-by":"publisher","first-page":"1443","DOI":"10.1016\/S0378-4266(02)00271-6","volume":"26","author":"RT Rockafellar","year":"2002","unstructured":"Rockafellar, R. T., & Uryasev, S. (2002). Conditional value-at-risk for general loss distributions. Journal of Banking & Finance, 26(7), 1443\u20131471.","journal-title":"Journal of Banking & Finance"},{"issue":"3","key":"3011_CR40","doi-asserted-by":"publisher","first-page":"569","DOI":"10.1109\/TPAMI.2009.187","volume":"32","author":"JD Rodr\u00edguez","year":"2010","unstructured":"Rodr\u00edguez, J. D., Perez, A., & Lozano, J. A. (2010). Sensitivity analysis of k-fold cross validation in prediction error estimation. IEEE Transactions on Pattern Analysis and Machine Intelligence, 32(3), 569\u2013575.","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"3011_CR41","unstructured":"Royset, J. O., Sukumar, N., & Wets, R. J. B. (2013). Uncertainty quantification using exponential epi-splines. In Proceedings of the international conference on structural safety and reliability."},{"key":"3011_CR42","doi-asserted-by":"crossref","unstructured":"Royset, J. O., & Wets, R. J. B. (2014). From data to assessments and decisions: Epi-spline technology. In Bridging data and decisions, chap. 3, pp. 27\u201353.","DOI":"10.21236\/ADA603986"},{"issue":"2","key":"3011_CR43","doi-asserted-by":"publisher","first-page":"532","DOI":"10.1016\/j.ejor.2015.06.034","volume":"247","author":"JO Royset","year":"2015","unstructured":"Royset, J. O., & Wets, R. J. B. (2015). Fusion of hard and soft information in nonparametric density estimation. European Journal of Operational Research, 247(2), 532\u2013547.","journal-title":"European Journal of Operational Research"},{"key":"3011_CR44","doi-asserted-by":"crossref","unstructured":"Royset, J. O., & Wets, R. J. B. (2016). Multivariate epi-splines and evolving function identification problems. Set-Valued and Variational Analysis, 24(4), 517\u2013545. Erratum, pp. 547\u2013549.","DOI":"10.1007\/s11228-015-0329-4"},{"issue":"1","key":"3011_CR45","doi-asserted-by":"publisher","first-page":"295","DOI":"10.1007\/s10479-011-0885-4","volume":"186","author":"N Safaei","year":"2011","unstructured":"Safaei, N., Banjevic, D., & Jardine, A. K. (2011). Workforce-constrained maintenance scheduling for military aircraft fleet: A case study. Annals of Operations Research, 186(1), 295\u2013316.","journal-title":"Annals of Operations Research"},{"issue":"422","key":"3011_CR46","doi-asserted-by":"publisher","first-page":"486","DOI":"10.1080\/01621459.1993.10476299","volume":"88","author":"J Shao","year":"1993","unstructured":"Shao, J. (1993). Linear model selection by cross-validation. Journal of the American Statistical Association, 88(422), 486\u2013494.","journal-title":"Journal of the American Statistical Association"},{"key":"3011_CR47","unstructured":"Shapiro, A., & Philpott, A. (2007). A tutorial on stochastic programming. http:\/\/www2.isye.gatech.edu\/ashapiro\/publications.html ."},{"key":"3011_CR48","unstructured":"Singham, D. I., Royset, J. O., & Wets, R. J. B. (2013). Density estimation of simulation output using exponential epi-splines. In Proceedings of the 2013 winter simulation conference: simulation: making decisions in a complex world (pp. 755\u2013765). IEEE Press."},{"issue":"3","key":"3011_CR49","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1016\/j.ijar.2006.06.024","volume":"45","author":"WJ Tastle","year":"2007","unstructured":"Tastle, W. J., & Wierman, M. J. (2007). Consensus and dissension: A measure of ordinal dispersion. International Journal of Approximate Reasoning, 45(3), 531\u2013545.","journal-title":"International Journal of Approximate Reasoning"},{"key":"3011_CR50","unstructured":"Teter, M., Brickey, A., Newman, A., & Royset, J. (2018). Optimizing underground mine scheduling under production rate uncertainty. Working paper."},{"key":"3011_CR51","doi-asserted-by":"crossref","unstructured":"Teter, M. D. (2014). Applying subject matter expertise (SME) elicitation techniques to TRAC studies. Tech. rep., DTIC Document.","DOI":"10.21236\/ADA616463"},{"issue":"1","key":"3011_CR52","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.sorms.2016.05.001","volume":"21","author":"MD Teter","year":"2016","unstructured":"Teter, M. D., Newman, A. M., & Weiss, M. (2016). Consistent notation for presenting complex optimization models in technical writing. Surveys in Operations Research and Management Science, 21(1), 1\u201317.","journal-title":"Surveys in Operations Research and Management Science"},{"key":"3011_CR53","unstructured":"Under Secretary of Defense (Acquisition, Technology, and Logistics). (2002). DOD deskbook 5000.2-R Appendix 6: Technology readiness levels and their definitions. Department of Defense."},{"key":"3011_CR54","doi-asserted-by":"crossref","unstructured":"Van Buuren, S., & Groothuis-Oudshoorn, K. (2011). mice: Multivariate imputation by chained equations in R. Journal of Statistical Software, 45(3), 1\u201367. http:\/\/www.jstatsoft.org\/v45\/i03\/ .","DOI":"10.18637\/jss.v045.i03"},{"issue":"4","key":"3011_CR55","doi-asserted-by":"publisher","first-page":"247","DOI":"10.1007\/s11579-014-0140-2","volume":"9","author":"RJB Wets","year":"2015","unstructured":"Wets, R. J. B., & Rios, I. (2015). Modeling and estimating commodity prices: Copper prices. Mathematics and Financial Economics, 9(4), 247\u2013270.","journal-title":"Mathematics and Financial Economics"}],"container-title":["Annals of Operations Research"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10479-018-3011-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10479-018-3011-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10479-018-3011-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,9]],"date-time":"2024-07-09T23:49:14Z","timestamp":1720568954000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10479-018-3011-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,9,5]]},"references-count":55,"journal-issue":{"issue":"1-2","published-print":{"date-parts":[[2019,9]]}},"alternative-id":["3011"],"URL":"https:\/\/doi.org\/10.1007\/s10479-018-3011-z","relation":{},"ISSN":["0254-5330","1572-9338"],"issn-type":[{"value":"0254-5330","type":"print"},{"value":"1572-9338","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,9,5]]},"assertion":[{"value":"5 September 2018","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}