{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,6]],"date-time":"2026-06-06T16:45:16Z","timestamp":1780764316978,"version":"3.54.1"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2022,5,11]],"date-time":"2022-05-11T00:00:00Z","timestamp":1652227200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,5,11]],"date-time":"2022-05-11T00:00:00Z","timestamp":1652227200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"name":"DOE","award":["DE-EE0007660"],"award-info":[{"award-number":["DE-EE0007660"]}]},{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","award":["R01GM109718"],"award-info":[{"award-number":["R01GM109718"]}],"id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1832587"],"award-info":[{"award-number":["1832587"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006168","name":"Department of Energy\u2019s National Nuclear Security Administration","doi-asserted-by":"crossref","award":["DE-NA0003525"],"award-info":[{"award-number":["DE-NA0003525"]}],"id":[{"id":"10.13039\/100006168","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Auton Agent Multi-Agent Syst"],"published-print":{"date-parts":[[2022,10]]},"DOI":"10.1007\/s10458-022-09559-5","type":"journal-article","created":{"date-parts":[[2022,5,11]],"date-time":"2022-05-11T06:03:01Z","timestamp":1652248981000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["A framework for the comparison of agent-based models"],"prefix":"10.1007","volume":"36","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4075-0327","authenticated-orcid":false,"given":"Swapna","family":"Thorve","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhihao","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kiran","family":"Lakkaraju","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joshua","family":"Letchford","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anil","family":"Vullikanti","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Achla","family":"Marathe","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Samarth","family":"Swarup","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,5,11]]},"reference":[{"key":"9559_CR1","doi-asserted-by":"crossref","unstructured":"Adiga, A., Barrett, C., Eubank, S., Kuhlman, C.J., Marathe, M. V., Mortveit, H., Ravi, S. S., Rosenkrantz, D. J., Stearns, R. E., Swarup, S. & Vullikanti, A. (2019). Validating agent-based models of large networked systems. In: Winter simulation conference (WSC)","DOI":"10.1109\/WSC40007.2019.9004718"},{"issue":"2","key":"9559_CR2","doi-asserted-by":"publisher","first-page":"123","DOI":"10.1007\/BF01299065","volume":"1","author":"R Axtell","year":"1996","unstructured":"Axtell, R., Axelrod, R., Epstein, J. M., & Cohen, M. D. (1996). Aligning simulation models: A case study and results. Computational and Mathematical Organization Theory, 1(2), 123\u00e2141.","journal-title":"Computational and Mathematical Organization Theory"},{"key":"9559_CR3","doi-asserted-by":"publisher","unstructured":"Barde, S. & van\u00a0der Hoog, S. (2017). An empirical validation protocol for large-scale agent-based models. In: Bielefeld working papers in economics and management No. 04-2017. https:\/\/doi.org\/10.2139\/ssrn.2992473","DOI":"10.2139\/ssrn.2992473"},{"key":"9559_CR4","doi-asserted-by":"crossref","unstructured":"Barton, R. R. & Meckesheimer, M. (22006). Metamodel-based simulation optimization. In: S. G. Henderson, B. L. Nelson (eds.) Simulation, Handbooks in operations research and management science, Elsevier, vol.\u00a013, pp. 535 \u00e2 574","DOI":"10.1016\/S0927-0507(06)13018-2"},{"key":"9559_CR5","doi-asserted-by":"crossref","unstructured":"Bharathy, G. K. & Silverman, B. (2010). Validating agent based social systems models. In: Proceedings of the winter simulation conference, WSC \u201910, pp. 441\u00e2453. Winter simulation conference. http:\/\/dl.acm.org\/citation.cfm?id=2433508.2433559","DOI":"10.1109\/WSC.2010.5679142"},{"issue":"1","key":"9559_CR6","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1111\/j.2517-6161.1951.tb00067.x","volume":"13","author":"GEP Box","year":"1951","unstructured":"Box, G. E. P., & Wilson, K. B. (1951). On the experimental attainment of optimum conditions. Journal of the Royal Statistical Society Series B (Methodological), 13(1), 1\u00e24.","journal-title":"Journal of the Royal Statistical Society Series B (Methodological)"},{"key":"9559_CR7","doi-asserted-by":"crossref","unstructured":"Brueckner, S. A. & Parunak, H. V. D. (2003). Resource-aware exploration of the emergent dynamics of simulated systems. In: Proceedings of the international conference on autonomous agents and multi-agents systems (AAMAS). Melbourne, Australia","DOI":"10.1145\/860575.860701"},{"key":"9559_CR8","unstructured":"Burton, R. M. & Obel, B. (1999). The challenge of validation and docking. In: Proceedings of the workshop on agent simulation: Applications, models, and tools. Argonne National Laboratory, pp. 216\u00e2221."},{"key":"9559_CR9","doi-asserted-by":"crossref","unstructured":"Carley, K. M., Kamneva, N. Y., & Reminga, J. (2004). Response surface methodology. CASOS Technical report CMU-ISRI-04-136, Carnegie Mellon University","DOI":"10.21236\/ADA459032"},{"issue":"2","key":"9559_CR10","doi-asserted-by":"publisher","first-page":"230","DOI":"10.1287\/ijoc.1120.0498","volume":"25","author":"KH Chang","year":"2013","unstructured":"Chang, K. H., Hong, L. J., & Wan, H. (2013). Stochastic trust-region response-surface method (strong)-a new response-surface framework for simulation optimization. INFORMS Journal on Computing, 25(2), 230\u00e2243. https:\/\/doi.org\/10.1287\/ijoc.1120.0498","journal-title":"INFORMS Journal on Computing"},{"key":"9559_CR11","first-page":"2003","volume":"36","author":"N Collier","year":"2003","unstructured":"Collier, N. (2003). Repast: An extensible framework for agent simulation. The University of Chicago\u00e2s Social Science Research, 36, 2003.","journal-title":"The University of Chicago\u2019s Social Science Research"},{"issue":"4","key":"9559_CR12","doi-asserted-by":"publisher","first-page":"1685","DOI":"10.1137\/17M1161233","volume":"6","author":"A Fadikar","year":"2018","unstructured":"Fadikar, A., Higdon, D., Chen, J., Lewis, B. L., Venkatramanan, S., & Marathe, M. V. (2018). Calibrating a stochastic agent-based model using quantile-based emulation. SIAM\/ASA J. Uncertainty Quantification, 6(4), 1685\u00e21706.","journal-title":"SIAM\/ASA J. Uncertainty Quantification"},{"key":"9559_CR13","doi-asserted-by":"publisher","unstructured":"Forrest, S. & Uri, W. (2011) Finding forms of flocking: Evolutionary search in ABM parameter-spaces. Multi-agent-based simulation XI. MABS 2010. Lecture Notes in Computer Science 6532 (2011). https:\/\/doi.org\/10.1007\/978-3-642-18345-4_5","DOI":"10.1007\/978-3-642-18345-4_5"},{"key":"9559_CR14","doi-asserted-by":"crossref","unstructured":"Forrester, A. I. J., Sobester, A., & Keane, A. J. (2008). Engineering design via surrogate modelling\u00e2A practical guide. Wiley","DOI":"10.1002\/9780470770801"},{"issue":"23","key":"9559_CR15","doi-asserted-by":"publisher","first-page":"2760","DOI":"10.1016\/j.ecolmodel.2010.08.019","volume":"221","author":"V Grimm","year":"2010","unstructured":"Grimm, V., Berger, U., DeAngelis, D., Polhill, J., Giske, J., & Railsback, S. (2010). The odd protocol: A review and first update. Ecological Modelling, 221(23), 2760\u00e22768. https:\/\/doi.org\/10.1016\/j.ecolmodel.2010.08.019","journal-title":"Ecological Modelling"},{"key":"9559_CR16","unstructured":"Gupta, A., Hu, Z., Marathe, A., Swarup, S., & Vullikanti, A.: Predictors of rooftop solar adoption in rural Virginia. In: Proceedings of the computational social science conference (2018)"},{"key":"9559_CR17","unstructured":"Gupta, A., Swarup, S., Marathe, A., Vullikanti, A., Lakkaraju, K., & Letchford, J. (2018). Designing incentives to maximize the adoption of rooftop solar technology (extended abstract). In: Proceedings of the 17th international conference on autonomous agents and multi-agents systems (AAMAS). Stockholm, Sweden (2018)"},{"key":"9559_CR18","doi-asserted-by":"crossref","unstructured":"Hastie, T., Tibshirani, R., & Friedman, J. (2001). The elements of statistical l. Springer series in statistics. Springer, New York, NY, USA","DOI":"10.1007\/978-0-387-21606-5"},{"key":"9559_CR19","unstructured":"Hu, Z., Deng, X., Marathe, A., Swarup, S., & Vullikanti, A. (2019). Decision-adjusted modeling for imbalanced classification: Predicting rooftop solar panel adoption in rural virginia. In: Proceedings of the computational social science conference"},{"key":"9559_CR20","doi-asserted-by":"crossref","unstructured":"Kempe, D., Kleinberg, J., & \u00c3va, T. (2003). Maximizing the spread of influence through a social network. In: Proceedings of KDD. Washington, DC, USA","DOI":"10.1145\/956750.956769"},{"key":"9559_CR21","doi-asserted-by":"publisher","first-page":"366","DOI":"10.1016\/j.jedc.2018.03.011","volume":"90","author":"F Lamperti","year":"2018","unstructured":"Lamperti, F., Roventini, A., & Sani, A. (2018). Agent-based model calibration using machine learning surrogates. Journal of Economic Dynamic and Control, 90, 366\u00e2389.","journal-title":"Journal of Economic Dynamic and Control"},{"key":"9559_CR22","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1007\/978-1-4471-2099-5_1","volume-title":"SIGIR \u201994","author":"DD Lewis","year":"1994","unstructured":"Lewis, D. D., & Gale, W. A. (1994). A sequential algorithm for training text classifiers. In B. W. Croft & C. J. van Rijsbergen (Eds.), SIGIR \u00e294 (pp. 3\u00e212). London: Springer."},{"key":"9559_CR23","doi-asserted-by":"publisher","unstructured":"Lookman, T., Balachandran, P. V., Xue, D., & Yuan, R. (2009) Active learning in materials science with emphasis on adaptive sampling using uncertainties for targeted design. In: npj computational materials. https:\/\/doi.org\/10.1038\/s41524-019-0153-8","DOI":"10.1038\/s41524-019-0153-8"},{"key":"9559_CR24","unstructured":"Louie, M. A., Carley, K. M., Haghshenass, L., Kunz, J. C., Levitt, R. E., et\u00a0al. (2003). Model comparisons: docking orgahead and simvision. In: Proceedings of NAACSOS conference, Pittsburgh, PA. Citeseer"},{"key":"9559_CR25","unstructured":"Mao, H., Guo, F., Deng, X., & Doerzaph, Z. (2019). Decision-adjusted driver risk predictive model using kinematics information. submitted to IEEE transactions on intelligent transportation systems"},{"issue":"7","key":"9559_CR26","doi-asserted-by":"publisher","first-page":"88","DOI":"10.1145\/2483852.2483871","volume":"56","author":"M Marathe","year":"2013","unstructured":"Marathe, M., & Vullikanti, A. (2013). Computational epidemiology. Communications of the ACM, 56(7), 88\u00e296.","journal-title":"Communications of the ACM"},{"key":"9559_CR27","unstructured":"Minar, N., Burkhart, R., Langton, C., Askenazi, M., et\u00a0al.: The swarm simulation system: A toolkit for building multi-agent simulations. Technical report (1996)"},{"key":"9559_CR28","volume-title":"An introduction to sequential dynamical systems","author":"H Mortveit","year":"2007","unstructured":"Mortveit, H., & Reidys, C. (2007). An introduction to sequential dynamical systems. Springer: Universitext."},{"key":"9559_CR29","doi-asserted-by":"publisher","unstructured":"Murphy, R. F. (2011). An active role for machine learning in drug development. In: Nature chemical biology, p. 327\u00e2330. https:\/\/doi.org\/10.1038\/nchembio.576","DOI":"10.1038\/nchembio.576"},{"key":"9559_CR30","doi-asserted-by":"crossref","unstructured":"Neddermeijer, H. G., van Oortmarssen, G. J., Piersma, N., & Dekker, R. (2000). A framework for response surface methodology for simulation optimization. In: J. A. Joines, R. R. Barton, K.\u00a0Kang, P. A. Fishwick (eds.) Proceedings of the 32nd conference on winter simulation, WSC \u201900, pp. 129\u00e2136. Society for Computer Simulation International, San Diego, CA, USA. http:\/\/dl.acm.org\/citation.cfm?id=510378.510401","DOI":"10.1109\/WSC.2000.899707"},{"key":"9559_CR31","volume-title":"The beer dock: Three and a half implementations of the beer distribution game","author":"M North","year":"2002","unstructured":"North, M., & Macal, C. M. (2002). The beer dock: Three and a half implementations of the beer distribution game. Swarm Development Group: SwarmFest."},{"issue":"12","key":"9559_CR32","doi-asserted-by":"publisher","first-page":"2495","DOI":"10.2514\/2.1593","volume":"40","author":"VM P\u00c3\u00a9rez","year":"2002","unstructured":"P\u00c3\u00a9rez, V. M., Renaud, J. E., & Watson, L. T. (2002). Adaptive experimental design for construction of response surface approximations. AIAA Journal, 40(12), 2495\u00e22503.","journal-title":"AIAA Journal"},{"key":"9559_CR33","doi-asserted-by":"crossref","unstructured":"Scheffer, T., Decomain, C., & Wrobel, S. (2001). Active hidden markov models for information extraction. In: Proceedings of the 4th international conference on advances in intelligent data analysis, IDA \u201901, p. 309\u00e2318. Springer, Berlin, Heidelberg","DOI":"10.1007\/3-540-44816-0_31"},{"key":"9559_CR34","unstructured":"Thorve, S., Hu, Z., Lakkaraju, K., Letchford, J., Vullikanti, A., Marathe, A., & Swarup, S. (2020). An active learning method for the comparison of agent-based models. In: Proceedings of the 19th international conference on autonomous agents and multi-agent systems (AAMAS)"},{"key":"9559_CR35","doi-asserted-by":"crossref","unstructured":"Volk-Makarewicz, W. & Cleophas, C. (2017) A meta-algorithm for validating agent-based simulation models to support decision making. In: Proceedings of the 2017 winter simulation conference, WSC \u201917, pp. 102:1\u00e2102:12. IEEE Press, Piscataway, NJ, USA. http:\/\/dl.acm.org\/citation.cfm?id=3242181.3242289","DOI":"10.1109\/WSC.2017.8247879"},{"key":"9559_CR36","unstructured":"Xu, J., Gao, Y., & Madey, G. (2003) A docking experiment: Swarm and repast for social network modeling. In: Seventh annual swarm researchers meeting (Swarm2003), pp. 1\u00e29"},{"issue":"6","key":"9559_CR37","doi-asserted-by":"publisher","first-page":"1023","DOI":"10.1007\/s10458-016-9326-8","volume":"30","author":"H Zhang","year":"2016","unstructured":"Zhang, H., Vorobeychik, Y., Letchford, J., & Lakkaraju, K. (2016). Data-driven agent-based modeling, with application to rooftop solar adoption. Auton Agent Multi-Agent Syst, 30(6), 1023\u00e21049. https:\/\/doi.org\/10.1007\/s10458-016-9326-8","journal-title":"Auton Agent Multi-Agent Syst"},{"key":"9559_CR38","doi-asserted-by":"publisher","unstructured":"Zhang, Y., Li, Z., & Zhang, Y. (2020). Validation and calibration of an agent-based model: A surrogate approach. In: Discrete dynamics in nature and society. https:\/\/doi.org\/10.1155\/2020\/6946370","DOI":"10.1155\/2020\/6946370"},{"key":"9559_CR39","doi-asserted-by":"publisher","unstructured":"Zou, H., & Hastie, T. (2005). Regularization and variable selection via the elastic net. Journal of the Royal Statistical Society: Series B (Statistical Methodology), 67(2), 301\u00e2320. https:\/\/doi.org\/10.1111\/j.1467-9868.2005.00503.x.","DOI":"10.1111\/j.1467-9868.2005.00503.x"}],"container-title":["Autonomous Agents and Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-022-09559-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10458-022-09559-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-022-09559-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,10,29]],"date-time":"2022-10-29T12:49:44Z","timestamp":1667047784000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10458-022-09559-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,5,11]]},"references-count":39,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2022,10]]}},"alternative-id":["9559"],"URL":"https:\/\/doi.org\/10.1007\/s10458-022-09559-5","relation":{},"ISSN":["1387-2532","1573-7454"],"issn-type":[{"value":"1387-2532","type":"print"},{"value":"1573-7454","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5,11]]},"assertion":[{"value":"12 April 2022","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 May 2022","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"32"}}