{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,19]],"date-time":"2026-06-19T16:26:20Z","timestamp":1781886380124,"version":"3.54.5"},"reference-count":49,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2021,2,4]],"date-time":"2021-02-04T00:00:00Z","timestamp":1612396800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,2,4]],"date-time":"2021-02-04T00:00:00Z","timestamp":1612396800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100012456","name":"National Social Science Foundation of China","doi-asserted-by":"crossref","award":["BCA170073"],"award-info":[{"award-number":["BCA170073"]}],"id":[{"id":"10.13039\/501100012456","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Educ Inf Technol"],"published-print":{"date-parts":[[2021,7]]},"DOI":"10.1007\/s10639-021-10452-z","type":"journal-article","created":{"date-parts":[[2021,2,4]],"date-time":"2021-02-04T03:03:05Z","timestamp":1612407785000},"page":"3825-3844","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":12,"title":["Implications of the Delphi method in the evaluation of sustainability open education resource repositories"],"prefix":"10.1007","volume":"26","author":[{"given":"Xiaochen","family":"Wang","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tingting","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yihan","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4836-835X","authenticated-orcid":false,"given":"Harrison Hao","family":"Yang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,2,4]]},"reference":[{"key":"10452_CR1","unstructured":"Achieve.org (2011), Rubrics for evaluating open education resource (OER) objects. Washington, D.C. https:\/\/www.achieve.org\/files\/AchieveOERRubrics.pdf (accessed January 6, 2021)."},{"key":"10452_CR2","unstructured":"Aguilar, J, V, B. (2011). Rubrics to Evaluate Open Educational Resources. https:\/\/dev1.oerafrica.org\/resource\/rubrics-evalute-open-educational-resources (accessed January 14, 2021)."},{"issue":"5","key":"10452_CR3","doi-asserted-by":"publisher","first-page":"1","DOI":"10.19173\/irrodl.v16i5.2133","volume":"16","author":"D Annand","year":"2015","unstructured":"Annand, D. (2015). Developing a sustainable financial model in higher education for open educational resources. The International Review of Research in Open and Distributed Learning, 16(5), 1\u201315. https:\/\/doi.org\/10.19173\/irrodl.v16i5.2133 (accessed January 14, 2021).","journal-title":"The International Review of Research in Open and Distributed Learning"},{"issue":"2","key":"10452_CR4","first-page":"22","volume":"1","author":"J Atenas","year":"2013","unstructured":"Atenas, J., & Havemann, L. (2013). Quality assurance in the open: An evaluation of OER repositories. The International Journal for Innovation and Quality in Learning, 1(2), 22\u201334 http:\/\/oro.open.ac.uk\/56347\/1\/30-288-1-PB%5B1%5D.pdf (accessed January 14, 2021).","journal-title":"The International Journal for Innovation and Quality in Learning"},{"key":"10452_CR5","doi-asserted-by":"publisher","unstructured":"Atenas, J., & Havemann, L. (2014). Questions of quality in repositories of open educational resources: a literature review. Research in Learning Technology, 22. https:\/\/doi.org\/10.3402\/rlt.v22.20889 (accessed January 14, 2021).","DOI":"10.3402\/rlt.v22.20889"},{"issue":"1","key":"10452_CR6","doi-asserted-by":"publisher","first-page":"29","DOI":"10.5944\/openpraxis.6.1.81","volume":"6","author":"J Atenas","year":"2014","unstructured":"Atenas, J., Havemann, L., & Priego, E. (2014). Opening teaching landscapes: The importance of quality assurance in the delivery of open educational resources. Open Praxis, 6(1), 29\u201343. https:\/\/doi.org\/10.5944\/openpraxis.6.1.81 (accessed January 14, 2021).","journal-title":"Open Praxis"},{"key":"10452_CR7","unstructured":"Atkins, D, E., Brown, J, S., & Hammond, A, L. (2007). A review of the open educational resources (OER) movement: Achievements, challenges, and new opportunities. Mountain View: Creative common. https:\/\/core.ac.uk\/download\/pdf\/71348489.pdf (accessed January 14, 2021)."},{"key":"10452_CR8","unstructured":"Butcher, N. (2011). A basic guide to open education resources (OER). Vancouver: Commonwealth of Learning. http:\/\/oasis.col.org\/bitstream\/handle\/11599\/36\/2011_UNESCO_COL_A-Basic-Guide-to-OER.pdf?sequence=5&isAllowed=y (accessed January 14, 2021)."},{"key":"10452_CR9","doi-asserted-by":"publisher","unstructured":"Camilleri, A, F., Ehlers, U., & Pawlowski, J. (2014). State of the art review of quality issues related to open educational resources (OER). Publications Office of the European Union. https:\/\/doi.org\/10.2791\/80171 (accessed January 14, 2021).","DOI":"10.2791\/80171"},{"key":"10452_CR10","doi-asserted-by":"publisher","unstructured":"Caswell, T., Henson, S., Jensen, M., & Wiley, D. (2008). Open content and open educational resources: Enabling universal education. The International Review of Research in Open and Distributed Learning, 9(1). https:\/\/doi.org\/10.19173\/irrodl.v9i1.469 (accessed January 6, 2021).","DOI":"10.19173\/irrodl.v9i1.469"},{"key":"10452_CR11","doi-asserted-by":"publisher","first-page":"1098","DOI":"10.1016\/j.chb.2015.03.026","volume":"51","author":"K Clements","year":"2015","unstructured":"Clements, K., Pawlowski, J., & Manouselis, N. (2015). Open educational resources repositories literature review-towards a comprehensive quality approaches framework. Computers in Human Behavior, 51, 1098\u20131106. https:\/\/doi.org\/10.1016\/j.chb.2015.03.026.","journal-title":"Computers in Human Behavior"},{"key":"10452_CR12","unstructured":"Coursera.org (2021). Meet our partners. https:\/\/www.coursera.org\/about\/partners (accessed January 14, 2021)."},{"issue":"2","key":"10452_CR13","doi-asserted-by":"publisher","first-page":"96","DOI":"10.1109\/TLT.2009.34","volume":"3","author":"HC Davis","year":"2009","unstructured":"Davis, H. C., Carr, L. A., Hey, J. M., Howard, Y., Millard, D., Morris, D., & White, S. (2009). Bootstrapping a culture of sharing to facilitate open educational resources. IEEE Transactions on Learning Technologies, 3(2), 96\u2013109. https:\/\/doi.org\/10.1109\/TLT.2009.34 (accessed January 14, 2021).","journal-title":"IEEE Transactions on Learning Technologies"},{"issue":"1","key":"10452_CR14","doi-asserted-by":"publisher","first-page":"22","DOI":"10.1080\/15228959.2015.1108893","volume":"12","author":"E Davis","year":"2016","unstructured":"Davis, E., Cochran, D., Fagerheim, B., & Thoms, B. (2016). Enhancing teaching and learning: Libraries and open educational resources in the classroom. Public Services Quarterly, 12(1), 22\u201335. https:\/\/doi.org\/10.1080\/15228959.2015.1108893.","journal-title":"Public Services Quarterly"},{"key":"10452_CR15","unstructured":"De los Arcos, B., Farrow, R., Perryman, L, A., Pitt, R. & Weller, M. (2014). OER evidence report 2013\u20132014. OER Research Hub. https:\/\/oerresearchhub.files.wordpress.com\/2014\/11\/oerrh-evidence-report-2014.pdf (accessed January 6, 2021)."},{"issue":"1","key":"10452_CR16","first-page":"29","volume":"3","author":"S Downes","year":"2007","unstructured":"Downes, S. (2007). Models for sustainable open educational resources. Interdisciplinary Journal of E-Learning and Learning Objects, 3(1), 29\u201344 http:\/\/www.ijello.org\/Volume3\/IJKLOv3p029-044Downes.pdf (accessed January 6, 2021).","journal-title":"Interdisciplinary Journal of E-Learning and Learning Objects"},{"key":"10452_CR17","unstructured":"edX.org (2021). Schools and partners. https:\/\/www.edx.org\/schools-partners (accessed January 14, 2021)."},{"issue":"2","key":"10452_CR18","first-page":"1","volume":"15","author":"UD Ehlers","year":"2011","unstructured":"Ehlers, U. D. (2011). Extending the territory: From open educational resources to open educational practices. Journal of Open, Flexible, and Distance Learning, 15(2), 1\u201310.","journal-title":"Journal of Open, Flexible, and Distance Learning"},{"key":"10452_CR19","doi-asserted-by":"publisher","unstructured":"Geser, G. (2007). Open educational practices and resources. OLCOS Roadmap, 2012. Open e-Learning Content Observatory Services. https:\/\/doi.org\/10.7238\/rusc.v4i1.295 (accessed January 14, 2021), 4","DOI":"10.7238\/rusc.v4i1.295"},{"issue":"4","key":"10452_CR20","first-page":"8","volume":"3","author":"A Habibi","year":"2014","unstructured":"Habibi, A., Sarafrazi, A., & Izadyar, S. (2014). Delphi technique theoretical framework in qualitative research. The International Journal of Engineering and Science, 3(4), 8\u201313 https:\/\/parsmodir.com\/wp-content\/uploads\/2018\/11\/Delphi2014-En.pdf (accessed January 14, 2021).","journal-title":"The International Journal of Engineering and Science"},{"key":"10452_CR21","unstructured":"Hyl\u00e9n, J. (2006). Open educational resources: Opportunities and challenges. Proceedings of open education. https:\/\/www.knowledgeall.net\/files\/Additional_Readings-Consolidated.pdf (accessed January 14, 2021)."},{"issue":"4","key":"10452_CR22","doi-asserted-by":"publisher","first-page":"162","DOI":"10.1016\/j.cnre.2016.11.005","volume":"3","author":"X-P Jiang","year":"2016","unstructured":"Jiang, X.-P., Yan, L., Zheng, X.-L., Liu, X., & Wei, X.-Q. (2016). Development and evaluation of a new curriculum based on the Delphi method for master of nursing programs in China. Chinese Nursing Research, 3(4), 162\u2013167. https:\/\/doi.org\/10.1016\/j.cnre.2016.11.005 (accessed January 14, 2021).","journal-title":"Chinese Nursing Research"},{"key":"10452_CR23","first-page":"15","volume":"3","author":"S Johnstone","year":"2005","unstructured":"Johnstone, S. (2005). Forum on the impact of open courseware for higher education in developing countries. Education Quarterly, 3, 15\u201318.","journal-title":"Education Quarterly"},{"issue":"2","key":"10452_CR24","doi-asserted-by":"publisher","first-page":"65","DOI":"10.1080\/08923641003696588","volume":"24","author":"A Kanwar","year":"2010","unstructured":"Kanwar, A., Kodhandaraman, B., & Umar, A. (2010). Toward sustainable open education resources: A perspective from the global south. American Journal of Distance Education, 24(2), 65\u201380. https:\/\/doi.org\/10.1080\/08923641003696588.","journal-title":"American Journal of Distance Education"},{"key":"10452_CR25","first-page":"3","volume":"40","author":"P Kawachi","year":"2015","unstructured":"Kawachi, P. (2015). Quality assurance for OER: Current state of the art and the TIPS framework. eLearning Papers, 40, 3\u201313.","journal-title":"eLearning Papers"},{"issue":"5","key":"10452_CR26","doi-asserted-by":"publisher","first-page":"467","DOI":"10.1016\/j.techfore.2005.09.002","volume":"73","author":"J Landeta","year":"2006","unstructured":"Landeta, J. (2006). Current validity of the Delphi method in social sciences. Technological Forecasting and Social Change, 73(5), 467\u2013482. https:\/\/doi.org\/10.1016\/j.techfore.2005.09.002.","journal-title":"Technological Forecasting and Social Change"},{"key":"10452_CR27","volume-title":"Using the Delphi method","author":"KK Lilja","year":"2011","unstructured":"Lilja, K. K., Laakso, K., & Palomki, J. (2011). Using the Delphi method. Portland: Technology Management in the Energy Smart World."},{"key":"10452_CR28","doi-asserted-by":"publisher","unstructured":"McAndrew, P., Farrow, R., Elliott-Cirigottis, G., & Law, P. (2012). Learning the lessons of openness. Journal of Interactive Media in Education, 2. https:\/\/doi.org\/10.5334\/2012-10 (accessed January 14, 2021)","DOI":"10.5334\/2012-10"},{"issue":"7","key":"10452_CR29","doi-asserted-by":"publisher","first-page":"293","DOI":"10.19173\/irrodl.v18i7.3541","volume":"19","author":"R McGreal","year":"2017","unstructured":"McGreal, R. (2017). Special report on the role of open educational resources in supporting the sustainable development goal 4: Quality education challenges and opportunities. The International Review of Research in Open and Distributed Learning, 19(7), 293\u2013305. https:\/\/doi.org\/10.19173\/irrodl.v18i7.3541 (accessed January 14, 2021).","journal-title":"The International Review of Research in Open and Distributed Learning"},{"key":"10452_CR30","unstructured":"Oercommons.org (2021). Professional learning & teacher training. https:\/\/www.oercommons.org\/training (accessed January 14, 2021)."},{"issue":"1","key":"10452_CR31","doi-asserted-by":"publisher","first-page":"15","DOI":"10.1016\/j.im.2003.11.002","volume":"42","author":"C Okoli","year":"2004","unstructured":"Okoli, C., & Pawlowski, S. D. (2004). The Delphi method as a research tool: An example, design considerations and applications. Information & Management, 42(1), 15\u201329. https:\/\/doi.org\/10.1016\/j.im.2003.11.002.","journal-title":"Information & Management"},{"key":"10452_CR32","unstructured":"Oxford Advanced American Dictionary, (2021). X factor term https:\/\/www.oxfordlearnersdictionaries.com\/definition\/english\/x-factor?q=X+factor (accessed January 14, 2021)."},{"key":"10452_CR33","doi-asserted-by":"publisher","unstructured":"P\u00e9rez, L, I, G., Montoya, M, S, R., & Garc\u00eda-Pe\u00f1alvo, F, J. (2016). Open access to educational resources in energy and sustainability: Usability evaluation prototype for repositories. Proceedings of the Fourth International Conference on Technological Ecosystems for Enhancing Multiculturality-TEEM (pp.1103-1108). https:\/\/doi.org\/10.1145\/3012430.3012654.","DOI":"10.1145\/3012430.3012654"},{"key":"10452_CR34","unstructured":"Quality Matter.org (n.d.). Helping you deliver on your online promise. https:\/\/www.qualitymatters.org\/ (accessed January 14, 2021)."},{"issue":"54","key":"10452_CR35","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1080\/19345747.2010.487927","volume":"26","author":"MS Ram\u00edrez-Montoya","year":"2018","unstructured":"Ram\u00edrez-Montoya, M. S., Garc\u00eda-Pe\u00f1alvo, F. J., & McGreal, R. (2018). Shared science and knowledge. Comunicar, 26(54), 1\u20135. https:\/\/doi.org\/10.1080\/19345747.2010.487927.","journal-title":"Comunicar"},{"issue":"6423","key":"10452_CR36","doi-asserted-by":"publisher","first-page":"130","DOI":"10.1126\/science.aav7958","volume":"363","author":"J Reich","year":"2019","unstructured":"Reich, J., & Ruip\u00e9rez-Valiente, J. A. (2019). The MOOC pivot. Science, 363(6423), 130\u2013131. https:\/\/doi.org\/10.1126\/science.aav7958.","journal-title":"Science"},{"issue":"4","key":"10452_CR37","doi-asserted-by":"publisher","first-page":"255","DOI":"10.1109\/te.2014.2309554","volume":"57","author":"A Ruiz-Iniesta","year":"2014","unstructured":"Ruiz-Iniesta, A., Jimenez-Diaz, G., & Gomez-Albarran, M. (2014). A semantically enriched context-aware OER recommendation strategy and its application to a computer science OER repository. IEEE Transactions on Education, 57(4), 255\u2013260. https:\/\/doi.org\/10.1109\/te.2014.2309554.","journal-title":"IEEE Transactions on Education"},{"issue":"5","key":"10452_CR38","doi-asserted-by":"publisher","first-page":"84","DOI":"10.19173\/irrodl.v18i5.3063","volume":"18","author":"G Santos-Hermosa","year":"2017","unstructured":"Santos-Hermosa, G., Ferran-Ferrer, N., & Abadal, E. (2017). Repositories of open educational resources: An assessment of reuse and educational aspects. The International Review of Research in Open and Distributed Learning, 18(5), 84\u2013120. https:\/\/doi.org\/10.19173\/irrodl.v18i5.3063 (accessed January 14, 2021).","journal-title":"The International Review of Research in Open and Distributed Learning"},{"issue":"1","key":"10452_CR39","doi-asserted-by":"publisher","first-page":"226","DOI":"10.22237\/jmasm\/1241137140","volume":"8","author":"HA Shah","year":"2009","unstructured":"Shah, H. A., & Kalaian, S. A. (2009). Which is the best parametric statistical method for analyzing Delphi data? Journal of Modern Applied Statistical Methods, 8(1), 226\u2013232. https:\/\/doi.org\/10.22237\/jmasm\/1241137140 (accessed January 14, 2021).","journal-title":"Journal of Modern Applied Statistical Methods"},{"issue":"5","key":"10452_CR40","doi-asserted-by":"publisher","first-page":"8","DOI":"10.3200\/CHNG.38.5.8-17","volume":"38","author":"MS Smith","year":"2006","unstructured":"Smith, M. S., & Casserly, C. M. (2006). The promise of open educational resources. Change: The Magazine of higher learning, 38(5), 8\u201317. https:\/\/doi.org\/10.3200\/CHNG.38.5.8-17.","journal-title":"Change: The Magazine of higher learning"},{"issue":"1","key":"10452_CR41","doi-asserted-by":"publisher","first-page":"58","DOI":"10.1111\/ejed.12019","volume":"48","author":"I Tuomi","year":"2013","unstructured":"Tuomi, I. (2013). Open educational resources and the transformation of education. European Journal of Education, 48(1), 58\u201378. https:\/\/doi.org\/10.1111\/ejed.12019.","journal-title":"European Journal of Education"},{"key":"10452_CR42","unstructured":"UNESCO (2018). The Paris OER Declaration 2012. https:\/\/en.unesco.org\/oer\/paris-declaration (accessed January 14, 2021)."},{"key":"10452_CR43","unstructured":"UNESCO (2020a). UNESCO recommendation on open educational resources (OER). https:\/\/en.unesco.org\/news\/unesco-recommendation-open-educational-resources-oer (accessed January 14, 2021)."},{"key":"10452_CR44","unstructured":"UNESCO (2020b). OER dynamic coalition. https:\/\/en.unesco.org\/themes\/building-knowledge-societies\/oer\/dynamic-coalition\/ (accessed January 14, 2021)."},{"key":"10452_CR45","volume-title":"Towards a quality model for open courseware and open educational resources. In international conference on web-based learning (pp. 213\u2013220)","author":"M Vl\u0103doiu","year":"2012","unstructured":"Vl\u0103doiu, M. (2012). Towards a quality model for open courseware and open educational resources. In international conference on web-based learning (pp. 213\u2013220). Berlin, Heidelberg: Springer."},{"issue":"12","key":"10452_CR46","first-page":"106","volume":"38","author":"XC Wang","year":"2017","unstructured":"Wang, X. C., Liu, M. R., Sun, Y. X., Li, Q. H., & Gao, Y. (2017). Systematic analysis of international OER platforms and implications of educational resource construction of the B&R initiative. E-education Research, 38(12), 106\u2013113.","journal-title":"E-education Research"},{"key":"10452_CR47","doi-asserted-by":"crossref","unstructured":"Wiley, D., Bliss, T, J., & McEwen, M. (2014). Open educational resources: A review of the literature. In Handbook of research on educational communications\u00a0and technology (pp. 781\u2013789). Berlin, Heidelberg: Springer.","DOI":"10.1007\/978-1-4614-3185-5_63"},{"issue":"2","key":"10452_CR48","doi-asserted-by":"publisher","first-page":"185","DOI":"10.1080\/01587919.2012.692051","volume":"33","author":"J Willems","year":"2012","unstructured":"Willems, J., & Bossu, C. (2012). Equity considerations for open educational resources in the glocalization of education. Distance Education, 33(2), 185\u2013199. https:\/\/doi.org\/10.1080\/01587919.2012.692051.","journal-title":"Distance Education"},{"key":"10452_CR49","unstructured":"Zeedick, D, M. (2010). The modified Delphi method to analyze the application of instructional design theory to online graduate education [unpublished doctoral dissertation]. The George Washington University. https:\/\/search.proquest.com\/openview\/26828804f13b2945857549df75dd9c5c\/1?pq-origsite=gscholar&cbl=18750&diss=y (accessed January 14, 2021)."}],"container-title":["Education and Information Technologies"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10639-021-10452-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10639-021-10452-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10639-021-10452-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,7,20]],"date-time":"2021-07-20T05:29:06Z","timestamp":1626758946000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10639-021-10452-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,2,4]]},"references-count":49,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,7]]}},"alternative-id":["10452"],"URL":"https:\/\/doi.org\/10.1007\/s10639-021-10452-z","relation":{},"ISSN":["1360-2357","1573-7608"],"issn-type":[{"value":"1360-2357","type":"print"},{"value":"1573-7608","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,2,4]]},"assertion":[{"value":"26 October 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 January 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"4 February 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare that they have no conflict of interest\/competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflicts of interest\/competing interests"}}]}}