{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,10]],"date-time":"2026-01-10T03:37:27Z","timestamp":1768016247695,"version":"3.49.0"},"publisher-location":"Cham","reference-count":28,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030447274","type":"print"},{"value":"9783030447281","type":"electronic"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-44728-1_3","type":"book-chapter","created":{"date-parts":[[2020,3,25]],"date-time":"2020-03-25T12:02:56Z","timestamp":1585137776000},"page":"49-68","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":20,"title":["Enabling Continuous Testing of HPC Systems Using ReFrame"],"prefix":"10.1007","author":[{"given":"Vasileios","family":"Karakasis","sequence":"first","affiliation":[]},{"given":"Theofilos","family":"Manitaras","sequence":"additional","affiliation":[]},{"given":"Victor Holanda","family":"Rusu","sequence":"additional","affiliation":[]},{"given":"Rafael","family":"Sarmiento-P\u00e9rez","sequence":"additional","affiliation":[]},{"given":"Christopher","family":"Bignamini","sequence":"additional","affiliation":[]},{"given":"Matthias","family":"Kraushaar","sequence":"additional","affiliation":[]},{"given":"Andreas","family":"Jocksch","sequence":"additional","affiliation":[]},{"given":"Samuel","family":"Omlin","sequence":"additional","affiliation":[]},{"given":"Guilherme","family":"Peretti-Pezzi","sequence":"additional","affiliation":[]},{"given":"Jo\u00e3o P. S. C.","family":"Augusto","sequence":"additional","affiliation":[]},{"given":"Brian","family":"Friesen","sequence":"additional","affiliation":[]},{"given":"Yun","family":"He","sequence":"additional","affiliation":[]},{"given":"Lisa","family":"Gerhardt","sequence":"additional","affiliation":[]},{"given":"Brandon","family":"Cook","sequence":"additional","affiliation":[]},{"given":"Zhi-Qiang","family":"You","sequence":"additional","affiliation":[]},{"given":"Samuel","family":"Khuvis","sequence":"additional","affiliation":[]},{"given":"Karen","family":"Tomko","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,3,26]]},"reference":[{"key":"3_CR1","unstructured":"OpenHPC: Community building blocks for HPC systems. \nhttps:\/\/github.com\/openhpc\/ohpc"},{"key":"3_CR2","unstructured":"Pavilion2. \nhttps:\/\/github.com\/lanl-preteam\/pavilion2"},{"key":"3_CR3","unstructured":"Cray Lightweight Log Manager (LLM) (2019). \nhttps:\/\/pubs.cray.com\/content\/S-2393\/CLE%207.0.UP00\/xctm-series-system-administration-guide\/cray-lightweight-log-manager-llm"},{"key":"3_CR4","doi-asserted-by":"crossref","unstructured":"Adams, M., Brown, J., Shalf, J., Straalen, B.V., Strohmaier, E., Williams, S.: HPGMG 1.0: a benchmark for ranking high performance computing systems. Technical report, LBNL-6630E, Lawrence Berkeley National Laboratory, May 2014. \nhttp:\/\/escholarship.org\/uc\/item\/00r9w79m","DOI":"10.2172\/1131029"},{"key":"3_CR5","doi-asserted-by":"publisher","unstructured":"Checconi, F., Petrini, F., Willcock, J., Lumsdaine, A., Choudhury, A.R., Sabharwal, Y.: Breaking the speed and scalability Barriers for Graph exploration on distributed-memory machines. In: SC 2012: Proceedings of the International Conference on High Performance Computing, Networking, Storage and Analysis, pp. 1\u201312, November 2012. \nhttps:\/\/doi.org\/10.1109\/SC.2012.25","DOI":"10.1109\/SC.2012.25"},{"key":"3_CR6","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"20","DOI":"10.1007\/11516798_2","volume-title":"Principles of Distributed Systems","author":"BN Chun","year":"2005","unstructured":"Chun, B.N.: DART: distributed automated regression testing for large-scale network applications. In: Higashino, T. (ed.) OPODIS 2004. LNCS, vol. 3544, pp. 20\u201336. Springer, Heidelberg (2005). \nhttps:\/\/doi.org\/10.1007\/11516798_2"},{"key":"3_CR7","doi-asserted-by":"publisher","unstructured":"Colby, K., Maji, A.K., Rahman, J., Bottum, J.: Testpilot: A flexible framework for user-centric testing of HPC clusters. In: Proceedings of the Fourth International Workshop on HPC User Support Tools, HUST 2017, pp. 5:1\u20135:10. ACM, New York (2017). \nhttps:\/\/doi.org\/10.1145\/3152493.3152555\n\n. \nhttp:\/\/doi.acm.org\/10.1145\/3152493.3152555","DOI":"10.1145\/3152493.3152555"},{"key":"3_CR8","unstructured":"Dongarra, J., Heroux, M.A., Luszczek, P.: HPCG benchmark: a new metric for ranking high performance computing systems. Technical report, UT-EECS-15-736, Electrical Engineering and Compute Science Department, University of Tennessee, Knoxville, November 2015. \nhttps:\/\/library.eecs.utk.edu\/storage\/594phpwDhjVNut-eecs-15-736.pdf"},{"issue":"4","key":"3_CR9","doi-asserted-by":"publisher","first-page":"69","DOI":"10.1109\/MCSE.2012.84","volume":"14","author":"PF Dubois","year":"2012","unstructured":"Dubois, P.F.: Testing scientific programs. Comput. Sci. Eng. 14(4), 69\u201373 (2012). \nhttps:\/\/doi.org\/10.1109\/MCSE.2012.84","journal-title":"Comput. Sci. Eng."},{"key":"3_CR10","unstructured":"Furlani, J.L., Osel, P.W.: Abstract yourself with modules. In: Proceedings of the 10th USENIX Conference on System Administration, LISA 1996, pp. 193\u2013204. USENIX Association, Berkeley (1996). \nhttp:\/\/dl.acm.org\/citation.cfm?id=1029824.1029858"},{"key":"3_CR11","doi-asserted-by":"publisher","unstructured":"Gamblin, T., et al.: The Spack package manager: bringing order to HPC software chaos. In: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis, SC 2015, pp. 40:1\u201340:12. ACM, New York (2015). \nhttps:\/\/doi.org\/10.1145\/2807591.2807623\n\n. \nhttp:\/\/doi.acm.org\/10.1145\/2807591.2807623","DOI":"10.1145\/2807591.2807623"},{"key":"3_CR12","unstructured":"GrafanaLabs: Grafana: The open platform for beautiful analytics and monitoring. \nhttps:\/\/grafana.com\/"},{"key":"3_CR13","unstructured":"Graylog Community: Enterprise Log Management for All. \nhttps:\/\/www.graylog.org\/"},{"issue":"1","key":"3_CR14","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1137\/080715962","volume":"32","author":"I Horenko","year":"2010","unstructured":"Horenko, I.: Finite element approach to clustering of multidimensional time series. SIAM J. Sci. Comput. 32(1), 62\u201383 (2010). \nhttps:\/\/doi.org\/10.1137\/080715962","journal-title":"SIAM J. Sci. Comput."},{"key":"3_CR15","doi-asserted-by":"publisher","unstructured":"Hoste, K., Timmerman, J., Georges, A., Weirdt, S.D.: Easybuild: building software with ease. In: 2012 IEEE International Conference on Services Computing (SCC), pp. 572\u2013582, November 2013. \nhttps:\/\/doi.org\/10.1109\/SC.Companion.2012.81\n\n. doi.ieeecomputersociety.org\/10.1109\/SC.Companion.2012.81","DOI":"10.1109\/SC.Companion.2012.81"},{"key":"3_CR16","unstructured":"J\u00fclich Supercomputing Centre: JUBE Benchmarking Environment. \nhttps:\/\/apps.fz-juelich.de\/jsc\/jube\/jube2\/docu\/index.html"},{"key":"3_CR17","doi-asserted-by":"publisher","unstructured":"Khuvis, S., et al.: A continuous integration-based framework for software management. In: Proceedings of the Practice and Experience in Advanced Research Computing on Rise of the Machines (Learning), PEARC 2019, pp. 28:1\u201328:7. ACM, New York (2019). \nhttps:\/\/doi.org\/10.1145\/3332186.3332219\n\n. \nhttp:\/\/doi.acm.org\/10.1145\/3332186.3332219","DOI":"10.1145\/3332186.3332219"},{"key":"3_CR18","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"334","DOI":"10.1007\/978-3-319-67630-2_25","volume-title":"High Performance Computing","author":"T Kurth","year":"2017","unstructured":"Kurth, T., et al.: Analyzing performance of selected NESAP applications on the Cori HPC system. In: Kunkel, J.M., Yokota, R., Taufer, M., Shalf, J. (eds.) ISC High Performance 2017. LNCS, vol. 10524, pp. 334\u2013347. Springer, Cham (2017). \nhttps:\/\/doi.org\/10.1007\/978-3-319-67630-2_25"},{"key":"3_CR19","unstructured":"Lockwood, G.: IOR and mdtest (2019). \nhttps:\/\/github.com\/hpc\/ior"},{"key":"3_CR20","doi-asserted-by":"publisher","unstructured":"Ma, Wenjing, Ao, Yulong, Yang, Chao, Williams, Samuel: Solving a trillion unknowns per second with HPGMG on Sunway TaihuLight. Cluster Comput. 1\u201315 (2019). \nhttps:\/\/doi.org\/10.1007\/s10586-019-02938-w","DOI":"10.1007\/s10586-019-02938-w"},{"key":"3_CR21","unstructured":"McLay, R.: Lmod: A New Environment Module System. \nhttps:\/\/lmod.readthedocs.io\/"},{"key":"3_CR22","doi-asserted-by":"publisher","unstructured":"Merchant, S., Prabhakar, G.: Tool for performance tuning and regression analyses of HPC systems and applications. In: 2012 19th International Conference on High Performance Computing, pp. 1\u20136, December 2012. \nhttps:\/\/doi.org\/10.1109\/HiPC.2012.6507528","DOI":"10.1109\/HiPC.2012.6507528"},{"key":"3_CR23","unstructured":"Open Source: Environment Modules. \nhttp:\/\/modules.sourceforge.net\/"},{"key":"3_CR24","unstructured":"Sauers, J.: Onyx Point works with Exascale Computing Project to bring CI to supercomputing centers (2018). \nhttps:\/\/www.onyxpoint.com\/onyxpoint-works-with-ecp-to-bring-ci-to-supercomputers\/"},{"key":"3_CR25","doi-asserted-by":"publisher","unstructured":"Shan, H., Williams, S., Zheng, Y., Kamil, A., Yelick, K.: Implementing high-performance geometric multigrid solver with naturally grained messages. In: 2015 9th International Conference on Partitioned Global Address Space Programming Models, pp. 38\u201346, September 2015. \nhttps:\/\/doi.org\/10.1109\/PGAS.2015.12","DOI":"10.1109\/PGAS.2015.12"},{"key":"3_CR26","unstructured":"Siddiqui, S.: Buildtest: A HPC Application Testing Framework. \nhttps:\/\/github.com\/HPC-buildtest\/buildtest"},{"key":"3_CR27","unstructured":"Whitney, C., Bautista, E., Davis, T.: The NERSC Data Collect Environment. In: Cray User Group 2016. CUG16 (2016). \nhttps:\/\/cug.org\/proceedings\/cug2016_proceedings\/includes\/files\/pap101s2-file1.pdf"},{"key":"3_CR28","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"44","DOI":"10.1007\/10968987_3","volume-title":"Job Scheduling Strategies for Parallel Processing","author":"AB Yoo","year":"2003","unstructured":"Yoo, A.B., Jette, M.A., Grondona, M.: SLURM: simple Linux utility for resource management. In: Feitelson, D., Rudolph, L., Schwiegelshohn, U. (eds.) JSSPP 2003. LNCS, vol. 2862, pp. 44\u201360. Springer, Heidelberg (2003). \nhttps:\/\/doi.org\/10.1007\/10968987_3\n\n. \nhttps:\/\/slurm.schedmd.com\/"}],"container-title":["Communications in Computer and Information Science","Tools and Techniques for High Performance Computing"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-44728-1_3","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,3,25]],"date-time":"2020-03-25T23:07:41Z","timestamp":1585177661000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/978-3-030-44728-1_3"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030447274","9783030447281"],"references-count":28,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-44728-1_3","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"value":"1865-0929","type":"print"},{"value":"1865-0937","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"26 March 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"HUST","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Annual Workshop on HPC User Support Tools","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Denver, CO","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"USA","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2019","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 November 2019","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"11 November 2019","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"6","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"hust2019","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/hust-workshop.github.io\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Single-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Linklings","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"100% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}