{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,2]],"date-time":"2026-07-02T07:47:55Z","timestamp":1782978475595,"version":"3.54.5"},"publisher-location":"Cham","reference-count":32,"publisher":"Springer International Publishing","isbn-type":[{"value":"9783030925109","type":"print"},{"value":"9783030925116","type":"electronic"}],"license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021]]},"DOI":"10.1007\/978-3-030-92511-6_5","type":"book-chapter","created":{"date-parts":[[2021,12,7]],"date-time":"2021-12-07T14:05:12Z","timestamp":1638885912000},"page":"72-92","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["ComBench: A\u00a0Benchmarking Framework for\u00a0Publish\/Subscribe Communication Protocols Under Network Limitations"],"prefix":"10.1007","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-7960-0823","authenticated-orcid":false,"given":"Stefan","family":"Herrnleben","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maximilian","family":"Leidinger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7481-4099","authenticated-orcid":false,"given":"Veronika","family":"Lesch","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Thomas","family":"Prantl","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9643-6543","authenticated-orcid":false,"given":"Johannes","family":"Grohmann","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7275-0738","authenticated-orcid":false,"given":"Christian","family":"Krupitzer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Samuel","family":"Kounev","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,12,8]]},"reference":[{"issue":"4","key":"5_CR1","doi-asserted-by":"publisher","first-page":"2347","DOI":"10.1109\/COMST.2015.2444095","volume":"17","author":"A Al-Fuqaha","year":"2015","unstructured":"Al-Fuqaha, A., Guizani, M., Mohammadi, M., Aledhari, M., Ayyash, M.: Internet of things: a survey on enabling technologies, protocols, and applications. IEEE Commun. Surv. Tutor. 17(4), 2347\u20132376 (2015). https:\/\/doi.org\/10.1109\/COMST.2015.2444095","journal-title":"IEEE Commun. Surv. Tutor."},{"key":"5_CR2","doi-asserted-by":"crossref","unstructured":"Appel, S., Sachs, K., Buchmann, A.: Towards benchmarking of AMQP. In: Proceedings of the Fourth ACM International Conference on Distributed Event-Based Systems, pp. 99\u2013100 (2010)","DOI":"10.1145\/1827418.1827438"},{"key":"5_CR3","doi-asserted-by":"crossref","unstructured":"Bansal, S., Kumar, D.: IoT application layer protocols: performance analysis and significance in smart city. In: 2019 10th International Conference on Computing, Communication and Networking Technologies (ICCCNT), pp. 1\u20136. IEEE (2019)","DOI":"10.1109\/ICCCNT45670.2019.8944807"},{"key":"5_CR4","doi-asserted-by":"crossref","unstructured":"Boano, C.A., et al.: IoTBench: towards a benchmark for low-power wireless networking. In: 2018 IEEE Workshop on Benchmarking Cyber-Physical Networks and Systems (CPSBench). IEEE (2018)","DOI":"10.1109\/CPSBench.2018.00013"},{"key":"5_CR5","doi-asserted-by":"crossref","unstructured":"Chen, Y., Kunz, T.: Performance evaluation of IoT protocols under a constrained wireless access network. In: 2016 International Conference on Selected Topics in Mobile & Wireless Networking (MoWNeT), pp. 1\u20137. IEEE (2016)","DOI":"10.1109\/MoWNet.2016.7496622"},{"key":"5_CR6","doi-asserted-by":"crossref","unstructured":"Collina, M., Bartolucci, M., Vanelli-Coralli, A., Corazza, G.E.: Internet of Things application layer protocol analysis over error and delay prone links. In: 2014 7th Advanced Satellite Multimedia Systems Conference and the 13th Signal Processing for Space Communications Workshop (ASMS\/SPSC), pp. 398\u2013404. IEEE (2014)","DOI":"10.1109\/ASMS-SPSC.2014.6934573"},{"key":"5_CR7","unstructured":"Community: Locust Website (2021). https:\/\/locust.io\/, Accessed 15 Apr 2021"},{"key":"5_CR8","unstructured":"Community: MZBench Website (2021). https:\/\/github.com\/satori-com\/mzbench, Accessed 13 Apr 2021"},{"key":"5_CR9","unstructured":"Corp, G.: Gatling Website (2021). https:\/\/gatling.io\/, Accessed 13 Apr 2021"},{"key":"5_CR10","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3292674","volume":"51","author":"J Dizdarevi\u0107","year":"2019","unstructured":"Dizdarevi\u0107, J., Carpio, F., Jukan, A., Masip-Bruin, X.: A survey of communication protocols for internet of things and related challenges of fog and cloud computing integration. ACM Comput. Surv. (CSUR) 51, 1\u201329 (2019)","journal-title":"ACM Comput. Surv. (CSUR)"},{"issue":"2","key":"5_CR11","doi-asserted-by":"publisher","first-page":"114","DOI":"10.1145\/857076.857078","volume":"35","author":"PT Eugster","year":"2003","unstructured":"Eugster, P.T., Felber, P.A., Guerraoui, R., Kermarrec, A.M.: The many faces of publish\/subscribe. ACM Comput. Surv. (CSUR) 35(2), 114\u2013131 (2003)","journal-title":"ACM Comput. Surv. (CSUR)"},{"key":"5_CR12","doi-asserted-by":"crossref","unstructured":"G\u00fcndo\u011fran, C., Kietzmann, P., Lenders, M., Petersen, H., Schmidt, T.C., W\u00e4hlisch, M.: NDN, CoAP, and MQTT: a comparative measurement study in the IoT. In: Proceedings of the 5th ACM Conference on Information-Centric Networking, pp. 159\u2013171 (2018)","DOI":"10.1145\/3267955.3267967"},{"key":"5_CR13","volume-title":"Apache JMeter","author":"E Halili","year":"2008","unstructured":"Halili, E.: Apache JMeter. Packt Publishing, Birmingham (2008)"},{"key":"5_CR14","unstructured":"Herrnleben, S., et al.: Evaluation results of ComBench as open data. Technical report, University of Wuerzburg (2021). https:\/\/doi.org\/10.5281\/zenodo.4723344, Accessed 30 Apr 2021"},{"key":"5_CR15","doi-asserted-by":"crossref","unstructured":"Iglesias-Urkia, M., Orive, A., Barcelo, M., Moran, A., Bilbao, J., Urbieta, A.: Towards a lightweight protocol for industry 4.0: an implementation based benchmark. In: 2017 IEEE International Workshop of Electronics, Control, Measurement, Signals and their Application to Mechatronics (ECMSM) (2017)","DOI":"10.1109\/ECMSM.2017.7945894"},{"key":"5_CR16","doi-asserted-by":"crossref","unstructured":"Jacob, R., Boano, C.A., Raza, U., Zimmerling, M., Thiele, L.: Towards a methodology for experimental evaluation in low-power wireless networking. In: Proceedings of the 2nd Workshop on Benchmarking Cyber-Physical Systems and Internet of Things, pp. 18\u201323 (2019)","DOI":"10.1145\/3312480.3313173"},{"key":"5_CR17","doi-asserted-by":"crossref","unstructured":"Kayal, P., Perros, H.: A comparison of IoT application layer protocols through a smart parking implementation. In: 2017 20th Conference on Innovations in Clouds, Internet and Networks (ICIN), pp. 331\u2013336. IEEE (2017)","DOI":"10.1109\/ICIN.2017.7899436"},{"key":"5_CR18","doi-asserted-by":"crossref","unstructured":"von Kistowski, J., Arnold, J.A., Huppler, K., Lange, K.D., Henning, J.L., Cao, P.: How to build a benchmark. In: Proceedings of the 6th ACM\/SPEC International Conference on Performance Engineering (ICPE 2015), ICPE \u201915. ACM, New York (2015)","DOI":"10.1145\/2668930.2688819"},{"key":"5_CR19","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-41705-5","volume-title":"Systems Benchmarking","author":"S Kounev","year":"2020","unstructured":"Kounev, S., Lange, K.D., von Kistowski, J.: Systems Benchmarking. Springer, Heidelberg (2020). https:\/\/doi.org\/10.1007\/978-3-030-41705-5"},{"key":"5_CR20","doi-asserted-by":"crossref","unstructured":"Naik, N.: Choice of effective messaging protocols for IoT systems: MQTT, CoAP, AMQP and HTTP. In: 2017 IEEE International Systems Engineering Symposium (ISSE), pp. 1\u20137. IEEE (2017)","DOI":"10.1109\/SysEng.2017.8088251"},{"key":"5_CR21","doi-asserted-by":"crossref","unstructured":"Neagoe, T., Cristea, V., Banica, L.: NTP versus PTP in computer networks clock synchronization. In: 2006 IEEE International Symposium on Industrial Electronics, vol. 1, pp. 317\u2013362. IEEE (2006)","DOI":"10.1109\/ISIE.2006.295613"},{"key":"5_CR22","doi-asserted-by":"crossref","unstructured":"Pohl, M., Kubela, J., Bosse, S., Turowski, K.: Performance evaluation of application layer protocols for the internet-of-things. In: 2018 Sixth International Conference on Enterprise Systems (ES), pp. 180\u2013187. IEEE (2018)","DOI":"10.1109\/ES.2018.00035"},{"key":"5_CR23","doi-asserted-by":"crossref","unstructured":"Profanter, S., Tekat, A., Dorofeev, K., Rickert, M., Knoll, A.: OPC UA versus ROS, DDS, and MQTT: performance evaluation of industry 4.0 protocols. In: Proceedings of the IEEE International Conference on Industrial Technology (ICIT) (2019)","DOI":"10.1109\/ICIT.2019.8755050"},{"key":"5_CR24","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"203","DOI":"10.1007\/978-3-642-14589-6_21","volume-title":"Database Systems for Advanced Applications","author":"K Sachs","year":"2010","unstructured":"Sachs, K., Appel, S., Kounev, S., Buchmann, A.: Benchmarking publish\/subscribe-based messaging systems. In: Yoshikawa, M., Meng, X., Yumoto, T., Ma, Q., Sun, L., Watanabe, C. (eds.) DASFAA 2010. LNCS, vol. 6193, pp. 203\u2013214. Springer, Heidelberg (2010). https:\/\/doi.org\/10.1007\/978-3-642-14589-6_21"},{"issue":"8","key":"5_CR25","doi-asserted-by":"publisher","first-page":"410","DOI":"10.1016\/j.peva.2009.01.003","volume":"66","author":"K Sachs","year":"2009","unstructured":"Sachs, K., Kounev, S., Bacon, J., Buchmann, A.: Performance evaluation of message-oriented middleware using the SPECjms2007 benchmark. Perf. Eval. 66(8), 410\u2013434 (2009)","journal-title":"Perf. Eval."},{"issue":"21","key":"5_CR26","doi-asserted-by":"publisher","first-page":"e4257","DOI":"10.1002\/cpe.4257","volume":"29","author":"A Shukla","year":"2017","unstructured":"Shukla, A., Chaturvedi, S., Simmhan, Y.: RIoTBench: an IoT benchmark for distributed stream processing systems. Concurr. Comput. Pract. Exp. 29(21), e4257 (2017)","journal-title":"Concurr. Comput. Pract. Exp."},{"key":"5_CR27","unstructured":"Statista, IHS: Internet of Things - number of connected devices worldwide 2015\u20132025 (2018). https:\/\/www.statista.com\/statistics\/471264\/iot-number-of-connected-devices-worldwide\/"},{"key":"5_CR28","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.cmpb.2016.03.004","volume":"129","author":"A Talaminos-Barroso","year":"2016","unstructured":"Talaminos-Barroso, A., Estudillo-Valderrama, M.A., Roa, L.M., Reina-Tosina, J., Ortega-Ruiz, F.: A machine-to-machine protocol benchmark for ehealth applications - use case: respiratory rehabilitation. Comput. Methods Prog. Biomed. 129, 1\u201311 (2016)","journal-title":"Comput. Methods Prog. Biomed."},{"key":"5_CR29","unstructured":"University of Wuerzburg, Institute of Computer Science, Germany, Chair of Software Engineering: Git repository of ComBench (2021). https:\/\/github.com\/DescartesResearch\/ComBench"},{"key":"5_CR30","doi-asserted-by":"crossref","unstructured":"Wirawan, I.M., Wahyono, I.D., Idfi, G., Kusumo, G.R.: IoT communication system using publish-subscribe. In: 2018 International Seminar on Application for Technology of Information and Communication, pp. 61\u201365. IEEE (2018)","DOI":"10.1109\/ISEMANTIC.2018.8549814"},{"key":"5_CR31","doi-asserted-by":"crossref","unstructured":"Zhang, H.L., Zhang, S., Li, X.J., Zhang, P., Liu, S.B.: Research of load testing and result application based on LoadRunner. In: 2012 National Conference on Information Technology and Computer Science. Atlantis Press (2012)","DOI":"10.2991\/citcs.2012.271"},{"key":"5_CR32","doi-asserted-by":"crossref","unstructured":"Zhang, K., Rabl, T., Sun, Y.P., Kumar, R., Zen, N., Jacobsen, H.A.: PSBench: a benchmark for content-and topic-based publish\/subscribe systems. In: Proceedings of the Posters & Demos Session, pp. 17\u201318. Association for Computing Machinery (2014)","DOI":"10.1145\/2678508.2678517"}],"container-title":["Lecture Notes of the Institute for Computer Sciences, Social Informatics and Telecommunications Engineering","Performance Evaluation Methodologies and Tools"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-92511-6_5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,13]],"date-time":"2024-09-13T23:01:57Z","timestamp":1726268517000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-92511-6_5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"ISBN":["9783030925109","9783030925116"],"references-count":32,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-92511-6_5","relation":{},"ISSN":["1867-8211","1867-822X"],"issn-type":[{"value":"1867-8211","type":"print"},{"value":"1867-822X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]},"assertion":[{"value":"8 December 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"VALUETOOLS","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"EAI International Conference on Performance Evaluation Methodologies and Tools","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2021","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"30 October 2021","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"31 October 2021","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"valuetools2021","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Confy +","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"32","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"16","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"50% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}