{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,2,18]],"date-time":"2026-02-18T23:22:34Z","timestamp":1771456954018,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":41,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,10,29]],"date-time":"2024-10-29T00:00:00Z","timestamp":1730160000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-sa\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,10,29]]},"DOI":"10.1145\/3671127.3698182","type":"proceedings-article","created":{"date-parts":[[2024,10,30]],"date-time":"2024-10-30T00:30:41Z","timestamp":1730248241000},"page":"110-120","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":8,"title":["Office-in-the-Loop for Building HVAC Control with Multimodal Foundation Models"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5762-0783","authenticated-orcid":false,"given":"Tomoya","family":"Sawada","sequence":"first","affiliation":[{"name":"MITSUBISHI ELECTRIC, Yokohama, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-7020-6386","authenticated-orcid":false,"given":"Takaomi","family":"Hasegawa","sequence":"additional","affiliation":[{"name":"Matsuo Institute, Inc, Tokyo, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0009-0002-2575-3917","authenticated-orcid":false,"given":"Keiichi","family":"Yokoyama","sequence":"additional","affiliation":[{"name":"Matsuo Institute, Inc, Tokyo, Japan"}]},{"ORCID":"https:\/\/orcid.org\/0009-0006-1466-5806","authenticated-orcid":false,"given":"Masahiro","family":"Mizuno","sequence":"additional","affiliation":[{"name":"MITSUBISHI ELECTRIC, Yokohama, Japan"}]}],"member":"320","published-online":{"date-parts":[[2024,10,29]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.buildenv.2013.11.016"},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.3390\/buildings13112680"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.apenergy.2022.119104"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.3390\/s20030781"},{"key":"e_1_3_2_2_5_1","volume-title":"Is gpt-4 a good data analyst? arXiv preprint arXiv:2305.15038","author":"Cheng Liying","year":"2023","unstructured":"Liying Cheng, Xingxuan Li, and Lidong Bing. 2023. Is gpt-4 a good data analyst? arXiv preprint arXiv:2305.15038 (2023)."},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.buildenv.2022.109438"},{"key":"e_1_3_2_2_7_1","volume-title":"The 18th ACM\/IEEE HRI.","author":"Cui Yuchen","unstructured":"Yuchen Cui, Karamcheti Siddharth, Raj Palleti, Nidhya Shivakumar, Percy Liang, and Dorsa Sadigh. 2023. \"No, to the Right\" - Online Language Corrections for Robotic Manipulation via Shared Autonomy. In The 18th ACM\/IEEE HRI."},{"key":"e_1_3_2_2_8_1","unstructured":"Danny Driess Fei Xia Mehdi S. M. Sajjadi Corey Lynch Aakanksha Chowdhery Brian Ichter Ayzaan Wahid Jonathan Tompson Quan Vuong Tianhe Yu Wenlong Huang Yevgen Chebotar Pierre Sermanet Daniel Duckworth Sergey Levine Vincent Vanhoucke Karol Hausman Marc Toussaint Klaus Greff Andy Zeng Igor Mordatch and Pete Florence. 2023. PaLM-E: An Embodied Multimodal Language Model. In arXiv preprint arXiv:2303.03378."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-022-30761-2"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"crossref","unstructured":"Daocheng Fu Xin Li Licheng Wen Min Dou Pinlong Cai Botian Shi and Qiao Yu. 2024. Drive Like a Human: Rethinking Autonomous Driving with Large Language Models.. In 2024 IEEE\/CVF WACVW. 910--919.","DOI":"10.1109\/WACVW60836.2024.00102"},{"key":"e_1_3_2_2_11_1","volume-title":"Unlocking multimodal understanding across millions of tokens of context. arXiv preprint arXiv:2403.05530","author":"Google Gemini Team","year":"2024","unstructured":"Gemini Team Google. 2024. Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context. arXiv preprint arXiv:2403.05530 (2024)."},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/IROS55552.2023.10341989"},{"key":"e_1_3_2_2_13_1","volume-title":"The Challenge of Using LLMs to Simulate Human Behavior: A Causal Inference Perspective. Columbia Business School Research Paper 4750172 (March","author":"Gui George","year":"2024","unstructured":"George Gui and Olivier Toubia. 2024. The Challenge of Using LLMs to Simulate Human Behavior: A Causal Inference Perspective. Columbia Business School Research Paper 4750172 (March 2024)."},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.enbuild.2021.111793"},{"key":"e_1_3_2_2_15_1","volume-title":"Mathprompter: Mathematical reasoning using large language models. arXiv preprint arXiv:2303.05398","author":"Imani Shima","year":"2023","unstructured":"Shima Imani, Liang Du, and Harsh Shrivastava. 2023. Mathprompter: Mathematical reasoning using large language models. arXiv preprint arXiv:2303.05398 (2023)."},{"key":"e_1_3_2_2_16_1","volume-title":"Foundations and Trends in Multimodal Machine Learning: Principles, Challenges, and Open Questions. arXiv preprint arXiv:2209.03430","author":"Liang Paul Pu","year":"2022","unstructured":"Paul Pu Liang, Amir Zadeh, and Louis-Philippe Morency. 2022. Foundations and Trends in Multimodal Machine Learning: Principles, Challenges, and Open Questions. arXiv preprint arXiv:2209.03430 (2022)."},{"key":"e_1_3_2_2_17_1","volume-title":"Eureka: Human-Level Reward Design via Coding Large Language Models. In The 12th ICLR.","author":"Ma Yecheng Jason","year":"2024","unstructured":"Yecheng Jason Ma, William Liang, Guanzhi Wang, De-An Huang, Osbert Bastani, Dinesh Jayaraman, Yuke Zhu, Linxi Fan, and Anima Anandkumar. 2024. Eureka: Human-Level Reward Design via Coding Large Language Models. In The 12th ICLR."},{"key":"e_1_3_2_2_18_1","volume-title":"NeurIPS 2023 Foundation Models Workshop.","author":"Mao Jiageng","year":"2023","unstructured":"Jiageng Mao, Yuxi Qian, Hang Zhao, and Yue Wang. 2023. GPT-Driver: Learning to Drive with GPT. In NeurIPS 2023 Foundation Models Workshop."},{"key":"e_1_3_2_2_19_1","volume-title":"Harlan M Krumholz, Jure Leskovec, Eric J Topol, and Pranav Rajpurkar.","author":"Moor Michael","year":"2023","unstructured":"Michael Moor, Oishi Banerjee, Zahra Shakeri Hossein Abad, Harlan M Krumholz, Jure Leskovec, Eric J Topol, and Pranav Rajpurkar. 2023. Foundation models for generalist medical artificial intelligence. Nature 616, 7956 (2023), 259--265."},{"key":"e_1_3_2_2_20_1","unstructured":"OpenAI. 2024. GPT-4 Technical Report. arXiv preprint arXiv:2303.08774 (2024)."},{"key":"e_1_3_2_2_21_1","volume-title":"Proceedings of the ACM UIST.","author":"Park Joon Sung","unstructured":"Joon Sung Park, Lindsay Popowski, Carrie Jun Cai, Meredith Ringel Morris, Percy Liang, and Michael S. Bernstein. 2022. Social Simulacra: Creating Populated Prototypes for Social Computing Systems. In Proceedings of the ACM UIST."},{"key":"e_1_3_2_2_22_1","volume-title":"NeurIPS 2022 ML Safety Workshop","volume":"2211","author":"Raj Harsh","year":"2022","unstructured":"Harsh Raj, Domenic Rosati, and Subhabrata Majumdar. 2022. Measuring Reliability of Large Language Models through Semantic Consistency. In NeurIPS 2022 ML Safety Workshop, Vol. abs\/2211.05853."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"crossref","unstructured":"Anjali Rajith Sakurai Soki and Mine Hiroshi. 2018. Real-time optimized HVAC control system on top of an IoT framework. In 2018 Third FMEC. 181--186.","DOI":"10.1109\/FMEC.2018.8364062"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1109\/LRA.2023.3312035"},{"key":"e_1_3_2_2_25_1","volume-title":"Proceedings of the JSRAE Annual Conference","volume":"2023","author":"Semitsu Takayuki","year":"2023","unstructured":"Takayuki Semitsu, Yousuke Kaizu, Masayuki Komatsu, Shinji Nakamura, Satoshia Akagi, and Mamoru Hamada. 2023. Development of Model for Predicting Air-conditioning Power Consumption in Offices using Machine Learning Models.. In Proceedings of the JSRAE Annual Conference, Vol. 2023."},{"key":"e_1_3_2_2_26_1","volume-title":"Masahoshi Tomizuka, Wei Zhan, and Mingyu Ding.","author":"Sha Hao","year":"2023","unstructured":"Hao Sha, Yao Mu, Yuxuan Jiang, Li Chen, Chenfeng Xu, Ping Luo, Shengbo Li Eben, Masahoshi Tomizuka, Wei Zhan, and Mingyu Ding. 2023. LanguageMPC: Large Language Models as Decision Makers for Autonomous Driving. CoRR (October 2023)."},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52688.2022.01519"},{"key":"e_1_3_2_2_28_1","volume-title":"Predicting Thermal Comfort in Buildings With Machine Learning and Occupant Feedback. In 2023 IEEE International Workshop on MetroLivEnv. 34--39","author":"Skaloumpakas Panagiotis","year":"2023","unstructured":"Panagiotis Skaloumpakas, Elissaios Sarmas, Zoi Mylona, Alessio Cavadenti, Francesca Santori, and Vangelis Marinakis. 2023. Predicting Thermal Comfort in Buildings With Machine Learning and Occupant Feedback. In 2023 IEEE International Workshop on MetroLivEnv. 34--39."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV51070.2023.00280"},{"key":"e_1_3_2_2_30_1","volume-title":"Pre-Trained Large Language Models for Industrial Control. arXiv preprint arXiv:2308.03028abs\/2308.03028","author":"Song Lei","year":"2023","unstructured":"Lei Song, Chuheng Zhang, Li Zhao, and Jiang Bian. 2023. Pre-Trained Large Language Models for Industrial Control. arXiv preprint arXiv:2308.03028abs\/2308.03028 (2023)."},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jobe.2022.105067"},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.bff007ab"},{"key":"e_1_3_2_2_33_1","first-page":"5696","article-title":"Omnivl: One foundation model for image-language and video-language tasks","volume":"35","author":"Wang Junke","year":"2022","unstructured":"Junke Wang, Dongdong Chen, Zuxuan Wu, Chong Luo, Luowei Zhou, Yucheng Zhao, Yujia Xie, Ce Liu, Yu-Gang Jiang, and Lu Yuan. 2022. Omnivl: One foundation model for image-language and video-language tasks. NeurIPS 35 (2022), 5696--5710.","journal-title":"NeurIPS"},{"key":"e_1_3_2_2_34_1","volume-title":"Sharan Narang, Aakanksha Chowdhery, and Denny Zhou.","author":"Wang Xuezhi","year":"2023","unstructured":"Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc V Le, Ed H. Chi, Sharan Narang, Aakanksha Chowdhery, and Denny Zhou. 2023. Self-Consistency Improves Chain of Thought Reasoning in Language Models. In The 11th ICLR."},{"key":"e_1_3_2_2_35_1","volume-title":"Visual chatgpt: Talking, drawing and editing with visual foundation models. arXiv preprint arXiv:2303.04671","author":"Wu Chenfei","year":"2023","unstructured":"Chenfei Wu, Shengming Yin, Weizhen Qi, Xiaodong Wang, Zecheng Tang, and Nan Duan. 2023. Visual chatgpt: Talking, drawing and editing with visual foundation models. arXiv preprint arXiv:2303.04671 (2023)."},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1109\/TPAMI.2023.3275156"},{"key":"e_1_3_2_2_37_1","volume-title":"Set-of-mark prompting unleashes extraordinary visual grounding in gpt-4v. arXiv preprint arXiv:2310.11441","author":"Yang Jianwei","year":"2023","unstructured":"Jianwei Yang, Hao Zhang, Feng Li, Xueyan Zou, Chunyuan Li, and Jianfeng Gao. 2023. Set-of-mark prompting unleashes extraordinary visual grounding in gpt-4v. arXiv preprint arXiv:2310.11441 (2023)."},{"key":"e_1_3_2_2_38_1","volume-title":"The dawn of lmms: Preliminary explorations with gpt-4v (ision). arXiv preprint arXiv:2309.17421 9, 1","author":"Yang Zhengyuan","year":"2023","unstructured":"Zhengyuan Yang, Linjie Li, Kevin Lin, Jianfeng Wang, Chung-Ching Lin, Zicheng Liu, and Lijuan Wang. 2023. The dawn of lmms: Preliminary explorations with gpt-4v (ision). arXiv preprint arXiv:2309.17421 9, 1 (2023), 1."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.buildenv.2021.107952"},{"key":"e_1_3_2_2_40_1","volume-title":"Hao-Tien Lewis Chiang, Tom Erez, Leonard Hasenclever, Jan Humplik, Brian Ichter, Ted Xiao, Peng Xu, Andy Zeng, Tingnan Zhang, Nicolas Heess, Dorsa Sadigh, Jie Tan, Yuval Tassa, and Fei Xia.","author":"Yu Wenhao","year":"2023","unstructured":"Wenhao Yu, Nimrod Gileadi, Chuyuan Fu, Sean Kirmani, Kuang-Huei Lee, Montse Gonzalez Arenas, Hao-Tien Lewis Chiang, Tom Erez, Leonard Hasenclever, Jan Humplik, Brian Ichter, Ted Xiao, Peng Xu, Andy Zeng, Tingnan Zhang, Nicolas Heess, Dorsa Sadigh, Jie Tan, Yuval Tassa, and Fei Xia. 2023. Language to Rewards for Robotic Skill Synthesis. arXiv:2306.08647"},{"key":"e_1_3_2_2_41_1","volume-title":"Language Models Know the Value of Numbers. arXiv preprint arXiv:2401.03735","author":"Zhu Fangwei","year":"2024","unstructured":"Fangwei Zhu, Damai Dai, and Zhifang Sui. 2024. Language Models Know the Value of Numbers. arXiv preprint arXiv:2401.03735 (2024)."}],"event":{"name":"BuildSys '24: The 11th ACM International Conference on Systems for Energy-Efficient Buildings, Cities, and Transportation","location":"Hangzhou China","acronym":"BuildSys '24"},"container-title":["Proceedings of the 11th ACM International Conference on Systems for Energy-Efficient Buildings, Cities, and Transportation"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3671127.3698182","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3671127.3698182","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,4]],"date-time":"2025-11-04T23:25:36Z","timestamp":1762298736000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3671127.3698182"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,29]]},"references-count":41,"alternative-id":["10.1145\/3671127.3698182","10.1145\/3671127"],"URL":"https:\/\/doi.org\/10.1145\/3671127.3698182","relation":{},"subject":[],"published":{"date-parts":[[2024,10,29]]},"assertion":[{"value":"2024-10-29","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}