{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T20:50:26Z","timestamp":1776113426122,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":52,"publisher":"ACM","license":[{"start":{"date-parts":[[2025,4,25]],"date-time":"2025-04-25T00:00:00Z","timestamp":1745539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1145\/3706599.3719914","type":"proceedings-article","created":{"date-parts":[[2025,4,23]],"date-time":"2025-04-23T20:03:57Z","timestamp":1745438637000},"page":"1-9","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Design Principles and Guidelines for LLM Observability: Insights from Developers"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0007-6825-6212","authenticated-orcid":false,"given":"Xin","family":"Chen","sequence":"first","affiliation":[{"name":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0003-9443-9777","authenticated-orcid":false,"given":"Yan","family":"Li","sequence":"additional","affiliation":[{"name":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0001-1050-8364","authenticated-orcid":false,"given":"Xiaoming","family":"Wang","sequence":"additional","affiliation":[{"name":"Alibaba Cloud Computing, Alibaba Group, Hangzhou, China"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,4,25]]},"reference":[{"key":"e_1_3_3_1_1_2","doi-asserted-by":"publisher","unstructured":"Jim Waldo and Soline Boussard. 2024. GPTs and Hallucination: Why do large language models hallucinate? Queue 22 4 https:\/\/doi.org\/ 10.1145\/3688007","DOI":"10.1145\/3688007"},{"key":"e_1_3_3_1_2_2","doi-asserted-by":"publisher","unstructured":"Badhan Chandra Das M. Hadi Amini and Yanzhao Wu. 2025. Security and Privacy Challenges of Large Language Models: A Survey. ACM Comput. Surv. 57 6 https:\/\/doi.org\/ 10.1145\/3712001","DOI":"10.1145\/3712001"},{"key":"e_1_3_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1109\/SC41406.2024.00089"},{"key":"e_1_3_3_1_4_2","doi-asserted-by":"publisher","unstructured":"Samuel Greengard. 2024. Is It Possible to Truly Understand Performance in LLMs? Commun. ACM 67 12 https:\/\/doi.org\/ 10.1145\/3695860","DOI":"10.1145\/3695860"},{"key":"e_1_3_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.1145\/3605764.3623985"},{"key":"e_1_3_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.1145\/3691620.3695004"},{"key":"e_1_3_3_1_7_2","unstructured":"Tian Yu Liu Stefano Soatto Matteo Marchi Pratik Chaudhari and Paulo Tabuada. 2024. Meanings and Feelings of Large Language Models: Observability of Latent States in Generative AI. ArXiv abs\/2405.14061"},{"key":"e_1_3_3_1_8_2","unstructured":"Liming Dong Qinghua Lu and Liming Zhu. 2024. AgentOps: Enabling Observability of LLM Agents. ArXiv abs\/2411.05285"},{"key":"e_1_3_3_1_9_2","doi-asserted-by":"crossref","unstructured":"P Ganesan. 2024. LLM-Powered Observability Enhancing Monitoring and Diagnostics. J Artif Intell Mach Learn & Data Sci 2024 2 2","DOI":"10.51219\/JAIMLD\/premkumar-ganesan\/304"},{"key":"e_1_3_3_1_10_2","volume-title":"Interactive Planning Using Large Language Models for Partially Observable Robotic Tasks. 2024 IEEE International Conference on Robotics and Automation (ICRA)","author":"Sun Lingfeng","year":"2023","unstructured":"Lingfeng Sun, Devesh K. Jha, Chiori Hori, Siddarth Jain, Radu Corcodel, Xinghao Zhu, Masayoshi Tomizuka, and Diego Romeres. 2023. Interactive Planning Using Large Language Models for Partially Observable Robotic Tasks. 2024 IEEE International Conference on Robotics and Automation (ICRA)"},{"key":"e_1_3_3_1_11_2","doi-asserted-by":"publisher","DOI":"10.1145\/3687151.3687158"},{"key":"e_1_3_3_1_12_2","unstructured":"R.A. R. 2024. Mastering Large Language Models with Python: Unleash the Power of Advanced Natural Language Processing for Enterprise Innovation and Efficiency Using Large Language Models (LLMs) with Python. Orange Education Pvt Limited."},{"key":"e_1_3_3_1_13_2","volume-title":"Iker Lasa Ojanguren, and Ana I Torre-Bastida","author":"Diaz-De-Arcaya Josu","year":"2024","unstructured":"Josu Diaz-De-Arcaya, Juan L\u00f3pez-De-Armentia, Ra\u00fal Mi\u00f1\u00f3n, Iker Lasa Ojanguren, and Ana I Torre-Bastida 2024. Large Language Model Operations (LLMOps): Definition, Challenges, and Lifecycle Management. IEEE, City."},{"key":"e_1_3_3_1_14_2","doi-asserted-by":"crossref","unstructured":"Chakkrit Kla Tantithamthavorn Fabio Palomba Foutse Khomh and Joselito Joey Chua 2025. MLOps LLMOps FMOps and Beyond. IEEE Computer Society City.","DOI":"10.1109\/MS.2024.3477014"},{"key":"e_1_3_3_1_15_2","doi-asserted-by":"crossref","unstructured":"Saurabh Pahune and Zahid Akhtar. 2025. Transitioning from MLOps to LLMOps: Navigating the Unique Challenges of Large Language Models. Information 16 2","DOI":"10.3390\/info16020087"},{"key":"e_1_3_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.1145\/97243.97281"},{"key":"e_1_3_3_1_17_2","unstructured":"Ben Shneiderman Catherine Plaisant Maxine Cohen Steven Jacobs Niklas Elmqvist and Nicholas Diakopoulos. 2016. Designing the User Interface: Strategies for Effective Human-Computer Interaction. Pearson."},{"key":"e_1_3_3_1_18_2","unstructured":"Nigel Bevan 2005. Guidelines and standards for web usability. Citeseer City."},{"key":"e_1_3_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300233"},{"key":"e_1_3_3_1_20_2","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376301"},{"key":"e_1_3_3_1_21_2","doi-asserted-by":"publisher","unstructured":"Vikas Hassija Vinay Chamola Atmesh Mahapatra Abhinandan Singal Divyansh Goel Kaizhu Huang Simone Scardapane Indro Spinelli Mufti Mahmud and Amir Hussain. 2024. Interpreting Black-Box Models: A Review on Explainable Artificial Intelligence. Cognitive Computation 16 1 https:\/\/doi.org\/ 10.1007\/s12559-023-10179-8","DOI":"10.1007\/s12559-023-10179-8"},{"key":"e_1_3_3_1_22_2","doi-asserted-by":"publisher","unstructured":"Gesina Schwalbe and Bettina Finzel. 2024. A comprehensive taxonomy for explainable artificial intelligence: a systematic survey of surveys on methods and concepts. Data Mining and Knowledge Discovery 38 5 https:\/\/doi.org\/ 10.1007\/s10618-022-00867-8","DOI":"10.1007\/s10618-022-00867-8"},{"key":"e_1_3_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"e_1_3_3_1_24_2","doi-asserted-by":"crossref","unstructured":"Prachi Zodage Hussain Harianawala Hafsa Shaikh and Asad Kharodia. 2024. Explainable AI (XAI): History Basic Ideas and Methods. International Journal of Advanced Research in Science Communication and Technology","DOI":"10.48175\/IJARSCT-16988"},{"key":"e_1_3_3_1_25_2","doi-asserted-by":"publisher","DOI":"10.1007\/s44230-023-00038-y"},{"key":"e_1_3_3_1_26_2","unstructured":"Mark O. Riedl. 2019. Human-Centered Artificial Intelligence and Machine Learning. ArXiv abs\/1901.11184"},{"key":"e_1_3_3_1_27_2","volume-title":"Riedl","author":"Ehsan Upol","year":"2020","unstructured":"Upol Ehsan, and Mark O. Riedl 2020. Human-Centered Explainable AI: Towards a Reflective Sociotechnical Approach. Springer International Publishing, City."},{"key":"e_1_3_3_1_28_2","unstructured":"Finale Doshi-Velez and Been Kim. 2017. Towards A Rigorous Science of Interpretable Machine Learning. arXiv: Machine Learning"},{"key":"e_1_3_3_1_29_2","doi-asserted-by":"crossref","unstructured":"Peter Hase and Mohit Bansal 2020. Evaluating Explainable AI: Which Algorithmic Explanations Help Users Predict Model Behavior? City.","DOI":"10.18653\/v1\/2020.acl-main.491"},{"key":"e_1_3_3_1_30_2","doi-asserted-by":"publisher","unstructured":"Meike Nauta Jan Trienes Shreyasi Pathak Elisa Nguyen Michelle Peters Yasmin Schmitt J\u00f6rg Schl\u00f6tterer Maurice van Keulen and Christin Seifert. 2023. From Anecdotal Evidence to Quantitative Evaluation Methods: A Systematic Review on Evaluating Explainable AI. ACM Comput. Surv. 55 13s https:\/\/doi.org\/ 10.1145\/3583558","DOI":"10.1145\/3583558"},{"key":"e_1_3_3_1_31_2","unstructured":"Richard J. Tomsett Daniel Harborne Supriyo Chakraborty Prudhvi K. Gurram and Alun David Preece. 2019. Sanity Checks for Saliency Metrics. ArXiv abs\/1912.01451"},{"key":"e_1_3_3_1_32_2","volume-title":"Proceedings of the Proceedings of the 39th International Conference on Machine Learning, 2022, Proceedings of Machine Learning Research. PMLR","author":"Rong Yao","year":"2022","unstructured":"Yao Rong, Tobias Leemann, Vadim Borisov, Gjergji Kasneci, and Enkelejda Kasneci. 2022. A Consistent and Efficient Evaluation Strategy for Attribution Methods. In Proceedings of the Proceedings of the 39th International Conference on Machine Learning, 2022, Proceedings of Machine Learning Research. PMLR, 18770\u201318795."},{"key":"e_1_3_3_1_33_2","doi-asserted-by":"crossref","unstructured":"Dong Nguyen 2018. Comparing Automatic and Human Evaluation of Local Explanations for Text Classification. City.","DOI":"10.18653\/v1\/N18-1097"},{"key":"e_1_3_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173704"},{"key":"e_1_3_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1109\/tpami.2023.3331846"},{"key":"e_1_3_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1145\/3654777.3676414"},{"key":"e_1_3_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.1145\/3387166"},{"key":"e_1_3_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174098"},{"key":"e_1_3_3_1_39_2","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858529"},{"key":"e_1_3_3_1_40_2","doi-asserted-by":"publisher","DOI":"10.1145\/3581641.3584037"},{"key":"e_1_3_3_1_41_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-76827-9_9"},{"key":"e_1_3_3_1_42_2","doi-asserted-by":"publisher","DOI":"10.1145\/3698038.3698568"},{"key":"e_1_3_3_1_43_2","volume-title":"Are Software Developers Just Users of Development Tools? Assessing Developer Experience of a Graphical User Interface Designer","author":"Kati Kuusinen","unstructured":"Kati Kuusinen 2016. Are Software Developers Just Users of Development Tools? Assessing Developer Experience of a Graphical User Interface Designer. Springer International Publishing, City."},{"key":"e_1_3_3_1_44_2","doi-asserted-by":"crossref","unstructured":"Masitah Ghazali and Alfian Naufal Ravi Hidayat. 2023. Enhancing the Developer Experience (DX) in Docker Supported Projects. International Journal of Innovative Computing","DOI":"10.11113\/ijic.v13n1.393"},{"key":"e_1_3_3_1_45_2","doi-asserted-by":"publisher","unstructured":"Abdul Razzaq Jim Buckley Qin Lai Tingting Yu and Goetz Botterweck. 2024. A Systematic Literature Review on the Influence of Enhanced Developer Experience on Developers' Productivity: Factors Practices and Recommendations. ACM Comput. Surv. 57 1 https:\/\/doi.org\/ 10.1145\/3687299","DOI":"10.1145\/3687299"},{"key":"e_1_3_3_1_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/3698322.3698345"},{"key":"e_1_3_3_1_47_2","doi-asserted-by":"publisher","DOI":"10.5555\/3103196.3103209"},{"key":"e_1_3_3_1_48_2","doi-asserted-by":"publisher","DOI":"10.1145\/1134285.1134355"},{"key":"e_1_3_3_1_49_2","volume-title":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems","author":"Cherubini Mauro","unstructured":"Mauro Cherubini, Gina Venolia, Robert A Deline, and Amy J. Ko. 2007. Let's go to the whiteboard: how and why software developers use drawings. Proceedings of the SIGCHI Conference on Human Factors in Computing Systems"},{"key":"e_1_3_3_1_50_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2013.03.084"},{"key":"e_1_3_3_1_51_2","doi-asserted-by":"publisher","unstructured":"G. Johnson and Mario Verdicchio. 2023. Ethical AI is Not about AI. Commun. ACM 66 2 https:\/\/doi.org\/ 10.1145\/3576932","DOI":"10.1145\/3576932"},{"key":"e_1_3_3_1_52_2","doi-asserted-by":"publisher","DOI":"10.1145\/3652592"}],"event":{"name":"CHI EA '25: Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI EA '25","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the Extended Abstracts of the CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3719914","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3706599.3719914","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,19]],"date-time":"2025-06-19T01:18:35Z","timestamp":1750295915000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3706599.3719914"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,25]]},"references-count":52,"alternative-id":["10.1145\/3706599.3719914","10.1145\/3706599"],"URL":"https:\/\/doi.org\/10.1145\/3706599.3719914","relation":{},"subject":[],"published":{"date-parts":[[2025,4,25]]},"assertion":[{"value":"2025-04-25","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}