{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,2]],"date-time":"2026-06-02T21:15:30Z","timestamp":1780434930914,"version":"3.54.1"},"publisher-location":"Republic and Canton of Geneva, Switzerland","reference-count":33,"publisher":"International World Wide Web Conferences Steering Committee","license":[{"start":{"date-parts":[[2015,5,18]],"date-time":"2015-05-18T00:00:00Z","timestamp":1431907200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2015,5,18]]},"DOI":"10.1145\/2736277.2741669","type":"proceedings-article","created":{"date-parts":[[2016,2,5]],"date-time":"2016-02-05T20:37:12Z","timestamp":1454704632000},"page":"506-516","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":101,"title":["Automatic Online Evaluation of Intelligent Assistants"],"prefix":"10.1145","author":[{"given":"Jiepu","family":"Jiang","sequence":"first","affiliation":[{"name":"University of Massachusetts Amherst, Amherst, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ahmed","family":"Hassan Awadallah","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rosie","family":"Jones","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Umut","family":"Ozertem","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Imed","family":"Zitouni","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ranjitha","family":"Gurunath Kulkarni","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Omar Zia","family":"Khan","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2015,5,18]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009965"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","DOI":"10.1093\/oso\/9780195091878.001.0001","volume-title":"Spoken Natural Language Dialog Systems: A Practical Approach","author":"Smith R.W.","year":"1995","unstructured":"Smith R.W. and Hipp , D.R . ( 1995 ). Spoken Natural Language Dialog Systems: A Practical Approach . Oxford University Press . Smith R.W. and Hipp, D.R. (1995). Spoken Natural Language Dialog Systems: A Practical Approach. Oxford University Press."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835458"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1059981.1059982"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1214\/aos\/1016218223"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2348283.2348323"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1718487.1718515"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505682"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063599"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556221"},{"key":"e_1_3_2_1_11_1","volume-title":"Proceedings of the First Workshop on Speech, Language and Audio in Multimedia, 96--101","author":"Heck L.P.","year":"2013","unstructured":"Heck , L.P. , Hakkani-T\u00fcr , D. , Chinthakunta , M. , T\u00fcr , G. , Iyer , R. , Parthasarathy , P. , Stifelman , L. , Shriberg , E. and Fidler , A . ( 2013 ). Multi-modal conversational search and browse . Proceedings of the First Workshop on Speech, Language and Audio in Multimedia, 96--101 . Heck, L.P., Hakkani-T\u00fcr, D., Chinthakunta, M., T\u00fcr, G., Iyer, R., Parthasarathy, P., Stifelman, L., Shriberg, E. and Fidler, A. (2013). Multi-modal conversational search and browse. Proceedings of the First Workshop on Speech, Language and Audio in Multimedia, 96--101."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICASSP.2013.6639103"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277839"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.5555\/1793274.1793280"},{"key":"e_1_3_2_1_16_1","volume-title":"Proc. HCIR","author":"Jeng W.","year":"2013","unstructured":"Jeng , W. , Jiang , J. and He , D . ( 2013 ). Users' Perceived Difficulties and Corresponding Reformulation Strategies in Voice Search . Proc. HCIR 2013. Jeng, W., Jiang, J. and He, D. (2013). Users' Perceived Difficulties and Corresponding Reformulation Strategies in Voice Search. Proc. HCIR 2013."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685319"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609633"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484092"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073146"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609468"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556220"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009922"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2013.09.002"},{"issue":"12","key":"e_1_3_2_1_25_1","first-page":"39","article-title":"Hanging on the Metaphone","volume":"7","author":"Philips L.","year":"1990","unstructured":"Philips , L. ( 1990 ). Hanging on the Metaphone . Computer Language , 7 ( 12 ), 39 -- 44 . Philips, L. (1990). Hanging on the Metaphone. Computer Language, 7(12), 39--44.","journal-title":"Computer Language"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609497"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1162\/089120100561737"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1093\/jos\/17.1.7"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"crossref","DOI":"10.1002\/9781119992691","volume-title":"Spoken language understanding: Systems for extracting semantic information from speech","author":"Tur G.","year":"2011","unstructured":"Tur , G. and De Mori , R. ( 2011 ). Spoken language understanding: Systems for extracting semantic information from speech . John Wiley & Sons . Tur, G. and De Mori, R. (2011). Spoken language understanding: Systems for extracting semantic information from speech. John Wiley & Sons."},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"crossref","DOI":"10.1007\/3-540-36678-4","volume-title":"SmartKom: foundations of multimodal dialogue systems","author":"Wahlster W.","year":"2006","unstructured":"Wahlster , W. ( 2006 ). SmartKom: foundations of multimodal dialogue systems . Springer . Wahlster, W. (2006). SmartKom: foundations of multimodal dialogue systems. Springer."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.3115\/976909.979652"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609607"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1109\/JPROC.2012.2225812"}],"event":{"name":"WWW '15: 24th International World Wide Web Conference","location":"Florence Italy","acronym":"WWW '15","sponsor":["IW3C2 International World Wide Web Conference Committee","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web"]},"container-title":["Proceedings of the 24th International Conference on World Wide Web"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2736277.2741669","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2736277.2741669","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T06:12:30Z","timestamp":1750227150000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2736277.2741669"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,5,18]]},"references-count":33,"alternative-id":["10.1145\/2736277.2741669","10.5555\/2736277"],"URL":"https:\/\/doi.org\/10.1145\/2736277.2741669","relation":{},"subject":[],"published":{"date-parts":[[2015,5,18]]},"assertion":[{"value":"2015-05-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}