{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,15]],"date-time":"2026-07-15T05:22:30Z","timestamp":1784092950235,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":51,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,8,7]],"date-time":"2017-08-07T00:00:00Z","timestamp":1502064000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"name":"Tsinghua University Initiative Scientific Research Program","award":["2014Z21032"],"award-info":[{"award-number":["2014Z21032"]}]},{"name":"National Key Basic Research Program","award":["2015CB358700"],"award-info":[{"award-number":["2015CB358700"]}]},{"name":"Natural Science Foundation of China","award":["61622208, 61532011, 61672311"],"award-info":[{"award-number":["61622208, 61532011, 61672311"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2017,8,7]]},"DOI":"10.1145\/3077136.3080804","type":"proceedings-article","created":{"date-parts":[[2017,7,28]],"date-time":"2017-07-28T19:35:01Z","timestamp":1501270501000},"page":"15-24","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":46,"title":["Meta-evaluation of Online and Offline Web Search Evaluation Metrics"],"prefix":"10.1145","author":[{"given":"Ye","family":"Chen","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ke","family":"Zhou","sequence":"additional","affiliation":[{"name":"University of Nottingham, Nottingham, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yiqun","family":"Liu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Min","family":"Zhang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shaoping","family":"Ma","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2017,8,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2009965"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"crossref","unstructured":"Azzah Al-Maskari Mark Sanderson and Paul Clough. 2007. The relationship between IR effectiveness measures and user satisfaction SIGIR'07. ACM 773--774. Azzah Al-Maskari Mark Sanderson and Paul Clough. 2007. The relationship between IR effectiveness measures and user satisfaction SIGIR'07. ACM 773--774.","DOI":"10.1145\/1277741.1277902"},{"key":"e_1_3_2_1_3_1","unstructured":"Lorin W. Anderson David R. Krathwohl and Benjamin Samuel Bloom. 2001. A taxonomy for learning teaching and assessing: A revision of Bloom's taxonomy of educational objectives. Allyn & Bacon. Lorin W. Anderson David R. Krathwohl and Benjamin Samuel Bloom. 2001. A taxonomy for learning teaching and assessing: A revision of Bloom's taxonomy of educational objectives. Allyn & Bacon."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-00296-0_5"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/792550.792552"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345543"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"Stefan B\u00fcttcher Charles L. A. Clarke Peter C. K. Yeung and Ian Soboroff. 2007. Reliable information retrieval evaluation with incomplete and biased judgements SIGIR'07. ACM 63--70. Stefan B\u00fcttcher Charles L. A. Clarke Peter C. K. Yeung and Ian Soboroff. 2007. Reliable information retrieval evaluation with incomplete and biased judgements SIGIR'07. ACM 63--70.","DOI":"10.1145\/1277741.1277755"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835675"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806473"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484071"},{"key":"e_1_3_2_1_12_1","unstructured":"Cyril Cleverdon Jack Mills and Michael Keen. 1966. FACTORS DETERMINING THE PERFORMANCE OF INDEXING SYSTEMS VOLUME 1. DESIGN. (1966). Cyril Cleverdon Jack Mills and Michael Keen. 1966. FACTORS DETERMINING THE PERFORMANCE OF INDEXING SYSTEMS VOLUME 1. DESIGN. (1966)."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939700"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.2307\/591118"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1059981.1059982"},{"key":"e_1_3_2_1_16_1","volume-title":"ACM","author":"Guo Qi","year":"2012"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505682"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/2063576.2063599"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000051"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277839"},{"key":"e_1_3_2_1_21_1","volume-title":"Xiaolin Shi, and Ryen W. White.","author":"Jiang Jiepu","year":"2015"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609633"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/775047.775067"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010056"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_26_1","volume-title":"Predicting User Satisfaction with Intelligent Assistants SIGIR'16","author":"Kiseleva Julia","year":"2016"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10618-008-0114-1"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"crossref","unstructured":"Dmitry Lagun Chih-Hung Hsieh Dale Webster and Vidhya Navalpakkam. 2014. Towards better measurement of attention and satisfaction in mobile search SIGIR'14. ACM 113--122. Dmitry Lagun Chih-Hung Hsieh Dale Webster and Vidhya Navalpakkam. 2014. Towards better measurement of attention and satisfaction in mobile search SIGIR'14. ACM 113--122.","DOI":"10.1145\/2600428.2609631"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1571951"},{"key":"e_1_3_2_1_30_1","volume-title":"Jin Young Kim, and Imed Zitouni","author":"Li Lihong","year":"2015"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767721"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"crossref","unstructured":"Zeyang Liu Yiqun Liu Ke Zhou Min Zhang and Shaoping Ma. 2015. Influence of vertical result in web search examination SIGIR'15. ACM 193--202. Zeyang Liu Yiqun Liu Ke Zhou Min Zhang and Shaoping Ma. 2015. Influence of vertical result in web search examination SIGIR'15. ACM 193--202.","DOI":"10.1145\/2766462.2767714"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-016-9282-6"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911507"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"crossref","unstructured":"Vidhya Navalpakkam LaDawn Jentzsch Rory Sayres Sujith Ravi Amr Ahmed and Alex Smola. 2013. Measurement and modeling of eye-mouse behavior in the presence of nonlinear page layouts WWW'13. ACM 953--964. Vidhya Navalpakkam LaDawn Jentzsch Rory Sayres Sujith Ravi Amr Ahmed and Alex Smola. 2013. Measurement and modeling of eye-mouse behavior in the presence of nonlinear page layouts WWW'13. ACM 953--964.","DOI":"10.1145\/2488388.2488471"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"crossref","unstructured":"Filip Radlinski and Nick Craswell. 2010. Comparing the sensitivity of information retrieval metrics SIGIR'10. ACM 667--674. Filip Radlinski and Nick Craswell. 2010. Comparing the sensitivity of information retrieval metrics SIGIR'10. ACM 667--674.","DOI":"10.1145\/1835449.1835560"},{"key":"e_1_3_2_1_38_1","volume-title":"Score estimation, incomplete judgments, and significance testing in IR evaluation AIRS'10","author":"Ravana Sri Devi"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"crossref","unstructured":"Tetsuya Sakai. 2006. Evaluating evaluation metrics based on the bootstrap SIGIR'06. ACM 525--532. Tetsuya Sakai. 2006. Evaluating evaluation metrics based on the bootstrap SIGIR'06. ACM 525--532.","DOI":"10.1145\/1148170.1148261"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_2"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9059-7"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"crossref","volume-title":"Test collection based evaluation of information retrieval systems","author":"Sanderson Mark","DOI":"10.1561\/1500000009"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"crossref","unstructured":"Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting search satisfaction metrics with interleaved comparisons SIGIR'15. ACM 463--472. Anne Schuth Katja Hofmann and Filip Radlinski. 2015. Predicting search satisfaction metrics with interleaved comparisons SIGIR'15. ACM 463--472.","DOI":"10.1145\/2766462.2767695"},{"key":"e_1_3_2_1_44_1","volume-title":"Proceedings of the international communication of association for computing machinery conference.","author":"Sch\u00fctze Hinrich","year":"2008"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(92)90007-M"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10334"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"crossref","unstructured":"Hongning Wang Yang Song Ming-Wei Chang Xiaodong He Ahmed Hassan and Ryen W White. 2014. Modeling action-level satisfaction for search task satisfaction prediction SIGIR'14. ACM 123--132. Hongning Wang Yang Song Ming-Wei Chang Xiaodong He Ahmed Hassan and Ryen W White. 2014. Modeling action-level satisfaction for search task satisfaction prediction SIGIR'14. ACM 123--132.","DOI":"10.1145\/2600428.2609607"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/1852102.1852106"},{"key":"e_1_3_2_1_49_1","volume-title":"Aslam","author":"Yilmaz Emine","year":"2008"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-009-9116-x"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911500"}],"event":{"name":"SIGIR '17: The 40th International ACM SIGIR conference on research and development in Information Retrieval","location":"Shinjuku Tokyo Japan","acronym":"SIGIR '17","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080804","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3077136.3080804","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T18:41:55Z","timestamp":1750790515000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3077136.3080804"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,8,7]]},"references-count":51,"alternative-id":["10.1145\/3077136.3080804","10.1145\/3077136"],"URL":"https:\/\/doi.org\/10.1145\/3077136.3080804","relation":{},"subject":[],"published":{"date-parts":[[2017,8,7]]},"assertion":[{"value":"2017-08-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}