{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,23]],"date-time":"2025-08-23T05:24:26Z","timestamp":1755926666680,"version":"3.40.3"},"publisher-location":"Cham","reference-count":29,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030997359"},{"type":"electronic","value":"9783030997366"}],"license":[{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2022,1,1]],"date-time":"2022-01-01T00:00:00Z","timestamp":1640995200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.1007\/978-3-030-99736-6_9","type":"book-chapter","created":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T23:02:47Z","timestamp":1649113367000},"page":"128-141","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":7,"title":["WANDS: Dataset for Product Search Relevance Assessment"],"prefix":"10.1007","author":[{"given":"Yan","family":"Chen","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shujian","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zheng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Weiyi","family":"Sun","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Linas","family":"Baltrunas","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Benjamin","family":"Schroeder","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2022,4,5]]},"reference":[{"key":"9_CR1","unstructured":"Home depot product search relevance. https:\/\/www.kaggle.com\/c\/home-depot-product-search-relevance\/overview"},{"key":"9_CR2","unstructured":"Gov2 dataset. http:\/\/ir.dcs.gla.ac.uk\/test_collections\/gov2-summary.htm"},{"key":"9_CR3","unstructured":"Clueweb09 dataset. http:\/\/boston.lti.cs.cmu.edu\/Data\/clueweb09\/"},{"key":"9_CR4","unstructured":"Crowdflower search results relevance. https:\/\/www.kaggle.com\/c\/crowdflower-search-relevance\/overview"},{"key":"9_CR5","first-page":"23","volume-title":"Million Query Track 2008 Overview","author":"J Allan","year":"2008","unstructured":"Allan, J., Aslam, J., Carterette, B., Pavlu, V., Kanoulas, E.: Million Query Track 2008 Overview, p. 23. NIST Special Publication, Gaithersburg (2008)"},{"key":"9_CR6","doi-asserted-by":"crossref","unstructured":"Allan, J., Carterette, B., Dachev, B., Aslam, J.A., Pavlu, V., Kanoulas, E.: Million query track 2007 overview. In: Proceedings of the Sixteenth Text REtrieval Conference, TREC 2007 (2007)","DOI":"10.21236\/ADA477388"},{"key":"9_CR7","doi-asserted-by":"crossref","unstructured":"Buckley, C., Voorhees, E.M.: Retrieval evaluation with incomplete information. In: Proceedings of the 27th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 25\u201332 (2004)","DOI":"10.1145\/1008992.1009000"},{"key":"9_CR8","doi-asserted-by":"crossref","unstructured":"Carmel, D., Haramaty, E., Lazerson, A., Lewin-Eytan, L., Maarek, Y.: Why do people buy seemingly irrelevant items in voice product search? On the relation between product relevance and customer satisfaction in ecommerce. In: Proceedings of the 13th International Conference on Web Search and Data Mining, pp. 79\u201387 (2020)","DOI":"10.1145\/3336191.3371780"},{"key":"9_CR9","doi-asserted-by":"crossref","unstructured":"Carterette, B., Pavluy, V., Fang, H., Kanoulas, E.: Million query track 2009 overview. In: TREC, January 2009","DOI":"10.6028\/NIST.SP.500-278.million-query-overview"},{"key":"9_CR10","unstructured":"Choi, J.I., Kallumadi, S., Mitra, B., Agichtein, E., Javed, F.: Semantic product search for matching structured product catalogs in e-commerce (2020). https:\/\/arxiv.org\/pdf\/2008.08180.pdf"},{"key":"9_CR11","doi-asserted-by":"crossref","unstructured":"Cleverdon, C.: The Cranfield tests on index language devices. In: Aslib Proceedings. MCB UP Ltd. (1967)","DOI":"10.1108\/eb050097"},{"issue":"1","key":"9_CR12","doi-asserted-by":"publisher","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"J Cohen","year":"1960","unstructured":"Cohen, J.: A coefficient of agreement for nominal scales. Educ. Psychol. Measure. 20(1), 37\u201346 (1960)","journal-title":"Educ. Psychol. Measure."},{"key":"9_CR13","doi-asserted-by":"crossref","unstructured":"Deng, A., Shi, X.: Data-driven metric development for online controlled experiments: seven lessons learned. In: Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 77\u201386 (2016)","DOI":"10.1145\/2939672.2939700"},{"issue":"4","key":"9_CR14","doi-asserted-by":"publisher","first-page":"416","DOI":"10.1007\/s10791-016-9282-6","volume":"19","author":"X Lu","year":"2016","unstructured":"Lu, X., Moffat, A., Culpepper, J.S.: The effect of pooling and evaluation depth on IR metrics. Inf. Retrieval J. 19(4), 416\u2013445 (2016)","journal-title":"Inf. Retrieval J."},{"key":"9_CR15","doi-asserted-by":"crossref","unstructured":"Magnani, A., Liu, F., Xie, M., Banerjee, S.: Neural product retrieval at walmart.com. In: Companion Proceedings of the 2019 World Wide Conference (2019)","DOI":"10.1145\/3308560.3316603"},{"key":"9_CR16","doi-asserted-by":"crossref","unstructured":"Nigam, P., et al.: Semantic product search. In: Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (2019)","DOI":"10.1145\/3292500.3330759"},{"key":"9_CR17","doi-asserted-by":"publisher","unstructured":"Pitoura, E., Stefanidis, K., Koutrika, G.: Fairness in rankings and recommendations: an overview. VLDB J. 1\u201328 (2021). https:\/\/doi.org\/10.1007\/s00778-021-00697-y","DOI":"10.1007\/s00778-021-00697-y"},{"key":"9_CR18","unstructured":"Qin, T., Liu, T.: Introducing LETOR 4.0 datasets. CoRR abs\/1306.2597 (2013). http:\/\/arxiv.org\/abs\/1306.2597"},{"issue":"5","key":"9_CR19","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1007\/s10791-008-9059-7","volume":"11","author":"T Sakai","year":"2008","unstructured":"Sakai, T., Kando, N.: On information retrieval metrics designed for evaluation with incomplete relevance assessments. Inf. Retrieval 11(5), 447\u2013470 (2008)","journal-title":"Inf. Retrieval"},{"key":"9_CR20","doi-asserted-by":"crossref","unstructured":"Sanderson, M., Zobel, J.: Information retrieval system evaluation: effort, sensitivity, and reliability. In: Proceedings of the 28th ACM SIGIR Conference on Research and Development in Information Retrieval (2005)","DOI":"10.1145\/1076034.1076064"},{"key":"9_CR21","doi-asserted-by":"crossref","unstructured":"Valcarce, D., Bellog\u00edn, A., Parapar, J., Castells, P.: On the robustness and discriminative power of information retrieval metrics for top-n recommendation. In: Proceedings of the 12th ACM Conference on Recommender Systems, pp. 260\u2013268 (2018)","DOI":"10.1145\/3240323.3240347"},{"key":"9_CR22","doi-asserted-by":"crossref","unstructured":"Voorhees, E.: The sixteenth text retrieval conference (TREC 2007) (2007). https:\/\/tsapps.nist.gov\/publication\/get_pdf.cfm?pub_id=890068","DOI":"10.6028\/NIST.SP.500-274"},{"key":"9_CR23","series-title":"Lecture Notes in Computer Science","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1007\/3-540-45691-0_34","volume-title":"Evaluation of Cross-Language Information Retrieval Systems","author":"EM Voorhees","year":"2002","unstructured":"Voorhees, E.M.: The philosophy of information retrieval evaluation. In: Peters, C., Braschler, M., Gonzalo, J., Kluck, M. (eds.) CLEF 2001. LNCS, vol. 2406, pp. 355\u2013370. Springer, Heidelberg (2002). https:\/\/doi.org\/10.1007\/3-540-45691-0_34"},{"key":"9_CR24","doi-asserted-by":"crossref","unstructured":"Xia, X., et al.: SearchGCN: powering embedding retrieval by graph convolution networks for e-commerce search. In: Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval (2021)","DOI":"10.1145\/3404835.3464927"},{"key":"9_CR25","doi-asserted-by":"crossref","unstructured":"Zhang, H., et al.: Towards personalized and semantic retrieval: an end-to-end solution for e-commerce search via embedding learning. In: Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval (2020)","DOI":"10.1145\/3397271.3401446"},{"key":"9_CR26","unstructured":"Zhang, H., Wang, T., Meng, X., Hu, Y., Wang, H.: Improving semantic matching via multi-task learning in e-commerce. In: Proceedings of the 42nd International ACM SIGIR on Research and Development in Information and Retrieval, Workshop on eCommerce (2019)"},{"key":"9_CR27","doi-asserted-by":"crossref","unstructured":"Zhang, J., Liu, Y., Ma, S., Tian, Q.: Relevance estimation with multiple information sources on search engine result pages. In: Proceedings of the 2018 ACM on Conference on Information and Knowledge Management (2018)","DOI":"10.1145\/3269206.3271673"},{"key":"9_CR28","unstructured":"Zhang, J., Liu, Y., Ma, S., Tian, Q.: Que2Search: fast and accurate query and document understanding for search at Facebook. In: Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining (2021)"},{"key":"9_CR29","doi-asserted-by":"crossref","unstructured":"Zheng, Y., Fan, Z., Liu, Y., Luo, C., Zhang, M., Ma, S.: Sogou-QCL: a new dataset with click relevance label. In: Proceedings of the 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (2018)","DOI":"10.1145\/3209978.3210092"}],"container-title":["Lecture Notes in Computer Science","Advances in Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-99736-6_9","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,9,21]],"date-time":"2024-09-21T15:41:25Z","timestamp":1726933285000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-99736-6_9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"ISBN":["9783030997359","9783030997366"],"references-count":29,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-99736-6_9","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"type":"print","value":"0302-9743"},{"type":"electronic","value":"1611-3349"}],"subject":[],"published":{"date-parts":[[2022]]},"assertion":[{"value":"5 April 2022","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECIR","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"European Conference on Information Retrieval","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Stavanger","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Norway","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2022","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"10 April 2022","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 April 2022","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"44","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecir2022","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecir2022.org","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"395","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"35","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"29","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"9% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4-6","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Additionally, there are other papers: 11 reproducibility, 12 doctoral, 13 CLEF Labs, 5 workshops and 4 tutorials.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}