{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,10,8]],"date-time":"2025-10-08T16:24:06Z","timestamp":1759940646733,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":16,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,7,28]],"date-time":"2013-07-28T00:00:00Z","timestamp":1374969600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,7,28]]},"DOI":"10.1145\/2484028.2484150","type":"proceedings-article","created":{"date-parts":[[2013,7,30]],"date-time":"2013-07-30T13:40:50Z","timestamp":1375191650000},"page":"765-768","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":23,"title":["Is relevance hard work?"],"prefix":"10.1145","author":[{"given":"Robert","family":"Villa","sequence":"first","affiliation":[{"name":"University of Sheffield, Sheffield, United Kingdom"}]},{"given":"Martin","family":"Halvey","sequence":"additional","affiliation":[{"name":"Glasgow Caledonian University, Glasgow, United Kingdom"}]}],"member":"320","published-online":{"date-parts":[[2013,7,28]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199709)48:9%3C810::AID-ASI6%3E3.0.CO;2-U"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(98)00025-9"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148219"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00081-2"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2011.04.005"},{"key":"e_1_3_2_1_7_1","volume-title":"SIGIR 2011 Information Retrieval for E-Discovery (SIRE) Workshop","author":"Wang J.","year":"2011","unstructured":"Wang , J. 2011 . Accuracy, agreement, speed, and perceived difficulty of users' relevance judgments for e-discovery . SIGIR 2011 Information Retrieval for E-Discovery (SIRE) Workshop , Beijing, China , July 28, 2011. Wang, J. 2011. Accuracy, agreement, speed, and perceived difficulty of users' relevance judgments for e-discovery. SIGIR 2011 Information Retrieval for E-Discovery (SIRE) Workshop, Beijing, China, July 28, 2011."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/345508.345512"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.v61:11"},{"key":"e_1_3_2_1_11_1","volume-title":"Assessing Cognitive Load on Web Search Tasks. The Ergonomics Open Journal. Bentham Open Access","author":"Gwizdka J.","year":"2009","unstructured":"Gwizdka , J. ( 2009 ). Assessing Cognitive Load on Web Search Tasks. The Ergonomics Open Journal. Bentham Open Access . Gwizdka, J. (2009). Assessing Cognitive Load on Web Search Tasks. The Ergonomics Open Journal. Bentham Open Access."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-02713-0_73"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Hart S.G. Staveland L.E. 1988. Development of a NASA-TLX (Task load index): Results of empirical and theoretical research. In: Hancock P.A. Meshkati N. (eds.) Human Mental Workload 139--183.  Hart S.G. Staveland L.E. 1988. Development of a NASA-TLX (Task load index): Results of empirical and theoretical research. In: Hancock P.A. Meshkati N. (eds.) Human Mental Workload 139--183.","DOI":"10.1016\/S0166-4115(08)62386-9"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1016\/0306-4573(92)90007-M"},{"key":"e_1_3_2_1_15_1","volume-title":"TREC 2005","author":"Allan J.","year":"2005","unstructured":"Allan , J. 2005 . HARD Track Overview , TREC 2005 Allan, J. 2005. HARD Track Overview, TREC 2005"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010134"}],"event":{"name":"SIGIR '13: The 36th International ACM SIGIR conference on research and development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Dublin Ireland","acronym":"SIGIR '13"},"container-title":["Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2484028.2484150","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2484028.2484150","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:35:47Z","timestamp":1750235747000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2484028.2484150"}},"subtitle":["evaluating the effort of making relevant assessments"],"short-title":[],"issued":{"date-parts":[[2013,7,28]]},"references-count":16,"alternative-id":["10.1145\/2484028.2484150","10.1145\/2484028"],"URL":"https:\/\/doi.org\/10.1145\/2484028.2484150","relation":{},"subject":[],"published":{"date-parts":[[2013,7,28]]},"assertion":[{"value":"2013-07-28","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}