{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,16]],"date-time":"2026-07-16T05:11:16Z","timestamp":1784178676894,"version":"3.55.0"},"publisher-location":"New York, NY, USA","reference-count":41,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,8,9]],"date-time":"2023-08-09T00:00:00Z","timestamp":1691539200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"European Commission","award":["860721"],"award-info":[{"award-number":["860721"]}]},{"name":"Australian Research Council","award":["DP210104043"],"award-info":[{"award-number":["DP210104043"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,8,9]]},"DOI":"10.1145\/3578337.3605135","type":"proceedings-article","created":{"date-parts":[[2023,8,9]],"date-time":"2023-08-09T22:12:46Z","timestamp":1691619166000},"page":"125-133","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":5,"title":["Outcome-based Evaluation of Systematic Review Automation"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4420-4147","authenticated-orcid":false,"given":"Wojciech","family":"Kusa","sequence":"first","affiliation":[{"name":"TU Wien, Vienna, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0271-5563","authenticated-orcid":false,"given":"Guido","family":"Zuccon","sequence":"additional","affiliation":[{"name":"The University of Queensland, Brisbane, QLD, Australia"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1161-7359","authenticated-orcid":false,"given":"Petr","family":"Knoth","sequence":"additional","affiliation":[{"name":"The Open University, Milton Keynes, United Kingdom"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7149-5843","authenticated-orcid":false,"given":"Allan","family":"Hanbury","sequence":"additional","affiliation":[{"name":"TU Wien, Vienna, Austria"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,8,9]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1186\/S13643-019-0942--7"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.envsci.2014.05.010"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2022.3148206"},{"key":"e_1_3_2_1_4_1","volume-title":"AMIA annual symposium proceedings","author":"Cohen Aaron M","unstructured":"Aaron M Cohen, Kyle Ambert, and Marian McDonagh. 2010. A prospective evaluation of an automated classification system to support evidence-based medicine and systematic review. In AMIA annual symposium proceedings, Vol. 2010. American Medical Informatics Association, 121."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M1929"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X12457243"},{"key":"e_1_3_2_1_7_1","volume-title":"Douglas G Altman, and Cochrane Statistical Methods Group.","author":"Deeks Jonathan J","year":"2019","unstructured":"Jonathan J Deeks, Julian PT Higgins, Douglas G Altman, and Cochrane Statistical Methods Group. 2019. Analysing data and undertaking meta-analyses. Cochrane handbook for systematic reviews of interventions (2019), 241--284."},{"key":"e_1_3_2_1_8_1","volume-title":"Higgins","author":"Deeks Jonathan J.","year":"2010","unstructured":"Jonathan J. Deeks and Julian P. T. Higgins. 2010. Statistical algorithms in Review Manager 5. Statistical Methods Group of The Cochrane Collaboration, Vol. 1, 11 (2010)."},{"key":"e_1_3_2_1_9_1","volume-title":"TREC 2016 Total Recall Track Overview.. In TREC.","author":"Grossman Maura R","year":"2016","unstructured":"Maura R Grossman, Gordon V Cormack, and Adam Roegiest. 2016. TREC 2016 Total Recall Track Overview.. In TREC."},{"key":"e_1_3_2_1_10_1","volume-title":"Choosing effect measures and computing estimates of effect. Cochrane handbook for systematic reviews of interventions","author":"Higgins Julian PT","year":"2019","unstructured":"Julian PT Higgins, Tianjing Li, and Jonathan J Deeks. 2019. Choosing effect measures and computing estimates of effect. Cochrane handbook for systematic reviews of interventions (2019), 143--176."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1186\/s13643-016-0263-z"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1016\/J.ENVINT.2020.105623"},{"key":"e_1_3_2_1_13_1","volume-title":"Burch Jane, Chambers Duncan, Eastwood Alison, Fayter Debra, Hempel Susanne, Light Kate, Rice Stephen, Rithalia Amber, Stewart Lesley, Stock Christian, Wilson Paul, and Woolacott Nerys.","author":"Jo Akers","year":"2009","unstructured":"Akers Jo, Aguiar-Ib\u00e1 nez Raquel, Burch Jane, Chambers Duncan, Eastwood Alison, Fayter Debra, Hempel Susanne, Light Kate, Rice Stephen, Rithalia Amber, Stewart Lesley, Stock Christian, Wilson Paul, and Woolacott Nerys. 2009. Systematic Reviews: CRD's guidance for undertaking reviews in health care. CRD, University of York, York. www.york.ac.uk\/inst\/crd"},{"key":"e_1_3_2_1_14_1","volume-title":"CLEF 2017 technologically assisted reviews in empirical medicine overview. CEUR Workshop Proceedings","volume":"1866","author":"Kanoulas Evangelos","year":"2017","unstructured":"Evangelos Kanoulas, Dan Li, Leif Azzopardi, and Rene Spijker. 2017. CLEF 2017 technologically assisted reviews in empirical medicine overview. CEUR Workshop Proceedings, Vol. 1866 (9 2017), 1--29. https:\/\/pureportal.strath.ac.uk\/en\/publications\/clef-2017-technologically-assisted-reviews-in-empirical-medicine-"},{"key":"e_1_3_2_1_15_1","volume-title":"CLEF 2018 technologically assisted reviews in empirical medicine overview. CEUR Workshop Proceedings","volume":"2125","author":"Kanoulas Evangelos","year":"2018","unstructured":"Evangelos Kanoulas, Dan Li, Leif Azzopardi, and Rene Spijker. 2018. CLEF 2018 technologically assisted reviews in empirical medicine overview. CEUR Workshop Proceedings, Vol. 2125 (7 2018). https:\/\/pureportal.strath.ac.uk\/en\/publications\/clef-2018-technologically-assisted-reviews-in-empirical-medicine-"},{"key":"e_1_3_2_1_16_1","volume-title":"CLEF 2019 Technology Assisted Reviews in Empirical Medicine Overview. In CLEF.","author":"Kanoulas E.","year":"2019","unstructured":"E. Kanoulas, Dan Li, Leif Azzopardi, and Ren\u00e9 Spijker. 2019. CLEF 2019 Technology Assisted Reviews in Empirical Medicine Overview. In CLEF."},{"key":"e_1_3_2_1_17_1","unstructured":"Staffs Keele et al. 2007. Guidelines for performing systematic literature reviews in software engineering. Technical Report. Technical report ver. 2.3 ebse technical report. ebse."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswax.2020.100030"},{"key":"e_1_3_2_1_19_1","volume-title":"Advances in Information Retrieval, Matthias Hagen, Suzan Verberne, Craig Macdonald, Christin Seifert, Krisztian Balog, Kjetil N\u00f8rv\u00e5g","author":"Kusa Wojciech","unstructured":"Wojciech Kusa, Allan Hanbury, and Petr Knoth. 2022. Automation of Citation Screening for Systematic Literature Reviews Using Neural Networks: A Replicability Study. In Advances in Information Retrieval, Matthias Hagen, Suzan Verberne, Craig Macdonald, Christin Seifert, Krisztian Balog, Kjetil N\u00f8rv\u00e5g, and Vinay Setty (Eds.). Springer International Publishing, Cham, 584--598. https:\/\/arxiv.org\/abs\/2201.07534v1"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.iswa.2023.200193"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3539618.3591802"},{"key":"e_1_3_2_1_22_1","volume-title":"Visualizing the Pareto Frontier. Multiobjective optimization","author":"Lotov Alexander V","year":"2008","unstructured":"Alexander V Lotov and Kaisa Miettinen. 2008. Visualizing the Pareto Frontier. Multiobjective optimization, Vol. 5252 (2008), 213--243."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835551"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1136\/JAMIA.2010.004325"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2019.10.011"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jclinepi.2018.05.022"},{"key":"e_1_3_2_1_27_1","volume-title":"Using text mining for study identification in systematic reviews: a systematic review of current approaches. Systematic reviews","author":"O'Mara-Eves Alison","year":"2015","unstructured":"Alison O'Mara-Eves, James Thomas, John McNaught, Makoto Miwa, and Sophia Ananiadou. 2015. Using text mining for study identification in systematic reviews: a systematic review of current approaches. Systematic reviews, Vol. 4, 1 (2015), 1--22."},{"key":"e_1_3_2_1_28_1","volume-title":"Systematic reviews in the social sciences: A practical guide","author":"Petticrew Mark","unstructured":"Mark Petticrew and Helen Roberts. 2008. Systematic reviews in the social sciences: A practical guide. John Wiley & Sons."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-45439-5_27"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-020-09381-1"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380185"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080707"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1186\/S13643-018-0724--7"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2021.106589"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835804.1835829"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1137\/1.9781611972818.16"},{"key":"e_1_3_2_1_37_1","volume-title":"Semi-automated screening of biomedical citations for systematic reviews. BMC bioinformatics","author":"Wallace Byron C","year":"2010","unstructured":"Byron C Wallace, Thomas A Trikalinos, Joseph Lau, Carla Brodley, and Christopher H Schmid. 2010b. Semi-automated screening of biomedical citations for systematic reviews. BMC bioinformatics, Vol. 11, 1 (2010), 1--11."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.iswa.2022.200141"},{"key":"e_1_3_2_1_39_1","volume-title":"Can ChatGPT write a good boolean query for systematic review literature search? arXiv preprint arXiv:2302.03495","author":"Wang Shuai","year":"2023","unstructured":"Shuai Wang, Harrisen Scells, Bevan Koopman, and Guido Zuccon. 2023a. Can ChatGPT write a good boolean query for systematic review literature search? arXiv preprint arXiv:2302.03495 (2023)."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3572960.3572980"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99736-6_34"}],"event":{"name":"ICTIR '23: The 2023 ACM SIGIR International Conference on the Theory of Information Retrieval","location":"Taipei Taiwan","acronym":"ICTIR '23","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3578337.3605135","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3578337.3605135","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:47:05Z","timestamp":1750178825000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3578337.3605135"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,8,9]]},"references-count":41,"alternative-id":["10.1145\/3578337.3605135","10.1145\/3578337"],"URL":"https:\/\/doi.org\/10.1145\/3578337.3605135","relation":{},"subject":[],"published":{"date-parts":[[2023,8,9]]},"assertion":[{"value":"2023-08-09","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}