{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,9]],"date-time":"2025-12-09T11:20:50Z","timestamp":1765279250230,"version":"3.44.0"},"publisher-location":"New York, NY, USA","reference-count":34,"publisher":"ACM","funder":[{"name":"Hereditary","award":["101137074"],"award-info":[{"award-number":["101137074"]}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2025,7,13]]},"DOI":"10.1145\/3726302.3730286","type":"proceedings-article","created":{"date-parts":[[2025,7,14]],"date-time":"2025-07-14T01:21:38Z","timestamp":1752456098000},"page":"3488-3497","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["Doctron: A Web-based Collaborative Annotation Tool for Ground Truth Creation in IR"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2284-5699","authenticated-orcid":false,"given":"Ornella","family":"Irrera","sequence":"first","affiliation":[{"name":"University of Padua, Padova, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0362-5893","authenticated-orcid":false,"given":"Stefano","family":"Marchesin","sequence":"additional","affiliation":[{"name":"University of Padua, Padova, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-8174-0082","authenticated-orcid":false,"given":"Farzad","family":"Shami","sequence":"additional","affiliation":[{"name":"University of Padua, Padova, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4970-4554","authenticated-orcid":false,"given":"Gianmaria","family":"Silvello","sequence":"additional","affiliation":[{"name":"University of Padua, Padova, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2025,7,13]]},"reference":[{"volume-title":"The Practice of Crowdsourcing","author":"Alonso O.","key":"e_1_3_2_1_1_1","unstructured":"O. Alonso. 2022. The Practice of Crowdsourcing. Springer Nature."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1007\/S10579-013-9215-6"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1093\/DATABASE\/BAU033"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1177\/0013164460020001"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-70011-8_33"},{"key":"e_1_3_2_1_6_1","volume-title":"Overview of the TREC 2019 deep learning track. CoRR","volume":"2003","author":"Craswell N.","year":"2020","unstructured":"N. Craswell, B. Mitra, E. Yilmaz, D. Campos, and E. M. Voorhees. 2020. Overview of the TREC 2019 deep learning track. CoRR, Vol. abs\/2003.07820 (2020). arXiv:2003.07820 https:\/\/arxiv.org\/abs\/2003.07820"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-79725-6_64"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","unstructured":"J. L. Fleiss J. C. Nee and J. R. Landis. 1979. Large sample variance of kappa in the case of different sets of raters. Psychological bulletin Vol. 86 5 (1979) 974. https:\/\/doi.org\/10.1177\/0013164420973080","DOI":"10.1177\/0013164420973080"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1186\/S12911-021-01706-4"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-99739-7_35"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1186\/S12859-024-05730-9"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1093\/NAR\/GKAA333"},{"key":"e_1_3_2_1_13_1","volume-title":"The INCEpTION Platform: Machine-Assisted and Knowledge-Oriented Interactive Annotation. In COLING 2018, The 27th International Conference on Computational Linguistics: System Demonstrations","author":"Klie J. C.","year":"2018","unstructured":"J. C. Klie, M. Bugert, B. Boullosa, R. E. de Castilho, and I. Gurevych. 2018. The INCEpTION Platform: Machine-Assisted and Knowledge-Oriented Interactive Annotation. In COLING 2018, The 27th International Conference on Computational Linguistics: System Demonstrations, Santa Fe, New Mexico, August 20-26, 2018. Association for Computational Linguistics, 5-9. https:\/\/aclanthology.org\/C18-2002\/"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1177\/00131644700300010"},{"key":"e_1_3_2_1_15_1","unstructured":"K. Krippendorff. 2011. Computing Krippendorff's alpha-reliability."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1093\/DATABASE\/BAU067"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1093\/NAR\/GKY428"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-04346-8_62"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3463254"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.11613\/BM.2012.031"},{"key":"e_1_3_2_1_21_1","volume-title":"Doccano: Text annotation tool for human. Software available from https:\/\/github. com\/doccano\/doccano","author":"Nakayama H.","year":"2018","unstructured":"H. Nakayama, T. Kubo, J. Kamura, Y. Taniguchi, and X. Liang. 2018. Doccano: Text annotation tool for human. Software available from https:\/\/github. com\/doccano\/doccano, Vol. 34 (2018)."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1093\/BIB\/BBZ130"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1093\/BIB\/BBS084"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2022.EMNLP-DEMOS.33"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2021.EMNLP-DEMO.3"},{"key":"e_1_3_2_1_26_1","unstructured":"J. Pustejovsky and A. Stubbs. 2012. Natural Language Annotation for Machine Learning: A guide to corpus-building for applications. '' O'Reilly Media Inc.''."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1093\/BIOINFORMATICS\/BTS435"},{"key":"e_1_3_2_1_28_1","volume-title":"Proceedings of the Eleventh International Conference on Language Resources and Evaluation, LREC 2018","author":"Shindo H.","year":"2018","unstructured":"H. Shindo, Y. Munesada, and Y. Matsumoto. 2018. PDFAnno: a Web-based Linguistic Annotation Tool for PDF Documents. In Proceedings of the Eleventh International Conference on Language Resources and Evaluation, LREC 2018, Miyazaki, Japan, May 7-12, 2018. European Language Resources Association (ELRA). http:\/\/www.lrec-conf.org\/proceedings\/lrec2018\/summaries\/680.html"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/2380921.2380942"},{"key":"e_1_3_2_1_30_1","unstructured":"M. Tkachenko M. Malyuk A. Holmanyuk and N. Liubimov. 2020. Label studio: Data labeling software. Open source software available from https:\/\/github. com\/heartexlabs\/label-studio Vol. 2022 (2020)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/1067268.1067272"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.18653\/V1\/2023.EACL-DEMO.17"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.31193\/ssap.01.9787520126595"},{"key":"e_1_3_2_1_34_1","unstructured":"S. M. Yimam I. Gurevych R. E. de Castilho and C. Biemann. 2013. WebAnno: A Flexible Web-based and Visually Supported System for Distributed Annotations. In 51st Annual Meeting of the Association for Computational Linguistics ACL 2013 Proceedings of the Conference System Demonstrations 4-9 August 2013 Sofia Bulgaria. The Association for Computer Linguistics 1-6. https:\/\/aclanthology.org\/P13-4001\/"}],"event":{"name":"SIGIR '25: The 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"],"location":"Padua Italy","acronym":"SIGIR '25"},"container-title":["Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3726302.3730286","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T10:02:14Z","timestamp":1755856934000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3726302.3730286"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,7,13]]},"references-count":34,"alternative-id":["10.1145\/3726302.3730286","10.1145\/3726302"],"URL":"https:\/\/doi.org\/10.1145\/3726302.3730286","relation":{},"subject":[],"published":{"date-parts":[[2025,7,13]]},"assertion":[{"value":"2025-07-13","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}