{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:06:27Z","timestamp":1755839187328,"version":"3.41.0"},"publisher-location":"New York, NY, USA","reference-count":58,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,8,25]],"date-time":"2021-08-25T00:00:00Z","timestamp":1629849600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2021,8,25]]},"DOI":"10.1145\/3465336.3475109","type":"proceedings-article","created":{"date-parts":[[2021,8,25]],"date-time":"2021-08-25T21:11:41Z","timestamp":1629925901000},"page":"165-175","source":"Crossref","is-referenced-by-count":6,"title":["What Is Unclear? Computational Assessment of Task Clarity in Crowdsourcing"],"prefix":"10.1145","author":[{"given":"Zahra","family":"Nouri","sequence":"first","affiliation":[{"name":"Paderborn University, Paderborn, Germany"}]},{"given":"Ujwal","family":"Gadiraju","sequence":"additional","affiliation":[{"name":"Delft University of Technology, Delft, Netherlands"}]},{"given":"Gregor","family":"Engels","sequence":"additional","affiliation":[{"name":"Paderborn University, Paderborn, Germany"}]},{"given":"Henning","family":"Wachsmuth","sequence":"additional","affiliation":[{"name":"Paderborn University, Paderborn, Germany"}]}],"member":"320","published-online":{"date-parts":[[2021,8,29]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"[n.d.]. Natural Language Toolkit official website. https:\/\/www.nltk.org. Accessed: 2021-04--30.  [n.d.]. Natural Language Toolkit official website. https:\/\/www.nltk.org. Accessed: 2021-04--30."},{"key":"e_1_3_2_2_2_1","unstructured":"[n.d.]. An open source tool for predictive data analysis. https:\/\/scikit-learn.org\/stable\/. Accessed: 2021-05--14.  [n.d.]. An open source tool for predictive data analysis. https:\/\/scikit-learn.org\/stable\/. Accessed: 2021-05--14."},{"key":"e_1_3_2_2_3_1","unstructured":"[n.d.]. Pretrained Bert models: A list of pretrained BERT models with a short presentation. https:\/\/huggingface.co\/transformers\/pretrained_models.html. Accessed: 2021-05--11.  [n.d.]. Pretrained Bert models: A list of pretrained BERT models with a short presentation. https:\/\/huggingface.co\/transformers\/pretrained_models.html. Accessed: 2021-05--11."},{"key":"e_1_3_2_2_4_1","unstructured":"[n.d.]. Pypi official website Readability project description. https:\/\/pypi.org\/project\/readability\/. Accessed: 2021-04--30.  [n.d.]. Pypi official website Readability project description. https:\/\/pypi.org\/project\/readability\/. Accessed: 2021-04--30."},{"key":"e_1_3_2_2_5_1","unstructured":"[n.d.]. PyTorch Pretrained BERT: The Big and Extending Repository of pretrained Transformers. https:\/\/pypi.org\/project\/pytorch-pretrained-bert\/. Accessed: 2021-05--14.  [n.d.]. PyTorch Pretrained BERT: The Big and Extending Repository of pretrained Transformers. https:\/\/pypi.org\/project\/pytorch-pretrained-bert\/. Accessed: 2021-05--14."},{"key":"e_1_3_2_2_6_1","unstructured":"[n.d.]. SpaCy official website Named Entity Recognition documentation. https:\/\/spacy.io\/models\/en#en_core_web_lg. Accessed: 2021-04--30.  [n.d.]. SpaCy official website Named Entity Recognition documentation. https:\/\/spacy.io\/models\/en#en_core_web_lg. Accessed: 2021-04--30."},{"key":"e_1_3_2_2_7_1","unstructured":"[n.d.]. The Stanford Natural Language Processing Group official website Stanford Log-linear Part-Of-Speech Tagger. https:\/\/nlp.stanford.edu\/software\/tagger.shtml. Accessed: 2021-04--30.  [n.d.]. The Stanford Natural Language Processing Group official website Stanford Log-linear Part-Of-Speech Tagger. https:\/\/nlp.stanford.edu\/software\/tagger.shtml. Accessed: 2021-04--30."},{"key":"e_1_3_2_2_8_1","unstructured":"[n.d.]. TextBlob official website: Simplified Text Processing. https:\/\/textblob.readthedocs.io\/en\/dev\/. Accessed: 2021-04--30.  [n.d.]. TextBlob official website: Simplified Text Processing. https:\/\/textblob.readthedocs.io\/en\/dev\/. Accessed: 2021-04--30."},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.5555\/1996889.1996910"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866078"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3242587.3242598"},{"volume-title":"Readability revisited: The new Dale-Chall readability formula","author":"Chall Jeanne Sternlicht","key":"e_1_3_2_2_12_1"},{"volume-title":"Risks and Rewards of Crowdsourcing Marketplaces","author":"Chandler Jesse","key":"e_1_3_2_2_13_1","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4614-8806-4_30"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3026044"},{"key":"e_1_3_2_2_15_1","first-page":"1","article-title":"Opportunities for crowdsourcing research on amazon mechanical turk","volume":"5","author":"Chen Jenny J","year":"2011","journal-title":"Interfaces"},{"volume-title":"Proceedings of the ACM on Human- Computer Interaction 2, CSCW","year":"2018","author":"Chiang Chun-Wei","key":"e_1_3_2_2_16_1"},{"volume-title":"Proceedings of the human language technology conference of the North American chapter of the association for computational linguistics: HLT-NAACL","year":"2004","author":"Collins-Thompson Kevyn","key":"e_1_3_2_2_17_1"},{"volume-title":"Bert: Pre-training of deep bidirectional transformers for language understanding. arXiv preprint arXiv:1810.04805","year":"2018","author":"Devlin Jacob","key":"e_1_3_2_2_18_1"},{"volume-title":"Proceedings of the 24th international conference on world wide web. 238--247","year":"2015","author":"Difallah Djellel Eddine","key":"e_1_3_2_2_19_1"},{"volume-title":"13th ACM Web Science Conference","year":"2021","author":"Edixhoven Tom","key":"e_1_3_2_2_20_1"},{"volume-title":"Proceedings of the ACM on Human-Computer Interaction 4, CSCW2","year":"2020","author":"Fan Shaoyang","key":"e_1_3_2_2_21_1"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2499149.2499168"},{"volume-title":"How unclear terms affect survey data. Public Opinion Quarterly 56, 2 (01","year":"1992","author":"Fowler Floyd Jackson","key":"e_1_3_2_2_23_1"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/1989323.1989331"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3342220.3343644"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2631775.2631819"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3078714.3078715"},{"volume-title":"Companion of the 2017 ACM conference on computer supported cooperative work and social computing. 1--4.","year":"2017","author":"Gaikwad Snehalkumar","key":"e_1_3_2_2_28_1"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.5555\/1866696.1866723"},{"volume-title":"Fantasktic: Improving quality of results for novice crowdsourcing users. EECS Dept., Univ","year":"2012","author":"Gutheim Philipp","key":"e_1_3_2_2_30_1"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3336191.3371857"},{"volume-title":"Proceedings of the 2013 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 1120--1130","year":"2013","author":"Hovy Dirk","key":"e_1_3_2_2_32_1"},{"volume-title":"The rise of crowdsourcing. Wired magazine 14, 6","year":"2006","author":"Howe Jeff","key":"e_1_3_2_2_33_1"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.14778\/3007263.3007293"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1869086.1869094"},{"volume-title":"Aditya Parameswaran, and Jennifer Widom.","year":"2017","author":"Jain Ayush","key":"e_1_3_2_2_36_1"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"crossref","first-page":"97","DOI":"10.1075\/itl.165.2.01col","article-title":"Computational assessment of text readability","volume":"165","author":"Kevyn Collins-Thompson","year":"2014","journal-title":"ITL-International Journal of Applied Linguistics"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/1926180.1926195"},{"volume-title":"Richard L Rogers, and Brad S Chissom.","year":"1975","author":"Kincaid J Peter","key":"e_1_3_2_2_39_1"},{"key":"e_1_3_2_2_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357127"},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"volume-title":"Proceedings of the 24th annual ACM symposium on User interface software and technology. 43--52","year":"2011","author":"Kittur Aniket","key":"e_1_3_2_2_42_1"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2012.72"},{"volume-title":"CHI'11 extended abstracts on human factors in computing systems. 2053--2058","year":"2011","author":"Kulkarni Anand P","key":"e_1_3_2_2_44_1"},{"volume-title":"Identifying Featured Articles in Wikipedia: Writing Style Matters. In 19th International Conference on World Wide Web (WWW","year":"2010","author":"Lipka Nedim","key":"e_1_3_2_2_45_1"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1866029.1866040"},{"volume-title":"Companion Proceedings of The 2019 World Wide Web Conference. 1121--1130","author":"Chaithanya Manam VK","key":"e_1_3_2_2_47_1"},{"volume-title":"Sixth AAAI Conference on Human Computation and Crowdsourcing.","year":"2018","author":"Chaithanya Manam VK","key":"e_1_3_2_2_48_1"},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858539"},{"volume-title":"Proceedings of the 28th International Conference on Computational Linguistics. International Committee on Computational Linguistics, Barcelona, Spain (Online), 6264--6276","year":"2020","author":"Nouri Zahra","key":"e_1_3_2_2_50_1"},{"key":"e_1_3_2_2_51_1","volume-title":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","volume":"3","author":"Papoutsaki Alexandra","year":"2015"},{"volume-title":"Proceedings of the SIGCHI conference on human factors in computing systems. 1403--1412","year":"2011","author":"Quinn Alexander J","key":"e_1_3_2_2_52_1"},{"volume-title":"Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing. 1890--","year":"2017","author":"Salehi Niloufar","key":"e_1_3_2_2_53_1"},{"key":"e_1_3_2_2_54_1","unstructured":"Thimo Schulze Stefan Seedorf David Geiger Nicolas Kaufmann and Martin Schader. 2011. Exploring task properties in crowdsourcing--An empirical study on Mechanical Turk. (2011).  Thimo Schulze Stefan Seedorf David Geiger Nicolas Kaufmann and Martin Schader. 2011. Exploring task properties in crowdsourcing--An empirical study on Mechanical Turk. (2011)."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837891"},{"volume-title":"Proceedings of the 28th International Conference on Computational Linguistics. International Committee on Computational Linguistics, Barcelona, Spain (Online), 6739--6745","year":"2020","author":"Wachsmuth Henning","key":"e_1_3_2_2_56_1"},{"volume-title":"Artificial intelligence and collective intelligence. Handbook of Collective Intelligence","year":"2015","author":"Weld Daniel S","key":"e_1_3_2_2_57_1"},{"volume-title":"Fifth AAAI Conference on Human Computation and Crowdsourcing.","year":"2017","author":"Wu Meng-Han","key":"e_1_3_2_2_58_1"}],"event":{"name":"HT '21: 32nd ACM Conference on Hypertext and Social Media","sponsor":["SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGCHI ACM Special Interest Group on Computer-Human Interaction"],"location":"Virtual Event USA","acronym":"HT '21"},"container-title":["Proceedings of the 32st ACM Conference on Hypertext and Social Media"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3465336.3475109","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:12Z","timestamp":1750191432000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3465336.3475109"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,8,25]]},"references-count":58,"alternative-id":["10.1145\/3465336.3475109","10.1145\/3465336"],"URL":"https:\/\/doi.org\/10.1145\/3465336.3475109","relation":{},"subject":[],"published":{"date-parts":[[2021,8,25]]}}}