{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,3,27]],"date-time":"2025-03-27T13:57:07Z","timestamp":1743083827097,"version":"3.40.3"},"publisher-location":"Cham","reference-count":26,"publisher":"Springer International Publishing","isbn-type":[{"type":"print","value":"9783030659646"},{"type":"electronic","value":"9783030659653"}],"license":[{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,1,1]],"date-time":"2020-01-01T00:00:00Z","timestamp":1577836800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.1007\/978-3-030-65965-3_24","type":"book-chapter","created":{"date-parts":[[2021,2,1]],"date-time":"2021-02-01T19:48:52Z","timestamp":1612208932000},"page":"363-373","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2,"title":["Assessing the Difficulty of Labelling an Instance in Crowdworking"],"prefix":"10.1007","author":[{"given":"Neetha","family":"Jambigi","sequence":"first","affiliation":[]},{"given":"Tirtha","family":"Chanda","sequence":"additional","affiliation":[]},{"given":"Vishnu","family":"Unnikrishnan","sequence":"additional","affiliation":[]},{"given":"Myra","family":"Spiliopoulou","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2021,2,2]]},"reference":[{"key":"24_CR1","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1016\/j.is.2015.04.007","volume":"53","author":"S Aghabozorgi","year":"2015","unstructured":"Aghabozorgi, S., Shirkhorshidi, A.S., Wah, T.Y.: Time-series clustering-a decade review. Inf. Syst. 53, 16\u201338 (2015)","journal-title":"Inf. Syst."},{"key":"24_CR2","doi-asserted-by":"crossref","unstructured":"Agirre, E., et al.: Semeval-2015 task 2: semantic textual similarity, English, Spanish and pilot on interpretability. In: Proceedings of the 9th International Workshop on Semantic Evaluation, pp. 252\u2013263 (2015)","DOI":"10.18653\/v1\/S15-2045"},{"issue":"2","key":"24_CR3","doi-asserted-by":"publisher","first-page":"76","DOI":"10.1109\/MIC.2013.20","volume":"17","author":"M Allahbakhsh","year":"2013","unstructured":"Allahbakhsh, M., Benatallah, B., Ignjatovic, A., Motahari-Nezhad, H.R., Bertino, E., Dustdar, S.: Quality control in crowdsourcing systems: issues and directions. IEEE Internet Comput. 17(2), 76\u201381 (2013)","journal-title":"IEEE Internet Comput."},{"issue":"360","key":"24_CR4","first-page":"312","volume":"1418","author":"L Anthony","year":"2011","unstructured":"Anthony, L., Carrington, P., Chu, P., Kidd, C., Lai, J., Sears, A.: Gesture dynamics: features sensitive to task difficulty and correlated with physiological sensors. Stress 1418(360), 312\u2013316 (2011)","journal-title":"Stress"},{"issue":"3","key":"24_CR5","doi-asserted-by":"publisher","first-page":"606","DOI":"10.1007\/s10618-016-0483-9","volume":"31","author":"A Bagnall","year":"2016","unstructured":"Bagnall, A., Lines, J., Bostrom, A., Large, J., Keogh, E.: The great time series classification bake off: a review and experimental evaluation of recent algorithmic advances. Data Mining Knowl. Disc. 31(3), 606\u2013660 (2016). https:\/\/doi.org\/10.1007\/s10618-016-0483-9","journal-title":"Data Mining Knowl. Disc."},{"key":"24_CR6","doi-asserted-by":"publisher","first-page":"377","DOI":"10.1007\/978-1-4614-8806-4_30","volume-title":"Handbook of Human Computation","author":"J Chandler","year":"2013","unstructured":"Chandler, J., Paolacci, G., Mueller, P.: Risks and rewards of crowdsourcing marketplaces. In: Michelucci, P. (ed.) Handbook of Human Computation, pp. 377\u2013392. Springer, New York (2013). https:\/\/doi.org\/10.1007\/978-1-4614-8806-4_30"},{"key":"24_CR7","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1016\/j.neucom.2018.03.067","volume":"307","author":"M Christ","year":"2018","unstructured":"Christ, M., Braun, N., Neuffer, J., Kempa-Liehr, A.W.: Time series feature extraction on basis of scalable hypothesis tests (tsfresh-a python package). Neurocomputing 307, 72\u201377 (2018)","journal-title":"Neurocomputing"},{"key":"24_CR8","first-page":"666","volume":"78","author":"H Critchley","year":"2013","unstructured":"Critchley, H., Nagai, Y.: Electrodermal activity (EDA). Encycl. Behav. Med. 78, 666\u2013669 (2013)","journal-title":"Encycl. Behav. Med."},{"key":"24_CR9","unstructured":"Dua, D., Graff, C.: UCI machine learning repository (2017). http:\/\/archive.ics.uci.edu\/ml"},{"key":"24_CR10","unstructured":"D\u00f6bler, A., Moczalla, G.: Design and evaluation of similarity assessment configurations with help of crowdsourcing and active learning. Master Thesis OVGU (2018)"},{"issue":"2","key":"24_CR11","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1007\/s10791-011-9181-9","volume":"16","author":"C Eickhoff","year":"2013","unstructured":"Eickhoff, C., de Vries, A.P.: Increasing cheat robustness of crowdsourcing tasks. Inf. Retriev. 16(2), 121\u2013137 (2013)","journal-title":"Inf. Retriev."},{"key":"24_CR12","unstructured":"Gadiraju, U.: Its Getting Crowded!. Improving the Effectiveness of Microtask Crowdsourcing, Gesellschaft f\u00fcr Informatik eV (2018)"},{"issue":"2","key":"24_CR13","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1109\/TITS.2005.848368","volume":"6","author":"JA Healey","year":"2005","unstructured":"Healey, J.A., Picard, R.W.: Detecting stress during real-world driving tasks using physiological sensors. IEEE Trans. Intell. Transp. Syst. 6(2), 156\u2013166 (2005)","journal-title":"IEEE Trans. Intell. Transp. Syst."},{"key":"24_CR14","doi-asserted-by":"crossref","unstructured":"Ipeirotis, P.G., Provost, F., Wang, J.: Quality management on amazon mechanical turk. In: Proceedings of the ACM SIGKDD Workshop on Human Computation, pp. 64\u201367 (2010)","DOI":"10.1145\/1837885.1837906"},{"issue":"1","key":"24_CR15","first-page":"3233","volume":"18","author":"S Jagabathula","year":"2017","unstructured":"Jagabathula, S., Subramanian, L., Venkataraman, A.: Identifying unreliable and adversarial workers in crowdsourced labeling tasks. J. Mach. Learn. Res. 18(1), 3233\u20133299 (2017)","journal-title":"J. Mach. Learn. Res."},{"issue":"3","key":"24_CR16","doi-asserted-by":"publisher","first-page":"794","DOI":"10.1109\/TKDE.2014.2327026","volume":"27","author":"A Kurve","year":"2014","unstructured":"Kurve, A., Miller, D.J., Kesidis, G.: Multicategory crowdsourcing accounting for variable task difficulty, worker skill, and worker intention. IEEE Trans. Knowl. Data Eng. 27(3), 794\u2013809 (2014)","journal-title":"IEEE Trans. Knowl. Data Eng."},{"issue":"2","key":"24_CR17","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1007\/s10462-014-9423-5","volume":"44","author":"N Luz","year":"2014","unstructured":"Luz, N., Silva, N., Novais, P.: A survey of task-oriented crowdsourcing. Artif. Intell. Rev. 44(2), 187\u2013213 (2014). https:\/\/doi.org\/10.1007\/s10462-014-9423-5","journal-title":"Artif. Intell. Rev."},{"key":"24_CR18","doi-asserted-by":"crossref","unstructured":"Paparrizos, J., Gravano, L.: k-shape: efficient and accurate clustering of time series. In: Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data, pp. 1855\u20131870 (2015)","DOI":"10.1145\/2723372.2737793"},{"key":"24_CR19","doi-asserted-by":"crossref","unstructured":"R\u00e4biger, S., Gezici, G., Sayg\u0131n, Y., Spiliopoulou, M.: Predicting worker disagreement for more effective crowd labeling. In: 2018 IEEE 5th International Conference on Data Science and Advanced Analytics (DSAA), pp. 179\u2013188. IEEE (2018)","DOI":"10.1109\/DSAA.2018.00028"},{"key":"24_CR20","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1016\/j.ins.2018.05.036","volume":"457","author":"S R\u00e4biger","year":"2018","unstructured":"R\u00e4biger, S., Spiliopoulou, M., Sayg\u0131n, Y.: How do annotators label short texts? Toward understanding the temporal dynamics of tweet labeling. Inf. Sci. 457, 29\u201347 (2018)","journal-title":"Inf. Sci."},{"key":"24_CR21","unstructured":"Sch\u00e4fer, P.: Bag-of-SFA-symbols in vector space (boss vs) (2015)"},{"key":"24_CR22","doi-asserted-by":"crossref","unstructured":"Sch\u00e4fer, P., Leser, U.: Fast and accurate time series classification with weasel. In: Proceedings of the 2017 ACM on Conference on Information and Knowledge Management, pp. 637\u2013646. ACM (2017)","DOI":"10.1145\/3132847.3132980"},{"key":"24_CR23","doi-asserted-by":"crossref","unstructured":"Sheng, V.S., Provost, F., Ipeirotis, P.G.: Get another label? Improving data quality and data mining using multiple, noisy labelers. In: Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, pp. 614\u2013622 (2008)","DOI":"10.1145\/1401890.1401965"},{"key":"24_CR24","doi-asserted-by":"crossref","unstructured":"Snow, R., O\u2019connor, B., Jurafsky, D., Ng, A.Y.: Cheap and fast-but is it good? Evaluating non-expert annotations for natural language tasks. In: Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing, pp. 254\u2013263 (2008)","DOI":"10.3115\/1613715.1613751"},{"issue":"1","key":"24_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s41060-019-00177-1","volume":"9","author":"V Unnikrishnan","year":"2019","unstructured":"Unnikrishnan, V., et al.: Entity-level stream classification: exploiting entity similarity to label the future observations referring to an entity. Int. J. Data Sci. Anal. 9(1), 1\u201315 (2019). https:\/\/doi.org\/10.1007\/s41060-019-00177-1","journal-title":"Int. J. Data Sci. Anal."},{"key":"24_CR26","unstructured":"Welinder, P., Branson, S., Perona, P., Belongie, S.J.: The multidimensional wisdom of crowds. In: Advances in Neural Information Processing Systems, pp. 2424\u20132432 (2010)"}],"container-title":["Communications in Computer and Information Science","ECML PKDD 2020 Workshops"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-030-65965-3_24","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,4,24]],"date-time":"2021-04-24T23:23:20Z","timestamp":1619306600000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-030-65965-3_24"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"ISBN":["9783030659646","9783030659653"],"references-count":26,"URL":"https:\/\/doi.org\/10.1007\/978-3-030-65965-3_24","relation":{},"ISSN":["1865-0929","1865-0937"],"issn-type":[{"type":"print","value":"1865-0929"},{"type":"electronic","value":"1865-0937"}],"subject":[],"published":{"date-parts":[[2020]]},"assertion":[{"value":"2 February 2021","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Ghent","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Belgium","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2020","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"14 September 2020","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2020","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2020","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/ecmlpkdd2020.net\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"EasyChair","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"945","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"195","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"21% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4,5","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4,4","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"No","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"The conference took place virtually due to the COVID-19 pandemic","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}