{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T14:13:26Z","timestamp":1778595206234,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":43,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,1,30]],"date-time":"2019-01-30T00:00:00Z","timestamp":1548806400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/501100010429","name":"Erasmus+","doi-asserted-by":"publisher","award":["60171990"],"award-info":[{"award-number":["60171990"]}],"id":[{"id":"10.13039\/501100010429","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100011105","name":"European Commission","doi-asserted-by":"publisher","award":["732328"],"award-info":[{"award-number":["732328"]}],"id":[{"id":"10.13039\/100011105","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,1,30]]},"DOI":"10.1145\/3289600.3291035","type":"proceedings-article","created":{"date-parts":[[2019,3,11]],"date-time":"2019-03-11T12:33:01Z","timestamp":1552307581000},"page":"321-329","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":42,"title":["All Those Wasted Hours"],"prefix":"10.1145","author":[{"given":"Lei","family":"Han","sequence":"first","affiliation":[{"name":"University of Queensland, St Lucia, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Kevin","family":"Roitero","sequence":"additional","affiliation":[{"name":"University of Udine, Udine, Italy"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ujwal","family":"Gadiraju","sequence":"additional","affiliation":[{"name":"L3S Research Center, Hanover, Germany"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Cristina","family":"Sarasua","sequence":"additional","affiliation":[{"name":"University of Zurich, Zurich, Switzerland"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Alessandro","family":"Checco","sequence":"additional","affiliation":[{"name":"University of Sheffield, Sheffield, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Eddy","family":"Maddalena","sequence":"additional","affiliation":[{"name":"University of Southampton, Southampton, United Kingdom"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Gianluca","family":"Demartini","sequence":"additional","affiliation":[{"name":"University of Queensland, St Lucia, Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2019,1,30]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148177"},{"key":"e_1_3_2_1_2_1","volume-title":"Qualitative research methods for the social sciences","author":"Berg Bruce Lawrence","unstructured":"Bruce Lawrence Berg , Howard Lune , and Howard Lune . 2004. Qualitative research methods for the social sciences . Vol. 5 . Pearson Boston , MA. Bruce Lawrence Berg, Howard Lune, and Howard Lune. 2004. Qualitative research methods for the social sciences. Vol. 5. Pearson Boston, MA."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207710"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1367497.1367505"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/2452376.2452451"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1177\/1354856507084420"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187900"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1561\/1800000025"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/2488388.2488421"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398399"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1979742.1979826"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159654"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1177\/0165551512437638"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130914"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2631775.2631819"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/2702123.2702443"},{"key":"e_1_3_2_1_17_1","volume-title":"Proceedings of the 6th AAAI Conference on Human Computation and Crowdsourcing (HCOMP) .","author":"Goyal Tanya","year":"2018","unstructured":"Tanya Goyal , Tyler McDonnell , Mucahid Kutlu , Tamer Elsayed , and Matthew Lease . 2018 . Your Behavior Signals Your Reliability: Modeling Crowd Behavioral Traces to Ensure Quality Relevance Annotations . In Proceedings of the 6th AAAI Conference on Human Computation and Crowdsourcing (HCOMP) . Tanya Goyal, Tyler McDonnell, Mucahid Kutlu, Tamer Elsayed, and Matthew Lease. 2018. Your Behavior Signals Your Reliability: Modeling Crowd Behavioral Traces to Ensure Quality Relevance Annotations. In Proceedings of the 6th AAAI Conference on Human Computation and Crowdsourcing (HCOMP) ."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1080\/0267257X.2017.1325389"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174023"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2505682"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/1837885.1837906"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.5555\/3122009.3176837"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2396761.2398697"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2835776.2835835"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556220"},{"key":"e_1_3_2_1_26_1","volume-title":"Content analysis: An introduction to its methodology","author":"Kirppendorff K","year":"1989","unstructured":"K Kirppendorff . 1989. Content analysis: An introduction to its methodology . Beverley Hills : Sage ( 1989 ). K Kirppendorff. 1989. Content analysis: An introduction to its methodology. Beverley Hills: Sage (1989)."},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741681"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2835776.2835833"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2568033"},{"key":"e_1_3_2_1_31_1","volume-title":"Fourth AAAI Conference on Human Computation and Crowdsourcing .","author":"Maddalena Eddy","year":"2016","unstructured":"Eddy Maddalena , Marco Basaldella , Dario De Nart , Dante Degl'Innocenti , Stefano Mizzaro , and Gianluca Demartini . 2016 . Crowdsourcing relevance assessments: The unexpected benefits of limiting the time to judge . In Fourth AAAI Conference on Human Computation and Crowdsourcing . Eddy Maddalena, Marco Basaldella, Dario De Nart, Dante Degl'Innocenti, Stefano Mizzaro, and Gianluca Demartini. 2016. Crowdsourcing relevance assessments: The unexpected benefits of limiting the time to judge. In Fourth AAAI Conference on Human Computation and Crowdsourcing ."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3002172"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121060"},{"key":"e_1_3_2_1_34_1","volume-title":"Fourth AAAI Conference on Human Computation and Crowdsourcing .","author":"McDonnell Tyler","year":"2016","unstructured":"Tyler McDonnell , Matthew Lease , Mucahid Kutlu , and Tamer Elsayed . 2016 . Why is that relevant? Collecting annotator rationales for relevance judgments . In Fourth AAAI Conference on Human Computation and Crowdsourcing . Tyler McDonnell, Matthew Lease, Mucahid Kutlu, and Tamer Elsayed. 2016. Why is that relevant? Collecting annotator rationales for relevance judgments. In Fourth AAAI Conference on Human Computation and Crowdsourcing ."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3132847.3133001"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210052"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753846.1753873"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2047196.2047199"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/564376.564433"},{"key":"e_1_3_2_1_40_1","volume-title":"Qualitative analysis for social scientists","author":"Strauss Anselm L","unstructured":"Anselm L Strauss . 1987. Qualitative analysis for social scientists . Cambridge University Press . Anselm L Strauss. 1987. Qualitative analysis for social scientists .Cambridge University Press."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/2566486.2567989"},{"key":"e_1_3_2_1_42_1","volume-title":"Overview of The Eighth Text REtrieval Conference (TREC 8), 1--24","author":"Voorhees EM","year":"1999","unstructured":"EM Voorhees and DK Harman . 1999 . Overview of The Eighth Text REtrieval Conference (TREC 8), 1--24 . NIST Special Publication (1999). EM Voorhees and DK Harman. 1999. Overview of The Eighth Text REtrieval Conference (TREC 8), 1--24. NIST Special Publication (1999)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-012-9206-z"}],"event":{"name":"WSDM '19: The Twelfth ACM International Conference on Web Search and Data Mining","location":"Melbourne VIC Australia","acronym":"WSDM '19","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGWEB ACM Special Interest Group on Hypertext, Hypermedia, and Web","SIGKDD ACM Special Interest Group on Knowledge Discovery in Data","SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3289600.3291035","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3289600.3291035","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T01:02:21Z","timestamp":1750208541000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3289600.3291035"}},"subtitle":["On Task Abandonment in Crowdsourcing"],"short-title":[],"issued":{"date-parts":[[2019,1,30]]},"references-count":43,"alternative-id":["10.1145\/3289600.3291035","10.1145\/3289600"],"URL":"https:\/\/doi.org\/10.1145\/3289600.3291035","relation":{},"subject":[],"published":{"date-parts":[[2019,1,30]]},"assertion":[{"value":"2019-01-30","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}