{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T05:27:22Z","timestamp":1782278842739,"version":"3.54.5"},"publisher-location":"New York, NY, USA","reference-count":64,"publisher":"ACM","license":[{"start":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T00:00:00Z","timestamp":1720569600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3626772.3657796","type":"proceedings-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T12:40:05Z","timestamp":1720701605000},"page":"1051-1061","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2,"title":["Evaluating Search System Explainability with Psychometrics and Crowdsourcing"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0009-8734-436X","authenticated-orcid":false,"given":"Catherine","family":"Chen","sequence":"first","affiliation":[{"name":"Brown University, Providence, RI, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9895-4061","authenticated-orcid":false,"given":"Carsten","family":"Eickhoff","sequence":"additional","affiliation":[{"name":"University of T\u00fcbingen, T\u00fcbingen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Peeking inside the black-box: a survey on explainable artificial intelligence (XAI)","author":"Adadi Amina","year":"2018","unstructured":"Amina Adadi and Mohammed Berrada. 2018. Peeking inside the black-box: a survey on explainable artificial intelligence (XAI). IEEE access , Vol. 6 (2018), 52138--52160."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.inffus.2019.12.012"},{"key":"e_1_3_2_1_3_1","volume-title":"How to determine the number of factors to retain in exploratory factor analysis: A comparison of extraction methods under realistic conditions. Psychological methods","author":"Auerswald Max","year":"2019","unstructured":"Max Auerswald and Morten Moshagen. 2019. How to determine the number of factors to retain in exploratory factor analysis: A comparison of extraction methods under realistic conditions. Psychological methods, Vol. 24, 4 (2019), 468."},{"key":"e_1_3_2_1_4_1","volume-title":"On the importance and implications of studying technology non-use. interactions","author":"Baumer Eric PS","year":"2015","unstructured":"Eric PS Baumer, Jenna Burrell, Morgan G Ames, Jed R Brubaker, and Paul Dourish. 2015. On the importance and implications of studying technology non-use. interactions, Vol. 22, 2 (2015), 52--56."},{"key":"e_1_3_2_1_5_1","volume-title":"The viability of crowdsourcing for survey research. Behavior research methods","author":"Behrend Tara S","year":"2011","unstructured":"Tara S Behrend, David J Sharek, Adam W Meade, and Eric N Wiebe. 2011. The viability of crowdsourcing for survey research. Behavior research methods, Vol. 43, 3 (2011), 800--813."},{"key":"e_1_3_2_1_6_1","volume-title":"Significance tests and goodness of fit in the analysis of covariance structures. Psychological bulletin","author":"Bentler Peter M","year":"1980","unstructured":"Peter M Bentler and Douglas G Bonett. 1980. Significance tests and goodness of fit in the analysis of covariance structures. Psychological bulletin, Vol. 88, 3 (1980), 588."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462571"},{"key":"e_1_3_2_1_8_1","unstructured":"Adrien Bibal and Beno^it Fr\u00e9nay. 2016. Interpretability of machine learning models and representations: an introduction.. In ESANN."},{"key":"e_1_3_2_1_9_1","volume-title":"Confirmatory factor analysis. Handbook of structural equation modeling","author":"Brown Timothy A","year":"2012","unstructured":"Timothy A Brown and Michael T Moore. 2012. Confirmatory factor analysis. Handbook of structural equation modeling , Vol. 361 (2012), 379."},{"key":"e_1_3_2_1_10_1","volume-title":"The scree test for the number of factors. Multivariate behavioral research","author":"Cattell Raymond B","year":"1966","unstructured":"Raymond B Cattell. 1966. The scree test for the number of factors. Multivariate behavioral research, Vol. 1, 2 (1966), 245--276."},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462951"},{"key":"e_1_3_2_1_12_1","volume-title":"A first course in factor analysis","author":"Comrey Andrew L","unstructured":"Andrew L Comrey and Howard B Lee. 2013. A first course in factor analysis. Psychology press."},{"key":"e_1_3_2_1_13_1","volume-title":"Best practices in exploratory factor analysis: Four recommendations for getting the most from your analysis. Practical assessment, research, and evaluation","author":"Costello Anna B","year":"2005","unstructured":"Anna B Costello and Jason Osborne. 2005. Best practices in exploratory factor analysis: Four recommendations for getting the most from your analysis. Practical assessment, research, and evaluation, Vol. 10, 1 (2005), 7."},{"key":"e_1_3_2_1_14_1","volume-title":"Factor analysis at 100: Historical developments and future directions","author":"Cudeck Robert","unstructured":"Robert Cudeck and Robert C MacCallum. 2007. Factor analysis at 100: Historical developments and future directions. Routledge."},{"key":"e_1_3_2_1_15_1","volume-title":"Opportunities and challenges in explainable artificial intelligence (xai): A survey. arXiv preprint arXiv:2006.11371","author":"Das Arun","year":"2020","unstructured":"Arun Das and Paul Rad. 2020. Opportunities and challenges in explainable artificial intelligence (xai): A survey. arXiv preprint arXiv:2006.11371 (2020)."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1080\/02664763.2011.610445"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159661"},{"key":"e_1_3_2_1_18_1","volume-title":"Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608","author":"Doshi-Velez Finale","year":"2017","unstructured":"Finale Doshi-Velez and Been Kim. 2017. Towards a rigorous science of interpretable machine learning. arXiv preprint arXiv:1702.08608 (2017)."},{"key":"e_1_3_2_1_19_1","volume-title":"Explainable and interpretable models in computer vision and machine learning","author":"Doshi-Velez Finale","unstructured":"Finale Doshi-Velez and Been Kim. 2018. Considerations for evaluation and generalization in interpretable machine learning. In Explainable and interpretable models in computer vision and machine learning. Springer, 3--17."},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"crossref","unstructured":"Mike Furr. 2011. Scale construction and psychometrics for social and personality psychology. SAGE publications ltd.","DOI":"10.4135\/9781446287866"},{"key":"e_1_3_2_1_21_1","unstructured":"R Michael Furr. 2021. Psychometrics: an introduction. SAGE publications."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1109\/DSAA.2018.00018"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.48.1.82"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(02)00032-8"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174023"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1037\/e577632012-009"},{"key":"e_1_3_2_1_27_1","volume-title":"The promise and peril of human evaluation for model interpretability. arXiv preprint arXiv:1711.07414","author":"Herman Bernease","year":"2017","unstructured":"Bernease Herman. 2017. The promise and peril of human evaluation for model interpretability. arXiv preprint arXiv:1711.07414 (2017)."},{"key":"e_1_3_2_1_28_1","volume-title":"Structural equation modelling: Guidelines for determining model fit. Electronic journal of business research methods","author":"Hooper Daire","year":"2008","unstructured":"Daire Hooper, Joseph Coughlan, and Michael R Mullen. 2008. Structural equation modelling: Guidelines for determining model fit. Electronic journal of business research methods, Vol. 6, 1 (2008), pp53--60."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1007\/BF02289447"},{"key":"e_1_3_2_1_30_1","volume-title":"Cutoff criteria for fit indexes in covariance structure analysis: Conventional criteria versus new alternatives. Structural equation modeling: a multidisciplinary journal","author":"Bentler Hu","year":"1999","unstructured":"Li-tze Hu and Peter M Bentler. 1999. Cutoff criteria for fit indexes in covariance structure analysis: Conventional criteria versus new alternatives. Structural equation modeling: a multidisciplinary journal, Vol. 6, 1 (1999), 1--55."},{"key":"e_1_3_2_1_31_1","volume-title":"Towards faithfully interpretable NLP systems: How should we define and evaluate faithfulness? arXiv preprint arXiv:2004.03685","author":"Jacovi Alon","year":"2020","unstructured":"Alon Jacovi and Yoav Goldberg. 2020. Towards faithfully interpretable NLP systems: How should we define and evaluate faithfulness? arXiv preprint arXiv:2004.03685 (2020)."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jrp.2004.09.009"},{"key":"e_1_3_2_1_33_1","volume-title":"The application of electronic computers to factor analysis. Educational and psychological measurement","author":"Kaiser Henry F","year":"1960","unstructured":"Henry F Kaiser. 1960. The application of electronic computers to factor analysis. Educational and psychological measurement, Vol. 20, 1 (1960), 141--151."},{"key":"e_1_3_2_1_34_1","volume-title":"Examples are not enough, learn to criticize! criticism for interpretability. Advances in neural information processing systems","author":"Kim Been","year":"2016","unstructured":"Been Kim, Rajiv Khanna, and Oluwasanmi O Koyejo. 2016. Examples are not enough, learn to criticize! criticism for interpretability. Advances in neural information processing systems , Vol. 29 (2016)."},{"key":"e_1_3_2_1_35_1","volume-title":"Understanding the Negative Aspects of User Experience in Human-likeness of Voice-based Conversational Agents. In Designing Interactive Systems Conference. 1418--1427","author":"Kim Hyeji","year":"2022","unstructured":"Hyeji Kim, Inchan Jung, and Youn-kyung Lim. 2022. Understanding the Negative Aspects of User Experience in Human-likeness of Voice-based Conversational Agents. In Designing Interactive Systems Conference. 1418--1427."},{"key":"e_1_3_2_1_36_1","volume-title":"Psychological testing: A practical approach to design and evaluation","author":"Kline Theresa","unstructured":"Theresa Kline. 2005. Psychological testing: A practical approach to design and evaluation. Sage."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5280"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236386.3241340"},{"key":"e_1_3_2_1_39_1","volume-title":"A unified approach to interpreting model predictions. Advances in neural information processing systems","author":"Lundberg Scott M","year":"2017","unstructured":"Scott M Lundberg and Su-In Lee. 2017. A unified approach to interpreting model predictions. Advances in neural information processing systems , Vol. 30 (2017)."},{"key":"e_1_3_2_1_40_1","volume-title":"Sample size in factor analysis. Psychological methods","author":"MacCallum Robert C","year":"1999","unstructured":"Robert C MacCallum, Keith F Widaman, Shaobo Zhang, and Sehee Hong. 1999. Sample size in factor analysis. Psychological methods, Vol. 4, 1 (1999), 84."},{"key":"e_1_3_2_1_41_1","volume-title":"Identifying careless responses in survey data. Psychological methods","author":"Meade Adam W","year":"2012","unstructured":"Adam W Meade and S Bartholomew Craig. 2012. Identifying careless responses in survey data. Psychological methods, Vol. 17, 3 (2012), 437."},{"key":"e_1_3_2_1_42_1","volume-title":"From anecdotal evidence to quantitative evaluation methods: A systematic review on evaluating explainable ai. arXiv preprint arXiv:2201.08164","author":"Nauta Meike","year":"2022","unstructured":"Meike Nauta, Jan Trienes, Shreyasi Pathak, Elisa Nguyen, Michelle Peters, Yasmin Schmitt, J\u00f6rg Schl\u00f6tterer, Maurice van Keulen, and Christin Seifert. 2022. From anecdotal evidence to quantitative evaluation methods: A systematic review on evaluating explainable ai. arXiv preprint arXiv:2201.08164 (2022)."},{"key":"e_1_3_2_1_43_1","volume-title":"On quantitative aspects of model interpretability. arXiv preprint arXiv:2007.07584","author":"Mar'ia Rodr'iguez Mart'inez Nguyen","year":"2020","unstructured":"An-phi Nguyen and Mar'ia Rodr'iguez Mart'inez. 2020. On quantitative aspects of model interpretability. arXiv preprint arXiv:2007.07584 (2020)."},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300698"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753685"},{"key":"e_1_3_2_1_46_1","volume-title":"Reputation as a sufficient condition for data quality on Amazon Mechanical Turk. Behavior research methods","author":"Peer Eyal","year":"2014","unstructured":"Eyal Peer, Joachim Vosgerau, and Alessandro Acquisti. 2014. Reputation as a sufficient condition for data quality on Amazon Mechanical Turk. Behavior research methods, Vol. 46, 4 (2014), 1023--1031."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3404835.3462799"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401277"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484084"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401279"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401117"},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/3289600.3290620"},{"key":"e_1_3_2_1_54_1","volume-title":"Assessing the local interpretability of machine learning models. arXiv preprint arXiv:1902.03501","author":"Slack Dylan","year":"2019","unstructured":"Dylan Slack, Sorelle A Friedler, Carlos Scheidegger, and Chitradeep Dutta Roy. 2019. Assessing the local interpretability of machine learning models. arXiv preprint arXiv:1902.03501 (2019)."},{"key":"e_1_3_2_1_55_1","volume-title":"Understanding the limitations of global fit assessment in structural equation modeling. Personality and Individual differences","author":"Steiger James H","year":"2007","unstructured":"James H Steiger. 2007. Understanding the limitations of global fit assessment in structural equation modeling. Personality and Individual differences, Vol. 42, 5 (2007), 893--898."},{"key":"e_1_3_2_1_56_1","volume-title":"Using multivariate statistics","author":"Tabachnick Barbara G","unstructured":"Barbara G Tabachnick, Linda S Fidell, and Jodie B Ullman. 2007. Using multivariate statistics. Vol. 5. pearson Boston, MA."},{"key":"e_1_3_2_1_57_1","volume-title":"Multiple-factor analysis","author":"Thurstone Louis Leon","year":"1947","unstructured":"Louis Leon Thurstone. 1947. Multiple-factor analysis; a development and expansion of The Vectors of Mind. (1947)."},{"key":"e_1_3_2_1_58_1","volume-title":"Structural equation modeling. Handbook of Psychology","author":"Ullman Jodie B","year":"2012","unstructured":"Jodie B Ullman and Peter M Bentler. 2012. Structural equation modeling. Handbook of Psychology, Second Edition , Vol. 2 (2012)."},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.6028\/NIST.SP.500--261"},{"key":"e_1_3_2_1_60_1","volume-title":"Assessing reliability and stability in panel models. Sociological methodology","author":"Wheaton Blair","year":"1977","unstructured":"Blair Wheaton, Bengt Muthen, Duane F Alwin, and Gene F Summers. 1977. Assessing reliability and stability in panel models. Sociological methodology , Vol. 8 (1977), 84--136."},{"key":"e_1_3_2_1_61_1","volume-title":"Exploratory factor analysis: A five-step guide for novices. Australasian journal of paramedicine","author":"Williams Brett","year":"2010","unstructured":"Brett Williams, Andrys Onsman, and Ted Brown. 2010. Exploratory factor analysis: A five-step guide for novices. Australasian journal of paramedicine, Vol. 8, 3 (2010)."},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1177\/0011000006288127"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3477495.3532067"},{"key":"e_1_3_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609577"}],"event":{"name":"SIGIR 2024: The 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","location":"Washington DC USA","acronym":"SIGIR 2024","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657796","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3626772.3657796","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,8,22]],"date-time":"2025-08-22T05:42:01Z","timestamp":1755841321000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3626772.3657796"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":64,"alternative-id":["10.1145\/3626772.3657796","10.1145\/3626772"],"URL":"https:\/\/doi.org\/10.1145\/3626772.3657796","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-11","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}