{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T22:42:06Z","timestamp":1776120126611,"version":"3.50.1"},"reference-count":71,"publisher":"Informa UK Limited","issue":"1","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["International Journal of Human-Computer Interaction"],"published-print":{"date-parts":[[2003,2]]},"DOI":"10.1207\/s15327590ijhc1501_13","type":"journal-article","created":{"date-parts":[[2004,1,14]],"date-time":"2004-01-14T09:30:03Z","timestamp":1074072603000},"page":"145-181","source":"Crossref","is-referenced-by-count":144,"title":["Criteria For Evaluating Usability Evaluation Methods"],"prefix":"10.1080","volume":"15","author":[{"given":"H. Rex","family":"Hartson","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Terence S.","family":"Andre","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Robert C.","family":"Williges","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"301","reference":[{"key":"p_1_183","doi-asserted-by":"crossref","unstructured":"Andre, T. S., Belz, S. M., McCreary, F. A. & Hartson, H. R. (2000). Testing a framework for reliable classification of usability problems. In Human Factors and Ergonomics Society Annual Meeting (pp. 573-576). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193120004403707"},{"key":"p_2_184","unstructured":"Andre, T. S., Williges, R. C. & Hartson, H. R. (1999). The effectiveness of usability evaluation methods: Determining the appropriate criteria. In Human Factors and Ergonomics Society 43rd Annual Meeting (pp. 1090-1094). Santa Monica, CA: Human Factors and Ergonomics Society."},{"key":"p_3_185","doi-asserted-by":"crossref","first-page":"105","DOI":"10.1080\/10447319509526114","volume":"7","author":"Bastien J. M. C.","year":"1995","journal-title":"International Journal of Human-Computer Interaction"},{"key":"p_4_186","doi-asserted-by":"crossref","unstructured":"Bastien, J. M. C., Scapin, D. L. & Leulier, C. (1996). Looking for usability problems with the ergonomic criteria and with the ISO 9241-10 dialogue principles. In CHI Conference on Human Factors in Computing Systems (pp. 77-78). New York: ACM.","DOI":"10.1145\/257089.257153"},{"key":"p_5_187","doi-asserted-by":"crossref","unstructured":"Beer, T., Anodenko, T. & Sears, A. (1997). A pair of techniques for effective interface evaluation: Cognitive walkthroughs and think-aloud evaluations. In Proceedings of the Human Factors and Ergonomics Society 41st Annual Meeting (pp. 380-384). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/107118139704100184"},{"key":"p_6_188","doi-asserted-by":"publisher","DOI":"10.1109\/52.84220"},{"key":"p_7_189","unstructured":"Bradford, J. S. (1994). Evaluating high-level design: Synergistic use of inspection and usability methods for evaluating early software designs. In J. Nielsen & R. L. Mack (Eds.), Usability inspection methods (pp. 235-253). New York: Wiley."},{"key":"p_8_190","unstructured":"Card, S. K., Moran, T. P. & Newell, A. (1983). The psychology of human-computer interaction. Hillsdale, NJ: Lawrence Erlbaum Associates, Inc."},{"key":"p_9_191","doi-asserted-by":"crossref","first-page":"247","DOI":"10.1080\/01449299208924345","volume":"11","author":"Carroll J. M.","year":"1992","journal-title":"Behaviour & Information Technology"},{"key":"p_10_192","doi-asserted-by":"crossref","unstructured":"Chin, J. P., Diehl, V. A. & Norman, K. L. (1988). Development of an instrument measuring user satisfaction of the human-computer interface. In CHI Conference on Human Factors in Computing Systems (pp. 213-218). New York: ACM.","DOI":"10.1145\/57167.57203"},{"key":"p_11_193","unstructured":"Cockton, G. & Lavery, D. (1999). A framework for usability problem extraction. In INTERACT '99 (pp. 344-352). London: IOS Press."},{"key":"p_12_194","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1177\/001316446002000104","volume":"20","author":"Cohen J.","year":"1960","journal-title":"Educational and Psychological Measurement"},{"key":"p_13_195","unstructured":"Cuomo, D. L. & Bowen, C. D. (1992). Stages of user activity model as a basis for user-centered interface evaluation. In Annual Human Factors Society Conference (pp. 1254-1258). Santa Monica, CA: Human Factors Society."},{"key":"p_14_196","doi-asserted-by":"publisher","DOI":"10.1016\/0953-5438(94)90006-X"},{"key":"p_15_197","doi-asserted-by":"crossref","unstructured":"del Galdo, E. M., Williges, R. C., Williges, B. H. & Wixon, D. R. (1986). An evaluation of critical incidents for software documentation design. In Thirtieth Annual Human Factors Society Conference (pp. 19-23). Anaheim, CA: Human Factors Society.","DOI":"10.1177\/154193128603000105"},{"key":"p_16_198","doi-asserted-by":"crossref","unstructured":"del Galdo, E. M., Williges, R. C., Williges, B. H. & Wixon, D. R. (1987). A critical incident evaluation tool for software documentation. In L. S. Mark, J. S. Warm, & R. L. Huston (Eds.), Ergonomics and human factors (pp. 253-258). New York: Springer-Verlag.","DOI":"10.1007\/978-1-4612-4756-2_29"},{"key":"p_17_199","unstructured":"Desurvire, H. W., Kondziela, J. M. & Atwood, M. E. (1992). What is gained and lost when using evaluation methods other than empirical testing. In A. Monk, D. Diaper, & M. D. Harrison (Eds.), People and computers Volume VII (pp. 89-102). Cambridge, England: Cambridge University Press."},{"key":"p_18_200","unstructured":"Desurvire, H. W. & Thomas, J. C. (1993). Enhancing the performance of interface evaluators. In Proceedings of the Human Factors and Ergonomics Society 37th Annual Meeting (pp. 1132-1136). Seattle, WA: Human Factors and Ergonomics Society."},{"key":"p_19_201","doi-asserted-by":"crossref","unstructured":"Doubleday, A., Ryan, M., Springett, M. & Sutcliffe, A. (1997). A comparison of usability techniques for evaluating design. In Designing interactive Systems (DIS '97) Conference (pp. 101-110). New York: ACM.","DOI":"10.1145\/263552.263583"},{"key":"p_20_202","doi-asserted-by":"crossref","unstructured":"Dutt, A., Johnson, H. & Johnson, P. (1994). Evaluating evaluation methods. In G. Cockton, S. W. Draper, & G. R. S. Weir (Eds.), People and computers, Volume IX (pp. 109-121). Cambridge, England: Cambridge University Press.","DOI":"10.1017\/CBO9780511600821.010"},{"key":"p_21_203","unstructured":"Egan, J. P. (1975). Signal detection theory and ROC analysis. New York: Academic."},{"key":"p_22_204","unstructured":"Ericsson, A. & Simon, H. (1984). Protocol analysis: Verbal reports as data. Cambridge, MA: MIT Press."},{"key":"p_23_205","doi-asserted-by":"crossref","first-page":"327","DOI":"10.1037\/h0061470","volume":"51","author":"Flanagan J. C.","year":"1954","journal-title":"Psychological Bulletin"},{"key":"p_24_206","doi-asserted-by":"crossref","first-page":"378","DOI":"10.1037\/h0031619","volume":"76","author":"Fleiss J. L.","year":"1971","journal-title":"Psychological Bulletin"},{"key":"p_25_207","doi-asserted-by":"crossref","first-page":"203","DOI":"10.1207\/s15327051hci1303_2","volume":"13","author":"Gray W. D.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_26_208","unstructured":"Hartson, H. R., Andre, T. S., Williges, R. C. & van Rens, L. S. (1999). The user action framework: A theory-based foundation for inspection and classification of usability problems. In H.J. Bullinger & J. Ziegler (Eds.), Human-computer interaction: Ergonomics and user interfaces, Volume 1 (pp. 1058-1062). Mahwah, NJ: Lawrence Erlbaum Associates, Inc."},{"key":"p_27_209","doi-asserted-by":"crossref","unstructured":"Hartson, H. R. & Castillo, J. C. (1998). Remote evaluation for post-deployment usability improvement. In Advanced Visual Interfaces '98 (pp. 22-29). L'Aquila, Italy: ACM Press.","DOI":"10.1145\/948496.948499"},{"key":"p_28_210","doi-asserted-by":"crossref","unstructured":"Hartson, H. R., Castillo, J. C., Kelso, J., Kamler, J. & Neale, W. C. (1996). Remote evaluation: The network as an extension of the usability laboratory. In CHI Conference on Human Factors in Computing Systems (pp. 228-235). New York: ACM.","DOI":"10.1145\/238386.238511"},{"key":"p_29_211","unstructured":"Hix, D. & Hartson, H. R. (1993a). Developing user interfaces: Ensuring usability through product & process. New York: Wiley."},{"key":"p_30_212","unstructured":"Hix, D. & Hartson, H. R. (1993b). Formative evaluation: Ensuring usability in user interfaces. In L. Bass & P. Dewan (Eds.), Trends in software, Volume 1: User interface software (pp. 1-30). New York: Wiley."},{"key":"p_31_213","doi-asserted-by":"crossref","unstructured":"Jeffries, R., Miller, J. R., Wharton, C. & Uyeda, K. M. (1991). User interface evaluation in the real world: A comparison of four techniques. In CHI Conference on Human Factors in Computing Systems (pp. 119-124). New York: ACM.","DOI":"10.1145\/108844.108862"},{"key":"p_32_214","doi-asserted-by":"publisher","DOI":"10.1080\/014492997119789"},{"key":"p_33_215","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199711)48:11<1004::AID-ASI4>3.3.CO;2-T"},{"key":"p_34_216","unstructured":"Kahn, M. J. & Prail, A. (1994). Formal usability inspections. In J. Nielsen & R. L. Mack (Eds.), Usability inspection methods (pp. 141-171). New York: Wiley."},{"key":"p_35_217","doi-asserted-by":"crossref","unstructured":"Karat, C.M., Campbell, R. & Fiegel, T. (1992). Comparison of empirical testing and walkthrough methods in user interface evaluation. In CHI Conference on Human Factors in Computing Systems (pp. 397-404). New York: ACM.","DOI":"10.1145\/142750.142873"},{"key":"p_36_218","first-page":"265","volume":"13","author":"Karat J.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_37_219","unstructured":"Keppel, G. (1991). Design and analysis: A researcher's handbook. Englewood Cliffs, NJ: Prentice Hall."},{"key":"p_38_220","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-4571(199807)49:9<776::AID-ASI3>3.0.CO;2-P"},{"key":"p_39_221","unstructured":"Landauer, T. K. (1995). The trouble with computers: Usefulness, usability, and productivity. Cambridge, MA: MIT Press."},{"key":"p_40_222","doi-asserted-by":"publisher","DOI":"10.1080\/014492997119824"},{"key":"p_41_223","doi-asserted-by":"crossref","unstructured":"Lewis, C., Polson, P., Wharton, C. & Rieman, J. (1990). Testing a walkthrough methodology for theory-based design of walk-up-and-use interfaces. In CHI Conference on Human Factors in Computing Systems (pp. 235-242). New York: ACM.","DOI":"10.1145\/97243.97279"},{"key":"p_42_224","doi-asserted-by":"crossref","first-page":"368","DOI":"10.1177\/001872089403600215","volume":"36","author":"Lewis J. R.","year":"1994","journal-title":"Human Factors"},{"key":"p_43_225","first-page":"276","volume":"13","author":"Lund A. M.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_44_226","unstructured":"Manning, C. D. & Schutze, H. (1999). Foundations of statistical natural language processing. Cambridge, MA: MIT Press."},{"key":"p_45_227","unstructured":"Marchetti, R. (1994). Using usability inspections to find usability problems early in the lifecycle. In Pacific Northwest Software Quality Conference (pp. 1-19). Palo Alto, CA: Hewlett Packard."},{"key":"p_46_228","unstructured":"Meister, D., Andre, T. S. & Aretz, A. J. (1997). System analysis. In T. S. Andre & A. W. Schopper (Eds.), Human factors engineering in system design (pp. 21-55). Dayton, OH: Crew System Ergonomics Information Analysis Center."},{"key":"p_47_229","first-page":"296","volume":"13","author":"Monk A. F.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_48_230","doi-asserted-by":"crossref","unstructured":"Neale, D. C., Dunlap, R., Isenhour, P. & Carroll, J. M. (2000). Collaborative critical incident development. In Human Factors and Ergonomics Society 43rd Annual Meeting (pp. 598-601). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193120004403714"},{"key":"p_49_231","first-page":"316","volume":"13","author":"Newman W. M.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_50_232","unstructured":"Nielsen, J. (1989). Usability engineering at a discount. In G. Salvendy & M. J. Smith (Eds.), Designing and using human-computer interfaces and knowledge-based systems (pp. 394-401). Amsterdam: Elsevier."},{"key":"p_51_233","unstructured":"Nielsen, J. (1990). Evaluating the thinking aloud technique for use by computer scientists. In H. R. Hartson & D. Hix (Eds.), Advances in human-computer interaction (pp. 69-82). Norwood, NJ: Ablex."},{"key":"p_52_234","doi-asserted-by":"crossref","unstructured":"Nielsen, J. (1992). Finding usability problems through heuristic evaluation. In CHI Conference on Human Factors in Computing Systems (pp. 373-380). New York: ACM.","DOI":"10.1145\/142750.142834"},{"key":"p_53_235","unstructured":"Nielsen, J. (1994). Heuristic evaluation. In J. Nielsen & R. L. Mack (Eds.), Usability inspection methods (pp. 25-62). New York: Wiley."},{"key":"p_54_236","doi-asserted-by":"crossref","unstructured":"Nielsen, J. & Mack, R. L. (Eds.). (1994). Usability inspection methods. New York: Wiley.","DOI":"10.1145\/259963.260531"},{"key":"p_55_237","doi-asserted-by":"crossref","unstructured":"Nielsen, J. & Molich, R. (1990). Heuristic evaluation of user interfaces. In CHI Conference on Human Factors in Computing Systems (pp. 249-256). New York: ACM.","DOI":"10.1145\/97243.97281"},{"key":"p_56_238","doi-asserted-by":"crossref","unstructured":"Norman, D. A. (1986). Cognitive engineering. In D. A. Norman & S. W. Draper (Eds.), User centered system design (pp. 31-61). Hillsdale, NJ: Lawrence Erlbaum Associates, Inc.","DOI":"10.1201\/b15703"},{"key":"p_57_239","doi-asserted-by":"crossref","first-page":"263","DOI":"10.1207\/s15327051hci1303_3","volume":"13","author":"Olson G. M.","year":"1998","journal-title":"Human-Computer Interaction"},{"key":"p_58_240","unstructured":"Rubin, J. (1994). Handbook of usability testing. New York: Wiley."},{"key":"p_59_241","unstructured":"Salton, G. & McGill, M. J. (1983). Introduction to modern information retrieval. New York: McGraw-Hill."},{"key":"p_60_242","unstructured":"Scriven, M. (1967). The methodology of evaluation. In R. Tyler, R. Gagne, & M. Scriven (Eds.), Perspectives of curriculum evaluation (pp. 39-83). Chicago: Rand McNally."},{"key":"p_61_243","doi-asserted-by":"crossref","first-page":"213","DOI":"10.1207\/s15327590ijhc0903_2","volume":"9","author":"Sears A.","year":"1997","journal-title":"International Journal of Human-Computer Interaction"},{"key":"p_62_244","doi-asserted-by":"crossref","unstructured":"Smith, S. L. & Mosier, J. N. (1986). Guidelines for designing user interface software (Technical Report No. ESD- TR-86-278\/MTR 10090). Bedford, MA: MITRE Corporation.","DOI":"10.21236\/ADA177198"},{"key":"p_63_245","unstructured":"Swets, J. A. (1964). Signal detection and recognition by human observers. New York: Wiley."},{"key":"p_64_246","doi-asserted-by":"crossref","unstructured":"Thompson, J. A. & Williges, R. C. (2000). Web-based collection of critical incidents during remote usability evaluation. In IEA 2000\/HFES 2000 Congress (pp. 602-605). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193120004403715"},{"key":"p_65_247","unstructured":"van Rijsbergen, C. J. (1979). Information retrieval (2nd ed.). London: Butterworths."},{"key":"p_66_248","unstructured":"Virzi, R. A. (1990). Streamlining the design process: Running fewer subjects. In Human Factors and Ergonomics Society 34th Annual Meeting (pp. 291-294). Santa Monica, CA: Human Factors and Ergonomics Society."},{"key":"p_67_249","doi-asserted-by":"crossref","first-page":"457","DOI":"10.1177\/001872089203400407","volume":"34","author":"Virzi R. A.","year":"1992","journal-title":"Human Factors"},{"key":"p_68_250","doi-asserted-by":"crossref","unstructured":"Virzi, R. A., Sorce, J. & Herbert, L. B. (1993). A comparison of three usability evaluation methods: Heuristic, think-aloud, and performance testing. In Proceedings of the Human Factors and Ergonomics Society 36th Annual Meeting (pp. 309-313). Seattle, WA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193129303700412"},{"key":"p_69_251","doi-asserted-by":"crossref","unstructured":"Wharton, C., Bradford, J., Jeffries, R. & Franzke, M. (1992). Applying cognitive walkthroughs to more complex user interfaces: Experiences, issues, and recommendations. In CHI Conference on Human Factors in Computing Systems (pp. 381-388). New York: ACM.","DOI":"10.1145\/142750.142864"},{"key":"p_70_252","unstructured":"Winer, B. J., Brown, D. R. & Michels, K. M. (1991). Statistical principles in experimental design. New York: McGraw-Hill."},{"key":"p_71_253","doi-asserted-by":"crossref","first-page":"891","DOI":"10.1016\/S0020-7373(05)80167-1","volume":"35","author":"Wright P.","year":"1991","journal-title":"International Journal of Man-Machine Studies"}],"container-title":["International Journal of Human-Computer Interaction"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/www.tandfonline.com\/doi\/pdf\/10.1207\/S15327590IJHC1501_13","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2018,4,5]],"date-time":"2018-04-05T05:33:08Z","timestamp":1522906388000},"score":1,"resource":{"primary":{"URL":"http:\/\/www.tandfonline.com\/doi\/abs\/10.1207\/S15327590IJHC1501_13"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2003,2]]},"references-count":71,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2003,2]]}},"alternative-id":["10.1207\/S15327590IJHC1501_13"],"URL":"https:\/\/doi.org\/10.1207\/s15327590ijhc1501_13","relation":{},"ISSN":["1044-7318","1532-7590"],"issn-type":[{"value":"1044-7318","type":"print"},{"value":"1532-7590","type":"electronic"}],"subject":[],"published":{"date-parts":[[2003,2]]}}}