{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,12,17]],"date-time":"2025-12-17T17:37:40Z","timestamp":1765993060020},"reference-count":44,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2005,4,1]],"date-time":"2005-04-01T00:00:00Z","timestamp":1112313600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2005,4]]},"DOI":"10.1007\/s10664-004-6189-4","type":"journal-article","created":{"date-parts":[[2005,2,14]],"date-time":"2005-02-14T14:30:56Z","timestamp":1108391456000},"page":"113-147","source":"Crossref","is-referenced-by-count":21,"title":["Collecting Feedback during Software Engineering Experiments"],"prefix":"10.1007","volume":"10","author":[{"given":"Amela","family":"Karahasanovi\ufffd","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bente","family":"Anda","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Erik","family":"Arisholm","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Siw Elisabeth","family":"Hove","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Magne","family":"J\ufffdrgensen","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Dag I K","family":"Sj\ufffdberg","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Ray","family":"Welland","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","reference":[{"key":"CR1","unstructured":"Abelow, D. 1993. Automating feedback on software product use. CASE Trends December: 15?17."},{"key":"CR2","unstructured":"Anda, B., and Sj\u00f8berg, D. I. K. 2003. Applying use cases to design versus validate class diagrams?A controlled experiment using a professional modelling tool. IEEE International Symposium on Empirical Software Engineering (ISESE 2003). Rome, Italy, 50?60."},{"key":"CR3","doi-asserted-by":"crossref","first-page":"467","DOI":"10.1017\/S0140525X00023554","volume":"10","author":"J. R. Anderson","year":"1987","unstructured":"Anderson, J. R. 1987. Methodologies for studying human knowledge. Behavioural and Brain Science 10:467?505.","journal-title":"Behavioural and Brain Science"},{"issue":"4","key":"CR4","first-page":"231","volume":"9","author":"E. Arisholm","year":"2002","unstructured":"Arisholm, E., Sj\u00f8berg, D. I. K., Carelius, G., and Lindsj\u00f8rn, Y. 2002. A web-based support environment for software engineering experiments. Nordic Journal of Computing 9(4): 231?247.","journal-title":"Nordic Journal of Computing"},{"key":"CR5","unstructured":"Arisholm, E., Ali, S. A., and Hove, S. E. 2003. An initial controlled experiment to evaluate the effect of UML design documentation on the maintainability of object-oriented software in a realistic programming environment. Simula Research Laboratory, Technical Report 2003?2004."},{"key":"CR6","doi-asserted-by":"crossref","first-page":"273","DOI":"10.1126\/science.118.3062.273","volume":"118","author":"E. Aserinsky","year":"1953","unstructured":"Aserinsky, E., and Kleitman, N., 1953. Regularly occurring periods of eye mobility and concomitant phenomena during sleep. Science 118: 273?374.","journal-title":"Science"},{"issue":"4","key":"CR7","doi-asserted-by":"crossref","first-page":"456","DOI":"10.1109\/32.799939","volume":"25","author":"V. R. Basili","year":"1999","unstructured":"Basili, V. R., Shull, F., and Lanubile, F. 1999. Building knowledge through families of experiments. IEEE Transactions on Software Engineering 25(4): July?Aug., 456?473.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"CR8","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1016\/0049-089X(82)90006-0","volume":"11","author":"H. Bernard","year":"1982","unstructured":"Bernard, H., Killorth, P., and Sailer, L. 1982. Informant accurracy in social-network data. An experimental attempt to predict actual communication from recall data. Social Science Research 11: 30?36.","journal-title":"Social Science Research"},{"issue":"3","key":"CR9","doi-asserted-by":"crossref","first-page":"175","DOI":"10.1080\/01449299008924235","volume":"9","author":"D. C. Berry","year":"1990","unstructured":"Berry, D. C., and Broadbent, D. E. 1990. The role of instruction and verbalization in improving performance on complex search tasks. Behaviour and Information Technology 9(3): May?June, 175?190.","journal-title":"Behaviour and Information Technology"},{"key":"CR10","doi-asserted-by":"crossref","unstructured":"Boren, M. T., and Ramey, J., 2000. Thinking-aloud: Reconciling theory and practice. IEEE Transactions on Professional Communication 43(3): September, 261?278.","DOI":"10.1109\/47.867942"},{"key":"CR11","volume-title":"Experimental and Quasi-Experimental Designs for Research","author":"D. T. Campbell","year":"1963","unstructured":"Campbell, D. T., and Stanley, J. C. 1963. Experimental and Quasi-Experimental Designs for Research. Boston, MA, USA: Houghton Mifflin Company."},{"key":"CR12","unstructured":"Chen, H., and Nilan, M. 1998. An exploration of web users? internal experience: Application of the experience sampling method to the web environment. WebNet 98 World Conference. Orlando, Florida."},{"key":"CR13","doi-asserted-by":"crossref","first-page":"315","DOI":"10.1016\/0378-8733(83)90031-X","volume":"5","author":"D. W. Conrath","year":"1983","unstructured":"Conrath, D. W., Higgins, C. A., and McClean, R. J. 1983. A comparison of questionnaire versus diary data. Social Networks 5: 315?322.","journal-title":"Social Networks"},{"key":"CR14","unstructured":"Davidson, G. V., Shorter, L., Crum, A., and Lane, J. 1996. Children?s use of learning strategies and decision making in hypertext computer lesson. ED MEDIA 96, Conference on Educational Multimedia and Hypermedia. Assoc. Adv. Comp. Educ., Charlottesville, VA, USA."},{"key":"CR15","doi-asserted-by":"crossref","unstructured":"Denning, S., Hoiem, D., Simpson, M., and Sullivan, K. 1990. The value of thinking-aloud protocols in industry: A case study of microsoft. Proceedings of the Human Factors Society?34th Annual Meeting. Santa Monica, 1285?1289.","DOI":"10.1177\/154193129003401723"},{"key":"CR16","unstructured":"Ericsson, K. A. 2003. Valid and non-reactive verbalisation of thoughts. In Preparation."},{"key":"CR17","doi-asserted-by":"crossref","DOI":"10.7551\/mitpress\/5657.001.0001","volume-title":"Protocol Analysis: Verbal Reports as Data","author":"K. A. Ericsson","year":"1993","unstructured":"Ericsson, K. A., and Simon, H. A. 1993. Protocol Analysis: Verbal Reports as Data. Cambridge, Massachusetts: The MIT Press."},{"issue":"3","key":"CR18","doi-asserted-by":"crossref","first-page":"178","DOI":"10.1207\/s15327884mca0503_3","volume":"5","author":"K. A. Ericsson","year":"1998","unstructured":"Ericsson, K. A., and Simon, H. A. 1998. How to study thinking in everyday life: Contrasting think-aloud protocols with descriptions and explanations of thinking. Mind, Culture and Activity 5(3): 178?186.","journal-title":"Mind, Culture and Activity"},{"issue":"2","key":"CR19","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1002\/spe.498","volume":"33","author":"H. Evans","year":"2003","unstructured":"Evans, H., Atkinson, M., Brown, M., Cargill, J., Crease, M., Draper, S., Gray, P., and Thomas, R. 2003. The pervasiveness of evolution in GRUMPS software. Software-Practice & Experience 33(2): 99?120.","journal-title":"Software-Practice & Experience"},{"key":"CR20","unstructured":"Freund, R. J., and Wilson, W. J. 1998. Regression Analysis: Statistical Modeling of a Response Variable. Academic Press."},{"key":"CR21","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1016\/B978-0-12-742460-6.50011-6","volume-title":"Learning and Study Strategies: Issues in Assessment, Instruction, and Evaluation","author":"R. Garner","year":"1988","unstructured":"Garner, R. 1988. Verbal-report data on cognitive and metacognitive strategies. In: Weinstein C. E. et al., (eds), Learning and Study Strategies: Issues in Assessment, Instruction, and Evaluation. San Diego, CA: Academic Press, Inc., 63?100."},{"key":"CR22","first-page":"223","volume-title":"Cognitive Assessment","author":"M. Genest","year":"1981","unstructured":"Genest, M., and Turk, D. C. (1981). Think-aloud approaches to cognitive assessment. In Merluzzi C. R. et al. (eds), Cognitive Assessment. Guilford Press, New York, pp. 223?269."},{"key":"CR23","unstructured":"Haak, M. J., and Jong, D. T. M. 2003. Exploring two methods of usability testing: Concurrent versus retrospective think-aloud protocols. IEEE Computer Society 285?287."},{"key":"CR24","doi-asserted-by":"crossref","first-page":"103","DOI":"10.1016\/0092-6566(79)90045-X","volume":"13","author":"R. T. Hurlburt","year":"1979","unstructured":"Hurlburt, R. T. 1979. Random sampling of cognitions and behaviour. Journal of Research in Personality 13: 103?111.","journal-title":"Journal of Research in Personality"},{"issue":"4","key":"CR25","doi-asserted-by":"crossref","first-page":"470","DOI":"10.1145\/503112.503114","volume":"33","author":"M. I. Ivory","year":"2001","unstructured":"Ivory, M. I., and Hearst, M. A. 2001. The state of the art in automating usability evaluation of user interfaces. ACM Computing Surveys 33(4): 470?516.","journal-title":"ACM Computing Surveys"},{"issue":"4","key":"CR26","doi-asserted-by":"crossref","first-page":"243","DOI":"10.1023\/A:1011462726040","volume":"16","author":"A. Jungk","year":"2000","unstructured":"Jungk, A., Thull, B., Hoeft, A., and Rau, G. 2000. Evaluation of two new ecological interface approaches for the anesthesia workplace. Journal of Clinical Monitoring and Computing 16(4): 243?258.","journal-title":"Journal of Clinical Monitoring and Computing"},{"key":"CR27","unstructured":"Karahasanovi?, A. 2000. SEMT?A tool for finding impacts of schema changes. NWPER?2000 Nordic Workshop on Programming Environment Research. Lillehammer, Norway, pp. 60?75."},{"key":"CR28","unstructured":"Karahasanovi?, A. 2002. Supporting Application Consistency in Evolving Object-Oriented Systems by Impacts Analysis and Visualisation, Ph.D. Thesis, Faculty of Mathematics and Natural Science, University of Oslo, Unipub, ISSN 1501?7710, Nr. 234."},{"key":"CR29","unstructured":"Karahasanovi?, A., and Sj\u00f8berg, D. I. K. 2001. Visualising impacts of database schema changes?A controlled experiment. 2001 IEEE Symposium on Visual\/Multimedia Approaches to Programming and Software Engineering. Stresa, Italy: IEEE Computer Society, 358?365."},{"key":"CR30","unstructured":"Karahasanovi?, A., and Sj\u00f8berg, D. I. K. 2002. Visualising impacts of change in evolving object-oriented systems: An explorative study. International Workshop on Graph-Based Tools GraBaTs?02. Barcelona, Spain, pp. 22?31."},{"key":"CR31","unstructured":"Karahasanovi?, A., Sj\u00f8berg, D. I. K., and J\u00f8rgensen, M. 2001. Data collection in software engineering experiments. Information Resources Management Association International Conference, Soft. Eng. Track. Toronto, Ontario, Canada, pp. 1027?1028."},{"key":"CR32","first-page":"42","volume-title":"Naturalistic Approaches to Studying Social Interaction","author":"R. Larson","year":"1983","unstructured":"Larson, R., and Csikszentmihalyi, M. 1983. The experience sampling method. In R. H. T. (ed), Naturalistic Approaches to Studying Social Interaction. Jossey-Bass, San Francisco, pp. 42?56."},{"issue":"3?4","key":"CR33","doi-asserted-by":"crossref","first-page":"101","DOI":"10.1080\/14626269109408294","volume":"2","author":"M. J. Nathan","year":"1991","unstructured":"Nathan, M. J. 1991. A simple learning environment improves mathematical reasoning. Intelligent Tutoring Media 2(3?4): 101?111.","journal-title":"Intelligent Tutoring Media"},{"key":"CR34","doi-asserted-by":"crossref","unstructured":"Nielsen, J. 1993. Usability Engineering. AP Professional.","DOI":"10.1016\/B978-0-08-052029-2.50007-3"},{"issue":"3","key":"CR35","doi-asserted-by":"crossref","first-page":"147","DOI":"10.1016\/S1386-5056(01)00165-4","volume":"63","author":"V. L. Patel","year":"2001","unstructured":"Patel, V. L., Arocha, J. F., Diermeier, M., How, J., and Mottur-Pilson, C. 2001. Cognitive psychological studies of representation and use of clinical practice guidelines. International Journal of Medical Informatics 63(3): 147?167.","journal-title":"International Journal of Medical Informatics"},{"key":"CR36","doi-asserted-by":"crossref","unstructured":"Sanderson, P. M. 1990. Verbal protocol analysis in three experimental domains using SHAPA. Proceedings of the Human Factors Society, 34th Annual Meeting. Santa Monica, CA, pp. 1280?1284.","DOI":"10.1177\/154193129003401722"},{"issue":"4","key":"CR37","doi-asserted-by":"crossref","first-page":"557","DOI":"10.1109\/32.799955","volume":"25","author":"C. Seaman","year":"1999","unstructured":"Seaman, C. 1999. Qualitative methods in empirical studies of software engineering. IEEE Transactions on Software Engineering 25(4): July\/August, 557?572.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"CR38","doi-asserted-by":"crossref","first-page":"287","DOI":"10.1023\/A:1011998412776","volume":"6","author":"J. Singer","year":"2001","unstructured":"Singer, J., and Vinson, N. 2001. Why and how research ethics matters to you. Yes, you! Empirical Software Engineering 6: 287?290.","journal-title":"Empirical Software Engineering"},{"issue":"4","key":"CR39","doi-asserted-by":"crossref","first-page":"526","DOI":"10.1109\/32.799950","volume":"25","author":"A. Mayrhauser von","year":"1999","unstructured":"von Mayrhauser, A., and Lang, S. 1999. A coding scheme to support systematic analysis of software comprehension. IEEE Transactions on Software Engineering 25(4): 526?540.","journal-title":"IEEE Transactions on Software Engineering"},{"key":"CR40","volume-title":"Thought and Language","author":"L. S. Vygotsky","year":"1994","unstructured":"Vygotsky, L. S. 1994. Thought and Language. Cambridge, MA: MIT Press."},{"key":"CR41","unstructured":"Welland R., Sj\u00f8berg D., and Atkinson, M. 1997. Empirical analysis based on automatic tool logging. Empirical Assessment & Evaluation in Software Engineering (EASE97). Keele, UK."},{"key":"CR42","volume-title":"Experimentation in Software Engineering?An Introduction","author":"C. Wohlin","year":"1999","unstructured":"Wohlin, C., Runeson, P., H\u00f6st, M., Ohlsson, M. C., Regnell, B., and Wessl\u00e9n, A. 1999. Experimentation in Software Engineering?An Introduction. Boston: Kluwer Academic Publishers."},{"key":"CR43","doi-asserted-by":"crossref","unstructured":"Wright, R. B., and Converse, S. A. 1992. Method bias and concurrent verbal protocol in software usability testing. Proc Human Factors Society 36th Annual Meeting. Atlanta, Georgia, pp. 1220?1224.","DOI":"10.1177\/154193129203601608"},{"key":"CR44","volume-title":"Case Study Research","author":"R. K. Yin","year":"1994","unstructured":"Yin, R. K. 1994. Case Study Research. Thousand Oaks, California: SAGE Publications."}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-004-6189-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10664-004-6189-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-004-6189-4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,7,5]],"date-time":"2021-07-05T02:31:36Z","timestamp":1625452296000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s10664-004-6189-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2005,4]]},"references-count":44,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2005,4]]}},"alternative-id":["6189"],"URL":"https:\/\/doi.org\/10.1007\/s10664-004-6189-4","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2005,4]]}}}