{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,13]],"date-time":"2026-05-13T21:06:28Z","timestamp":1778706388727,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":50,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,6,1]],"date-time":"2016-06-01T00:00:00Z","timestamp":1464739200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,6]]},"DOI":"10.1145\/2915970.2916008","type":"proceedings-article","created":{"date-parts":[[2016,5,24]],"date-time":"2016-05-24T22:58:31Z","timestamp":1464130711000},"page":"1-6","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":150,"title":["The need for multivocal literature reviews in software engineering"],"prefix":"10.1145","author":[{"given":"Vahid","family":"Garousi","sequence":"first","affiliation":[{"name":"Hacettepe University, Ankara, Turkey"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Michael","family":"Felderer","sequence":"additional","affiliation":[{"name":"University of Innsbruck, Austria"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Mika V.","family":"M\u00e4ntyl\u00e4","sequence":"additional","affiliation":[{"name":"University of Oulu, Oulu, Finland"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"320","published-online":{"date-parts":[[2016,6]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2008.09.009"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.03.007"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.04.001"},{"key":"e_1_3_2_1_4_1","unstructured":"R. L. Glass and T. DeMarco Software Creativity 2.0: developer.* Books 2006.   R. L. Glass and T. DeMarco Software Creativity 2.0 : developer.* Books 2006."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.3102\/00346543061003265"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.3102\/00346543061003287"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"crossref","unstructured":"W. F. Whyte Participatory Action Research SAGE Publications 1990.  W. F. Whyte Participatory Action Research SAGE Publications 1990.","DOI":"10.4135\/9781412985383"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.3102\/00346543061003293"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1741-6787.2006.00051.x"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1002\/jrsm.1106"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"crossref","unstructured":"S. Hopewell M. Clarke and S. Mallett \"Grey literature and systematic reviews \" in Publication Bias in Meta-Analysis: Prevention Assessment and Adjustments H. R. Rothstein A. J. Sutton and M. Borenstein Eds. ed: John Wiley & Sons 2006.  S. Hopewell M. Clarke and S. Mallett \"Grey literature and systematic reviews \" in Publication Bias in Meta-Analysis: Prevention Assessment and Adjustments H. R. Rothstein A. J. Sutton and M. Borenstein Eds. ed: John Wiley & Sons 2006.","DOI":"10.1002\/0470870168.ch4"},{"key":"e_1_3_2_1_12_1","article-title":"Grey literature in meta-analyses of randomized trials of health care interventions","author":"H. S, M. S, C. M","year":"2007","journal-title":"Cochrane Database Systematic Reviews"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2012.12.052"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"crossref","unstructured":"I. Kulesovs \"iOS Applications Testing \" vol. 3 pp. 138--150 2015.  I. Kulesovs \"iOS Applications Testing \" vol. 3 pp. 138--150 2015.","DOI":"10.17770\/etr2015vol3.187"},{"key":"e_1_3_2_1_15_1","first-page":"1","article-title":"A Systematic Literature Review of Software Process Improvement in Small and Medium Web Companies,\" in Advances in Software Engineering. vol. 59, D. \u015al\u0119zak, T.-h. Kim, A. Kiumi, T. Jiang, J. Verner, and S. Abrah\u00e3o, Eds., ed","author":"Sulayman M.","year":"2009","journal-title":"Springer Berlin Heidelberg"},{"key":"e_1_3_2_1_16_1","volume-title":"Sweden","author":"Yasin A.","year":"2012"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2013.03.004"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2015.02.005"},{"key":"e_1_3_2_1_19_1","first-page":"16","article-title":"A Consolidated Understanding of Technical debt","author":"Tom E.","year":"2012","journal-title":"European Conference on Information Systems"},{"key":"e_1_3_2_1_20_1","unstructured":"D. Kovalenko Selenium Design Patterns and Best Practices: Packt Publishing Ltd 2014.   D. Kovalenko Selenium Design Patterns and Best Practices : Packt Publishing Ltd 2014."},{"key":"e_1_3_2_1_21_1","unstructured":"N. Garg Test Automation Using Selenium Webdriver with Java: Step by Step Guide: AdactIn Group Pty 2014.   N. Garg Test Automation Using Selenium Webdriver with Java: Step by Step Guide : AdactIn Group Pty 2014."},{"key":"e_1_3_2_1_22_1","unstructured":"D. Dary \"GTAC 2014: Selendroid - Selenium for Android \" https:\/\/www.youtube.com\/watch?v=gIPxxF4lhGo Last accessed: Dec. 2015.  D. Dary \"GTAC 2014: Selendroid - Selenium for Android \" https:\/\/www.youtube.com\/watch?v=gIPxxF4lhGo Last accessed: Dec. 2015."},{"key":"e_1_3_2_1_23_1","unstructured":"J. Kaasila \"GTAC 2015: Mobile Game Test Automation Using Real Devices \" https:\/\/www.youtube.com\/watch?v=WFBfRk-GLRo Last accessed: Dec. 2015.  J. Kaasila \"GTAC 2015: Mobile Game Test Automation Using Real Devices \" https:\/\/www.youtube.com\/watch?v=WFBfRk-GLRo Last accessed: Dec. 2015."},{"key":"e_1_3_2_1_24_1","unstructured":"G. Zhu and A. Momtaz \"GTAC 2013: Android UI Automation \" https:\/\/www.youtube.com\/watch?v=O1u8iBLUFL0 Last accessed: Dec. 2015.  G. Zhu and A. Momtaz \"GTAC 2013: Android UI Automation \" https:\/\/www.youtube.com\/watch?v=O1u8iBLUFL0 Last accessed: Dec. 2015."},{"key":"e_1_3_2_1_25_1","unstructured":"S. Anantharamiah \"Simplify GUI test automation by using a labor-saving design pattern \" http:\/\/www.ibm.com\/developerworks\/rational\/library\/10\/simplify-gui-test-automation-using-a-design-pattern\/ 2010 Last accessed: Dec. 2015.  S. Anantharamiah \"Simplify GUI test automation by using a labor-saving design pattern \" http:\/\/www.ibm.com\/developerworks\/rational\/library\/10\/simplify-gui-test-automation-using-a-design-pattern\/ 2010 Last accessed: Dec. 2015."},{"key":"e_1_3_2_1_26_1","unstructured":"D. Nicolette Software Development Metrics: Manning Publications 2015.  D. Nicolette Software Development Metrics : Manning Publications 2015."},{"key":"e_1_3_2_1_27_1","unstructured":"D. S. Vacanti Actionable Agile Metrics for Predictability: An Introduction: Leanpub 2015.  D. S. Vacanti Actionable Agile Metrics for Predictability: An Introduction : Leanpub 2015."},{"key":"e_1_3_2_1_28_1","unstructured":"A. Maurya Scaling Lean: Mastering the Key Metrics for Startup Growth: Portfolio 2016.  A. Maurya Scaling Lean: Mastering the Key Metrics for Startup Growth : Portfolio 2016."},{"key":"e_1_3_2_1_29_1","unstructured":"Scaled Agile Inc. (2015 Last accessed: Jan. 2016). Metrics for Scaled Agile framework. Available: http:\/\/scaledagileframework.com\/metrics\/  Scaled Agile Inc. (2015 Last accessed: Jan. 2016). Metrics for Scaled Agile framework . Available: http:\/\/scaledagileframework.com\/metrics\/"},{"key":"e_1_3_2_1_30_1","unstructured":"CA Technologies. (2015 Last accessed: Jan. 2016). Agile Reports and Metrics. Available: https:\/\/www.rallydev.com\/product-feature\/reporting-analytics  CA Technologies. (2015 Last accessed: Jan. 2016). Agile Reports and Metrics . Available: https:\/\/www.rallydev.com\/product-feature\/reporting-analytics"},{"key":"e_1_3_2_1_31_1","unstructured":"S. Pillai. (2015 Last accessed: Jan. 2016). Agile Project Reporting and Metrics. Available: https:\/\/www.scrumalliance.org\/community\/articles\/2013\/july\/agile-project-reporting-and-metrics  S. Pillai. (2015 Last accessed: Jan. 2016). Agile Project Reporting and Metrics . Available: https:\/\/www.scrumalliance.org\/community\/articles\/2013\/july\/agile-project-reporting-and-metrics"},{"key":"e_1_3_2_1_32_1","unstructured":"TechEd North America. (Last accessed: Jan. 2016). From Vanity to Value Metrics That Matter: Improving Lean and Agile Kanban and Scrum. Available: https:\/\/www.youtube.com\/watch?v=Y8ymtdLiBnA  TechEd North America. (Last accessed: Jan. 2016). From Vanity to Value Metrics That Matter: Improving Lean and Agile Kanban and Scrum . Available: https:\/\/www.youtube.com\/watch?v=Y8ymtdLiBnA"},{"key":"e_1_3_2_1_33_1","unstructured":"M. Levison. (2009 Last accessed: Jan. 2016). What is a Good Agile Metric? Available: http:\/\/www.infoq.com\/news\/2009\/11\/good-agile-metrics  M. Levison. (2009 Last accessed: Jan. 2016). What is a Good Agile Metric? Available: http:\/\/www.infoq.com\/news\/2009\/11\/good-agile-metrics"},{"key":"e_1_3_2_1_34_1","unstructured":"W. Hayes. (2014 Last accessed: Jan. 2016). Agile Metrics: Seven Categories. Available: https:\/\/insights.sei.cmu.edu\/sei_blog\/2014\/09\/agile-metrics-seven-categories.html  W. Hayes. (2014 Last accessed: Jan. 2016). Agile Metrics: Seven Categories . Available: https:\/\/insights.sei.cmu.edu\/sei_blog\/2014\/09\/agile-metrics-seven-categories.html"},{"key":"e_1_3_2_1_35_1","unstructured":"K. Schwaber and J. Sutherland The scrum guide: Scrum.org 2013.  K. Schwaber and J. Sutherland The scrum guide : Scrum.org 2013."},{"key":"e_1_3_2_1_36_1","unstructured":"K. Beck and C. Andres Extreme programming explained: embrace change: Addison-Wesley Professional 2004.   K. Beck and C. Andres Extreme programming explained: embrace change : Addison-Wesley Professional 2004."},{"key":"e_1_3_2_1_37_1","unstructured":"D. J. Anderson Kanban: Blue Hole Press 2010.  D. J. Anderson Kanban : Blue Hole Press 2010."},{"key":"e_1_3_2_1_38_1","unstructured":"M. Poppendieck and T. Poppendieck Lean software development: An agile toolkit: Addison-Wesley Professional 2003.   M. Poppendieck and T. Poppendieck Lean software development: An agile toolkit : Addison-Wesley Professional 2003."},{"key":"e_1_3_2_1_39_1","unstructured":"C. Sterling Managing software debt: building for inevitable change: Addison-Wesley Professional 2010.   C. Sterling Managing software debt: building for inevitable change : Addison-Wesley Professional 2010."},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"crossref","unstructured":"G. Suryanarayana G. Samarthyam and T. Sharma Refactoring for Software Design Smells: Managing Technical Debt: Morgan Kaufmann 2014.   G. Suryanarayana G. Samarthyam and T. Sharma Refactoring for Software Design Smells: Managing Technical Debt : Morgan Kaufmann 2014.","DOI":"10.1016\/B978-0-12-801397-7.00001-1"},{"key":"e_1_3_2_1_41_1","unstructured":"J. Shore. (2004 Last accessed: Jan. 2016). Design Debt. Available: http:\/\/www.jamesshore.com\/Articles\/Business\/Software%20Profitability%20Newsletter\/Design%20Debt.html  J. Shore. (2004 Last accessed: Jan. 2016). Design Debt . Available: http:\/\/www.jamesshore.com\/Articles\/Business\/Software%20Profitability%20Newsletter\/Design%20Debt.html"},{"key":"e_1_3_2_1_42_1","unstructured":"N. Zakharenko. (2015 Last accessed: Jan. 2016). Technical Debt. Available: https:\/\/www.youtube.com\/watch?v=JKYktDRoRxw  N. Zakharenko. (2015 Last accessed: Jan. 2016). Technical Debt . Available: https:\/\/www.youtube.com\/watch?v=JKYktDRoRxw"},{"key":"e_1_3_2_1_43_1","unstructured":"M. Oak \"How to Minimize Technical Debt? \" 2015.  M. Oak \"How to Minimize Technical Debt? \" 2015."},{"key":"e_1_3_2_1_44_1","unstructured":"D. Racodon. (2015 Last accessed: Jan. 2016). Technical Debt. Available: http:\/\/docs.sonarqube.org\/display\/SONAR\/Technical+Debt  D. Racodon. (2015 Last accessed: Jan. 2016). Technical Debt . Available: http:\/\/docs.sonarqube.org\/display\/SONAR\/Technical+Debt"},{"key":"e_1_3_2_1_45_1","unstructured":"C. Wohlin \"Software Engineering Research under the Lamppost \" in Proceedings of the International Joint Conference on Software Technologies 2013.  C. Wohlin \"Software Engineering Research under the Lamppost \" in Proceedings of the International Joint Conference on Software Technologies 2013."},{"key":"e_1_3_2_1_46_1","unstructured":"V. Garousi and M. V. M\u00e4ntyl\u00e4 \"Online Paper Repository for the SLR on 'When and What to Automate in Software Testing?' \" in http:\/\/goo.gl\/zwY1sj Last accessed: Sept. 2015.  V. Garousi and M. V. M\u00e4ntyl\u00e4 \"Online Paper Repository for the SLR on 'When and What to Automate in Software Testing?' \" in http:\/\/goo.gl\/zwY1sj Last accessed: Sept. 2015."},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2014.12.027"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2011.12.003"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1109\/ESEM.2009.5316010"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"crossref","unstructured":"C. Wohlin P. Runeson M. H\u00f6st M. C. Ohlsson B. Regnell and A. Wessl\u00e9n Experimentation in Software Engineering: An Introduction: Kluwer Academic Publishers 2000.   C. Wohlin P. Runeson M. H\u00f6st M. C. Ohlsson B. Regnell and A. Wessl\u00e9n Experimentation in Software Engineering: An Introduction : Kluwer Academic Publishers 2000.","DOI":"10.1007\/978-1-4615-4625-2"}],"event":{"name":"EASE '16: 20th International Conference on Evaluation and Assessment in Software Engineering","location":"Limerick Ireland","acronym":"EASE '16"},"container-title":["Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2915970.2916008","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2915970.2916008","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:56:19Z","timestamp":1750222579000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2915970.2916008"}},"subtitle":["complementing systematic literature reviews with grey literature"],"short-title":[],"issued":{"date-parts":[[2016,6]]},"references-count":50,"alternative-id":["10.1145\/2915970.2916008","10.1145\/2915970"],"URL":"https:\/\/doi.org\/10.1145\/2915970.2916008","relation":{},"subject":[],"published":{"date-parts":[[2016,6]]},"assertion":[{"value":"2016-06-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}