{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,8]],"date-time":"2026-07-08T23:38:01Z","timestamp":1783553881766,"version":"3.55.0"},"reference-count":57,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2021,5,8]],"date-time":"2021-05-08T00:00:00Z","timestamp":1620432000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,5,8]],"date-time":"2021-05-08T00:00:00Z","timestamp":1620432000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100007523","name":"Division of Advanced Cyberinfrastructure","doi-asserted-by":"publisher","award":["OAC-1640864"],"award-info":[{"award-number":["OAC-1640864"]}],"id":[{"id":"10.13039\/100007523","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100004586","name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004586","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NYU Moore-Sloan Data Science Environment"},{"DOI":"10.13039\/501100004586","name":"Funda\u00e7\u00e3o Carlos Chagas Filho de Amparo \u00e0 Pesquisa do Estado do Rio de Janeiro","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100004586","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002322","name":"Coordena\u00e7\u00e3o de Aperfei\u00e7oamento de Pessoal de N\u00edvel Superior","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002322","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003593","name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100003593","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2021,7]]},"DOI":"10.1007\/s10664-021-09961-9","type":"journal-article","created":{"date-parts":[[2021,5,8]],"date-time":"2021-05-08T11:03:35Z","timestamp":1620471815000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":77,"title":["Understanding and improving the quality and reproducibility of Jupyter notebooks"],"prefix":"10.1007","volume":"26","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6680-7470","authenticated-orcid":false,"given":"Jo\u00e3o Felipe","family":"Pimentel","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5173-1247","authenticated-orcid":false,"given":"Leonardo","family":"Murta","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1184-8192","authenticated-orcid":false,"given":"Vanessa","family":"Braganholo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3915-7075","authenticated-orcid":false,"given":"Juliana","family":"Freire","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,5,8]]},"reference":[{"key":"9961_CR1","unstructured":"Agrawal R, Srikant R, et al. (1994) Fast algorithms for mining association rules. In: VLDB conference, VLDB, vol 1215, pp 487\u2013499"},{"key":"9961_CR2","unstructured":"Anaconda (2018) Anaconda software distribution. https:\/\/www.anaconda.com. Accessed: 2019-10-01"},{"issue":"1","key":"9961_CR3","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1007\/s10664-014-9350-8","volume":"21","author":"V Arnaoudova","year":"2016","unstructured":"Arnaoudova V, Di Penta M, Antoniol G (2016) Linguistic antipatterns: what they are and how developers perceive them. Empir Softw Eng 21 (1):104\u2013158","journal-title":"Empir Softw Eng"},{"issue":"6","key":"9961_CR4","doi-asserted-by":"publisher","first-page":"574","DOI":"10.1080\/10447310802205776","volume":"24","author":"A Bangor","year":"2008","unstructured":"Bangor A, Kortum P T, Miller J T (2008) An empirical evaluation of the system usability scale. Int J Hum\u2013Comput Interact 24(6):574\u2013594","journal-title":"Int J Hum\u2013Comput Interact"},{"issue":"8\u201312","key":"9961_CR5","first-page":"57","volume":"2003","author":"J Benedek","year":"2002","unstructured":"Benedek J, Miner T (2002) Measuring desirability: new methods for evaluating desirability in a usability lab setting. Proc Usabil Prof Assoc 2003(8\u201312):57","journal-title":"Proc Usabil Prof Assoc"},{"key":"9961_CR6","unstructured":"Brachmann M, Spoth W, Kennedy O, Glavic B, Mueller H, Castelo S, Bautista C, Freire J (2020) Your notebook is not crumby enough, replace it. In: Conference on innovative data systems research, CIDR, pp 1\u201316"},{"key":"9961_CR7","unstructured":"Brooke J (1996) Sus: a \u201cquick and dirty\u201d usability. Usability Evaluation in Industry, p 189"},{"key":"9961_CR8","unstructured":"Burns T, Ward G (2013) ipython-nose. https:\/\/github.com\/taavi\/ipython_nose. Accessed: 2019-10-01"},{"key":"9961_CR9","unstructured":"Cannon B, Smith N, Stufft D (2016) Pep 518: specifying minimum build system requirements for python projects. https:\/\/www.python.org\/dev\/peps\/pep-0518\/. Accessed: 22 Sep 2020"},{"key":"9961_CR10","doi-asserted-by":"crossref","unstructured":"Chirigati F, Rampin R, Shasha D, Freire J (2016) Reprozip: computational reproducibility with ease. In: International conference on management of data, ACM, SIGMOD, pp 2085\u20132088","DOI":"10.1145\/2882903.2899401"},{"key":"9961_CR11","unstructured":"Collberg C, Proebsting T, Moraila G, Shankaran A, Shi Z, Warren A M (2014) Measuring reproducibility in computer systems research. Tech rep. Department of Computer Science, University of Arizona"},{"key":"9961_CR12","unstructured":"Danilak MM (2016) langdetect. https:\/\/pypi.org\/project\/langdetect\/. Accessed: 2019-10-02"},{"issue":"3","key":"9961_CR13","doi-asserted-by":"publisher","first-page":"11","DOI":"10.1109\/MCSE.2008.79","volume":"10","author":"J Freire","year":"2008","unstructured":"Freire J, Koop D, Santos E, Silva C (2008) Provenance for computational tasks: a survey. Comput Sci Eng 10(3):11\u201321. https:\/\/doi.org\/10.1109\/MCSE.2008.79","journal-title":"Comput Sci Eng"},{"key":"9961_CR14","doi-asserted-by":"publisher","first-page":"52","DOI":"10.1016\/j.jss.2017.12.013","volume":"138","author":"V Garousi","year":"2018","unstructured":"Garousi V, K\u00fc\u00e7\u00fck B (2018) Smells in software test code: a survey of knowledge in industry and academia. J Syst Softw 138:52\u201381","journal-title":"J Syst Softw"},{"key":"9961_CR15","unstructured":"Grus J (2018) I don\u2019t like notebooks. https:\/\/conferences.oreilly.com\/jupyter\/jup-ny\/public\/schedule\/detail\/68282, jupyterCon"},{"key":"9961_CR16","unstructured":"Han J, Pei J, Kamber M (2011) Data mining: concepts and techniques. Elsevier"},{"key":"9961_CR17","doi-asserted-by":"publisher","unstructured":"Hook D, Kelly D (2009) Testing for trustworthiness in scientific software. In: ICSE workshop on software engineering for computational science and engineering, SE-CSE. https:\/\/doi.org\/10.1109\/SECSE.2009.5069163, pp 59\u201364","DOI":"10.1109\/SECSE.2009.5069163"},{"key":"9961_CR18","doi-asserted-by":"crossref","unstructured":"Horwitz S, Reps T (1992) The use of program dependence graphs in software engineering. In: International conference on software engineering, ACM, ICSE, pp 392\u2013411","DOI":"10.1109\/ICSE.1992.753516"},{"key":"9961_CR19","unstructured":"H\u00fcrsch W L, Lopes C V (1995) Separation of concerns. Tech. rep., Northeastern University"},{"key":"9961_CR20","unstructured":"Israel G D (1992) Determining sample size. Tech. rep., University of Florida"},{"key":"9961_CR21","doi-asserted-by":"crossref","unstructured":"K\u00e4ll\u00e9n M, Sigvardsson U, Wrigstad T (2020) Jupyter notebooks on github: characteristics and code clones. arXiv:200710146","DOI":"10.22152\/programming-journal.org\/2021\/5\/15"},{"key":"9961_CR22","doi-asserted-by":"crossref","unstructured":"Kalliamvakou E, Gousios G, Blincoe K, Singer L, German D M, Damian D (2014) The promises and perils of mining github. In: Working conference on mining software repositories. ACM, MSR, pp 92\u2013101","DOI":"10.1145\/2597073.2597074"},{"key":"9961_CR23","doi-asserted-by":"publisher","unstructured":"Kery MB, Radensky M, Arya M, John BE, Myers BA (2018) The story in the notebook: exploratory data science using a literate programming tool. In: CHI conference on human factors in computing systems. https:\/\/doi.org\/10.1145\/3173574.3173748. ACM, CHI, New York, pp 174:1\u2013174:11","DOI":"10.1145\/3173574.3173748"},{"key":"9961_CR24","doi-asserted-by":"crossref","unstructured":"Kluyver T, Ragan-Kelley B, P\u00e9rez F, Granger BE, Bussonnier M, Frederic J, Kelley K, Hamrick JB, Grout J, Corlay S et al (2016) Jupyter notebooks\u2014a publishing format for reproducible computational workflows. In: Loizides F, Scmidt B (eds) International conference on electronic publishing. https:\/\/eprints.soton.ac.uk\/403913\/. IOS Press, ELPUB, G\u00f6ttingen, pp 87\u201390","DOI":"10.3233\/978-1-61499-649-1-87"},{"issue":"2","key":"9961_CR25","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1093\/comjnl\/27.2.97","volume":"27","author":"DE Knuth","year":"1984","unstructured":"Knuth D E (1984) Literate programming. Comput J 27(2):97\u2013111","journal-title":"Comput J"},{"key":"9961_CR26","doi-asserted-by":"crossref","unstructured":"Koenzen A, Ernst N, Storey MA (2020) Code duplication and reuse in jupyter notebooks. arXiv:200513709","DOI":"10.1109\/VL\/HCC50065.2020.9127202"},{"key":"9961_CR27","unstructured":"Koop D, Patel J (2017) Dataflow notebooks: encoding and tracking dependencies of cells. In: Workshop on the theory and practice of provenance. USENIX, TaPP, Seattle, pp 1\u20137"},{"key":"9961_CR28","unstructured":"Krassowski M (2019) Language server protocol integration for jupyter(lab). https:\/\/github.com\/krassowski\/jupyterlab-lsp\/, accessed: 2020-10-13"},{"key":"9961_CR29","unstructured":"Lewine D (1991) POSIX programmers guide. O\u2019Reilly Media, Inc."},{"key":"9961_CR30","unstructured":"McNutt A (2019) Jupyterlab-flake8. https:\/\/github.com\/mlshapiro\/jupyterlab-flake8. Accessed: 2020-10-13"},{"key":"9961_CR31","unstructured":"Microsoft (2018) Naming files, paths, and namespaces. Windows Dev Center. https:\/\/docs.microsoft.com\/en-us\/windows\/desktop\/FileIO\/naming-a-File. Accessed: 2019-10-01"},{"key":"9961_CR32","unstructured":"Mueller A (2018) 5 reasons why jupyter notebooks suck. https:\/\/towardsdatascience.com\/5-reasons-why-jupyter-notebooks-suck-4dc201e27086\/, accessed: 2019-10-01"},{"issue":"6","key":"9961_CR33","doi-asserted-by":"publisher","first-page":"3219","DOI":"10.1007\/s10664-017-9512-6","volume":"22","author":"N Munaiah","year":"2017","unstructured":"Munaiah N, Kroh S, Cabrey C, Nagappan M (2017) Curating github for engineered software projects. Empir Softw Eng 22(6):3219\u20133253","journal-title":"Empir Softw Eng"},{"key":"9961_CR34","volume-title":"The art of software testing, vol 2","author":"GJ Myers","year":"2004","unstructured":"Myers GJ, Badgett T, Thomas TM, Sandler C (2004) The art of software testing, vol 2. Wiley Online Library, Hoboken"},{"key":"9961_CR35","unstructured":"Neglectos (2018) A preliminary analysis on the use of python notebooks. https:\/\/blog.bitergia.com\/2018\/04\/02\/a-preliminary-analysis-on-the-use-of-python-notebooks\/, Accessed: 2019-10-01"},{"key":"9961_CR36","unstructured":"Parente P (2020) nbestimate. https:\/\/nbviewer.jupyter.org\/github\/parente\/nbestimate\/blob\/master\/estimate.ipynb, accessed: 2020-12-03"},{"issue":"3","key":"9961_CR37","doi-asserted-by":"publisher","first-page":"21","DOI":"10.1109\/MCSE.2007.53","volume":"9","author":"F P\u00e9rez","year":"2007","unstructured":"P\u00e9rez F, Granger BE (2007) Ipython: a system for interactive scientific computing. Comput Sci Eng 9(3):21\u201329","journal-title":"Comput Sci Eng"},{"key":"9961_CR38","unstructured":"Pimentel JF (2016) ipython-unittest. https:\/\/github.com\/JoaoFelipe\/ipython-unittest. Accessed: 2019-10-01"},{"key":"9961_CR39","unstructured":"Pimentel J F N, Braganholo V, Murta L, Freire J (2015) Collecting and analyzing provenance on interactive notebooks: when ipython meets noworkflow. In: Workshop on the theory and practice of provenance. USENIX, TaPP, Edinburgh, pp 155\u2013167"},{"issue":"3","key":"9961_CR40","doi-asserted-by":"publisher","first-page":"47:1","DOI":"10.1145\/3311955","volume":"52","author":"JF Pimentel","year":"2019","unstructured":"Pimentel JF, Freire J, Murta L, Braganholo V (2019a) A survey on collecting, managing, and analyzing provenance from scripts. ACM Comput Surv 52 (3):47:1\u201347:38. https:\/\/doi.org\/10.1145\/3311955. http:\/\/doi.acm.org\/10.1145\/3311955","journal-title":"ACM Comput Surv"},{"key":"9961_CR41","doi-asserted-by":"crossref","unstructured":"Pimentel J F, Murta L, Braganholo V, Freire J (2019b) A large-scale study about quality and reproducibility of jupyter notebooks. In: Proceedings of the international conference on mining software repositories. IEEE Press, MSR, pp 507\u2013517","DOI":"10.1109\/MSR.2019.00077"},{"key":"9961_CR42","unstructured":"Pomogajko K (2015) Why I don\u2019t like Jupyter (FKA IPython Notebook). https:\/\/yihui.name\/en\/2018\/09\/notebook-war\/. Accessed: 2019-10-01"},{"key":"9961_CR43","unstructured":"PyPA (2020) Python Packaging Documentation: install_requires vs requirements files. https:\/\/packaging.python.org\/discussions\/install-requires-vs-requirements\/#requirements-files. Accessed: 2020-10-08"},{"key":"9961_CR44","unstructured":"Python-Wiki (2019) Python testing tools taxonomy. https:\/\/wiki.python.org\/moin\/PythonTestingToolsTaxonomy. Accessed: 2019-10-01"},{"key":"9961_CR45","unstructured":"ReproZip (2017) Making Jupyter Notebooks Reproducible with ReproZip. https:\/\/docs.reprozip.org\/en\/1.0.x\/jupyter.html. Accessed: 2019-10-01"},{"key":"9961_CR46","doi-asserted-by":"publisher","unstructured":"Rule A, Tabard A, Hollan JD (2018) Exploration and explanation in computational notebooks. In: Proceedings of the CHI conference on human factors in computing systems. https:\/\/doi.org\/10.1145\/3173574.3173606. http:\/\/doi.acm.org\/10.1145\/3173574.3173606. ACM, CHI, New York, pp 32:1\u201332:12","DOI":"10.1145\/3173574.3173606"},{"key":"9961_CR47","unstructured":"Samuel S, K\u00f6nig-Ries B (2018) Provbook: provenance-based semantic enrichment of interactive notebooks for reproducibility. In: The international semantic web conference, ISWC. Monterey, Springer, pp 1\u20134"},{"key":"9961_CR48","doi-asserted-by":"crossref","unstructured":"Samuel S, K\u00f6nig-Ries B (2020) Reproducemegit: a visualization tool for analyzing reproducibility of jupyter notebooks. In: ProvenanceWeek, pp 1\u20132","DOI":"10.1007\/978-3-030-80960-7_12"},{"issue":"7525","key":"9961_CR49","doi-asserted-by":"publisher","first-page":"151","DOI":"10.1038\/515151a","volume":"515","author":"H Shen","year":"2014","unstructured":"Shen H (2014) Interactive notebooks: sharing the code. Nature News 515(7525):151","journal-title":"Nature News"},{"key":"9961_CR50","unstructured":"Staley T (2017) Making git and jupyter notebooks play nice. http:\/\/timstaley.co.uk\/posts\/making-git-and-jupyter-notebooks-play-nice\/. Accessed: 2019-10-01"},{"key":"9961_CR51","unstructured":"Tim, Doorknob (2014) Is space not allowed in a filename? Unix & Linux. https:\/\/unix.stackexchange.com\/q\/148043, accessed: 2019-10-01"},{"issue":"4","key":"9961_CR52","first-page":"e1838","volume":"29","author":"M Tufano","year":"2017","unstructured":"Tufano M, Palomba F, Bavota G, Di Penta M, Oliveto R, De Lucia A, Poshyvanyk D (2017) There and back again: can you compile that snapshot? J Softw: Evol Process 29(4):e1838","journal-title":"J Softw: Evol Process"},{"key":"9961_CR53","unstructured":"Udacity (2017) Deep learning nanodegree foundation. https:\/\/github.com\/udacity\/deep-learning. Accessed: 2019-10-01"},{"key":"9961_CR54","unstructured":"van Rossum G, Warsaw B, Coghlan N (2001) Pep 8: style guide for python code. https:\/\/www.python.org\/dev\/peps\/pep-0008\/. Accessed: 2019-10-01"},{"issue":"2","key":"9961_CR55","first-page":"11","volume":"1","author":"N Vavrov\u00e1","year":"2017","unstructured":"Vavrov\u00e1 N, Zaytsev V (2017) Does python smell like java? The Art. Sci Eng Program 1(2):11\u20131","journal-title":"Sci Eng Program"},{"key":"9961_CR56","doi-asserted-by":"crossref","unstructured":"Wang J, Li L, Zeller A (2020) Better code, better sharing: on the need of analyzing jupyter notebooks. In: International conference on software engineering: new ideas and emerging results, ICSE, pp 53\u201356","DOI":"10.1145\/3377816.3381724"},{"issue":"1","key":"9961_CR57","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1371\/journal.pbio.1001745","volume":"12","author":"G Wilson","year":"2014","unstructured":"Wilson G, Aruliah DA, Brown CT, Chue Hong NP, Davis M, Guy RT, Haddock SHD, Huff KD, Mitchell IM, Plumbley MD, Waugh B, White EP, Wilson P (2014) Best practices for scientific computing. PLOS Biol 12(1):1\u20137. https:\/\/doi.org\/10.1371\/journal.pbio.1001745","journal-title":"PLOS Biol"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-021-09961-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-021-09961-9\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-021-09961-9.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,9]],"date-time":"2025-04-09T09:00:41Z","timestamp":1744189241000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-021-09961-9"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,8]]},"references-count":57,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2021,7]]}},"alternative-id":["9961"],"URL":"https:\/\/doi.org\/10.1007\/s10664-021-09961-9","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,5,8]]},"assertion":[{"value":"19 March 2021","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 May 2021","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}],"article-number":"65"}}