diff --git a/evaluation/badges.qmd b/evaluation/badges.qmd index 2ffe6f6..29e28e0 100644 --- a/evaluation/badges.qmd +++ b/evaluation/badges.qmd @@ -8,7 +8,7 @@ bibliography: ../quarto_site/references.bib This page evaluates the extent to which the author-published research artefacts meet the criteria of badges related to reproducibility from various organisations and journals. -*Caveat: Please note that these criteria are based on available information about each badge online, and that we have likely differences in our procedure (e.g. allowed troubleshooting for execution and reproduction, not under tight time pressure to complete). Moreover, we focus only on reproduction of the discrete-event simulation, and not on other aspects of the article. We cannot guarantee that the badges below would have been awarded in practice by these journals.* +*Caveat: Please note that these criteria are based on available information about each badge online. Moreover, we focus only on reproduction of the discrete-event simulation, and not on other aspects of the article. We cannot guarantee that the badges below would have been awarded in practice by these journals.* ## Criteria @@ -19,35 +19,34 @@ import pandas as pd # Criteria and their definitions criteria = { - 'archive': 'Stored in a permanent archive that is publicly and openly accessible', - 'id': 'Has a persistent identifier', - 'license': 'Includes an open license', - 'relevant': '''Artefacts are relevant to and contribute to the article's results''', - 'complete': 'Complete set of materials shared (as would be needed to fully reproduce article)', - 'structure': 'Artefacts are well structured/organised (e.g. to the extent that reuse and repurposing is facilitated, adhering to norms and standards of research community)', - 'documentation_sufficient': 'Artefacts are sufficiently documented (i.e. to understand how it works, to enable it to be run, including package versions)', - 'documentation_careful': 'Artefacts are carefully documented (more than sufficient - i.e. to the extent that reuse and repurposing is facilitated - e.g. changing parameters, reusing for own purpose)', - # This criteria is kept seperate to documentation_careful, as it specifically requires a README file - 'documentation_readme': 'Artefacts are clearly documented and accompanied by a README file with step-by-step instructions on how to reproduce results in the manuscript', + 'archive': 'Artefacts are archived in a repository that is: (a) public (b) guarantees persistence (c) gives a unique identifier (e.g. DOI)', + 'licence': 'Open licence', + 'complete': 'Complete (all relevant artefacts available)', + 'docs1': 'Documents (a) how code is used (b) how it relates to article (c) software, systems, packages and versions', + 'docs2': 'Documents (a) inventory of artefacts (b) sufficient description for artefacts to be exercised', + 'relevant': 'Artefacts relevant to paper', 'execute': 'Scripts can be successfully executed', - 'regenerated': 'Independent party regenerated results using the authors research artefacts', - 'hour': 'Reproduced within approximately one hour (excluding compute time)', + 'careful': 'Artefacts are carefully documented and well-structured to the extent that reuse and repurposing is facilitated, adhering to norms and standards', + 'reproduce': 'Reproduced results (assuming (a) acceptably similar (b) reasonable time frame (c) only minor troubleshooting)', + 'readme': 'README file with step-by-step instructions to run analysis', + 'dependencies': 'Dependencies (e.g. package versions) stated', + 'correspond': 'Clear how output of analysis corresponds to article' } # Evaluation for this study eval = pd.Series({ 'archive': 0, - 'id': 0, - 'license': 1, - 'relevant': 1, + 'licence': 1, 'complete': 0, - 'structure': 0, - 'documentation_sufficient': 0, - 'documentation_careful': 0, - 'documentation_readme': 0, + 'docs1': 0, + 'docs2': 0, + 'relevant': 1, 'execute': 1, - 'regenerated': 1, - 'hour': 0, + 'careful': 0, + 'reproduce': 0, + 'readme': 1, + 'dependencies': 0, + 'correspond': 0 }) # Get list of criteria met (True/False) overall @@ -82,10 +81,10 @@ def create_criteria_list(criteria_dict): return(formatted_list) # Define groups of criteria -criteria_share_how = ['archive', 'id', 'license'] -criteria_share_what = ['relevant', 'complete'] -criteria_doc_struc = ['structure', 'documentation_sufficient', 'documentation_careful', 'documentation_readme'] -criteria_run = ['execute', 'regenerated', 'hour'] +criteria_share_how = ['archive', 'licence'] +criteria_share_what = ['complete', 'relevant'] +criteria_doc_struc = ['docs1', 'docs2', 'careful', 'readme', 'dependencies', 'correspond'] +criteria_run = ['execute', 'reproduce'] # Create text section display(Markdown(f''' @@ -118,9 +117,9 @@ Criteria related to running and reproducing results - # Full badge names badge_names = { # Open objects + 'open_acm': 'ACM "Artifacts Available"', 'open_niso': 'NISO "Open Research Objects (ORO)"', 'open_niso_all': 'NISO "Open Research Objects - All (ORO-A)"', - 'open_acm': 'ACM "Artifacts Available"', 'open_cos': 'COS "Open Code"', 'open_ieee': 'IEEE "Code Available"', # Object review @@ -128,8 +127,8 @@ badge_names = { 'review_acm_reusable': 'ACM "Artifacts Evaluated - Reusable"', 'review_ieee': 'IEEE "Code Reviewed"', # Results reproduced - 'reproduce_niso': 'NISO "Results Reproduced (ROR-R)"', 'reproduce_acm': 'ACM "Results Reproduced"', + 'reproduce_niso': 'NISO "Results Reproduced (ROR-R)"', 'reproduce_ieee': 'IEEE "Code Reproducible"', 'reproduce_psy': 'Psychological Science "Computational Reproducibility"' } @@ -137,20 +136,20 @@ badge_names = { # Criteria required by each badge badges = { # Open objects - 'open_niso': ['archive', 'id', 'license'], - 'open_niso_all': ['archive', 'id', 'license', 'complete'], - 'open_acm': ['archive', 'id'], - 'open_cos': ['archive', 'id', 'license', 'complete', 'documentation_sufficient'], + 'open_acm': ['archive'], + 'open_niso': ['archive', 'licence'], + 'open_niso_all': ['archive', 'licence', 'complete'], + 'open_cos': ['archive', 'licence', 'docs1'], 'open_ieee': ['complete'], # Object review - 'review_acm_functional': ['documentation_sufficient', 'relevant', 'complete', 'execute'], - 'review_acm_reusable': ['documentation_sufficient', 'documentation_careful', 'relevant', 'complete', 'execute', 'structure'], + 'review_acm_functional': ['docs2', 'relevant', 'complete', 'execute'], + 'review_acm_reusable': ['docs2', 'relevant', 'complete', 'execute', 'careful'], 'review_ieee': ['complete', 'execute'], # Results reproduced - 'reproduce_niso': ['regenerated'], - 'reproduce_acm': ['regenerated'], - 'reproduce_ieee': ['regenerated'], - 'reproduce_psy': ['regenerated', 'hour', 'structure', 'documentation_readme'], + 'reproduce_acm': ['reproduce'], + 'reproduce_niso': ['reproduce'], + 'reproduce_ieee': ['reproduce'], + 'reproduce_psy': ['reproduce', 'readme', 'dependencies', 'correspond'] } # Identify which badges would be awarded based on criteria @@ -256,12 +255,12 @@ create_badge_callout({k: v for (k, v) in award.items() if k.startswith('reproduc * "Open Code" -**Institute of Electrical and Electronics Engineers (IEEE)** (@institute_of_electrical_and_electronics_engineers_ieee_about_nodate) +**Institute of Electrical and Electronics Engineers (IEEE)** (@institute_of_electrical_and_electronics_engineers_ieee_about_2024) * "Code Available" * "Code Reviewed" * "Code Reproducible" -**Psychological Science** (@hardwicke_transparency_2023 and @association_for_psychological_science_aps_psychological_2023) +**Psychological Science** (@hardwicke_transparency_2024 and @association_for_psychological_science_aps_psychological_2024) * "Computational Reproducibility" \ No newline at end of file diff --git a/evaluation/reproduction_report.qmd b/evaluation/reproduction_report.qmd index 8669952..7a62354 100644 --- a/evaluation/reproduction_report.qmd +++ b/evaluation/reproduction_report.qmd @@ -91,7 +91,7 @@ eval_dict = { 'STARS (essential)': [4, 2, 2, 0], 'STARS (optional)': [0, 0, 5, 0], 'Badges (criteria)': [4, 0, 8, 0], - 'Badges (badges)': [3, 0, 9, 0], + 'Badges (badges)': [0, 0, 12, 0], 'STRESS-DES': [16, 1, 2, 5], 'ISPOR-SDM': [15, 0, 2, 1] } diff --git a/logbook/posts/2024_11_21/index.qmd b/logbook/posts/2024_11_21/index.qmd new file mode 100644 index 0000000..3c86d66 --- /dev/null +++ b/logbook/posts/2024_11_21/index.qmd @@ -0,0 +1,25 @@ +--- +title: "Day 21" +author: "Amy Heather" +date: "2024-11-21" +categories: [evaluation] +--- + +::: {.callout-note} + +Redid badge evaluation. + +::: + +## 11.06-11.09: Revisit evaluation + +Revisited and revised the badge criteria to (a) make them up-to-date, and (b) make sure they are *specific* to the descriptions from each badge. Hence, redoing evaluations for all eight studies. + +Notes: + +* reproduce - no due to time + troubleshooting +* docs1 - no as doesn't describe relationship between outputs and article, and incomplete package list with only a few minimum versions + +## Untimed: Update summary report + +With new badge evaluation results. \ No newline at end of file diff --git a/quarto_site/references.bib b/quarto_site/references.bib index 5b6fa1d..fa37cec 100644 --- a/quarto_site/references.bib +++ b/quarto_site/references.bib @@ -445,12 +445,13 @@ @misc{blohowiak_badges_2023 file = {Snapshot:/home/amy/Zotero/storage/Z62G42T5/tvyxz.html:text/html}, } -@misc{institute_of_electrical_and_electronics_engineers_ieee_about_nodate, +@misc{institute_of_electrical_and_electronics_engineers_ieee_about_2024, title = {About {Content} in {IEEE} {Xplore}}, url = {https://ieeexplore.ieee.org/Xplorehelp/overview-of-ieee-xplore/about-content}, urldate = {2024-05-20}, journal = {IEEE Explore}, author = {{Institute of Electrical and Electronics Engineers (IEEE)}}, + year = {2024}, file = {About Content in IEEE Xplore:/home/amy/Zotero/storage/EI83P87Z/about-content.html:text/html}, } @@ -465,16 +466,6 @@ @misc{springer_nature_springer_2018 file = {Springer Nature Open data badge - Canvas Badges:/home/amy/Zotero/storage/JCSU3UQI/xhW4FLHBRe6Tzz2Cj4Q1tA.html:text/html}, } -@article{hardwicke_transparency_2023, - title = {Transparency {Is} {Now} the {Default} at {Psychological} {Science}}, - volume = {0}, - doi = {https://doi.org/10.1177/09567976231221573}, - number = {0}, - journal = {Psychological Science}, - author = {Hardwicke, Tom E. and Vazire, Simine}, - year = {2023}, -} - @article{van_lissa_worcs_2021, title = {{WORCS}: {A} workflow for open reproducible code in science}, volume = {4}, @@ -495,14 +486,14 @@ @article{van_lissa_worcs_2021 file = {Full Text PDF:/home/amy/Zotero/storage/N5P5IQ8K/Van Lissa et al. - 2021 - WORCS A workflow for open reproducible code in sc.pdf:application/pdf}, } -@misc{association_for_psychological_science_aps_psychological_2023, +@misc{association_for_psychological_science_aps_psychological_2024, title = {Psychological {Science} {Submission} {Guidelines}}, url = {https://www.psychologicalscience.org/publications/psychological_science/ps-submissions}, - urldate = {2024-05-20}, + urldate = {2024-11-20}, journal = {APS}, author = {{Association for Psychological Science (APS)}}, - month = dec, - year = {2023}, + month = sep, + year = {2024}, file = {Psychological Science Submission Guidelines – Association for Psychological Science – APS:/home/amy/Zotero/storage/MEJIJ7DC/ps-submissions.html:text/html}, } @@ -936,20 +927,6 @@ @article{glover_discrete_2018 file = {SAGE PDF Full Text:/home/amy/Zotero/storage/P2JPSDTZ/Glover et al. - 2018 - Discrete Event Simulation for Decision Modeling in.pdf:application/pdf}, } -@article{heather_protocol_2024, - title = {Protocol for assessing the computational reproducibility of discrete-event simulation models on {STARS}}, - url = {https://zenodo.org/records/12179846}, - abstract = {This protocol will be used to assess the computational reproducibility of published healthcare discrete-event simulation (DES) models created using Python or R. It forms part of the project STARS: “Sharing Tools and Artefacts for Reproducible Simulations in healthcare”.}, - language = {eng}, - urldate = {2024-08-19}, - author = {Heather, Amy and Monks, Thomas and Harper, Alison and Mustafee, Navonil and Mayne, Andrew}, - month = jun, - year = {2024}, - note = {Publisher: Zenodo}, - keywords = {Data science, Health care sciences, Health care services, Health sciences, Medical and health sciences, Simulation software}, - file = {Full Text PDF:/home/amy/Zotero/storage/FZSBV4J4/Heather et al. - 2024 - Protocol for assessing the computational reproduci.pdf:application/pdf}, -} - @article{thompson_screening_2018, title = {Screening women aged 65 years or over for abdominal aortic aneurysm: a modelling study and health economic evaluation}, volume = {22}, @@ -1153,3 +1130,1022 @@ @article{sadatsafavi_development_2019 pages = {152--167}, file = {SAGE PDF Full Text:/home/amy/Zotero/storage/PD7PRXSQ/Sadatsafavi et al. - 2019 - Development and Validation of the Evaluation Platf.pdf:application/pdf}, } + +@article{cadwallader_survey_2022, + title = {A survey of researchers’ code sharing and code reuse practices, and assessment of interactive notebook prototypes}, + volume = {10}, + copyright = {https://creativecommons.org/licenses/by/4.0/}, + issn = {2167-8359}, + url = {https://peerj.com/articles/13933}, + doi = {10.7717/peerj.13933}, + abstract = {This research aimed to understand the needs and habits of researchers in relation to code sharing and reuse; gather feedback on prototype code notebooks created by NeuroLibre; and help determine strategies that publishers could use to increase code sharing. We surveyed 188 researchers in computational biology. Respondents were asked about how often and why they look at code, which methods of accessing code they find useful and why, what aspects of code sharing are important to them, and how satisfied they are with their ability to complete these tasks. Respondents were asked to look at a prototype code notebook and give feedback on its features. Respondents were also asked how much time they spent preparing code and if they would be willing to increase this to use a code sharing tool, such as a notebook. As a reader of research articles the most common reason (70\%) for looking at code was to gain a better understanding of the article. The most commonly encountered method for code sharing–linking articles to a code repository–was also the most useful method of accessing code from the reader’s perspective. As authors, the respondents were largely satisfied with their ability to carry out tasks related to code sharing. The most important of these tasks were ensuring that the code was running in the correct environment, and sharing code with good documentation. The average researcher, according to our results, is unwilling to incur additional costs (in time, effort or expenditure) that are currently needed to use code sharing tools alongside a publication. We infer this means we need different models for funding and producing interactive or executable research outputs if they are to reach a large number of researchers. For the purpose of increasing the amount of code shared by authors, + PLOS Computational Biology + is, as a result, focusing on policy rather than tools.}, + language = {en}, + urldate = {2024-11-06}, + journal = {PeerJ}, + author = {Cadwallader, Lauren and Hrynaszkiewicz, Iain}, + month = aug, + year = {2022}, + pages = {e13933}, + file = {Full Text:/home/amy/Zotero/storage/2PVYDHZH/Cadwallader and Hrynaszkiewicz - 2022 - A survey of researchers’ code sharing and code reu.pdf:application/pdf}, +} + +@article{noauthor_revisiting_2022, + title = {Revisiting code reusability}, + volume = {4}, + copyright = {2022 Springer Nature Limited}, + issn = {2522-5839}, + url = {https://www.nature.com/articles/s42256-022-00554-9}, + doi = {10.1038/s42256-022-00554-9}, + abstract = {We introduced reusability reports, an article type to highlight code reusability, almost two years ago. On the basis of the results and positive feedback from authors and referees, we remain enthusiastic about the format.}, + language = {en}, + number = {10}, + urldate = {2024-11-06}, + journal = {Nature Machine Intelligence}, + month = oct, + year = {2022}, + note = {Publisher: Nature Publishing Group}, + keywords = {Computer science, Scientific community}, + pages = {801--801}, + file = {Full Text PDF:/home/amy/Zotero/storage/Y55ZFNDA/2022 - Revisiting code reusability.pdf:application/pdf}, +} + +@article{mejba_evolution_2023, + title = {The {Evolution} and {Impact} of {Code} {Reuse}: {A} {Deep} {Dive} into {Challenges}, {Reuse} {Strategies} and {Security}}, + copyright = {Creative Commons Attribution 4.0 International}, + shorttitle = {The {Evolution} and {Impact} of {Code} {Reuse}}, + url = {https://zenodo.org/doi/10.5281/zenodo.10141558}, + doi = {10.5281/ZENODO.10141558}, + abstract = {Code reuse, the practice of using pre-existing software code in new applications, is a widely adopted strategy in software development due to its potential to increase productivity, improve code quality, and reduce errors. However, it also presents unique challenges such as understanding and integrating reused code, managing potential bugs and security vulnerabilities, and dealing with licensing and ownership issues. This paper provides an in-depth analysis of code reuse, exploring its benefits, challenges, and strategies for effective implementation. It delves into strategies such as modular design, use of libraries and frameworks, design patterns, object-oriented programming, APIs and microservices, code documentation, automated testing, refactoring, code reviews, and continuous integration/continuous deployment. The paper also discusses common problems encountered during code reuse and provides comprehensive solutions to these problems. The aim is to provide a thorough understanding of code reuse and a guide for developers to maximize its potential benefits while mitigating its challenges. By implementing the strategies and solutions discussed in this paper, developers can create more robust, maintainable, and secure software through effective code reuse.}, + language = {en}, + urldate = {2024-11-06}, + author = {Mejba, Raoha Bin and Miazi, Sabbir Morshed and Palash, Ashikur Rahaman and Sobuz, Touhidur Rahman and Harshana Ranasinghe, Ranasinghe Arachchige Tharindu}, + month = nov, + year = {2023}, + note = {Publisher: Zenodo}, + file = {Mejba et al. - 2023 - The Evolution and Impact of Code Reuse A Deep Div.pdf:/home/amy/Zotero/storage/DUH6LF7N/Mejba et al. - 2023 - The Evolution and Impact of Code Reuse A Deep Div.pdf:application/pdf}, +} + +@article{curty_attitudes_2017, + title = {Attitudes and norms affecting scientists’ data reuse}, + volume = {12}, + issn = {1932-6203}, + url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0189288}, + doi = {10.1371/journal.pone.0189288}, + abstract = {The value of sharing scientific research data is widely appreciated, but factors that hinder or prompt the reuse of data remain poorly understood. Using the Theory of Reasoned Action, we test the relationship between the beliefs and attitudes of scientists towards data reuse, and their self-reported data reuse behaviour. To do so, we used existing responses to selected questions from a worldwide survey of scientists developed and administered by the DataONE Usability and Assessment Working Group (thus practicing data reuse ourselves). Results show that the perceived efficacy and efficiency of data reuse are strong predictors of reuse behaviour, and that the perceived importance of data reuse corresponds to greater reuse. Expressed lack of trust in existing data and perceived norms against data reuse were not found to be major impediments for reuse contrary to our expectations. We found that reported use of models and remotely-sensed data was associated with greater reuse. The results suggest that data reuse would be encouraged and normalized by demonstration of its value. We offer some theoretical and practical suggestions that could help to legitimize investment and policies in favor of data sharing.}, + language = {en}, + number = {12}, + urldate = {2024-11-06}, + journal = {PLOS ONE}, + author = {Curty, Renata Gonçalves and Crowston, Kevin and Specht, Alison and Grant, Bruce W. and Dalton, Elizabeth D.}, + month = dec, + year = {2017}, + note = {Publisher: Public Library of Science}, + keywords = {Metadata, Careers, Careers in research, Data management, Psychological attitudes, Scientists, Survey research, Surveys}, + pages = {e0189288}, + file = {Full Text PDF:/home/amy/Zotero/storage/AMF8JQV4/Curty et al. - 2017 - Attitudes and norms affecting scientists’ data reu.pdf:application/pdf}, +} + +@article{dupre_beyond_2022, + title = {Beyond advertising: {New} infrastructures for publishing integrated research objects}, + volume = {18}, + issn = {1553-7358}, + shorttitle = {Beyond advertising}, + url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1009651}, + doi = {10.1371/journal.pcbi.1009651}, + language = {en}, + number = {1}, + urldate = {2024-11-06}, + journal = {PLOS Computational Biology}, + author = {DuPre, Elizabeth and Holdgraf, Chris and Karakuzu, Agah and Tetrel, Loïc and Bellec, Pierre and Stikov, Nikola and Poline, Jean-Baptiste}, + month = jan, + year = {2022}, + note = {Publisher: Public Library of Science}, + keywords = {Scientific publishing, Peer review, Data management, Scientists, Advertising, Cloud computing, Clouds, Internet}, + pages = {e1009651}, + file = {Full Text PDF:/home/amy/Zotero/storage/DQHBHPQV/DuPre et al. - 2022 - Beyond advertising New infrastructures for publis.pdf:application/pdf}, +} + +@article{benureau_re-run_2018, + title = {Re-run, {Repeat}, {Reproduce}, {Reuse}, {Replicate}: {Transforming} {Code} into {Scientific} {Contributions}}, + volume = {11}, + issn = {1662-5196}, + shorttitle = {Re-run, {Repeat}, {Reproduce}, {Reuse}, {Replicate}}, + url = {https://www.frontiersin.org/journals/neuroinformatics/articles/10.3389/fninf.2017.00069/full}, + doi = {10.3389/fninf.2017.00069}, + abstract = {{\textless}p{\textgreater}Scientific code is different from production software. Scientific code, by producing results that are then analyzed and interpreted, participates in the elaboration of scientific conclusions. This imposes specific constraints on the code that are often overlooked in practice. We articulate, with a small example, five characteristics that a scientific code in computational science should possess: re-runnable, repeatable, reproducible, reusable, and replicable. The code should be executable (re-runnable) and produce the same result more than once (repeatable); it should allow an investigator to reobtain the published results (reproducible) while being easy to use, understand and modify (reusable), and it should act as an available reference for any ambiguity in the algorithmic descriptions of the article (replicable).{\textless}/p{\textgreater}}, + language = {English}, + urldate = {2024-11-06}, + journal = {Frontiers in Neuroinformatics}, + author = {Benureau, Fabien C. Y. and Rougier, Nicolas P.}, + month = jan, + year = {2018}, + note = {Publisher: Frontiers}, + keywords = {Reproducibility of Results, best practices, computational science, replicability, Reproducible Research, reproducible science, Software Development}, + file = {Full Text PDF:/home/amy/Zotero/storage/J7CEL7LP/Benureau and Rougier - 2018 - Re-run, Repeat, Reproduce, Reuse, Replicate Trans.pdf:application/pdf}, +} + +@article{wilson_good_2017, + title = {Good enough practices in scientific computing}, + volume = {13}, + issn = {1553-7358}, + url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1005510}, + doi = {10.1371/journal.pcbi.1005510}, + abstract = {Author summary Computers are now essential in all branches of science, but most researchers are never taught the equivalent of basic lab skills for research computing. As a result, data can get lost, analyses can take much longer than necessary, and researchers are limited in how effectively they can work with software and data. Computing workflows need to follow the same practices as lab projects and notebooks, with organized data, documented steps, and the project structured for reproducibility, but researchers new to computing often don't know where to start. This paper presents a set of good computing practices that every researcher can adopt, regardless of their current level of computational skill. These practices, which encompass data management, programming, collaborating with colleagues, organizing projects, tracking work, and writing manuscripts, are drawn from a wide variety of published sources from our daily lives and from our work with volunteer organizations that have delivered workshops to over 11,000 people since 2010.}, + language = {en}, + number = {6}, + urldate = {2024-11-06}, + journal = {PLOS Computational Biology}, + author = {Wilson, Greg and Bryan, Jennifer and Cranston, Karen and Kitzes, Justin and Nederbragt, Lex and Teal, Tracy K.}, + month = jun, + year = {2017}, + note = {Publisher: Public Library of Science}, + keywords = {Metadata, Reproducibility, Data management, Computer software, Control systems, Programming languages, Software tools, Source code}, + pages = {e1005510}, + file = {Full Text PDF:/home/amy/Zotero/storage/IF5UE78V/Wilson et al. - 2017 - Good enough practices in scientific computing.pdf:application/pdf}, +} + +@article{gomes_why_2022, + title = {Why don't we share data and code? {Perceived} barriers and benefits to public archiving practices}, + volume = {289}, + shorttitle = {Why don't we share data and code?}, + url = {https://royalsocietypublishing.org/doi/full/10.1098/rspb.2022.1113}, + doi = {10.1098/rspb.2022.1113}, + abstract = {The biological sciences community is increasingly recognizing the value of open, reproducible and transparent research practices for science and society at large. Despite this recognition, many researchers fail to share their data and code publicly. This pattern may arise from knowledge barriers about how to archive data and code, concerns about its reuse, and misaligned career incentives. Here, we define, categorize and discuss barriers to data and code sharing that are relevant to many research fields. We explore how real and perceived barriers might be overcome or reframed in the light of the benefits relative to costs. By elucidating these barriers and the contexts in which they arise, we can take steps to mitigate them and align our actions with the goals of open science, both as individual scientists and as a scientific community.}, + number = {1987}, + urldate = {2024-11-06}, + journal = {Proceedings of the Royal Society B: Biological Sciences}, + author = {Gomes, Dylan G. E. and Pottier, Patrice and Crystal-Ornelas, Robert and Hudgins, Emma J. and Foroughirad, Vivienne and Sánchez-Reyes, Luna L. and Turba, Rachel and Martinez, Paula Andrea and Moreau, David and Bertram, Michael G. and Smout, Cooper A. and Gaynor, Kaitlyn M.}, + month = nov, + year = {2022}, + note = {Publisher: Royal Society}, + keywords = {open science, reproducibility, data science, code reuse‌, data reuse, transparency}, + pages = {20221113}, + file = {Full Text PDF:/home/amy/Zotero/storage/LCAAHYDR/Gomes et al. - 2022 - Why don't we share data and code Perceived barrie.pdf:application/pdf}, +} + +@article{connolly_software_2023, + title = {Software {Engineering} {Practices} in {Academia}: {Promoting} the {3Rs}—{Readability}, {Resilience}, and {Reuse}}, + volume = {5}, + shorttitle = {Software {Engineering} {Practices} in {Academia}}, + url = {https://hdsr.mitpress.mit.edu/pub/f0f7h5cu}, + doi = {10.1162/99608f92.018bf012}, + abstract = {Over the past decade as data science has become integral to the research workflow, we, like many others, have learned that good data science requires high-quality software engineering. Unfortunately, our experience is that many data science projects can be limited by the absence of software engineering processes. We advocate that data science projects should incorporate what we call the 3Rs of software engineering: readability (human understandable codes), resilience (fails rarely/gracefully), and reuse (can easily be used by others and can be embedded in other software). This article discusses engineering practices that promote 3R software in academia. We emphasize that best practices in academia may differ from those in industry because of substantial differences in project scope (most academic projects have a single developer who is the sole user) and the reward systems in place in academia. We provide a framework for selecting a level of software engineering rigor that aligns well with the project scope, something that may change over time. We further discuss how to improve training in software engineering skills in an academic environment and how to build communities of practice that span across disciplines.}, + language = {en}, + number = {2}, + urldate = {2024-11-06}, + journal = {Harvard Data Science Review}, + author = {Connolly, Andrew and Hellerstein, Joseph and Alterman, Naomi and Beck, David and Fatland, Rob and Lazowska, Ed and Mandava, Vani and Stone, Sarah}, + month = apr, + year = {2023}, + file = {Connolly et al. - 2023 - Software Engineering Practices in Academia Promot.pdf:/home/amy/Zotero/storage/PCTS9VWW/Connolly et al. - 2023 - Software Engineering Practices in Academia Promot.pdf:application/pdf}, +} + +@misc{hrynaszkiewicz_survey_2021, + title = {A survey of code sharing practice and policy in computational biology}, + url = {https://osf.io/f73a6}, + doi = {10.31219/osf.io/f73a6}, + abstract = {Sharing of code supports reproducible research but fewer journals have policies on code sharing compared to data sharing, and there is little evidence on researchers’ attitudes and experiences with code sharing. Before introducing a stronger policy on sharing of code, the Editors and publisher of the journal PLOS Computational Biology wished to test, via an online survey, the suitability of a proposed mandatory code sharing policy with its community of authors. Previous research has established, in 2019, 41\% of papers in the journal linked to shared code. We also wanted to understand the potential impact of the proposed policy on authors' submissions to the journal, and their concerns about code sharing. + +We received 214 completed survey responses, all of whom had generated code in their research previously. 80\% had published in PLOS Computational Biology and 88\% of whom were based in Europe or North America. Overall, respondents reported they were more likely to submit to the journal if it had a mandatory code sharing policy and US researchers were more positive than the average for all respondents. Researchers whose main discipline is Medicine and Health sciences viewed the proposed policy less favourably, as did the most senior researchers (those with more than 100 publications) compared to early and mid-career researchers. + +The authors surveyed report that, on average, 71\% of their research articles have associated code, and that for the average author, code has not been shared for 32\% of these papers. The most common reasons for not sharing code previously are practical issues, which are unlikely to prevent compliance with the policy. A lack of time to share code was the most common reason. 22\% of respondents who had not shared their code in the past cited intellectual property (IP) concerns - a concern that might prevent public sharing of code under a mandatory code sharing policy. The results also imply that 18\% of the respondents’ previous publications did not have the associated code shared and IP concerns were not cited, suggesting more papers in the journal could share code. + +To remain inclusive of all researchers in the community, the policy was designed to allow researchers who can demonstrate they are legally restricted from sharing their code to be granted an exemption to public sharing of code. + +As a secondary goal of the survey we wanted to determine if researchers have unmet needs in their ability to share their own code, and to access other researchers' code. Consistent with our previous research on data sharing, we found potential opportunities for new products or features that support code accessibility or reuse. We found researchers were on average satisfied with their ability to share their own code, suggesting that offering new products or features to support sharing in the absence of a stronger policy would not increase the availability of code with the journal's publications.}, + language = {en-us}, + urldate = {2024-11-06}, + publisher = {OSF}, + author = {Hrynaszkiewicz, Iain and Harney, James and Cadwallader, Lauren}, + month = apr, + year = {2021}, + keywords = {Open science, Journal policy, Code sharing, Computational biology, Open research, Publishing practices, Survey results}, +} + +@article{eglen_toward_2017, + title = {Toward standard practices for sharing computer code and programs in neuroscience}, + volume = {20}, + copyright = {2017 Springer Nature America, Inc.}, + issn = {1546-1726}, + url = {https://www.nature.com/articles/nn.4550}, + doi = {10.1038/nn.4550}, + abstract = {Computational techniques are central in many areas of neuroscience and are relatively easy to share. This paper describes why computer programs underlying scientific publications should be shared and lists simple steps for sharing. Together with ongoing efforts in data sharing, this should aid reproducibility of research.}, + language = {en}, + number = {6}, + urldate = {2024-11-06}, + journal = {Nature Neuroscience}, + author = {Eglen, Stephen J. and Marwick, Ben and Halchenko, Yaroslav O. and Hanke, Michael and Sufi, Shoaib and Gleeson, Padraig and Silver, R. Angus and Davison, Andrew P. and Lanyon, Linda and Abrams, Mathew and Wachtler, Thomas and Willshaw, David J. and Pouzat, Christophe and Poline, Jean-Baptiste}, + month = jun, + year = {2017}, + note = {Publisher: Nature Publishing Group}, + keywords = {Scientific community, Neuroscience}, + pages = {770--773}, + file = {Full Text PDF:/home/amy/Zotero/storage/BQKS8F5K/Eglen et al. - 2017 - Toward standard practices for sharing computer cod.pdf:application/pdf}, +} + +@inproceedings{stodden_enabling_2018, + address = {New York, NY, USA}, + series = {P-{RECS}'18}, + title = {Enabling the {Verification} of {Computational} {Results}: {An} {Empirical} {Evaluation} of {Computational} {Reproducibility}}, + isbn = {978-1-4503-5861-3}, + shorttitle = {Enabling the {Verification} of {Computational} {Results}}, + url = {https://dl.acm.org/doi/10.1145/3214239.3214242}, + doi = {10.1145/3214239.3214242}, + abstract = {The ability to independently regenerate published computational claims is widely recognized as a key component of scientific reproducibility. In this article we take a narrow interpretation of this goal, and attempt to regenerate published claims from author-supplied information, including data, code, inputs, and other provided specifications, on a different computational system than that used by the original authors. We are motivated by Claerbout and Donoho's exhortation of the importance of providing complete information for reproducibility of the published claim. We chose the Elsevier journal, the Journal of Computational Physics, which has stated author guidelines that encourage the availability of computational digital artifacts that support scholarly findings. In an IRB approved study at the University of Illinois at Urbana-Champaign (IRB \#17329) we gathered artifacts from a sample of authors who published in this journal in 2016 and 2017. We then used the ICERM criteria generated at the 2012 ICERM workshop "Reproducibility in Computational and Experimental Mathematics" to evaluate the sufficiency of the information provided in the publications and the ease with which the digital artifacts afforded computational reproducibility. We find that, for the articles for which we obtained computational artifacts, we could not easily regenerate the findings for 67\% of them, and we were unable to easily regenerate all the findings for any of the articles. We then evaluated the artifacts we did obtain (55 of 306 articles) and find that the main barriers to computational reproducibility are inadequate documentation of code, data, and workflow information (70.9\%), missing code function and setting information, and missing licensing information (75\%). We recommend improvements based on these findings, including the deposit of supporting digital artifacts for reproducibility as a condition of publication, and verification of computational findings via re-execution of the code when possible.}, + urldate = {2024-11-06}, + booktitle = {Proceedings of the {First} {International} {Workshop} on {Practical} {Reproducible} {Evaluation} of {Computer} {Systems}}, + publisher = {Association for Computing Machinery}, + author = {Stodden, Victoria and Krafczyk, Matthew S. and Bhaskar, Adhithya}, + month = jun, + year = {2018}, + pages = {1--5}, + file = {Full Text PDF:/home/amy/Zotero/storage/RUW2MJET/Stodden et al. - 2018 - Enabling the Verification of Computational Results.pdf:application/pdf}, +} + +@article{mccullough_lessons_2006, + title = {Lessons from the {JMCB} {Archive}}, + volume = {38}, + issn = {0022-2879}, + url = {https://www.jstor.org/stable/3838995}, + abstract = {We examine the online archive of the Journal of Money, Credit, and Banking, in which an author is required to deposit the data and code that replicate the results of his paper. We find that most authors do not fulfill this requirement. Of more than 150 empirical articles, fewer than 15 could be replicated. Despite all this, there is no doubt that a data/code archive is more conducive to replicable research than the alternatives. We make recommendations to improve the functioning of the archive.}, + number = {4}, + urldate = {2024-11-06}, + journal = {Journal of Money, Credit and Banking}, + author = {McCullough, B. D. and McGeary, Kerry Anne and Harrison, Teresa D.}, + year = {2006}, + note = {Publisher: [Wiley, Ohio State University Press]}, + pages = {1093--1107}, +} + +@misc{galiani_incentives_2017, + type = {Working {Paper}}, + series = {Working {Paper} {Series}}, + title = {Incentives for {Replication} in {Economics}}, + url = {https://www.nber.org/papers/w23576}, + doi = {10.3386/w23576}, + abstract = {Replication is a critical component of scientific credibility as it increases our confidence in the reliability of the knowledge generated by original research. Yet, replication is the exception rather than the rule in economics. In this paper, we examine why replication is so rare and propose changes to the incentives to replicate. Our study focuses on software code replication, which seeks to replicate the results in the original paper using the same data as the original study and verifying that the analysis code is correct. We analyse the effectiveness of the current model for code replication in the context of three desirable characteristics: unbiasedness, fairness and efficiency. We find substantial evidence of “overturn bias” that likely leads to many false positives in terms of “finding” or claiming mistakes in the original analysis. Overturn bias comes from the fact that replications that overturn original results are much easier to publish than those that confirm original results. In a survey of editors, almost all responded they would in principle publish a replication study that overturned the results of the original study, but only 29\% responded that they would consider publishing a replication study that confirmed the original study results. We also find that most replication effort is devoted to so called important papers and that the cost of replication is high in that posited data and software are very hard to use. We outline a new model for the journals to take over replication post acceptance and prepublication that would solve the incentive problems raised in this paper.}, + urldate = {2024-11-06}, + publisher = {National Bureau of Economic Research}, + author = {Galiani, Sebastian and Gertler, Paul and Romero, Mauricio}, + month = jul, + year = {2017}, + file = {Full Text PDF:/home/amy/Zotero/storage/RZKSMHA9/Galiani et al. - 2017 - Incentives for Replication in Economics.pdf:application/pdf}, +} + +@article{chang_is_2022, + title = {Is {Economics} {Research} {Replicable}? {Sixty} {Published} {Papers} {From} {Thirteen} {Journals} {Say} “{Often} {Not}”}, + volume = {11}, + issn = {2164-5744, 2164-5760}, + shorttitle = {Is {Economics} {Research} {Replicable}?}, + url = {http://www.nowpublishers.com/article/Details/CFR-0053}, + doi = {10.1561/104.00000053}, + language = {en}, + number = {1}, + urldate = {2024-11-06}, + journal = {Critical Finance Review}, + author = {Chang, Andrew and Li, Phillip}, + year = {2022}, + pages = {185--206}, +} + +@article{stockemer_data_2018, + title = {Data {Access}, {Transparency}, and {Replication}: {New} {Insights} from the {Political} {Behavior} {Literature}}, + volume = {51}, + issn = {1049-0965, 1537-5935}, + shorttitle = {Data {Access}, {Transparency}, and {Replication}}, + url = {https://www.cambridge.org/core/journals/ps-political-science-and-politics/article/data-access-transparency-and-replication-new-insights-from-the-political-behavior-literature/64CA07CBA652E299079FF32BC5A6DCB3}, + doi = {10.1017/S1049096518000926}, + abstract = {Do researchers share their quantitative data and are the quantitative results that are published in political science journals replicable? We attempt to answer these questions by analyzing all articles published in the 2015 issues of three political behaviorist journals (i.e., Electoral Studies, Party Politics, and Journal of Elections, Public Opinion \& Parties)—all of which did not have a binding data-sharing and replication policy as of 2015. We found that authors are still reluctant to share their data; only slightly more than half of the authors in these journals do so. For those who share their data, we mainly confirmed the initial results reported in the respective articles in roughly 70\% of the times. Only roughly 5\% of the articles yielded significantly different results from those reported in the publication. However, we also found that roughly 25\% of the articles organized the data and/or code so poorly that replication was impossible.}, + language = {en}, + number = {4}, + urldate = {2024-11-07}, + journal = {PS: Political Science \& Politics}, + author = {Stockemer, Daniel and Koehler, Sebastian and Lentz, Tobias}, + month = oct, + year = {2018}, + pages = {799--803}, + file = {Full Text PDF:/home/amy/Zotero/storage/Y6HAFSM7/Stockemer et al. - 2018 - Data Access, Transparency, and Replication New In.pdf:application/pdf}, +} + +@article{naudet_data_2018, + title = {Data sharing and reanalysis of randomized controlled trials in leading biomedical journals with a full data sharing policy: survey of studies published in {The} {BMJ} and {PLOS} {Medicine}}, + volume = {360}, + copyright = {Published by the BMJ Publishing Group Limited. For permission to use (where not already granted under a licence) please go to http://group.bmj.com/group/rights-licensing/permissions. This is an Open Access article distributed in accordance with the Creative Commons Attribution Non Commercial (CC BY-NC 4.0) license, which permits others to distribute, remix, adapt, build upon this work non-commercially, and license their derivative works on different terms, provided the original work is properly cited and the use is non-commercial. See: http://creativecommons.org/licenses/by-nc/4.0/.}, + issn = {0959-8138, 1756-1833}, + shorttitle = {Data sharing and reanalysis of randomized controlled trials in leading biomedical journals with a full data sharing policy}, + url = {https://www.bmj.com/content/360/bmj.k400}, + doi = {10.1136/bmj.k400}, + abstract = {Objectives To explore the effectiveness of data sharing by randomized controlled trials (RCTs) in journals with a full data sharing policy and to describe potential difficulties encountered in the process of performing reanalyses of the primary outcomes. +Design Survey of published RCTs. +Setting PubMed/Medline. +Eligibility criteria RCTs that had been submitted and published by The BMJ and PLOS Medicine subsequent to the adoption of data sharing policies by these journals. +Main outcome measure The primary outcome was data availability, defined as the eventual receipt of complete data with clear labelling. Primary outcomes were reanalyzed to assess to what extent studies were reproduced. Difficulties encountered were described. +Results 37 RCTs (21 from The BMJ and 16 from PLOS Medicine) published between 2013 and 2016 met the eligibility criteria. 17/37 (46\%, 95\% confidence interval 30\% to 62\%) satisfied the definition of data availability and 14 of the 17 (82\%, 59\% to 94\%) were fully reproduced on all their primary outcomes. Of the remaining RCTs, errors were identified in two but reached similar conclusions and one paper did not provide enough information in the Methods section to reproduce the analyses. Difficulties identified included problems in contacting corresponding authors and lack of resources on their behalf in preparing the datasets. In addition, there was a range of different data sharing practices across study groups. +Conclusions Data availability was not optimal in two journals with a strong policy for data sharing. When investigators shared data, most reanalyses largely reproduced the original results. Data sharing practices need to become more widespread and streamlined to allow meaningful reanalyses and reuse of data. +Trial registration Open Science Framework osf.io/c4zke.}, + language = {en}, + urldate = {2024-11-07}, + journal = {BMJ}, + author = {Naudet, Florian and Sakarovitch, Charlotte and Janiaud, Perrine and Cristea, Ioana and Fanelli, Daniele and Moher, David and Ioannidis, John P. A.}, + month = feb, + year = {2018}, + pmid = {29440066}, + note = {Publisher: British Medical Journal Publishing Group +Section: Research}, + pages = {k400}, + file = {Full Text PDF:/home/amy/Zotero/storage/AQJYDIMS/Naudet et al. - 2018 - Data sharing and reanalysis of randomized controll.pdf:application/pdf}, +} + +@inproceedings{raff_step_2019, + title = {A {Step} {Toward} {Quantifying} {Independently} {Reproducible} {Machine} {Learning} {Research}}, + volume = {32}, + url = {https://proceedings.neurips.cc/paper/2019/hash/c429429bf1f2af051f2021dc92a8ebea-Abstract.html}, + abstract = {What makes a paper independently reproducible? Debates on reproducibility center around intuition or assumptions but lack empirical results. Our field focuses on releasing code, which is important, but is not sufficient for determining reproducibility. We take the first step toward a quantifiable answer by manually attempting to implement 255 papers published from 1984 until 2017, recording features of each paper, and performing statistical analysis of the results. For each paper, we did not look at the authors code, if released, in order to prevent bias toward discrepancies between code and paper.}, + urldate = {2024-11-07}, + booktitle = {Advances in {Neural} {Information} {Processing} {Systems}}, + publisher = {Curran Associates, Inc.}, + author = {Raff, Edward}, + year = {2019}, + file = {Full Text PDF:/home/amy/Zotero/storage/7KJGWDLU/Raff - 2019 - A Step Toward Quantifying Independently Reproducib.pdf:application/pdf}, +} + +@article{stagge_assessing_2019, + title = {Assessing data availability and research reproducibility in hydrology and water resources}, + volume = {6}, + copyright = {2019 The Author(s)}, + issn = {2052-4463}, + url = {https://www.nature.com/articles/sdata201930}, + doi = {10.1038/sdata.2019.30}, + abstract = {There is broad interest to improve the reproducibility of published research. We developed a survey tool to assess the availability of digital research artifacts published alongside peer-reviewed journal articles (e.g. data, models, code, directions for use) and reproducibility of article results. We used the tool to assess 360 of the 1,989 articles published by six hydrology and water resources journals in 2017. Like studies from other fields, we reproduced results for only a small fraction of articles (1.6\% of tested articles) using their available artifacts. We estimated, with 95\% confidence, that results might be reproduced for only 0.6\% to 6.8\% of all 1,989 articles. Unlike prior studies, the survey tool identified key bottlenecks to making work more reproducible. Bottlenecks include: only some digital artifacts available (44\% of articles), no directions (89\%), or all artifacts available but results not reproducible (5\%). The tool (or extensions) can help authors, journals, funders, and institutions to self-assess manuscripts, provide feedback to improve reproducibility, and recognize and reward reproducible articles as examples for others.}, + language = {en}, + number = {1}, + urldate = {2024-11-07}, + journal = {Scientific Data}, + author = {Stagge, James H. and Rosenberg, David E. and Abdallah, Adel M. and Akbar, Hadia and Attallah, Nour A. and James, Ryan}, + month = feb, + year = {2019}, + note = {Publisher: Nature Publishing Group}, + keywords = {Publishing, Hydrology, Research data}, + pages = {190030}, + file = {Full Text PDF:/home/amy/Zotero/storage/K2NFJIGZ/Stagge et al. - 2019 - Assessing data availability and research reproduci.pdf:application/pdf}, +} + +@article{eubank_lessons_2016, + title = {Lessons from a {Decade} of {Replications} at the {Quarterly} {Journal} of {Political} {Science}}, + volume = {49}, + issn = {1049-0965, 1537-5935}, + url = {https://www.cambridge.org/core/journals/ps-political-science-and-politics/article/lessons-from-a-decade-of-replications-at-the-quarterly-journal-of-political-science/284B2830BFD99888B42D4CEABC28B9EE}, + doi = {10.1017/S1049096516000196}, + abstract = {To allow researchers to investigate not only whether a paper’s methods are theoretically sound but also whether they have been properly implemented and are robust to alternative specifications, it is necessary that published papers be accompanied by their underlying data and code. This article describes experiences and lessons learned at the Quarterly Journal of Political Science since it began requiring authors to provide this type of replication code in 2005. It finds that of the 24 empirical papers subjected to in-house replication review since September 2012, only four packages did not require any modifications. Most troubling, 14 packages (58\%) had results in the paper that differed from those generated by the author’s own code. Based on these experiences, this article presents a set of guidelines for authors and journals for improving the reliability and usability of replication packages.}, + language = {en}, + number = {2}, + urldate = {2024-11-07}, + journal = {PS: Political Science \& Politics}, + author = {Eubank, Nicholas}, + month = apr, + year = {2016}, + pages = {273--276}, + file = {Full Text PDF:/home/amy/Zotero/storage/98TKVFYM/Eubank - 2016 - Lessons from a Decade of Replications at the Quart.pdf:application/pdf}, +} + +@article{artner_reproducibility_2021, + title = {The reproducibility of statistical results in psychological research: {An} investigation using unpublished raw data}, + volume = {26}, + issn = {1939-1463}, + shorttitle = {The reproducibility of statistical results in psychological research}, + doi = {10.1037/met0000365}, + abstract = {We investigated the reproducibility of the major statistical conclusions drawn in 46 articles published in 2012 in three APA journals. After having identified 232 key statistical claims, we tried to reproduce, for each claim, the test statistic, its degrees of freedom, and the corresponding p value, starting from the raw data that were provided by the authors and closely following the Method section in the article. Out of the 232 claims, we were able to successfully reproduce 163 (70\%), 18 of which only by deviating from the article’s analytical description. Thirteen (7\%) of the 185 claims deemed significant by the authors are no longer so. The reproduction successes were often the result of cumbersome and time-consuming trial-and-error work, suggesting that APA style reporting in conjunction with raw data makes numerical verification at least hard, if not impossible. This article discusses the types of mistakes we could identify and the tediousness of our reproduction efforts in the light of a newly developed taxonomy for reproducibility. We then link our findings with other findings of empirical research on this topic, give practical recommendations on how to achieve reproducibility, and discuss the challenges of large-scale reproducibility checks as well as promising ideas that could considerably increase the reproducibility of psychological research. (PsycInfo Database Record (c) 2021 APA, all rights reserved)}, + number = {5}, + journal = {Psychological Methods}, + author = {Artner, Richard and Verliefde, Thomas and Steegen, Sara and Gomes, Sara and Traets, Frits and Tuerlinckx, Francis and Vanpaemel, Wolf}, + year = {2021}, + note = {Place: US +Publisher: American Psychological Association}, + keywords = {Statistical Analysis, Statistics, American Psychological Association, Data Sets, Errors, Experimental Replication, Experimentation, Scientific Communication}, + pages = {527--546}, + file = {Accepted Version:/home/amy/Zotero/storage/3ZUKR4JJ/Artner et al. - 2021 - The reproducibility of statistical results in psyc.pdf:application/pdf}, +} + +@article{maassen_reproducibility_2020, + title = {Reproducibility of individual effect sizes in meta-analyses in psychology}, + volume = {15}, + issn = {1932-6203}, + url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0233107}, + doi = {10.1371/journal.pone.0233107}, + abstract = {To determine the reproducibility of psychological meta-analyses, we investigated whether we could reproduce 500 primary study effect sizes drawn from 33 published meta-analyses based on the information given in the meta-analyses, and whether recomputations of primary study effect sizes altered the overall results of the meta-analysis. Results showed that almost half (k = 224) of all sampled primary effect sizes could not be reproduced based on the reported information in the meta-analysis, mostly because of incomplete or missing information on how effect sizes from primary studies were selected and computed. Overall, this led to small discrepancies in the computation of mean effect sizes, confidence intervals and heterogeneity estimates in 13 out of 33 meta-analyses. We provide recommendations to improve transparency in the reporting of the entire meta-analytic process, including the use of preregistration, data and workflow sharing, and explicit coding practices.}, + language = {en}, + number = {5}, + urldate = {2024-11-07}, + journal = {PLOS ONE}, + author = {Maassen, Esther and Assen, Marcel A. L. M. van and Nuijten, Michèle B. and Olsson-Collentine, Anton and Wicherts, Jelte M.}, + month = may, + year = {2020}, + note = {Publisher: Public Library of Science}, + keywords = {Psychology, Systematic reviews, Reproducibility, Peer review, Clinical psychology, Metaanalysis, Publication ethics, Research reporting guidelines}, + pages = {e0233107}, + file = {Full Text PDF:/home/amy/Zotero/storage/P3KYHGG6/Maassen et al. - 2020 - Reproducibility of individual effect sizes in meta.pdf:application/pdf}, +} + +@article{chambers_verification_2020, + title = {\textit{{Verification} {Reports}}: {A} new article type at \textit{{Cortex}}}, + volume = {129}, + issn = {0010-9452}, + shorttitle = {\textit{{Verification} {Reports}}}, + url = {https://www.sciencedirect.com/science/article/pii/S0010945220301738}, + doi = {10.1016/j.cortex.2020.04.020}, + urldate = {2024-11-07}, + journal = {Cortex}, + author = {Chambers, Christopher D.}, + month = aug, + year = {2020}, + pages = {A1--A3}, + file = {Accepted Version:/home/amy/Zotero/storage/TRMF2L5W/Chambers - 2020 - Verification Reports A new article type at.pdf:application/pdf;ScienceDirect Snapshot:/home/amy/Zotero/storage/ITQFC56F/S0010945220301738.html:text/html}, +} + +@article{ostermann_advancing_2017, + title = {Advancing {Science} with {VGI}: {Reproducibility} and {Replicability} of {Recent} {Studies} using {VGI}}, + volume = {21}, + copyright = {© 2015 John Wiley \& Sons Ltd}, + issn = {1467-9671}, + shorttitle = {Advancing {Science} with {VGI}}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/tgis.12195}, + doi = {10.1111/tgis.12195}, + abstract = {In scientific research, reproducibility and replicability are requirements to ensure the advancement of our body of knowledge. This holds true also for VGI-related research and studies. However, the characteristics of VGI suggest particular difficulties in ensuring reproducibility and replicability. In this article, we aim to examine the current situation in VGI-related research, and identify strategies to ensure realization of its full potential. To do so, we first investigate the different aspects of reproducibility and replicability and their impact on VGI-related research. These impacts are different depending on the objectives of the study. Therefore, we examine the study focus of VGI-related research to assess the current body of research and structure our assessment. This work is based on a rigorous review of the elements of reproducibility and a systematic mapping and analysis of 58 papers on the use of VGI in the crisis management field. Results of our investigation show that reproducibility issues related to data are a serious concern, while reproducibility issues related to analysis methods and processes face fewer challenges. However, since most studies still focus on analyzing the source data, reproducibility and replicability are still an unsolved problem in VGI-related research. Therefore, we show initiatives tackling the problem, and finally formulate strategies to improve the situation.}, + language = {en}, + number = {2}, + urldate = {2024-11-07}, + journal = {Transactions in GIS}, + author = {Ostermann, Frank O. and Granell, Carlos}, + year = {2017}, + note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/tgis.12195}, + pages = {224--237}, + file = {Full Text PDF:/home/amy/Zotero/storage/AMFJQG7F/Ostermann and Granell - 2017 - Advancing Science with VGI Reproducibility and Re.pdf:application/pdf;Snapshot:/home/amy/Zotero/storage/C67Q8JQR/tgis.html:text/html}, +} + +@article{gil_toward_2016, + title = {Toward the {Geoscience} {Paper} of the {Future}: {Best} practices for documenting and sharing research from data to software to provenance}, + volume = {3}, + issn = {2333-5084}, + shorttitle = {Toward the {Geoscience} {Paper} of the {Future}}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/2015EA000136}, + doi = {10.1002/2015EA000136}, + abstract = {Geoscientists now live in a world rich with digital data and methods, and their computational research cannot be fully captured in traditional publications. The Geoscience Paper of the Future (GPF) presents an approach to fully document, share, and cite all their research products including data, software, and computational provenance. This article proposes best practices for GPF authors to make data, software, and methods openly accessible, citable, and well documented. The publication of digital objects empowers scientists to manage their research products as valuable scientific assets in an open and transparent way that enables broader access by other scientists, students, decision makers, and the public. Improving documentation and dissemination of research will accelerate the pace of scientific discovery by improving the ability of others to build upon published work.}, + language = {en}, + number = {10}, + urldate = {2024-11-07}, + journal = {Earth and Space Science}, + author = {Gil, Yolanda and David, Cédric H. and Demir, Ibrahim and Essawy, Bakinam T. and Fulweiler, Robinson W. and Goodall, Jonathan L. and Karlstrom, Leif and Lee, Huikyo and Mills, Heath J. and Oh, Ji-Hyun and Pierce, Suzanne A. and Pope, Allen and Tzeng, Mimi W. and Villamizar, Sandra R. and Yu, Xuan}, + year = {2016}, + note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/2015EA000136}, + keywords = {reproducibility, workflow, data sharing, geoscience paper of the future, provenance, software reuse}, + pages = {388--415}, + file = {Snapshot:/home/amy/Zotero/storage/PXARVXD5/2015EA000136.html:text/html}, +} + +@article{skaggs_reproducible_2015, + title = {Reproducible {Research} in {Vadose} {Zone} {Sciences}}, + volume = {14}, + issn = {1539-1663}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.2136/vzj2015.06.0088}, + doi = {10.2136/vzj2015.06.0088}, + abstract = {Core Ideas A significant portion of present-day geoscience research is computational. Science would benefit from greater transparency in computational research. Vadose Zone Journal is launching a Reproducible Research program. Code and data underlying a research article will be published alongside articles. A significant portion of present-day soil and Earth science research is computational, involving complex data analysis pipelines, advanced mathematical and statistical models, and sophisticated computer codes. Opportunities for scientific progress are greatly diminished if reproducing and building on published research is difficult or impossible due to the complexity of these computational systems. Vadose Zone Journal (VZJ) is launching a Reproducible Research (RR) program in which code and data underlying a research article will be published alongside the article, thereby enabling readers to analyze data in a manner similar to that presented in the article and build on results in future research and applications. In this article, we discuss reproducible research, its background and use across other disciplines, its value to the scientific community, and its implementation in VZJ.}, + language = {en}, + number = {10}, + urldate = {2024-11-07}, + journal = {Vadose Zone Journal}, + author = {Skaggs, T.h. and Young, M.h. and Vrugt, J.a.}, + year = {2015}, + note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.2136/vzj2015.06.0088}, + pages = {vzj2015.06.0088}, + file = {Full Text:/home/amy/Zotero/storage/J34H3Z8L/Skaggs et al. - 2015 - Reproducible Research in Vadose Zone Sciences.pdf:application/pdf;Snapshot:/home/amy/Zotero/storage/8EF2K5HI/vzj2015.06.html:text/html}, +} + +@article{hardwicke_data_2018, + title = {Data availability, reusability, and analytic reproducibility: evaluating the impact of a mandatory open data policy at the journal {Cognition}}, + volume = {5}, + shorttitle = {Data availability, reusability, and analytic reproducibility}, + url = {https://royalsocietypublishing.org/doi/10.1098/rsos.180448}, + doi = {10.1098/rsos.180448}, + abstract = {Access to data is a critical feature of an efficient, progressive and ultimately self-correcting scientific ecosystem. But the extent to which in-principle benefits of data sharing are realized in practice is unclear. Crucially, it is largely unknown whether published findings can be reproduced by repeating reported analyses upon shared data (‘analytic reproducibility’). To investigate this, we conducted an observational evaluation of a mandatory open data policy introduced at the journal Cognition. Interrupted time-series analyses indicated a substantial post-policy increase in data available statements (104/417, 25\% pre-policy to 136/174, 78\% post-policy), although not all data appeared reusable (23/104, 22\% pre-policy to 85/136, 62\%, post-policy). For 35 of the articles determined to have reusable data, we attempted to reproduce 1324 target values. Ultimately, 64 values could not be reproduced within a 10\% margin of error. For 22 articles all target values were reproduced, but 11 of these required author assistance. For 13 articles at least one value could not be reproduced despite author assistance. Importantly, there were no clear indications that original conclusions were seriously impacted. Mandatory open data policies can increase the frequency and quality of data sharing. However, suboptimal data curation, unclear analysis specification and reporting errors can impede analytic reproducibility, undermining the utility of data sharing and the credibility of scientific findings.}, + number = {8}, + urldate = {2024-11-07}, + journal = {Royal Society Open Science}, + author = {Hardwicke, Tom E. and Mathur, Maya B. and MacDonald, Kyle and Nilsonne, Gustav and Banks, George C. and Kidwell, Mallory C. and Hofelich Mohr, Alicia and Clayton, Elizabeth and Yoon, Erica J. and Henry Tessler, Michael and Lenne, Richie L. and Altman, Sara and Long, Bria and Frank, Michael C.}, + month = aug, + year = {2018}, + note = {Publisher: Royal Society}, + keywords = {open science, reproducibility, open data, journal policy, interrupted time series, meta-science}, + pages = {180448}, + file = {Full Text PDF:/home/amy/Zotero/storage/E56SYYI4/Hardwicke et al. - 2018 - Data availability, reusability, and analytic repro.pdf:application/pdf}, +} + +@article{sandve_ten_2013, + title = {Ten {Simple} {Rules} for {Reproducible} {Computational} {Research}}, + volume = {9}, + issn = {1553-7358}, + url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1003285}, + doi = {10.1371/journal.pcbi.1003285}, + language = {en}, + number = {10}, + urldate = {2024-11-13}, + journal = {PLOS Computational Biology}, + author = {Sandve, Geir Kjetil and Nekrutenko, Anton and Taylor, James and Hovig, Eivind}, + month = oct, + year = {2013}, + note = {Publisher: Public Library of Science}, + keywords = {Replication studies, Reproducibility, Source code, Archives, Computer and information sciences, Computer applications, Genome analysis, Habits}, + pages = {e1003285}, + file = {Full Text PDF:/home/amy/Zotero/storage/UEDNQV5Y/Sandve et al. - 2013 - Ten Simple Rules for Reproducible Computational Re.pdf:application/pdf}, +} + +@misc{eynden_survey_2016, + title = {Survey of {Wellcome} researchers and their attitudes to open research}, + copyright = {Creative Commons Attribution 4.0 International}, + url = {https://doi.org/10.6084/m9.figshare.4055448.v1}, + abstract = {Results of a survey of Wellcome researchers to find out what they think about open research, how they practice it, and some of the barriers they face. {\textless}br{\textgreater}This work was commissioned by the Wellcome Trust and conducted by the London School of Hygiene and Tropical Medicine and the UK Data Service.{\textless}br{\textgreater}}, + urldate = {2024-11-13}, + publisher = {Wellcome Trust}, + author = {Eynden, Veerle Van Den and Knight, Gareth and Vlad, Anca and Radler, Barry and Tenopir, Carol and Leon, David and Manista, Frank and Whitworth, Jimmy and Corti, Louise}, + year = {2016}, + keywords = {Survey Results}, +} + +@article{colavizza_analysis_2024, + title = {An analysis of the effects of sharing research data, code, and preprints on citations}, + volume = {19}, + issn = {1932-6203}, + url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0311493}, + doi = {10.1371/journal.pone.0311493}, + abstract = {Calls to make scientific research more open have gained traction with a range of societal stakeholders. Open Science practices include but are not limited to the early sharing of results via preprints and openly sharing outputs such as data and code to make research more reproducible and extensible. Existing evidence shows that adopting Open Science practices has effects in several domains. In this study, we investigate whether adopting one or more Open Science practices leads to significantly higher citations for an associated publication, which is one form of academic impact. We use a novel dataset known as Open Science Indicators, produced by PLOS and DataSeer, which includes all PLOS publications from 2018 to 2023 as well as a comparison group sampled from the PMC Open Access Subset. In total, we analyze circa 122’000 publications. We calculate publication and author-level citation indicators and use a broad set of control variables to isolate the effect of Open Science Indicators on received citations. We show that Open Science practices are adopted to different degrees across scientific disciplines. We find that the early release of a publication as a preprint correlates with a significant positive citation advantage of about 20.2\% (±.7) on average. We also find that sharing data in an online repository correlates with a smaller yet still positive citation advantage of 4.3\% (±.8) on average. However, we do not find a significant citation advantage for sharing code. Further research is needed on additional or alternative measures of impact beyond citations. Our results are likely to be of interest to researchers, as well as publishers, research funders, and policymakers.}, + language = {en}, + number = {10}, + urldate = {2024-11-14}, + journal = {PLOS ONE}, + author = {Colavizza, Giovanni and Cadwallader, Lauren and LaFlamme, Marcel and Dozot, Grégory and Lecorney, Stéphane and Rappo, Daniel and Hrynaszkiewicz, Iain}, + month = oct, + year = {2024}, + note = {Publisher: Public Library of Science}, + keywords = {Open science, Reproducibility, Careers, Bibliometrics, Citation analysis, Linear regression analysis, Research assessment, Science policy}, + pages = {e0311493}, + file = {Full Text PDF:/home/amy/Zotero/storage/HZKJK3YR/Colavizza et al. - 2024 - An analysis of the effects of sharing research dat.pdf:application/pdf}, +} + +@article{trisovic_large-scale_2022, + title = {A large-scale study on research code quality and execution}, + volume = {9}, + copyright = {2022 The Author(s)}, + issn = {2052-4463}, + url = {https://www.nature.com/articles/s41597-022-01143-6}, + doi = {10.1038/s41597-022-01143-6}, + abstract = {This article presents a study on the quality and execution of research code from publicly-available replication datasets at the Harvard Dataverse repository. Research code is typically created by a group of scientists and published together with academic papers to facilitate research transparency and reproducibility. For this study, we define ten questions to address aspects impacting research reproducibility and reuse. First, we retrieve and analyze more than 2000 replication datasets with over 9000 unique R files published from 2010 to 2020. Second, we execute the code in a clean runtime environment to assess its ease of reuse. Common coding errors were identified, and some of them were solved with automatic code cleaning to aid code execution. We find that 74\% of R files failed to complete without error in the initial execution, while 56\% failed when code cleaning was applied, showing that many errors can be prevented with good coding practices. We also analyze the replication datasets from journals’ collections and discuss the impact of the journal policy strictness on the code re-execution rate. Finally, based on our results, we propose a set of recommendations for code dissemination aimed at researchers, journals, and repositories.}, + language = {en}, + number = {1}, + urldate = {2024-11-14}, + journal = {Scientific Data}, + author = {Trisovic, Ana and Lau, Matthew K. and Pasquier, Thomas and Crosas, Mercè}, + month = feb, + year = {2022}, + note = {Publisher: Nature Publishing Group}, + keywords = {Software, Information technology, Research data}, + pages = {60}, + file = {Full Text PDF:/home/amy/Zotero/storage/YX35W8EE/Trisovic et al. - 2022 - A large-scale study on research code quality and e.pdf:application/pdf}, +} + +@misc{heather_protocol_2024, + title = {Protocol for assessing the computational reproducibility of discrete-event simulation models on {STARS}}, + url = {https://doi.org/10.5281/zenodo.12179846}, + abstract = {This protocol will be used to assess the computational reproducibility of published healthcare discrete-event simulation (DES) models created using Python or R. It forms part of the project STARS: “Sharing Tools and Artefacts for Reproducible Simulations in healthcare”.}, + publisher = {Zenodo}, + author = {Heather, Amy and Monks, Thomas and Harper, Alison and Mustafee, Navonil and Mayne, Andrew}, + month = jun, + year = {2024}, + doi = {10.5281/zenodo.12179846}, + file = {Full Text:/home/amy/Zotero/storage/Z4QU8XLV/Heather et al. - 2024 - Protocol for assessing the computational reproduci.pdf:application/pdf}, +} + +@article{andersen_time_2021, + title = {Time from submission to publication varied widely for biomedical journals: a systematic review}, + volume = {37}, + issn = {0300-7995}, + shorttitle = {Time from submission to publication varied widely for biomedical journals}, + url = {https://doi.org/10.1080/03007995.2021.1905622}, + doi = {10.1080/03007995.2021.1905622}, + abstract = {Fast dissemination of research is important for improving treatments and thus benefitting patients, caregivers, and researchers. However, getting scientific papers published may take a long time. The editorial handling time can be delayed by several processes both before and after acceptance of the paper. The aim of this study was to systematically review the editorial handling time of biomedical peer-reviewed literature (i.e. time from submission to publication). The protocol for this systematic review was registered in PROSPERO (CRD42020196238). PubMed and EMBASE were searched systematically on 29 May 2020. We included publications on the timespan between submission and publication for accepted articles published in biomedical journals. Of the 4197 unique studies identified in the search, 69 were included in the systematic review. The mean timespan from submission to publication varied from 91 to 639 days, while the median timespan varied from 70 to 558 days. Submission to acceptance and acceptance to publication timespans showed similar disparity with means ranging from 50 to 276 and 11 to 362 days, respectively. Data were too statistically heterogeneous to perform meta-analyses. Editorial handling times of journals varied widely from a few months to almost two years, which delays the availability of new evidence. The editorial handling time did not differ between submission-to-acceptance-time and acceptance-to-publication-time. Examining differences in editorial processes between journals with long and short editorial handling times may help uncover, which processes are frequent causes of delay and thereby where to improve.}, + number = {6}, + urldate = {2024-11-14}, + journal = {Current Medical Research and Opinion}, + author = {Andersen, Mikkel Zola and Fonnes, Siv and Rosenberg, Jacob}, + month = jun, + year = {2021}, + pmid = {33735591}, + note = {Publisher: Taylor \& Francis +\_eprint: https://doi.org/10.1080/03007995.2021.1905622}, + keywords = {systematic review, acceptance, journalology, publication delay, Publication time, submission}, + pages = {985--993}, + file = {Full Text PDF:/home/amy/Zotero/storage/YS2ZRM22/Andersen et al. - 2021 - Time from submission to publication varied widely .pdf:application/pdf}, +} + +@article{stodden_empirical_2018, + title = {An empirical analysis of journal policy effectiveness for computational reproducibility}, + volume = {115}, + url = {https://www.pnas.org/doi/full/10.1073/pnas.1708290115}, + doi = {10.1073/pnas.1708290115}, + abstract = {A key component of scientific communication is sufficient information for other researchers in the field to reproduce published findings. For computational and data-enabled research, this has often been interpreted to mean making available the raw data from which results were generated, the computer code that generated the findings, and any additional information needed such as workflows and input parameters. Many journals are revising author guidelines to include data and code availability. This work evaluates the effectiveness of journal policy that requires the data and code necessary for reproducibility be made available postpublication by the authors upon request. We assess the effectiveness of such a policy by (i) requesting data and code from authors and (ii) attempting replication of the published findings. We chose a random sample of 204 scientific papers published in the journal Science after the implementation of their policy in February 2011. We found that we were able to obtain artifacts from 44\% of our sample and were able to reproduce the findings for 26\%. We find this policy—author remission of data and code postpublication upon request—an improvement over no policy, but currently insufficient for reproducibility.}, + number = {11}, + urldate = {2024-11-14}, + journal = {Proceedings of the National Academy of Sciences}, + author = {Stodden, Victoria and Seiler, Jennifer and Ma, Zhaokun}, + month = mar, + year = {2018}, + note = {Publisher: Proceedings of the National Academy of Sciences}, + pages = {2584--2589}, + file = {Full Text PDF:/home/amy/Zotero/storage/8Z287AHJ/Stodden et al. - 2018 - An empirical analysis of journal policy effectiven.pdf:application/pdf}, +} + +@article{fisar_reproducibility_2024, + title = {Reproducibility in {Management} {Science}}, + volume = {70}, + issn = {0025-1909}, + url = {https://pubsonline.informs.org/doi/10.1287/mnsc.2023.03556}, + doi = {10.1287/mnsc.2023.03556}, + abstract = {With the help of more than 700 reviewers, we assess the reproducibility of nearly 500 articles published in the journal Management Science before and after the introduction of a new Data and Code Disclosure policy in 2019. When considering only articles for which data accessibility and hardware and software requirements were not an obstacle for reviewers, the results of more than 95\% of articles under the new disclosure policy could be fully or largely computationally reproduced. However, for 29\% of articles, at least part of the data set was not accessible to the reviewer. Considering all articles in our sample reduces the share of reproduced articles to 68\%. These figures represent a significant increase compared with the period before the introduction of the disclosure policy, where only 12\% of articles voluntarily provided replication materials, of which 55\% could be (largely) reproduced. Substantial heterogeneity in reproducibility rates across different fields is mainly driven by differences in data set accessibility. Other reasons for unsuccessful reproduction attempts include missing code, unresolvable code errors, weak or missing documentation, and software and hardware requirements and code complexity. Our findings highlight the importance of journal code and data disclosure policies and suggest potential avenues for enhancing their effectiveness. This paper was accepted by David Simchi-Levi, behavioral economics and decision analysis–fast track. Supplemental Material: The online appendices and data are available at https://doi.org/10.1287/mnsc.2023.03556.}, + number = {3}, + urldate = {2024-11-14}, + journal = {Management Science}, + author = {Fišar, Miloš and Greiner, Ben and Huber, Christoph and Katok, Elena and Ozkes, Ali I.}, + month = mar, + year = {2024}, + note = {Publisher: INFORMS}, + keywords = {reproducibility, crowd science, replication}, + pages = {1343--1356}, + file = {Full Text PDF:/home/amy/Zotero/storage/ULZ3XRPX/Fišar et al. - 2024 - Reproducibility in Management Science.pdf:application/pdf}, +} + +@misc{springer_nature_springer_2024, + title = {Springer {Nature} announces unified open code policy to better support open research practices}, + url = {https://group.springernature.com/gp/group/media/press-releases/unified-code-sharing-policy-promoting-open-science/26789930}, + urldate = {2024-11-14}, + author = {{Springer Nature}}, + month = feb, + year = {2024}, + file = {Springer Nature announces unified open code policy to better support open research practices | Springer Nature Group | Springer Nature:/home/amy/Zotero/storage/YAQ68D3J/26789930.html:text/html}, +} + +@article{hamilton_prevalence_2023, + title = {Prevalence and predictors of data and code sharing in the medical and health sciences: systematic review with meta-analysis of individual participant data}, + volume = {382}, + copyright = {© Author(s) (or their employer(s)) 2019. Re-use permitted under CC BY-NC. No commercial re-use. See rights and permissions. Published by BMJ.. http://creativecommons.org/licenses/by-nc/4.0/This is an Open Access article distributed in accordance with the Creative Commons Attribution Non Commercial (CC BY-NC 4.0) license, which permits others to distribute, remix, adapt, build upon this work non-commercially, and license their derivative works on different terms, provided the original work is properly cited and the use is non-commercial. See: http://creativecommons.org/licenses/by-nc/4.0/.}, + issn = {1756-1833}, + shorttitle = {Prevalence and predictors of data and code sharing in the medical and health sciences}, + url = {https://www.bmj.com/content/382/bmj-2023-075767}, + doi = {10.1136/bmj-2023-075767}, + abstract = {Objectives To synthesise research investigating data and code sharing in medicine and health to establish an accurate representation of the prevalence of sharing, how this frequency has changed over time, and what factors influence availability. +Design Systematic review with meta-analysis of individual participant data. +Data sources Ovid Medline, Ovid Embase, and the preprint servers medRxiv, bioRxiv, and MetaArXiv were searched from inception to 1 July 2021. Forward citation searches were also performed on 30 August 2022. +Review methods Meta-research studies that investigated data or code sharing across a sample of scientific articles presenting original medical and health research were identified. Two authors screened records, assessed the risk of bias, and extracted summary data from study reports when individual participant data could not be retrieved. Key outcomes of interest were the prevalence of statements that declared that data or code were publicly or privately available (declared availability) and the success rates of retrieving these products (actual availability). The associations between data and code availability and several factors (eg, journal policy, type of data, trial design, and human participants) were also examined. A two stage approach to meta-analysis of individual participant data was performed, with proportions and risk ratios pooled with the Hartung-Knapp-Sidik-Jonkman method for random effects meta-analysis. +Results The review included 105 meta-research studies examining 2 121 580 articles across 31 specialties. Eligible studies examined a median of 195 primary articles (interquartile range 113-475), with a median publication year of 2015 (interquartile range 2012-2018). Only eight studies (8\%) were classified as having a low risk of bias. Meta-analyses showed a prevalence of declared and actual public data availability of 8\% (95\% confidence interval 5\% to 11\%) and 2\% (1\% to 3\%), respectively, between 2016 and 2021. For public code sharing, both the prevalence of declared and actual availability were estimated to be {\textless}0.5\% since 2016. Meta-regressions indicated that only declared public data sharing prevalence estimates have increased over time. Compliance with mandatory data sharing policies ranged from 0\% to 100\% across journals and varied by type of data. In contrast, success in privately obtaining data and code from authors historically ranged between 0\% and 37\% and 0\% and 23\%, respectively. +Conclusions The review found that public code sharing was persistently low across medical research. Declarations of data sharing were also low, increasing over time, but did not always correspond to actual sharing of data. The effectiveness of mandatory data sharing policies varied substantially by journal and type of data, a finding that might be informative for policy makers when designing policies and allocating resources to audit compliance. +Systematic review registration Open Science Framework doi:10.17605/OSF.IO/7SX8U.}, + language = {en}, + urldate = {2024-11-15}, + journal = {BMJ}, + author = {Hamilton, Daniel G. and Hong, Kyungwan and Fraser, Hannah and Rowhani-Farid, Anisa and Fidler, Fiona and Page, Matthew J.}, + month = jul, + year = {2023}, + pmid = {37433624}, + note = {Publisher: British Medical Journal Publishing Group +Section: Research}, + pages = {e075767}, + file = {Full Text PDF:/home/amy/Zotero/storage/FILBGNV5/Hamilton et al. - 2023 - Prevalence and predictors of data and code sharing.pdf:application/pdf}, +} + +@article{loder_mandatory_2024, + title = {Mandatory data and code sharing for research published by {The} {BMJ}}, + volume = {384}, + copyright = {Published by the BMJ Publishing Group Limited. For permission to use (where not already granted under a licence) please go to http://group.bmj.com/group/rights-licensing/permissions}, + issn = {1756-1833}, + url = {https://www.bmj.com/content/384/bmj.q324}, + doi = {10.1136/bmj.q324}, + abstract = {{\textless}p{\textgreater}New policy requires authors to share analytic codes from all studies and data from all trials {\textless}/p{\textgreater}}, + language = {en}, + urldate = {2024-11-15}, + journal = {BMJ}, + author = {Loder, Elizabeth and Macdonald, Helen and Bloom, Theodora and Abbasi, Kamran}, + month = mar, + year = {2024}, + pmid = {38443070}, + note = {Publisher: British Medical Journal Publishing Group +Section: Editorial}, + pages = {q324}, + file = {Full Text PDF:/home/amy/Zotero/storage/AV7LWM7G/Loder et al. - 2024 - Mandatory data and code sharing for research publi.pdf:application/pdf}, +} + +@article{noauthor_code_2023, + title = {Code sharing in the spotlight}, + volume = {3}, + copyright = {2023 Springer Nature America, Inc.}, + issn = {2662-8457}, + url = {https://www.nature.com/articles/s43588-023-00566-4}, + doi = {10.1038/s43588-023-00566-4}, + abstract = {The Year of Open Science has highlighted the importance of sharing the code associated with peer-reviewed manuscripts. We at Nature Computational Science provide support — via policies and implementations within our submission system — to facilitate this task.}, + language = {en}, + number = {11}, + urldate = {2024-11-15}, + journal = {Nature Computational Science}, + month = nov, + year = {2023}, + note = {Publisher: Nature Publishing Group}, + keywords = {Software, Computational science, Publishing, Policy}, + pages = {907--907}, + file = {Full Text PDF:/home/amy/Zotero/storage/6BIB6HPL/2023 - Code sharing in the spotlight.pdf:application/pdf}, +} + +@article{cadwallader_collaborating_2021, + title = {Collaborating with our community to increase code sharing}, + volume = {17}, + issn = {1553-7358}, + url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1008867}, + doi = {10.1371/journal.pcbi.1008867}, + language = {en}, + number = {3}, + urldate = {2024-11-15}, + journal = {PLOS Computational Biology}, + author = {Cadwallader, Lauren and Papin, Jason A. and Gabhann, Feilim Mac and Kirk, Rebecca}, + month = mar, + year = {2021}, + note = {Publisher: Public Library of Science}, + keywords = {Scientific publishing, Reproducibility, Scientists, Survey research, Surveys, Computational biology, Publication ethics, Science policy}, + pages = {e1008867}, + file = {Full Text PDF:/home/amy/Zotero/storage/KMY2QHSW/Cadwallader et al. - 2021 - Collaborating with our community to increase code .pdf:application/pdf}, +} + +@article{noauthor_promoting_2024, + title = {Promoting reproduction and replication at scale}, + volume = {8}, + copyright = {2024 Springer Nature Limited}, + issn = {2397-3374}, + url = {https://www.nature.com/articles/s41562-024-01818-7}, + doi = {10.1038/s41562-024-01818-7}, + abstract = {Nature Human Behaviour is partnering with the Institute for Replication for the large-scale reproduction and replication of our published research.}, + language = {en}, + number = {1}, + urldate = {2024-11-15}, + journal = {Nature Human Behaviour}, + month = jan, + year = {2024}, + note = {Publisher: Nature Publishing Group}, + keywords = {general, Behavioral Sciences, Experimental Psychology, Life Sciences, Microeconomics, Neurosciences, Personality and Social Psychology}, + pages = {1--1}, + file = {Full Text PDF:/home/amy/Zotero/storage/LV3CJYG2/2024 - Promoting reproduction and replication at scale.pdf:application/pdf}, +} + +@article{hardwicke_transparency_2024, + title = {Transparency {Is} {Now} the {Default} at {Psychological} {Science}}, + volume = {35}, + issn = {0956-7976}, + url = {https://doi.org/10.1177/09567976231221573}, + doi = {10.1177/09567976231221573}, + language = {en}, + number = {7}, + urldate = {2024-11-15}, + journal = {Psychological Science}, + author = {Hardwicke, Tom E. and Vazire, Simine}, + month = jul, + year = {2024}, + note = {Publisher: SAGE Publications Inc}, + pages = {708--711}, + file = {SAGE PDF Full Text:/home/amy/Zotero/storage/MQQMBMNC/Hardwicke and Vazire - 2024 - Transparency Is Now the Default at Psychological S.pdf:application/pdf}, +} + +@misc{centre_for_open_science_top_nodate, + title = {{TOP} {Guidelines}}, + url = {https://www.cos.io/initiatives/top-guidelines}, + urldate = {2024-11-15}, + journal = {Centre for Open Science}, + author = {{Centre for Open Science}}, + file = {TOP Guidelines:/home/amy/Zotero/storage/QWL4AEIS/top-guidelines.html:text/html}, +} + +@article{philip_simulation_2023, + title = {Simulation modelling of hospital outpatient department: a review of the literature and bibliometric analysis}, + volume = {99}, + issn = {0037-5497}, + shorttitle = {Simulation modelling of hospital outpatient department}, + url = {https://doi.org/10.1177/00375497221139282}, + doi = {10.1177/00375497221139282}, + abstract = {The increase in demand for outpatient departments (OPDs) has contributed to overcrowded clinics and patient dissatisfaction. Computer simulation can help decision-makers meet the operational challenge of balancing the demand for outpatient services with considerations of available capacity. The paper presents a synthesis of the literature on simulation modeling in OPD using two approaches: a bibliometric analysis (employing keyword co-occurrence network) and a literature classification focusing on OPD strategy, OPD performance measures, and simulation techniques. Our review is based on 161 papers, published between 2006 and 2020, identified through a methodological search of the literature. The objective of the review is threefold: (1) to identify the major and emerging research issues in general and specialized OPD, (2) to find the commonly used performance measures in OPD and how it is associated with the strategies used to improve the performance, and (3) to identify the commonly used simulation methods for OPD modeling. A key finding from the bibliometric analysis is that most OPD research can be classified under one of the four clusters—“organization and management,”“patient satisfaction,”“overbooking,” and “performance.” We also find that patient waiting time has received much attention among the performance measures reported in the literature, followed by server idle time/overtime (server here is the OPD consultant or other healthcare resource). Our review serves as a key reference point for scholars, practitioners, students, and healthcare stakeholders, and those who use quantitative tools to aid operational decision-making.}, + language = {en}, + number = {6}, + urldate = {2024-11-15}, + journal = {SIMULATION}, + author = {Philip, Aby M and Prasannavenkatesan, Shanmugam and Mustafee, Navonil}, + month = jun, + year = {2023}, + note = {Publisher: SAGE Publications Ltd STM}, + pages = {573--597}, + file = {SAGE PDF Full Text:/home/amy/Zotero/storage/4RHLN4JM/Philip et al. - 2023 - Simulation modelling of hospital outpatient depart.pdf:application/pdf}, +} + +@article{roy_healthcare_2021, + title = {Healthcare services: {A} systematic review of patient-centric logistics issues using simulation}, + volume = {72}, + issn = {0160-5682}, + shorttitle = {Healthcare services}, + url = {https://doi.org/10.1080/01605682.2020.1790306}, + doi = {10.1080/01605682.2020.1790306}, + abstract = {Healthcare has material-centric external and patient-centric internal logistics. Researchers widely use simulation approaches to model healthcare internal logistics due to the problem complexity. There is a need to map the existing knowledge base to systematically identify the emerging research themes of this domain. This work presents a systematic literature review to identify the patient-centric logistics issues in healthcare modelled using simulation. In all, 583 papers published from 2008 to 2017 in the Clarivate Analytics Web of Science database have been collected; 238 articles were shortlisted for the review. Using keyword co-occurrence and cluster analysis, thirteen research clusters are identified, with eight of them central to the research domain and deserving further attention. Among the simulation approaches, discrete event simulation is most prevalent followed by hybrid approaches (that use two or more simulation techniques or simulation combined with analytical methods), system dynamics, agent-based simulation, and Monte Carlo simulation.}, + number = {10}, + urldate = {2024-11-15}, + journal = {Journal of the Operational Research Society}, + author = {Roy, Sumanta and Prasanna Venkatesan, Shanmugam and Goh, Mark}, + month = oct, + year = {2021}, + note = {Publisher: Taylor \& Francis +\_eprint: https://doi.org/10.1080/01605682.2020.1790306}, + keywords = {simulation, bibliometric, co-occurrence analysis, Healthcare logistics, literature review}, + pages = {2342--2364}, + file = {Full Text PDF:/home/amy/Zotero/storage/PG7B7NGA/Roy et al. - 2021 - Healthcare services A systematic review of patien.pdf:application/pdf}, +} + +@article{salleh_simulation_2017, + title = {Simulation {Modelling} in {Healthcare}: {An} {Umbrella} {Review} of {Systematic} {Literature} {Reviews}}, + volume = {35}, + issn = {1179-2027}, + shorttitle = {Simulation {Modelling} in {Healthcare}}, + url = {https://doi.org/10.1007/s40273-017-0523-3}, + doi = {10.1007/s40273-017-0523-3}, + abstract = {Numerous studies examine simulation modelling in healthcare. These studies present a bewildering array of simulation techniques and applications, making it challenging to characterise the literature.}, + language = {en}, + number = {9}, + urldate = {2024-11-15}, + journal = {PharmacoEconomics}, + author = {Salleh, Syed and Thokala, Praveen and Brennan, Alan and Hughes, Ruby and Booth, Andrew}, + month = sep, + year = {2017}, + pages = {937--949}, + file = {Full Text PDF:/home/amy/Zotero/storage/8G6ZQFMT/Salleh et al. - 2017 - Simulation Modelling in Healthcare An Umbrella Re.pdf:application/pdf}, +} + +@article{salmon_structured_2018, + title = {A structured literature review of simulation modelling applied to {Emergency} {Departments}: {Current} patterns and emerging trends}, + volume = {19}, + issn = {2211-6923}, + shorttitle = {A structured literature review of simulation modelling applied to {Emergency} {Departments}}, + url = {https://www.sciencedirect.com/science/article/pii/S2211692317301042}, + doi = {10.1016/j.orhc.2018.01.001}, + abstract = {The public importance, wait-for-treatment ethos and clear geographic layout of Emergency Departments (EDs) has contributed to them being one of the most commonly modelled systems in healthcare Operational Research (OR). EDs are presently contending with higher than ever attendances, to which clinical research does not appear to have a comprehensive solution, whilst OR methodologies still need to command the trust of decision makers. With potentially greater acceptance of OR methodologies driven by heightened efforts to engage clinicians in evidence based approaches, we present a comprehensive review of the current literature. Whilst not the first in this area, our review is more broadly focused and thus able to serve both as a resource for modellers of methodology and study design, and as an introduction for decision makers. Our systematic literature search aimed to identify all English language papers from the year 2000 onward. We categorise papers using the defined dimensions of purpose, application area, method, scope and sponsor (originator). Of 254 retrievals, we find that new publications are currently appearing at approximately 25 per year, up seven fold since 2000. We find positive trends in terms of recent publications (75\% since 2008) as well as a trend towards achieving publication in journals, including healthcare related journals, which may assist in bringing simulation to a clinical audience and facilitating future engagement. The majority of projects appear to be of academic origin, based on Discrete Event Simulation, and focused on capacity, process and workforce issues at an operational level. However, the use of hybrid modelling may be associated with a more strategic outlook, as do projects originated at the request of healthcare organisations. We present a selection of case studies to illustrate both our classification and findings, and suggest directions for further research.}, + urldate = {2024-11-15}, + journal = {Operations Research for Health Care}, + author = {Salmon, Andrew and Rachuba, Sebastian and Briscoe, Simon and Pitt, Martin}, + month = dec, + year = {2018}, + keywords = {Simulation, 4-hour target, Emergency Department, Systematic, Taxonomy, Winter pressure}, + pages = {1--13}, + file = {ScienceDirect Snapshot:/home/amy/Zotero/storage/QFLYWWNC/S2211692317301042.html:text/html}, +} + +@article{so_reusability_2023, + title = {Reusability report: {Evaluating} reproducibility and reusability of a fine-tuned model to predict drug response in cancer patient samples}, + volume = {5}, + copyright = {2023 The Author(s), under exclusive licence to Springer Nature Limited}, + issn = {2522-5839}, + shorttitle = {Reusability report}, + url = {https://www.nature.com/articles/s42256-023-00688-4}, + doi = {10.1038/s42256-023-00688-4}, + abstract = {Machine learning and artificial intelligence methods are increasingly being used in personalized medicine, including precision oncology. Ma et al. (Nature Cancer 2021) have developed a new method called ‘transfer of cell line response prediction’ (TCRP) to train predictors of drug response in cancer cell lines and optimize their performance in higher complex cancer model systems via few-shot learning. TCRP has been presented as a successful modelling approach in multiple case studies. Given the importance of this approach for assisting clinicians in their treatment decision processes, we sought to independently reproduce the authors’ findings and improve the reusability of TCRP in new case studies, including validation in clinical-trial datasets—a high bar for drug-response prediction. Our reproducibility results, while not reaching the same level of superiority as those of the original authors, were able to confirm the superiority of TCRP in the original clinical context. Our reusability results indicate that, in the majority of novel clinical contexts, TCRP remains the superior method for predicting response for both preclinical and clinical settings. Our results thus support the superiority of TCRP over established statistical and machine learning approaches in preclinical and clinical settings. We also developed new resources to increase the reusability of the TCRP model for future improvements and validation studies.}, + language = {en}, + number = {7}, + urldate = {2024-11-15}, + journal = {Nature Machine Intelligence}, + author = {So, Emily and Yu, Fengqing and Wang, Bo and Haibe-Kains, Benjamin}, + month = jul, + year = {2023}, + note = {Publisher: Nature Publishing Group}, + keywords = {Computational models, Predictive medicine}, + pages = {792--798}, + file = {Full Text PDF:/home/amy/Zotero/storage/TP2B5XHD/So et al. - 2023 - Reusability report Evaluating reproducibility and.pdf:application/pdf}, +} + +@article{xu_reusability_2024, + title = {Reusability report: {Uncovering} associations in biomedical bipartite networks via a bilinear attention network with domain adaptation}, + volume = {6}, + copyright = {2024 The Author(s), under exclusive licence to Springer Nature Limited}, + issn = {2522-5839}, + shorttitle = {Reusability report}, + url = {https://www.nature.com/articles/s42256-024-00822-w}, + doi = {10.1038/s42256-024-00822-w}, + abstract = {Conditional domain adversarial learning presents a promising approach for enhancing the generalizability of deep learning-based methods. Inspired by the efficacy of conditional domain adversarial networks, Bai and colleagues introduced DrugBAN, a methodology designed to explicitly learn pairwise local interactions between drugs and targets. DrugBAN leverages drug molecular graphs and target protein sequences, employing conditional domain adversarial networks to improve the ability to adapt to out-of-distribution data and thereby ensuring superior prediction accuracy for new drug–target pairs. Here we examine the reusability of DrugBAN and extend the evaluation of its generalizability across a wider range of biomedical contexts beyond the original datasets. Various clustering-based strategies are implemented to resplit the source and target domains to assess the robustness of DrugBAN. We also apply this cross-domain adaptation technique to the prediction of cell line–drug responses and mutation–drug associations. The analysis serves as a stepping-off point to better understand and establish a general template applicable to link prediction tasks in biomedical bipartite networks.}, + language = {en}, + number = {4}, + urldate = {2024-11-15}, + journal = {Nature Machine Intelligence}, + author = {Xu, Tao and Shi, Haoyuan and Gao, Wanling and Wang, Xiaosong and Yue, Zhenyu}, + month = apr, + year = {2024}, + note = {Publisher: Nature Publishing Group}, + keywords = {Computational biology and bioinformatics, Drug discovery}, + pages = {461--466}, + file = {Full Text PDF:/home/amy/Zotero/storage/SU4HG8YH/Xu et al. - 2024 - Reusability report Uncovering associations in bio.pdf:application/pdf}, +} + +@article{mcmanus_barriers_2019, + title = {Barriers and {Facilitators} to {Model} {Replication} {Within} {Health} {Economics}}, + volume = {22}, + issn = {1098-3015}, + url = {https://www.sciencedirect.com/science/article/pii/S1098301519321928}, + doi = {10.1016/j.jval.2019.04.1928}, + abstract = {Background +Model replication is important because it enables researchers to check research integrity and transparency and, potentially, to inform the model conceptualization process when developing a new or updated model. +Objective +The aim of this study was to evaluate the replicability of published decision analytic models and to identify the barriers and facilitators to replication. +Methods +Replication attempts of 5 published economic modeling studies were made. The replications were conducted using only publicly available information within the manuscripts and supplementary materials. The replicator attempted to reproduce the key results detailed in the paper, for example, the total cost, total outcomes, and if applicable, incremental cost-effectiveness ratio reported. Although a replication attempt was not explicitly defined as a success or failure, the replicated results were compared for percentage difference to the original results. +Results +In conducting the replication attempts, common barriers and facilitators emerged. For most case studies, the replicator needed to make additional assumptions when recreating the model. This was often exacerbated by conflicting information being presented in the text and the tables. Across the case studies, the variation between original and replicated results ranged from −4.54\% to 108.00\% for costs and −3.81\% to 0.40\% for outcomes. +Conclusion +This study demonstrates that although models may appear to be comprehensively reported, it is often not enough to facilitate a precise replication. Further work is needed to understand how to improve model transparency and in turn increase the chances of replication, thus ensuring future usability.}, + number = {9}, + urldate = {2024-11-19}, + journal = {Value in Health}, + author = {McManus, Emma and Turner, David and Gray, Ewan and Khawar, Haseeb and Okoli, Toochukwu and Sach, Tracey}, + month = sep, + year = {2019}, + keywords = {reproducibility, replication, decision-analytic modeling, reporting transparency}, + pages = {1018--1025}, + file = {Accepted Version:/home/amy/Zotero/storage/VSFIY5LG/McManus et al. - 2019 - Barriers and Facilitators to Model Replication Wit.pdf:application/pdf;ScienceDirect Snapshot:/home/amy/Zotero/storage/W7TCSVPA/S1098301519321928.html:text/html}, +} + +@article{philips_good_2006, + title = {Good {Practice} {Guidelines} for {Decision}-{Analytic} {Modelling} in {Health} {Technology} {Assessment}}, + volume = {24}, + issn = {1179-2027}, + url = {https://doi.org/10.2165/00019053-200624040-00006}, + doi = {10.2165/00019053-200624040-00006}, + abstract = {The use of decision-analytic modelling for the purpose of health technology assessment (HTA) has increased dramatically in recent years. Several guidelines for best practice have emerged in the literature; however, there is no agreed standard for what constitutes a ‘good model’ or how models should be formally assessed. The objective of this paper is to identify, review and consolidate existing guidelines on the use of decision-analytic modelling for the purpose of HTA and to develop a consistent framework against which the quality of models may be assessed.}, + language = {en}, + number = {4}, + urldate = {2024-11-19}, + journal = {PharmacoEconomics}, + author = {Philips, Zoë and Bojke, Laura and Sculpher, Mark and Claxton, Karl and Golder, Su}, + month = apr, + year = {2006}, + keywords = {Appraisal Committee, Decision Uncertainty, Health Technology Assessment, Probabilistic Sensitivity Analysis, Review Team}, + pages = {355--371}, + file = {Full Text PDF:/home/amy/Zotero/storage/C3H9RES7/Philips et al. - 2006 - Good Practice Guidelines for Decision-Analytic Mod.pdf:application/pdf}, +} + +@article{robinson_simulation_2023, + title = {Simulation as a tool to model potential workflow enhancements in radiotherapy treatment pathways – {A} systematic review}, + volume = {24}, + copyright = {© 2023 The Authors. Journal of Applied Clinical Medical Physics published by Wiley Periodicals, LLC on behalf of The American Association of Physicists in Medicine.}, + issn = {1526-9914}, + url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/acm2.14132}, + doi = {10.1002/acm2.14132}, + abstract = {This systematic review aimed to synthesize and summarize the use of simulation of radiotherapy pathways. The objective was to establish the suitability of those simulations in modeling the potential introduction of processes and technologies to speed up radiotherapy pathways. A systematic literature search was carried out using PubMed and Scopus databases to evaluate the use of simulation in radiotherapy pathways. Full journal articles and conference proceedings were considered, and the search was limited to the English language only. To be eligible for inclusion, articles had to model multiple sequential processes in the radiotherapy pathway concurrently to demonstrate the suitability of simulation modeling in typical pathways. Papers solely modeling scheduling, capacity, or queuing strategies were excluded. In total, 151 potential studies were identified and screened to find 18 relevant studies in October 2022. Studies showed that various pathways could be modeled, including the entire pathway from referral to end of treatment or the constituent phases such as pre-treatment, treatment, or other subcomponents. The data required to generate models varied from study to study, but at least 3 months of data were needed. This review demonstrates that modeling and simulation of radiotherapy pathways are feasible and that model output matches real-world systems. Validated models give researchers confidence to modify models with potential workflow enhancements to assess their potential effect on real-world systems. It is recommended that researchers follow best practice guidelines when building models to ensure that they are fit for purpose and to enable decision makers to have confidence in their results.}, + language = {en}, + number = {10}, + urldate = {2024-11-19}, + journal = {Journal of Applied Clinical Medical Physics}, + author = {Robinson, Andrew and Asaduzzaman, Md and Jena, Raj and Naemi, Roozbeh}, + year = {2023}, + note = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/acm2.14132}, + keywords = {discrete event simulation, computer simulations, operations research, pathway, radiotherapy, waiting times}, + pages = {e14132}, + file = {Full Text PDF:/home/amy/Zotero/storage/I7U9L36U/Robinson et al. - 2023 - Simulation as a tool to model potential workflow e.pdf:application/pdf;Snapshot:/home/amy/Zotero/storage/3X648CQY/acm2.html:text/html}, +} + +@phdthesis{nwanosike_direct_2023, + title = {Direct {Oral} {Anticoagulants} ({DOACs}) use in patients with renal insufficiency and obesity}, + url = {https://pure.hud.ac.uk/ws/portalfiles/portal/83727839/Final_thesis_Nwanosike.pdf}, + language = {en}, + school = {University of Huddersfield}, + author = {Nwanosike, Ezekwesiri Michael}, + month = sep, + year = {2023}, + file = {Nwanosike - DIRECT ORAL ANTICOAGULANTS (DOACs) USE IN PATIENTS.pdf:/home/amy/Zotero/storage/95SMRJJ5/Nwanosike - DIRECT ORAL ANTICOAGULANTS (DOACs) USE IN PATIENTS.pdf:application/pdf}, +} + +@misc{heather_stars_2024, + title = {{STARS}: {Computational} reproducibility of {Hernandez} et al. 2015}, + shorttitle = {{STARS}}, + url = {https://doi.org/10.5281/zenodo.13971045}, + abstract = {This repository forms part of work package 1 on the project STARS: Sharing Tools and Artefacts for Reproducible Simulations. It assesses the computational reproducibility of: Hernandez, I., Ramirez-Marquez, J., Starr, D., McKay, R., Guthartz, S., Motherwell, M., Barcellona, J. Optimal staffing strategies for points of dispensing. Computers \& Industrial Engineering 83 (2015). https://doi.org/10.1016/j.cie.2015.02.015.}, + urldate = {2024-11-19}, + publisher = {Zenodo}, + author = {Heather, Amy and Monks, Thomas and Harper, Alison}, + month = oct, + year = {2024}, + doi = {10.5281/zenodo.13971045}, + file = {Snapshot:/home/amy/Zotero/storage/SZR7H3ND/13971045.html:text/html}, +} + +@misc{anagnostou_charm_2022, + title = {{CHARM}: {dynamiC} {Hospital} {wARd} {Management}}, + url = {https://doi.org/10.17633/rd.brunel.18517892.v1}, + publisher = {Brunel University London}, + author = {Anagnostou, Anastasia}, + month = jan, + year = {2022}, +}