references.bib

@article{borchering_public_2023,
	title = {Public health impact of the {U}.{S}. {Scenario} {Modeling} {Hub}},
	volume = {44},
	doi = {10.1016/j.epidem.2023.100705},
	abstract = {Beginning in December 2020, the COVID-19 Scenario Modeling Hub has provided quantitative scenario-based projections for cases, hospitalizations, and deaths, aggregated across up to nine modeling groups. Projections spanned multiple months into the future and provided timely information on potential impacts of epidemiological uncertainties and interventions. Projections results were shared with the public, public health partners, and the Centers for Disease Control COVID-19 Response Team. The projections provided insights on situational awareness and informed decision-making to mitigate COVID-19 disease burden (e.g., vaccination strategies). By aggregating projections from multiple modeling teams, the Scenario Modeling Hub provided rapidly synthesized information in times of great uncertainty and conveyed possible trajectories in the presence of emerging variants. Here we detail several use cases of these projections in public health practice and communication, including assessments of whether modeling results directly or indirectly informed public health communication or guidance. These include multiple examples where comparisons of projected COVID-19 disease outcomes under different vaccination scenarios were used to inform Advisory Committee for Immunization Practices recommendations. We also describe challenges and lessons learned during this highly beneficial collaboration.},
	language = {eng},
	journal = {Epidemics},
	author = {Borchering, Rebecca K. and Healy, Jessica M. and Cadwell, Betsy L. and Johansson, Michael A. and Slayton, Rachel B. and Wallace, Megan and Biggerstaff, Matthew},
	month = sep,
	year = {2023},
	keywords = {COVID-19, Humans, Public health, Public Health, Scenario projections, Vaccination},
	pages = {100705},
}

@article{yamana_superensemble_2016,
	title = {Superensemble forecasts of dengue outbreaks},
	volume = {13},
	doi = {10.1098/rsif.2016.0410},
	abstract = {In recent years, a number of systems capable of predicting future infectious disease incidence have been developed. As more of these systems are operationalized, it is important that the forecasts generated by these different approaches be formally reconciled so that individual forecast error and bias are reduced. Here we present a first example of such multi-system, or superensemble, forecast. We develop three distinct systems for predicting dengue, which are applied retrospectively to forecast outbreak characteristics in San Juan, Puerto Rico. We then use Bayesian averaging methods to combine the predictions from these systems and create superensemble forecasts. We demonstrate that on average, the superensemble approach produces more accurate forecasts than those made from any of the individual forecasting systems.},
	number = {123},
	journal = {Journal of The Royal Society Interface},
	author = {Yamana, Teresa K. and Kandula, Sasikiran and Shaman, Jeffrey},
	month = oct,
	year = {2016},
	keywords = {infectious disease, dengue, forecast, Bayesian model averaging, superensemble},
	pages = {20160410},
}

@article{colon-gonzalez_probabilistic_2021,
	title = {Probabilistic seasonal dengue forecasting in {Vietnam}: {A} modelling study using superensembles},
	volume = {18},
	shorttitle = {Probabilistic seasonal dengue forecasting in {Vietnam}},
	doi = {10.1371/journal.pmed.1003542},
	abstract = {Background With enough advanced notice, dengue outbreaks can be mitigated. As a climate-sensitive disease, environmental conditions and past patterns of dengue can be used to make predictions about future outbreak risk. These predictions improve public health planning and decision-making to ultimately reduce the burden of disease. Past approaches to dengue forecasting have used seasonal climate forecasts, but the predictive ability of a system using different lead times in a year-round prediction system has been seldom explored. Moreover, the transition from theoretical to operational systems integrated with disease control activities is rare. Methods and findings We introduce an operational seasonal dengue forecasting system for Vietnam where Earth observations, seasonal climate forecasts, and lagged dengue cases are used to drive a superensemble of probabilistic dengue models to predict dengue risk up to 6 months ahead. Bayesian spatiotemporal models were fit to 19 years (2002–2020) of dengue data at the province level across Vietnam. A superensemble of these models then makes probabilistic predictions of dengue incidence at various future time points aligned with key Vietnamese decision and planning deadlines. We demonstrate that the superensemble generates more accurate predictions of dengue incidence than the individual models it incorporates across a suite of time horizons and transmission settings. Using historical data, the superensemble made slightly more accurate predictions (continuous rank probability score [CRPS] = 66.8, 95\% CI 60.6–148.0) than a baseline model which forecasts the same incidence rate every month (CRPS = 79.4, 95\% CI 78.5–80.5) at lead times of 1 to 3 months, albeit with larger uncertainty. The outbreak detection capability of the superensemble was considerably larger (69\%) than that of the baseline model (54.5\%). Predictions were most accurate in southern Vietnam, an area that experiences semi-regular seasonal dengue transmission. The system also demonstrated added value across multiple areas compared to previous practice of not using a forecast. We use the system to make a prospective prediction for dengue incidence in Vietnam for the period May to October 2020. Prospective predictions made with the superensemble were slightly more accurate (CRPS = 110, 95\% CI 102–575) than those made with the baseline model (CRPS = 125, 95\% CI 120–168) but had larger uncertainty. Finally, we propose a framework for the evaluation of probabilistic predictions. Despite the demonstrated value of our forecasting system, the approach is limited by the consistency of the dengue case data, as well as the lack of publicly available, continuous, and long-term data sets on mosquito control efforts and serotype-specific case data. Conclusions This study shows that by combining detailed Earth observation data, seasonal climate forecasts, and state-of-the-art models, dengue outbreaks can be predicted across a broad range of settings, with enough lead time to meaningfully inform dengue control. While our system omits some important variables not currently available at a subnational scale, the majority of past outbreaks could be predicted up to 3 months ahead. Over the next 2 years, the system will be prospectively evaluated and, if successful, potentially extended to other areas and other climate-sensitive disease systems.},
	language = {en},
	number = {3},
	journal = {PLOS Medicine},
	author = {Colón-González, Felipe J. and Bastos, Leonardo Soares and Hofmann, Barbara and Hopkin, Alison and Harpham, Quillon and Crocker, Tom and Amato, Rosanna and Ferrario, Iacopo and Moschini, Francesca and James, Samuel and Malde, Sajni and Ainscoe, Eleanor and Nam, Vu Sinh and Tan, Dang Quang and Khoa, Nguyen Duc and Harrison, Mark and Tsarouchi, Gina and Lumbroso, Darren and Brady, Oliver J. and Lowe, Rachel},
	month = mar,
	year = {2021},
	keywords = {Forecasting, Dengue fever, Public and occupational health, Decision making, Seasons, Humidity, Mosquitoes, Vietnam},
	pages = {e1003542},
}

@article{paireau_ensemble_2022,
	title = {An ensemble model based on early predictors to forecast {COVID}-19 health care demand in {France}},
	volume = {119},
	doi = {10.1073/pnas.2103302119},
	number = {18},
	journal = {Proceedings of the National Academy of Sciences},
	author = {Paireau, Juliette and Andronico, Alessio and Hozé, Nathanaël and Layan, Maylis and Crépey, Pascal and Roumagnac, Alix and Lavielle, Marc and Boëlle, Pierre-Yves and Cauchemez, Simon},
	month = may,
	year = {2022},
	pages = {e2103302119},
}


@article{ray_prediction_2018,
	title = {Prediction of infectious disease epidemics via weighted density ensembles},
	volume = {14},
	doi = {10.1371/journal.pcbi.1005910},
	abstract = {Accurate and reliable predictions of infectious disease dynamics can be valuable to public health organizations that plan interventions to decrease or prevent disease transmission. A great variety of models have been developed for this task, using different model structures, covariates, and targets for prediction. Experience has shown that the performance of these models varies; some tend to do better or worse in different seasons or at different points within a season. Ensemble methods combine multiple models to obtain a single prediction that leverages the strengths of each model. We considered a range of ensemble methods that each form a predictive density for a target of interest as a weighted sum of the predictive densities from component models. In the simplest case, equal weight is assigned to each component model; in the most complex case, the weights vary with the region, prediction target, week of the season when the predictions are made, a measure of component model uncertainty, and recent observations of disease incidence. We applied these methods to predict measures of influenza season timing and severity in the United States, both at the national and regional levels, using three component models. We trained the models on retrospective predictions from 14 seasons (1997/1998-2010/2011) and evaluated each model's prospective, out-of-sample performance in the five subsequent influenza seasons. In this test phase, the ensemble methods showed average performance that was similar to the best of the component models, but offered more consistent performance across seasons than the component models. Ensemble methods offer the potential to deliver more reliable predictions to public health decision makers.},
	language = {eng},
	number = {2},
	journal = {PLOS computational biology},
	author = {Ray, Evan L. and Reich, Nicholas G.},
	month = feb,
	year = {2018},
	keywords = {Epidemics, Humans, Models, Statistical, Reproducibility of Results, Public Health, Computational Biology, United States, Communicable Diseases, Incidence, Centers for Disease Control and Prevention, U.S., Influenza, Human, Seasons, Algorithms, Infectious Disease Medicine, Models, Biological, Predictive Value of Tests, Prospective Studies, Retrospective Studies, Uncertainty},
	pages = {e1005910},
}


@article{ray_comparing_2023,
	title = {Comparing trained and untrained probabilistic ensemble forecasts of {COVID}-19 cases and deaths in the {United} {States}},
	journal = {International Journal of Forecasting},
	volume = {39},
	doi = {10.1016/j.ijforecast.2022.06.005},
	abstract = {The U.S. COVID-19 Forecast Hub aggregates forecasts of the short-term burden of COVID-19 in the United States from many contributing teams. We study methods for building an ensemble that combines forecasts from these teams. These experiments have informed the ensemble methods used by the Hub. To be most useful to policymakers, ensemble forecasts must have stable performance in the presence of two key characteristics of the component forecasts: (1) occasional misalignment with the reported data, and (2) instability in the relative performance of component forecasters over time. Our results indicate that in the presence of these challenges, an untrained and robust approach to ensembling using an equally weighted median of all component forecasts is a good choice to support public health decision-makers. In settings where some contributing forecasters have a stable record of good performance, trained ensembles that give those forecasters higher weight can also be helpful.},
	number = {3},
	author = {Ray, Evan L. and Brooks, Logan C. and Bien, Jacob and Biggerstaff, Matthew and Bosse, Nikos I. and Bracher, Johannes and Cramer, Estee Y. and Funk, Sebastian and Gerding, Aaron and Johansson, Michael A. and Rumack, Aaron and Wang, Yijin and Zorn, Martha and Tibshirani, Ryan J. and Reich, Nicholas G.},
	month = jul,
	year = {2023},
	keywords = {Epidemiology, COVID-19, Ensemble, Health forecasting, Quantile combination},
	pages = {1366--1383},
}


@article{reich_accuracy_2019,
	title = {Accuracy of real-time multi-model ensemble forecasts for seasonal influenza in the {U}.{S}},
	volume = {15},
	doi = {10.1371/journal.pcbi.1007486},
	abstract = {Seasonal influenza results in substantial annual morbidity and mortality in the United States and worldwide. Accurate forecasts of key features of influenza epidemics, such as the timing and severity of the peak incidence in a given season, can inform public health response to outbreaks. As part of ongoing efforts to incorporate data and advanced analytical methods into public health decision-making, the United States Centers for Disease Control and Prevention (CDC) has organized seasonal influenza forecasting challenges since the 2013/2014 season. In the 2017/2018 season, 22 teams participated. A subset of four teams created a research consortium called the FluSight Network in early 2017. During the 2017/2018 season they worked together to produce a collaborative multi-model ensemble that combined 21 separate component models into a single model using a machine learning technique called stacking. This approach creates a weighted average of predictive densities where the weight for each component is determined by maximizing overall ensemble accuracy over past seasons. In the 2017/2018 influenza season, one of the largest seasonal outbreaks in the last 15 years, this multi-model ensemble performed better on average than all individual component models and placed second overall in the CDC challenge. It also outperformed the baseline multi-model ensemble created by the CDC that took a simple average of all models submitted to the forecasting challenge. This project shows that collaborative efforts between research teams to develop ensemble forecasting approaches can bring measurable improvements in forecast accuracy and important reductions in the variability of performance from year to year. Efforts such as this, that emphasize real-time testing and evaluation of forecasting models and facilitate the close collaboration between public health officials and modeling researchers, are essential to improving our understanding of how best to use forecasts to improve public health response to seasonal and emerging epidemic threats.},
	language = {eng},
	number = {11},
	journal = {PLOS computational biology},
	author = {Reich, Nicholas G. and McGowan, Craig J. and Yamana, Teresa K. and Tushar, Abhinav and Ray, Evan L. and Osthus, Dave and Kandula, Sasikiran and Brooks, Logan C. and Crawford-Crudell, Willow and Gibson, Graham Casey and Moore, Evan and Silva, Rebecca and Biggerstaff, Matthew and Johansson, Michael A. and Rosenfeld, Roni and Shaman, Jeffrey},
	month = nov,
	year = {2019},
	keywords = {Forecasting, Epidemics, Humans, Models, Statistical, Models, Theoretical, Public Health, United States, Incidence, Centers for Disease Control and Prevention, U.S., Disease Outbreaks, Influenza, Human, Seasons, Computer Simulation, Models, Biological, Data Accuracy, Data Collection, Machine Learning},
	pages = {e1007486},
}

@article{mcandrew_aggregating_2021,
	title = {Aggregating predictions from experts: {A} review of statistical methods, experiments, and applications},
	volume = {13},
	copyright = {© 2020 Wiley Periodicals LLC.},
	shorttitle = {Aggregating predictions from experts},
	doi = {doi.org/10.1002/wics.1514},
	abstract = {Forecasts support decision making in a variety of applications. Statistical models can produce accurate forecasts given abundant training data, but when data is sparse or rapidly changing, statistical models may not be able to make accurate predictions. Expert judgmental forecasts—models that combine expert-generated predictions into a single forecast—can make predictions when training data is limited by relying on human intuition. Researchers have proposed a wide array of algorithms to combine expert predictions into a single forecast, but there is no consensus on an optimal aggregation model. This review surveyed recent literature on aggregating expert-elicited predictions. We gathered common terminology, aggregation methods, and forecasting performance metrics, and offer guidance to strengthen future work that is growing at an accelerated pace. This article is categorized under: Statistical Learning and Exploratory Methods of the Data Sciences {\textgreater} Clustering and Classification Statistical Learning and Exploratory Methods of the Data Sciences {\textgreater} Exploratory Data Analysis Statistical Learning and Exploratory Methods of the Data Sciences {\textgreater} Modeling Methods Statistical and Graphical Methods of Data Analysis {\textgreater} Multivariate Analysis},
	language = {en},
	number = {2},
	journal = {WIREs Computational Statistics},
	author = {McAndrew, Thomas and Wattanachit, Nutcha and Gibson, Graham C. and Reich, Nicholas G.},
	year = {2021},
	keywords = {consensus, expert judgment, forecast aggregation, forecast combination, judgmental forecasting},
	pages = {e1514},
}

@Manual{bosse_stackr_2023,
	title = {stackr: {Create} {Mixture} {Models} {From} {Predictive} {Samples}},
	author = {Bosse, Nikos and Yao, Yuling and Abbott, Sam and Funk, Sebastian},
	year = {2023},
	note = {R package version 0.1.0, https://github.com/epiforecasts/stackr, Accessed: 2024-01-05},
}

@Manual{couch_stacks_2023,
  title = {stacks: Tidy Model Stacking},
  author = {Simon Couch and Max Kuhn},
  year = {2023},
  note = {R package version 1.0.3, https://github.com/tidymodels/stacks, Accessed: 2024-01-05},
}

@article{pedregosa_scikit-learn_2011,
	title = {Scikit-learn: {Machine} {Learning} in {Python}},
	volume = {12},
	shorttitle = {Scikit-learn},
	abstract = {Scikit-learn is a Python module integrating a wide range of state-of-the-art machine learning algorithms for medium-scale supervised and unsupervised problems. This package focuses on bringing machine learning to non-specialists using a general-purpose high-level language. Emphasis is put on ease of use, performance, documentation, and API consistency. It has minimal dependencies and is distributed under the simplified BSD license, encouraging its use in both academic and commercial settings. Source code, binaries, and documentation can be downloaded from http://scikit-learn.sourceforge.net.},
	number = {85},
	journal = {Journal of Machine Learning Research},
	author = {Pedregosa, Fabian and Varoquaux, Gaël and Gramfort, Alexandre and Michel, Vincent and Thirion, Bertrand and Grisel, Olivier and Blondel, Mathieu and Prettenhofer, Peter and Weiss, Ron and Dubourg, Vincent and Vanderplas, Jake and Passos, Alexandre and Cournapeau, David and Brucher, Matthieu and Perrot, Matthieu and Duchesnay, Édouard},
	year = {2011},
	pages = {2825--2830},
	doi = {10.5555/1953048.2078195},
}


@article{viboud2018,
	title = {The RAPIDD ebola forecasting challenge: Synthesis and lessons learnt},
	author = {Viboud, {Cécile} and Sun, Kaiyuan and Gaffey, Robert and Ajelli, Marco and Fumanelli, Laura and Merler, Stefano and Zhang, Qian and Chowell, Gerardo and Simonsen, Lone and Vespignani, Alessandro},
	year = {2018},
	month = {03},
	date = {2018-03-01},
	journal = {Epidemics},
	pages = {13--21},
	series = {The RAPIDD Ebola Forecasting Challenge},
	volume = {22},
	doi = {10.1016/j.epidem.2017.08.002},
	langid = {en}
}

@article{weiss2019,
	title = {Forecast Combinations in R using the ForecastComb Package},
	author = {Weiss, {Christoph,E.} and Raviv, Eran and Roetzer, Gernot},
	year = {2019},
	date = {2019},
	journal = {The R Journal},
	pages = {262},
	volume = {10},
	number = {2},
	doi = {10.32614/RJ-2018-052},
	langid = {en}
}


@article{cramer2022,
	title = {Evaluation of individual and ensemble probabilistic forecasts of COVID-19 mortality in the United States},
	author = {Cramer, Estee Y. and Ray, Evan L. and Lopez, Velma K. and Bracher, Johannes and Brennen, Andrea and Castro Rivadeneira, Alvaro J. and Gerding, Aaron and Gneiting, Tilmann and House, Katie H. and Huang, Yuxin and Jayawardena, Dasuni and Kanji, Abdul H. and Khandelwal, Ayush and Le, Khoa and {Mühlemann}, Anja and Niemi, Jarad and Shah, Apurv and Stark, Ariane and Wang, Yijin and Wattanachit, Nutcha and Zorn, Martha W. and Gu, Youyang and Jain, Sansiddh and Bannur, Nayana and Deva, Ayush and Kulkarni, Mihir and Merugu, Srujana and Raval, Alpan and Shingi, Siddhant and Tiwari, Avtansh and White, Jerome and Abernethy, Neil F. and Woody, Spencer and Dahan, Maytal and Fox, Spencer and Gaither, Kelly and Lachmann, Michael and Meyers, Lauren Ancel and Scott, James G. and Tec, Mauricio and Srivastava, Ajitesh and George, Glover E. and Cegan, Jeffrey C. and Dettwiller, Ian D. and England, William P. and Farthing, Matthew W. and Hunter, Robert H. and Lafferty, Brandon and Linkov, Igor and Mayo, Michael L. and Parno, Matthew D. and Rowland, Michael A. and Trump, Benjamin D. and Zhang-James, Yanli and Chen, Samuel and Faraone, Stephen V. and Hess, Jonathan and Morley, Christopher P. and Salekin, Asif and Wang, Dongliang and Corsetti, Sabrina M. and Baer, Thomas M. and Eisenberg, Marisa C. and Falb, Karl and Huang, Yitao and Martin, Emily T. and McCauley, Ella and Myers, Robert L. and Schwarz, Tom and Sheldon, Daniel and Gibson, Graham Casey and Yu, Rose and Gao, Liyao and Ma, Yian and Wu, Dongxia and Yan, Xifeng and Jin, Xiaoyong and Wang, Yu-Xiang and Chen, YangQuan and Guo, Lihong and Zhao, Yanting and Gu, Quanquan and Chen, Jinghui and Wang, Lingxiao and Xu, Pan and Zhang, Weitong and Zou, Difan and Biegel, Hannah and Lega, Joceline and McConnell, Steve and Nagraj, V. P. and Guertin, Stephanie L. and Hulme-Lowe, Christopher and Turner, Stephen D. and Shi, Yunfeng and Ban, Xuegang and Walraven, Robert and Hong, Qi-Jun and Kong, Stanley and van de Walle, Axel and Turtle, James A. and Ben-Nun, Michal and Riley, Steven and Riley, Pete and Koyluoglu, Ugur and DesRoches, David and Forli, Pedro and Hamory, Bruce and Kyriakides, Christina and Leis, Helen and Milliken, John and Moloney, Michael and Morgan, James and Nirgudkar, Ninad and Ozcan, Gokce and Piwonka, Noah and Ravi, Matt and Schrader, Chris and Shakhnovich, Elizabeth and Siegel, Daniel and Spatz, Ryan and Stiefeling, Chris and Wilkinson, Barrie and Wong, Alexander and Cavany, Sean and {España}, Guido and Moore, Sean and Oidtman, Rachel and Perkins, Alex and Kraus, David and Kraus, Andrea and Gao, Zhifeng and Bian, Jiang and Cao, Wei and Lavista Ferres, Juan and Li, Chaozhuo and Liu, Tie-Yan and Xie, Xing and Zhang, Shun and Zheng, Shun and Vespignani, Alessandro and Chinazzi, Matteo and Davis, Jessica T. and Mu, Kunpeng and Pastore y Piontti, Ana and Xiong, Xinyue and Zheng, Andrew and Baek, Jackie and Farias, Vivek and Georgescu, Andreea and Levi, Retsef and Sinha, Deeksha and Wilde, Joshua and Perakis, Georgia and Bennouna, Mohammed Amine and Nze-Ndong, David and Singhvi, Divya and Spantidakis, Ioannis and Thayaparan, Leann and Tsiourvas, Asterios and Sarker, Arnab and Jadbabaie, Ali and Shah, Devavrat and Della Penna, Nicolas and Celi, Leo A. and Sundar, Saketh and Wolfinger, Russ and Osthus, Dave and Castro, Lauren and Fairchild, Geoffrey and Michaud, Isaac and Karlen, Dean and Kinsey, Matt and Mullany, Luke C. and Rainwater-Lovett, Kaitlin and Shin, Lauren and Tallaksen, Katharine and Wilson, Shelby and Lee, Elizabeth C. and Dent, Juan and Grantz, Kyra H. and Hill, Alison L. and Kaminsky, Joshua and Kaminsky, Kathryn and Keegan, Lindsay T. and Lauer, Stephen A. and Lemaitre, Joseph C. and Lessler, Justin and Meredith, Hannah R. and Perez-Saez, Javier and Shah, Sam and Smith, Claire P. and Truelove, Shaun A. and Wills, Josh and Marshall, Maximilian and Gardner, Lauren and Nixon, Kristen and Burant, John C. and Wang, Lily and Gao, Lei and Gu, Zhiling and Kim, Myungjin and Li, Xinyi and Wang, Guannan and Wang, Yueying and Yu, Shan and Reiner, Robert C. and Barber, Ryan and Gakidou, Emmanuela and Hay, Simon I. and Lim, Steve and Murray, Chris and Pigott, David and Gurung, Heidi L. and Baccam, Prasith and Stage, Steven A. and Suchoski, Bradley T. and Prakash, B. Aditya and Adhikari, Bijaya and Cui, Jiaming and {Rodríguez}, Alexander and Tabassum, Anika and Xie, Jiajia and Keskinocak, Pinar and Asplund, John and Baxter, Arden and Oruc, Buse Eylul and Serban, Nicoleta and Arik, Sercan O. and Dusenberry, Mike and Epshteyn, Arkady and Kanal, Elli and Le, Long T. and Li, Chun-Liang and Pfister, Tomas and Sava, Dario and Sinha, Rajarishi and Tsai, Thomas and Yoder, Nate and Yoon, Jinsung and Zhang, Leyou and Abbott, Sam and Bosse, Nikos I. and Funk, Sebastian and Hellewell, Joel and Meakin, Sophie R. and Sherratt, Katharine and Zhou, Mingyuan and Kalantari, Rahi and Yamana, Teresa K. and Pei, Sen and Shaman, Jeffrey and Li, Michael L. and Bertsimas, Dimitris and Skali Lami, Omar and Soni, Saksham and Tazi Bouardi, Hamza and Ayer, Turgay and Adee, Madeline and Chhatwal, Jagpreet and Dalgic, Ozden O. and Ladd, Mary A. and Linas, Benjamin P. and Mueller, Peter and Xiao, Jade and Wang, Yuanjia and Wang, Qinxia and Xie, Shanghong and Zeng, Donglin and Green, Alden and Bien, Jacob and Brooks, Logan and Hu, Addison J. and Jahja, Maria and McDonald, Daniel and Narasimhan, Balasubramanian and Politsch, Collin and Rajanala, Samyak and Rumack, Aaron and Simon, Noah and Tibshirani, Ryan J. and Tibshirani, Rob and Ventura, Valerie and Wasserman, Larry and {O{\textquoteright}Dea}, Eamon B. and Drake, John M. and Pagano, Robert and Tran, Quoc T. and Ho, Lam Si Tung and Huynh, Huong and Walker, Jo W. and Slayton, Rachel B. and Johansson, Michael A. and Biggerstaff, Matthew and Reich, Nicholas G.},
	year = {2022},
	month = {04},
	date = {2022-04-12},
	journal = {Proceedings of the National Academy of Sciences},
	pages = {e2113561119},
	volume = {119},
	number = {15},
	doi = {10.1073/pnas.2113561119},
}

@article{johansson2019,
	title = {An open challenge to advance probabilistic forecasting for dengue epidemics},
	author = {Johansson, Michael A. and Apfeldorf, Karyn M. and Dobson, Scott and Devita, Jason and Buczak, Anna L. and Baugher, Benjamin and Moniz, Linda J. and Bagley, Thomas and Babin, Steven M. and Guven, Erhan and Yamana, Teresa K. and Shaman, Jeffrey and Moschou, Terry and Lothian, Nick and Lane, Aaron and Osborne, Grant and Jiang, Gao and Brooks, Logan C. and Farrow, David C. and Hyun, Sangwon and Tibshirani, Ryan J. and Rosenfeld, Roni and Lessler, Justin and Reich, Nicholas G. and Cummings, Derek A. T. and Lauer, Stephen A. and Moore, Sean M. and Clapham, Hannah E. and Lowe, Rachel and Bailey, Trevor C. and {García-Díez}, Markel and Carvalho, {Marilia Sá} and {Rodó}, Xavier and Sardar, Tridip and Paul, Richard and Ray, Evan L. and Sakrejda, Krzysztof and Brown, Alexandria C. and Meng, Xi and Osoba, Osonde and Vardavas, Raffaele and Manheim, David and Moore, Melinda and Rao, Dhananjai M. and Porco, Travis C. and Ackley, Sarah and Liu, Fengchen and Worden, Lee and Convertino, Matteo and Liu, Yang and Reddy, Abraham and Ortiz, Eloy and Rivero, Jorge and Brito, Humberto and Juarrero, Alicia and Johnson, Leah R. and Gramacy, Robert B. and Cohen, Jeremy M. and Mordecai, Erin A. and Murdock, Courtney C. and Rohr, Jason R. and Ryan, Sadie J. and Stewart-Ibarra, Anna M. and Weikel, Daniel P. and Jutla, Antarpreet and Khan, Rakibul and Poultney, Marissa and Colwell, Rita R. and {Rivera-García}, Brenda and Barker, Christopher M. and Bell, Jesse E. and Biggerstaff, Matthew and Swerdlow, David and Mier-Y-Teran-Romero, Luis and Forshey, Brett M. and Trtanj, Juli and Asher, Jason and Clay, Matt and Margolis, Harold S. and Hebbeler, Andrew M. and George, Dylan and Chretien, Jean-Paul},
	year = {2019},
	month = {11},
	date = {2019-11-26},
	journal = {Proceedings of the National Academy of Sciences},
	pages = {24268--24274},
	volume = {116},
	number = {48},
	doi = {10.1073/pnas.1909865116},
	langid = {eng}
}

@article{reich2022,
	title = {Collaborative Hubs: Making the Most of Predictive Epidemic Modeling},
	author = {Reich, Nicholas G. and Lessler, Justin and Funk, Sebastian and Viboud, Cecile and Vespignani, Alessandro and Tibshirani, Ryan J. and Shea, Katriona and Schienle, Melanie and Runge, Michael C. and Rosenfeld, Roni and Ray, Evan L. and Niehus, Rene and Johnson, Helen C. and Johansson, Michael A. and Hochheiser, Harry and Gardner, Lauren and Bracher, Johannes and Borchering, Rebecca K. and Biggerstaff, Matthew},
	year = {2022},
	date = {2022},
	journal = {American Journal of Public Health},
	pages = {839--842},
	volume = {112},
	number = {6},
	doi = {10.2105/AJPH.2022.306831},
}

@article{tebaldi2007,
	title = {The Use of the Multi-Model Ensemble in Probabilistic Climate Projections},
	author = {Tebaldi, Claudia and Knutti, Reto},
	year = {2007},
	date = {2007},
	journal = {Philosophical Transactions: Mathematical, Physical and Engineering Sciences},
	pages = {2053--2075},
	volume = {365},
	number = {1857},
	doi = {10.1098/rsta.2007.2076},
}

@article{winkler2015,
	title = {Equal Versus Differential Weighting in Combining Forecasts},
	author = {Winkler, Robert L.},
	year = {2015},
	date = {2015},
	journal = {Risk Analysis},
	pages = {16--18},
	volume = {35},
	number = {1},
	doi = {10.1111/risa.12302},
	langid = {en}
}

@article{alley2019,
	title = {Advances in weather prediction},
	author = {Alley, Richard B. and Emanuel, Kerry A. and Zhang, Fuqing},
	year = {2019},
	month = {01},
	date = {2019-01-25},
	journal = {Science},
	pages = {342--344},
	volume = {363},
	number = {6425},
	doi = {10.1126/science.aav7274},
}

@article{aastveit2018,
	title = {The Evolution of Forecast Density Combinations in Economics},
	author = {Aastveit, Knut Are and Mitchell, James and Ravazzolo, Francesco and van Dijk, Herman K.},
	year = {2018},
	date = {2018},
	journal = {Tinbergen Institute Discussion Papers},
	url = {https://hdl.handle.net/10419/185588},
	address = {Amsterdam and Rotterdam},
	langid = {eng}
}

@article{mcgowan2019,
	title = {Collaborative efforts to forecast seasonal influenza in the United States, 2015{\textendash}2016},
	author = {McGowan, Craig J. and Biggerstaff, Matthew and Johansson, Michael and Apfeldorf, Karyn M. and Ben-Nun, Michal and Brooks, Logan and Convertino, Matteo and Erraguntla, Madhav and Farrow, David C. and Freeze, John and Ghosh, Saurav and Hyun, Sangwon and Kandula, Sasikiran and Lega, Joceline and Liu, Yang and Michaud, Nicholas and Morita, Haruka and Niemi, Jarad and Ramakrishnan, Naren and Ray, Evan L. and Reich, Nicholas G. and Riley, Pete and Shaman, Jeffrey and Tibshirani, Ryan and Vespignani, Alessandro and Zhang, Qian and Reed, Carrie},
	year = {2019},
	month = {01},
	date = {2019-01-24},
	journal = {Scientific Reports},
	pages = {683},
	volume = {9},
	number = {1},
	doi = {10.1038/s41598-018-36361-9},
	langid = {en}
}

@article{hibon2005,
	title = {To combine or not to combine: selecting among forecasts and their combinations},
	author = {Hibon, {Michèle} and Evgeniou, Theodoros},
	year = {2005},
	month = {01},
	date = {2005-01-01},
	journal = {International Journal of Forecasting},
	pages = {15--24},
	volume = {21},
	number = {1},
	doi = {10.1016/j.ijforecast.2004.05.002},
	langid = {en}
}

@article{clemen1989,
	title = {Combining forecasts: A review and annotated bibliography},
	author = {Clemen, Robert T.},
	year = {1989},
	month = {01},
	date = {1989-01-01},
	journal = {International Journal of Forecasting},
	pages = {559--583},
	volume = {5},
	number = {4},
	doi = {10.1016/0169-2070(89)90012-5},
	langid = {en}
}

@inbook{timmermann2006,
	title = {Chapter 4 Forecast Combinations},
	author = {Timmermann, Allan},
	year = {2006},
	month = {01},
	date = {2006-01-01},
	publisher = {Elsevier},
	pages = {135--196},
	volume = {1},
	doi = {10.1016/S1574-0706(05)01004-9},
	langid = {en}
}

@misc{hubverse_docs,
  author = {{Consortium of Infectious Disease Modeling Hubs}},
  title = {The hubverse: open tools for collaborative forecasting},
  url = {https://hubverse.io/en/latest/index.html},
  year = {2024},
  note = "Accessed: 2024-05-02"
}

@article{stone1961,
	title = {The Opinion Pool},
	author = {Stone, M.},
	year = {1961},
	date = {1961},
	journal = {The Annals of Mathematical Statistics},
	pages = {1339--1342},
	volume = {32},
	number = {4},
}

@phdthesis{vincent1912,
	title = {The function of the vibrissae in the behavior of the white rat.},
	author = {Vincent, Stella Burnham},
	year = {1912},
	date = {1912},
	note = {OCLC: 17104960},
	school = {University of Chicago},
	address = {Cambridge MA},
	langid = {English}
}

@article{lichtendahl2013,
	title = {Is It Better to Average Probabilities or Quantiles?},
	author = {Lichtendahl, Kenneth C. and Grushka-Cockayne, Yael and Winkler, Robert L.},
	year = {2013},
	date = {2013},
	journal = {Management Science},
	pages = {1594--1611},
	volume = {59},
	number = {7},
	doi = {10.1287/mnsc.1120.1667},
}

@article{howerton2023,
	title = {Context-dependent representation of within- and between-model uncertainty: aggregating probabilistic predictions in infectious disease epidemiology},
	author = {Howerton, Emily and Runge, Michael C. and Bogich, Tiffany L. and Borchering, Rebecca K. and Inamine, Hidetoshi and Lessler, Justin and Mullany, Luke C. and Probert, William J. M. and Smith, Claire P. and Truelove, Shaun and Viboud, {Cécile} and Shea, Katriona},
	year = {2023},
	month = {01},
	date = {2023-01-25},
	journal = {Journal of The Royal Society Interface},
	pages = {20220659},
	volume = {20},
	number = {198},
	doi = {10.1098/rsif.2022.0659},
}

@article{disease_economics,
	author = {Kristine M. Smith and Catherine C. Machalaba and Richard Seifman and Yasha Feferholtz and William B. Karesh},
	title = {Infectious disease and economics: The case for considering multi-sectoral impacts},
	journal = {One Health},
	year = {2019},
	volume = {7},
	pages = {100080},
	doi = {https://doi.org/10.1016/j.onehlt.2018.100080},
}

@misc{lauer,
  author = {Stephen A. Lauer and Alexandria C. Brown and Nicholas G. Reich},
  title = {Infectious Disease Forecasting for Public Health},
  publisher = {arXiv},
  year = {2020},
  copyright = {Creative Commons Attribution 4.0 International},
  doi = {10.48550/ARXIV.2006.00073}
}

@article{bracher_evaluating_2021,
	title = {Evaluating epidemic forecasts in an interval format},
	volume = {17},
	doi = {10.1371/journal.pcbi.1008618},
	abstract = {For practical reasons, many forecasts of case, hospitalization, and death counts in the context of the current Coronavirus Disease 2019 (COVID-19) pandemic are issued in the form of central predictive intervals at various levels. This is also the case for the forecasts collected in the COVID-19 Forecast Hub (https://covid19forecasthub.org/). Forecast evaluation metrics like the logarithmic score, which has been applied in several infectious disease forecasting challenges, are then not available as they require full predictive distributions. This article provides an overview of how established methods for the evaluation of quantile and interval forecasts can be applied to epidemic forecasts in this format. Specifically, we discuss the computation and interpretation of the weighted interval score, which is a proper score that approximates the continuous ranked probability score. It can be interpreted as a generalization of the absolute error to probabilistic forecasts and allows for a decomposition into a measure of sharpness and penalties for over- and underprediction.},
	language = {en},
	number = {2},
	journal = {PLOS Computational Biology},
	author = {Bracher, Johannes and Ray, Evan L. and Gneiting, Tilmann and Reich, Nicholas G.},
	month = feb,
	year = {2021},
	keywords = {Probability distribution, Public and occupational health, COVID 19, Forecasting, Pandemics, Binomials, Epidemiological methods and statistics, Instrument calibration},
	pages = {e1008618},
}

@book{niederreiter1992quasirandom,
  title={Random number generation and quasi-Monte Carlo methods},
  author={Niederreiter, Harald},
  year={1992},
  publisher={Society for Industrial and Applied Mathematics},
  address = {Philadelphia PA}
}

@misc{cdc_flusight,
  author = {CDC},
  title = {About Flu Forecasting},
  url = {https://www.cdc.gov/flu/weekly/flusight/how-flu-forecasting.htm},
  year = {2023},
  note = "Accessed: 2024-05-02"
}

@Manual{distfromq,
  title = {distfromq: Reconstruct a Distribution from a Collection of Quantiles},
  author = {Evan L. Ray and Aaron Gerding},
  year = {2024},
  note = {R package version 1.0.3, http://github.com/reichlab/distfromq, Accessed: 2024-01-08}
}

@article{reich_zoltar_2021,
	title = {The {Zoltar} forecast archive, a tool to standardize and store interdisciplinary prediction research},
	volume = {8},
	copyright = {2021 The Author(s)},
	doi = {10.1038/s41597-021-00839-5},
	abstract = {Forecasting has emerged as an important component of informed, data-driven decision-making in a wide array of fields. We introduce a new data model for probabilistic predictions that encompasses a wide range of forecasting settings. This framework clearly defines the constituent parts of a probabilistic forecast and proposes one approach for representing these data elements. The data model is implemented in Zoltar, a new software application that stores forecasts using the data model and provides standardized API access to the data. In one real-time case study, an instance of the Zoltar web application was used to store, provide access to, and evaluate real-time forecast data on the order of 108 rows, provided by over 40 international research teams from academia and industry making forecasts of the COVID-19 outbreak in the US. Tools and data infrastructure for probabilistic forecasts, such as those introduced here, will play an increasingly important role in ensuring that future forecasting research adheres to a strict set of rigorous and reproducible standards.},
	language = {en},
	number = {1},
	journal = {Scientific Data},
	author = {Reich, Nicholas G. and Cornell, Matthew and Ray, Evan L. and House, Katie and Le, Khoa},
	month = feb,
	year = {2021},
	keywords = {Databases, Infectious diseases},
	pages = {59},
}