@article{andersonAsymptoticTheoryCertain1952,
  title = {Asymptotic {{Theory}} of {{Certain}} "{{Goodness}} of {{Fit}}" {{Criteria Based}} on {{Stochastic Processes}}},
  author = {Anderson, T. W. and Darling, D. A.},
  year = {1952},
  journal = {The Annals of Mathematical Statistics},
  volume = {23},
  number = {2},
  eprint = {2236446},
  eprinttype = {jstor},
  pages = {193--212},
  publisher = {Institute of Mathematical Statistics},
  issn = {0003-4851},
  urldate = {2020-08-21},
  abstract = {The statistical problem treated is that of testing the hypothesis that n independent, identically distributed random variables have a specified continuous distribution function F(x). If Fn(x) is the empirical cumulative distribution function and {$\psi$}(t) is some nonnegative weight function (0 {$\leq$} t {$\leq$} 1), we consider \$n{\textasciicircum}\{{\textbackslash}frac\{1\}\{2\}\} {\textbackslash}sup\_\{-{\textbackslash}infty\vphantom\}}
}

@article{angusProbabilityIntegralTransform1994,
  title = {The {{Probability Integral Transform}} and {{Related Results}}},
  author = {Angus, John E.},
  year = {1994},
  month = dec,
  journal = {SIAM Review},
  volume = {36},
  number = {4},
  pages = {652--654},
  publisher = {{Society for Industrial and Applied Mathematics}},
  issn = {0036-1445},
  doi = {10.1137/1036146},
  urldate = {2020-08-12},
  abstract = {A simple proof of the probability integral transform theorem in probability and statistics is given that depends only on probabilistic concepts and elementary properties of continuous functions. This proof yields the theorem in its fullest generality. A similar theorem that forms the basis for the inverse method of random number generation is also discussed and contrasted to the probability integral transform theorem. Typical applications are discussed. Despite their generality and far reaching consequences, these theorems are remarkable in their simplicity and ease of proof.},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/8K3YQL5Q/1036146.html}
}

@article{bosseScoringEpidemiologicalForecasts2023,
  title = {Scoring Epidemiological Forecasts on Transformed Scales},
  author = {Bosse, Nikos I. and Abbott, Sam and Cori, Anne and van Leeuwen, Edwin and Bracher, Johannes and Funk, Sebastian},
  year = {2023},
  month = aug,
  journal = {PLOS Computational Biology},
  volume = {19},
  number = {8},
  pages = {e1011393},
  publisher = {Public Library of Science},
  issn = {1553-7358},
  doi = {10.1371/journal.pcbi.1011393},
  urldate = {2023-11-20},
  abstract = {Forecast evaluation is essential for the development of predictive epidemic models and can inform their use for public health decision-making. Common scores to evaluate epidemiological forecasts are the Continuous Ranked Probability Score (CRPS) and the Weighted Interval Score (WIS), which can be seen as measures of the absolute distance between the forecast distribution and the observation. However, applying these scores directly to predicted and observed incidence counts may not be the most appropriate due to the exponential nature of epidemic processes and the varying magnitudes of observed values across space and time. In this paper, we argue that transforming counts before applying scores such as the CRPS or WIS can effectively mitigate these difficulties and yield epidemiologically meaningful and easily interpretable results. Using the CRPS on log-transformed values as an example, we list three attractive properties: Firstly, it can be interpreted as a probabilistic version of a relative error. Secondly, it reflects how well models predicted the time-varying epidemic growth rate. And lastly, using arguments on variance-stabilizing transformations, it can be shown that under the assumption of a quadratic mean-variance relationship, the logarithmic transformation leads to expected CRPS values which are independent of the order of magnitude of the predicted quantity. Applying a transformation of log(x + 1) to data and forecasts from the European COVID-19 Forecast Hub, we find that it changes model rankings regardless of stratification by forecast date, location or target types. Situations in which models missed the beginning of upward swings are more strongly emphasised while failing to predict a downturn following a peak is less severely penalised when scoring transformed forecasts as opposed to untransformed ones. We conclude that appropriate transformations, of which the natural logarithm is only one particularly attractive option, should be considered when assessing the performance of different models in the context of infectious disease incidence.},
  langid = {english},
  keywords = {COVID 19,Epidemiology,Europe,Forecasting,Normal distribution,Probability distribution,Public and occupational health,Statistical dispersion},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/NDS44ERL/Bosse et al. - 2023 - Scoring epidemiological forecasts on transformed s.pdf}
}

@article{bracherEvaluatingEpidemicForecasts2021,
  title = {Evaluating Epidemic Forecasts in an Interval Format},
  author = {Bracher, Johannes and Ray, Evan L. and Gneiting, Tilmann and Reich, Nicholas G.},
  year = {2021},
  month = feb,
  journal = {PLoS computational biology},
  volume = {17},
  number = {2},
  pages = {e1008618},
  issn = {1553-7358},
  doi = {10.1371/journal.pcbi.1008618},
  abstract = {For practical reasons, many forecasts of case, hospitalization, and death counts in the context of the current Coronavirus Disease 2019 (COVID-19) pandemic are issued in the form of central predictive intervals at various levels. This is also the case for the forecasts collected in the COVID-19 Forecast Hub (https://covid19forecasthub.org/). Forecast evaluation metrics like the logarithmic score, which has been applied in several infectious disease forecasting challenges, are then not available as they require full predictive distributions. This article provides an overview of how established methods for the evaluation of quantile and interval forecasts can be applied to epidemic forecasts in this format. Specifically, we discuss the computation and interpretation of the weighted interval score, which is a proper score that approximates the continuous ranked probability score. It can be interpreted as a generalization of the absolute error to probabilistic forecasts and allows for a decomposition into a measure of sharpness and penalties for over- and underprediction.},
  langid = {english},
  pmcid = {PMC7880475},
  pmid = {33577550},
  keywords = {Communicable Diseases,COVID-19,Forecasting,Humans,Pandemics,Probability,SARS-CoV-2},
  annotation = {note: DOI 10.1371/journal.pcbi.1008618},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/EX37R6J8/Bracher et al. - 2021 - Evaluating epidemic forecasts in an interval forma.pdf}
}

@article{bracherNationalSubnationalShortterm2022,
  title = {National and Subnational Short-Term Forecasting of {{COVID-19}} in {{Germany}} and {{Poland}} during Early 2021},
  author = {Bracher, Johannes and Wolffram, Daniel and Deuschel, Jannik and G{\"o}rgen, Konstantin and Ketterer, Jakob L. and Ullrich, Alexander and Abbott, Sam and Barbarossa, Maria V. and Bertsimas, Dimitris and Bhatia, Sangeeta and Bodych, Marcin and Bosse, Nikos I. and Burgard, Jan Pablo and Castro, Lauren and Fairchild, Geoffrey and Fiedler, Jochen and Fuhrmann, Jan and Funk, Sebastian and Gambin, Anna and Gogolewski, Krzysztof and Heyder, Stefan and Hotz, Thomas and Kheifetz, Yuri and Kirsten, Holger and Krueger, Tyll and Krymova, Ekaterina and Leith{\"a}user, Neele and Li, Michael L. and Meinke, Jan H. and Miasojedow, B{\l}a{\.z}ej and Michaud, Isaac J. and Mohring, Jan and Nouvellet, Pierre and Nowosielski, Jedrzej M. and Ozanski, Tomasz and Radwan, Maciej and Rakowski, Franciszek and Scholz, Markus and Soni, Saksham and Srivastava, Ajitesh and Gneiting, Tilmann and Schienle, Melanie},
  year = {2022},
  month = oct,
  journal = {Communications Medicine},
  volume = {2},
  number = {1},
  pages = {1--17},
  publisher = {Nature Publishing Group},
  issn = {2730-664X},
  doi = {10.1038/s43856-022-00191-8},
  urldate = {2023-02-22},
  abstract = {During the COVID-19 pandemic there has been a strong interest in forecasts of the short-term development of epidemiological indicators to inform decision makers. In this study we evaluate probabilistic real-time predictions of confirmed cases and deaths from COVID-19 in Germany and Poland for the period from January through April 2021.},
  copyright = {2022 The Author(s)},
  langid = {english},
  keywords = {Epidemiology,Viral infection},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/N3WD8QGU/Bracher et al. - 2022 - National and subnational short-term forecasting of.pdf}
}

@article{bracherPreregisteredShorttermForecasting2021a,
  title = {A Pre-Registered Short-Term Forecasting Study of {{COVID-19}} in {{Germany}} and {{Poland}} during the Second Wave},
  author = {Bracher, Johannes and Wolffram, Daniel and Deuschel, Jannik and G{\"o}rgen, Konstantin and Ketterer, Jakob L. and Ullrich, Alexander and Abbott, Sam and Barbarossa, Maria V. and Bertsimas, Dimitris and Bhatia, Sangeeta and Bodych, Marcin and Bosse, Nikos I. and Burgard, Jan Pablo and Castro, Lauren and Fairchild, Geoffrey and Fuhrmann, Jan and Funk, Sebastian and Gogolewski, Krzysztof and Gu, Quanquan and Heyder, Stefan and Hotz, Thomas and Kheifetz, Yuri and Kirsten, Holger and Krueger, Tyll and Krymova, Ekaterina and Li, Michael L. and Meinke, Jan H. and Michaud, Isaac J. and Niedzielewski, K. and O{\.z}a{\'n}ski, T. and Rakowski, F. and Scholz, Markus and Soni, Saksham and Srivastava, Ajitesh and Zieli{\'n}ski, J. and Zou, Difan and Gneiting, Tilmann and Schienle, Melanie},
  year = {2021},
  month = aug,
  journal = {Nature Communications},
  volume = {12},
  number = {1},
  pages = {5173},
  publisher = {Nature Publishing Group},
  issn = {2041-1723},
  doi = {10.1038/s41467-021-25207-0},
  urldate = {2023-12-07},
  abstract = {Disease modelling has had considerable policy impact during the ongoing COVID-19 pandemic, and it is increasingly acknowledged that combining multiple models can improve the reliability of outputs. Here we report insights from ten weeks of collaborative short-term forecasting of COVID-19 in Germany and Poland (12 October--19 December 2020). The study period covers the onset of the second wave in both countries, with tightening non-pharmaceutical interventions (NPIs) and subsequently a decay (Poland) or plateau and renewed increase (Germany) in reported cases. Thirteen independent teams provided probabilistic real-time forecasts of COVID-19 cases and deaths. These were reported for lead times of one to four weeks, with evaluation focused on one- and two-week horizons, which are less affected by changing NPIs. Heterogeneity between forecasts was considerable both in terms of point predictions and forecast spread. Ensemble forecasts showed good relative performance, in particular in terms of coverage, but did not clearly dominate single-model predictions. The study was preregistered and will be followed up in future phases of the pandemic.},
  copyright = {2021 The Author(s)},
  langid = {english},
  keywords = {Computational models,Epidemiology,SARS-CoV-2,Statistics},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/8ADJWVQ6/Bracher et al. - 2021 - A pre-registered short-term forecasting study of C.pdf}
}

@article{brierVerificationForecastsExpressed1950,
  title = {Verification of {{Forecasts Expressed}} in {{Terms}} of {{Probability}}},
  author = {Brier, Glenn W.},
  year = {1950},
  month = jan,
  journal = {Monthly Weather Review},
  volume = {78},
  number = {1},
  pages = {1--3},
  publisher = {American Meteorological Society},
  issn = {1520-0493, 0027-0644},
  doi = {10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2},
  urldate = {2022-01-21},
  abstract = {Abstract No Abstract Available.},
  chapter = {Monthly Weather Review},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/ZCBG3Z38/Brier - 1950 - VERIFICATION OF FORECASTS EXPRESSED IN TERMS OF PR.pdf;/Users/nikos/github-synced/zotero-nikos/storage/I83583N3/1520-0493_1950_078_0001_vofeit_2_0_co_2.html}
}

@article{brockerDecompositionsProperScores,
  title = {Decompositions of {{Proper Scores}}},
  author = {Brocker, Jochen},
  pages = {9},
  abstract = {Scoring rules are an important tool for evaluating the performance of probabilistic forecasts. A popular example is the Brier score, which allows for a decomposition into terms related to the sharpness (or information content) and to the reliability of the forecast. This feature renders the Brier score a very intuitive measure of forecast quality. In this paper, it is demonstrated that all strictly proper scoring rules allow for a similar decomposition into reliability and sharpness related terms. This finding underpins the importance of proper scores and yields further credence to the practice of measuring forecast quality by proper scores. Furthermore, the effect of averaging multiple probabilistic forecasts on the score is discussed. It is well known that the Brier score of a mixture of several forecasts is never worse that the average score of the individual forecasts. This property hinges on the convexity of the Brier score, a property not universal among proper scores. Arguably, this phenomenon portends epistemological questions which require clarification.},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/P9IB98P5/Brocker - Decompositions of Proper Scores.pdf}
}

@misc{cramerCOVID19ForecastHub2020,
  title = {{{COVID-19 Forecast Hub}}: 4 {{December}} 2020 Snapshot},
  shorttitle = {{{COVID-19 Forecast Hub}}},
  author = {Cramer, Estee and Nicholas G Reich and Serena Yijin Wang and Jarad Niemi and Abdul Hannan and Katie House and Youyang Gu and Shanghong Xie and Steve Horstman and {aniruddhadiga} and Robert Walraven and {starkari} and Michael Lingzhi Li and Graham Gibson and Lauren Castro and Dean Karlen and Nutcha Wattanachit and {jinghuichen} and {zyt9lsb} and {aagarwal1996} and Spencer Woody and Evan Ray and Frost Tianjian Xu and Hannah Biegel and GuidoEspana and Xinyue X and Johannes Bracher and Elizabeth Lee and {har96} and {leyouz}},
  year = {2020},
  month = dec,
  publisher = {Zenodo},
  doi = {10.5281/zenodo.3963371},
  urldate = {2021-05-29},
  abstract = {This update to the COVID-19 Forecast Hub repository is a snapshot as of 4 December 2020 of the data hosted by and visualized at~https://covid19forecasthub.org/.},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/AVWA2UPE/4305938.html}
}

@article{cramerEvaluationIndividualEnsemble2021,
  title = {Evaluation of Individual and Ensemble Probabilistic Forecasts of {{COVID-19}} Mortality in the {{US}}},
  author = {Cramer, Estee and Ray, Evan L. and Lopez, Velma K. and Bracher, Johannes and Brennen, Andrea and Rivadeneira, Alvaro J. Castro and Gerding, Aaron and Gneiting, Tilmann and House, Katie H. and Huang, Yuxin and Jayawardena, Dasuni and Kanji, Abdul H. and Khandelwal, Ayush and Le, Khoa and M{\"u}hlemann, Anja and Niemi, Jarad and Shah, Apurv and Stark, Ariane and Wang, Yijin and Wattanachit, Nutcha and Zorn, Martha W. and Gu, Youyang and Jain, Sansiddh and Bannur, Nayana and Deva, Ayush and Kulkarni, Mihir and Merugu, Srujana and Raval, Alpan and Shingi, Siddhant and Tiwari, Avtansh and White, Jerome and Woody, Spencer and Dahan, Maytal and Fox, Spencer and Gaither, Kelly and Lachmann, Michael and Meyers, Lauren Ancel and Scott, James G. and Tec, Mauricio and Srivastava, Ajitesh and George, Glover E. and Cegan, Jeffrey C. and Dettwiller, Ian D. and England, William P. and Farthing, Matthew W. and Hunter, Robert H. and Lafferty, Brandon and Linkov, Igor and Mayo, Michael L. and Parno, Matthew D. and Rowland, Michael A. and Trump, Benjamin D. and Corsetti, Sabrina M. and Baer, Thomas M. and Eisenberg, Marisa C. and Falb, Karl and Huang, Yitao and Martin, Emily T. and McCauley, Ella and Myers, Robert L. and Schwarz, Tom and Sheldon, Daniel and Gibson, Graham Casey and Yu, Rose and Gao, Liyao and Ma, Yian and Wu, Dongxia and Yan, Xifeng and Jin, Xiaoyong and Wang, Yu-Xiang and Chen, YangQuan and Guo, Lihong and Zhao, Yanting and Gu, Quanquan and Chen, Jinghui and Wang, Lingxiao and Xu, Pan and Zhang, Weitong and Zou, Difan and Biegel, Hannah and Lega, Joceline and Snyder, Timothy L. and Wilson, Davison D. and McConnell, Steve and Walraven, Robert and Shi, Yunfeng and Ban, Xuegang and Hong, Qi-Jun and Kong, Stanley and Turtle, James A. and {Ben-Nun}, Michal and Riley, Pete and Riley, Steven and Koyluoglu, Ugur and DesRoches, David and Hamory, Bruce and Kyriakides, Christina and Leis, Helen and Milliken, John and Moloney, Michael and Morgan, James and Ozcan, Gokce and Schrader, Chris and Shakhnovich, Elizabeth and Siegel, Daniel and Spatz, Ryan and Stiefeling, Chris and Wilkinson, Barrie and Wong, Alexander and Gao, Zhifeng and Bian, Jiang and Cao, Wei and Ferres, Juan Lavista and Li, Chaozhuo and Liu, Tie-Yan and Xie, Xing and Zhang, Shun and Zheng, Shun and Vespignani, Alessandro and Chinazzi, Matteo and Davis, Jessica T. and Mu, Kunpeng and y Piontti, Ana Pastore and Xiong, Xinyue and Zheng, Andrew and Baek, Jackie and Farias, Vivek and Georgescu, Andreea and Levi, Retsef and Sinha, Deeksha and Wilde, Joshua and Penna, Nicolas D. and Celi, Leo A. and Sundar, Saketh and Cavany, Sean and Espa{\~n}a, Guido and Moore, Sean and Oidtman, Rachel and Perkins, Alex and Osthus, Dave and Castro, Lauren and Fairchild, Geoffrey and Michaud, Isaac and Karlen, Dean and Lee, Elizabeth C. and Dent, Juan and Grantz, Kyra H. and Kaminsky, Joshua and Kaminsky, Kathryn and Keegan, Lindsay T. and Lauer, Stephen A. and Lemaitre, Joseph C. and Lessler, Justin and Meredith, Hannah R. and {Perez-Saez}, Javier and Shah, Sam and Smith, Claire P. and Truelove, Shaun A. and Wills, Josh and Kinsey, Matt and Obrecht, R. F. and Tallaksen, Katharine and Burant, John C. and Wang, Lily and Gao, Lei and Gu, Zhiling and Kim, Myungjin and Li, Xinyi and Wang, Guannan and Wang, Yueying and Yu, Shan and Reiner, Robert C. and Barber, Ryan and Gaikedu, Emmanuela and Hay, Simon and Lim, Steve and Murray, Chris and Pigott, David and Prakash, B. Aditya and Adhikari, Bijaya and Cui, Jiaming and Rodr{\'i}guez, Alexander and Tabassum, Anika and Xie, Jiajia and Keskinocak, Pinar and Asplund, John and Baxter, Arden and Oruc, Buse Eylul and Serban, Nicoleta and Arik, Sercan O. and Dusenberry, Mike and Epshteyn, Arkady and Kanal, Elli and Le, Long T. and Li, Chun-Liang and Pfister, Tomas and Sava, Dario and Sinha, Rajarishi and Tsai, Thomas and Yoder, Nate and Yoon, Jinsung and Zhang, Leyou and Abbott, Sam and Bosse, Nikos I. and Funk, Sebastian and Hellewel, Joel and Meakin, Sophie R. and Munday, James D. and Sherratt, Katherine and Zhou, Mingyuan and Kalantari, Rahi and Yamana, Teresa K. and Pei, Sen and Shaman, Jeffrey and Ayer, Turgay and Adee, Madeline and Chhatwal, Jagpreet and Dalgic, Ozden O. and Ladd, Mary A. and Linas, Benjamin P. and Mueller, Peter and Xiao, Jade and Li, Michael L. and Bertsimas, Dimitris and Lami, Omar Skali and Soni, Saksham and Bouardi, Hamza Tazi and Wang, Yuanjia and Wang, Qinxia and Xie, Shanghong and Zeng, Donglin and Green, Alden and Bien, Jacob and Hu, Addison J. and Jahja, Maria and Narasimhan, Balasubramanian and Rajanala, Samyak and Rumack, Aaron and Simon, Noah and Tibshirani, Ryan and Tibshirani, Rob and Ventura, Valerie and Wasserman, Larry and O'Dea, Eamon B. and Drake, John M. and Pagano, Robert and Walker, Jo W. and Slayton, Rachel B. and Johansson, Michael and Biggerstaff, Matthew and Reich, Nicholas G.},
  year = {2021},
  month = feb,
  journal = {medRxiv},
  pages = {2021.02.03.21250974},
  publisher = {Cold Spring Harbor Laboratory Press},
  doi = {10.1101/2021.02.03.21250974},
  urldate = {2021-04-06},
  abstract = {{$<$}h3{$>$}Abstract{$<$}/h3{$>$} {$<$}p{$>$}Short-term probabilistic forecasts of the trajectory of the COVID-19 pandemic in the United States have served as a visible and important communication channel between the scientific modeling community and both the general public and decision-makers. Forecasting models provide specific, quantitative, and evaluable predictions that inform short-term decisions such as healthcare staffing needs, school closures, and allocation of medical supplies. In 2020, the COVID-19 Forecast Hub (https://covid19forecasthub.org/) collected, disseminated, and synthesized hundreds of thousands of specific predictions from more than 50 different academic, industry, and independent research groups. This manuscript systematically evaluates 23 models that regularly submitted forecasts of reported weekly incident COVID-19 mortality counts in the US at the state and national level. One of these models was a multi-model ensemble that combined all available forecasts each week. The performance of individual models showed high variability across time, geospatial units, and forecast horizons. Half of the models evaluated showed better accuracy than a na{\"i}ve baseline model. In combining the forecasts from all teams, the ensemble showed the best overall probabilistic accuracy of any model. Forecast accuracy degraded as models made predictions farther into the future, with probabilistic accuracy at a 20-week horizon more than 5 times worse than when predicting at a 1-week horizon. This project underscores the role that collaboration and active coordination between governmental public health agencies, academic modeling teams, and industry partners can play in developing modern modeling capabilities to support local, state, and federal response to outbreaks.{$<$}/p{$>$}},
  copyright = {{\copyright} 2021, Posted by Cold Spring Harbor Laboratory. This article is a US Government work. It is not subject to copyright under 17 USC 105 and is also made available for use under a CC0 license},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/W82X9ZN5/Cramer et al. - 2021 - Evaluation of individual and ensemble probabilisti.pdf;/Users/nikos/github-synced/zotero-nikos/storage/7MC6LGTC/2021.02.03.21250974v1.html}
}

@article{czadoPredictiveModelAssessment2009,
  title = {Predictive {{Model Assessment}} for {{Count Data}}},
  author = {Czado, Claudia and Gneiting, Tilmann and Held, Leonhard},
  year = {2009},
  journal = {Biometrics},
  volume = {65},
  number = {4},
  pages = {1254--1261},
  issn = {1541-0420},
  doi = {10.1111/j.1541-0420.2009.01191.x},
  urldate = {2020-08-12},
  abstract = {We discuss tools for the evaluation of probabilistic forecasts and the critique of statistical models for count data. Our proposals include a nonrandomized version of the probability integral transform, marginal calibration diagrams, and proper scoring rules, such as the predictive deviance. In case studies, we critique count regression models for patent data, and assess the predictive performance of Bayesian age-period-cohort models for larynx cancer counts in Germany. The toolbox applies in Bayesian or classical and parametric or nonparametric settings and to any type of ordered discrete outcomes.},
  copyright = {{\copyright} 2009, The International Biometric Society},
  langid = {english},
  keywords = {Calibration,Forecast verification,Model diagnostics,Predictive deviance,Probability integral transform,Proper scoring rule},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/SKRXXEYJ/Czado et al. - 2009 - Predictive Model Assessment for Count Data.pdf;/Users/nikos/github-synced/zotero-nikos/storage/Y3Z8AY8F/j.1541-0420.2009.01191.html}
}

@article{dawidCoherentDispersionCriteria1999,
  title = {Coherent Dispersion Criteria for Optimal Experimental Design},
  author = {Dawid, A. Philip and Sebastiani, Paola},
  year = {1999},
  month = mar,
  journal = {The Annals of Statistics},
  volume = {27},
  number = {1},
  pages = {65--81},
  publisher = {Institute of Mathematical Statistics},
  issn = {0090-5364, 2168-8966},
  doi = {10.1214/aos/1018031101},
  urldate = {2022-01-21},
  abstract = {We characterize those coherent design criteria which depend only on the dispersion matrix (assumed proper and nonsingular) of the ``state of nature,'' which may be a parameter-vector or a set of future observables, and describe the associated decision problems. Connections are established with the classical approach to optimal design theory for the normal linear model, based on concave functions of the information matrix. Implications of the theory for more general models are also considered.},
  keywords = {62C10,62K05,Bayesian decision theory,Coherence,concavity,dispersion standard,optimal design,optimality criterion,proper scoring rule,uncertainty function},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/BLEK95JQ/Dawid and Sebastiani - 1999 - Coherent dispersion criteria for optimal experimen.pdf;/Users/nikos/github-synced/zotero-nikos/storage/KIXNL5J2/1018031101.html}
}

@article{dawidPresentPositionPotential1984,
  title = {Present {{Position}} and {{Potential Developments}}: {{Some Personal Views Statistical Theory}} the {{Prequential Approach}}},
  shorttitle = {Present {{Position}} and {{Potential Developments}}},
  author = {Dawid, A. P.},
  year = {1984},
  journal = {Journal of the Royal Statistical Society: Series A (General)},
  volume = {147},
  number = {2},
  pages = {278--290},
  issn = {2397-2327},
  doi = {10.2307/2981683},
  urldate = {2020-08-12},
  abstract = {The prequential approach is founded on the premiss that the purpose of statistical inference is to make sequential probability forecasts for future observations, rather than to express information about parameters. Many traditional parametric concepts, such as consistency and efficiency, prove to have natural counterparts in this formulation, which sheds new light on these and suggests fruitful extensions.},
  copyright = {{\copyright} 1984 Royal Statistical Society},
  langid = {english},
  keywords = {consistency,efficiency,likelihood,prequential principle,probability forecasting},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/PX9RNJBW/Dawid - 1984 - Present Position and Potential Developments Some .pdf;/Users/nikos/github-synced/zotero-nikos/storage/UXRWFPAE/2981683.html}
}

@article{dawidProperLocalScoring2012,
  title = {Proper Local Scoring Rules on Discrete Sample Spaces},
  author = {Dawid, A. Philip and Lauritzen, Steffen and Parry, Matthew},
  year = {2012},
  month = feb,
  journal = {The Annals of Statistics},
  volume = {40},
  number = {1},
  eprint = {1104.2224},
  issn = {0090-5364},
  doi = {10.1214/12-AOS972},
  urldate = {2021-12-17},
  abstract = {A scoring rule is a loss function measuring the quality of a quoted probability distribution \$Q\$ for a random variable \$X\$, in the light of the realized outcome \$x\$ of \$X\$; it is proper if the expected score, under any distribution \$P\$ for \$X\$, is minimized by quoting \$Q=P\$. Using the fact that any differentiable proper scoring rule on a finite sample space \$\{{\textbackslash}mathcal\{X\}\}\$ is the gradient of a concave homogeneous function, we consider when such a rule can be local in the sense of depending only on the probabilities quoted for points in a nominated neighborhood of \$x\$. Under mild conditions, we characterize such a proper local scoring rule in terms of a collection of homogeneous functions on the cliques of an undirected graph on the space \$\{{\textbackslash}mathcal\{X\}\}\$. A useful property of such rules is that the quoted distribution \$Q\$ need only be known up to a scale factor. Examples of the use of such scoring rules include Besag's pseudo-likelihood and Hyv{\textbackslash}"\{a\}rinen's method of ratio matching.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {Mathematics - Statistics Theory},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/M7CJGNKN/Dawid et al. - 2012 - Proper local scoring rules on discrete sample spac.pdf}
}

@article{elliottForecastingEconomicsFinance2016,
  title = {Forecasting in {{Economics}} and {{Finance}}},
  author = {Elliott, Graham and Timmermann, Allan},
  year = {2016},
  journal = {Annual Review of Economics},
  volume = {8},
  number = {1},
  pages = {81--110},
  doi = {10.1146/annurev-economics-080315-015346},
  urldate = {2021-12-15},
  abstract = {Practices used to address economic forecasting problems have undergone substantial changes over recent years. We review how such changes have influenced the ways in which a range of forecasting questions are being addressed. We also discuss the promises and challenges arising from access to big data. Finally, we review empirical evidence and experience accumulated from the use of forecasting methods to a range of economic and financial variables.},
  keywords = {big data,forecast evaluation,forecast models,model instability,model misspecification,parameter estimation,risk},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/3AXCLA59/Elliott and Timmermann - 2016 - Forecasting in Economics and Finance.pdf}
}

@article{epsteinScoringSystemProbability1969,
  title = {A {{Scoring System}} for {{Probability Forecasts}} of {{Ranked Categories}}},
  author = {Epstein, Edward S.},
  year = {1969},
  month = dec,
  journal = {Journal of Applied Meteorology},
  volume = {8},
  number = {6},
  pages = {985--987},
  publisher = {American Meteorological Society},
  issn = {0021-8952},
  doi = {10.1175/1520-0450(1969)008<0985:ASSFPF>2.0.CO;2},
  urldate = {2020-08-13},
  langid = {english},
  keywords = {ranked probability score,RPS},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/XAVX39GC/Epstein - 1969 - A Scoring System for Probability Forecasts of Rank.pdf;/Users/nikos/github-synced/zotero-nikos/storage/CVK2YPKP/A-Scoring-System-for-Probability-Forecasts-of.html}
}

@misc{europeancovid-19forecasthubEuropeanCovid19Forecast2021,
  title = {European {{Covid-19 Forecast Hub}}},
  author = {{European Covid-19 Forecast Hub}},
  year = {2021},
  urldate = {2021-05-30},
  howpublished = {https://covid19forecasthub.eu/},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/JRFUHRDI/covid19forecasthub.eu.html}
}

@article{funkAssessingPerformanceRealtime2019,
  title = {Assessing the Performance of Real-Time Epidemic Forecasts: {{A}} Case Study of {{Ebola}} in the {{Western Area}} Region of {{Sierra Leone}}, 2014-15},
  shorttitle = {Assessing the Performance of Real-Time Epidemic Forecasts},
  author = {Funk, Sebastian and Camacho, Anton and Kucharski, Adam J. and Lowe, Rachel and Eggo, Rosalind M. and Edmunds, W. John},
  year = {2019},
  month = feb,
  journal = {PLOS Computational Biology},
  volume = {15},
  number = {2},
  pages = {e1006785},
  issn = {1553-7358},
  doi = {10.1371/journal.pcbi.1006785},
  urldate = {2019-09-16},
  abstract = {Real-time forecasts based on mathematical models can inform critical decision-making during infectious disease outbreaks. Yet, epidemic forecasts are rarely evaluated during or after the event, and there is little guidance on the best metrics for assessment. Here, we propose an evaluation approach that disentangles different components of forecasting ability using metrics that separately assess the calibration, sharpness and bias of forecasts. This makes it possible to assess not just how close a forecast was to reality but also how well uncertainty has been quantified. We used this approach to analyse the performance of weekly forecasts we generated in real time for Western Area, Sierra Leone, during the 2013--16 Ebola epidemic in West Africa. We investigated a range of forecast model variants based on the model fits generated at the time with a semi-mechanistic model, and found that good probabilistic calibration was achievable at short time horizons of one or two weeks ahead but model predictions were increasingly unreliable at longer forecasting horizons. This suggests that forecasts may have been of good enough quality to inform decision making based on predictions a few weeks ahead of time but not longer, reflecting the high level of uncertainty in the processes driving the trajectory of the epidemic. Comparing forecasts based on the semi-mechanistic model to simpler null models showed that the best semi-mechanistic model variant performed better than the null models with respect to probabilistic calibration, and that this would have been identified from the earliest stages of the outbreak. As forecasts become a routine part of the toolkit in public health, standards for evaluation of performance will be important for assessing quality and improving credibility of mathematical models, and for elucidating difficulties and trade-offs when aiming to make the most useful and reliable forecasts.},
  langid = {english},
  keywords = {Epidemiology,Forecasting,Infectious disease epidemiology,Infectious diseases,Mathematical models,Probability distribution,Public and occupational health,Sierra Leone},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/X6Z9PIFT/Funk et al. - 2019 - Assessing the performance of real-time epidemic fo.pdf;/Users/nikos/github-synced/zotero-nikos/storage/JN28VVKF/article.html}
}

@article{funkShorttermForecastsInform2020,
  title = {Short-Term Forecasts to Inform the Response to the {{Covid-19}} Epidemic in the {{UK}}},
  author = {Funk, Sebastian and Abbott, Sam and Atkins, B. D. and Baguelin, M. and Baillie, J. K. and Birrell, P. and Blake, J. and Bosse, Nikos I. and Burton, J. and Carruthers, J. and Davies, N. G. and Angelis, D. De and Dyson, L. and Edmunds, W. J. and Eggo, R. M. and Ferguson, N. M. and Gaythorpe, K. and Gorsich, E. and {Guyver-Fletcher}, G. and Hellewell, J. and Hill, E. M. and Holmes, A. and House, T. A. and Jewell, C. and Jit, M. and Jombart, T. and Joshi, I. and Keeling, M. J. and Kendall, E. and Knock, E. S. and Kucharski, A. J. and Lythgoe, K. A. and Meakin, S. R. and Munday, J. D. and Openshaw, P. J. M. and Overton, C. E. and Pagani, F. and Pearson, J. and {Perez-Guzman}, P. N. and Pellis, L. and Scarabel, F. and Semple, M. G. and Sherratt, K. and Tang, M. and Tildesley, M. J. and {van Leeuwen}, E. and Whittles, L. K. and Group, CMMID COVID-19 Working and Team, Imperial College COVID-19 Response and Investigators, Isaric4c},
  year = {2020},
  month = nov,
  journal = {medRxiv},
  pages = {2020.11.11.20220962},
  publisher = {Cold Spring Harbor Laboratory Press},
  doi = {10.1101/2020.11.11.20220962},
  urldate = {2020-11-28},
  abstract = {{$<$}p{$>$}Background: Short-term forecasts of infectious disease can create situational awareness and inform planning for outbreak response. Here, we report on multi-model forecasts of Covid-19 in the UK that were generated at regular intervals starting at the end of March 2020, in order to monitor expected healthcare utilisation and population impacts in real time. Methods: We evaluated the performance of individual model forecasts generated between 24 March and 14 July 2020, using a variety of metrics including the weighted interval score as well as metrics that assess the calibration, sharpness, bias and absolute error of forecasts separately. We further combined the predictions from individual models to ensemble forecasts using a simple mean as well as a quantile regression average that aimed to maximise performance. We further compared model performance to a null model of no change. Results: In most cases, individual models performed better than the null model, and ensembles models were well calibrated and performed comparatively to the best individual models. The quantile regression average did not noticeably outperform the mean ensemble. Conclusions: Ensembles of multi-model forecasts can inform the policy response to the Covid-19 pandemic by assessing future resource needs and expected population impact of morbidity and mortality.{$<$}/p{$>$}},
  copyright = {{\copyright} 2020, Posted by Cold Spring Harbor Laboratory. This pre-print is available under a Creative Commons License (Attribution 4.0 International), CC BY 4.0, as described at http://creativecommons.org/licenses/by/4.0/},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/9RK57885/Funk et al. - 2020 - Short-term forecasts to inform the response to the.pdf;/Users/nikos/github-synced/zotero-nikos/storage/AKDY6PAQ/2020.11.11.20220962v1.full.html}
}

@article{gelmanUnderstandingPredictiveInformation2014,
  title = {Understanding Predictive Information Criteria for {{Bayesian}} Models},
  author = {Gelman, Andrew and Hwang, Jessica and Vehtari, Aki},
  year = {2014},
  month = nov,
  journal = {Statistics and Computing},
  volume = {24},
  number = {6},
  pages = {997--1016},
  issn = {1573-1375},
  doi = {10.1007/s11222-013-9416-2},
  urldate = {2019-02-28},
  abstract = {We review the Akaike, deviance, and Watanabe-Akaike information criteria from a Bayesian perspective, where the goal is to estimate expected out-of-sample-prediction error using a bias-corrected adjustment of within-sample error. We focus on the choices involved in setting up these measures, and we compare them in three simple examples, one theoretical and two applied. The contribution of this paper is to put all these information criteria into a Bayesian predictive context and to better understand, through small examples, how these methods can apply in practice.},
  langid = {english},
  keywords = {AIC,Bayes,Cross-validation,DIC,Prediction,WAIC},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/B2VVZDAP/Gelman et al. - 2014 - Understanding predictive information criteria for .pdf}
}

@article{gneitingProbabilisticForecastsCalibration2007,
  title = {Probabilistic Forecasts, Calibration and Sharpness},
  author = {Gneiting, Tilmann and Balabdaoui, Fadoua and Raftery, Adrian E.},
  year = {2007},
  journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
  volume = {69},
  number = {2},
  pages = {243--268},
  issn = {1467-9868},
  doi = {10.1111/j.1467-9868.2007.00587.x},
  urldate = {2020-02-17},
  abstract = {Summary. Probabilistic forecasts of continuous variables take the form of predictive densities or predictive cumulative distribution functions. We propose a diagnostic approach to the evaluation of predictive performance that is based on the paradigm of maximizing the sharpness of the predictive distributions subject to calibration. Calibration refers to the statistical consistency between the distributional forecasts and the observations and is a joint property of the predictions and the events that materialize. Sharpness refers to the concentration of the predictive distributions and is a property of the forecasts only. A simple theoretical framework allows us to distinguish between probabilistic calibration, exceedance calibration and marginal calibration. We propose and study tools for checking calibration and sharpness, among them the probability integral transform histogram, marginal calibration plots, the sharpness diagram and proper scoring rules. The diagnostic approach is illustrated by an assessment and ranking of probabilistic forecasts of wind speed at the Stateline wind energy centre in the US Pacific Northwest. In combination with cross-validation or in the time series context, our proposal provides very general, nonparametric alternatives to the use of information criteria for model diagnostics and model selection.},
  langid = {english},
  keywords = {Cross-validation,Density forecast,Ensemble prediction system,Ex post evaluation,Forecast verification,Model diagnostics,Posterior predictive assessment,Predictive distribution,Prequential principle,Probability integral transform,Proper scoring rule},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/BUWD6CGT/Gneiting et al. - 2007 - Probabilistic forecasts, calibration and sharpness.pdf;/Users/nikos/github-synced/zotero-nikos/storage/EUCMSBKN/j.1467-9868.2007.00587.html}
}

@article{gneitingStrictlyProperScoring2007,
  title = {Strictly {{Proper Scoring Rules}}, {{Prediction}}, and {{Estimation}}},
  author = {Gneiting, Tilmann and Raftery, Adrian E},
  year = {2007},
  month = mar,
  journal = {Journal of the American Statistical Association},
  volume = {102},
  number = {477},
  pages = {359--378},
  issn = {0162-1459, 1537-274X},
  doi = {10.1198/016214506000001437},
  urldate = {2020-03-22},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/P599P5ZY/Gneiting and Raftery - 2007 - Strictly Proper Scoring Rules, Prediction, and Est.pdf}
}

@article{gneitingWeatherForecastingEnsemble2005,
  title = {Weather {{Forecasting}} with {{Ensemble Methods}}},
  author = {Gneiting, Tilmann and Raftery, Adrian E.},
  year = {2005},
  month = oct,
  journal = {Science},
  volume = {310},
  number = {5746},
  pages = {248--249},
  publisher = {American Association for the Advancement of Science},
  issn = {0036-8075, 1095-9203},
  doi = {10.1126/science.1115255},
  urldate = {2021-05-30},
  abstract = {{$<$}p{$>$} Traditional weather forecasting has been built on a foundation of deterministic modeling--start with initial conditions, put them into a supercomputer model, and end up with a prediction about future weather. But as Gneiting and Raftery discuss in their Perspective, a new approach--ensemble forecasting--was introduced in the early 1990s. In this method, up to 100 different computer runs, each with slightly different starting conditions or model assumptions, are combined into a weather forecast. In concert with statistical techniques, ensembles can provide accurate statements about the uncertainty in daily and seasonal forecasting. The challenge now is to improve the modeling, statistical analysis, and visualization technologies for disseminating the ensemble results. {$<$}/p{$>$}},
  chapter = {Perspective},
  copyright = {{\copyright} 2005 American Association for the Advancement of Science},
  langid = {english},
  pmid = {16224011},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/VRJMN77J/Gneiting and Raftery - 2005 - Weather Forecasting with Ensemble Methods.pdf;/Users/nikos/github-synced/zotero-nikos/storage/8Q5UA2FU/248.html}
}

@article{goodRationalDecisions1952,
  title = {Rational {{Decisions}}},
  author = {Good, I. J.},
  year = {1952},
  journal = {Journal of the Royal Statistical Society. Series B (Methodological)},
  volume = {14},
  number = {1},
  eprint = {2984087},
  eprinttype = {jstor},
  pages = {107--114},
  publisher = {[Royal Statistical Society, Wiley]},
  issn = {0035-9246},
  urldate = {2020-08-13},
  abstract = {This paper deals first with the relationship between the theory of probability and the theory of rational behaviour. A method is then suggested for encouraging people to make accurate probability estimates, a connection with the theory of information being mentioned. Finally Wald's theory of statistical decision functions is summarised and generalised and its relation to the theory of rational behaviour is discussed.},
  keywords = {Log Score,LogS},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/23458422/2020 - Rational Decisions.pdf}
}

@article{hamillInterpretationRankHistograms2001a,
  title = {Interpretation of {{Rank Histograms}} for {{Verifying Ensemble Forecasts}}},
  author = {Hamill, Thomas M.},
  year = {2001},
  month = mar,
  journal = {Monthly Weather Review},
  volume = {129},
  number = {3},
  pages = {550--560},
  publisher = {American Meteorological Society},
  issn = {1520-0493, 0027-0644},
  doi = {10.1175/1520-0493(2001)129<0550:IORHFV>2.0.CO;2},
  urldate = {2022-01-21},
  abstract = {Abstract Rank histograms are a tool for evaluating ensemble forecasts. They are useful for determining the reliability of ensemble forecasts and for diagnosing errors in its mean and spread. Rank histograms are generated by repeatedly tallying the rank of the verification (usually an observation) relative to values from an ensemble sorted from lowest to highest. However, an uncritical use of the rank histogram can lead to misinterpretations of the qualities of that ensemble. For example, a flat rank histogram, usually taken as a sign of reliability, can still be generated from unreliable ensembles. Similarly, a U-shaped rank histogram, commonly understood as indicating a lack of variability in the ensemble, can also be a sign of conditional bias. It is also shown that flat rank histograms can be generated for some model variables if the variance of the ensemble is correctly specified, yet if covariances between model grid points are improperly specified, rank histograms for combinations of model variables may not be flat. Further, if imperfect observations are used for verification, the observational errors should be accounted for, otherwise the shape of the rank histogram may mislead the user about the characteristics of the ensemble. If a statistical hypothesis test is to be performed to determine whether the differences from uniformity of rank are statistically significant, then samples used to populate the rank histogram must be located far enough away from each other in time and space to be considered independent.},
  chapter = {Monthly Weather Review},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/FJYU9QZH/Hamill - 2001 - Interpretation of Rank Histograms for Verifying En.pdf;/Users/nikos/github-synced/zotero-nikos/storage/SH65U38N/1520-0493_2001_129_0550_iorhfv_2.0.co_2.html}
}

@article{hersbachDecompositionContinuousRanked2000a,
  title = {Decomposition of the {{Continuous Ranked Probability Score}} for {{Ensemble Prediction Systems}}},
  author = {Hersbach, Hans},
  year = {2000},
  month = oct,
  journal = {Weather and Forecasting},
  volume = {15},
  number = {5},
  pages = {559--570},
  publisher = {American Meteorological Society},
  issn = {1520-0434, 0882-8156},
  doi = {10.1175/1520-0434(2000)015<0559:DOTCRP>2.0.CO;2},
  urldate = {2022-03-14},
  abstract = {Abstract Some time ago, the continuous ranked probability score (CRPS) was proposed as a new verification tool for (probabilistic) forecast systems. Its focus is on the entire permissible range of a certain (weather) parameter. The CRPS can be seen as a ranked probability score with an infinite number of classes, each of zero width. Alternatively, it can be interpreted as the integral of the Brier score over all possible threshold values for the parameter under consideration. For a deterministic forecast system the CRPS reduces to the mean absolute error. In this paper it is shown that for an ensemble prediction system the CRPS can be decomposed into a reliability part and a resolution/uncertainty part, in a way that is similar to the decomposition of the Brier score. The reliability part of the CRPS is closely connected to the rank histogram of the ensemble, while the resolution/uncertainty part can be related to the average spread within the ensemble and the behavior of its outliers. The usefulness of such a decomposition is illustrated for the ensemble prediction system running at the European Centre for Medium-Range Weather Forecasts. The evaluation of the CRPS and its decomposition proposed in this paper can be extended to systems issuing continuous probability forecasts, by realizing that these can be interpreted as the limit of ensemble forecasts with an infinite number of members.},
  chapter = {Weather and Forecasting},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/8C93QRYF/Hersbach - 2000 - Decomposition of the Continuous Ranked Probability.pdf;/Users/nikos/github-synced/zotero-nikos/storage/7PARNA8T/1520-0434_2000_015_0559_dotcrp_2_0_co_2.html}
}

@misc{https://epiforecasts.io/scoringutilsUtilitiesScoringAssessing,
  title = {Utilities for {{Scoring}} and {{Assessing Predictions}}},
  author = {{https://epiforecasts.io/scoringutils}},
  urldate = {2024-02-27},
  abstract = {Provides a collection of metrics and proper scoring rules (Tilmann Gneiting \& Adrian E Raftery (2007) {$<$}doi:10.1198/016214506000001437{$>$}, Jordan, A., Kr{\"u}ger, F., \& Lerch, S. (2019) {$<$}doi:10.18637/jss.v090.i12{$>$}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) {$<$}doi:10.1371/journal.pcbi.1006785{$>$}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. Find more information about the package in the accompanying paper ({$<$}doi:10.48550/arXiv.2205.07090{$>$}).},
  howpublished = {https://epiforecasts.io/scoringutils/},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/FZIQXXJW/scoringutils.html}
}

@article{jordanEvaluatingProbabilisticForecasts2019,
  title = {Evaluating {{Probabilistic Forecasts}} with {{{\textbf{scoringRules}}}}},
  author = {Jordan, Alexander and Kr{\"u}ger, Fabian and Lerch, Sebastian},
  year = {2019},
  journal = {Journal of Statistical Software},
  volume = {90},
  number = {12},
  issn = {1548-7660},
  doi = {10.18637/jss.v090.i12},
  urldate = {2020-02-13},
  abstract = {Probabilistic forecasts in the form of probability distributions over future events have become popular in several fields including meteorology, hydrology, economics, and demography. In typical applications, many alternative statistical models and data sources can be used to produce probabilistic forecasts. Hence, evaluating and selecting among competing methods is an important task. The scoringRules package for R provides functionality for comparative evaluation of probabilistic models based on proper scoring rules, covering a wide range of situations in applied work. This paper discusses implementation and usage details, presents case studies from meteorology and economics, and points to the relevant background literature.},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/DSYW6QUF/Jordan et al. - 2019 - Evaluating Probabilistic Forecasts with bscoring.pdf}
}

@article{joseCharacterizationSphericalScoring2009,
  title = {A {{Characterization}} for the {{Spherical Scoring Rule}}},
  author = {Jose, Victor Richmond},
  year = {2009},
  month = mar,
  journal = {Theory and Decision},
  volume = {66},
  number = {3},
  pages = {263--281},
  issn = {1573-7187},
  doi = {10.1007/s11238-007-9067-x},
  urldate = {2022-03-14},
  abstract = {Strictly proper scoring rules have been studied widely in statistical decision theory and recently in experimental economics because of their ability to encourage assessors to honestly provide their true subjective probabilities. In this article, we study the spherical scoring rule by analytically examining some of its properties and providing some new geometric interpretations for this rule. Moreover, we state a theorem which provides an axiomatic characterization for the spherical scoring rule. The objective of this analysis is to provide a better understanding of one of the most commonly available scoring rules, which could aid decision makers in the selection of an appropriate tool for evaluating and assessing probabilistic forecasts.},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/TSSMRAB9/Jose - 2009 - A Characterization for the Spherical Scoring Rule.pdf}
}

@article{kukkonenReviewOperationalRegionalscale2012,
  title = {A Review of Operational, Regional-Scale, Chemical Weather Forecasting Models in {{Europe}}},
  author = {Kukkonen, J. and Olsson, T. and Schultz, D. M. and Baklanov, A. and Klein, T. and Miranda, A. I. and Monteiro, A. and Hirtl, M. and Tarvainen, V. and Boy, M. and Peuch, V.-H. and Poupkou, A. and Kioutsioukis, I. and Finardi, S. and Sofiev, M. and Sokhi, R. and Lehtinen, K. E. J. and Karatzas, K. and San Jos{\'e}, R. and Astitha, M. and Kallos, G. and Schaap, M. and Reimer, E. and Jakobs, H. and Eben, K.},
  year = {2012},
  month = jan,
  journal = {Atmospheric Chemistry and Physics},
  volume = {12},
  number = {1},
  pages = {1--87},
  publisher = {Copernicus GmbH},
  issn = {1680-7316},
  doi = {10.5194/acp-12-1-2012},
  urldate = {2021-12-15},
  abstract = {{$<$}p{$><$}strong class="journal-contentHeaderColor"{$>$}Abstract.{$<$}/strong{$>$} Numerical models that combine weather forecasting and atmospheric chemistry are here referred to as chemical weather forecasting models. Eighteen operational chemical weather forecasting models on regional and continental scales in Europe are described and compared in this article. Topics discussed in this article include how weather forecasting and atmospheric chemistry models are integrated into chemical weather forecasting systems, how physical processes are incorporated into the models through parameterization schemes, how the model architecture affects the predicted variables, and how air chemistry and aerosol processes are formulated. In addition, we discuss sensitivity analysis and evaluation of the models, user operational requirements, such as model availability and documentation, and output availability and dissemination. In this manner, this article allows for the evaluation of the relative strengths and weaknesses of the various modelling systems and modelling approaches. Finally, this article highlights the most prominent gaps of knowledge for chemical weather forecasting models and suggests potential priorities for future research directions, for the following selected focus areas: emission inventories, the integration of numerical weather prediction and atmospheric chemical transport models, boundary conditions and nesting of models, data assimilation of the various chemical species, improved understanding and parameterization of physical processes, better evaluation of models against data and the construction of model ensembles.{$<$}/p{$>$}},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/X3N7D4HE/Kukkonen et al. - 2012 - A review of operational, regional-scale, chemical .pdf;/Users/nikos/github-synced/zotero-nikos/storage/XWR2S6F8/2012.html}
}

@article{liboschikTscountPackageAnalysis2017,
  title = {Tscount: {{An R Package}} for {{Analysis}} of {{Count Time Series Following Generalized Linear Models}}},
  shorttitle = {Tscount},
  author = {Liboschik, Tobias and Fokianos, Konstantinos and Fried, Roland},
  year = {2017},
  month = nov,
  journal = {Journal of Statistical Software},
  volume = {82},
  pages = {1--51},
  issn = {1548-7660},
  doi = {10.18637/jss.v082.i05},
  urldate = {2022-01-21},
  abstract = {The R package tscount provides likelihood-based estimation methods for analysis and modeling of count time series following generalized linear models. This is a flexible class of models which can describe serial correlation in a parsimonious way. The conditional mean of the process is linked to its past values, to past observations and to potential covariate effects. The package allows for models with the identity and with the logarithmic link function. The conditional distribution can be Poisson or negative binomial. An important special case of this class is the so-called INGARCH model and its log-linear extension. The package includes methods for model fitting and assessment, prediction and intervention analysis. This paper summarizes the theoretical background of these methods. It gives details on the implementation of the package and provides simulation results for models which have not been studied theoretically before. The usage of the package is illustrated by two data examples. Additionally, we provide a review of R packages which can be used for count time series analysis. This includes a detailed comparison of tscount to those packages.},
  copyright = {Copyright (c) 2017 Tobias Liboschik, Konstantinos Fokianos, Roland Fried},
  langid = {english},
  keywords = {serial correlation},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/NAKUKRYZ/Liboschik et al. - 2017 - tscount An R Package for Analysis of Count Time S.pdf}
}

@article{macheteContrastingProbabilisticScoring2012,
  title = {Contrasting {{Probabilistic Scoring Rules}}},
  author = {Machete, Reason Lesego},
  year = {2012},
  month = jul,
  journal = {arXiv:1112.4530 [math, stat]},
  eprint = {1112.4530},
  primaryclass = {math, stat},
  urldate = {2020-03-21},
  abstract = {There are several scoring rules that one can choose from in order to score probabilistic forecasting models or estimate model parameters. Whilst it is generally agreed that proper scoring rules are preferable, there is no clear criterion for preferring one proper scoring rule above another. This manuscript compares and contrasts some commonly used proper scoring rules and provides guidance on scoring rule selection. In particular, it is shown that the logarithmic scoring rule prefers erring with more uncertainty, the spherical scoring rule prefers erring with lower uncertainty, whereas the other scoring rules are indifferent to either option.},
  archiveprefix = {arXiv},
  langid = {english},
  keywords = {62B10 62C05 62G05 62G07 62F99 62P05 62P12 62P20,Mathematics - Statistics Theory},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/8FYPC3Y4/Machete - 2012 - Contrasting Probabilistic Scoring Rules.pdf}
}

@article{mannTestWhetherOne1947,
  title = {On a {{Test}} of {{Whether}} One of {{Two Random Variables}} Is {{Stochastically Larger}} than the {{Other}}},
  author = {Mann, H. B. and Whitney, D. R.},
  year = {1947},
  month = mar,
  journal = {The Annals of Mathematical Statistics},
  volume = {18},
  number = {1},
  pages = {50--60},
  publisher = {Institute of Mathematical Statistics},
  issn = {0003-4851, 2168-8990},
  doi = {10.1214/aoms/1177730491},
  urldate = {2022-01-21},
  abstract = {Let \$x\$ and \$y\$ be two random variables with continuous cumulative distribution functions \$f\$ and \$g\$. A statistic \$U\$ depending on the relative ranks of the \$x\$'s and \$y\$'s is proposed for testing the hypothesis \$f = g\$. Wilcoxon proposed an equivalent test in the Biometrics Bulletin, December, 1945, but gave only a few points of the distribution of his statistic. Under the hypothesis \$f = g\$ the probability of obtaining a given \$U\$ in a sample of \$n x's\$ and \$m y's\$ is the solution of a certain recurrence relation involving \$n\$ and \$m\$. Using this recurrence relation tables have been computed giving the probability of \$U\$ for samples up to \$n = m = 8\$. At this point the distribution is almost normal. From the recurrence relation explicit expressions for the mean, variance, and fourth moment are obtained. The 2rth moment is shown to have a certain form which enabled us to prove that the limit distribution is normal if \$m, n\$ go to infinity in any arbitrary manner. The test is shown to be consistent with respect to the class of alternatives \$f(x) {$>$} g(x)\$ for every \$x\$.},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/YTWX67GQ/Mann and Whitney - 1947 - On a Test of Whether one of Two Random Variables i.pdf;/Users/nikos/github-synced/zotero-nikos/storage/8DA4G3H8/1177730491.html}
}

@article{mathesonScoringRulesContinuous1976,
  title = {Scoring {{Rules}} for {{Continuous Probability Distributions}}},
  author = {Matheson, James E. and Winkler, Robert L.},
  year = {1976},
  month = jun,
  journal = {Management Science},
  volume = {22},
  number = {10},
  pages = {1087--1096},
  publisher = {INFORMS},
  issn = {0025-1909},
  doi = {10.1287/mnsc.22.10.1087},
  urldate = {2020-08-13},
  abstract = {Personal, or subjective, probabilities are used as inputs to many inferential and decision-making models, and various procedures have been developed for the elicitation of such probabilities. Included among these elicitation procedures are scoring rules, which involve the computation of a score based on the assessor's stated probabilities and on the event that actually occurs. The development of scoring rules has, in general, been restricted to the elicitation of discrete probability distributions. In this paper, families of scoring rules for the elicitation of continuous probability distributions are developed and discussed.},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/SVJ7YPP7/Matheson and Winkler - 1976 - Scoring Rules for Continuous Probability Distribut.pdf;/Users/nikos/github-synced/zotero-nikos/storage/H5CNZS4U/mnsc.22.10.html}
}

@article{murphyNoteRankedProbability1971a,
  title = {A {{Note}} on the {{Ranked Probability Score}}},
  author = {Murphy, Allan H.},
  year = {1971},
  month = feb,
  journal = {Journal of Applied Meteorology and Climatology},
  volume = {10},
  number = {1},
  pages = {155--156},
  publisher = {American Meteorological Society},
  issn = {1520-0450},
  doi = {10.1175/1520-0450(1971)010<0155:ANOTRP>2.0.CO;2},
  urldate = {2022-03-14},
  abstract = {Abstract},
  chapter = {Journal of Applied Meteorology and Climatology},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/G4ZUDF6B/Murphy - 1971 - A Note on the Ranked Probability Score.pdf;/Users/nikos/github-synced/zotero-nikos/storage/DA47KK7K/1520-0450_1971_010_0155_anotrp_2_0_co_2.html}
}

@article{reichCollaborativeMultiyearMultimodel2019,
  title = {A Collaborative Multiyear, Multimodel Assessment of Seasonal Influenza Forecasting in the {{United States}}},
  author = {Reich, Nicholas G. and Brooks, Logan C. and Fox, Spencer J. and Kandula, Sasikiran and McGowan, Craig J. and Moore, Evan and Osthus, Dave and Ray, Evan L. and Tushar, Abhinav and Yamana, Teresa K. and Biggerstaff, Matthew and Johansson, Michael A. and Rosenfeld, Roni and Shaman, Jeffrey},
  year = {2019},
  month = feb,
  journal = {Proceedings of the National Academy of Sciences},
  volume = {116},
  number = {8},
  pages = {3146--3154},
  publisher = {National Academy of Sciences},
  issn = {0027-8424, 1091-6490},
  doi = {10.1073/pnas.1812594116},
  urldate = {2021-10-13},
  abstract = {Influenza infects an estimated 9--35 million individuals each year in the United States and is a contributing cause for between 12,000 and 56,000 deaths annually. Seasonal outbreaks of influenza are common in temperate regions of the world, with highest incidence typically occurring in colder and drier months of the year. Real-time forecasts of influenza transmission can inform public health response to outbreaks. We present the results of a multiinstitution collaborative effort to standardize the collection and evaluation of forecasting models for influenza in the United States for the 2010/2011 through 2016/2017 influenza seasons. For these seven seasons, we assembled weekly real-time forecasts of seven targets of public health interest from 22 different models. We compared forecast accuracy of each model relative to a historical baseline seasonal average. Across all regions of the United States, over half of the models showed consistently better performance than the historical baseline when forecasting incidence of influenza-like illness 1 wk, 2 wk, and 3 wk ahead of available data and when forecasting the timing and magnitude of the seasonal peak. In some regions, delays in data reporting were strongly and negatively associated with forecast accuracy. More timely reporting and an improved overall accessibility to novel and traditional data sources are needed to improve forecasting accuracy and its integration with real-time public health decision making.},
  chapter = {PNAS Plus},
  copyright = {Copyright {\copyright} 2019 the Author(s). Published by PNAS.. https://creativecommons.org/licenses/by-nc-nd/4.0/This open access article is distributed under Creative Commons Attribution-NonCommercial-NoDerivatives License 4.0 (CC BY-NC-ND).},
  langid = {english},
  pmid = {30647115},
  keywords = {forecasting,infectious disease,influenza,public health,statistics},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/XEKLR37W/Reich et al. - 2019 - A collaborative multiyear, multimodel assessment o.pdf;/Users/nikos/github-synced/zotero-nikos/storage/QID3PLW4/3146.html}
}

@misc{ScoringEpidemiologicalForecasts,
  title = {Scoring Epidemiological Forecasts on Transformed Scales {\textbar} {{PLOS Computational Biology}}},
  urldate = {2023-11-16},
  howpublished = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1011393},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/ZF63XZW2/article.html}
}

@article{sherrattPredictivePerformanceMultimodel2022,
  title = {Predictive Performance of Multi-Model Ensemble Forecasts of {{COVID-19}} across  {{European}} Nation},
  author = {Sherratt, K. and Gruson, H. and Grah, R. and Johnson, H. and Niehus, R. and Prasse, B. and Sandman, F. and Deuschel, J. and Wolffram, D. and Abbott, S. and Ullrich, A. and Gibson, G. and Ray, {\relax EL}. and Reich, {\relax NG}. and Sheldon, D. and Wang, Y. and Wattanachit, N. and Wang, L. and Trnka, J. and Obozinski, G. and Sun, T. and Thanou, D. and Pottier, L. and Krymova, E. and Barbarossa, {\relax MV}. and Leith{\"a}user, N. and Mohring, J. and Schneider, J. and Wlazlo, J. and Fuhrmann, J. and Lange, B. and Rodiah, I. and Baccam, P. and Gurung, H. and Stage, S. and Suchoski, B. and Budzinski, J. and Walraven, R. and Villanueva, I. and Tucek, V. and {\v S}m{\'i}d, M. and Zaj{\'i}cek, M. and P{\'e}rez, {\'A}lvarez C. and Reina, B. and Bosse, {\relax NI}. and Meakin, S. and Di Loro, Alaimo and Maruotti, A. and Eclerov{\'a}, V. and Kraus, A. and Kraus, D. and Pribylova, L. and Dimitris, B. and Li, {\relax ML}. and Saksham, S. and Dehning, J. and Mohr, S. and Priesemann, V. and Redlarski, G. and Bejar, B. and Ardenghi, G. and Parolini, N. and Ziarelli, G. and Bock, W. and Heyder, S. and Hotz, T. and E., Singh D. and {Guzman-Merino}, M. and Aznarte, {\relax JL}. and Mori{\~n}a, D. and Alonso, S. and {\'A}lvarez, E. and L{\'o}pez, D. and Prats, C. and Burgard, {\relax JP}. and Rodloff, A. and Zimmermann, T. and Kuhlmann, A. and Zibert, J. and Pennoni, F. and Divino, F. and Catal{\`a}, M. and Lovison, G. and Giudici, P. and Tarantino, B. and Bartolucci, F. and Jona, Lasinio G. and Mingione, M. and Farcomeni, A. and Srivastava, A. and {Montero-Manso}, P. and Adiga, A. and Hurt, B. and Lewis, B. and Marathe, M. and Porebski, P. and Venkatramanan, S. and Bartczuk, R. and Dreger, F. and Gambin, A. and Gogolewski, K. and {Gruziel-Slomka}, M. and Krupa, B. and Moszynski, A. and Niedzielewski, K. and Nowosielski, J. and Radwan, M. and Rakowski, F. and Semeniuk, M. and Szczurek, E. and Zielinski, J. and Kisielewski, J. and Pabjan, B. and Holger, K. and Kheifetz, Y. and Scholz, M. and Bodych, M. and Filinski, M. and Idzikowski, R. and Krueger, T. and Ozanski, T. and Bracher, J. and Funk, S.},
  year = {2022},
  journal = {Europe PMC},
  doi = {10.1101/2022.06.16.22276024},
  urldate = {2023-02-07},
  abstract = {Background  Short-term forecasts of infectious disease burden can contribute to  situational awareness and aid capacity planning. Based on best practice in  other fields and recent insights in infectious disease epidemiology, one can  maximise the predictive performance of such forecasts if multiple models are  combined into an ensemble. Here we report on the performance of ensembles in  predicting COVID-19 cases and deaths across Europe between 08 March 2021 and  07 March 2022. Methods  We used open-source tools to develop a public European COVID-19 Forecast  Hub. We invited groups globally to contribute weekly forecasts for COVID-19  cases and deaths reported from a standardised source over the next one to  four weeks. Teams submitted forecasts from March 2021 using standardised  quantiles of the predictive distribution. Each week we created an ensemble  forecast, where each predictive quantile was calculated as the  equally-weighted average (initially the mean and then from 26th July the  median) of all individual models' predictive quantiles. We measured the  performance of each model using the relative Weighted Interval Score (WIS),  comparing models' forecast accuracy relative to all other models. We  retrospectively explored alternative methods for ensemble forecasts,  including weighted averages based on models' past predictive  performance. Results  Over 52 weeks we collected and combined up to 28 forecast models for 32  countries. We found a weekly ensemble had a consistently strong performance  across countries over time. Across all horizons and locations, the ensemble  performed better on relative WIS than 84\% of participating models' forecasts  of incident cases (with a total N=862), and 92\% of participating models'  forecasts of deaths (N=746). Across a one to four week time horizon,  ensemble performance declined with longer forecast periods when forecasting  cases, but remained stable over four weeks for incident death forecasts. In  every forecast across 32 countries, the ensemble outperformed most  contributing models when forecasting either cases or deaths, frequently  outperforming all of its individual component models. Among several choices  of ensemble methods we found that the most influential and best choice was  to use a median average of models instead of using the mean, regardless of  methods of weighting component forecast models. Conclusions  Our results support the use of combining forecasts from individual  models into an ensemble in order to improve predictive performance across  epidemiological targets and populations during infectious disease epidemics.  Our findings further suggest that median ensemble methods yield better  predictive performance more than ones based on means. Our findings also  highlight that forecast consumers should place more weight on incident death  forecasts than incident case forecasts at forecast horizons greater than two  weeks. Code and data availability  All data and code are publicly available on Github:  covid19-forecast-hub-europe/euro-hub-ensemble.},
  copyright = {cc by},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/HK3RSINT/Sherratt et al. - 2022 - Predictive performance of multi-model ensemble for.pdf}
}

@article{thoreyOnlineLearningContinuous2017a,
  title = {Online Learning with the {{Continuous Ranked Probability Score}} for Ensemble Forecasting},
  author = {Thorey, J. and Mallet, V. and Baudin, P.},
  year = {2017},
  month = jan,
  journal = {Quarterly Journal of the Royal Meteorological Society},
  volume = {143},
  number = {702},
  pages = {521--529},
  issn = {0035-9009, 1477-870X},
  doi = {10.1002/qj.2940},
  urldate = {2022-03-14},
  abstract = {Ensemble forecasting resorts to multiple individual forecasts to produce a discrete probability distribution which accurately represents the uncertainties. Before every forecast, a weighted empirical distribution function is derived from the ensemble, so as to minimize the Continuous Ranked Probability Score (CRPS). We apply online learning techniques, which have previously been used for deterministic forecasting, and we adapt them for the minimization of the CRPS. The proposed method theoretically guarantees that the aggregated forecast competes, in terms of CRPS, against the best weighted empirical distribution function with weights constant in time. This is illustrated on synthetic data. Besides, our study improves the knowledge of the CRPS expectation for model mixtures. We generalize results on the bias of the CRPS computed with ensemble forecasts, and propose a new scheme to achieve fair CRPS minimization, without any assumption on the distributions.},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/R96EA9GW/Thorey et al. - 2017 - Online learning with the Continuous Ranked Probabi.pdf}
}

@article{timmermannForecastingMethodsFinance2018,
  title = {Forecasting {{Methods}} in {{Finance}}},
  author = {Timmermann, Allan},
  year = {2018},
  journal = {Annual Review of Financial Economics},
  volume = {10},
  number = {1},
  pages = {449--479},
  doi = {10.1146/annurev-financial-110217-022713},
  urldate = {2020-12-14},
  abstract = {Our review highlights some of the key challenges in financial forecasting problems and opportunities arising from the unique features of financial data. We analyze the difficulty of establishing predictability in an environment with a low signal-to-noise ratio, persistent predictors, and instability in predictive relations arising from competitive pressures and investors' learning. We discuss approaches for forecasting the mean, variance, and probability distribution of asset returns. Finally, we discuss how to evaluate financial forecasts while accounting for the possibility that numerous forecasting models may have been considered, leading to concerns of data mining.}
}

@article{timmermannForecastingMethodsFinance2018a,
  title = {Forecasting {{Methods}} in {{Finance}}},
  author = {Timmermann, Allan},
  year = {2018},
  month = nov,
  journal = {Annual Review of Financial Economics},
  volume = {10},
  number = {1},
  pages = {449--479},
  issn = {1941-1367, 1941-1375},
  doi = {10.1146/annurev-financial-110217-022713},
  urldate = {2021-12-15},
  abstract = {Our review highlights some of the key challenges in financial forecasting problems and opportunities arising from the unique features of financial data. We analyze the difficulty of establishing predictability in an environment with a low signal-to-noise ratio, persistent predictors, and instability in predictive relations arising from competitive pressures and investors' learning. We discuss approaches for forecasting the mean, variance, and probability distribution of asset returns. Finally, we discuss how to evaluate financial forecasts while accounting for the possibility that numerous forecasting models may have been considered, leading to concerns of data mining.},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/9FHI5V4F/Timmermann - 2018 - Forecasting Methods in Finance.pdf}
}

@article{winklerScoringRulesEvaluation1996,
  title = {Scoring Rules and the Evaluation of Probabilities},
  author = {Winkler, R. L. and Mu{\~n}oz, Javier and Cervera, Jos{\'e} L. and Bernardo, Jos{\'e} M. and Blattenberger, Gail and Kadane, Joseph B. and Lindley, Dennis V. and Murphy, Allan H. and Oliver, Robert M. and {R{\'i}os-Insua}, David},
  year = {1996},
  month = jun,
  journal = {Test},
  volume = {5},
  number = {1},
  pages = {1--60},
  issn = {1863-8260},
  doi = {10.1007/BF02562681},
  urldate = {2021-03-04},
  abstract = {In Bayesian inference and decision analysis, inferences and predictions are inherently probabilistic in nature. Scoring rules, which involve the computation of a score based on probability forecasts and what actually occurs, can be used to evaluate probabilities and to provide appropriate incentives for ``good'' probabilities. This paper review scoring rules and some related measures for evaluating probabilities, including decompositions of scoring rules and attributes of ``goodness'' of probabilites, comparability of scores, and the design of scoring rules for specific inferential and decision-making problems},
  langid = {english},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/VHTQR6BK/Winkler et al. - 1996 - Scoring rules and the evaluation of probabilities.pdf}
}

@article{zellnerSurveyHumanJudgement2021,
  title = {A Survey of Human Judgement and Quantitative Forecasting Methods},
  author = {Zellner, Maximilian and Abbas, Ali E. and Budescu, David V. and Galstyan, Aram},
  year = {2021},
  month = feb,
  journal = {Royal Society Open Science},
  volume = {8},
  number = {2},
  pages = {201187},
  publisher = {Royal Society},
  doi = {10.1098/rsos.201187},
  urldate = {2023-02-01},
  abstract = {This paper's top-level goal is to provide an overview of research conducted in the many academic domains concerned with forecasting. By providing a summary encompassing these domains, this survey connects them, establishing a common ground for future discussions. To this end, we survey literature on human judgement and quantitative forecasting as well as hybrid methods that involve both humans and algorithmic approaches. The survey starts with key search terms that identified more than 280 publications in the fields of computer science, operations research, risk analysis, decision science, psychology and forecasting. Results show an almost 10-fold increase in the application-focused forecasting literature between the 1990s and the current decade, with a clear rise of quantitative, data-driven forecasting models. Comparative studies of quantitative methods and human judgement show that (1) neither method is universally superior, and (2) the better method varies as a function of factors such as availability, quality, extent and format of data, suggesting that (3) the two approaches can complement each other to yield more accurate and resilient models. We also identify four research thrusts in the human/machine-forecasting literature: (i) the choice of the appropriate quantitative model, (ii) the nature of the interaction between quantitative models and human judgement, (iii) the training and incentivization of human forecasters, and (iv) the combination of multiple forecasts (both algorithmic and human) into one. This review surveys current research in all four areas and argues that future research in the field of human/machine forecasting needs to consider all of them when investigating predictive performance. We also address some of the ethical dilemmas that might arise due to the combination of quantitative models with human judgement.},
  keywords = {forecast combination,forecasting,human judgment,quantitative forecasting methods},
  file = {/Users/nikos/github-synced/zotero-nikos/storage/WLF44NUH/Zellner et al. - 2021 - A survey of human judgement and quantitative forec.pdf}
}
