\relax 
\bibstyle{jss}
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\citation{timmermannForecastingMethodsFinance2018,elliottForecastingEconomicsFinance2016}
\citation{gneitingWeatherForecastingEnsemble2005,kukkonenReviewOperationalRegionalscale2012}
\citation{reichCollaborativeMultiyearMultimodel2019,funkShorttermForecastsInform2020,cramerEvaluationIndividualEnsemble2021,bracherNationalSubnationalShortterm2022,sherrattPredictivePerformanceMultimodel2022}
\citation{goodRationalDecisions1952}
\citation{epsteinScoringSystemProbability1969,murphyNoteRankedProbability1971a,mathesonScoringRulesContinuous1976}
\citation{gneitingProbabilisticForecastsCalibration2007}
\citation{funkAssessingPerformanceRealtime2019}
\citation{gneitingStrictlyProperScoring2007}
\citation{bracherEvaluatingEpidemicForecasts2021}
\newlabel{introduction}{{1}{1}{}{section.1}{}}
\citation{R}
\citation{tscount}
\citation{topmodels}
\citation{GLMMadaptive}
\citation{cvGEE}
\citation{fabletools}
\citation{scoringRules}
\citation{Metrics}
\citation{MLmetrics}
\citation{verification}
\citation{SpecsVerification}
\citation{surveillance}
\citation{predtools}
\citation{probably}
\citation{scoring}
\citation{yardstick}
\citation{tidymodels}
\citation{gneitingProbabilisticForecastsCalibration2007}
\citation{reichCollaborativeMultiyearMultimodel2019,cramerCOVID19ForecastHub2020,sherrattPredictivePerformanceMultimodel2022,bracherNationalSubnationalShortterm2022}
\citation{bosseScoringEpidemiologicalForecasts2023}
\citation{data.table}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Illustration of the suggested workflow for evaluating forecasts with {\fontseries  {m}\fontseries  {b}\selectfont  scoringutils}. A: Workflow for working with forecasts in a \bgroup \catcode `\_12\relax \catcode `\~12\relax \catcode `\$12\relax {\normalfont  \ttfamily  \hyphenchar \font =-1 data.table}\egroup -based format. The left side shows the core workflow of the package: 1) validating and processing inputs, 2) scoring forecasts and 3) summarising scores. The right side shows additional functionality that is available at the different stages of the evaluation process. The part in blue is covered by Section \ref {sec:inputs} and includes all functions related to processing and validating inputs as well as obtaining additional information about the forecasts. The part in green is covered by Section \ref {sec:scoring} and includes all functions related to scoring forecasts and obtaining additional information about the scores. The part in red is covered by Section \ref {sec:summarising} and includes all functions related to summarising scores and additional visualisations based on summarised scores. B: An alternative workflow, allowing users to call scoring rules directly with vectors/matrices as inputs.}}{3}{figure.caption.1}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{fig:workflow-scoringutils}{{1}{3}{Illustration of the suggested workflow for evaluating forecasts with \pkg {scoringutils}. A: Workflow for working with forecasts in a \code {data.table}-based format. The left side shows the core workflow of the package: 1) validating and processing inputs, 2) scoring forecasts and 3) summarising scores. The right side shows additional functionality that is available at the different stages of the evaluation process. The part in blue is covered by Section \ref {sec:inputs} and includes all functions related to processing and validating inputs as well as obtaining additional information about the forecasts. The part in green is covered by Section \ref {sec:scoring} and includes all functions related to scoring forecasts and obtaining additional information about the scores. The part in red is covered by Section \ref {sec:summarising} and includes all functions related to summarising scores and additional visualisations based on summarised scores. B: An alternative workflow, allowing users to call scoring rules directly with vectors/matrices as inputs}{figure.caption.1}{}}
\newlabel{paper-outline-and-package-workflow}{{1.0.1}{3}{}{subsubsection.1.0.1}{}}
\citation{sherrattPredictivePerformanceMultimodel2022}
\newlabel{sec:inputs}{{2}{4}{}{section.2}{}}
\newlabel{input-formats-and-types-of-forecasts}{{2.1}{4}{}{subsection.2.1}{}}
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Formatting requirements for data inputs. For binary forecasts, the column \texttt  {observed} must be of type factor with two levels and the column \texttt  {predicted} must be a numeric between 0 and 1. For nominal forecasts, the observed value must be a factor with $N$ levels (where $N$ is the number of possible outcomes) and a column \texttt  {predicted\_label} must denote the outcome for which a probability was made. For all other forecast types, both \texttt  {observed} and \texttt  {predicted} must be of type numeric. Forecasts in a sample-based format require an additional numeric column \texttt  {sample\_id} and forecasts in a quantile-based format require an additional numeric column \texttt  {quantile\_level} with values between 0 and 1.}}{5}{table.caption.2}\protected@file@percent }
\newlabel{tab:input-score}{{1}{5}{Formatting requirements for data inputs. For binary forecasts, the column \texttt {observed} must be of type factor with two levels and the column \texttt {predicted} must be a numeric between 0 and 1. For nominal forecasts, the observed value must be a factor with $N$ levels (where $N$ is the number of possible outcomes) and a column \texttt {predicted\_label} must denote the outcome for which a probability was made. For all other forecast types, both \texttt {observed} and \texttt {predicted} must be of type numeric. Forecasts in a sample-based format require an additional numeric column \texttt {sample\_id} and forecasts in a quantile-based format require an additional numeric column \texttt {quantile\_level} with values between 0 and 1}{table.caption.2}{}}
\newlabel{the-unit-of-a-single-forecast}{{2.1.1}{6}{}{subsubsection.2.1.1}{}}
\newlabel{sec:validation}{{2.2}{6}{}{subsection.2.2}{}}
\newlabel{diagnostic-helper-functions}{{2.3}{7}{}{subsection.2.3}{}}
\citation{bosseScoringEpidemiologicalForecasts2023}
\newlabel{transforming-forecasts}{{2.4}{9}{}{subsection.2.4}{}}
\citation{gneitingProbabilisticForecastsCalibration2007}
\newlabel{additional-functionality-related-to-forecast-objects}{{2.5}{10}{}{subsection.2.5}{}}
\newlabel{displaying-the-number-of-forecasts-available}{{2.5.1}{10}{}{subsubsection.2.5.1}{}}
\newlabel{probabilistic-calibration-and-pit-histograms}{{2.5.2}{10}{}{subsubsection.2.5.2}{}}
\citation{dawidPresentPositionPotential1984}
\citation{angusProbabilityIntegralTransform1994}
\citation{czadoPredictiveModelAssessment2009}
\citation{funkAssessingPerformanceRealtime2019}
\citation{funkAssessingPerformanceRealtime2019}
\citation{gneitingProbabilisticForecastsCalibration2007,hamillInterpretationRankHistograms2001a}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Visualistion of forecast counts for the example data}}{11}{figure.caption.3}\protected@file@percent }
\newlabel{fig:plot-forecast-counts}{{2}{11}{Visualistion of forecast counts for the example data}{figure.caption.3}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces PIT histograms of all models stratified by forecast target}}{12}{figure.caption.4}\protected@file@percent }
\newlabel{fig:pit-plots}{{3}{12}{PIT histograms of all models stratified by forecast target}{figure.caption.4}{}}
\newlabel{probabilistic-calibration-and-coverage-plots}{{2.5.3}{12}{}{subsubsection.2.5.3}{}}
\newlabel{sec:scoring}{{3}{13}{}{section.3}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Interval coverage (A) and quantile coverage (B) plots}}{14}{figure.caption.5}\protected@file@percent }
\newlabel{fig:coverage}{{4}{14}{Interval coverage (A) and quantile coverage (B) plots}{figure.caption.5}{}}
\newlabel{score-and-working-with-scoring-rules}{{3.1}{14}{}{subsection.3.1}{}}
\newlabel{composing-a-custom-list-of-metrics-and-scoring-rules}{{3.1.1}{15}{}{subsubsection.3.1.1}{}}
\citation{cramerEvaluationIndividualEnsemble2021}
\newlabel{pairwisetheory}{{3.2}{16}{}{subsection.3.2}{}}
\citation{mannTestWhetherOne1947}
\newlabel{additional-functionality-related-to-scores-objects}{{3.3}{17}{}{subsection.3.3}{}}
\newlabel{displaying-mean-score-ratios-from-pairwise-comparisons}{{3.3.1}{17}{}{subsubsection.3.3.1}{}}
\newlabel{correlations-between-scores}{{3.3.2}{17}{}{subsubsection.3.3.2}{}}
\citation{gneitingStrictlyProperScoring2007}
\newlabel{sec:summarising}{{4}{18}{}{section.4}{}}
\newlabel{summarising-scores}{{4.1}{18}{}{subsection.4.1}{}}
\citation{bracherEvaluatingEpidemicForecasts2021}
\newlabel{additional-functionality-for-summarised-scores}{{4.2}{19}{}{subsection.4.2}{}}
\newlabel{heatmaps}{{4.2.1}{19}{}{subsubsection.4.2.1}{}}
\newlabel{weighted-interval-score-decomposition}{{4.2.2}{19}{}{subsubsection.4.2.2}{}}
\newlabel{sec:discussion}{{5}{20}{}{section.5}{}}
\newlabel{summary}{{5.0.1}{20}{}{subsubsection.5.0.1}{}}
\bibdata{references.bib,scoringutils-paper.bib}
\bibcite{angusProbabilityIntegralTransform1994}{{1}{1994}{{Angus}}{{}}}
\bibcite{bosseScoringEpidemiologicalForecasts2023}{{2}{2023}{{Bosse \emph  {et~al.}}}{{Bosse, Abbott, Cori, van Leeuwen, Bracher, and Funk}}}
\newlabel{acknowledgments}{{6}{21}{}{section.6}{}}
\bibcite{bracherEvaluatingEpidemicForecasts2021}{{3}{2021}{{Bracher \emph  {et~al.}}}{{Bracher, Ray, Gneiting, and Reich}}}
\bibcite{bracherNationalSubnationalShortterm2022}{{4}{2022}{{Bracher \emph  {et~al.}}}{{Bracher, Wolffram, Deuschel, G{\"o}rgen, Ketterer, Ullrich, Abbott, Barbarossa, Bertsimas, Bhatia, Bodych, Bosse, Burgard, Castro, Fairchild, Fiedler, Fuhrmann, Funk, Gambin, Gogolewski, Heyder, Hotz, Kheifetz, Kirsten, Krueger, Krymova, Leith{\"a}user, Li, Meinke, Miasojedow, Michaud, Mohring, Nouvellet, Nowosielski, Ozanski, Radwan, Rakowski, Scholz, Soni, Srivastava, Gneiting, and Schienle}}}
\bibcite{cramerEvaluationIndividualEnsemble2021}{{5}{2021}{{Cramer \emph  {et~al.}}}{{Cramer, Ray, Lopez, Bracher, Brennen, Rivadeneira, Gerding, Gneiting, House, Huang, Jayawardena, Kanji, Khandelwal, Le, M{\"u}hlemann, Niemi, Shah, Stark, Wang, Wattanachit, Zorn, Gu, Jain, Bannur, Deva, Kulkarni, Merugu, Raval, Shingi, Tiwari, White, Woody, Dahan, Fox, Gaither, Lachmann, Meyers, Scott, Tec, Srivastava, George, Cegan, Dettwiller, England, Farthing, Hunter, Lafferty, Linkov, Mayo, Parno, Rowland, Trump, Corsetti, Baer, Eisenberg, Falb, Huang, Martin, McCauley, Myers, Schwarz, Sheldon, Gibson, Yu, Gao, Ma, Wu, Yan, Jin, Wang, Chen, Guo, Zhao, Gu, Chen, Wang, Xu, Zhang, Zou, Biegel, Lega, Snyder, Wilson, McConnell, Walraven, Shi, Ban, Hong, Kong, Turtle, {Ben-Nun}, Riley, Riley, Koyluoglu, DesRoches, Hamory, Kyriakides, Leis, Milliken, Moloney, Morgan, Ozcan, Schrader, Shakhnovich, Siegel, Spatz, Stiefeling, Wilkinson, Wong, Gao, Bian, Cao, Ferres, Li, Liu, Xie, Zhang, Zheng, Vespignani, Chinazzi, Davis, Mu, y~Piontti, Xiong, Zheng, Baek, Farias, Georgescu, Levi, Sinha, Wilde, Penna, Celi, Sundar, Cavany, Espa{\~n}a, Moore, Oidtman, Perkins, Osthus, Castro, Fairchild, Michaud, Karlen, Lee, Dent, Grantz, Kaminsky, Kaminsky, Keegan, Lauer, Lemaitre, Lessler, Meredith, {Perez-Saez}, Shah, Smith, Truelove, Wills, Kinsey, Obrecht, Tallaksen, Burant, Wang, Gao, Gu, Kim, Li, Wang, Wang, Yu, Reiner, Barber, Gaikedu, Hay, Lim, Murray, Pigott, Prakash, Adhikari, Cui, Rodr{\'i}guez, Tabassum, Xie, Keskinocak, Asplund, Baxter, Oruc, Serban, Arik, Dusenberry, Epshteyn, Kanal, Le, Li, Pfister, Sava, Sinha, Tsai, Yoder, Yoon, Zhang, Abbott, Bosse, Funk, Hellewel, Meakin, Munday, Sherratt, Zhou, Kalantari, Yamana, Pei, Shaman, Ayer, Adee, Chhatwal, Dalgic, Ladd, Linas, Mueller, Xiao, Li, Bertsimas, Lami, Soni, Bouardi, Wang, Wang, Xie, Zeng, Green, Bien, Hu, Jahja, Narasimhan, Rajanala, Rumack, Simon, Tibshirani, Tibshirani, Ventura, Wasserman, O'Dea, Drake, Pagano, Walker, Slayton, Johansson, Biggerstaff, and Reich}}}
\bibcite{cramerCOVID19ForecastHub2020}{{6}{2020}{{Cramer \emph  {et~al.}}}{{Cramer, Reich, Wang, Niemi, Hannan, House, Gu, Xie, Horstman, {aniruddhadiga}, Walraven, {starkari}, Li, Gibson, Castro, Karlen, Wattanachit, {jinghuichen}, {zyt9lsb}, {aagarwal1996}, Woody, Ray, Xu, Biegel, GuidoEspana, X, Bracher, Lee, {har96}, and {leyouz}}}}
\bibcite{czadoPredictiveModelAssessment2009}{{7}{2009}{{Czado \emph  {et~al.}}}{{Czado, Gneiting, and Held}}}
\bibcite{dawidPresentPositionPotential1984}{{8}{1984}{{Dawid}}{{}}}
\bibcite{data.table}{{9}{2023}{{Dowle and Srinivasan}}{{}}}
\bibcite{elliottForecastingEconomicsFinance2016}{{10}{2016}{{Elliott and Timmermann}}{{}}}
\bibcite{epsteinScoringSystemProbability1969}{{11}{1969}{{Epstein}}{{}}}
\bibcite{funkShorttermForecastsInform2020}{{12}{2020}{{Funk \emph  {et~al.}}}{{Funk, Abbott, Atkins, Baguelin, Baillie, Birrell, Blake, Bosse, Burton, Carruthers, Davies, Angelis, Dyson, Edmunds, Eggo, Ferguson, Gaythorpe, Gorsich, {Guyver-Fletcher}, Hellewell, Hill, Holmes, House, Jewell, Jit, Jombart, Joshi, Keeling, Kendall, Knock, Kucharski, Lythgoe, Meakin, Munday, Openshaw, Overton, Pagani, Pearson, {Perez-Guzman}, Pellis, Scarabel, Semple, Sherratt, Tang, Tildesley, {van Leeuwen}, Whittles, Group, Team, and Investigators}}}
\bibcite{funkAssessingPerformanceRealtime2019}{{13}{2019}{{Funk \emph  {et~al.}}}{{Funk, Camacho, Kucharski, Lowe, Eggo, and Edmunds}}}
\bibcite{gneitingProbabilisticForecastsCalibration2007}{{14}{2007}{{Gneiting \emph  {et~al.}}}{{Gneiting, Balabdaoui, and Raftery}}}
\bibcite{gneitingWeatherForecastingEnsemble2005}{{15}{2005}{{Gneiting and Raftery}}{{}}}
\bibcite{gneitingStrictlyProperScoring2007}{{16}{2007}{{Gneiting and Raftery}}{{}}}
\bibcite{goodRationalDecisions1952}{{17}{1952}{{Good}}{{}}}
\bibcite{hamillInterpretationRankHistograms2001a}{{18}{2001}{{Hamill}}{{}}}
\bibcite{Metrics}{{19}{2018}{{Hamner and Frasco}}{{}}}
\bibcite{scoringRules}{{20}{2019}{{Jordan \emph  {et~al.}}}{{Jordan, Kr\"uger, and Lerch}}}
\bibcite{yardstick}{{21}{2023{a}}{{Kuhn \emph  {et~al.}}}{{Kuhn, Vaughan, and Hvitfeldt}}}
\bibcite{probably}{{22}{2023{b}}{{Kuhn \emph  {et~al.}}}{{Kuhn, Vaughan, and Ruiz}}}
\bibcite{tidymodels}{{23}{2020}{{Kuhn and Wickham}}{{}}}
\bibcite{kukkonenReviewOperationalRegionalscale2012}{{24}{2012}{{Kukkonen \emph  {et~al.}}}{{Kukkonen, Olsson, Schultz, Baklanov, Klein, Miranda, Monteiro, Hirtl, Tarvainen, Boy, Peuch, Poupkou, Kioutsioukis, Finardi, Sofiev, Sokhi, Lehtinen, Karatzas, San~Jos{\'e}, Astitha, Kallos, Schaap, Reimer, Jakobs, and Eben}}}
\bibcite{verification}{{25}{2015}{{Laboratory}}{{}}}
\bibcite{tscount}{{26}{2017}{{Liboschik \emph  {et~al.}}}{{Liboschik, Fokianos, and Fried}}}
\bibcite{mannTestWhetherOne1947}{{27}{1947}{{Mann and Whitney}}{{}}}
\bibcite{mathesonScoringRulesContinuous1976}{{28}{1976}{{Matheson and Winkler}}{{}}}
\bibcite{scoring}{{29}{2013}{{Merkle and Steyvers}}{{}}}
\bibcite{surveillance}{{30}{2017}{{Meyer \emph  {et~al.}}}{{Meyer, Held, and Höhle}}}
\bibcite{murphyNoteRankedProbability1971a}{{31}{1971}{{Murphy}}{{}}}
\bibcite{fabletools}{{32}{2023}{{O'Hara-Wild \emph  {et~al.}}}{{O'Hara-Wild, Hyndman, and Wang}}}
\bibcite{R}{{33}{2021}{{R Core Team}}{{}}}
\bibcite{reichCollaborativeMultiyearMultimodel2019}{{34}{2019}{{Reich \emph  {et~al.}}}{{Reich, Brooks, Fox, Kandula, McGowan, Moore, Osthus, Ray, Tushar, Yamana, Biggerstaff, Johansson, Rosenfeld, and Shaman}}}
\bibcite{cvGEE}{{35}{2019}{{Rizopoulos}}{{}}}
\bibcite{GLMMadaptive}{{36}{2023}{{Rizopoulos}}{{}}}
\bibcite{predtools}{{37}{2023}{{Sadatsafavi \emph  {et~al.}}}{{Sadatsafavi, Safari, and Lee}}}
\bibcite{sherrattPredictivePerformanceMultimodel2022}{{38}{2022}{{Sherratt \emph  {et~al.}}}{{Sherratt, Gruson, Grah, Johnson, Niehus, Prasse, Sandman, Deuschel, Wolffram, Abbott, Ullrich, Gibson, Ray, Reich, Sheldon, Wang, Wattanachit, Wang, Trnka, Obozinski, Sun, Thanou, Pottier, Krymova, Barbarossa, Leith{\"a}user, Mohring, Schneider, Wlazlo, Fuhrmann, Lange, Rodiah, Baccam, Gurung, Stage, Suchoski, Budzinski, Walraven, Villanueva, Tucek, {\v S}m{\'i}d, Zaj{\'i}cek, P{\'e}rez, Reina, Bosse, Meakin, Di~Loro, Maruotti, Eclerov{\'a}, Kraus, Kraus, Pribylova, Dimitris, Li, Saksham, Dehning, Mohr, Priesemann, Redlarski, Bejar, Ardenghi, Parolini, Ziarelli, Bock, Heyder, Hotz, E., {Guzman-Merino}, Aznarte, Mori{\~n}a, Alonso, {\'A}lvarez, L{\'o}pez, Prats, Burgard, Rodloff, Zimmermann, Kuhlmann, Zibert, Pennoni, Divino, Catal{\`a}, Lovison, Giudici, Tarantino, Bartolucci, Jona, Mingione, Farcomeni, Srivastava, {Montero-Manso}, Adiga, Hurt, Lewis, Marathe, Porebski, Venkatramanan, Bartczuk, Dreger, Gambin, Gogolewski, {Gruziel-Slomka}, Krupa, Moszynski, Niedzielewski, Nowosielski, Radwan, Rakowski, Semeniuk, Szczurek, Zielinski, Kisielewski, Pabjan, Holger, Kheifetz, Scholz, Bodych, Filinski, Idzikowski, Krueger, Ozanski, Bracher, and Funk}}}
\bibcite{SpecsVerification}{{39}{2020}{{Siegert}}{{}}}
\bibcite{timmermannForecastingMethodsFinance2018}{{40}{2018}{{Timmermann}}{{}}}
\bibcite{MLmetrics}{{41}{2016}{{Yan}}{{}}}
\bibcite{topmodels}{{42}{2022}{{Zeileis and Lang}}{{}}}
\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Overview of the inputs and outputs of the metrics and scoring rules exported by {\fontseries  {m}\fontseries  {b}\selectfont  scoringutils}. Dots indicate scalar values, while bars indicate vectors (comprised of values that belong together). Several bars (vectors) can be grouped into a matrix with rows representing the individual forecasts. All scoring functions used within \bgroup \catcode `\_12\relax \catcode `\~12\relax \catcode `\$12\relax {\normalfont  \ttfamily  \hyphenchar \font =-1 score()}\egroup  must accept the same input formats as the functions here. However, functions used within \bgroup \catcode `\_12\relax \catcode `\~12\relax \catcode `\$12\relax {\normalfont  \ttfamily  \hyphenchar \font =-1 score()}\egroup  do not necessarily have to have the same argument names (see Section \ref {sec:scoring}). Input formats directly correspond to the required columns for the different forecast types (see Table \ref {tab:input-score}). The only exception is the forecast type 'sample': Inputs require a column \bgroup \catcode `\_12\relax \catcode `\~12\relax \catcode `\$12\relax {\normalfont  \ttfamily  \hyphenchar \font =-1 sample\_id}\egroup  in \bgroup \catcode `\_12\relax \catcode `\~12\relax \catcode `\$12\relax {\normalfont  \ttfamily  \hyphenchar \font =-1 score()}\egroup , but no corresponding argument is necessary when calling scoring rules directly on vectors or matrices.}}{27}{figure.caption.6}\protected@file@percent }
\newlabel{fig:input-scoring-rules}{{5}{27}{Overview of the inputs and outputs of the metrics and scoring rules exported by \pkg {scoringutils}. Dots indicate scalar values, while bars indicate vectors (comprised of values that belong together). Several bars (vectors) can be grouped into a matrix with rows representing the individual forecasts. All scoring functions used within \fct {score} must accept the same input formats as the functions here. However, functions used within \fct {score} do not necessarily have to have the same argument names (see Section \ref {sec:scoring}). Input formats directly correspond to the required columns for the different forecast types (see Table \ref {tab:input-score}). The only exception is the forecast type 'sample': Inputs require a column \code {sample\_id} in \fct {score}, but no corresponding argument is necessary when calling scoring rules directly on vectors or matrices}{figure.caption.6}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces Illustration of the computation of relative skill scores through pairwise comparisons of three different forecast models, M1-M3}}{28}{figure.caption.7}\protected@file@percent }
\newlabel{fig:pairwise-comparison}{{6}{28}{Illustration of the computation of relative skill scores through pairwise comparisons of three different forecast models, M1-M3}{figure.caption.7}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces Ratios of mean weighted interval scores based on overlapping forecast sets}}{28}{figure.caption.8}\protected@file@percent }
\newlabel{fig:pairwise-plot}{{7}{28}{Ratios of mean weighted interval scores based on overlapping forecast sets}{figure.caption.8}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces Plot of correlations between different scores}}{29}{figure.caption.9}\protected@file@percent }
\newlabel{fig:correlation-plot}{{8}{29}{Plot of correlations between different scores}{figure.caption.9}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {9}{\ignorespaces Heatmap of bias values for different models across different locations and forecast targets}}{29}{figure.caption.10}\protected@file@percent }
\newlabel{fig:score-heatmap}{{9}{29}{Heatmap of bias values for different models across different locations and forecast targets}{figure.caption.10}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {10}{\ignorespaces Decomposition of the weighted interval score (WIS) into dispersion, overprediction and underprediction}}{30}{figure.caption.11}\protected@file@percent }
\newlabel{fig:wis-components}{{10}{30}{Decomposition of the weighted interval score (WIS) into dispersion, overprediction and underprediction}{figure.caption.11}{}}
\@writefile{lof}{\contentsline {figure}{\numberline {B.11}{\ignorespaces A}}{33}{figure.caption.12}\protected@file@percent }
\newlabel{fig:calibration-plots}{{B.11}{33}{A}{figure.caption.12}{}}
\newlabel{sec:wis}{{C}{34}{}{appendix.C}{}}
\gdef \@abspage@last{36}
