Skip to content

climate_ref_core.metric_values #

Metric Values

A metric is a single statistical evaluation contained within a diagnostic. A diagnostic may consist of more than one metric.

Examples include bias, root mean squared error (RMSE), Earth Mover's Distance, phase/timing of the seasonal cycle, amplitude of the seasonal cycle, spatial or temporal correlations, interannual variability. Not all metrics are useful for all variables or should be used with every observationally constrained dataset. Each metric may be converted into a performance score.

MetricValueKind = Literal['model', 'reference'] module-attribute #

The role of a metric value: a model value or a reference (observation) value.

ScalarMetricValue #

Bases: BaseModel

A scalar value with an associated dimensions

Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
class ScalarMetricValue(BaseModel):
    """
    A scalar value with an associated dimensions
    """

    dimensions: dict[str, str]
    """
    Key, value pairs that identify the dimensions of the metric

    These values are used for a faceted search of the metric values.
    """
    kind: MetricValueKind = "model"
    """
    Whether the value is a model value or a reference (observation) value.

    This is the first-class signal for the role of the value
    and replaces provider-specific conventions for distinguishing the two.
    """
    value: Value
    """
    A scalar value
    """
    attributes: dict[str, str | Value] | None = None
    """
    Additional unstructured attributes associated with the metric value
    """

attributes = None class-attribute instance-attribute #

Additional unstructured attributes associated with the metric value

dimensions instance-attribute #

Key, value pairs that identify the dimensions of the metric

These values are used for a faceted search of the metric values.

kind = 'model' class-attribute instance-attribute #

Whether the value is a model value or a reference (observation) value.

This is the first-class signal for the role of the value and replaces provider-specific conventions for distinguishing the two.

value instance-attribute #

A scalar value

SeriesMetricValue #

Bases: BaseModel

A 1-d array with an associated index and additional dimensions

These values are typically sourced from the CMEC metrics bundle

Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
class SeriesMetricValue(BaseModel):
    """
    A 1-d array with an associated index and additional dimensions

    These values are typically sourced from the CMEC metrics bundle
    """

    dimensions: dict[str, str]
    """
    Key, value pairs that identify the dimensions of the metric

    These values are used for a faceted search of the metric values.
    """
    kind: MetricValueKind = "model"
    """
    Whether the series is a model value or a reference (observation) value.

    This is the first-class signal for the role of the value
    and replaces provider-specific conventions for distinguishing the two.
    """
    reference_id: str | None = None
    """
    Content hash identifying the reference payload of the series.

    Stable across executions for an identical reference payload,
    so reference series can be deduplicated deterministically.
    It is ``None`` for model series and is populated at ingest for reference series.
    """
    values: Sequence[Value]
    """
    A 1-d array of values
    """
    index: Sequence[str | Value]
    """
    A 1-d array of index values

    Values must be strings or numbers and have the same length as values.
    Non-unique index values are not allowed.
    """

    index_name: str
    """
    The name of the index.

    This is used for presentation purposes and is not used in the controlled vocabulary.
    """

    value_units: str | None = None
    """
    Units of the series values (e.g. ``"K"``).

    Presentation metadata. Optional for now; falls back to ``attributes`` when absent.
    """
    value_long_name: str | None = None
    """
    Human-readable name of the series values (e.g. ``"Near-Surface Air Temperature"``).

    Presentation metadata. Optional for now; falls back to ``attributes`` when absent.
    """
    index_units: str | None = None
    """
    Units of the index (e.g. ``"days since 1850-01-01"``).

    Presentation metadata. Optional for now; falls back to ``attributes`` when absent.
    """
    calendar: str | None = None
    """
    Calendar of a time index (e.g. ``"360_day"``), when the index is temporal.

    Presentation metadata. Optional for now; falls back to ``attributes`` when absent.
    """

    attributes: dict[str, str | Value | None] | None = None
    """
    Additional unstructured attributes associated with the metric value
    """

    @model_validator(mode="after")
    def validate_index(self) -> Self:
        """Validate that index has the same length as values and contains no NaNs"""
        if len(self.index) != len(self.values):
            raise ValueError(
                f"Index length ({len(self.index)}) must match values length ({len(self.values)})"
            )
        for v in self.index:
            if isinstance(v, float) and not np.isfinite(v):
                raise ValueError("NaN or Inf values are not allowed in the index")
        return self

    @field_validator("values", mode="before")
    @classmethod
    def validate_values(cls, value: Any) -> Any:
        """
        Transform None values to NaN in the values field
        """
        if not isinstance(value, (list, tuple)):
            raise ValueError("`values` must be a list or tuple.")

        # Transform None values to NaN
        return [float("nan") if v is None else v for v in value]

    @classmethod
    def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
        """
        Dump a sequence of SeriesMetricValue to a JSON file.

        Parameters
        ----------
        path
            The path to the JSON file.

            The directory containing this file must already exist.
            This file will be overwritten if it already exists.
        series
            The series values to dump.
        """
        # Sort the series before serialising so the order is deterministic across platforms
        # and runs. Diagnostics may emit series in an implementation-defined order (e.g. set or
        # dict iteration that differs by platform), which otherwise produces spurious diffs and
        # breaks the positional regression comparator. ``dimensions`` and ``kind`` together
        # identify a series (``kind`` lives outside ``dimensions``); ``index_name`` is a stable
        # tie-breaker.
        ordered = sorted(
            series,
            key=lambda s: (json.dumps(s.dimensions, sort_keys=True), s.kind, s.index_name),
        )
        with open(path, "w") as f:
            json.dump(
                [s._dump_for_json() for s in ordered],
                f,
                indent=2,
                allow_nan=False,
                sort_keys=True,
            )

    def _dump_for_json(self) -> dict[str, Any]:
        """
        Serialise to a JSON-ready dict, omitting the added fields left at their default.

        See [_OMIT_WHEN_DEFAULT][climate_ref_core.metric_values.typing._OMIT_WHEN_DEFAULT].
        """
        data = self.model_dump(mode="json")
        fields = type(self).model_fields
        for field in _OMIT_WHEN_DEFAULT:
            if data.get(field) == fields[field].default:
                data.pop(field, None)
        return data

    @classmethod
    def load_from_json(
        cls,
        path: Path,
    ) -> list["SeriesMetricValue"]:
        """
        Load a sequence of SeriesMetricValue from a JSON file.

        Parameters
        ----------
        path
            The path to the JSON file.
        """
        with open(path) as f:
            data = json.load(f)

        if not isinstance(data, list):
            raise ValueError(f"Expected a list of series values, got {type(data)}")

        return [cls.model_validate(s, strict=True) for s in data]

attributes = None class-attribute instance-attribute #

Additional unstructured attributes associated with the metric value

calendar = None class-attribute instance-attribute #

Calendar of a time index (e.g. "360_day"), when the index is temporal.

Presentation metadata. Optional for now; falls back to attributes when absent.

dimensions instance-attribute #

Key, value pairs that identify the dimensions of the metric

These values are used for a faceted search of the metric values.

index instance-attribute #

A 1-d array of index values

Values must be strings or numbers and have the same length as values. Non-unique index values are not allowed.

index_name instance-attribute #

The name of the index.

This is used for presentation purposes and is not used in the controlled vocabulary.

index_units = None class-attribute instance-attribute #

Units of the index (e.g. "days since 1850-01-01").

Presentation metadata. Optional for now; falls back to attributes when absent.

kind = 'model' class-attribute instance-attribute #

Whether the series is a model value or a reference (observation) value.

This is the first-class signal for the role of the value and replaces provider-specific conventions for distinguishing the two.

reference_id = None class-attribute instance-attribute #

Content hash identifying the reference payload of the series.

Stable across executions for an identical reference payload, so reference series can be deduplicated deterministically. It is None for model series and is populated at ingest for reference series.

value_long_name = None class-attribute instance-attribute #

Human-readable name of the series values (e.g. "Near-Surface Air Temperature").

Presentation metadata. Optional for now; falls back to attributes when absent.

value_units = None class-attribute instance-attribute #

Units of the series values (e.g. "K").

Presentation metadata. Optional for now; falls back to attributes when absent.

values instance-attribute #

A 1-d array of values

dump_to_json(path, series) classmethod #

Dump a sequence of SeriesMetricValue to a JSON file.

Parameters:

Name Type Description Default
path Path

The path to the JSON file.

The directory containing this file must already exist. This file will be overwritten if it already exists.

 required
series Sequence[SeriesMetricValue]

The series values to dump.

 required
Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
@classmethod
def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
    """
    Dump a sequence of SeriesMetricValue to a JSON file.

    Parameters
    ----------
    path
        The path to the JSON file.

        The directory containing this file must already exist.
        This file will be overwritten if it already exists.
    series
        The series values to dump.
    """
    # Sort the series before serialising so the order is deterministic across platforms
    # and runs. Diagnostics may emit series in an implementation-defined order (e.g. set or
    # dict iteration that differs by platform), which otherwise produces spurious diffs and
    # breaks the positional regression comparator. ``dimensions`` and ``kind`` together
    # identify a series (``kind`` lives outside ``dimensions``); ``index_name`` is a stable
    # tie-breaker.
    ordered = sorted(
        series,
        key=lambda s: (json.dumps(s.dimensions, sort_keys=True), s.kind, s.index_name),
    )
    with open(path, "w") as f:
        json.dump(
            [s._dump_for_json() for s in ordered],
            f,
            indent=2,
            allow_nan=False,
            sort_keys=True,
        )

load_from_json(path) classmethod #

Load a sequence of SeriesMetricValue from a JSON file.

Parameters:

Name Type Description Default
path Path

The path to the JSON file.

 required
Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
@classmethod
def load_from_json(
    cls,
    path: Path,
) -> list["SeriesMetricValue"]:
    """
    Load a sequence of SeriesMetricValue from a JSON file.

    Parameters
    ----------
    path
        The path to the JSON file.
    """
    with open(path) as f:
        data = json.load(f)

    if not isinstance(data, list):
        raise ValueError(f"Expected a list of series values, got {type(data)}")

    return [cls.model_validate(s, strict=True) for s in data]

validate_index() #

Validate that index has the same length as values and contains no NaNs

Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
128
129
130
131
132
133
134
135
136
137
138
@model_validator(mode="after")
def validate_index(self) -> Self:
    """Validate that index has the same length as values and contains no NaNs"""
    if len(self.index) != len(self.values):
        raise ValueError(
            f"Index length ({len(self.index)}) must match values length ({len(self.values)})"
        )
    for v in self.index:
        if isinstance(v, float) and not np.isfinite(v):
            raise ValueError("NaN or Inf values are not allowed in the index")
    return self

validate_values(value) classmethod #

Transform None values to NaN in the values field

Source code in packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py 
140
141
142
143
144
145
146
147
148
149
150
@field_validator("values", mode="before")
@classmethod
def validate_values(cls, value: Any) -> Any:
    """
    Transform None values to NaN in the values field
    """
    if not isinstance(value, (list, tuple)):
        raise ValueError("`values` must be a list or tuple.")

    # Transform None values to NaN
    return [float("nan") if v is None else v for v in value]

sub-packages#

Sub-package Description
typing