climate_ref.datasets.esmvaltool_reference

Adapter for ESMValTool reference (observational/reanalysis) datasets.

ESMValTool reference data is not CMOR/obs4MIPs compliant, so metadata cannot be read from global attributes the way :mod:climate_ref.datasets.obs4mips does. Instead it is parsed from the ESMValCore DRS path and filename templates that ESMValTool itself uses to locate the data at run time (see climate_ref_esmvaltool.diagnostics.base):

• OBS / OBS6 (metadata from the filename): OBS/Tier{tier}/{dataset}/{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{timerange}.nc
• native6 (metadata from the directory, raw non-CMOR filename): native6/Tier{tier}/{dataset}/{version}/{frequency}/{short_name}/*.nc
• obs4MIPs: obs4MIPs/{dataset}/{version}/{short_name}_*.nc

ESMValToolReferenceDatasetAdapter

Bases: DatasetAdapter

Adapter for ESMValTool reference datasets.

See the module docstring for the layout conventions this adapter understands.

Source code in packages/climate-ref/src/climate_ref/datasets/esmvaltool_reference.py

class ESMValToolReferenceDatasetAdapter(DatasetAdapter):
    """
    Adapter for ESMValTool reference datasets.

    See the module docstring for the layout conventions this adapter understands.
    """

    dataset_cls: type[Dataset] = ESMValToolReferenceDataset
    slug_column = "instance_id"

    dataset_specific_metadata = (
        "project",
        "source_id",
        "variable_id",
        "frequency",
        "version",
        "data_type",
        "tier",
        "long_name",
        "units",
        "finalised",
        slug_column,
    )

    file_specific_metadata = ("start_time", "end_time", "path")
    version_metadata = "version"
    dataset_id_metadata = (
        "project",
        "source_id",
        "frequency",
        "variable_id",
    )

    def __init__(self, n_jobs: int = 1):
        self.n_jobs = n_jobs

    def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
        """
        Generate a data catalog from the specified file or directory.

        Each dataset may contain multiple files (rows). The unique dataset identifier is
        the ``instance_id`` slug in :attr:`slug_column`.
        """
        datasets = build_catalog(
            paths=[str(file_or_directory)],
            parsing_func=parse_esmvaltool_reference,
            include_patterns=["*.nc"],
            depth=10,
            n_jobs=self.n_jobs,
        )
        if datasets.empty:
            logger.error("No datasets found")
            raise ValueError("No ESMValTool reference datasets found")

        datasets["start_time"] = parse_cftime_dates(datasets["start_time"])
        datasets["end_time"] = parse_cftime_dates(datasets["end_time"])
        datasets["finalised"] = True
        return build_instance_id(datasets, list(_INSTANCE_ID_FACETS), prefix=_SLUG_PREFIX)

find_local_datasets(file_or_directory)

Generate a data catalog from the specified file or directory.

Each dataset may contain multiple files (rows). The unique dataset identifier is the instance_id slug in :attr:slug_column.

Source code in packages/climate-ref/src/climate_ref/datasets/esmvaltool_reference.py

def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
    """
    Generate a data catalog from the specified file or directory.

    Each dataset may contain multiple files (rows). The unique dataset identifier is
    the ``instance_id`` slug in :attr:`slug_column`.
    """
    datasets = build_catalog(
        paths=[str(file_or_directory)],
        parsing_func=parse_esmvaltool_reference,
        include_patterns=["*.nc"],
        depth=10,
        n_jobs=self.n_jobs,
    )
    if datasets.empty:
        logger.error("No datasets found")
        raise ValueError("No ESMValTool reference datasets found")

    datasets["start_time"] = parse_cftime_dates(datasets["start_time"])
    datasets["end_time"] = parse_cftime_dates(datasets["end_time"])
    datasets["finalised"] = True
    return build_instance_id(datasets, list(_INSTANCE_ID_FACETS), prefix=_SLUG_PREFIX)

parse_esmvaltool_reference(file, **kwargs)

Parse a single ESMValTool reference file into a metadata record.

Dispatches on the top-level project directory (OBS/native6/obs4MIPs) and reads metadata from the path/filename rather than the file contents, because the data is not CMOR compliant.

Source code in packages/climate-ref/src/climate_ref/datasets/esmvaltool_reference.py

def parse_esmvaltool_reference(file: str, **kwargs: Any) -> dict[str, Any]:
    """
    Parse a single ESMValTool reference file into a metadata record.

    Dispatches on the top-level project directory (``OBS``/``native6``/``obs4MIPs``) and
    reads metadata from the path/filename rather than the file contents, because the data
    is not CMOR compliant.
    """
    try:
        path = Path(file)
        parts = path.parts

        anchor_idx = next((i for i, part in enumerate(parts) if part in _PROJECT_ANCHORS), None)
        if anchor_idx is None:
            raise ValueError(
                f"{file} is not under a known ESMValTool reference project ({', '.join(_PROJECT_ANCHORS)})"
            )
        rel = parts[anchor_idx:]
        anchor = rel[0]

        if anchor == "OBS":
            info = _parse_obs(rel, path.name)
        elif anchor == "native6":
            info = _parse_native6(rel)
        else:
            info = _parse_obs4mips(rel, path.name)

        info["path"] = str(file)
        info["long_name"] = None
        info["units"] = None
        return info
    except (ValueError, IndexError) as err:
        logger.warning(str(err))
        return {"INVALID_ASSET": file, "TRACEBACK": str(err)}
    except Exception:
        logger.warning(traceback.format_exc())
        return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}