Skip to content

climate_ref.baseline_report.collect #

Read the baselines either side of a base ref and pair up everything that moved.

Collection reads only the repository, both the manifests and the committed bundle they track. Fetching native blobs from the store and building diffs is :mod:~climate_ref.baseline_report.analyse's job.

IMAGE_SUFFIXES = frozenset({'.png', '.jpg', '.jpeg', '.gif', '.svg'}) module-attribute #

Extensions rendered as a two-up image comparison.

NETCDF_SUFFIXES = frozenset({'.nc'}) module-attribute #

Extensions reported as a size delta and a link rather than a content diff.

TEXT_SUFFIXES = frozenset({'.json', '.csv', '.yml', '.yaml', '.html', '.txt', '.md', '.log'}) module-attribute #

Extensions whose blobs are worth fetching and diffing line by line.

CaseChange #

Everything that changed for a single test case.

Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
@frozen
class CaseChange:
    """Everything that changed for a single test case."""

    label: str
    """``provider/diagnostic/test-case``."""

    rel_path: str
    """Repo-relative path of the case's ``manifest.json``."""

    base: Manifest | None
    """The manifest on the base ref, or ``None`` when the case is new."""

    head: Manifest | None
    """The manifest on HEAD, or ``None`` when the case was removed."""

    files: tuple[FileChange, ...]
    """Every native file that moved, in name order."""

    committed: tuple[CommittedChange, ...]
    """Committed artefacts whose digest moved, in name order."""

    metadata: tuple[str, ...]
    """Scalar manifest fields that moved, described one per entry."""

    @property
    def is_new(self) -> bool:
        """Whether the whole test case is new on this branch."""
        return self.base is None

    @property
    def is_removed(self) -> bool:
        """Whether the whole test case was deleted on this branch."""
        return self.head is None

base instance-attribute #

The manifest on the base ref, or None when the case is new.

committed instance-attribute #

Committed artefacts whose digest moved, in name order.

files instance-attribute #

Every native file that moved, in name order.

head instance-attribute #

The manifest on HEAD, or None when the case was removed.

is_new property #

Whether the whole test case is new on this branch.

is_removed property #

Whether the whole test case was deleted on this branch.

label instance-attribute #

provider/diagnostic/test-case.

metadata instance-attribute #

Scalar manifest fields that moved, described one per entry.

rel_path instance-attribute #

Repo-relative path of the case's manifest.json.

CommittedChange #

One committed regression artefact whose digest moved.

Unlike a native output these bytes are tracked in git, so both sides are read from the repository rather than fetched from the store.

Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@frozen
class CommittedChange:
    """One committed regression artefact whose digest moved.

    Unlike a native output these bytes are tracked in git, so both sides are read from the
    repository rather than fetched from the store.
    """

    name: str
    """Path of the artefact relative to the test case's ``regression/`` directory."""

    rel_path: str
    """Repo-relative path of the artefact, which is where it appears in the pull request."""

    old: str | None
    """The digest on the base ref, or ``None`` when the artefact is new."""

    new: str | None
    """The digest on HEAD, or ``None`` when the artefact was removed."""

    old_text: str | None
    """The artefact's content on the base ref, or ``None`` when it could not be read."""

    new_text: str | None
    """The artefact's content on HEAD, or ``None`` when it could not be read."""

    @property
    def status(self) -> str:
        """``added``, ``removed`` or ``changed``."""
        return change_status(self.old, self.new)

name instance-attribute #

Path of the artefact relative to the test case's regression/ directory.

new instance-attribute #

The digest on HEAD, or None when the artefact was removed.

new_text instance-attribute #

The artefact's content on HEAD, or None when it could not be read.

old instance-attribute #

The digest on the base ref, or None when the artefact is new.

old_text instance-attribute #

The artefact's content on the base ref, or None when it could not be read.

rel_path instance-attribute #

Repo-relative path of the artefact, which is where it appears in the pull request.

status property #

added, removed or changed.

FileChange #

One native output file that was added, removed, or changed by the mint.

Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
@frozen
class FileChange:
    """One native output file that was added, removed, or changed by the mint."""

    name: str
    """Path of the file relative to the test case's native output directory."""

    old: NativeEntry | None
    """The manifest entry on the base ref, or ``None`` when the file is new."""

    new: NativeEntry | None
    """The manifest entry on HEAD, or ``None`` when the file was removed."""

    kind: FileKind
    """How the file should be rendered."""

    @property
    def status(self) -> str:
        """``added``, ``removed`` or ``changed``."""
        return change_status(self.old, self.new)

kind instance-attribute #

How the file should be rendered.

name instance-attribute #

Path of the file relative to the test case's native output directory.

new instance-attribute #

The manifest entry on HEAD, or None when the file was removed.

old instance-attribute #

The manifest entry on the base ref, or None when the file is new.

status property #

added, removed or changed.

FileKind #

Bases: Enum

How a native output file should be presented in the report.

Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
36
37
38
39
40
41
42
class FileKind(enum.Enum):
    """How a native output file should be presented in the report."""

    IMAGE = "image"
    TEXT = "text"
    NETCDF = "netcdf"
    OTHER = "other"

Report #

Every test case whose baseline moved on this branch.

Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
182
183
184
185
186
187
188
189
190
191
192
193
@frozen
class Report:
    """Every test case whose baseline moved on this branch."""

    base_ref: str
    """The git ref HEAD was compared against."""

    head_sha: str
    """The full sha of the commit that was compared."""

    cases: tuple[CaseChange, ...]
    """The changed cases, in label order."""

base_ref instance-attribute #

The git ref HEAD was compared against.

cases instance-attribute #

The changed cases, in label order.

head_sha instance-attribute #

The full sha of the commit that was compared.

build_case_change(repo, base, rel_path) #

Collect every change to one test case.

A case deleted on this branch has no head manifest. It is still reported, as a removal, because dropping a test case is a baseline change a reviewer needs to see.

Parameters:

Name Type Description Default
repo Repo

The repository to read from.

 required
base str

The git ref to compare against.

 required
rel_path str

Repo-relative path of the case's manifest.

 required

Returns:

Type Description
CaseChange | None

The case's changes, or None when the manifest is absent from both sides or cannot be parsed, both of which leave nothing to say.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def build_case_change(repo: Repo, base: str, rel_path: str) -> CaseChange | None:
    """
    Collect every change to one test case.

    A case deleted on this branch has no head manifest. It is still reported, as a removal,
    because dropping a test case is a baseline change a reviewer needs to see.

    Parameters
    ----------
    repo
        The repository to read from.
    base
        The git ref to compare against.
    rel_path
        Repo-relative path of the case's manifest.

    Returns
    -------
    :
        The case's changes, or ``None`` when the manifest is absent from both sides
        or cannot be parsed, both of which leave nothing to say.
    """
    head_path = Path(repo.working_tree_dir or ".") / rel_path
    try:
        head = Manifest.load(head_path) if head_path.exists() else None
        base_manifest = load_at_ref(repo, base, rel_path)
    except ValueError as exc:
        logger.warning(f"Skipping {rel_path}: {exc}")
        return None
    if head is None and base_manifest is None:
        return None

    old_native = base_manifest.native if base_manifest else {}
    new_native = head.native if head else {}
    files = []
    for name in sorted(set(old_native) | set(new_native)):
        old, new = old_native.get(name), new_native.get(name)
        if old is not None and new is not None and old.sha256 == new.sha256:
            continue
        files.append(FileChange(name=name, old=old, new=new, kind=classify(name)))

    return CaseChange(
        label=case_label(rel_path),
        rel_path=rel_path,
        base=base_manifest,
        head=head,
        files=tuple(files),
        committed=_committed_changes(repo, base, rel_path, base_manifest, head),
        metadata=_metadata_changes(base_manifest, head),
    )

case_label(rel_path) #

Derive a provider/diagnostic/test-case label from a manifest path.

packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json becomes pmp/annual-cycle/cmip6-ts.

Parameters:

Name Type Description Default
rel_path str

Repo-relative path of the manifest.

 required

Returns:

Type Description
str

The label.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
def case_label(rel_path: str) -> str:
    """
    Derive a ``provider/diagnostic/test-case`` label from a manifest path.

    ``packages/climate-ref-pmp/tests/test-data/annual-cycle/cmip6-ts/manifest.json``
    becomes ``pmp/annual-cycle/cmip6-ts``.

    Parameters
    ----------
    rel_path
        Repo-relative path of the manifest.

    Returns
    -------
    :
        The label.
    """
    parts = Path(rel_path).parts
    provider = parts[1].removeprefix("climate-ref-") if len(parts) > 1 else "?"
    tail = parts[-3:-1] if len(parts) >= _MIN_LABEL_PARTS else ()
    return "/".join((provider, *tail))

change_status(old, new) #

Describe which sides of the change an entry is present on.

Parameters:

Name Type Description Default
old str | NativeEntry | None

The entry on the base ref, or None when it is absent there.

 required
new str | NativeEntry | None

The entry on HEAD, or None when it is absent there.

 required

Returns:

Type Description
str

added, removed or changed.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def change_status(old: str | NativeEntry | None, new: str | NativeEntry | None) -> str:
    """
    Describe which sides of the change an entry is present on.

    Parameters
    ----------
    old
        The entry on the base ref, or ``None`` when it is absent there.
    new
        The entry on HEAD, or ``None`` when it is absent there.

    Returns
    -------
    :
        ``added``, ``removed`` or ``changed``.
    """
    if old is None:
        return "added"
    if new is None:
        return "removed"
    return "changed"

changed_manifests(repo, base) #

Return the repo-relative paths of every test-case manifest that differs from base.

Uses the merge-base (base...HEAD) so commits landing on the base branch after the feature branch forked are not misreported as baseline changes.

Parameters:

Name Type Description Default
repo Repo

The repository to diff in.

 required
base str

The git ref to compare against.

 required

Returns:

Type Description
list[str]

The manifest paths, sorted.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
def changed_manifests(repo: Repo, base: str) -> list[str]:
    """
    Return the repo-relative paths of every test-case manifest that differs from ``base``.

    Uses the merge-base (``base...HEAD``) so commits landing on the base branch after the
    feature branch forked are not misreported as baseline changes.

    Parameters
    ----------
    repo
        The repository to diff in.
    base
        The git ref to compare against.

    Returns
    -------
    :
        The manifest paths, sorted.
    """
    from git import GitCommandError  # noqa: PLC0415

    pathspec = ":(glob)packages/**/test-data/**/manifest.json"
    try:
        out = repo.git.diff("--name-only", f"{base}...HEAD", "--", pathspec)
    except GitCommandError:
        # A shallow clone may have no merge-base with the base ref. A two-dot diff over-reports
        # (it also shows base-branch commits), which is the safe direction for a report.
        out = repo.git.diff("--name-only", base, "HEAD", "--", pathspec)
    return sorted(line for line in out.splitlines() if line.strip())

classify(name) #

Classify a native output file by its extension.

Parameters:

Name Type Description Default
name str

The file's path relative to the test case's native output directory.

 required

Returns:

Type Description
FileKind

The kind that decides how the file is rendered.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def classify(name: str) -> FileKind:
    """
    Classify a native output file by its extension.

    Parameters
    ----------
    name
        The file's path relative to the test case's native output directory.

    Returns
    -------
    :
        The kind that decides how the file is rendered.
    """
    suffix = Path(name).suffix.lower()
    if suffix in IMAGE_SUFFIXES:
        return FileKind.IMAGE
    if suffix in TEXT_SUFFIXES:
        return FileKind.TEXT
    if suffix in NETCDF_SUFFIXES:
        return FileKind.NETCDF
    return FileKind.OTHER

collect(repo, base) #

Collect every baseline change on this branch.

Parameters:

Name Type Description Default
repo Repo

The repository to read from.

 required
base str

The git ref to compare against.

 required

Returns:

Type Description
Report

The report, with cases in label order.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
def collect(repo: Repo, base: str) -> Report:
    """
    Collect every baseline change on this branch.

    Parameters
    ----------
    repo
        The repository to read from.
    base
        The git ref to compare against.

    Returns
    -------
    :
        The report, with cases in label order.
    """
    cases = [
        case
        for rel_path in changed_manifests(repo, base)
        if (case := build_case_change(repo, base, rel_path)) is not None
    ]
    return Report(
        base_ref=base,
        head_sha=repo.head.commit.hexsha,
        cases=tuple(sorted(cases, key=lambda case: case.label)),
    )

load_at_ref(repo, ref, rel_path) #

Load a manifest as it exists at ref, or None when absent there.

Parameters:

Name Type Description Default
repo Repo

The repository to read from.

 required
ref str

The git ref to read the manifest at.

 required
rel_path str

Repo-relative path of the manifest.

 required

Returns:

Type Description
Manifest | None

The parsed manifest, or None when the path does not exist at ref.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def load_at_ref(repo: Repo, ref: str, rel_path: str) -> Manifest | None:
    """
    Load a manifest as it exists at ``ref``, or ``None`` when absent there.

    Parameters
    ----------
    repo
        The repository to read from.
    ref
        The git ref to read the manifest at.
    rel_path
        Repo-relative path of the manifest.

    Returns
    -------
    :
        The parsed manifest, or ``None`` when the path does not exist at ``ref``.
    """
    text = show_at_ref(repo, ref, rel_path)
    if text is None:
        return None
    return Manifest.loads(text, source=f"{ref}:{rel_path}")

show_at_ref(repo, ref, rel_path) #

Read a file's content as it exists at ref, or None when it cannot be read there.

Parameters:

Name Type Description Default
repo Repo

The repository to read from.

 required
ref str

The git ref to read the file at.

 required
rel_path str

Repo-relative path of the file.

 required

Returns:

Type Description
str | None

The file's text, or None when the path does not exist at ref or does not decode as UTF-8.
Source code in packages/climate-ref/src/climate_ref/baseline_report/collect.py 
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
def show_at_ref(repo: Repo, ref: str, rel_path: str) -> str | None:
    """
    Read a file's content as it exists at ``ref``, or ``None`` when it cannot be read there.

    Parameters
    ----------
    repo
        The repository to read from.
    ref
        The git ref to read the file at.
    rel_path
        Repo-relative path of the file.

    Returns
    -------
    :
        The file's text, or ``None`` when the path does not exist at ``ref`` or does not
        decode as UTF-8.
    """
    from git import GitCommandError  # noqa: PLC0415

    try:
        return str(repo.git.show(f"{ref}:{rel_path}"))
    except (GitCommandError, UnicodeDecodeError):
        return None