Skip to content

climate_ref.cli.test_cases.diff #

ref test-cases diff.

Renders an HTML report of every regression baseline that moved on this branch. Images are shown old and new side by side and text outputs diffed inline.

diff_baselines(ctx, html_dir, base='origin/main', no_fetch=False, upload=None, comment_output=None) #

Render an HTML report of the regression baselines changed on this branch.

Compares every committed manifest.json to its counterpart on --base and writes one page per changed test case.

Exits 0 whether or not anything changed.

Examples:

ref test-cases diff --html-dir build/baseline-diff
ref test-cases diff --base origin/develop --html-dir build/baseline-diff
ref test-cases diff --html-dir build/baseline-diff --no-fetch
ref test-cases diff --html-dir build/baseline-diff --upload 912/0c7e1d4abc12
Source code in packages/climate-ref/src/climate_ref/cli/test_cases/diff.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
@app.command(name="diff")
def diff_baselines(  # noqa: PLR0913
    ctx: typer.Context,
    html_dir: Annotated[Path, typer.Option(help="Directory to write the HTML report into")],
    base: Annotated[
        str,
        typer.Option(help="Git ref to compare against (the PR base branch)"),
    ] = "origin/main",
    no_fetch: Annotated[
        bool,
        typer.Option("--no-fetch", help="Skip blob downloads and report sizes only"),
    ] = False,
    upload: Annotated[
        str | None,
        typer.Option(help="Upload the report under this key prefix, e.g. 912/0c7e1d4abc12"),
    ] = None,
    comment_output: Annotated[
        Path | None,
        typer.Option(help="Write the pull request comment markdown here"),
    ] = None,
) -> None:
    """
    Render an HTML report of the regression baselines changed on this branch.

    Compares every committed ``manifest.json`` to its counterpart on ``--base``
    and writes one page per changed test case.

    Exits 0 whether or not anything changed.

    Examples
    --------
        ref test-cases diff --html-dir build/baseline-diff
        ref test-cases diff --base origin/develop --html-dir build/baseline-diff
        ref test-cases diff --html-dir build/baseline-diff --no-fetch
        ref test-cases diff --html-dir build/baseline-diff --upload 912/0c7e1d4abc12
    """
    from climate_ref.baseline_report.analyse import analyse
    from climate_ref.baseline_report.collect import collect
    from climate_ref.baseline_report.render import render_comment, write_site
    from climate_ref.baseline_report.upload import upload_site
    from climate_ref_core.regression.report_store import build_report_store
    from climate_ref_core.regression.store import NativeStoreUnavailableError, build_native_store

    config: Config = ctx.obj.config
    console: Console = ctx.obj.console

    repo = get_repo_for_path(Path.cwd())
    if repo is None:
        logger.error("test-cases diff must be run inside a git repository")
        raise typer.Exit(code=1)

    report = collect(repo, base)
    store = build_native_store(config.native_store, writable=False)
    with tempfile.TemporaryDirectory() as workdir:
        analysed = analyse(report, store, fetch=not no_fetch, workdir=Path(workdir))
        index = write_site(analysed, html_dir)

    console.print(f"Wrote {len(analysed.cases)} case page(s) to {index}")

    base_url = None
    if upload is not None:
        try:
            report_store = build_report_store(config.report_store, writable=True)
            base_url = report_store.url_for(upload)
        except ValueError as exc:
            logger.error(
                f"{exc} Set --upload to a relative key prefix such as 912/0c7e1d4abc12, and check "
                "REF_REPORT_STORE_URL, REF_REPORT_STORE_S3_ENDPOINT_URL and REF_REPORT_STORE_BUCKET."
            )
            raise typer.Exit(code=1) from exc
        try:
            report_store.preflight()
            console.print(f"Uploaded the report to {upload_site(html_dir, report_store, upload)}")
        except (ImportError, ValueError, NativeStoreUnavailableError) as exc:
            logger.error(
                f"Could not upload the report: {exc} Check REF_REPORT_STORE_ACCESS_KEY_ID / "
                "REF_REPORT_STORE_SECRET_ACCESS_KEY and the 'climate-ref-core[aws]' extra."
            )
            raise typer.Exit(code=1) from exc

    if comment_output is not None:
        comment_output.parent.mkdir(parents=True, exist_ok=True)
        markdown = render_comment(analysed, base_url or index.parent.resolve().as_uri())
        comment_output.write_text(markdown, encoding="utf-8")
        console.print(f"Wrote the pull request comment to {comment_output}")