From 610066f015ead4affd74e6072c1d99ec9c0f62b7 Mon Sep 17 00:00:00 2001 From: MrlixiangWE <102979255+MrlixiangWE@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:51:44 +0800 Subject: [PATCH 1/2] Define the public API in otava/__init__.py otava had no __init__.py, so `import otava` gave an empty namespace package and the types a library user needs were spread over three submodules. Re-export them from the package root, and document the surface in docs/API.md. Closes #101 --- docs/API.md | 102 ++++++++++++++++++++ docs/README.md | 1 + otava/__init__.py | 61 ++++++++++++ tests/core_install_smoke.py | 19 ++++ tests/public_api_test.py | 184 ++++++++++++++++++++++++++++++++++++ 5 files changed, 367 insertions(+) create mode 100644 docs/API.md create mode 100644 otava/__init__.py create mode 100644 tests/public_api_test.py diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 00000000..18b02c86 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,102 @@ + + +# Using Otava as a library + +Otava can be used without its command line interface and without any of its +importers: build a `Series` from data you already have, analyze it, and read the +change points back. + +```python +from otava import AnalysisOptions, Metric, Series + +# One measurement per commit. Metric.direction says which way is an improvement: +# 1 when higher is better, -1 when lower is better. +series = Series( + test_name="throughput", + branch=None, + time=list(range(60)), + metrics={"ops": Metric(direction=1, scale=1.0)}, + data={"ops": [100.0] * 30 + [80.0] * 30}, + attributes={"commit": [f"sha{i}" for i in range(60)]}, +) + +analyzed = series.analyze(AnalysisOptions(window_len=20, max_pvalue=0.001)) + +for group in analyzed.change_points_by_time: + for metric, change in group.changes.items(): + print( + group.attributes["commit"], + metric, + change.stats.mean_before(), + change.stats.mean_after(), + f"{change.stats.forward_change_percent():+.1f}%", + ) +``` + +`analyze()` does not interpret `time`, so a plain index works for data that is +not a time series, but keep it increasing: change points are collected in time +order, and a second group that is not later than the first raises `ValueError`. +Otava's own reports do interpret it, formatting it with +`datetime.fromtimestamp`, so anything printed by Otava reads it as seconds since +the epoch. + +`attributes` carries one value per data point, and the values at a change point +are returned on its group. That is how you get from a detected change back to +the commit that caused it. + +`Series.analyze()` is lazy: the change points are computed the first time any of +the change point properties on `AnalyzedSeries` is read, and cached from then on. +Errors the analysis raises therefore surface at that first read. + +## What is public + +The names in `otava.__all__`, re-exported from the package root: + +| Name | Purpose | +| --- | --- | +| `Series` | Input data: timestamps, metrics, values, per-point attributes | +| `Metric` | Direction, scale and unit of one metric | +| `AnalysisOptions` | `window_len`, `max_pvalue`, `min_magnitude`, `orig_edivisive` | +| `AnalyzedSeries` | Result of `Series.analyze()` | +| `ChangePoint` | One change in one metric; the statistics are on its `stats` | +| `ChangePointGroup` | Changes that share a point in time | +| `ChangePointsByMetric` | Change points keyed by metric | +| `ChangePointsByTime` | Change point groups in time order | +| `compute_change_points` | The detection algorithm on a bare sequence of floats; returns `(strong, weak)` | + +Everything else is internal, including every submodule path such as +`otava.importer`, `otava.config` and `otava.main`. Otava does not offer a stable +public API yet, so even the names above can still change in a minor or patch +release. + +## Options + +`AnalysisOptions` holds the settings the `analyze` command exposes as flags, +with the same defaults. The attribute names and the flag names differ; run +`otava analyze --help` for the flags. + +| Option | Default | Effect | +| --- | --- | --- | +| `window_len` | 50 | Number of points the algorithm looks at around a candidate | +| `max_pvalue` | 0.001 | Significance threshold a change point must meet | +| `min_magnitude` | 0.0 | Smallest relative change worth reporting | +| `orig_edivisive` | False | Use unmodified e-divisive instead of the windowed variant | + +See [Change Point Detection](MATH.md) for what these do to the results. diff --git a/docs/README.md b/docs/README.md index 4046d934..7003c93c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,6 +39,7 @@ POSTGRESQL BIG_QUERY INFLUXDB GRAFANA +API ``` ```{toctree} diff --git a/otava/__init__.py b/otava/__init__.py new file mode 100644 index 00000000..db3a472a --- /dev/null +++ b/otava/__init__.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Change detection for continuous performance engineering. + +The names re-exported here are the public API of Otava as a library: the types +needed to build a Series from data Otava did not import itself, analyze it, and +read the change points back. + + >>> from otava import AnalysisOptions, Metric, Series + >>> series = Series( + ... "throughput", + ... branch=None, + ... time=list(range(40)), + ... metrics={"ops": Metric(direction=1, scale=1.0)}, + ... data={"ops": [100.0] * 20 + [80.0] * 20}, + ... attributes={}, + ... ) + >>> analyzed = series.analyze(AnalysisOptions(window_len=20)) + >>> [group.time for group in analyzed.change_points_by_time] + [20] + +Anything reached through a submodule path, such as ``otava.importer`` or +``otava.main``, is internal to the command-line tool and may change without +notice. +""" + +from otava.analysis import compute_change_points +from otava.change_point_divisive.base import ( + ChangePoint, + ChangePointGroup, + ChangePointsByMetric, + ChangePointsByTime, +) +from otava.series import AnalysisOptions, AnalyzedSeries, Metric, Series + +__all__ = [ + "AnalysisOptions", + "AnalyzedSeries", + "ChangePoint", + "ChangePointGroup", + "ChangePointsByMetric", + "ChangePointsByTime", + "Metric", + "Series", + "compute_change_points", +] diff --git a/tests/core_install_smoke.py b/tests/core_install_smoke.py index 72b300d3..b992c95b 100644 --- a/tests/core_install_smoke.py +++ b/tests/core_install_smoke.py @@ -47,6 +47,24 @@ def assert_optional_distributions_are_absent(): raise AssertionError(f"{distribution} was installed by the default package") +def assert_public_api_is_installed(): + # A wheel that drops otava/__init__.py leaves `import otava` resolving to an + # empty namespace package again, which no unit test can see: they run from + # the source tree, where the file is always present. + import otava + + if otava.__file__ is None: + raise AssertionError("otava/__init__.py was not installed with the default package") + + missing = [name for name in otava.__all__ if not hasattr(otava, name)] + if missing: + raise AssertionError(f"public names missing from the installed package: {missing}") + + strong, _weak = otava.compute_change_points([10.0] * 20 + [30.0] * 20, window_len=10) + if [point.index for point in strong] != [20]: + raise AssertionError("compute_change_points found no change through the package root") + + def assert_cli_help_works(): result = subprocess.run( [sys.executable, "-m", "otava.main", "--help"], capture_output=True, text=True @@ -121,6 +139,7 @@ def assert_optional_operations_name_their_extras(): def main(): assert_optional_distributions_are_absent() + assert_public_api_is_installed() assert_cli_help_works() assert_csv_analysis_works() assert_optional_operations_name_their_extras() diff --git a/tests/public_api_test.py b/tests/public_api_test.py new file mode 100644 index 00000000..a0d8fad8 --- /dev/null +++ b/tests/public_api_test.py @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import doctest +import re +import subprocess +import sys +import textwrap +from pathlib import Path +from types import ModuleType + +import otava +from otava.series import AnalysisOptions + +REPO_ROOT = Path(__file__).parents[1] +DOCS_API = REPO_ROOT / "docs/API.md" + +# Where each re-exported name is defined. The package root must hand out the +# very same objects, otherwise isinstance() checks would depend on the import +# path a caller happened to use. +DEFINING_MODULES = { + "AnalysisOptions": "otava.series", + "AnalyzedSeries": "otava.series", + "ChangePoint": "otava.change_point_divisive.base", + "ChangePointGroup": "otava.change_point_divisive.base", + "ChangePointsByMetric": "otava.change_point_divisive.base", + "ChangePointsByTime": "otava.change_point_divisive.base", + "Metric": "otava.series", + "Series": "otava.series", + "compute_change_points": "otava.analysis", +} + + +def optional_client_modules(): + """Top-level modules the optional service clients import. + + Read off the import_optional_dependency() call sites rather than repeated + here, so that a new optional client is covered the day it is added. + """ + call_site = re.compile(r"""import_optional_dependency\(\s*["']([^"']+)["']""") + names = { + match.group(1).partition(".")[0] + for path in (REPO_ROOT / "otava").glob("*.py") + for match in call_site.finditer(path.read_text(encoding="utf-8")) + } + + assert names, "no import_optional_dependency() call sites found" + return sorted(names) + + +def docs_section(title): + """The body of one `## ` section of docs/API.md, up to the next one. + + Bounded at both ends on purpose: a regex that ran to the end of the file + would silently make `title` the only section allowed to come last. + """ + body = re.search( + rf"^## {re.escape(title)}\n(.*?)(?=^## |\Z)", + DOCS_API.read_text(encoding="utf-8"), + re.M | re.S, + ) + + assert body, f"no '{title}' section in {DOCS_API}" + return body.group(1) + + +def documented_public_names(): + return sorted(re.findall(r"^\| `([^`]+)` \|", docs_section("What is public"), re.M)) + + +def test_public_names_are_the_objects_from_their_defining_modules(): + assert set(otava.__all__) == set(DEFINING_MODULES) + + for name, module_name in DEFINING_MODULES.items(): + module = __import__(module_name, fromlist=[name]) + + assert getattr(otava, name) is getattr(module, name) + + +def test_package_root_exports_nothing_beyond_dunder_all(): + attributes = {name: getattr(otava, name) for name in vars(otava)} + exported = { + name + for name, value in attributes.items() + if not name.startswith("_") and not isinstance(value, ModuleType) + } + + assert exported == set(otava.__all__) + assert otava.__all__ == sorted(otava.__all__) + + # Module-valued attributes are the submodules Python binds on import. A + # module from anywhere else is something the package root pulled in, and the + # filter above would otherwise hide it. Underscore names are checked too: an + # `import os as _os` is just as much a leak as `import os`. + foreign = sorted( + value.__name__ + for name, value in attributes.items() + if isinstance(value, ModuleType) and not value.__name__.startswith("otava.") + ) + + assert foreign == [] + + +def test_documented_public_names_match_dunder_all(): + assert documented_public_names() == sorted(otava.__all__) + + +def test_documented_option_defaults_match_analysis_options(): + rows = re.findall(r"^\| `(\w+)` \| ([^|]+?) \|", docs_section("Options"), re.M) + defaults = AnalysisOptions() + + assert len(rows) == len(dict(rows)), "an option is documented twice" + assert dict(rows).keys() == type(defaults).model_fields.keys() + for option, value in rows: + assert str(getattr(defaults, option)) == value, option + + +def test_package_docstring_example_is_accurate(): + results = doctest.testmod(otava, verbose=False) + + assert results.attempted > 0 + assert results.failed == 0 + + +def test_documented_examples_run_and_report_a_change(capsys): + snippets = re.findall( + r"^```python\n(.*?)^```", DOCS_API.read_text(encoding="utf-8"), re.M | re.S + ) + + assert snippets, f"no python examples found in {DOCS_API}" + + for snippet in snippets: + # The page is about the package root. An example that reached into + # otava.series would still run, and would still document the wrong + # import, so no submodule path may appear at all. + assert "from otava import " in snippet + assert not re.search(r"^\s*(?:from|import) otava\.", snippet, re.M) + + exec(compile(snippet, str(DOCS_API), "exec"), {}) + + # An example that finds nothing would still run, and would still be wrong. + assert capsys.readouterr().out.strip() + + +def test_importing_the_package_does_not_load_optional_service_clients(): + # The extras introduced in #55 only hold if no code path a core-only + # installation reaches imports a service client. Importing the package root + # became such a path when it stopped being an empty namespace package. + # + # Poisoning sys.modules rather than builtins.__import__ is deliberate: + # otava/_optional.py reaches its clients through importlib.import_module(), + # which does not go through builtins.__import__, so patching that would miss + # every import written the way this repository writes them. + script = "blocked = " + repr(optional_client_modules()) + textwrap.dedent( + """ + import sys + + for name in blocked: + sys.modules[name] = None + + import otava + + for name in otava.__all__: + getattr(otava, name) + """ + ) + + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + assert result.returncode == 0, result.stderr From c4d6baa3bce702846e1d356f5c03c30368768896 Mon Sep 17 00:00:00 2001 From: MrlixiangWE <102979255+MrlixiangWE@users.noreply.github.com> Date: Sun, 20 Sep 2026 12:15:01 +0800 Subject: [PATCH 2/2] Clarify public API documentation --- docs/API.md | 14 +++++++------- otava/__init__.py | 6 +++--- tests/public_api_test.py | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/API.md b/docs/API.md index 18b02c86..9de0a346 100644 --- a/docs/API.md +++ b/docs/API.md @@ -53,9 +53,9 @@ for group in analyzed.change_points_by_time: `analyze()` does not interpret `time`, so a plain index works for data that is not a time series, but keep it increasing: change points are collected in time order, and a second group that is not later than the first raises `ValueError`. -Otava's own reports do interpret it, formatting it with -`datetime.fromtimestamp`, so anything printed by Otava reads it as seconds since -the epoch. +Otava's human-readable log and regression reports do interpret it, formatting +it with `datetime.fromtimestamp`, so those report formats read it as seconds +since the epoch. JSON reports preserve the numeric value. `attributes` carries one value per data point, and the values at a change point are returned on its group. That is how you get from a detected change back to @@ -81,10 +81,10 @@ The names in `otava.__all__`, re-exported from the package root: | `ChangePointsByTime` | Change point groups in time order | | `compute_change_points` | The detection algorithm on a bare sequence of floats; returns `(strong, weak)` | -Everything else is internal, including every submodule path such as -`otava.importer`, `otava.config` and `otava.main`. Otava does not offer a stable -public API yet, so even the names above can still change in a minor or patch -release. +The supported import surface is the names above when imported from `otava`. +Submodule paths such as `otava.importer`, `otava.config` and `otava.main` are +not part of the public API. Otava does not offer a stable public API yet, so +even the names above can still change in a minor or patch release. ## Options diff --git a/otava/__init__.py b/otava/__init__.py index db3a472a..0bb6e656 100644 --- a/otava/__init__.py +++ b/otava/__init__.py @@ -34,9 +34,9 @@ >>> [group.time for group in analyzed.change_points_by_time] [20] -Anything reached through a submodule path, such as ``otava.importer`` or -``otava.main``, is internal to the command-line tool and may change without -notice. +The supported import surface is the names re-exported from ``otava``. Submodule +paths such as ``otava.importer`` and ``otava.main`` are not part of the public +API and may change without notice. """ from otava.analysis import compute_change_points diff --git a/tests/public_api_test.py b/tests/public_api_test.py index a0d8fad8..d135ea12 100644 --- a/tests/public_api_test.py +++ b/tests/public_api_test.py @@ -46,10 +46,10 @@ def optional_client_modules(): - """Top-level modules the optional service clients import. + """Literal optional dependencies imported by top-level Otava modules. - Read off the import_optional_dependency() call sites rather than repeated - here, so that a new optional client is covered the day it is added. + Read the import_optional_dependency() call sites instead of repeating the + currently known module names here. """ call_site = re.compile(r"""import_optional_dependency\(\s*["']([^"']+)["']""") names = {