Define the public API in otava/__init__.py

docs/API.md

@@ -0,0 +1,102 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Using Otava as a library
+
+Otava can be used without its command line interface and without any of its
+importers: build a `Series` from data you already have, analyze it, and read the
+change points back.
+
+```python
+from otava import AnalysisOptions, Metric, Series
+
+# One measurement per commit. Metric.direction says which way is an improvement:
+# 1 when higher is better, -1 when lower is better.
+series = Series(
+    test_name="throughput",
+    branch=None,
+    time=list(range(60)),
+    metrics={"ops": Metric(direction=1, scale=1.0)},
+    data={"ops": [100.0] * 30 + [80.0] * 30},
+    attributes={"commit": [f"sha{i}" for i in range(60)]},
+)
+
+analyzed = series.analyze(AnalysisOptions(window_len=20, max_pvalue=0.001))
+
+for group in analyzed.change_points_by_time:
+    for metric, change in group.changes.items():
+        print(
+            group.attributes["commit"],
+            metric,
+            change.stats.mean_before(),
+            change.stats.mean_after(),
+            f"{change.stats.forward_change_percent():+.1f}%",
+        )
+```
+
+`analyze()` does not interpret `time`, so a plain index works for data that is
+not a time series, but keep it increasing: change points are collected in time
+order, and a second group that is not later than the first raises `ValueError`.
+Otava's human-readable log and regression reports do interpret it, formatting
+it with `datetime.fromtimestamp`, so those report formats read it as seconds
+since the epoch. JSON reports preserve the numeric value.
+
+`attributes` carries one value per data point, and the values at a change point
+are returned on its group. That is how you get from a detected change back to
+the commit that caused it.
+
+`Series.analyze()` is lazy: the change points are computed the first time any of
+the change point properties on `AnalyzedSeries` is read, and cached from then on.
+Errors the analysis raises therefore surface at that first read.
+
+## What is public
+
+The names in `otava.__all__`, re-exported from the package root:
+
+| Name | Purpose |
+| --- | --- |
+| `Series` | Input data: timestamps, metrics, values, per-point attributes |
+| `Metric` | Direction, scale and unit of one metric |
+| `AnalysisOptions` | `window_len`, `max_pvalue`, `min_magnitude`, `orig_edivisive` |
+| `AnalyzedSeries` | Result of `Series.analyze()` |
+| `ChangePoint` | One change in one metric; the statistics are on its `stats` |
+| `ChangePointGroup` | Changes that share a point in time |
+| `ChangePointsByMetric` | Change points keyed by metric |
+| `ChangePointsByTime` | Change point groups in time order |
+| `compute_change_points` | The detection algorithm on a bare sequence of floats; returns `(strong, weak)` |
+
+The supported import surface is the names above when imported from `otava`.
+Submodule paths such as `otava.importer`, `otava.config` and `otava.main` are
+not part of the public API. Otava does not offer a stable public API yet, so
+even the names above can still change in a minor or patch release.
+
+## Options
+
+`AnalysisOptions` holds the settings the `analyze` command exposes as flags,
+with the same defaults. The attribute names and the flag names differ; run
+`otava analyze --help` for the flags.
+
+| Option | Default | Effect |
+| --- | --- | --- |
+| `window_len` | 50 | Number of points the algorithm looks at around a candidate |
+| `max_pvalue` | 0.001 | Significance threshold a change point must meet |
+| `min_magnitude` | 0.0 | Smallest relative change worth reporting |
+| `orig_edivisive` | False | Use unmodified e-divisive instead of the windowed variant |
+
+See [Change Point Detection](MATH.md) for what these do to the results.

docs/README.md

@@ -39,6 +39,7 @@ POSTGRESQL
 BIG_QUERY
 INFLUXDB
 GRAFANA
+API
 ```
 
 ```{toctree}

otava/__init__.py

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Change detection for continuous performance engineering.
+
+The names re-exported here are the public API of Otava as a library: the types
+needed to build a Series from data Otava did not import itself, analyze it, and
+read the change points back.
+
+    >>> from otava import AnalysisOptions, Metric, Series
+    >>> series = Series(
+    ...     "throughput",
+    ...     branch=None,
+    ...     time=list(range(40)),
+    ...     metrics={"ops": Metric(direction=1, scale=1.0)},
+    ...     data={"ops": [100.0] * 20 + [80.0] * 20},
+    ...     attributes={},
+    ... )
+    >>> analyzed = series.analyze(AnalysisOptions(window_len=20))
+    >>> [group.time for group in analyzed.change_points_by_time]
+    [20]
+
+The supported import surface is the names re-exported from ``otava``. Submodule
+paths such as ``otava.importer`` and ``otava.main`` are not part of the public
+API and may change without notice.
+"""
+
+from otava.analysis import compute_change_points
+from otava.change_point_divisive.base import (
+    ChangePoint,
+    ChangePointGroup,
+    ChangePointsByMetric,
+    ChangePointsByTime,
+)
+from otava.series import AnalysisOptions, AnalyzedSeries, Metric, Series
+
+__all__ = [
+    "AnalysisOptions",
+    "AnalyzedSeries",
+    "ChangePoint",
+    "ChangePointGroup",
+    "ChangePointsByMetric",
+    "ChangePointsByTime",
+    "Metric",
+    "Series",
+    "compute_change_points",
+]

tests/core_install_smoke.py

@@ -47,6 +47,24 @@ def assert_optional_distributions_are_absent():
         raise AssertionError(f"{distribution} was installed by the default package")
 
 
+def assert_public_api_is_installed():
+    # A wheel that drops otava/__init__.py leaves `import otava` resolving to an
+    # empty namespace package again, which no unit test can see: they run from
+    # the source tree, where the file is always present.
+    import otava
+
+    if otava.__file__ is None:
+        raise AssertionError("otava/__init__.py was not installed with the default package")
+
+    missing = [name for name in otava.__all__ if not hasattr(otava, name)]
+    if missing:
+        raise AssertionError(f"public names missing from the installed package: {missing}")
+
+    strong, _weak = otava.compute_change_points([10.0] * 20 + [30.0] * 20, window_len=10)
+    if [point.index for point in strong] != [20]:
+        raise AssertionError("compute_change_points found no change through the package root")
+
+
 def assert_cli_help_works():
     result = subprocess.run(
         [sys.executable, "-m", "otava.main", "--help"], capture_output=True, text=True
@@ -121,6 +139,7 @@ def assert_optional_operations_name_their_extras():
 
 def main():
     assert_optional_distributions_are_absent()
+    assert_public_api_is_installed()
     assert_cli_help_works()
     assert_csv_analysis_works()
     assert_optional_operations_name_their_extras()

tests/public_api_test.py

@@ -0,0 +1,184 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import doctest
+import re
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+from types import ModuleType
+
+import otava
+from otava.series import AnalysisOptions
+
+REPO_ROOT = Path(__file__).parents[1]
+DOCS_API = REPO_ROOT / "docs/API.md"
+
+# Where each re-exported name is defined. The package root must hand out the
+# very same objects, otherwise isinstance() checks would depend on the import
+# path a caller happened to use.
+DEFINING_MODULES = {
+    "AnalysisOptions": "otava.series",
+    "AnalyzedSeries": "otava.series",
+    "ChangePoint": "otava.change_point_divisive.base",
+    "ChangePointGroup": "otava.change_point_divisive.base",
+    "ChangePointsByMetric": "otava.change_point_divisive.base",
+    "ChangePointsByTime": "otava.change_point_divisive.base",
+    "Metric": "otava.series",
+    "Series": "otava.series",
+    "compute_change_points": "otava.analysis",
+}
+
+
+def optional_client_modules():
+    """Literal optional dependencies imported by top-level Otava modules.
+
+    Read the import_optional_dependency() call sites instead of repeating the
+    currently known module names here.
+    """
+    call_site = re.compile(r"""import_optional_dependency\(\s*["']([^"']+)["']""")
+    names = {
+        match.group(1).partition(".")[0]
+        for path in (REPO_ROOT / "otava").glob("*.py")
+        for match in call_site.finditer(path.read_text(encoding="utf-8"))
+    }
+
+    assert names, "no import_optional_dependency() call sites found"
+    return sorted(names)
+
+
+def docs_section(title):
+    """The body of one `## ` section of docs/API.md, up to the next one.
+
+    Bounded at both ends on purpose: a regex that ran to the end of the file
+    would silently make `title` the only section allowed to come last.
+    """
+    body = re.search(
+        rf"^## {re.escape(title)}\n(.*?)(?=^## |\Z)",
+        DOCS_API.read_text(encoding="utf-8"),
+        re.M | re.S,
+    )
+
+    assert body, f"no '{title}' section in {DOCS_API}"
+    return body.group(1)
+
+
+def documented_public_names():
+    return sorted(re.findall(r"^\| `([^`]+)` \|", docs_section("What is public"), re.M))
+
+
+def test_public_names_are_the_objects_from_their_defining_modules():
+    assert set(otava.__all__) == set(DEFINING_MODULES)
+
+    for name, module_name in DEFINING_MODULES.items():
+        module = __import__(module_name, fromlist=[name])
+
+        assert getattr(otava, name) is getattr(module, name)
+
+
+def test_package_root_exports_nothing_beyond_dunder_all():
+    attributes = {name: getattr(otava, name) for name in vars(otava)}
+    exported = {
+        name
+        for name, value in attributes.items()
+        if not name.startswith("_") and not isinstance(value, ModuleType)
+    }
+
+    assert exported == set(otava.__all__)
+    assert otava.__all__ == sorted(otava.__all__)
+
+    # Module-valued attributes are the submodules Python binds on import. A
+    # module from anywhere else is something the package root pulled in, and the
+    # filter above would otherwise hide it. Underscore names are checked too: an
+    # `import os as _os` is just as much a leak as `import os`.
+    foreign = sorted(
+        value.__name__
+        for name, value in attributes.items()
+        if isinstance(value, ModuleType) and not value.__name__.startswith("otava.")
+    )
+
+    assert foreign == []
+
+
+def test_documented_public_names_match_dunder_all():
+    assert documented_public_names() == sorted(otava.__all__)
+
+
+def test_documented_option_defaults_match_analysis_options():
+    rows = re.findall(r"^\| `(\w+)` \| ([^|]+?) \|", docs_section("Options"), re.M)
+    defaults = AnalysisOptions()
+
+    assert len(rows) == len(dict(rows)), "an option is documented twice"
+    assert dict(rows).keys() == type(defaults).model_fields.keys()
+    for option, value in rows:
+        assert str(getattr(defaults, option)) == value, option
+
+
+def test_package_docstring_example_is_accurate():
+    results = doctest.testmod(otava, verbose=False)
+
+    assert results.attempted > 0
+    assert results.failed == 0
+
+
+def test_documented_examples_run_and_report_a_change(capsys):
+    snippets = re.findall(
+        r"^```python\n(.*?)^```", DOCS_API.read_text(encoding="utf-8"), re.M | re.S
+    )
+
+    assert snippets, f"no python examples found in {DOCS_API}"
+
+    for snippet in snippets:
+        # The page is about the package root. An example that reached into
+        # otava.series would still run, and would still document the wrong
+        # import, so no submodule path may appear at all.
+        assert "from otava import " in snippet
+        assert not re.search(r"^\s*(?:from|import) otava\.", snippet, re.M)
+
+        exec(compile(snippet, str(DOCS_API), "exec"), {})
+
+    # An example that finds nothing would still run, and would still be wrong.
+    assert capsys.readouterr().out.strip()
+
+
+def test_importing_the_package_does_not_load_optional_service_clients():
+    # The extras introduced in #55 only hold if no code path a core-only
+    # installation reaches imports a service client. Importing the package root
+    # became such a path when it stopped being an empty namespace package.
+    #
+    # Poisoning sys.modules rather than builtins.__import__ is deliberate:
+    # otava/_optional.py reaches its clients through importlib.import_module(),
+    # which does not go through builtins.__import__, so patching that would miss
+    # every import written the way this repository writes them.
+    script = "blocked = " + repr(optional_client_modules()) + textwrap.dedent(
+        """
+        import sys
+
+        for name in blocked:
+            sys.modules[name] = None
+
+        import otava
+
+        for name in otava.__all__:
+            getattr(otava, name)
+        """
+    )
+
+    result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True)
+
+    assert result.returncode == 0, result.stderr