Add --deterministic-edivisive variation

otava/analysis.py

@@ -213,6 +213,44 @@ def compute_change_points_orig(series: Sequence[SupportsFloat], max_pvalue: floa
     return change_points, None
 
 
+def compute_change_points_deterministic(series: Sequence[SupportsFloat], max_pvalue: float = 0.001, min_magnitude: float = 0.0) -> Tuple[PermCPList, Optional[PermCPList]]:
+    """
+    Same as the original algorithm but with deterministic Student T significance test at the end.
+
+    The motivation for this variation follows from fixing the bug explained at the top of https://github.com/apache/otava/pull/96
+    The intuition is that the split-merge approach introduced by Datastax is addressing the same problem that the _kappa_ variable
+    does in the original paper. Now that we compute correctly over all values of kappa, the split-merge part should be unnecessary,
+    as the original algorithm with kappa bug fixed, will find the same change points, and more. Therefore the conclusion is we want
+    to go back as much as possible to the original and real algorithm from the Matteson & James paper. But even then, we find that
+    Student T as significance test is both much faster but also qualitatively produces better results for the use case we're in at least,
+    that we want to continue using T test and not random permutations for the significance test.
+
+    TBD: Whether weak change points are still helpful or not. By reading the problem they fix appears unrelated from the split-merge vs **kappa** symptoms.
+
+    TODO: Support incremental e-divisive. This was easy to implement on top of the split-merge variation. Not clear what is the correct way here.
+    An easy solution is to rerun from the last change-point, but the problem is the last change point could itself be influenced by the new data
+    appended.
+    """
+    tester = TTestSignificanceTester(max_pvalue=max_pvalue)
+    detector = ChangePointDetector(significance_tester=tester, calculator=PairDistanceCalculator)
+    all_change_points = detector.get_change_points(series=series)
+    if min_magnitude > 0.0:
+        above_threshold_change_points = [cp for cp in all_change_points if cp.stats.change_magnitude() >= min_magnitude]
+    else:
+        above_threshold_change_points = all_change_points
+    return above_threshold_change_points, all_change_points
+
+
+def compute_change_points_split(
+    series: Sequence[SupportsFloat], window_len: int = 50, max_pvalue: float = 0.001, min_magnitude: float = 0.0,
+    new_data: Optional[int] = None, old_weak_cp: Optional[GenCPList] = None
+) -> Tuple[GenCPList, Optional[GenCPList]]:
+    """
+    This function added mainly for symmetry and anticipating a future where this is no longer the default
+    """
+    return compute_change_points(series, window_len, max_pvalue, min_magnitude, new_data, old_weak_cp)
+
+
 def compute_change_points(
     series: Sequence[SupportsFloat], window_len: int = 50, max_pvalue: float = 0.001, min_magnitude: float = 0.0,
     new_data: Optional[int] = None, old_weak_cp: Optional[GenCPList] = None

otava/main.py

@@ -422,14 +422,28 @@ def setup_analysis_options_parser(parser: argparse.ArgumentParser):
         "as noise so it is best to keep it short enough to include not more "
         "than a few change points (optimally at most 1)",
     )
-    parser.add_argument(
+    ediv_group = parser.add_mutually_exclusive_group()
+    ediv_group.add_argument(
         "--orig-edivisive",
         action="store_true",
-        default=False,
         dest="orig_edivisive",
         help="use the original edivisive algorithm with no windowing "
         "and weak change points analysis improvements",
     )
+    ediv_group.add_argument(
+        "--deterministic-edivisive",
+        action="store_true",
+        dest="deterministic_edivisive",
+        help="EXPERIMENTAL: use the original edivisive algorithm, but using "
+        "Student T for significance test. (TBD: May include weak change points later.)",
+    )
+    ediv_group.add_argument(
+        "--split-edivisive",
+        action="store_true",
+        dest="split_edivisive",
+        help="use 'hunter' version of this algorithm, from 2023, featuring "
+        "split of data into smaller windows, weak change points and Student T test. (Default)",
+    )
 
 
 def analysis_options_from_args(args: argparse.Namespace) -> AnalysisOptions:
@@ -440,8 +454,14 @@ def analysis_options_from_args(args: argparse.Namespace) -> AnalysisOptions:
         conf.min_magnitude = args.magnitude
     if args.window is not None:
         conf.window_len = args.window
-    if args.orig_edivisive is not None:
-        conf.orig_edivisive = args.orig_edivisive
+
+    conf.orig_edivisive = args.orig_edivisive
+    conf.deterministic_edivisive = args.deterministic_edivisive
+    conf.split_edivisive = args.split_edivisive
+    if not (args.split_edivisive or args.deterministic_edivisive or args.orig_edivisive):
+        # Default:
+        conf.split_edivisive = True
+
     return conf
 
 

otava/serialization.py

@@ -30,6 +30,8 @@ class AnalysisOptionsModel(BaseModel):
     max_pvalue: float = 0.001
     min_magnitude: float = 0.0
     orig_edivisive: bool = False
+    deterministic_edivisive: bool = False
+    split_edivisive: bool = True
 
 
 class MetricModel(BaseModel):

otava/series.py

@@ -25,6 +25,7 @@
 from otava.analysis import (
     TTestStats,
     compute_change_points,
+    compute_change_points_deterministic,
     compute_change_points_orig,
 )
 from otava.change_point_divisive.base import (
@@ -194,6 +195,20 @@ def __compute_change_points(
                         changes={metric: c},
                     )
                     result.append(cpg)
+            elif options.deterministic_edivisive:
+                change_points, _ = compute_change_points_deterministic(
+                    values,
+                    max_pvalue=options.max_pvalue,
+                    min_magnitude=options.min_magnitude,
+                )
+                for c in change_points:
+                    c.metric = metric
+                    cpg = ChangePointGroup(
+                        time=series.time[c.index],
+                        attributes=series.attributes_at(c.index),
+                        changes={metric: c},
+                    )
+                    result.append(cpg)
             else:
                 change_points, weak_cps = compute_change_points(
                     values,

tests/cli_help_test.py

@@ -185,7 +185,7 @@ def test_otava_analyze_help_output():
                      [--output {{log,json,regressions_only}}] [--branch [STRING]] [--metrics LIST]
 {usage_filter_lines}
                      [--last COUNT] [-P, --p-value PVALUE] [-M MAGNITUDE] [--window WINDOW]
-                     [--orig-edivisive]
+                     [--orig-edivisive | --deterministic-edivisive | --split-edivisive]
                      tests [tests ...]
 
 positional arguments:
@@ -237,6 +237,11 @@ def test_otava_analyze_help_output():
                         enough to include not more than a few change points (optimally at most 1)
   --orig-edivisive      use the original edivisive algorithm with no windowing and weak change
                         points analysis improvements
+  --deterministic-edivisive
+                        EXPERIMENTAL: use the original edivisive algorithm, but using Student T
+                        for significance test. (TBD: May include weak change points later.)
+  --split-edivisive     use 'hunter' version of this algorithm, from 2023, featuring split of data
+                        into smaller windows, weak change points and Student T test. (Default)
 
 CSV Options:
   Options for CSV configuration

tests/cli_options_test.py

@@ -46,18 +46,50 @@ def test_default_cli_option(self):
 
             assert otava.series.compute_change_points.call_count == 2
 
-    # Failing due to lack of cp.metric see pull#141
-    # def test_orig_cli_option(self):
-    #     with patch('otava.series.compute_change_points') as mock_orig:
-    #         mock_orig.return_value = ([], None)
-    #         with tempfile.TemporaryDirectory() as td:
-    #             td_path = Path(td)
-    #             csv_path, timestamps, config_path, test_name = _create_files_in_temp_dir(td_path)
-    #             # _uv_run(td_path, test_name)
-    #             config_path_str = "" + str(config_path)
-    #             script_main(args=["analyze", "--config", config_path_str, "--orig-edivisive", "true", test_name])
-    #
-    #         assert otava.series.compute_change_points_orig.call_count == 2
+    def test_split_cli_option(self):
+        with patch("otava.series.compute_change_points") as mock_split:
+            mock_split.return_value = ([], [])
+            with tempfile.TemporaryDirectory() as td:
+                td_path = Path(td)
+                csv_path, timestamps, config_path, test_name = _create_files_in_temp_dir(td_path)
+                config_path_str = "" + str(config_path)
+                script_main(
+                    args=["analyze", "--config", config_path_str, "--split-edivisive", test_name]
+                )
+
+            assert otava.series.compute_change_points.call_count == 2
+
+    def test_orig_cli_option(self):
+        with patch("otava.series.compute_change_points_orig") as mock_orig:
+            mock_orig.return_value = ([], None)
+            with tempfile.TemporaryDirectory() as td:
+                td_path = Path(td)
+                csv_path, timestamps, config_path, test_name = _create_files_in_temp_dir(td_path)
+                config_path_str = "" + str(config_path)
+                script_main(
+                    args=["analyze", "--config", config_path_str, "--orig-edivisive", test_name]
+                )
+
+            assert otava.series.compute_change_points_orig.call_count == 2
+
+    def test_deterministic_cli_option(self):
+        with patch("otava.series.compute_change_points_deterministic") as mock_deterministic:
+            mock_deterministic.return_value = ([], None)
+            with tempfile.TemporaryDirectory() as td:
+                td_path = Path(td)
+                csv_path, timestamps, config_path, test_name = _create_files_in_temp_dir(td_path)
+                config_path_str = "" + str(config_path)
+                script_main(
+                    args=[
+                        "analyze",
+                        "--config",
+                        config_path_str,
+                        "--deterministic-edivisive",
+                        test_name,
+                    ]
+                )
+
+            assert otava.series.compute_change_points_deterministic.call_count == 2
 
 
 def _create_files_in_temp_dir(td_path: Path):

tests/deterministic_tigerbeetle_test.py

@@ -0,0 +1,128 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from otava.analysis import compute_change_points_deterministic
+from tests.tigerbeetle_test import tigerbeetle_demo_data as _get_series
+
+
+def test_tb_magnitude0_p2():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.2)
+    indexes = [c.index for c in cps]
+    assert indexes == [10, 11, 15, 61, 71, 95, 117, 131, 142, 148, 192, 212, 260, 363]
+
+
+def test_tb_magnitude0_p15():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.15)
+    indexes = [c.index for c in cps]
+    assert indexes == [10, 11, 15, 61, 71, 95, 117, 131, 142, 148, 192, 212, 260, 363]
+
+
+def test_tb_magnitude0_p14():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.14)
+    indexes = [c.index for c in cps]
+    assert indexes == [10, 11, 15, 61, 71, 95, 117, 131, 142, 148, 192, 212, 260, 363]
+
+
+def test_tb_magnitude0_p13():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.13)
+    indexes = [c.index for c in cps]
+    assert indexes == [10, 11, 15, 61, 71, 95, 117, 131, 142, 148, 192, 212, 260, 363]
+
+
+def test_tb_magnitude0_p127():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.129)
+    indexes = [c.index for c in cps]
+    assert indexes == [10, 11, 15, 61, 71, 95, 117, 131, 142, 148, 192, 212, 260, 363]
+
+
+def test_tb_magnitude0_p125():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.125)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 95, 131, 192]
+
+
+def test_tb_magnitude0_p12():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.12)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 95, 131, 192]
+
+
+def test_tb_magnitude0_p11():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.11)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 95, 131, 192]
+
+
+def test_tb_magnitude0_p1():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.1)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 95, 131, 192]
+
+
+def test_tb_magnitude0_p01():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.01)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 95, 131, 192]
+
+
+def test_tb_magnitude0_p001():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.001)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 192]
+
+
+def test_tb_magnitude0_p0001():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.0001)
+    indexes = [c.index for c in cps]
+    assert indexes == [15, 71, 192]
+
+
+def test_tb_magnitude0_p00001():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.00001)
+    indexes = [c.index for c in cps]
+    print(cps)
+    assert indexes == [15, 71, 192]
+
+
+def test_tb_magnitude0_p7x01():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.00000001)
+    indexes = [c.index for c in cps]
+    print(cps)
+    assert indexes == [71, 192]
+
+
+def test_tb_magnitude0_p31x01():
+    series = _get_series()
+    cps, weak_cps = compute_change_points_deterministic(series, max_pvalue=0.00000000000000000000000000000001)
+    indexes = [c.index for c in cps]
+    print(cps)
+    assert indexes == [71, 192]