From abd3a7df78f80a354b6c1a9d6bd07d0e35759b1c Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Fri, 4 Sep 2026 16:17:30 -0400 Subject: [PATCH 1/7] fix chunking system to be more accurate and complete and make all pted faster --- README.md | 91 +++++-- src/pted/pted.py | 180 ++++++++----- src/pted/utils.py | 619 +++++++++++++++++++++++++++++++++++--------- tests/test_pted.py | 148 ++++++++++- tests/test_utils.py | 273 ++++++++++++++++++- 5 files changed, 1096 insertions(+), 215 deletions(-) diff --git a/README.md b/README.md index fca8df8..d808013 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,16 @@ You can also generate a Probability Integral Transform (PIT) plot via `pit_plot PTED uses the energy distance of the two samples `x` and `y`, this is computed as: -$$d = \frac{2}{n_xn_y}\sum_{i,j}||x_i - y_j|| - \frac{1}{n_x^2}\sum_{i,j}||x_i - x_j|| - \frac{1}{n_y^2}\sum_{i,j}||y_i - y_j||$$ +$$d = \frac{2}{n_xn_y}\sum_{i,j}||x_i - y_j|| - \frac{1}{n_x(n_x-1)}\sum_{i\neq j}||x_i - x_j|| - \frac{1}{n_y(n_y-1)}\sum_{i\neq j}||y_i - y_j||$$ + +The within-group sums run over distinct pairs, skipping the `i = j` terms which +are zero by construction. That makes `d` an unbiased estimator of the population +energy distance, so under the null it scatters around zero and can come out +slightly negative (and strongly negative if `x` and `y` are *more* alike than +chance allows, e.g. if they share samples). Only its rank among the permuted +values matters. For equal sample sizes this ranks the permutations identically +to the `1/n^2` normalisation of Székely & Rizzo, so the p-value is unchanged; for +unequal sizes the two differ slightly. The energy distance measures distances between pairs of points[^2]. It becomes more positive if the `x` and `y` samples tend to be further from each other than from @@ -298,6 +307,9 @@ def pted( chunk_size: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, + n_columns: Optional[int] = None, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: ``` @@ -305,9 +317,12 @@ def pted( * **y** *(Union[np.ndarray, Tensor, jax.Array])*: second set of samples. Shape (M, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If `chunk_size >= len(x)` and `chunk_size >= len(y)`, PTED falls back to the full (non-chunked) computation. If None, use the full dataset. +* **chunk_size** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. Only distances from every sample to `min(chunk_size, len(x)) + min(chunk_size, len(y))` "column" points drawn from the pooled sample are computed, so the cost drops from `O(n^2 d)` to `O(n c d)`. If `chunk_size` covers both full datasets, PTED falls back to the exact full-matrix computation. If None, use the full dataset. Mutually exclusive with `n_columns`. * **two_tailed** *(bool)*: if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. If False, only checks for dissimilarity but is more sensitive. Default is True. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of permutation tests. Default is False. +* **n_columns** *(Optional[int])*: the number of column points `c` directly. This is the preferred arg; `chunk_size` is the older one and simply maps onto it. Mutually exclusive with `chunk_size`. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. +* **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ### Coverage test @@ -324,6 +339,9 @@ def pted_coverage_test( pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, + n_columns: Optional[int] = None, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: ``` @@ -331,12 +349,17 @@ def pted_coverage_test( * **s** *(Union[np.ndarray, Tensor, jax.Array])*: Posterior samples. Shape (n_samples, n_sims, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If None, use the full dataset. +* **chunk_size** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. Only distances from every sample to `min(chunk_size, len(x)) + min(chunk_size, len(y))` "column" points drawn from the pooled sample are computed, so the cost drops from `O(n^2 d)` to `O(n c d)`. If `chunk_size` covers both full datasets, PTED falls back to the exact full-matrix computation. If None, use the full dataset. Mutually exclusive with `n_columns`. + + Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches only `n - c` distinct label assignments. Keep `chunk_size` well below the number of posterior samples. * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. * **pit_confidence** *(float)*: Confidence level for the KS confidence band in the PIT plot. Default is 0.95 (95%). Only used when `pit_plot` is not None. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of simulations. Default is False. +* **n_columns** *(Optional[int])*: the number of column points `c` directly. This is the preferred arg; `chunk_size` is the older one and simply maps onto it. Mutually exclusive with `chunk_size`. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. +* **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ## GPU Compatibility @@ -373,38 +396,56 @@ print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 If a GPU isn't enough to get PTED running fast enough for you, or if you are running into memory limitations, there are still options! We can use an -approximation of the energy distance, in this case the test is still exact but -less sensitive than it would be otherwise. Instead of building the full -`(n_samp_x + n_samp_y) x (n_samp_x + n_samp_y)` pairwise distance matrix, PTED -can build a smaller rectangular `(n_samp_x + n_samp_y) x (nxc + nyc)` matrix of -distances from every sample to a fixed set of "landmark" samples, where `nxc = -min(chunk_size, n_samp_x)` landmarks are drawn from x and `nyc = min(chunk_size, -n_samp_y)` from y. Just set the `chunk_size` parameter to control how many -landmarks are used. This rectangular matrix is computed only once; the -permutation distribution is then obtained purely by re-indexing (permuting) its -rows, which reassigns samples to the x/y groups without recomputing any -distances. The larger the chunk size, the closer the estimate will be to the -true energy distance, but it will take more compute. - -Note that the computational complexity for standard PTED goes as `O((n_samp_x + -n_samp_y)^2)` to build the full distance matrix, while the chunked version goes -as `O((n_samp_x + n_samp_y) * (nxc + nyc))`, so plan your chunking accordingly. -For a given chunk size, the computational complexity of PTED grows linearly -with dataset size, much like other large scale (machine learning oriented) two -sample tests. +approximation of the energy distance; the test stays exact, it just becomes less +sensitive than it would otherwise be. Instead of building the full `n x n` +pairwise distance matrix (`n = n_samp_x + n_samp_y`), PTED builds a smaller +rectangular `n x c` matrix of distances from every sample to `c` "column" points +drawn from the pooled sample. Set `n_columns = c` to choose how many columns. + +Building the full matrix costs `O(n^2 d)` and each permutation `O(n^2)`; the +rectangular matrix costs `O(n c d)` and each permutation `O(n c)`. So for a +fixed number of columns PTED grows linearly with dataset size, much like other +large scale (machine learning oriented) two sample tests. Permutations are +evaluated in batches as a single matrix product rather than one at a time, and +everything heavy stays on whichever backend your arrays live on. Example: ```python from pted import pted import numpy as np -x = np.random.normal(size = (500, 10)) # (n_samples_x, n_dimensions) -y = np.random.normal(size = (400, 10)) # (n_samples_y, n_dimensions) +x = np.random.normal(size = (5000, 10)) # (n_samples_x, n_dimensions) +y = np.random.normal(size = (4000, 10)) # (n_samples_y, n_dimensions) -p_value = pted(x, y, chunk_size = 50) +p_value = pted(x, y, n_columns = 200) print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 ``` +### Why chunking is still exact + +The subtlety is that the columns cannot simply be re-split between the two +groups after each shuffle. The column set `C` is chosen using the group labels +(so that both groups are represented), which means the labels are no longer +uniformly distributed once you condition on `C`. PTED handles this by confining +permutations to the subgroup that fixes `C`: labels are shuffled within the +column positions and within their complement, never between. `C` then stays put +under every permutation and the observed labelling is exchangeable with the permuted +ones. That is what makes the p-value exact. (`C` is chosen from sample positions +and group labels only, never from the data values — picking columns by maximin, +k-means or leverage would break the argument.) + +What you give up is sensitivity: the null spread grows like `sqrt(n / c)`, so the +smallest detectable energy distance scales as `(n c)^-0.5` rather than `n^-1`. +The detection threshold degrades as one over the square root of the compute. + +You also give up some p-value resolution, and this is most significant when one +group holds a single point — which is exactly the per-simulation test inside +`pted_coverage_test`. There the permutation subgroup only reaches `n - c` +distinct label assignments, so the smallest attainable p-value is about `1 / (n +- c)` no matter how many permutations you draw. Keep `chunk_size` well below the +number of posterior samples in that case; PTED raises a +`PermutationResolutionWarning` when the column count is too greedy. + ## Citation If you use PTED in your work, please include a citation to the [zenodo diff --git a/src/pted/pted.py b/src/pted/pted.py index 06f33a5..64e6cac 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -3,8 +3,8 @@ import numpy as np from .utils import ( - pted as _pted_impl, - pted_chunk as _pted_chunk_impl, + permutation_energy_test as _energy_test, + _as_rng, two_tailed_p, confidence_alert, simulation_based_calibration_histogram, @@ -22,6 +22,9 @@ def pted( chunk_size: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, + n_columns: Optional[int] = None, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: """ Two sample null hypothesis test using a permutation test on the energy @@ -53,6 +56,13 @@ def pted( p = sum(permute_stats > test_stat) return (1 + p) / (1 + permutations) + The energy distance is computed with the within-group means taken over + distinct pairs, excluding the zero self-pairs. This makes the statistic an + unbiased estimator of the population energy distance, so it may come out + slightly negative when x and y are drawn from the same distribution (and + strongly negative when they are more alike than chance allows, e.g. shared + samples). Only its rank among the permuted statistics matters. + Example ------- import numpy as np @@ -75,46 +85,58 @@ def pted( return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): if not None, use a landmark-based - rectangular distance matrix to estimate the energy distance - instead of the full pairwise matrix. ``nxc = min(chunk_size, - len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` - samples from y are used as fixed "landmarks"; only distances from - every sample to these landmarks are computed. If ``chunk_size`` - covers both full datasets, this falls back to the exact - (non-chunked) computation. If None, use the full dataset. + chunk_size (Optional[int]): if not None, estimate the energy distance + from a rectangular distance matrix instead of the full pairwise + matrix. Only distances from every sample to ``min(chunk_size, + len(x)) + min(chunk_size, len(y))`` "column" points drawn from the + pooled sample are computed, so the cost drops from ``O(n^2 d)`` to + ``O(n c d)``. If ``chunk_size`` covers both full datasets, this + falls back to the exact full-matrix computation. If None, use the + full dataset. Mutually exclusive with ``n_columns``. two_tailed (bool): if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. Default is True. prog_bar (bool): if True, show a progress bar to track the progress of permutation tests. Default is False. - + n_columns (Optional[int]): the number of column points ``c`` directly. + This is the preferred arg; ``chunk_size`` is the older one and + simply maps onto it. Mutually exclusive with ``chunk_size``. + batch_size (Optional[int]): number of permutations evaluated per matrix + product. Larger values are faster (especially on GPU) at + ``O(batch_size * n)`` extra memory. None picks a size that keeps a + batch to a few million elements. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state, so ``np.random.seed`` still controls reproducibility. Note ---- - PTED has O(n^2 * D) time complexity to build the distance matrix, - where n is the number of samples in x and y and D is the number of - dimensions; each of the P permutations then costs O(n^2) to - re-evaluate the test statistic from the re-indexed matrix. For large - datasets this can get unwieldy, so chunking is recommended. When - chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances - from every sample to a fixed set of landmark samples is computed, so - this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. - The permutation distribution is then obtained by re-indexing (permuting) - the rows of this matrix, which costs O(n * nc) per permutation and - never recomputes distances. PTED remains an exact p-value test even - when chunking, it simply becomes less sensitive to the difference - between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and - ``len(y)``. + The full test builds the ``(n, n)`` distance matrix in ``O(n^2 d)`` + time, where n is the pooled sample size and d the dimension. Each + permutation then costs ``O(n^2)``, though permutations are evaluated in + batches as a single matrix product rather than one at a time. For large + datasets this gets unwieldy, so chunking is recommended: only a + rectangular ``(n, c)`` matrix of distances to ``c`` column points is + built, in ``O(n c d)`` time, and each permutation costs ``O(n c)``. + + Chunking keeps the p-value exact. Permutations are drawn from the + subgroup that holds the column set fixed, shuffling labels within the + columns and within their complement but never between, so the observed + labelling is exchangeable with the permuted ones. The test simply + becomes less sensitive as ``c`` shrinks: the null spread grows like + ``sqrt(n / c)``, so the detectable energy distance scales as + ``(n c)^-0.5`` rather than ``n^-1``. + + The one thing chunking does cost is p-value resolution, and it bites + hardest when one group holds a single point (the per-simulation test + inside ``pted_coverage_test``). There the subgroup reaches only + ``n - c`` distinct label assignments, so the smallest attainable + p-value is about ``1 / (n - c)`` however many permutations are drawn. + Keep ``chunk_size`` well below ``len(s)`` in that case; a + ``PermutationResolutionWarning`` is raised when it is too large. """ assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" assert len(y.shape) >= 2, f"y must be at least 2D, not {y.shape}" - if chunk_size is not None: - assert chunk_size > 0, "chunk_size must be > 0" - # If chunk_size covers both full datasets, chunking adds no benefit - if chunk_size >= len(x) and chunk_size >= len(y): - chunk_size = None assert ( x.shape[1:] == y.shape[1:] ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" @@ -123,18 +145,18 @@ def pted( if len(y.shape) > 2: y = y.reshape(y.shape[0], -1) - if chunk_size is not None: - test, permute = _pted_chunk_impl( - x, - y, - permutations=permutations, - chunk_size=int(chunk_size), - prog_bar=prog_bar, - ) - else: - test, permute = _pted_impl(x, y, permutations=permutations, prog_bar=prog_bar) - - permute = np.array(permute) + # The column controls are resolved once, inside the test itself; a count + # covering the whole pooled sample lands in the exact full-matrix regime. + test, permute = _energy_test( + x, + y, + permutations=permutations, + chunk_size=chunk_size, + n_columns=n_columns, + prog_bar=prog_bar, + batch_size=batch_size, + rng=rng, + ) # Compute p-value if two_tailed: @@ -162,6 +184,9 @@ def pted_coverage_test( pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, + n_columns: Optional[int] = None, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: """ Coverage test using a permutation test on the energy distance (Euclidean). @@ -216,14 +241,14 @@ def pted_coverage_test( return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): If not None, use a landmark-based - rectangular distance matrix to estimate the energy distance - instead of the full pairwise matrix. ``nxc = min(chunk_size, - len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` - samples from y are used as fixed "landmarks"; only distances from - every sample to these landmarks are computed. If ``chunk_size`` - covers both full datasets, this falls back to the exact - (non-chunked) computation. If None, use the full dataset. + chunk_size (Optional[int]): if not None, estimate the energy distance + from a rectangular distance matrix instead of the full pairwise + matrix. Only distances from every sample to ``min(chunk_size, + len(x)) + min(chunk_size, len(y))`` "column" points drawn from the + pooled sample are computed, so the cost drops from ``O(n^2 d)`` to + ``O(n c d)``. If ``chunk_size`` covers both full datasets, this + falls back to the exact full-matrix computation. If None, use the + full dataset. Mutually exclusive with ``n_columns``. sbc_histogram (Optional[str]): If given, the path/filename to save a Simulation-Based-Calibration histogram. sbc_bins (Optional[int]): If given, force the histogram to have the provided @@ -239,23 +264,42 @@ def pted_coverage_test( ``pit_plot`` is not None. prog_bar (bool): If True, show a progress bar to track the progress of simulations. Default is False. + n_columns (Optional[int]): the number of column points ``c`` directly. + This is the preferred arg; ``chunk_size`` is the older one and + simply maps onto it. Mutually exclusive with ``chunk_size``. + batch_size (Optional[int]): number of permutations evaluated per matrix + product. Larger values are faster (especially on GPU) at + ``O(batch_size * n)`` extra memory. None picks a size that keeps a + batch to a few million elements. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state, so ``np.random.seed`` still controls reproducibility. Note ---- - PTED has O(n^2 * D) time complexity to build the distance matrix, - where n is the number of samples in x and y and D is the number of - dimensions; each of the P permutations then costs O(n^2) to - re-evaluate the test statistic from the re-indexed matrix. For large - datasets this can get unwieldy, so chunking is recommended. When - chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances - from every sample to a fixed set of landmark samples is computed, so - this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. - The permutation distribution is then obtained by re-indexing (permuting) - the rows of this matrix, which costs O(n * nc) per permutation and - never recomputes distances. PTED remains an exact p-value test even - when chunking, it simply becomes less sensitive to the difference - between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and - ``len(y)``. + The full test builds the ``(n, n)`` distance matrix in ``O(n^2 d)`` + time, where n is the pooled sample size and d the dimension. Each + permutation then costs ``O(n^2)``, though permutations are evaluated in + batches as a single matrix product rather than one at a time. For large + datasets this gets unwieldy, so chunking is recommended: only a + rectangular ``(n, c)`` matrix of distances to ``c`` column points is + built, in ``O(n c d)`` time, and each permutation costs ``O(n c)``. + + Chunking keeps the p-value exact. Permutations are drawn from the + subgroup that holds the column set fixed, shuffling labels within the + columns and within their complement but never between, so every + normalising constant in the statistic is a constant and the observed + labelling is exchangeable with the permuted ones. The test simply + becomes less sensitive as ``c`` shrinks: the null spread grows like + ``sqrt(n / c)``, so the detectable energy distance scales as + ``(n c)^-0.5`` rather than ``n^-1``. + + The one thing chunking does cost is p-value resolution, and it bites + hardest when one group holds a single point (the per-simulation test + inside ``pted_coverage_test``). There the subgroup reaches only + ``n - c`` distinct label assignments, so the smallest attainable + p-value is about ``1 / (n - c)`` however many permutations are drawn. + Keep ``chunk_size`` well below ``len(s)`` in that case; a + ``PermutationResolutionWarning`` fires when it is too large. """ nsamp, nsim, *_ = s.shape assert nsim > 0, "need some simulations to run test, got 0 simulations" @@ -266,6 +310,11 @@ def pted_coverage_test( s = s.reshape(nsamp, nsim, -1) g = g.reshape(1, nsim, -1) + # Coerce once, so the generator's state advances across simulations. Passing + # a bare seed straight through would hand every simulation the same + # permutation draws. + rng = _as_rng(rng) + test_stats = [] permute_stats = [] pvals = [] @@ -277,6 +326,9 @@ def pted_coverage_test( return_all=True, two_tailed=False, chunk_size=chunk_size, + n_columns=n_columns, + batch_size=batch_size, + rng=rng, ) test_stats.append(test) permute_stats.append(permute) diff --git a/src/pted/utils.py b/src/pted/utils.py index a5be282..5a87fd0 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -1,10 +1,70 @@ -from typing import Optional, Union +"""Backend-agnostic machinery for the permutation test on the energy distance. + +The pooled sample is ``z = [x; y]`` of size ``n = n1 + n2``. Rather than the +full ``n x n`` distance matrix of Szekely & Rizzo (2004), the test may be run +on a rectangular ``n x c`` matrix ``D = cdist(z, z[cols])``, where ``cols`` +("columns", elsewhere called landmarks) indexes ``c`` points drawn from the +pooled sample. That is ``O(n * c)`` distances and ``O(n * c)`` work per +permutation rather than ``O(n^2)``. Setting ``c = n`` recovers the exact +full-matrix test, so both live in one code path. + +Validity +-------- +Permutations are always confined to the subgroup ``S_C x S_{C^c}``: labels are +shuffled within the column positions and within their complement, never across. +Since ``sigma(C) = C`` identically, the column set is invariant and the test is +exact conditional on ``C``. This is what licenses choosing ``C`` by design +(balanced, or aligned with the group structure). The one requirement is that +``C`` must not be chosen using the data values. + +Column allocation +----------------- +Dispatched on ``n_s = min(n1, n2)``, the smaller group: + +``full`` + ``c >= n``. Every point is a column, ``S_C x S_{C^c}`` is the full + symmetric group, and the statistic is the exact energy distance. + +``singleton`` + ``n_s == 1``. All columns come from the large group, so the lone + small-group element sits in ``C^c``. Its within-group mean is identically + zero (a single point has no within-group pairs). The label roams over + ``C^c``: ``n - c`` distinct assignments. + +``small_group_in_C`` + ``n_s <= c // 2``. ``C`` holds ALL of the small group plus ``c - n_s`` from + the large group. Every permuted small group is then a subset of ``C``, so + every within-small-group distance appears in ``D`` and that term is + computed exactly rather than subsampled. + +``proportional`` + Otherwise. ``c_s ~ c * n_s / n``, and both within-group terms are + subsampled. + +Cost +---- +Null spread grows like ``sqrt(n / c)``, so the detectable energy distance +scales as ``(n * c)^{-1/2}`` versus ``n^{-1}`` for the full test: the detection +threshold degrades as one over the square root of the compute. See Janson +(1984) on incomplete U-statistics. + +In the ``singleton`` regime the reference set is ``n - c``, so columns come +straight out of p-value resolution. A single row is nearly sufficient for the +statistic there, so keep ``c`` small -- roughly 5-15% of ``n``. + +Block means exclude the zero self-pairs, making the statistic unbiased for the +population energy distance. It can therefore be negative under H0, like +unbiased distance covariance. Only its rank among the permutations matters. +""" + +from math import comb +from typing import Optional from warnings import warn import numpy as np from scipy.spatial.distance import cdist from scipy.stats import chi2 as chi2_dist, binom, kstwo, kstest -from tqdm.auto import trange +from tqdm.auto import tqdm try: import torch @@ -28,8 +88,9 @@ class torch: __all__ = ( "is_torch_tensor", "is_jax_array", - "pted", - "pted_chunk", + "permutation_energy_test", + "allocate_columns", + "PermutationResolutionWarning", "two_tailed_p", "confidence_alert", "simulation_based_calibration_histogram", @@ -87,14 +148,30 @@ def _to_scalar(x, backend: str) -> float: return float(x) -def _to_backend(x, backend: str): +def _to_numpy(x, backend: str) -> np.ndarray: if backend == "torch": - return torch.as_tensor(x) - if backend == "jax": - return jnp.asarray(x) + return x.detach().cpu().numpy() return np.asarray(x) +def _asarray_like(x, ref, backend: str): + """Move ``x`` onto the device and dtype of ``ref``.""" + if backend == "torch": + return torch.as_tensor(x, dtype=ref.dtype, device=ref.device) + if backend == "jax": + return jnp.asarray(x, dtype=ref.dtype) + return np.asarray(x, dtype=ref.dtype) + + +def _index_like(idx, ref, backend: str): + """Move an integer index array onto the device of ``ref``.""" + if backend == "torch": + return torch.as_tensor(idx, dtype=torch.long, device=ref.device) + if backend == "jax": + return jnp.asarray(idx) + return np.asarray(idx) + + def _random_permutation(D, backend: str = "numpy"): if backend == "torch": I = torch.randperm(D.shape[0], device=D.device) @@ -118,93 +195,403 @@ def _cdist(a, b, backend: str): return cdist(a, b, metric="euclidean") -def _energy_distance_precompute( - D: Union[np.ndarray, torch.Tensor], - nx: int, - ny: int, - nxc: Optional[int] = None, - nyc: Optional[int] = None, -) -> Union[float, torch.Tensor]: - """Energy distance from a rectangular (nx+ny, nxc+nyc) distance matrix. - - If ``D`` is a full (nx+ny, nx+ny) distance matrix, then ``nxc=nx`` and - ``nyc=ny`` and this is the standard energy distance formula. Otherwise, if - ``D`` is a rectangular (nx+ny, nxc+nyc) distance matrix, then ``D`` holds - distances from every sample (rows: ``nx`` of x followed by ``ny`` of y) to a - set of landmark samples (columns: ``nxc`` from x followed by ``nyc`` from - y). Exx/Eyy are estimated from each group against its own landmarks, and Exy - is estimated by averaging the two cross blocks (x-rows vs y-landmarks, and - y-rows vs x-landmarks). +def _zero_self_pairs(D, cols, backend: str): + """Set ``D[cols[k], k] = 0``, the entries pairing a point with itself. + + Called after ``D`` has been shifted by a constant, to restore the property + that a point is at distance zero from itself. Self-pairs are excluded from + every pair count, so leaving them shifted would bias the block means. + + Only positional self-pairs are touched. A genuine duplicate point elsewhere + in the sample is a real pair at distance zero and stays counted. This also + scrubs the ~1e-7 that ``torch.cdist`` leaves on the diagonal once it + switches to its matmul-based algorithm. """ - nxc = nx if nxc is None else nxc - nyc = ny if nyc is None else nyc - Exx = D[:nx, :nxc].sum() / (nx * nxc) - Eyy = D[nx:, nxc:].sum() / (ny * nyc) - Exy = (D[:nx, nxc:].sum() + D[nx:, :nxc].sum()) / (nx * nyc + ny * nxc) - return 2 * Exy - Exx - Eyy + k = np.arange(len(cols)) + if backend == "jax": + return D.at[jnp.asarray(cols), jnp.asarray(k)].set(0.0) + D[_index_like(cols, D, backend), _index_like(k, D, backend)] = 0.0 + return D -def pted( - x, - y, - permutations: int, - prog_bar: bool = False, -) -> tuple[float, list[float]]: - """Permutation-based energy distance test statistic and its permutation distribution. +def _as_rng(rng) -> np.random.Generator: + """Coerce ``rng`` to a numpy ``Generator``. - Works transparently with numpy arrays, torch tensors, and jax arrays. + ``None`` seeds a fresh generator from the legacy global state, so that + ``np.random.seed(...)`` still controls reproducibility for callers who + rely on it. An existing ``Generator`` is returned untouched, so a caller + that has already coerced its own argument can pass the result down without + reseeding anything. """ - if is_torch_tensor(x): - assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" - if is_jax_array(x): - assert jax is not None, "JAX is not installed! try: `pip install jax`" + if isinstance(rng, np.random.Generator): + return rng + if rng is None: + return np.random.default_rng(np.random.randint(0, 2**32)) + return np.random.default_rng(rng) - backend = _backend(x) - z = _concatenate((x, y), backend) - assert _all_finite(z, backend), "Input contains NaN or Inf!" - dmatrix = _cdist(z, z, backend) - assert _all_finite( - dmatrix, backend - ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." - if backend == "jax": # numpy is faster for eager stuff - dmatrix = np.asarray(dmatrix) - backend = "numpy" - nx = len(x) - ny = len(y) - - test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend) - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - dmatrix = _random_permutation(dmatrix, backend) - permute_stats.append(_to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend)) - return test_stat, permute_stats +class PermutationResolutionWarning(UserWarning): + """The permutation group is too small to resolve the requested p-value.""" -def pted_chunk( +# comb(n, k) is astronomically large for balanced samples; we only ever need to +# know whether the reference set is small, so cap it. +_REFERENCE_CAP = 10**15 + +# Target element count for a batch of permutations, bounding peak memory. +_BATCH_ELEMENTS = 2**22 +_MAX_BATCH = 256 + + +def _capped_comb(n: int, k: int) -> int: + if k > 50 and n - k > 50: + return _REFERENCE_CAP + return min(comb(n, k), _REFERENCE_CAP) + + +# --------------------------------------------------------------- allocation + + +def allocate_columns(n1: int, n2: int, n_columns: int, rng=None) -> dict: + """Choose which pooled-sample points supply the columns of ``D``. + + Group 0 is x (pooled indices ``0..n1-1``), group 1 is y (``n1..n-1``). + Selection uses only group sizes and positions, never the data values, which + together with the restricted permutation group is what keeps the test exact. + + Parameters + ---------- + n1 (int): number of samples in x. + n2 (int): number of samples in y. + n_columns (int): number of column points ``c``. ``c >= n1 + n2`` + requests the exact full-matrix test. + rng: seed, ``np.random.Generator``, or None (draw from the global state). + + Returns + ------- + dict with keys ``cols``, ``regime``, ``small_idx`` (pooled indices + of the smaller group), ``n_small``, ``n_large``, ``c_small``, + ``c_large``, ``n_columns``, + ``reference_size`` (distinct label assignments the subgroup reaches) + and ``exact_within`` (whether the small group's within-group term is + computed exactly rather than subsampled). + """ + rng = _as_rng(rng) + n = n1 + n2 + n_columns = int(n_columns) + if min(n1, n2) < 1: + raise ValueError(f"both samples need at least one point, got n1={n1}, n2={n2}") + if n_columns < 2: + raise ValueError(f"need at least 2 columns, got n_columns={n_columns}") + if n_columns > n: + raise ValueError(f"n_columns={n_columns} exceeds the pooled sample size {n}") + + n_small, n_large = min(n1, n2), max(n1, n2) + x_is_small = n1 <= n2 + small_idx = np.arange(n1) if x_is_small else n1 + np.arange(n2) + large_idx = n1 + np.arange(n2) if x_is_small else np.arange(n1) + + if n_columns == n: + # C is everything, so S_C x S_{C^c} is the full symmetric group and + # this is the classical exact permutation test. + cols = np.arange(n) + regime, c_small = "full", n_small + elif n_small == 1: + # Keep the singleton OUT of C so its label can range over C^c, which is + # the larger of the two options whenever c < n / 2. Reaching here means + # c < n, and n_large == n - 1, so C fits inside the large group. + cols = rng.choice(large_idx, size=n_columns, replace=False) + regime, c_small = "singleton", 0 + elif n_small <= n_columns // 2: + # Put the whole small group in C so that every permuted small group is + # still a subset of C and its within-group distances are all present. + cols = np.concatenate( + [small_idx, rng.choice(large_idx, size=n_columns - n_small, replace=False)] + ) + regime, c_small = "small_group_in_C", n_small + else: + lo = max(1, n_columns - n_large) + hi = min(n_small, n_columns - 1) + c_small = int(np.clip(round(n_columns * n_small / n), lo, hi)) + cols = np.concatenate( + [ + rng.choice(small_idx, size=c_small, replace=False), + rng.choice(large_idx, size=n_columns - c_small, replace=False), + ] + ) + regime = "proportional" + + # The subgroup places c_small of the small labels among the c columns and + # the rest among the n - c remaining positions, independently. + reference = min( + _capped_comb(n_columns, c_small) * _capped_comb(n - n_columns, n_small - c_small), + _REFERENCE_CAP, + ) + + return { + "cols": np.sort(cols), + "regime": regime, + "small_idx": small_idx, + "n_small": n_small, + "n_large": n_large, + "c_small": c_small, + "c_large": n_columns - c_small, + "n_columns": n_columns, + "reference_size": reference, + "exact_within": c_small == n_small or n_small == 1, + } + + +def _warn_reference_size(alloc: dict, permutations: int) -> None: + """Warn when the permutation group cannot resolve the requested p-value. + + The observed labelling is itself a member of the group, so with only ``R`` + reachable assignments roughly one draw in ``R`` reproduces it exactly and + the p-value is floored near ``1/R`` however many permutations are drawn. + """ + reference = alloc["reference_size"] + # In the full regime the reference set is fixed by the sample sizes alone, + # so only flag it when it is degenerate. Everywhere else it is a + # consequence of the column count, which the caller can act on. + floor = 20 if alloc["regime"] == "full" else max(20, (permutations + 1) // 10) + if reference >= floor: + return + + hint = { + "singleton": ( + f"the small group holds a single point, so the reference set is " + f"n - n_columns; use fewer columns (currently {alloc['n_columns']})" + ), + "full": "the samples are too small for a permutation test at this resolution", + }.get( + alloc["regime"], + f"use more columns (currently {alloc['n_columns']}) or more samples", + ) + warn( + PermutationResolutionWarning( + f"the {alloc['regime']!r} column allocation reaches only {reference:,} distinct " + f"label assignments, so the smallest attainable p-value is about " + f"{1.0 / reference:.3g} no matter how many permutations are drawn " + f"({permutations} requested): {hint}." + ) + ) + + +def _draw_labels(base_small, idx_c, idx_o, n_draws: int, rngs) -> np.ndarray: + """``(n_draws, n)`` indicator of small-group membership. + + Drawn uniformly from the subgroup ``S_C x S_{C^c}`` by shuffling the labels + within the column positions ``idx_c`` and within their complement ``idx_o``, + never between. Permutation bookkeeping stays in numpy: it is pure integer + shuffling, and only the resulting indicator matrix crosses to the compute + backend. + + The two parts draw from separate generators in ``rngs``. ``permuted`` + consumes the stream row by row, so changing ``batch_size`` doesn't affect + the sequence of permutations. + """ + n = base_small.size + U = np.empty((n_draws, n), dtype=base_small.dtype) + for idx, rng in zip((idx_c, idx_o), rngs): + if idx.size == 0: + continue + block = rng.permuted(np.repeat(base_small[idx][None, :], n_draws, axis=0), axis=1) + if idx.size == n: + U[:] = block + else: + U[:, idx] = block + return U + + +# ---------------------------------------------------------------- statistic + + +def _prepare_statistic(D, alloc: dict, backend: str) -> dict: + """Precompute everything that is constant across permutations. + + Every block sum is a bilinear form in the row indicator ``u`` (length n) + and the column indicator ``v = u[cols]`` (length c): + + u @ D @ v = sum_{i in A} sum_{k: cols[k] in A} D[i, k] + + Only that one product is needed per permutation; the other three blocks + follow from the row sums, column sums and grand total, since the two + indicators of each pair sum to one. All denominators are constants, because + the subgroup ``S_C x S_{C^c}`` preserves the per-group column counts. + """ + n_s, n_l = alloc["n_small"], alloc["n_large"] + n = n_s + n_l + c_s, c_l = alloc["c_small"], alloc["c_large"] + cols = alloc["cols"] + + # allocate_columns guarantees c_l >= 1 in every regime, and c_s >= 1 + # unless n_s == 1 (the singleton regime, where the small group has no + # within-group pairs to estimate), so both block means below are defined. + + # The statistic is invariant to a constant offset on every non-self pair, + # so centre D before any reduction. In high dimension the pairwise + # distances concentrate far from zero, and the uncentred sums spend most of + # their significant digits on an offset that cancels anyway. Any constant + # cancels exactly, so the offset itself needs no precision -- a float32 + # mean is fine. That is not true of `total` below, whose error survives + # into the reported statistic, so that one accumulates in float64. + offset = _to_scalar(D.mean(), backend) + D = D - offset + D = _zero_self_pairs(D, cols, backend) + + row_sums = D.sum(1) + base_small = np.zeros(n) + base_small[alloc["small_idx"]] = 1.0 + in_c = np.zeros(n, dtype=bool) + in_c[cols] = True + + return { + "D": D, + # ``cols`` covers everything in the full regime, so skip the gather. + "cols": None if alloc["regime"] == "full" else _index_like(cols, D, backend), + "offset": offset, + "row_sums": row_sums, + "col_sums": D.sum(0), + "total": float(_to_numpy(row_sums, backend).sum(dtype=np.float64)), + "n": n, + "n_s": n_s, + "n_l": n_l, + "c_s": c_s, + "c_l": c_l, + # Column point cols[k] is also row cols[k], so each within-group block + # holds exactly c_g zero-valued self-pairs; drop them from the counts. + "pairs_ss": c_s * (n_s - 1), + "pairs_ll": c_l * (n_l - 1), + "base_small": base_small, + "idx_c": np.flatnonzero(in_c), + "idx_o": np.flatnonzero(~in_c), + "backend": backend, + } + + +def _evaluate_statistic(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of label assignments. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_statistic`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. + """ + backend = prep["backend"] + D = prep["D"] + + Us = _asarray_like(U, D, backend) + Vs = Us if prep["cols"] is None else Us[:, prep["cols"]] + + S_ss = ((Us @ D) * Vs).sum(-1) # small rows, small cols + S_sl = Us @ prep["row_sums"] - S_ss # small rows, large cols + S_ls = Vs @ prep["col_sums"] - S_ss # large rows, small cols + S_ll = prep["total"] - S_ss - S_sl - S_ls # large rows, large cols + + n_s, n_l = prep["n_s"], prep["n_l"] + c_s, c_l = prep["c_s"], prep["c_l"] + + # A group of one point has no within-group pairs, so its mean is zero on + # the raw distances -- which is -offset on the centred ones. Writing it + # that way keeps the 2 - 1 - 1 cancellation that makes the statistic + # independent of the centring, and so keeps the reported value equal to + # the uncentred statistic rather than merely rank-equivalent to it. + off = prep["offset"] + mu_ss = -off if n_s < 2 else S_ss / prep["pairs_ss"] + mu_ll = -off if n_l < 2 else S_ll / prep["pairs_ll"] + + # Cross term: mean over whichever orientations have columns available. + parts = [S_sl / (n_s * c_l)] + if c_s > 0: + parts.append(S_ls / (n_l * c_s)) + mu_sl = sum(parts) / len(parts) + + stat = (n_s * n_l / prep["n"]) * (2.0 * mu_sl - mu_ss - mu_ll) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +# --------------------------------------------------------------- public API + + +def _resolve_n_columns( + n1: int, n2: int, chunk_size: Optional[int] = None, n_columns: Optional[int] = None +) -> int: + """Map the user-facing column controls onto a single column count ``c``. + + ``n_columns`` is ``c`` directly. ``chunk_size`` is the legacy control and + maps to ``min(chunk_size, n1) + min(chunk_size, n2)``, which reproduces the + number of distance columns (and hence the compute cost) it used to request. + Neither one given means the exact full-matrix test. + + Only the arithmetic lives here. Whether the resulting count is usable is + :func:`allocate_columns`' business, so a nonsense ``n_columns`` is passed + through untouched and rejected there. + """ + n = n1 + n2 + if chunk_size is not None and n_columns is not None: + raise ValueError("give either chunk_size or n_columns, not both") + if n_columns is not None: + c = int(n_columns) + elif chunk_size is not None: + c = int(chunk_size) + if c < 1: + raise ValueError(f"chunk_size must be positive, got {c}") + c = min(c, n1) + min(c, n2) + else: + c = n + return min(c, n) + + +def permutation_energy_test( x, y, permutations: int, - chunk_size: int, + chunk_size: Optional[int] = None, + n_columns: Optional[int] = None, prog_bar: bool = False, -) -> tuple[float, list[float]]: - """Chunked variant of `pted` for large datasets. - - Rather than the full ``(nx+ny, nx+ny)`` pairwise distance matrix, this - builds a rectangular ``(nx+ny, nxc+nyc)`` matrix of distances from every - sample to a set of randomly chosen "landmark" samples, where ``nxc = - min(chunk_size, nx)`` landmarks are drawn from ``x`` and ``nyc = - min(chunk_size, ny)`` from ``y``. This matrix is computed only once; the - permutation distribution is then obtained by re-indexing (permuting) its - rows, which reassigns samples to the x/y groups without recomputing any - distances. Each landmark's row position is tracked and carried through the - same row permutation, so after every shuffle the landmarks are re-split into - "x-side"/"y-side" columns according to where they now fall, keeping rows and - columns consistent. In pathological cases where too few landmarks are - permuted onto either side (zero or one tenth of the original number of - landmarks), a new permutation is drawn. + batch_size: Optional[int] = None, + rng=None, +) -> tuple[float, np.ndarray]: + """Observed energy statistic and its permutation distribution. + + With neither ``chunk_size`` nor ``n_columns`` this is the exact + full-matrix test: the ``(n, n)`` distance matrix is built once and the + statistic is evaluated for batches of permutations as a bilinear form in + the group indicator, so no distance is recomputed and no matrix is + re-indexed. Asking for fewer columns switches to the rectangular + ``(n, c)`` matrix, cutting the cost from ``O(n^2 d)`` to ``O(n c d)``; + permutations are then confined to the subgroup that fixes the column set, + which keeps the p-value exact. See the module docstring for the column + allocation regimes and what they cost in p-value resolution. Works transparently with numpy arrays, torch tensors, and jax arrays. + Distances and all matrix algebra stay on the backend that ``x`` and ``y`` + live on; only the label bookkeeping and the resulting statistics touch + numpy. + + Parameters + ---------- + x, y: samples, shape ``(n1, d)`` and ``(n2, d)``. + permutations (int): number of permutations to draw. + chunk_size (Optional[int]): legacy control, equivalent to + ``n_columns = min(chunk_size, n1) + min(chunk_size, n2)``. + n_columns (Optional[int]): number of distance columns ``c``, the + preferred control. Mutually exclusive with ``chunk_size``. + prog_bar (bool): show a progress bar over permutations. + batch_size (Optional[int]): permutations evaluated per matrix product. + Larger is faster on GPU, at ``O(batch_size * n)`` extra memory. + None picks a size that bounds the batch at a few million elements. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state (so ``np.random.seed`` still applies). + + Returns + ------- + ``(test_stat, permute_stats)``, the observed statistic and a + ``(permutations,)`` array of permuted statistics. """ if is_torch_tensor(x): assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" @@ -214,45 +601,43 @@ def pted_chunk( backend = _backend(x) z = _concatenate((x, y), backend) assert _all_finite(z, backend), "Input contains NaN or Inf!" - nx = len(x) - ny = len(y) - nxc = min(chunk_size, nx) - nyc = min(chunk_size, ny) - landmark_pos = np.sort( - np.concatenate( - [ - np.random.choice(nx, size=nxc, replace=False), - nx + np.random.choice(ny, size=nyc, replace=False), - ] - ) + n1, n2 = len(x), len(y) + n = n1 + n2 + c = _resolve_n_columns(n1, n2, chunk_size=chunk_size, n_columns=n_columns) + + rng = _as_rng(rng) + alloc = allocate_columns(n1, n2, c, rng) + _warn_reference_size(alloc, permutations) + + cols = alloc["cols"] + zc = z if alloc["regime"] == "full" else z[_index_like(cols, z, backend)] + dmatrix = _cdist(z, zc, backend) + assert _all_finite(dmatrix, backend), ( + "Distance matrix contains NaN or Inf! Consider normalizing values to be " + "more stable (i.e. z-score norm)." ) - dmatrix = _cdist(z, z[_to_backend(landmark_pos, backend)], backend) - assert _all_finite( - dmatrix, backend - ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." - if backend == "jax": # numpy is faster for eager stuff - dmatrix = np.asarray(dmatrix) - backend = "numpy" - - test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny, nxc, nyc), backend) - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - while True: - I = np.random.permutation(len(z)) - # Track where each landmark moved to, then re-split columns by their new x/y side. - landmark_pos = np.argsort(I)[landmark_pos] - order = np.argsort(landmark_pos >= nx, kind="stable") - dmatrix_i = dmatrix[_to_backend(I, backend)][:, _to_backend(order, backend)] - landmark_pos = landmark_pos[order] - nxc_i = int(np.sum(landmark_pos < nx)) - nyc_i = nxc + nyc - nxc_i - if nxc_i < max(1, (nxc * 0.1)) or nyc_i < max(1, (nyc * 0.1)): - continue # Skip this permutation if too few landmarks remain on either side, to avoid unstable estimates. - permute_stats.append( - _to_scalar(_energy_distance_precompute(dmatrix_i, nx, ny, nxc_i, nyc_i), backend) - ) - break + + prep = _prepare_statistic(dmatrix, alloc, backend) + + test_stat = float(_evaluate_statistic(prep, prep["base_small"][None, :])[0]) + assert np.isfinite(test_stat), "Observed statistic is not finite!" + + if batch_size is None: + batch_size = min(_MAX_BATCH, max(1, _BATCH_ELEMENTS // max(n, c))) + batch_size = max(1, int(batch_size)) + + permute_stats = np.empty(permutations, dtype=np.float64) + label_rngs = rng.spawn(2) + done = 0 + with tqdm(total=permutations, disable=not prog_bar) as bar: + while done < permutations: + size = min(batch_size, permutations - done) + U = _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], size, label_rngs) + permute_stats[done : done + size] = _evaluate_statistic(prep, U) + done += size + bar.update(size) + return test_stat, permute_stats diff --git a/tests/test_pted.py b/tests/test_pted.py index 5acb823..ea5c35a 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -249,37 +249,41 @@ def test_is_jax_array_no_jax(monkeypatch): def test_pted_jax_no_jax(monkeypatch): - """pted raises AssertionError when a jax array is passed but JAX is not installed.""" + """permutation_energy_test raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) + pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) def test_pted_chunk_jax_no_jax(monkeypatch): - """pted_chunk raises AssertionError when a jax array is passed but JAX is not installed.""" + """permutation_energy_test raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) + pted.utils.permutation_energy_test( + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2 + ) def test_pted_torch_no_torch(monkeypatch): - """pted raises AssertionError when a torch tensor is passed but torch is not installed.""" + """permutation_energy_test raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) + pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) def test_pted_chunk_torch_no_torch(monkeypatch): - """pted_chunk raises AssertionError when a torch tensor is passed but torch is not installed.""" + """permutation_energy_test raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) + pted.utils.permutation_energy_test( + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2 + ) @pytest.mark.parametrize("backend", ["torch", "jax"]) @@ -297,3 +301,131 @@ def test_cdist_matches_scipy(backend): pted.utils._cdist(_to_backend(x_np, backend), _to_backend(y_np, backend), backend=backend) ) assert np.allclose(got, expected, rtol=1e-5) + + +# --------------------------------------------------------------------------- +# The new column/batching controls +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_n_columns(backend): + """n_columns names the column count directly, and agrees with the + chunk_size that maps onto the same value.""" + _require_backend(backend) + np.random.seed(11) + x = _to_backend(np.random.normal(size=(300, 6)), backend) + y = _to_backend(np.random.normal(size=(300, 6)), backend) + + # chunk_size=40 on two equal groups requests 40 + 40 columns + p_chunk = pted.pted(x, y, chunk_size=40, rng=0) + p_cols = pted.pted(x, y, n_columns=80, rng=0) + assert p_chunk == p_cols + + y_diff = _to_backend(np.random.uniform(size=(300, 6)), backend) + assert pted.pted(x, y_diff, n_columns=80) < 1e-2 + + # n_columns covering the pooled sample is just the full test + assert pted.pted(x, y, n_columns=10_000, rng=3) == pted.pted(x, y, rng=3) + + +def test_pted_column_args_are_exclusive(): + x = np.random.normal(size=(20, 3)) + y = np.random.normal(size=(20, 3)) + with pytest.raises(ValueError, match="not both"): + pted.pted(x, y, chunk_size=5, n_columns=10) + + +def test_pted_rng_is_reproducible(): + """An explicit rng pins the permutations; the global seed still works when + none is given.""" + x = np.random.normal(size=(60, 4)) + y = np.random.normal(size=(60, 4)) + + a = pted.pted(x, y, permutations=200, n_columns=30, rng=7, return_all=True) + b = pted.pted(x, y, permutations=200, n_columns=30, rng=7, return_all=True) + assert a[0] == b[0] and np.array_equal(a[1], b[1]) and a[2] == b[2] + + np.random.seed(4) + c = pted.pted(x, y, permutations=200, n_columns=30, return_all=True) + np.random.seed(4) + d = pted.pted(x, y, permutations=200, n_columns=30, return_all=True) + assert np.array_equal(c[1], d[1]), "np.random.seed should still pin the permutations" + + +@pytest.mark.parametrize( + "n1,n2,n_columns,regime", + [ + (60, 60, 24, "proportional"), + (1, 100, 11, "singleton"), + ], +) +def test_chunked_pvalues_are_calibrated(n1, n2, n_columns, regime): + """Type-I error under H0 must match the nominal rate. + + This is the property the chunked test exists to preserve, and the one the + earlier landmark-reshuffling scheme lost: because it re-split the columns + by group after each permutation while the observed statistic always used a + perfectly balanced split, the observed value was not exchangeable with the + permuted ones. It rejected at 29% (proportional) and 84% (singleton) for a + nominal 5%. Fixed seed, so this is deterministic rather than flaky. + """ + from pted.utils import allocate_columns + + assert allocate_columns(n1, n2, n_columns, rng=0)["regime"] == regime + + trials, permutations = 600, 49 + rng = np.random.default_rng(20240904) + pvals = np.empty(trials) + for t in range(trials): + # H0 is true: both groups come from the same distribution + x = rng.standard_normal((n1, 4)) + y = rng.standard_normal((n2, 4)) + pvals[t] = pted.pted( + x, y, permutations=permutations, n_columns=n_columns, two_tailed=False, rng=rng + ) + + for nominal, tol in ((0.10, 0.05), (0.20, 0.06)): + rate = float(np.mean(pvals <= nominal)) + assert ( + abs(rate - nominal) < tol + ), f"{regime}: rejected {rate:.3f} of {trials} null trials at nominal {nominal}" + + +def test_coverage_test_rng_is_not_shared_across_simulations(): + """A bare seed must not hand every simulation the same permutation draws, + while the run as a whole stays reproducible from that seed.""" + g = np.random.default_rng(0).standard_normal((6, 3)) + s = np.random.default_rng(1).standard_normal((40, 6, 3)) + + _, permute, _ = pted.pted_coverage_test(g, s, permutations=50, rng=7, return_all=True) + assert not any(np.array_equal(permute[0], permute[i]) for i in range(1, len(permute))) + + _, again, _ = pted.pted_coverage_test(g, s, permutations=50, rng=7, return_all=True) + assert np.array_equal(permute, again) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_coverage_test_with_chunking(backend): + """Chunked coverage runs end to end and still separates a calibrated + posterior from an over/under-confident one.""" + _require_backend(backend) + rng = np.random.default_rng(3) + nsim, nsamp, d = 40, 300, 3 + g, draws = [], [] + for _ in range(nsim): + loc = rng.standard_normal(d) * 5 + scale = rng.uniform(1, 4, size=d) + g.append(rng.normal(loc, scale, size=d)) + draws.append(rng.normal(loc, scale, size=(nsamp, d))) + g = _to_backend(np.array(g), backend) + ok = _to_backend(np.stack(draws, axis=1), backend) + over = _to_backend( + np.stack([v.mean(0) + (v - v.mean(0)) * 0.4 for v in draws], axis=1), backend + ) + + p_ok = pted.pted_coverage_test(g, ok, permutations=199, chunk_size=30) + assert 1e-2 < p_ok < 0.99, f"calibrated posterior gave p={p_ok}" + + p_over = pted.pted_coverage_test(g, over, permutations=199, chunk_size=30, warn_confidence=None) + assert p_over < 1e-2, f"overconfident posterior gave p={p_over}" diff --git a/tests/test_utils.py b/tests/test_utils.py index aac3f3a..541f0d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,26 @@ import sys, os +import warnings +from math import comb import numpy as np +from scipy.spatial.distance import cdist from pted.utils import ( + PermutationResolutionWarning, + allocate_columns, two_tailed_p, simulation_based_calibration_histogram, pit_plot, hdp_coverage_test, - _to_scalar, + permutation_energy_test, + _cdist, + _draw_labels, + _evaluate_statistic, + _index_like, + _prepare_statistic, _random_permutation, + _resolve_n_columns, + _to_scalar, ) try: @@ -26,6 +38,13 @@ import pytest +def _require_backend(backend): + if backend == "torch" and torch is None: + pytest.skip("torch not installed") + if backend == "jax" and jax is None: + pytest.skip("jax not installed") + + def test_two_tailed_p(): # assert np.isclose(two_tailed_p(4, 6), 1.0), "p-value at mode should be 1.0" @@ -118,3 +137,255 @@ def test_random_permutation(backend): assert set(np.array(permuted_D.flatten()).tolist()) == set( np.array(D.flatten()).tolist() ), "Permutation should contain all original elements" + + +# --------------------------------------------------------------------------- +# Rectangular / batched permutation backend +# --------------------------------------------------------------------------- + + +def _unbiased_energy(x, y): + """Energy statistic from first principles, within-group means over + distinct pairs only.""" + n1, n2 = len(x), len(y) + dxx, dyy = cdist(x, x), cdist(y, y) + exx = (dxx.sum() - np.trace(dxx)) / (n1 * (n1 - 1)) if n1 > 1 else 0.0 + eyy = (dyy.sum() - np.trace(dyy)) / (n2 * (n2 - 1)) if n2 > 1 else 0.0 + return (n1 * n2 / (n1 + n2)) * (2 * cdist(x, y).mean() - exx - eyy) + + +def _brute_block_means(z, small_indicator, cols, n): + """Energy statistic by explicit block sums over the rectangular matrix.""" + D = cdist(z, z[cols]) + is_s = small_indicator.astype(bool) + rows_s, rows_l = np.flatnonzero(is_s), np.flatnonzero(~is_s) + ks, kl = np.flatnonzero(is_s[cols]), np.flatnonzero(~is_s[cols]) + n_s, n_l = len(rows_s), len(rows_l) + + def block(rows, kk, drop_self): + tot = cnt = 0 + for i in rows: + for k in kk: + if drop_self and cols[k] == i: + continue + tot += D[i, k] + cnt += 1 + return tot, cnt + + s_ss, c_ss = block(rows_s, ks, True) + s_ll, c_ll = block(rows_l, kl, True) + s_sl, c_sl = block(rows_s, kl, False) + s_ls, c_ls = block(rows_l, ks, False) + mu_ss = 0.0 if c_ss == 0 else s_ss / c_ss + mu_ll = 0.0 if c_ll == 0 else s_ll / c_ll + parts = [s_sl / c_sl] + ([s_ls / c_ls] if c_ls else []) + return (n_s * n_l / n) * (2 * sum(parts) / len(parts) - mu_ss - mu_ll) + + +ALLOCATIONS = [ + # n1, n2, n_columns, expected regime + (100, 100, 200, "full"), + (3, 40, 43, "full"), + (1, 60, 12, "singleton"), + (6, 80, 40, "small_group_in_C"), + (40, 50, 30, "proportional"), + (40, 50, 3, "proportional"), +] + + +@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) +def test_allocate_columns_regimes(n1, n2, c, regime): + """Each size combination lands in the intended regime with a consistent + set of columns.""" + alloc = allocate_columns(n1, n2, c, rng=0) + cols = alloc["cols"] + n = n1 + n2 + + assert alloc["regime"] == regime + assert cols.shape == (c,) + assert len(np.unique(cols)) == c, "columns must be distinct points" + assert cols.min() >= 0 and cols.max() < n + assert np.all(np.diff(cols) > 0), "columns are returned sorted" + + # c_small must be the number of columns actually belonging to the small group + assert np.sum(np.isin(cols, alloc["small_idx"])) == alloc["c_small"] + assert alloc["c_small"] + alloc["c_large"] == c + assert alloc["c_large"] >= 1, "the large group must keep at least one column" + assert alloc["exact_within"] == (alloc["c_small"] == alloc["n_small"] or alloc["n_small"] == 1) + + +def test_allocate_columns_reference_size(): + """reference_size counts the label assignments the subgroup can reach.""" + for n1, n2, c, _ in ALLOCATIONS: + alloc = allocate_columns(n1, n2, c, rng=1) + n, n_s, c_s = n1 + n2, alloc["n_small"], alloc["c_small"] + expected = comb(c, c_s) * comb(n - c, n_s - c_s) + assert alloc["reference_size"] == min(expected, 10**15) + + +def test_allocate_columns_errors(): + """allocate_columns is the single gate on sample and column counts.""" + with pytest.raises(ValueError, match="at least 2 columns"): + allocate_columns(10, 10, 1) + with pytest.raises(ValueError, match="exceeds the pooled sample size"): + allocate_columns(10, 10, 21) + with pytest.raises(ValueError, match="both samples need at least one point"): + allocate_columns(0, 10, 5) + # and it is reached through the public API, not bypassed by the mapping + with pytest.raises(ValueError, match="at least 2 columns"): + permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=1, n_columns=1) + with pytest.raises(ValueError, match="both samples need at least one point"): + permutation_energy_test(np.zeros((0, 2)), np.zeros((5, 2)), permutations=1) + + +@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) +def test_draw_labels_stays_in_subgroup(n1, n2, c, regime): + """Permutations never move a label across the C / C^c boundary, so the + per-group column counts -- and hence every normalising constant -- are + fixed. This is what makes the observed labelling exchangeable with the + permuted ones.""" + rng = np.random.default_rng(3) + alloc = allocate_columns(n1, n2, c, rng) + n = n1 + n2 + base = np.zeros(n) + base[alloc["small_idx"]] = 1.0 + in_c = np.zeros(n, bool) + in_c[alloc["cols"]] = True + + U = _draw_labels(base, np.flatnonzero(in_c), np.flatnonzero(~in_c), 200, rng.spawn(2)) + + assert np.all(np.isin(U, [0.0, 1.0])) + assert np.all(U.sum(1) == alloc["n_small"]), "small group size is preserved" + assert np.all(U[:, alloc["cols"]].sum(1) == alloc["c_small"]), "column counts are preserved" + # and the labels really do move around within each part + if alloc["reference_size"] > 100: + assert len(np.unique(U, axis=0)) > 1 + + +@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) +def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): + """The bilinear-form shortcut agrees with explicit block sums, for the + observed labelling and for permuted ones.""" + rng = np.random.default_rng(7) + x = rng.standard_normal((n1, 4)) + y = rng.standard_normal((n2, 4)) + 0.4 + z = np.vstack([x, y]) + + alloc = allocate_columns(n1, n2, c, rng) + D = cdist(z, z[alloc["cols"]]) + prep = _prepare_statistic(D, alloc, "numpy") + + U = np.concatenate( + [ + prep["base_small"][None, :], + _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], 5, rng.spawn(2)), + ] + ) + got = _evaluate_statistic(prep, U) + expect = [_brute_block_means(z, U[b], alloc["cols"], n1 + n2) for b in range(len(U))] + assert np.allclose(got, expect, atol=1e-9) + + +@pytest.mark.parametrize("n1,n2", [(30, 30), (7, 23), (1, 40), (2, 2)]) +def test_full_matrix_is_the_energy_distance(n1, n2): + """With every point a column, the statistic is exactly the energy + distance with within-group means over distinct pairs.""" + rng = np.random.default_rng(11) + x = rng.standard_normal((n1, 3)) + y = rng.standard_normal((n2, 3)) + 0.6 + test_stat, _ = permutation_energy_test(x, y, permutations=0) + assert np.isclose(test_stat, _unbiased_energy(x, y), atol=1e-9) + + +@pytest.mark.parametrize("backend", ["torch", "jax"]) +def test_statistic_backend_agreement(backend): + """numpy, torch and jax evaluate the same statistic for the same labels.""" + _require_backend(backend) + rng = np.random.default_rng(5) + n1, n2, c = 40, 60, 30 + x = rng.standard_normal((n1, 5)) + y = rng.standard_normal((n2, 5)) + 0.3 + z = np.vstack([x, y]) + + alloc = allocate_columns(n1, n2, c, rng) + U = None + results = {} + for be in ("numpy", backend): + zb = {"numpy": lambda: z, "torch": lambda: torch.tensor(z), "jax": lambda: jnp.array(z)}[ + be + ]() + D = _cdist(zb, zb[_index_like(alloc["cols"], zb, be)], be) + prep = _prepare_statistic(D, alloc, be) + if U is None: + U = np.concatenate( + [ + prep["base_small"][None, :], + _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], 8, rng.spawn(2)), + ] + ) + results[be] = _evaluate_statistic(prep, U) + # jax defaults to float32, so compare with a float32-appropriate tolerance + assert np.allclose(results["numpy"], results[backend], rtol=1e-4, atol=1e-4) + + +def test_draw_labels_independent_of_batch_size(): + """The two label streams are consumed row by row, so splitting a run into + batches yields bit-identical permutations.""" + base = np.r_[np.ones(5), np.zeros(15)] + idx_c, idx_o = np.arange(12), np.arange(12, 20) + whole = _draw_labels(base, idx_c, idx_o, 10, np.random.default_rng(9).spawn(2)) + rngs = np.random.default_rng(9).spawn(2) + split = np.concatenate([_draw_labels(base, idx_c, idx_o, k, rngs) for k in (3, 3, 4)]) + assert np.array_equal(whole, split) + + +def test_statistic_independent_of_batch_size(): + """Batching is purely a memory/throughput knob: with the same seed it draws + the same permutations, so the statistics agree to within the reordering + that a different matmul shape costs in floating point.""" + rng_kwargs = dict(permutations=64, n_columns=30, rng=12345) + x = np.random.default_rng(0).standard_normal((50, 4)) + y = np.random.default_rng(1).standard_normal((70, 4)) + ref_stat, ref_perm = permutation_energy_test(x, y, batch_size=1, **rng_kwargs) + for batch in (7, 64, 1000): + stat, perm = permutation_energy_test(x, y, batch_size=batch, **rng_kwargs) + assert stat == ref_stat + assert np.allclose( + perm, ref_perm, rtol=1e-10, atol=1e-12 + ), f"batch_size={batch} changed the null draws" + + +def test_resolve_n_columns(): + # chunk_size reproduces the number of distance columns it used to request + assert _resolve_n_columns(1000, 1000, chunk_size=100) == 200 + assert _resolve_n_columns(200, 30, chunk_size=50) == 80 + # covering both datasets falls back to the full matrix + assert _resolve_n_columns(20, 30, chunk_size=40) == 50 + # n_columns is taken literally, capped at the pooled size + assert _resolve_n_columns(20, 30, n_columns=17) == 17 + assert _resolve_n_columns(20, 30, n_columns=500) == 50 + # neither means the full matrix + assert _resolve_n_columns(20, 30) == 50 + + with pytest.raises(ValueError, match="not both"): + _resolve_n_columns(20, 30, chunk_size=5, n_columns=5) + with pytest.raises(ValueError, match="chunk_size must be positive"): + _resolve_n_columns(20, 30, chunk_size=0) + # a nonsense n_columns passes through here and is rejected by + # allocate_columns, which owns every column-count check + assert _resolve_n_columns(20, 30, n_columns=1) == 1 + + +def test_permutation_resolution_warning(): + """Too many columns starves the singleton regime of distinct assignments.""" + rng = np.random.default_rng(2) + x = rng.standard_normal((1, 3)) + y = rng.standard_normal((60, 3)) + + with pytest.warns(PermutationResolutionWarning, match="distinct label assignments"): + permutation_energy_test(x, y, permutations=100, n_columns=55) + + # a sensible column count is quiet + with warnings.catch_warnings(): + warnings.simplefilter("error", PermutationResolutionWarning) + permutation_energy_test(x, y, permutations=100, n_columns=8) From 5ca9b63d63d00c80228de8082c75babb6352b8b7 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Fri, 4 Sep 2026 19:36:40 -0400 Subject: [PATCH 2/7] aligning docs minor change to guard edge case in singleton --- README.md | 42 +++++++++++++++++++++++++++++++++++++----- src/pted/pted.py | 22 ++++++++++++---------- src/pted/utils.py | 39 ++++++++++++++++++++++++++------------- tests/test_pted.py | 1 + tests/test_utils.py | 40 +++++++++++++++++++++++++++++++++------- 5 files changed, 109 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index d808013..bffb0a1 100644 --- a/README.md +++ b/README.md @@ -440,11 +440,43 @@ The detection threshold degrades as one over the square root of the compute. You also give up some p-value resolution, and this is most significant when one group holds a single point — which is exactly the per-simulation test inside -`pted_coverage_test`. There the permutation subgroup only reaches `n - c` -distinct label assignments, so the smallest attainable p-value is about `1 / (n -- c)` no matter how many permutations you draw. Keep `chunk_size` well below the -number of posterior samples in that case; PTED raises a -`PermutationResolutionWarning` when the column count is too greedy. +`pted_coverage_test`. There the lone point can sit inside `C`, its label roaming +over the `c` column positions, or outside it, roaming over the other `n - c`, so +the subgroup reaches `max(c, n - c)` distinct label assignments. PTED puts the +point on whichever side is larger, which holds the reference set at half the +pooled sample or better. The smallest attainable p-value is still about the +reciprocal of that, however many permutations you draw, so PTED raises a +`PermutationResolutionWarning` when the reachable set is too small to resolve +the p-value you asked for. + +### Does restricting the permutations still measure the energy distance? + +Yes — the restriction is on how the statistic is *calibrated*, not on what it +measures. Every block mean still averages genuine distances between genuine +members of the two groups: the cross term pairs **all** `n_x` samples against the +large-group columns and **all** `n_y` samples against the small-group columns, so +no sample is demoted to a mere marker. Each block mean is an unbiased estimate of +exactly the population quantity the full test estimates, which makes the whole +statistic an unbiased estimator of the population energy distance — an +*incomplete U-statistic* in the sense of Janson (1984), averaging over a subset +of the pairs rather than computing a different function of them. + +Freezing the per-group column counts is a restricted-randomisation device, the +same idea as conditioning on the margins in Fisher's exact test: it removes a +nuisance source of variability from the null rather than changing the estimand. +Empirically it is power-neutral. Compare it against the obvious alternative — +draw `C` uniformly by position, so that `C` is independent of the labels and the +full permutation group is legal — and both reject at the same rate (`n = 200`, +`c = 40`, 500 trials: 0.16 vs 0.16 at a 0.25σ shift, 0.31 vs 0.35 at 0.40σ). The +power lost relative to the full test comes from subsampling, not from the +subgroup. + +What the subgroup buys is the freedom to choose `C` *by design*. Drawing `C` +blind to the labels leaves the smaller group with no columns at all — and its +within-group term unestimable — distressingly often once the samples are +unbalanced: with `n_x = 3`, `n_y = 200` and `c = 40`, 52% of permutations have no +x-columns. Choosing `C` so that it covers the small group fixes that, and the +subgroup is what keeps the test exact when you do. ## Citation diff --git a/src/pted/pted.py b/src/pted/pted.py index 64e6cac..c5c934b 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -128,11 +128,12 @@ def pted( The one thing chunking does cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test - inside ``pted_coverage_test``). There the subgroup reaches only - ``n - c`` distinct label assignments, so the smallest attainable - p-value is about ``1 / (n - c)`` however many permutations are drawn. - Keep ``chunk_size`` well below ``len(s)`` in that case; a - ``PermutationResolutionWarning`` is raised when it is too large. + inside ``pted_coverage_test``). That lone point sits on whichever side + of the column set leaves more room, so the subgroup reaches + ``max(c, n - c)`` distinct label assignments and the smallest + attainable p-value is about the reciprocal of that, however many + permutations are drawn. A ``PermutationResolutionWarning`` is raised + when the reachable set is too small to resolve the p-value requested. """ assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" @@ -295,11 +296,12 @@ def pted_coverage_test( The one thing chunking does cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test - inside ``pted_coverage_test``). There the subgroup reaches only - ``n - c`` distinct label assignments, so the smallest attainable - p-value is about ``1 / (n - c)`` however many permutations are drawn. - Keep ``chunk_size`` well below ``len(s)`` in that case; a - ``PermutationResolutionWarning`` fires when it is too large. + inside ``pted_coverage_test``). That lone point sits on whichever side + of the column set leaves more room, so the subgroup reaches + ``max(c, n - c)`` distinct label assignments and the smallest + attainable p-value is about the reciprocal of that, however many + permutations are drawn. A ``PermutationResolutionWarning`` is raised + when the reachable set is too small to resolve the p-value requested. """ nsamp, nsim, *_ = s.shape assert nsim > 0, "need some simulations to run test, got 0 simulations" diff --git a/src/pted/utils.py b/src/pted/utils.py index 5a87fd0..c3781db 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -26,10 +26,12 @@ symmetric group, and the statistic is the exact energy distance. ``singleton`` - ``n_s == 1``. All columns come from the large group, so the lone - small-group element sits in ``C^c``. Its within-group mean is identically - zero (a single point has no within-group pairs). The label roams over - ``C^c``: ``n - c`` distinct assignments. + ``n_s == 1``. Its within-group mean is identically zero (a single point + has no within-group pairs), so nothing is unestimable and the lone point + may sit on either side of ``C``. It goes on whichever side is larger: + outside, its label roams over ``C^c`` for ``n - c`` assignments; inside, + over the columns for ``c``. The reference set is therefore never smaller + than ``ceil(n / 2)``. ``small_group_in_C`` ``n_s <= c // 2``. ``C`` holds ALL of the small group plus ``c - n_s`` from @@ -48,9 +50,10 @@ threshold degrades as one over the square root of the compute. See Janson (1984) on incomplete U-statistics. -In the ``singleton`` regime the reference set is ``n - c``, so columns come -straight out of p-value resolution. A single row is nearly sufficient for the -statistic there, so keep ``c`` small -- roughly 5-15% of ``n``. +In the ``singleton`` regime a single row is nearly sufficient for the +statistic, so keep ``c`` small -- roughly 5-15% of ``n``. Columns there come +straight out of p-value resolution until ``c`` passes ``n / 2``, beyond which +you may as well pay for the full matrix. Block means exclude the zero self-pairs, making the statistic unbiased for the population energy distance. It can therefore be negative under H0, like @@ -297,11 +300,20 @@ def allocate_columns(n1: int, n2: int, n_columns: int, rng=None) -> dict: cols = np.arange(n) regime, c_small = "full", n_small elif n_small == 1: - # Keep the singleton OUT of C so its label can range over C^c, which is - # the larger of the two options whenever c < n / 2. Reaching here means - # c < n, and n_large == n - 1, so C fits inside the large group. - cols = rng.choice(large_idx, size=n_columns, replace=False) - regime, c_small = "singleton", 0 + # A lone point has no within-group pairs, so it is free to sit on + # either side of C. Inside, its label roams over the c column + # positions; outside, over the n - c others. Take whichever side is + # larger, which holds the reference set at ceil(n / 2) or more instead + # of letting it collapse to 1 as c approaches n. + regime = "singleton" + if n_columns > n - n_columns: + cols = np.concatenate( + [small_idx, rng.choice(large_idx, size=n_columns - 1, replace=False)] + ) + c_small = 1 + else: + cols = rng.choice(large_idx, size=n_columns, replace=False) + c_small = 0 elif n_small <= n_columns // 2: # Put the whole small group in C so that every permuted small group is # still a subset of C and its within-group distances are all present. @@ -360,7 +372,8 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: hint = { "singleton": ( f"the small group holds a single point, so the reference set is " - f"n - n_columns; use fewer columns (currently {alloc['n_columns']})" + f"max(n_columns, n - n_columns); move n_columns (currently " + f"{alloc['n_columns']}) further from half the pooled sample size" ), "full": "the samples are too small for a permutation test at this resolution", }.get( diff --git a/tests/test_pted.py b/tests/test_pted.py index ea5c35a..d127d52 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -358,6 +358,7 @@ def test_pted_rng_is_reproducible(): [ (60, 60, 24, "proportional"), (1, 100, 11, "singleton"), + (1, 40, 30, "singleton"), # c > n/2, so the lone point sits inside C ], ) def test_chunked_pvalues_are_calibrated(n1, n2, n_columns, regime): diff --git a/tests/test_utils.py b/tests/test_utils.py index 541f0d2..a2d74f2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -186,7 +186,8 @@ def block(rows, kk, drop_self): # n1, n2, n_columns, expected regime (100, 100, 200, "full"), (3, 40, 43, "full"), - (1, 60, 12, "singleton"), + (1, 60, 12, "singleton"), # c < n/2 -> the lone point stays out of C + (1, 20, 15, "singleton"), # c > n/2 -> it moves into C instead (6, 80, 40, "small_group_in_C"), (40, 50, 30, "proportional"), (40, 50, 3, "proportional"), @@ -376,16 +377,41 @@ def test_resolve_n_columns(): assert _resolve_n_columns(20, 30, n_columns=1) == 1 +def test_singleton_picks_the_larger_side_of_C(): + """A lone point sits inside or outside C, whichever leaves more label + assignments reachable. Without this the reference set collapses to 1 as c + approaches n, and the test can only ever return p = 1.""" + n2 = 200 + n = 1 + n2 + for c in range(2, n): + alloc = allocate_columns(1, n2, c, rng=0) + assert alloc["regime"] == "singleton" + assert alloc["c_small"] == (1 if c > n - c else 0) + assert alloc["reference_size"] == max(c, n - c) + # so the reference set never drops below half the pooled sample + worst = min(allocate_columns(1, n2, c, rng=0)["reference_size"] for c in range(2, n)) + assert worst >= n // 2 + + def test_permutation_resolution_warning(): - """Too many columns starves the singleton regime of distinct assignments.""" + """Column counts that starve the permutation group are flagged.""" rng = np.random.default_rng(2) - x = rng.standard_normal((1, 3)) - y = rng.standard_normal((60, 3)) - with pytest.warns(PermutationResolutionWarning, match="distinct label assignments"): - permutation_energy_test(x, y, permutations=100, n_columns=55) + # singleton: only a genuinely tiny pooled sample is short of assignments now + with pytest.warns(PermutationResolutionWarning, match="single point"): + permutation_energy_test( + rng.standard_normal((1, 3)), rng.standard_normal((10, 3)), permutations=100, n_columns=5 + ) - # a sensible column count is quiet + # small_group_in_C: too FEW columns is the failure mode here + with pytest.warns(PermutationResolutionWarning, match="use more columns"): + permutation_energy_test( + rng.standard_normal((2, 3)), rng.standard_normal((30, 3)), permutations=100, n_columns=5 + ) + + # sensible column counts are quiet, at either end of the singleton range + x, y = rng.standard_normal((1, 3)), rng.standard_normal((60, 3)) with warnings.catch_warnings(): warnings.simplefilter("error", PermutationResolutionWarning) permutation_energy_test(x, y, permutations=100, n_columns=8) + permutation_energy_test(x, y, permutations=100, n_columns=55) From 60a234e69a7da28d6783531c3f6b1a49e6c823c6 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Tue, 8 Sep 2026 09:59:57 -0400 Subject: [PATCH 3/7] rename chunk to landmark --- README.md | 72 +++++------ src/pted/pted.py | 102 +++++++-------- src/pted/utils.py | 308 ++++++++++++++++++++------------------------ tests/test_pted.py | 77 +++++------ tests/test_utils.py | 150 +++++++++++---------- 5 files changed, 332 insertions(+), 377 deletions(-) diff --git a/README.md b/README.md index bffb0a1..f5fd6b4 100644 --- a/README.md +++ b/README.md @@ -304,10 +304,9 @@ def pted( y: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, - n_columns: Optional[int] = None, batch_size: Optional[int] = None, rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: @@ -317,10 +316,9 @@ def pted( * **y** *(Union[np.ndarray, Tensor, jax.Array])*: second set of samples. Shape (M, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. Only distances from every sample to `min(chunk_size, len(x)) + min(chunk_size, len(y))` "column" points drawn from the pooled sample are computed, so the cost drops from `O(n^2 d)` to `O(n c d)`. If `chunk_size` covers both full datasets, PTED falls back to the exact full-matrix computation. If None, use the full dataset. Mutually exclusive with `n_columns`. +* **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. * **two_tailed** *(bool)*: if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. If False, only checks for dissimilarity but is more sensitive. Default is True. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of permutation tests. Default is False. -* **n_columns** *(Optional[int])*: the number of column points `c` directly. This is the preferred arg; `chunk_size` is the older one and simply maps onto it. Mutually exclusive with `chunk_size`. * **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. * **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. @@ -333,13 +331,12 @@ def pted_coverage_test( permutations: int = 1000, warn_confidence: Optional[float] = 1e-3, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, sbc_histogram: Optional[str] = None, sbc_bins: Optional[int] = None, pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, - n_columns: Optional[int] = None, batch_size: Optional[int] = None, rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: @@ -349,15 +346,14 @@ def pted_coverage_test( * **s** *(Union[np.ndarray, Tensor, jax.Array])*: Posterior samples. Shape (n_samples, n_sims, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. Only distances from every sample to `min(chunk_size, len(x)) + min(chunk_size, len(y))` "column" points drawn from the pooled sample are computed, so the cost drops from `O(n^2 d)` to `O(n c d)`. If `chunk_size` covers both full datasets, PTED falls back to the exact full-matrix computation. If None, use the full dataset. Mutually exclusive with `n_columns`. +* **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. - Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches only `n - c` distinct label assignments. Keep `chunk_size` well below the number of posterior samples. + Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches `max(m, n - m)` distinct label assignments. Keep `n_landmarks` well below the number of posterior samples. * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. * **pit_confidence** *(float)*: Confidence level for the KS confidence band in the PIT plot. Default is 0.95 (95%). Only used when `pit_plot` is not None. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of simulations. Default is False. -* **n_columns** *(Optional[int])*: the number of column points `c` directly. This is the preferred arg; `chunk_size` is the older one and simply maps onto it. Mutually exclusive with `chunk_size`. * **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. * **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. @@ -399,12 +395,13 @@ running into memory limitations, there are still options! We can use an approximation of the energy distance; the test stays exact, it just becomes less sensitive than it would otherwise be. Instead of building the full `n x n` pairwise distance matrix (`n = n_samp_x + n_samp_y`), PTED builds a smaller -rectangular `n x c` matrix of distances from every sample to `c` "column" points -drawn from the pooled sample. Set `n_columns = c` to choose how many columns. +rectangular `n x m` matrix of distances from every sample to `m` "landmark" +points drawn from the pooled sample — the Nyström-style subsampling familiar +from kernel methods. Set `n_landmarks = m` to choose how many. Building the full matrix costs `O(n^2 d)` and each permutation `O(n^2)`; the -rectangular matrix costs `O(n c d)` and each permutation `O(n c)`. So for a -fixed number of columns PTED grows linearly with dataset size, much like other +rectangular matrix costs `O(n m d)` and each permutation `O(n m)`. So for a +fixed number of landmarks PTED grows linearly with dataset size, much like other large scale (machine learning oriented) two sample tests. Permutations are evaluated in batches as a single matrix product rather than one at a time, and everything heavy stays on whichever backend your arrays live on. @@ -417,32 +414,33 @@ import numpy as np x = np.random.normal(size = (5000, 10)) # (n_samples_x, n_dimensions) y = np.random.normal(size = (4000, 10)) # (n_samples_y, n_dimensions) -p_value = pted(x, y, n_columns = 200) +p_value = pted(x, y, n_landmarks = 200) print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 ``` -### Why chunking is still exact +### Why using landmarks is still exact -The subtlety is that the columns cannot simply be re-split between the two -groups after each shuffle. The column set `C` is chosen using the group labels +The subtlety is that the landmarks cannot simply be re-split between the two +groups after each shuffle. The landmark set `L` is chosen using the group labels (so that both groups are represented), which means the labels are no longer -uniformly distributed once you condition on `C`. PTED handles this by confining -permutations to the subgroup that fixes `C`: labels are shuffled within the -column positions and within their complement, never between. `C` then stays put +uniformly distributed once you condition on `L`. PTED handles this by confining +permutations to the subgroup that fixes `L`: labels are shuffled within the +landmark positions and within their complement, never between. `L` then stays put under every permutation and the observed labelling is exchangeable with the permuted -ones. That is what makes the p-value exact. (`C` is chosen from sample positions -and group labels only, never from the data values — picking columns by maximin, -k-means or leverage would break the argument.) +ones. That is what makes the p-value exact. (`L` is chosen from sample positions +and group labels only, never from the data values — picking landmarks by maximin, +k-means or leverage would break the argument. "Landmark" here means only "a point +everything is measured against", never "a point chosen for its importance".) -What you give up is sensitivity: the null spread grows like `sqrt(n / c)`, so the -smallest detectable energy distance scales as `(n c)^-0.5` rather than `n^-1`. +What you give up is sensitivity: the null spread grows like `sqrt(n / m)`, so the +smallest detectable energy distance scales as `(n m)^-0.5` rather than `n^-1`. The detection threshold degrades as one over the square root of the compute. You also give up some p-value resolution, and this is most significant when one group holds a single point — which is exactly the per-simulation test inside -`pted_coverage_test`. There the lone point can sit inside `C`, its label roaming -over the `c` column positions, or outside it, roaming over the other `n - c`, so -the subgroup reaches `max(c, n - c)` distinct label assignments. PTED puts the +`pted_coverage_test`. There the lone point can sit inside `L`, its label roaming +over the `m` landmark positions, or outside it, roaming over the other `n - m`, +so the subgroup reaches `max(m, n - m)` distinct label assignments. PTED puts the point on whichever side is larger, which holds the reference set at half the pooled sample or better. The smallest attainable p-value is still about the reciprocal of that, however many permutations you draw, so PTED raises a @@ -454,28 +452,28 @@ the p-value you asked for. Yes — the restriction is on how the statistic is *calibrated*, not on what it measures. Every block mean still averages genuine distances between genuine members of the two groups: the cross term pairs **all** `n_x` samples against the -large-group columns and **all** `n_y` samples against the small-group columns, so -no sample is demoted to a mere marker. Each block mean is an unbiased estimate of +large-group landmarks and **all** `n_y` samples against the small-group +landmarks, so no sample is demoted to a mere marker. Each block mean is an unbiased estimate of exactly the population quantity the full test estimates, which makes the whole statistic an unbiased estimator of the population energy distance — an *incomplete U-statistic* in the sense of Janson (1984), averaging over a subset of the pairs rather than computing a different function of them. -Freezing the per-group column counts is a restricted-randomisation device, the +Freezing the per-group landmark counts is a restricted-randomisation device, the same idea as conditioning on the margins in Fisher's exact test: it removes a nuisance source of variability from the null rather than changing the estimand. Empirically it is power-neutral. Compare it against the obvious alternative — -draw `C` uniformly by position, so that `C` is independent of the labels and the +draw `L` uniformly by position, so that `L` is independent of the labels and the full permutation group is legal — and both reject at the same rate (`n = 200`, -`c = 40`, 500 trials: 0.16 vs 0.16 at a 0.25σ shift, 0.31 vs 0.35 at 0.40σ). The +`m = 40`, 500 trials: 0.16 vs 0.16 at a 0.25σ shift, 0.31 vs 0.35 at 0.40σ). The power lost relative to the full test comes from subsampling, not from the subgroup. -What the subgroup buys is the freedom to choose `C` *by design*. Drawing `C` -blind to the labels leaves the smaller group with no columns at all — and its +What the subgroup buys is the freedom to choose `L` *by design*. Drawing `L` +blind to the labels leaves the smaller group with no landmarks at all — and its within-group term unestimable — distressingly often once the samples are -unbalanced: with `n_x = 3`, `n_y = 200` and `c = 40`, 52% of permutations have no -x-columns. Choosing `C` so that it covers the small group fixes that, and the +unbalanced: with `n_x = 3`, `n_y = 200` and `m = 40`, 52% of permutations have no +x-landmarks. Choosing `L` so that it covers the small group fixes that, and the subgroup is what keeps the test exact when you do. ## Citation diff --git a/src/pted/pted.py b/src/pted/pted.py index c5c934b..08514d0 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -19,10 +19,9 @@ def pted( y: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, - n_columns: Optional[int] = None, batch_size: Optional[int] = None, rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: @@ -85,22 +84,18 @@ def pted( return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): if not None, estimate the energy distance + n_landmarks (Optional[int]): if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise - matrix. Only distances from every sample to ``min(chunk_size, - len(x)) + min(chunk_size, len(y))`` "column" points drawn from the - pooled sample are computed, so the cost drops from ``O(n^2 d)`` to - ``O(n c d)``. If ``chunk_size`` covers both full datasets, this - falls back to the exact full-matrix computation. If None, use the - full dataset. Mutually exclusive with ``n_columns``. + matrix. ``n_landmarks`` points are drawn from the pooled sample as + "landmarks", and only the distance from every sample to each + landmark is computed, so the cost drops from ``O(n^2 d)`` to + ``O(n m d)`` for ``m = n_landmarks``. A value covering the whole + pooled sample, or None, runs the exact full-matrix computation. two_tailed (bool): if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. Default is True. prog_bar (bool): if True, show a progress bar to track the progress of permutation tests. Default is False. - n_columns (Optional[int]): the number of column points ``c`` directly. - This is the preferred arg; ``chunk_size`` is the older one and - simply maps onto it. Mutually exclusive with ``chunk_size``. batch_size (Optional[int]): number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at ``O(batch_size * n)`` extra memory. None picks a size that keeps a @@ -114,23 +109,23 @@ def pted( time, where n is the pooled sample size and d the dimension. Each permutation then costs ``O(n^2)``, though permutations are evaluated in batches as a single matrix product rather than one at a time. For large - datasets this gets unwieldy, so chunking is recommended: only a - rectangular ``(n, c)`` matrix of distances to ``c`` column points is - built, in ``O(n c d)`` time, and each permutation costs ``O(n c)``. - - Chunking keeps the p-value exact. Permutations are drawn from the - subgroup that holds the column set fixed, shuffling labels within the - columns and within their complement but never between, so the observed - labelling is exchangeable with the permuted ones. The test simply - becomes less sensitive as ``c`` shrinks: the null spread grows like - ``sqrt(n / c)``, so the detectable energy distance scales as - ``(n c)^-0.5`` rather than ``n^-1``. - - The one thing chunking does cost is p-value resolution, and it bites + datasets this gets unwieldy, so landmarks are recommended: only a + rectangular ``(n, m)`` matrix of distances to ``m`` landmark points is + built, in ``O(n m d)`` time, and each permutation costs ``O(n m)``. + + Landmarks keep the p-value exact. Permutations are drawn from the + subgroup that holds the landmark set fixed, shuffling labels within + the landmarks and within their complement but never between, so the + observed labelling is exchangeable with the permuted ones. The test + simply becomes less sensitive as ``m`` shrinks: the null spread grows + like ``sqrt(n / m)``, so the detectable energy distance scales as + ``(n m)^-0.5`` rather than ``n^-1``. + + The one thing landmarks do cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test inside ``pted_coverage_test``). That lone point sits on whichever side - of the column set leaves more room, so the subgroup reaches - ``max(c, n - c)`` distinct label assignments and the smallest + of the landmark set leaves more room, so the subgroup reaches + ``max(m, n - m)`` distinct label assignments and the smallest attainable p-value is about the reciprocal of that, however many permutations are drawn. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. @@ -146,14 +141,13 @@ def pted( if len(y.shape) > 2: y = y.reshape(y.shape[0], -1) - # The column controls are resolved once, inside the test itself; a count - # covering the whole pooled sample lands in the exact full-matrix regime. + # A landmark count covering the whole pooled sample lands in the exact + # full-matrix regime, so there is no separate branch for it here. test, permute = _energy_test( x, y, permutations=permutations, - chunk_size=chunk_size, - n_columns=n_columns, + n_landmarks=n_landmarks, prog_bar=prog_bar, batch_size=batch_size, rng=rng, @@ -179,13 +173,12 @@ def pted_coverage_test( permutations: int = 1000, warn_confidence: Optional[float] = 1e-3, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, sbc_histogram: Optional[str] = None, sbc_bins: Optional[int] = None, pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, - n_columns: Optional[int] = None, batch_size: Optional[int] = None, rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: @@ -242,14 +235,13 @@ def pted_coverage_test( return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): if not None, estimate the energy distance + n_landmarks (Optional[int]): if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise - matrix. Only distances from every sample to ``min(chunk_size, - len(x)) + min(chunk_size, len(y))`` "column" points drawn from the - pooled sample are computed, so the cost drops from ``O(n^2 d)`` to - ``O(n c d)``. If ``chunk_size`` covers both full datasets, this - falls back to the exact full-matrix computation. If None, use the - full dataset. Mutually exclusive with ``n_columns``. + matrix. ``n_landmarks`` points are drawn from the pooled sample as + "landmarks", and only the distance from every sample to each + landmark is computed, so the cost drops from ``O(n^2 d)`` to + ``O(n m d)`` for ``m = n_landmarks``. A value covering the whole + pooled sample, or None, runs the exact full-matrix computation. sbc_histogram (Optional[str]): If given, the path/filename to save a Simulation-Based-Calibration histogram. sbc_bins (Optional[int]): If given, force the histogram to have the provided @@ -265,9 +257,6 @@ def pted_coverage_test( ``pit_plot`` is not None. prog_bar (bool): If True, show a progress bar to track the progress of simulations. Default is False. - n_columns (Optional[int]): the number of column points ``c`` directly. - This is the preferred arg; ``chunk_size`` is the older one and - simply maps onto it. Mutually exclusive with ``chunk_size``. batch_size (Optional[int]): number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at ``O(batch_size * n)`` extra memory. None picks a size that keeps a @@ -281,24 +270,24 @@ def pted_coverage_test( time, where n is the pooled sample size and d the dimension. Each permutation then costs ``O(n^2)``, though permutations are evaluated in batches as a single matrix product rather than one at a time. For large - datasets this gets unwieldy, so chunking is recommended: only a - rectangular ``(n, c)`` matrix of distances to ``c`` column points is - built, in ``O(n c d)`` time, and each permutation costs ``O(n c)``. + datasets this gets unwieldy, so landmarks are recommended: only a + rectangular ``(n, m)`` matrix of distances to ``m`` landmark points is + built, in ``O(n m d)`` time, and each permutation costs ``O(n m)``. - Chunking keeps the p-value exact. Permutations are drawn from the - subgroup that holds the column set fixed, shuffling labels within the - columns and within their complement but never between, so every + Landmarks keep the p-value exact. Permutations are drawn from the + subgroup that holds the landmark set fixed, shuffling labels within + the landmarks and within their complement but never between, so every normalising constant in the statistic is a constant and the observed labelling is exchangeable with the permuted ones. The test simply - becomes less sensitive as ``c`` shrinks: the null spread grows like - ``sqrt(n / c)``, so the detectable energy distance scales as - ``(n c)^-0.5`` rather than ``n^-1``. + becomes less sensitive as ``m`` shrinks: the null spread grows like + ``sqrt(n / m)``, so the detectable energy distance scales as + ``(n m)^-0.5`` rather than ``n^-1``. - The one thing chunking does cost is p-value resolution, and it bites + The one thing landmarks do cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test inside ``pted_coverage_test``). That lone point sits on whichever side - of the column set leaves more room, so the subgroup reaches - ``max(c, n - c)`` distinct label assignments and the smallest + of the landmark set leaves more room, so the subgroup reaches + ``max(m, n - m)`` distinct label assignments and the smallest attainable p-value is about the reciprocal of that, however many permutations are drawn. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. @@ -327,8 +316,7 @@ def pted_coverage_test( permutations=permutations, return_all=True, two_tailed=False, - chunk_size=chunk_size, - n_columns=n_columns, + n_landmarks=n_landmarks, batch_size=batch_size, rng=rng, ) diff --git a/src/pted/utils.py b/src/pted/utils.py index c3781db..35695b0 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -2,57 +2,62 @@ The pooled sample is ``z = [x; y]`` of size ``n = n1 + n2``. Rather than the full ``n x n`` distance matrix of Szekely & Rizzo (2004), the test may be run -on a rectangular ``n x c`` matrix ``D = cdist(z, z[cols])``, where ``cols`` -("columns", elsewhere called landmarks) indexes ``c`` points drawn from the -pooled sample. That is ``O(n * c)`` distances and ``O(n * c)`` work per -permutation rather than ``O(n^2)``. Setting ``c = n`` recovers the exact -full-matrix test, so both live in one code path. +on a rectangular ``n x m`` matrix ``D = cdist(z, z[landmarks])``, where +``landmarks`` indexes ``m`` points drawn from the pooled sample. Every sample +is measured against those ``m`` landmarks and no other distance is computed: +``O(n * m)`` distances and ``O(n * m)`` work per permutation rather than +``O(n^2)``. Setting ``m = n`` recovers the exact full-matrix test, so both +live in one code path. This is the Nystrom-style subsampling familiar from +kernel methods, applied to the energy distance. Validity -------- -Permutations are always confined to the subgroup ``S_C x S_{C^c}``: labels are -shuffled within the column positions and within their complement, never across. -Since ``sigma(C) = C`` identically, the column set is invariant and the test is -exact conditional on ``C``. This is what licenses choosing ``C`` by design -(balanced, or aligned with the group structure). The one requirement is that -``C`` must not be chosen using the data values. - -Column allocation ------------------ +Permutations are always confined to the subgroup ``S_L x S_{L^c}``, where ``L`` +is the landmark set: labels are shuffled within the landmark positions and +within their complement, never across. Since ``sigma(L) = L`` identically, the +landmark set is invariant and the test is exact conditional on ``L``. This is +what licenses choosing ``L`` by design (balanced, or aligned with the group +structure). The one requirement is that ``L`` must not be chosen using the data +values -- sample positions and group labels are fine, but maximin, k-means or +leverage selection would break the argument. "Landmark" here means only "a +point everything is measured against", never "a point picked for importance". + +Landmark allocation +------------------- Dispatched on ``n_s = min(n1, n2)``, the smaller group: ``full`` - ``c >= n``. Every point is a column, ``S_C x S_{C^c}`` is the full + ``m >= n``. Every point is a landmark, ``S_L x S_{L^c}`` is the full symmetric group, and the statistic is the exact energy distance. ``singleton`` ``n_s == 1``. Its within-group mean is identically zero (a single point has no within-group pairs), so nothing is unestimable and the lone point - may sit on either side of ``C``. It goes on whichever side is larger: - outside, its label roams over ``C^c`` for ``n - c`` assignments; inside, - over the columns for ``c``. The reference set is therefore never smaller + may sit on either side of ``L``. It goes on whichever side is larger: + outside, its label roams over ``L^c`` for ``n - m`` assignments; inside, + over the landmarks for ``m``. The reference set is therefore never smaller than ``ceil(n / 2)``. -``small_group_in_C`` - ``n_s <= c // 2``. ``C`` holds ALL of the small group plus ``c - n_s`` from - the large group. Every permuted small group is then a subset of ``C``, so +``small_group_in_L`` + ``n_s <= m // 2``. ``L`` holds ALL of the small group plus ``m - n_s`` from + the large group. Every permuted small group is then a subset of ``L``, so every within-small-group distance appears in ``D`` and that term is computed exactly rather than subsampled. ``proportional`` - Otherwise. ``c_s ~ c * n_s / n``, and both within-group terms are + Otherwise. ``m_s ~ m * n_s / n``, and both within-group terms are subsampled. Cost ---- -Null spread grows like ``sqrt(n / c)``, so the detectable energy distance -scales as ``(n * c)^{-1/2}`` versus ``n^{-1}`` for the full test: the detection +Null spread grows like ``sqrt(n / m)``, so the detectable energy distance +scales as ``(n * m)^{-1/2}`` versus ``n^{-1}`` for the full test: the detection threshold degrades as one over the square root of the compute. See Janson (1984) on incomplete U-statistics. In the ``singleton`` regime a single row is nearly sufficient for the -statistic, so keep ``c`` small -- roughly 5-15% of ``n``. Columns there come -straight out of p-value resolution until ``c`` passes ``n / 2``, beyond which +statistic, so keep ``m`` small -- roughly 5-15% of ``n``. Landmarks there come +straight out of p-value resolution until ``m`` passes ``n / 2``, beyond which you may as well pay for the full matrix. Block means exclude the zero self-pairs, making the statistic unbiased for the @@ -92,7 +97,7 @@ class torch: "is_torch_tensor", "is_jax_array", "permutation_energy_test", - "allocate_columns", + "allocate_landmarks", "PermutationResolutionWarning", "two_tailed_p", "confidence_alert", @@ -198,8 +203,8 @@ def _cdist(a, b, backend: str): return cdist(a, b, metric="euclidean") -def _zero_self_pairs(D, cols, backend: str): - """Set ``D[cols[k], k] = 0``, the entries pairing a point with itself. +def _zero_self_pairs(D, landmarks, backend: str): + """Set ``D[landmarks[k], k] = 0``, the entries pairing a point with itself. Called after ``D`` has been shifted by a constant, to restore the property that a point is at distance zero from itself. Self-pairs are excluded from @@ -210,10 +215,10 @@ def _zero_self_pairs(D, cols, backend: str): scrubs the ~1e-7 that ``torch.cdist`` leaves on the diagonal once it switches to its matmul-based algorithm. """ - k = np.arange(len(cols)) + k = np.arange(len(landmarks)) if backend == "jax": - return D.at[jnp.asarray(cols), jnp.asarray(k)].set(0.0) - D[_index_like(cols, D, backend), _index_like(k, D, backend)] = 0.0 + return D.at[jnp.asarray(landmarks), jnp.asarray(k)].set(0.0) + D[_index_like(landmarks, D, backend), _index_like(k, D, backend)] = 0.0 return D @@ -242,8 +247,7 @@ class PermutationResolutionWarning(UserWarning): _REFERENCE_CAP = 10**15 # Target element count for a batch of permutations, bounding peak memory. -_BATCH_ELEMENTS = 2**22 -_MAX_BATCH = 256 +_BATCH_ELEMENTS = 2**30 def _capped_comb(n: int, k: int) -> int: @@ -255,8 +259,11 @@ def _capped_comb(n: int, k: int) -> int: # --------------------------------------------------------------- allocation -def allocate_columns(n1: int, n2: int, n_columns: int, rng=None) -> dict: - """Choose which pooled-sample points supply the columns of ``D``. +def allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng=None) -> dict: + """Choose which pooled-sample points serve as landmarks. + + The landmarks are the ``m`` points every sample is measured against: they + supply the columns of the rectangular distance matrix ``D``. Group 0 is x (pooled indices ``0..n1-1``), group 1 is y (``n1..n-1``). Selection uses only group sizes and positions, never the data values, which @@ -266,91 +273,92 @@ def allocate_columns(n1: int, n2: int, n_columns: int, rng=None) -> dict: ---------- n1 (int): number of samples in x. n2 (int): number of samples in y. - n_columns (int): number of column points ``c``. ``c >= n1 + n2`` + n_landmarks (int): number of landmarks ``m``. ``m >= n1 + n2`` requests the exact full-matrix test. rng: seed, ``np.random.Generator``, or None (draw from the global state). Returns ------- - dict with keys ``cols``, ``regime``, ``small_idx`` (pooled indices - of the smaller group), ``n_small``, ``n_large``, ``c_small``, - ``c_large``, ``n_columns``, - ``reference_size`` (distinct label assignments the subgroup reaches) + dict with keys ``landmarks``, ``regime``, ``small_idx`` (pooled indices + of the smaller group), ``n_small``, ``n_large``, ``n_small_landmarks``, + ``n_large_landmarks``, ``n_landmarks``, ``reference_size`` + (distinct label assignments the subgroup reaches) and ``exact_within`` (whether the small group's within-group term is computed exactly rather than subsampled). """ rng = _as_rng(rng) n = n1 + n2 - n_columns = int(n_columns) + n_landmarks = int(n_landmarks) if min(n1, n2) < 1: raise ValueError(f"both samples need at least one point, got n1={n1}, n2={n2}") - if n_columns < 2: - raise ValueError(f"need at least 2 columns, got n_columns={n_columns}") - if n_columns > n: - raise ValueError(f"n_columns={n_columns} exceeds the pooled sample size {n}") + if n_landmarks < 2: + raise ValueError(f"need at least 2 landmarks, got n_landmarks={n_landmarks}") + if n_landmarks > n: + raise ValueError(f"n_landmarks={n_landmarks} exceeds the pooled sample size {n}") n_small, n_large = min(n1, n2), max(n1, n2) x_is_small = n1 <= n2 small_idx = np.arange(n1) if x_is_small else n1 + np.arange(n2) large_idx = n1 + np.arange(n2) if x_is_small else np.arange(n1) - if n_columns == n: - # C is everything, so S_C x S_{C^c} is the full symmetric group and + if n_landmarks == n: + # L is everything, so S_L x S_{L^c} is the full symmetric group and # this is the classical exact permutation test. - cols = np.arange(n) - regime, c_small = "full", n_small + landmarks = np.arange(n) + regime, n_small_landmarks = "full", n_small elif n_small == 1: # A lone point has no within-group pairs, so it is free to sit on - # either side of C. Inside, its label roams over the c column - # positions; outside, over the n - c others. Take whichever side is + # either side of L. Inside, its label roams over the m landmark + # positions; outside, over the n - m others. Take whichever side is # larger, which holds the reference set at ceil(n / 2) or more instead - # of letting it collapse to 1 as c approaches n. + # of letting it collapse to 1 as m approaches n. regime = "singleton" - if n_columns > n - n_columns: - cols = np.concatenate( - [small_idx, rng.choice(large_idx, size=n_columns - 1, replace=False)] + if n_landmarks > n - n_landmarks: + landmarks = np.concatenate( + [small_idx, rng.choice(large_idx, size=n_landmarks - 1, replace=False)] ) - c_small = 1 + n_small_landmarks = 1 else: - cols = rng.choice(large_idx, size=n_columns, replace=False) - c_small = 0 - elif n_small <= n_columns // 2: - # Put the whole small group in C so that every permuted small group is - # still a subset of C and its within-group distances are all present. - cols = np.concatenate( - [small_idx, rng.choice(large_idx, size=n_columns - n_small, replace=False)] + landmarks = rng.choice(large_idx, size=n_landmarks, replace=False) + n_small_landmarks = 0 + elif n_small <= n_landmarks // 2: + # Put the whole small group in L so that every permuted small group is + # still a subset of L and its within-group distances are all present. + landmarks = np.concatenate( + [small_idx, rng.choice(large_idx, size=n_landmarks - n_small, replace=False)] ) - regime, c_small = "small_group_in_C", n_small + regime, n_small_landmarks = "small_group_in_L", n_small else: - lo = max(1, n_columns - n_large) - hi = min(n_small, n_columns - 1) - c_small = int(np.clip(round(n_columns * n_small / n), lo, hi)) - cols = np.concatenate( + lo = max(1, n_landmarks - n_large) + hi = min(n_small, n_landmarks - 1) + n_small_landmarks = int(np.clip(round(n_landmarks * n_small / n), lo, hi)) + landmarks = np.concatenate( [ - rng.choice(small_idx, size=c_small, replace=False), - rng.choice(large_idx, size=n_columns - c_small, replace=False), + rng.choice(small_idx, size=n_small_landmarks, replace=False), + rng.choice(large_idx, size=n_landmarks - n_small_landmarks, replace=False), ] ) regime = "proportional" - # The subgroup places c_small of the small labels among the c columns and - # the rest among the n - c remaining positions, independently. + # The subgroup places n_small_landmarks of the small labels among the m + # landmarks and the rest among the n - m other positions, independently. reference = min( - _capped_comb(n_columns, c_small) * _capped_comb(n - n_columns, n_small - c_small), + _capped_comb(n_landmarks, n_small_landmarks) + * _capped_comb(n - n_landmarks, n_small - n_small_landmarks), _REFERENCE_CAP, ) return { - "cols": np.sort(cols), + "landmarks": np.sort(landmarks), "regime": regime, "small_idx": small_idx, "n_small": n_small, "n_large": n_large, - "c_small": c_small, - "c_large": n_columns - c_small, - "n_columns": n_columns, + "n_small_landmarks": n_small_landmarks, + "n_large_landmarks": n_landmarks - n_small_landmarks, + "n_landmarks": n_landmarks, "reference_size": reference, - "exact_within": c_small == n_small or n_small == 1, + "exact_within": n_small_landmarks == n_small or n_small == 1, } @@ -364,7 +372,7 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: reference = alloc["reference_size"] # In the full regime the reference set is fixed by the sample sizes alone, # so only flag it when it is degenerate. Everywhere else it is a - # consequence of the column count, which the caller can act on. + # consequence of the landmark count, which the caller can act on. floor = 20 if alloc["regime"] == "full" else max(20, (permutations + 1) // 10) if reference >= floor: return @@ -372,17 +380,17 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: hint = { "singleton": ( f"the small group holds a single point, so the reference set is " - f"max(n_columns, n - n_columns); move n_columns (currently " - f"{alloc['n_columns']}) further from half the pooled sample size" + f"max(n_landmarks, n - n_landmarks); move n_landmarks (currently " + f"{alloc['n_landmarks']}) further from half the pooled sample size" ), "full": "the samples are too small for a permutation test at this resolution", }.get( alloc["regime"], - f"use more columns (currently {alloc['n_columns']}) or more samples", + f"use more landmarks (currently {alloc['n_landmarks']}) or more samples", ) warn( PermutationResolutionWarning( - f"the {alloc['regime']!r} column allocation reaches only {reference:,} distinct " + f"the {alloc['regime']!r} landmark allocation reaches only {reference:,} distinct " f"label assignments, so the smallest attainable p-value is about " f"{1.0 / reference:.3g} no matter how many permutations are drawn " f"({permutations} requested): {hint}." @@ -390,12 +398,12 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: ) -def _draw_labels(base_small, idx_c, idx_o, n_draws: int, rngs) -> np.ndarray: +def _draw_labels(base_small, idx_l, idx_o, n_draws: int, rngs) -> np.ndarray: """``(n_draws, n)`` indicator of small-group membership. - Drawn uniformly from the subgroup ``S_C x S_{C^c}`` by shuffling the labels - within the column positions ``idx_c`` and within their complement ``idx_o``, - never between. Permutation bookkeeping stays in numpy: it is pure integer + Drawn uniformly from the subgroup ``S_L x S_{L^c}`` by shuffling the labels + within the landmark positions ``idx_l`` and within their complement + ``idx_o``, never between. Permutation bookkeeping stays in numpy: it is pure integer shuffling, and only the resulting indicator matrix crosses to the compute backend. @@ -405,7 +413,7 @@ def _draw_labels(base_small, idx_c, idx_o, n_draws: int, rngs) -> np.ndarray: """ n = base_small.size U = np.empty((n_draws, n), dtype=base_small.dtype) - for idx, rng in zip((idx_c, idx_o), rngs): + for idx, rng in zip((idx_l, idx_o), rngs): if idx.size == 0: continue block = rng.permuted(np.repeat(base_small[idx][None, :], n_draws, axis=0), axis=1) @@ -423,21 +431,21 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: """Precompute everything that is constant across permutations. Every block sum is a bilinear form in the row indicator ``u`` (length n) - and the column indicator ``v = u[cols]`` (length c): + and the landmark indicator ``v = u[landmarks]`` (length m): - u @ D @ v = sum_{i in A} sum_{k: cols[k] in A} D[i, k] + u @ D @ v = sum_{i in A} sum_{k: landmarks[k] in A} D[i, k] Only that one product is needed per permutation; the other three blocks follow from the row sums, column sums and grand total, since the two indicators of each pair sum to one. All denominators are constants, because - the subgroup ``S_C x S_{C^c}`` preserves the per-group column counts. + the subgroup ``S_L x S_{L^c}`` preserves the per-group landmark counts. """ n_s, n_l = alloc["n_small"], alloc["n_large"] n = n_s + n_l - c_s, c_l = alloc["c_small"], alloc["c_large"] - cols = alloc["cols"] + m_s, m_l = alloc["n_small_landmarks"], alloc["n_large_landmarks"] + landmarks = alloc["landmarks"] - # allocate_columns guarantees c_l >= 1 in every regime, and c_s >= 1 + # allocate_landmarks guarantees m_l >= 1 in every regime, and m_s >= 1 # unless n_s == 1 (the singleton regime, where the small group has no # within-group pairs to estimate), so both block means below are defined. @@ -450,18 +458,18 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: # into the reported statistic, so that one accumulates in float64. offset = _to_scalar(D.mean(), backend) D = D - offset - D = _zero_self_pairs(D, cols, backend) + D = _zero_self_pairs(D, landmarks, backend) row_sums = D.sum(1) base_small = np.zeros(n) base_small[alloc["small_idx"]] = 1.0 - in_c = np.zeros(n, dtype=bool) - in_c[cols] = True + in_l = np.zeros(n, dtype=bool) + in_l[landmarks] = True return { "D": D, - # ``cols`` covers everything in the full regime, so skip the gather. - "cols": None if alloc["regime"] == "full" else _index_like(cols, D, backend), + # The landmarks cover everything in the full regime, so skip the gather. + "landmarks": None if alloc["regime"] == "full" else _index_like(landmarks, D, backend), "offset": offset, "row_sums": row_sums, "col_sums": D.sum(0), @@ -469,15 +477,15 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: "n": n, "n_s": n_s, "n_l": n_l, - "c_s": c_s, - "c_l": c_l, - # Column point cols[k] is also row cols[k], so each within-group block + "m_s": m_s, + "m_l": m_l, + # Landmark landmarks[k] is also row landmarks[k], so each within-group block # holds exactly c_g zero-valued self-pairs; drop them from the counts. - "pairs_ss": c_s * (n_s - 1), - "pairs_ll": c_l * (n_l - 1), + "pairs_ss": m_s * (n_s - 1), + "pairs_ll": m_l * (n_l - 1), "base_small": base_small, - "idx_c": np.flatnonzero(in_c), - "idx_o": np.flatnonzero(~in_c), + "idx_l": np.flatnonzero(in_l), + "idx_o": np.flatnonzero(~in_l), "backend": backend, } @@ -498,15 +506,15 @@ def _evaluate_statistic(prep: dict, U) -> np.ndarray: D = prep["D"] Us = _asarray_like(U, D, backend) - Vs = Us if prep["cols"] is None else Us[:, prep["cols"]] + Vs = Us if prep["landmarks"] is None else Us[:, prep["landmarks"]] - S_ss = ((Us @ D) * Vs).sum(-1) # small rows, small cols - S_sl = Us @ prep["row_sums"] - S_ss # small rows, large cols - S_ls = Vs @ prep["col_sums"] - S_ss # large rows, small cols - S_ll = prep["total"] - S_ss - S_sl - S_ls # large rows, large cols + S_ss = ((Us @ D) * Vs).sum(-1) # small rows, small landmarks + S_sl = Us @ prep["row_sums"] - S_ss # small rows, large landmarks + S_ls = Vs @ prep["col_sums"] - S_ss # large rows, small landmarks + S_ll = prep["total"] - S_ss - S_sl - S_ls # large rows, large landmarks n_s, n_l = prep["n_s"], prep["n_l"] - c_s, c_l = prep["c_s"], prep["c_l"] + m_s, m_l = prep["m_s"], prep["m_l"] # A group of one point has no within-group pairs, so its mean is zero on # the raw distances -- which is -offset on the centred ones. Writing it @@ -517,10 +525,10 @@ def _evaluate_statistic(prep: dict, U) -> np.ndarray: mu_ss = -off if n_s < 2 else S_ss / prep["pairs_ss"] mu_ll = -off if n_l < 2 else S_ll / prep["pairs_ll"] - # Cross term: mean over whichever orientations have columns available. - parts = [S_sl / (n_s * c_l)] - if c_s > 0: - parts.append(S_ls / (n_l * c_s)) + # Cross term: mean over whichever orientations have landmarks available. + parts = [S_sl / (n_s * m_l)] + if m_s > 0: + parts.append(S_ls / (n_l * m_s)) mu_sl = sum(parts) / len(parts) stat = (n_s * n_l / prep["n"]) * (2.0 * mu_sl - mu_ss - mu_ll) @@ -530,56 +538,25 @@ def _evaluate_statistic(prep: dict, U) -> np.ndarray: # --------------------------------------------------------------- public API -def _resolve_n_columns( - n1: int, n2: int, chunk_size: Optional[int] = None, n_columns: Optional[int] = None -) -> int: - """Map the user-facing column controls onto a single column count ``c``. - - ``n_columns`` is ``c`` directly. ``chunk_size`` is the legacy control and - maps to ``min(chunk_size, n1) + min(chunk_size, n2)``, which reproduces the - number of distance columns (and hence the compute cost) it used to request. - Neither one given means the exact full-matrix test. - - Only the arithmetic lives here. Whether the resulting count is usable is - :func:`allocate_columns`' business, so a nonsense ``n_columns`` is passed - through untouched and rejected there. - """ - n = n1 + n2 - if chunk_size is not None and n_columns is not None: - raise ValueError("give either chunk_size or n_columns, not both") - if n_columns is not None: - c = int(n_columns) - elif chunk_size is not None: - c = int(chunk_size) - if c < 1: - raise ValueError(f"chunk_size must be positive, got {c}") - c = min(c, n1) + min(c, n2) - else: - c = n - return min(c, n) - - def permutation_energy_test( x, y, permutations: int, - chunk_size: Optional[int] = None, - n_columns: Optional[int] = None, + n_landmarks: Optional[int] = None, prog_bar: bool = False, batch_size: Optional[int] = None, rng=None, ) -> tuple[float, np.ndarray]: """Observed energy statistic and its permutation distribution. - With neither ``chunk_size`` nor ``n_columns`` this is the exact - full-matrix test: the ``(n, n)`` distance matrix is built once and the + Without ``n_landmarks`` this is the exact full-matrix test: the ``(n, n)`` distance matrix is built once and the statistic is evaluated for batches of permutations as a bilinear form in the group indicator, so no distance is recomputed and no matrix is - re-indexed. Asking for fewer columns switches to the rectangular - ``(n, c)`` matrix, cutting the cost from ``O(n^2 d)`` to ``O(n c d)``; - permutations are then confined to the subgroup that fixes the column set, - which keeps the p-value exact. See the module docstring for the column - allocation regimes and what they cost in p-value resolution. + re-indexed. Asking for fewer landmarks switches to the rectangular + ``(n, m)`` matrix, cutting the cost from ``O(n^2 d)`` to ``O(n m d)``; + permutations are then confined to the subgroup that fixes the landmark + set, which keeps the p-value exact. See the module docstring for the + landmark allocation regimes and what they cost in p-value resolution. Works transparently with numpy arrays, torch tensors, and jax arrays. Distances and all matrix algebra stay on the backend that ``x`` and ``y`` @@ -590,10 +567,9 @@ def permutation_energy_test( ---------- x, y: samples, shape ``(n1, d)`` and ``(n2, d)``. permutations (int): number of permutations to draw. - chunk_size (Optional[int]): legacy control, equivalent to - ``n_columns = min(chunk_size, n1) + min(chunk_size, n2)``. - n_columns (Optional[int]): number of distance columns ``c``, the - preferred control. Mutually exclusive with ``chunk_size``. + n_landmarks (Optional[int]): number of landmarks ``m``, the points + every sample is measured against. None, or any value covering the + whole pooled sample, runs the exact full-matrix test. prog_bar (bool): show a progress bar over permutations. batch_size (Optional[int]): permutations evaluated per matrix product. Larger is faster on GPU, at ``O(batch_size * n)`` extra memory. @@ -617,14 +593,16 @@ def permutation_energy_test( n1, n2 = len(x), len(y) n = n1 + n2 - c = _resolve_n_columns(n1, n2, chunk_size=chunk_size, n_columns=n_columns) + # More landmarks than points is just the full matrix; anything else + # nonsensical is rejected by allocate_landmarks, which owns that check. + m = n if n_landmarks is None else min(int(n_landmarks), n) rng = _as_rng(rng) - alloc = allocate_columns(n1, n2, c, rng) + alloc = allocate_landmarks(n1, n2, m, rng) _warn_reference_size(alloc, permutations) - cols = alloc["cols"] - zc = z if alloc["regime"] == "full" else z[_index_like(cols, z, backend)] + landmarks = alloc["landmarks"] + zc = z if alloc["regime"] == "full" else z[_index_like(landmarks, z, backend)] dmatrix = _cdist(z, zc, backend) assert _all_finite(dmatrix, backend), ( "Distance matrix contains NaN or Inf! Consider normalizing values to be " @@ -637,7 +615,7 @@ def permutation_energy_test( assert np.isfinite(test_stat), "Observed statistic is not finite!" if batch_size is None: - batch_size = min(_MAX_BATCH, max(1, _BATCH_ELEMENTS // max(n, c))) + batch_size = max(1, _BATCH_ELEMENTS // (n * m)) batch_size = max(1, int(batch_size)) permute_stats = np.empty(permutations, dtype=np.float64) @@ -646,7 +624,7 @@ def permutation_energy_test( with tqdm(total=permutations, disable=not prog_bar) as bar: while done < permutations: size = min(batch_size, permutations - done) - U = _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], size, label_rngs) + U = _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], size, label_rngs) permute_stats[done : done + size] = _evaluate_statistic(prep, U) done += size bar.update(size) diff --git a/tests/test_pted.py b/tests/test_pted.py index d127d52..685afdf 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -121,7 +121,7 @@ def test_pted_coverage_full(backend): @pytest.mark.parametrize("backend", BACKENDS) -def test_pted_chunk(backend): +def test_landmarks(backend): _require_backend(backend) np.random.seed(42) @@ -129,29 +129,29 @@ def test_pted_chunk(backend): D = 10 x = _to_backend(np.random.normal(size=(1000, D)), backend) y = _to_backend(np.random.normal(size=(1000, D)), backend) - p = pted.pted(x, y, chunk_size=100) + p = pted.pted(x, y, n_landmarks=200) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" y = _to_backend(np.random.uniform(size=(1000, D)), backend) - p = pted.pted(x, y, chunk_size=100) + p = pted.pted(x, y, n_landmarks=200) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" @pytest.mark.parametrize("backend", BACKENDS) -def test_pted_chunk_mismatched_sizes(backend): +def test_landmarks_mismatched_sizes(backend): """Chunked PTED works correctly when x and y have different sizes.""" _require_backend(backend) np.random.seed(0) D = 5 - # x has 200 samples, y has 30 samples; chunk_size=50 → nxc=50, nyc=30 landmarks + # x has 200 samples, y has 30 samples; 80 landmarks puts all 30 y in L x = _to_backend(np.random.normal(size=(200, D)), backend) y = _to_backend(np.random.normal(size=(30, D)), backend) - p = pted.pted(x, y, chunk_size=50) + p = pted.pted(x, y, n_landmarks=80) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" # Different distributions should give small p-value even with mismatched sizes y_diff = _to_backend(np.random.uniform(size=(30, D)), backend) - p = pted.pted(x, y_diff, chunk_size=50) + p = pted.pted(x, y_diff, n_landmarks=80) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" @@ -256,13 +256,13 @@ def test_pted_jax_no_jax(monkeypatch): pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) -def test_pted_chunk_jax_no_jax(monkeypatch): +def test_landmarks_jax_no_jax(monkeypatch): """permutation_energy_test raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): pted.utils.permutation_energy_test( - np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2 + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, n_landmarks=2 ) @@ -275,14 +275,14 @@ def test_pted_torch_no_torch(monkeypatch): pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) -def test_pted_chunk_torch_no_torch(monkeypatch): +def test_landmarks_torch_no_torch(monkeypatch): """permutation_energy_test raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): pted.utils.permutation_energy_test( - np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2 + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, n_landmarks=2 ) @@ -309,31 +309,22 @@ def test_cdist_matches_scipy(backend): @pytest.mark.parametrize("backend", BACKENDS) -def test_pted_n_columns(backend): - """n_columns names the column count directly, and agrees with the - chunk_size that maps onto the same value.""" +def test_pted_n_landmarks(backend): + """n_landmarks selects how many points every sample is measured against.""" _require_backend(backend) np.random.seed(11) x = _to_backend(np.random.normal(size=(300, 6)), backend) y = _to_backend(np.random.normal(size=(300, 6)), backend) - # chunk_size=40 on two equal groups requests 40 + 40 columns - p_chunk = pted.pted(x, y, chunk_size=40, rng=0) - p_cols = pted.pted(x, y, n_columns=80, rng=0) - assert p_chunk == p_cols + p = pted.pted(x, y, n_landmarks=80) + assert 1e-2 < p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" y_diff = _to_backend(np.random.uniform(size=(300, 6)), backend) - assert pted.pted(x, y_diff, n_columns=80) < 1e-2 + assert pted.pted(x, y_diff, n_landmarks=80) < 1e-2 - # n_columns covering the pooled sample is just the full test - assert pted.pted(x, y, n_columns=10_000, rng=3) == pted.pted(x, y, rng=3) - - -def test_pted_column_args_are_exclusive(): - x = np.random.normal(size=(20, 3)) - y = np.random.normal(size=(20, 3)) - with pytest.raises(ValueError, match="not both"): - pted.pted(x, y, chunk_size=5, n_columns=10) + # a landmark count covering the pooled sample is just the full test + assert pted.pted(x, y, n_landmarks=10_000, rng=3) == pted.pted(x, y, rng=3) + assert pted.pted(x, y, n_landmarks=600, rng=3) == pted.pted(x, y, rng=3) def test_pted_rng_is_reproducible(): @@ -342,38 +333,38 @@ def test_pted_rng_is_reproducible(): x = np.random.normal(size=(60, 4)) y = np.random.normal(size=(60, 4)) - a = pted.pted(x, y, permutations=200, n_columns=30, rng=7, return_all=True) - b = pted.pted(x, y, permutations=200, n_columns=30, rng=7, return_all=True) + a = pted.pted(x, y, permutations=200, n_landmarks=30, rng=7, return_all=True) + b = pted.pted(x, y, permutations=200, n_landmarks=30, rng=7, return_all=True) assert a[0] == b[0] and np.array_equal(a[1], b[1]) and a[2] == b[2] np.random.seed(4) - c = pted.pted(x, y, permutations=200, n_columns=30, return_all=True) + c = pted.pted(x, y, permutations=200, n_landmarks=30, return_all=True) np.random.seed(4) - d = pted.pted(x, y, permutations=200, n_columns=30, return_all=True) + d = pted.pted(x, y, permutations=200, n_landmarks=30, return_all=True) assert np.array_equal(c[1], d[1]), "np.random.seed should still pin the permutations" @pytest.mark.parametrize( - "n1,n2,n_columns,regime", + "n1,n2,n_landmarks,regime", [ (60, 60, 24, "proportional"), (1, 100, 11, "singleton"), (1, 40, 30, "singleton"), # c > n/2, so the lone point sits inside C ], ) -def test_chunked_pvalues_are_calibrated(n1, n2, n_columns, regime): +def test_landmark_pvalues_are_calibrated(n1, n2, n_landmarks, regime): """Type-I error under H0 must match the nominal rate. - This is the property the chunked test exists to preserve, and the one the + This is the property the landmark test exists to preserve, and the one the earlier landmark-reshuffling scheme lost: because it re-split the columns by group after each permutation while the observed statistic always used a perfectly balanced split, the observed value was not exchangeable with the permuted ones. It rejected at 29% (proportional) and 84% (singleton) for a nominal 5%. Fixed seed, so this is deterministic rather than flaky. """ - from pted.utils import allocate_columns + from pted.utils import allocate_landmarks - assert allocate_columns(n1, n2, n_columns, rng=0)["regime"] == regime + assert allocate_landmarks(n1, n2, n_landmarks, rng=0)["regime"] == regime trials, permutations = 600, 49 rng = np.random.default_rng(20240904) @@ -383,7 +374,7 @@ def test_chunked_pvalues_are_calibrated(n1, n2, n_columns, regime): x = rng.standard_normal((n1, 4)) y = rng.standard_normal((n2, 4)) pvals[t] = pted.pted( - x, y, permutations=permutations, n_columns=n_columns, two_tailed=False, rng=rng + x, y, permutations=permutations, n_landmarks=n_landmarks, two_tailed=False, rng=rng ) for nominal, tol in ((0.10, 0.05), (0.20, 0.06)): @@ -407,8 +398,8 @@ def test_coverage_test_rng_is_not_shared_across_simulations(): @pytest.mark.parametrize("backend", BACKENDS) -def test_coverage_test_with_chunking(backend): - """Chunked coverage runs end to end and still separates a calibrated +def test_coverage_test_with_landmarks(backend): + """Landmark coverage runs end to end and still separates a calibrated posterior from an over/under-confident one.""" _require_backend(backend) rng = np.random.default_rng(3) @@ -425,8 +416,10 @@ def test_coverage_test_with_chunking(backend): np.stack([v.mean(0) + (v - v.mean(0)) * 0.4 for v in draws], axis=1), backend ) - p_ok = pted.pted_coverage_test(g, ok, permutations=199, chunk_size=30) + p_ok = pted.pted_coverage_test(g, ok, permutations=199, n_landmarks=30) assert 1e-2 < p_ok < 0.99, f"calibrated posterior gave p={p_ok}" - p_over = pted.pted_coverage_test(g, over, permutations=199, chunk_size=30, warn_confidence=None) + p_over = pted.pted_coverage_test( + g, over, permutations=199, n_landmarks=30, warn_confidence=None + ) assert p_over < 1e-2, f"overconfident posterior gave p={p_over}" diff --git a/tests/test_utils.py b/tests/test_utils.py index a2d74f2..34efce7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,7 +7,7 @@ from pted.utils import ( PermutationResolutionWarning, - allocate_columns, + allocate_landmarks, two_tailed_p, simulation_based_calibration_histogram, pit_plot, @@ -19,7 +19,6 @@ _index_like, _prepare_statistic, _random_permutation, - _resolve_n_columns, _to_scalar, ) @@ -154,19 +153,19 @@ def _unbiased_energy(x, y): return (n1 * n2 / (n1 + n2)) * (2 * cdist(x, y).mean() - exx - eyy) -def _brute_block_means(z, small_indicator, cols, n): +def _brute_block_means(z, small_indicator, landmarks, n): """Energy statistic by explicit block sums over the rectangular matrix.""" - D = cdist(z, z[cols]) + D = cdist(z, z[landmarks]) is_s = small_indicator.astype(bool) rows_s, rows_l = np.flatnonzero(is_s), np.flatnonzero(~is_s) - ks, kl = np.flatnonzero(is_s[cols]), np.flatnonzero(~is_s[cols]) + ks, kl = np.flatnonzero(is_s[landmarks]), np.flatnonzero(~is_s[landmarks]) n_s, n_l = len(rows_s), len(rows_l) def block(rows, kk, drop_self): tot = cnt = 0 for i in rows: for k in kk: - if drop_self and cols[k] == i: + if drop_self and landmarks[k] == i: continue tot += D[i, k] cnt += 1 @@ -183,58 +182,60 @@ def block(rows, kk, drop_self): ALLOCATIONS = [ - # n1, n2, n_columns, expected regime + # n1, n2, n_landmarks, expected regime (100, 100, 200, "full"), (3, 40, 43, "full"), (1, 60, 12, "singleton"), # c < n/2 -> the lone point stays out of C (1, 20, 15, "singleton"), # c > n/2 -> it moves into C instead - (6, 80, 40, "small_group_in_C"), + (6, 80, 40, "small_group_in_L"), (40, 50, 30, "proportional"), (40, 50, 3, "proportional"), ] @pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) -def test_allocate_columns_regimes(n1, n2, c, regime): +def test_allocate_landmarks_regimes(n1, n2, c, regime): """Each size combination lands in the intended regime with a consistent set of columns.""" - alloc = allocate_columns(n1, n2, c, rng=0) - cols = alloc["cols"] + alloc = allocate_landmarks(n1, n2, c, rng=0) + landmarks = alloc["landmarks"] n = n1 + n2 assert alloc["regime"] == regime - assert cols.shape == (c,) - assert len(np.unique(cols)) == c, "columns must be distinct points" - assert cols.min() >= 0 and cols.max() < n - assert np.all(np.diff(cols) > 0), "columns are returned sorted" - - # c_small must be the number of columns actually belonging to the small group - assert np.sum(np.isin(cols, alloc["small_idx"])) == alloc["c_small"] - assert alloc["c_small"] + alloc["c_large"] == c - assert alloc["c_large"] >= 1, "the large group must keep at least one column" - assert alloc["exact_within"] == (alloc["c_small"] == alloc["n_small"] or alloc["n_small"] == 1) + assert landmarks.shape == (c,) + assert len(np.unique(landmarks)) == c, "columns must be distinct points" + assert landmarks.min() >= 0 and landmarks.max() < n + assert np.all(np.diff(landmarks) > 0), "columns are returned sorted" + + # n_small_landmarks must be the number of columns actually belonging to the small group + assert np.sum(np.isin(landmarks, alloc["small_idx"])) == alloc["n_small_landmarks"] + assert alloc["n_small_landmarks"] + alloc["n_large_landmarks"] == c + assert alloc["n_large_landmarks"] >= 1, "the large group must keep at least one column" + assert alloc["exact_within"] == ( + alloc["n_small_landmarks"] == alloc["n_small"] or alloc["n_small"] == 1 + ) -def test_allocate_columns_reference_size(): +def test_allocate_landmarks_reference_size(): """reference_size counts the label assignments the subgroup can reach.""" for n1, n2, c, _ in ALLOCATIONS: - alloc = allocate_columns(n1, n2, c, rng=1) - n, n_s, c_s = n1 + n2, alloc["n_small"], alloc["c_small"] + alloc = allocate_landmarks(n1, n2, c, rng=1) + n, n_s, c_s = n1 + n2, alloc["n_small"], alloc["n_small_landmarks"] expected = comb(c, c_s) * comb(n - c, n_s - c_s) assert alloc["reference_size"] == min(expected, 10**15) -def test_allocate_columns_errors(): - """allocate_columns is the single gate on sample and column counts.""" - with pytest.raises(ValueError, match="at least 2 columns"): - allocate_columns(10, 10, 1) +def test_allocate_landmarks_errors(): + """allocate_landmarks is the single gate on sample and column counts.""" + with pytest.raises(ValueError, match="at least 2 landmarks"): + allocate_landmarks(10, 10, 1) with pytest.raises(ValueError, match="exceeds the pooled sample size"): - allocate_columns(10, 10, 21) + allocate_landmarks(10, 10, 21) with pytest.raises(ValueError, match="both samples need at least one point"): - allocate_columns(0, 10, 5) + allocate_landmarks(0, 10, 5) # and it is reached through the public API, not bypassed by the mapping - with pytest.raises(ValueError, match="at least 2 columns"): - permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=1, n_columns=1) + with pytest.raises(ValueError, match="at least 2 landmarks"): + permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=1, n_landmarks=1) with pytest.raises(ValueError, match="both samples need at least one point"): permutation_energy_test(np.zeros((0, 2)), np.zeros((5, 2)), permutations=1) @@ -246,18 +247,20 @@ def test_draw_labels_stays_in_subgroup(n1, n2, c, regime): fixed. This is what makes the observed labelling exchangeable with the permuted ones.""" rng = np.random.default_rng(3) - alloc = allocate_columns(n1, n2, c, rng) + alloc = allocate_landmarks(n1, n2, c, rng) n = n1 + n2 base = np.zeros(n) base[alloc["small_idx"]] = 1.0 - in_c = np.zeros(n, bool) - in_c[alloc["cols"]] = True + in_l = np.zeros(n, bool) + in_l[alloc["landmarks"]] = True - U = _draw_labels(base, np.flatnonzero(in_c), np.flatnonzero(~in_c), 200, rng.spawn(2)) + U = _draw_labels(base, np.flatnonzero(in_l), np.flatnonzero(~in_l), 200, rng.spawn(2)) assert np.all(np.isin(U, [0.0, 1.0])) assert np.all(U.sum(1) == alloc["n_small"]), "small group size is preserved" - assert np.all(U[:, alloc["cols"]].sum(1) == alloc["c_small"]), "column counts are preserved" + assert np.all( + U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"] + ), "column counts are preserved" # and the labels really do move around within each part if alloc["reference_size"] > 100: assert len(np.unique(U, axis=0)) > 1 @@ -272,18 +275,18 @@ def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): y = rng.standard_normal((n2, 4)) + 0.4 z = np.vstack([x, y]) - alloc = allocate_columns(n1, n2, c, rng) - D = cdist(z, z[alloc["cols"]]) + alloc = allocate_landmarks(n1, n2, c, rng) + D = cdist(z, z[alloc["landmarks"]]) prep = _prepare_statistic(D, alloc, "numpy") U = np.concatenate( [ prep["base_small"][None, :], - _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], 5, rng.spawn(2)), + _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], 5, rng.spawn(2)), ] ) got = _evaluate_statistic(prep, U) - expect = [_brute_block_means(z, U[b], alloc["cols"], n1 + n2) for b in range(len(U))] + expect = [_brute_block_means(z, U[b], alloc["landmarks"], n1 + n2) for b in range(len(U))] assert np.allclose(got, expect, atol=1e-9) @@ -308,20 +311,20 @@ def test_statistic_backend_agreement(backend): y = rng.standard_normal((n2, 5)) + 0.3 z = np.vstack([x, y]) - alloc = allocate_columns(n1, n2, c, rng) + alloc = allocate_landmarks(n1, n2, c, rng) U = None results = {} for be in ("numpy", backend): zb = {"numpy": lambda: z, "torch": lambda: torch.tensor(z), "jax": lambda: jnp.array(z)}[ be ]() - D = _cdist(zb, zb[_index_like(alloc["cols"], zb, be)], be) + D = _cdist(zb, zb[_index_like(alloc["landmarks"], zb, be)], be) prep = _prepare_statistic(D, alloc, be) if U is None: U = np.concatenate( [ prep["base_small"][None, :], - _draw_labels(prep["base_small"], prep["idx_c"], prep["idx_o"], 8, rng.spawn(2)), + _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], 8, rng.spawn(2)), ] ) results[be] = _evaluate_statistic(prep, U) @@ -333,10 +336,10 @@ def test_draw_labels_independent_of_batch_size(): """The two label streams are consumed row by row, so splitting a run into batches yields bit-identical permutations.""" base = np.r_[np.ones(5), np.zeros(15)] - idx_c, idx_o = np.arange(12), np.arange(12, 20) - whole = _draw_labels(base, idx_c, idx_o, 10, np.random.default_rng(9).spawn(2)) + idx_l, idx_o = np.arange(12), np.arange(12, 20) + whole = _draw_labels(base, idx_l, idx_o, 10, np.random.default_rng(9).spawn(2)) rngs = np.random.default_rng(9).spawn(2) - split = np.concatenate([_draw_labels(base, idx_c, idx_o, k, rngs) for k in (3, 3, 4)]) + split = np.concatenate([_draw_labels(base, idx_l, idx_o, k, rngs) for k in (3, 3, 4)]) assert np.array_equal(whole, split) @@ -344,7 +347,7 @@ def test_statistic_independent_of_batch_size(): """Batching is purely a memory/throughput knob: with the same seed it draws the same permutations, so the statistics agree to within the reordering that a different matmul shape costs in floating point.""" - rng_kwargs = dict(permutations=64, n_columns=30, rng=12345) + rng_kwargs = dict(permutations=64, n_landmarks=30, rng=12345) x = np.random.default_rng(0).standard_normal((50, 4)) y = np.random.default_rng(1).standard_normal((70, 4)) ref_stat, ref_perm = permutation_energy_test(x, y, batch_size=1, **rng_kwargs) @@ -356,25 +359,14 @@ def test_statistic_independent_of_batch_size(): ), f"batch_size={batch} changed the null draws" -def test_resolve_n_columns(): - # chunk_size reproduces the number of distance columns it used to request - assert _resolve_n_columns(1000, 1000, chunk_size=100) == 200 - assert _resolve_n_columns(200, 30, chunk_size=50) == 80 - # covering both datasets falls back to the full matrix - assert _resolve_n_columns(20, 30, chunk_size=40) == 50 - # n_columns is taken literally, capped at the pooled size - assert _resolve_n_columns(20, 30, n_columns=17) == 17 - assert _resolve_n_columns(20, 30, n_columns=500) == 50 - # neither means the full matrix - assert _resolve_n_columns(20, 30) == 50 - - with pytest.raises(ValueError, match="not both"): - _resolve_n_columns(20, 30, chunk_size=5, n_columns=5) - with pytest.raises(ValueError, match="chunk_size must be positive"): - _resolve_n_columns(20, 30, chunk_size=0) - # a nonsense n_columns passes through here and is rejected by - # allocate_columns, which owns every column-count check - assert _resolve_n_columns(20, 30, n_columns=1) == 1 +def test_n_landmarks_covering_the_sample_is_the_full_test(): + """None, or any count at or above the pooled size, is the exact test.""" + x = np.random.default_rng(0).standard_normal((20, 3)) + y = np.random.default_rng(1).standard_normal((30, 3)) + ref = permutation_energy_test(x, y, permutations=32, rng=5) + for m in (50, 500): + stat, perm = permutation_energy_test(x, y, permutations=32, n_landmarks=m, rng=5) + assert stat == ref[0] and np.array_equal(perm, ref[1]) def test_singleton_picks_the_larger_side_of_C(): @@ -384,12 +376,12 @@ def test_singleton_picks_the_larger_side_of_C(): n2 = 200 n = 1 + n2 for c in range(2, n): - alloc = allocate_columns(1, n2, c, rng=0) + alloc = allocate_landmarks(1, n2, c, rng=0) assert alloc["regime"] == "singleton" - assert alloc["c_small"] == (1 if c > n - c else 0) + assert alloc["n_small_landmarks"] == (1 if c > n - c else 0) assert alloc["reference_size"] == max(c, n - c) # so the reference set never drops below half the pooled sample - worst = min(allocate_columns(1, n2, c, rng=0)["reference_size"] for c in range(2, n)) + worst = min(allocate_landmarks(1, n2, c, rng=0)["reference_size"] for c in range(2, n)) assert worst >= n // 2 @@ -400,18 +392,24 @@ def test_permutation_resolution_warning(): # singleton: only a genuinely tiny pooled sample is short of assignments now with pytest.warns(PermutationResolutionWarning, match="single point"): permutation_energy_test( - rng.standard_normal((1, 3)), rng.standard_normal((10, 3)), permutations=100, n_columns=5 + rng.standard_normal((1, 3)), + rng.standard_normal((10, 3)), + permutations=100, + n_landmarks=5, ) - # small_group_in_C: too FEW columns is the failure mode here - with pytest.warns(PermutationResolutionWarning, match="use more columns"): + # small_group_in_L: too FEW columns is the failure mode here + with pytest.warns(PermutationResolutionWarning, match="use more landmarks"): permutation_energy_test( - rng.standard_normal((2, 3)), rng.standard_normal((30, 3)), permutations=100, n_columns=5 + rng.standard_normal((2, 3)), + rng.standard_normal((30, 3)), + permutations=100, + n_landmarks=5, ) # sensible column counts are quiet, at either end of the singleton range x, y = rng.standard_normal((1, 3)), rng.standard_normal((60, 3)) with warnings.catch_warnings(): warnings.simplefilter("error", PermutationResolutionWarning) - permutation_energy_test(x, y, permutations=100, n_columns=8) - permutation_energy_test(x, y, permutations=100, n_columns=55) + permutation_energy_test(x, y, permutations=100, n_landmarks=8) + permutation_energy_test(x, y, permutations=100, n_landmarks=55) From a192a4795b2ecfdbeb9713f907c6e52de6c09abf Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Tue, 8 Sep 2026 14:35:10 -0400 Subject: [PATCH 4/7] refactor for better PIT plot and perforamnce upgrade --- README.md | 25 ++- src/pted/pted.py | 47 ++++- src/pted/tests.py | 13 +- src/pted/utils.py | 503 +++++++++++++++++++++++++++++++++----------- tests/test_pted.py | 18 +- tests/test_utils.py | 343 +++++++++++++++++++++++------- 6 files changed, 724 insertions(+), 225 deletions(-) diff --git a/README.md b/README.md index f5fd6b4..3ff8c0e 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,12 @@ print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 Note, you can also provide a filename via a parameter: `sbc_histogram = "sbc_hist.pdf"` and this will generate an SBC histogram from the test[^1]. -You can also generate a Probability Integral Transform (PIT) plot via `pit_plot = "pit.pdf"` for `pted_coverage_test`. The PIT plot shows the empirical CDF of the p-values against the expected uniform CDF (1:1 diagonal), along with a shaded KS confidence band determined by `pit_confidence` (95% by default). Any portion of the curve that deviates outside the band is evidence that the p-values are not uniformly distributed, at a significance level of `1 - pit_confidence` (5% by default), indicating a potentially miscalibrated posterior. +You can also generate a Probability Integral Transform (PIT) plot via `pit_plot = "pit.pdf"` for `pted_coverage_test`. The PIT plot shows the empirical CDF of the p-values against the expected uniform CDF (1:1 diagonal), along with a shaded band determined by `pit_confidence` (95% by default). The band is simultaneous: under the null the whole curve stays inside it with probability `pit_confidence`, so any portion of the curve that pokes out is evidence of non-uniform p-values at significance `1 - pit_confidence`, indicating a potentially miscalibrated posterior. + +The band is built from the exact null distribution of the p-values. The ECDF can only move where the p-values can actually land, so rather than constraining order statistics the band constrains the counts there. +Working with counts rather than order statistics is what makes this usable here. Permutation p-values are discrete — after enumeration they are exactly uniform on a grid of `L = len(permute) + 1` points — and ties are precisely what a binomial count expects. +`pted_coverage_test` works out the lattice size and passes it in for you. +Ultimately, this means that the fact we only use a finite number of simulations to compute the coverage test does not affect the validity of the bands in the PIT plot; any point that sticks out of the grey shaded area indicates possible miscalibration at the given significance level (default 95%). ## How it works @@ -352,7 +357,7 @@ def pted_coverage_test( * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. -* **pit_confidence** *(float)*: Confidence level for the KS confidence band in the PIT plot. Default is 0.95 (95%). Only used when `pit_plot` is not None. +* **pit_confidence** *(float)*: Confidence level for the PIT plot's simultaneous band. Default is 0.95 (95%). Only used when `pit_plot` is not None. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of simulations. Default is False. * **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. * **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. @@ -418,6 +423,22 @@ p_value = pted(x, y, n_landmarks = 200) print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 ``` +### Enumerating small permutation groups + +If the permutation subgroup turns out to have no more than `permutations` +members, PTED evaluates *all* of it rather than sampling. Sampling a small +group keeps redrawing the observed labelling, and each of those ties inflates +the p-value under the `>=` convention: with a group of 151 and 199 draws, +`P(p <= 0.02)` came out at 0.0169 instead of 0.0200. Walking the group instead +puts it at 0.0195, leaving only the unavoidable granularity of a 151-point +lattice — and it costs *less* compute, since 151 evaluations beat 199. This +mostly matters for `pted_coverage_test`, where the ground truth is a single +point and the group is only `max(m, n - m)` large. + +One consequence: when this kicks in, `return_all` gives back +`reference_size - 1` permuted statistics rather than `permutations` of them. +The p-value is then exact rather than sampled. + ### Why using landmarks is still exact The subtlety is that the landmarks cannot simply be re-split between the two diff --git a/src/pted/pted.py b/src/pted/pted.py index 08514d0..7b687c3 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -83,6 +83,9 @@ def pted( accurately the p-value is computed. return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. + The permuted statistics are ``permutations`` random draws, or the + whole permutation subgroup when that is smaller (see the note + below), so check its length rather than assuming. bool (default: False) n_landmarks (Optional[int]): if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise @@ -129,6 +132,13 @@ def pted( attainable p-value is about the reciprocal of that, however many permutations are drawn. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. + When the permutation subgroup has no more than ``permutations`` + members, PTED walks the whole group instead of sampling it. That is + both cheaper and exact: sampling a small group keeps redrawing the + observed labelling, and every such tie inflates the p-value under the + ``>=`` convention. In that case the returned array of permuted + statistics holds ``reference_size - 1`` entries rather than + ``permutations``. """ assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" @@ -153,14 +163,17 @@ def pted( rng=rng, ) - # Compute p-value + # Compute p-value. The null set is `permutations` random draws, or the + # whole permutation subgroup when that is smaller, so count what came back + # rather than what was asked for. + n_null = len(permute) if two_tailed: q = 2 * min(np.sum(permute >= test), np.sum(permute <= test)) - q = min(q, permutations) + q = min(q, n_null) else: q = np.sum(permute >= test) - pval = (1.0 + q) / (1.0 + permutations) + pval = (1.0 + q) / (1.0 + n_null) if return_all: return test, permute, pval @@ -250,11 +263,11 @@ def pted_coverage_test( Probability Integral Transform (PIT) plot. The plot shows the empirical CDF of the per-simulation p-values against the expected uniform CDF (1:1 diagonal), together with a shaded KS confidence - band. Deviations outside the band indicate that the p-values are - not uniformly distributed. Default is None (no plot saved). - pit_confidence (float): Confidence level for the KS confidence band - in the PIT plot. Default is 0.95 (95%). Only used when - ``pit_plot`` is not None. + band built from the exact null of the p-values. Deviations + outside the band indicate that the p-values are not uniformly + distributed. Default is None (no plot saved). + pit_confidence (float): Confidence level for the PIT plot's band. + Default is 0.95 (95%). Only used when ``pit_plot`` is not None. prog_bar (bool): If True, show a progress bar to track the progress of simulations. Default is False. batch_size (Optional[int]): number of permutations evaluated per matrix @@ -291,6 +304,13 @@ def pted_coverage_test( attainable p-value is about the reciprocal of that, however many permutations are drawn. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. + When the permutation subgroup has no more than ``permutations`` + members, PTED walks the whole group instead of sampling it. That is + both cheaper and exact: sampling a small group keeps redrawing the + observed labelling, and every such tie inflates the p-value under the + ``>=`` convention. In that case the returned array of permuted + statistics holds ``reference_size - 1`` entries rather than + ``permutations``. """ nsamp, nsim, *_ = s.shape assert nsim > 0, "need some simulations to run test, got 0 simulations" @@ -329,12 +349,19 @@ def pted_coverage_test( # Simulation-Based-Calibration histogram if sbc_histogram is not None: - ranks = np.sum(test_stats[:, None] >= permute_stats, axis=1) / permutations + ranks = np.sum(test_stats[:, None] >= permute_stats, axis=1) / permute_stats.shape[1] simulation_based_calibration_histogram(ranks, sbc_histogram, bins=sbc_bins) # Probability Integral Transform (PIT) plot if pit_plot is not None: - _pit_plot(pvals, pit_plot, confidence=pit_confidence) + # Under H0 a p-value of (1 + q) / (1 + n_null) is uniform on that many + # lattice points, so the band can be built from the exact null. + _pit_plot( + pvals, + pit_plot, + confidence=pit_confidence, + lattice=permute_stats.shape[1] + 1, + ) # Compute p-value if nsim == 1: diff --git a/src/pted/tests.py b/src/pted/tests.py index 3ea834f..ad3062f 100644 --- a/src/pted/tests.py +++ b/src/pted/tests.py @@ -6,11 +6,14 @@ def test(): np.random.seed(42) # example 2 sample test D = 300 - for _ in range(20): - x = np.random.normal(size=(100, D)) - y = np.random.normal(size=(100, D)) - p = pted(x, y) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" + # Under the null these are U(0,1), so any single draw can land anywhere: + # check the batch, not each p-value. Asserting 0.01 < p < 0.99 on each of + # 20 uniform draws fails a third of the time on its own. + pvals = [ + pted(np.random.normal(size=(100, D)), np.random.normal(size=(100, D))) for _ in range(20) + ] + median = float(np.median(pvals)) + assert 0.1 < median < 0.9, f"null p-values are not U(0,1): median {median}, {pvals}" x = np.random.normal(size=(100, D)) y = np.random.uniform(size=(100, D)) diff --git a/src/pted/utils.py b/src/pted/utils.py index 35695b0..f3b2e6c 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -65,13 +65,14 @@ unbiased distance covariance. Only its rank among the permutations matters. """ +from itertools import combinations, islice, product from math import comb from typing import Optional from warnings import warn import numpy as np from scipy.spatial.distance import cdist -from scipy.stats import chi2 as chi2_dist, binom, kstwo, kstest +from scipy.stats import chi2 as chi2_dist, binom from tqdm.auto import tqdm try: @@ -148,14 +149,6 @@ def _all_finite(z, backend: str) -> bool: return bool(np.all(np.isfinite(z))) -def _to_scalar(x, backend: str) -> float: - if backend == "torch": - return float(x.item()) - if backend == "jax": - return float(x.item()) - return float(x) - - def _to_numpy(x, backend: str) -> np.ndarray: if backend == "torch": return x.detach().cpu().numpy() @@ -180,16 +173,6 @@ def _index_like(idx, ref, backend: str): return np.asarray(idx) -def _random_permutation(D, backend: str = "numpy"): - if backend == "torch": - I = torch.randperm(D.shape[0], device=D.device) - elif backend == "jax": - I = jax.random.permutation(jax.random.PRNGKey(np.random.randint(0, 1e6)), D.shape[0]) - else: - I = np.random.permutation(D.shape[0]) - return D[I][:, I] - - @jit def _jax_cdist(x, y): return jax.vmap(lambda xi: jnp.linalg.norm(xi - y, ord=2.0, axis=-1))(x) @@ -398,30 +381,94 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: ) -def _draw_labels(base_small, idx_l, idx_o, n_draws: int, rngs) -> np.ndarray: - """``(n_draws, n)`` indicator of small-group membership. +def _label_drawer(prep: dict, rng): + """Return ``draw(rows) -> (rows, n)`` small-group indicator matrices. - Drawn uniformly from the subgroup ``S_L x S_{L^c}`` by shuffling the labels - within the landmark positions ``idx_l`` and within their complement - ``idx_o``, never between. Permutation bookkeeping stays in numpy: it is pure integer - shuffling, and only the resulting indicator matrix crosses to the compute - backend. + Each part -- the landmark positions and their complement -- gets a + uniformly random subset of the size the subgroup fixes, found by ranking + uniform keys: the ``k`` smallest of ``|part|`` iid uniforms are a uniform + random ``k``-subset. Ranking is what makes this a device operation, so the + labels are built wherever the distance matrix lives rather than shuffled on + the host and shipped over -- for large ``n`` that shuffle costs more than + the matrix product it feeds, and no accelerator can help with it. - The two parts draw from separate generators in ``rngs``. ``permuted`` - consumes the stream row by row, so changing ``batch_size`` doesn't affect - the sequence of permutations. + The returned function carries the generator state, so successive calls + continue the same stream. Which permutations that yields depends on how the + run is batched; the test is exact either way, and a fixed ``rng`` with a + fixed ``batch_size`` reproduces a run exactly. """ - n = base_small.size - U = np.empty((n_draws, n), dtype=base_small.dtype) - for idx, rng in zip((idx_l, idx_o), rngs): - if idx.size == 0: - continue - block = rng.permuted(np.repeat(base_small[idx][None, :], n_draws, axis=0), axis=1) - if idx.size == n: - U[:] = block - else: - U[:, idx] = block - return U + backend, ref = prep["backend"], prep["ref"] + n = prep["base_small"].size + parts = [(_index_like(idx, ref, backend), k) for idx, k in prep["parts"] if k] + + if backend == "torch": + gen = torch.Generator(device=ref.device) + gen.manual_seed(int(rng.integers(0, 2**62))) + + def draw(rows): + U = torch.zeros((rows, n), dtype=ref.dtype, device=ref.device) + for idx, k in parts: + keys = torch.rand((rows, idx.numel()), generator=gen, device=ref.device) + pick = torch.topk(keys, k, dim=1, largest=False, sorted=False).indices + U.scatter_(1, idx[pick], 1.0) + return U + + elif backend == "jax": + key = jax.random.PRNGKey(int(rng.integers(0, 2**62))) + + def draw(rows): + nonlocal key + U = jnp.zeros((rows, n), dtype=ref.dtype) + index = jnp.arange(rows)[:, None] + for idx, k in parts: + key, subkey = jax.random.split(key) + keys = jax.random.uniform(subkey, (rows, int(idx.size))) + U = U.at[index, idx[jax.lax.top_k(-keys, k)[1]]].set(1.0) + return U + + else: + + def draw(rows): + U = np.zeros((rows, n), dtype=ref.dtype) + for idx, k in parts: + keys = rng.random((rows, idx.size)) + pick = np.argpartition(keys, k - 1, axis=1)[:, :k] + np.put_along_axis(U, idx[pick], U.dtype.type(1.0), axis=1) + return U + + return draw + + +def _sampled_label_blocks(prep: dict, permutations: int, batch_size: int, rng): + """Yield blocks of randomly drawn labels, ``permutations`` rows in total.""" + draw = _label_drawer(prep, rng) + for start in range(0, permutations, batch_size): + yield draw(min(batch_size, permutations - start)) + + +def _enumerate_label_blocks(prep: dict, batch_size: int): + """Yield blocks covering the whole subgroup, minus the observed labelling. + + The subgroup fixes how many small labels sit among the landmarks, so it is + exactly: which of the landmark positions carry a small label, crossed with + which of the remaining positions carry the rest. Dropping the observed + assignment leaves ``reference_size - 1`` rows, so the caller's + ``(1 + q) / (1 + len)`` becomes the exact rank of the observed statistic + within the group -- no sampling, and no ties from redrawing the observed + labelling. + + Only ever called when ``reference_size`` is small, so materialising the + group a block at a time is cheap. + """ + base = prep["base_small"] + assignments = product(*(combinations(idx, k) for idx, k in prep["parts"])) + while chunk := list(islice(assignments, batch_size)): + U = np.zeros((len(chunk), base.size), dtype=base.dtype) + for j, picks in enumerate(chunk): + U[j, [i for pick in picks for i in pick]] = 1.0 + keep = np.any(U != base, axis=1) + if keep.any(): + yield U[keep] # ---------------------------------------------------------------- statistic @@ -435,15 +482,24 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: u @ D @ v = sum_{i in A} sum_{k: landmarks[k] in A} D[i, k] - Only that one product is needed per permutation; the other three blocks - follow from the row sums, column sums and grand total, since the two - indicators of each pair sum to one. All denominators are constants, because - the subgroup ``S_L x S_{L^c}`` preserves the per-group landmark counts. + All the denominators are constants, because the subgroup ``S_L x S_{L^c}`` + preserves the per-group landmark counts. That lets the whole statistic + collapse into a single affine function of at most three per-permutation + scalars, whose coefficients are folded down to plain Python floats here: + + E = a_ss * S_ss + a_row * R + a_col * C + const + + where ``R = u @ D @ 1`` and ``C = v @ (1 @ D)``. Everything else -- the + other three block sums, the pair counts, the ``n_s n_l / n`` scaling -- is + absorbed into those four numbers, so a permutation costs one matrix + product and a handful of elementwise ops rather than a dozen small + kernels on device. """ n_s, n_l = alloc["n_small"], alloc["n_large"] n = n_s + n_l m_s, m_l = alloc["n_small_landmarks"], alloc["n_large_landmarks"] landmarks = alloc["landmarks"] + full = alloc["regime"] == "full" # allocate_landmarks guarantees m_l >= 1 in every regime, and m_s >= 1 # unless n_s == 1 (the singleton regime, where the small group has no @@ -456,36 +512,70 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: # cancels exactly, so the offset itself needs no precision -- a float32 # mean is fine. That is not true of `total` below, whose error survives # into the reported statistic, so that one accumulates in float64. - offset = _to_scalar(D.mean(), backend) + offset = float(D.mean()) D = D - offset D = _zero_self_pairs(D, landmarks, backend) row_sums = D.sum(1) + total = float(_to_numpy(row_sums, backend).sum(dtype=np.float64)) + + # --- fold 2*mu_sl - mu_ss - mu_ll into A*S_ss + B*S_sl + C*S_ls + const -- + a, b, c, const = 0.0, 0.0, 0.0, 0.0 + if m_s > 0: # both cross orientations are available + b += 1.0 / (n_s * m_l) + c += 1.0 / (n_l * m_s) + else: # no small landmarks, so only one orientation exists + b += 2.0 / (n_s * m_l) + if n_s < 2: # a lone point has no within-group pairs; its mean is -offset + const += offset + else: + a -= 1.0 / (m_s * (n_s - 1)) + if n_l < 2: + const += offset + else: # S_ll = total - S_ss - S_sl - S_ls + pairs_ll = m_l * (n_l - 1) + a += 1.0 / pairs_ll + b += 1.0 / pairs_ll + c += 1.0 / pairs_ll + const -= total / pairs_ll + + # S_sl = R - S_ss and S_ls = C - S_ss, so rewrite in terms of what a + # permutation actually computes, then absorb the n_s n_l / n scaling. + scale = n_s * n_l / n + a_ss, a_row, a_col = scale * (a - b - c), scale * b, scale * c + const *= scale + + # With no small landmarks both S_ss and C are identically zero, so the + # matrix product is not needed at all -- a matrix-vector product against + # the row sums is the whole statistic. + needs_matrix = m_s > 0 + if full: + # D is symmetric, so the column sums are the row sums and C == R. + a_row, a_col = a_row + a_col, 0.0 + base_small = np.zeros(n) base_small[alloc["small_idx"]] = 1.0 in_l = np.zeros(n, dtype=bool) in_l[landmarks] = True + idx_l, idx_o = np.flatnonzero(in_l), np.flatnonzero(~in_l) + # Whatever the statistic reads sets the device and dtype the labels are + # built in, so they never have to cross a bus. + ref = D if needs_matrix else row_sums return { - "D": D, + "D": D if needs_matrix else None, + "ref": ref, + # (positions, how many of them carry the small label) for each half of + # the subgroup; those counts are what the subgroup holds fixed. + "parts": ((idx_l, m_s), (idx_o, n_s - m_s)), # The landmarks cover everything in the full regime, so skip the gather. - "landmarks": None if alloc["regime"] == "full" else _index_like(landmarks, D, backend), - "offset": offset, - "row_sums": row_sums, - "col_sums": D.sum(0), - "total": float(_to_numpy(row_sums, backend).sum(dtype=np.float64)), - "n": n, - "n_s": n_s, - "n_l": n_l, - "m_s": m_s, - "m_l": m_l, - # Landmark landmarks[k] is also row landmarks[k], so each within-group block - # holds exactly c_g zero-valued self-pairs; drop them from the counts. - "pairs_ss": m_s * (n_s - 1), - "pairs_ll": m_l * (n_l - 1), + "landmarks": None if full else _index_like(landmarks, D, backend), + "col_sums": D.sum(0) if (needs_matrix and a_col) else None, + "a_ss": a_ss, + "a_row": a_row, + "a_col": a_col, + "const": const, "base_small": base_small, - "idx_l": np.flatnonzero(in_l), - "idx_o": np.flatnonzero(~in_l), "backend": backend, } @@ -502,36 +592,21 @@ def _evaluate_statistic(prep: dict, U) -> np.ndarray: ------- ``(B,)`` numpy array of statistics. """ - backend = prep["backend"] - D = prep["D"] + backend, D = prep["backend"], prep["D"] + Us = _asarray_like(U, prep["ref"], backend) + + if D is None: # no small landmarks, so ref is the row sums and they are the whole statistic + stat = prep["a_row"] * (Us @ prep["ref"]) + prep["const"] + return _to_numpy(stat, backend).astype(np.float64, copy=False) - Us = _asarray_like(U, D, backend) Vs = Us if prep["landmarks"] is None else Us[:, prep["landmarks"]] - S_ss = ((Us @ D) * Vs).sum(-1) # small rows, small landmarks - S_sl = Us @ prep["row_sums"] - S_ss # small rows, large landmarks - S_ls = Vs @ prep["col_sums"] - S_ss # large rows, small landmarks - S_ll = prep["total"] - S_ss - S_sl - S_ls # large rows, large landmarks - - n_s, n_l = prep["n_s"], prep["n_l"] - m_s, m_l = prep["m_s"], prep["m_l"] - - # A group of one point has no within-group pairs, so its mean is zero on - # the raw distances -- which is -offset on the centred ones. Writing it - # that way keeps the 2 - 1 - 1 cancellation that makes the statistic - # independent of the centring, and so keeps the reported value equal to - # the uncentred statistic rather than merely rank-equivalent to it. - off = prep["offset"] - mu_ss = -off if n_s < 2 else S_ss / prep["pairs_ss"] - mu_ll = -off if n_l < 2 else S_ll / prep["pairs_ll"] - - # Cross term: mean over whichever orientations have landmarks available. - parts = [S_sl / (n_s * m_l)] - if m_s > 0: - parts.append(S_ls / (n_l * m_s)) - mu_sl = sum(parts) / len(parts) - - stat = (n_s * n_l / prep["n"]) * (2.0 * mu_sl - mu_ss - mu_ll) + # One matrix product carries both scalars: W @ 1 is the row-sum term and + # (W * Vs) summed is the small-by-small block. + W = Us @ D + stat = prep["a_ss"] * (W * Vs).sum(-1) + prep["a_row"] * W.sum(-1) + prep["const"] + if prep["col_sums"] is not None: + stat = stat + prep["a_col"] * (Vs @ prep["col_sums"]) return _to_numpy(stat, backend).astype(np.float64, copy=False) @@ -549,10 +624,10 @@ def permutation_energy_test( ) -> tuple[float, np.ndarray]: """Observed energy statistic and its permutation distribution. - Without ``n_landmarks`` this is the exact full-matrix test: the ``(n, n)`` distance matrix is built once and the - statistic is evaluated for batches of permutations as a bilinear form in - the group indicator, so no distance is recomputed and no matrix is - re-indexed. Asking for fewer landmarks switches to the rectangular + Without ``n_landmarks`` this is the exact full-matrix test: the ``(n, n)`` + distance matrix is built once and the statistic is evaluated for batches of + permutations as a bilinear form in the group indicator, so no distance is + recomputed and no matrix is re-indexed. Asking for fewer landmarks switches to the rectangular ``(n, m)`` matrix, cutting the cost from ``O(n^2 d)`` to ``O(n m d)``; permutations are then confined to the subgroup that fixes the landmark set, which keeps the p-value exact. See the module docstring for the @@ -579,8 +654,12 @@ def permutation_energy_test( Returns ------- - ``(test_stat, permute_stats)``, the observed statistic and a - ``(permutations,)`` array of permuted statistics. + ``(test_stat, permute_stats)``, the observed statistic and an array of + permuted statistics. That array holds ``permutations`` random draws -- + unless the subgroup has no more than ``permutations`` members, in which + case it holds every one of them except the observed labelling + (``reference_size - 1`` entries) and the resulting p-value is exact + rather than sampled. """ if is_torch_tensor(x): assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" @@ -618,16 +697,26 @@ def permutation_energy_test( batch_size = max(1, _BATCH_ELEMENTS // (n * m)) batch_size = max(1, int(batch_size)) - permute_stats = np.empty(permutations, dtype=np.float64) - label_rngs = rng.spawn(2) + # Sampling more permutations than the subgroup has members just redraws the + # observed labelling over and over, and every such tie inflates q under the + # >= convention. Below that point, walking the whole group is both exact + # and cheaper. + if alloc["reference_size"] <= permutations: + n_null = int(alloc["reference_size"]) - 1 + blocks = _enumerate_label_blocks(prep, batch_size) + else: + n_null = permutations + blocks = _sampled_label_blocks(prep, permutations, batch_size, rng) + + permute_stats = np.empty(n_null, dtype=np.float64) done = 0 - with tqdm(total=permutations, disable=not prog_bar) as bar: - while done < permutations: - size = min(batch_size, permutations - done) - U = _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], size, label_rngs) + with tqdm(total=n_null, disable=not prog_bar) as bar: + for U in blocks: + size = len(U) permute_stats[done : done + size] = _evaluate_statistic(prep, U) done += size bar.update(size) + assert done == n_null, f"produced {done} null statistics, expected {n_null}" return test_stat, permute_stats @@ -697,27 +786,179 @@ def simulation_based_calibration_histogram(ranks, saveto, bins=None): plt.close() -def pit_plot(pvals, saveto, confidence=0.95): - """Create a Probability Integral Transform (PIT) plot. +def _lattice_counts(pvals, t): + """``N_k = #{p <= t_k}`` for each evaluation point, for one or many rows.""" + p = np.atleast_2d(pvals) + rows, k = p.shape[0], len(t) + flat = np.searchsorted(t, p, side="left") + np.arange(rows)[:, None] * k + return np.cumsum(np.bincount(flat.ravel(), minlength=rows * k).reshape(rows, k), axis=1) + + +def _band_bounds(local, eta): + """Counts whose local level clears ``eta``, per evaluation point. + + ``local[k]`` is a lower tail min'd with an upper tail, so it rises then + falls and the set is a contiguous interval of counts. + """ + inside = local > eta + lower = np.argmax(inside, axis=1) + return lower, lower + inside.sum(axis=1) - 1 - Plots the empirical CDF of the provided p-values against the expected - CDF for a uniform distribution (the 1:1 diagonal). A shaded confidence - region is drawn showing the range within which the empirical CDF should - fall with probability ``confidence`` if the p-values are truly uniform. - The confidence band is derived from the two-sided Kolmogorov-Smirnov - statistic. Any portion of the empirical CDF that lies outside this band - constitutes evidence that the p-values are not uniformly distributed. - The KS statistic and its p-value are annotated on the plot to quantify - the maximum deviation from the diagonal. +def _band_accept_probability(t, n, lower, upper): + """``P(lower_k <= N_k <= upper_k for every k)`` under the null, exactly. + + The counting process is Markov: given ``N_k = j`` the other ``n - j`` + p-values are iid on ``(t_k, 1]``, so the increment is + ``Binomial(n - j, rho_k)`` with ``rho_k = (F0(t_k+1) - F0(t_k)) / (1 - + F0(t_k))``. Pushing the distribution of ``N_k`` forward through those + transitions, zeroing everything outside the interval at each step, leaves + exactly the probability of never having escaped. No simulation needed. + """ + f = binom.pmf(np.arange(n + 1), n, t[0]) + f[: lower[0]] = 0.0 + f[upper[0] + 1 :] = 0.0 + for k in range(len(t) - 1): + rho = 1.0 if t[k] >= 1.0 else (t[k + 1] - t[k]) / (1.0 - t[k]) + j = np.arange(lower[k], upper[k] + 1) + j_next = np.arange(lower[k + 1], upper[k + 1] + 1) + step = binom.pmf(j_next[None, :] - j[:, None], (n - j)[:, None], rho) + nxt = np.zeros(n + 1) + nxt[lower[k + 1] : upper[k + 1] + 1] = f[lower[k] : upper[k] + 1] @ step + f = nxt + return float(f.sum()) + + +def _lattice_band(n, lattice, confidence, max_points=512): + """Simultaneous ECDF band built from the known null of the p-values. + + The ECDF only moves where the p-values can actually land, so this works + with the counts there rather than with order statistics: + + N_k = #{i : p_i <= t_k} ~ Binomial(n, F0(t_k)) exactly + + Every evaluation point gets the same two-sided *local* level ``eta``, + chosen as large as it can be while the probability of *any* count escaping + its interval stays within ``1 - confidence``. The ECDF then leaves the band + with at most that probability under H0, so "pokes out of the band" reads as + "reject", simultaneously over the whole curve. The boundary follows the + real ``n F0(t) (1 - F0(t))`` variance of the count and pinches to nothing + at both corners, rather than holding one constant half-width across the + plot. + + Working with counts rather than order statistics is what lets this handle + *discrete* p-values: ties are exactly what a binomial count expects, + whereas the Beta distribution of an order statistic assumes they never + happen. + + Everything here is exact -- both the escape probability, via + :func:`_band_accept_probability`, and the search for ``eta``, which is a + bisection over the finitely many local levels the table can take. There is + no simulation and no seed. + + Because the null is discrete the attainable levels are discrete too, so the + band is generally a little conservative; the level it actually achieves is + returned rather than assumed. + + Parameters + ---------- + n (int): number of p-values. + lattice (Optional[int]): ``L`` if the p-values are uniform on + ``{1/L, 2/L, ..., 1}`` under H0, as permutation p-values are. None + for a continuous null. + confidence (float): target simultaneous confidence level. + max_points (int): cap on evaluation points. A finer lattice is checked + on an evenly spaced subset, which stays valid and costs a little + power. + + Returns + ------- + ``(t, lower, upper, local, achieved)``: evaluation points, the allowed + ECDF range at each as a fraction of ``n``, the table of local levels + ``local[k, j]`` for count ``j`` at point ``k``, and the exact + simultaneous level the band achieves. + """ + if lattice is None: + points = min(max_points, max(200, 20 * n)) + t = np.arange(1, points + 1) / points + else: + lattice = int(lattice) + k = ( + np.arange(1, lattice + 1) + if lattice <= max_points + else np.unique(np.round(np.linspace(1, lattice, max_points)).astype(int)) + ) + t = k / lattice + + # F0(t_k) = t_k for both nulls; the lattice only changes where we may look. + j = np.arange(n + 1) + cdf = binom.cdf(j[None, :], n, t[:, None]) # P(N_k <= j) + tail = np.empty_like(cdf) + tail[:, 0] = 1.0 + tail[:, 1:] = 1.0 - cdf[:, :-1] # P(N_k >= j) + local = np.minimum(cdf, tail) + + # Raising eta tightens every interval, so the acceptance probability falls + # monotonically: bisect for the largest eta that still clears the target. + candidates = np.unique(local) + lo, hi = -1, len(candidates) # lo = -1 means "eta below every level" + while hi - lo > 1: + mid = (lo + hi) // 2 + if _band_accept_probability(t, n, *_band_bounds(local, candidates[mid])) >= confidence: + lo = mid + else: + hi = mid + eta = float(candidates[lo]) if lo >= 0 else -np.inf + + lower, upper = _band_bounds(local, eta) + return t, lower / n, upper / n, local, _band_accept_probability(t, n, lower, upper) + + +def pit_plot(pvals, saveto, confidence=0.95, lattice=None): + """Create a Probability Integral Transform (PIT) plot. + + Plots the empirical CDF of the provided p-values against the expected CDF + for a uniform distribution (the 1:1 diagonal), inside a shaded band that + the ECDF stays wholly within with probability ``confidence`` if the + p-values really are uniform. Any excursion outside the band rejects that + null at significance ``1 - confidence`` -- and the band is *simultaneous*, + so it is the whole curve being read at once, not each point separately. + + The band is built from the exact null distribution of the p-values. The + ECDF only moves where the p-values can land, so rather than constraining + order statistics it constrains the counts there, + ``N_k = #{p_i <= t_k} ~ Binomial(n, F0(t_k))``, giving every point the same + local level and taking that level as large as the target simultaneous level + allows. The boundary follows the real ``n F0(t) (1 - F0(t))`` variance of + the count and pinches to nothing at both corners, so it is far tighter + where miscalibrated p-values pile up -- at n = 100 it is half the width of + a Kolmogorov-Smirnov band at t = 0.05 -- while giving up about 12% near the + median. Worth roughly 1.2x the power of KS against skewed p-values and 2x + against U-shaped or over-central ones. + + Pass ``lattice=L`` when the p-values are uniform on ``{1/L, ..., 1}`` under + H0, as permutation p-values are; ``pted_coverage_test`` supplies it + automatically. Because the band constrains counts rather than order + statistics, ties are exactly what it expects, and it holds its level on + lattices as coarse as ``L = 151`` where an order-statistic band would + reject 60% of valid p-value sets. + + Both the escape probability and the search for the local level are exact, + so the band is deterministic and its achieved level is known rather than + estimated -- it is annotated on the plot alongside the worst local level + the data reached and its simultaneous p-value. Parameters ---------- pvals (array-like): Array of p-values in [0, 1]. saveto (str): File path where the plot will be saved. The format is inferred from the file extension (e.g. ".pdf", ".png"). - confidence (float): Confidence level for the KS confidence band. - Default is 0.95 (95%). + confidence (float): Confidence level for the band. Default is 0.95 + (95%), i.e. the ECDF escapes it 5% of the time under the null. + lattice (Optional[int]): ``L`` if the p-values are uniform on + ``{1/L, ..., 1}`` under H0. None treats them as continuous, which + over-rejects if they are in fact discrete -- a warning fires if + they look it. """ try: import matplotlib.pyplot as plt @@ -734,31 +975,43 @@ def pit_plot(pvals, saveto, confidence=0.95): sorted_pvals = np.sort(pvals) ecdf = np.arange(1, n + 1) / n - # Critical value for the two-sided KS statistic at the given confidence level. - d_crit = kstwo.ppf(confidence, n) - - # One-sample KS test against U(0,1) for annotation - ks_stat, ks_pval = kstest(pvals, "uniform") + if lattice is None and n - len(np.unique(sorted_pvals)) >= 3: + warn( + f"{n - len(np.unique(sorted_pvals))} of the {n} p-values are duplicates, so they " + f"are discrete, but no lattice was given. Pass lattice=L (permutation p-values " + f"from B permutations are uniform on L = B + 1 points) so the band is built from " + f"the right null; treating them as continuous over-rejects." + ) - x = np.linspace(0, 1, 500) + t, lower, upper, local, achieved = _lattice_band(n, lattice, confidence) + counts = _lattice_counts(sorted_pvals, t)[0] + worst = float(local[np.arange(len(t)), counts].min()) + # The simultaneous p-value is the probability of any point reaching a local + # level this small, which is one minus the acceptance probability at that + # threshold -- the same exact recursion that calibrated the band. + band_pval = 1.0 - _band_accept_probability(t, n, *_band_bounds(local, worst)) fig, ax = plt.subplots() ax.fill_between( - x, - np.maximum(x - d_crit, 0), - np.minimum(x + d_crit, 1), + t, + lower, + upper, + step="post", color="grey", alpha=0.3, linewidth=0, - label=f"{int(confidence * 100)}% KS confidence band", + label=f"{achieved:.1%} simultaneous band" + + ("" if lattice is None else f", lattice of {int(lattice)}"), ) + curve_label = f"Empirical CDF (worst local level={worst:.2g}, p={band_pval:.3f})" + ax.plot([0, 1], [0, 1], "k--", alpha=0.8, label="Expected (Uniform)") ax.step( np.concatenate([[0], sorted_pvals, [1]]), np.concatenate([[0], ecdf, [1]]), where="post", color="#A34F4F", - label=f"Empirical CDF (KS={ks_stat:.3f}, p={ks_pval:.3f})", + label=curve_label, ) ax.set_xlabel("p-value") ax.set_ylabel("Empirical CDF") diff --git a/tests/test_pted.py b/tests/test_pted.py index 685afdf..2bfc0a1 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -62,15 +62,19 @@ def test_pted_main(): def test_pted_progress_bar(capsys): - pted.pted(np.array([[1, 2], [3, 4]]), np.array([[3, 2], [1, 4]]), permutations=42) + # Large enough that the permutation group dwarfs 42, so 42 draws really are + # taken; a tiny sample would be enumerated instead and the bar would count + # the group rather than the request. + np.random.seed(0) + x = np.random.normal(size=(10, 2)) + y = np.random.normal(size=(10, 2)) + pted.pted(x, y, permutations=42) captured = capsys.readouterr().err assert ( "42/42" not in captured ), "progress bar showed up when prog_bar is set to False by default" - pted.pted( - np.array([[1, 2], [3, 4]]), np.array([[3, 2], [1, 4]]), permutations=42, prog_bar=True - ) + pted.pted(x, y, permutations=42, prog_bar=True) captured = capsys.readouterr().err assert "42/42" in captured, "progress bar did not show when prog_bar is set to True" @@ -146,12 +150,14 @@ def test_landmarks_mismatched_sizes(backend): # x has 200 samples, y has 30 samples; 80 landmarks puts all 30 y in L x = _to_backend(np.random.normal(size=(200, D)), backend) y = _to_backend(np.random.normal(size=(30, D)), backend) - p = pted.pted(x, y, n_landmarks=80) + # Pinned: under the null a two-tailed p-value is uniform, so an unpinned + # `p < 0.99` bound fails once in a hundred runs on its own. + p = pted.pted(x, y, n_landmarks=80, rng=0) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" # Different distributions should give small p-value even with mismatched sizes y_diff = _to_backend(np.random.uniform(size=(30, D)), backend) - p = pted.pted(x, y_diff, n_landmarks=80) + p = pted.pted(x, y_diff, n_landmarks=80, rng=0) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" diff --git a/tests/test_utils.py b/tests/test_utils.py index 34efce7..08914f3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,14 @@ import numpy as np from scipy.spatial.distance import cdist +from scipy.stats import kstwo + from pted.utils import ( + _band_accept_probability, + _lattice_band, + _sampled_label_blocks, + _lattice_counts, + _enumerate_label_blocks, PermutationResolutionWarning, allocate_landmarks, two_tailed_p, @@ -14,12 +21,10 @@ hdp_coverage_test, permutation_energy_test, _cdist, - _draw_labels, + _label_drawer, _evaluate_statistic, _index_like, _prepare_statistic, - _random_permutation, - _to_scalar, ) try: @@ -36,6 +41,8 @@ import pytest +BACKENDS = ["numpy", "torch", "jax"] + def _require_backend(backend): if backend == "torch" and torch is None: @@ -98,46 +105,6 @@ def test_hdp_coverage_test(): assert pvalue > 0.01, "p-value should not be small for underconfident and one_tailed test" -@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) -def test_to_scalar(backend): - if backend == "torch" and torch is None: - pytest.skip("torch not installed") - if backend == "jax" and jax is None: - pytest.skip("jax not installed") - - if backend == "numpy": - x = np.array(5) - - elif backend == "torch": - x = torch.tensor(5) - - elif backend == "jax": - x = jnp.array(5) - - # Test with scalar - assert _to_scalar(x, backend) == 5 - - -@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) -def test_random_permutation(backend): - if backend == "torch" and torch is None: - pytest.skip("torch not installed") - if backend == "jax" and jax is None: - pytest.skip("jax not installed") - - D = np.arange(16).reshape(4, 4) - if backend == "torch": - D = torch.tensor(D) - elif backend == "jax": - D = jnp.array(D) - - permuted_D = _random_permutation(D, backend=backend) - - assert set(np.array(permuted_D.flatten()).tolist()) == set( - np.array(D.flatten()).tolist() - ), "Permutation should contain all original elements" - - # --------------------------------------------------------------------------- # Rectangular / batched permutation backend # --------------------------------------------------------------------------- @@ -240,21 +207,27 @@ def test_allocate_landmarks_errors(): permutation_energy_test(np.zeros((0, 2)), np.zeros((5, 2)), permutations=1) +def _prep_for(n1, n2, c, seed=3, backend="numpy"): + """An allocation plus its prepared statistic, on the given backend.""" + rng = np.random.default_rng(seed) + alloc = allocate_landmarks(n1, n2, c, rng) + z = rng.standard_normal((n1 + n2, 4)) + if backend == "torch": + z = torch.tensor(z) + elif backend == "jax": + z = jnp.array(z) + zc = z if alloc["regime"] == "full" else z[_index_like(alloc["landmarks"], z, backend)] + return alloc, _prepare_statistic(_cdist(z, zc, backend), alloc, backend) + + @pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) def test_draw_labels_stays_in_subgroup(n1, n2, c, regime): - """Permutations never move a label across the C / C^c boundary, so the - per-group column counts -- and hence every normalising constant -- are + """Permutations never move a label across the L / L^c boundary, so the + per-group landmark counts -- and hence every normalising constant -- are fixed. This is what makes the observed labelling exchangeable with the permuted ones.""" - rng = np.random.default_rng(3) - alloc = allocate_landmarks(n1, n2, c, rng) - n = n1 + n2 - base = np.zeros(n) - base[alloc["small_idx"]] = 1.0 - in_l = np.zeros(n, bool) - in_l[alloc["landmarks"]] = True - - U = _draw_labels(base, np.flatnonzero(in_l), np.flatnonzero(~in_l), 200, rng.spawn(2)) + alloc, prep = _prep_for(n1, n2, c) + U = np.asarray(_label_drawer(prep, np.random.default_rng(17))(200)) assert np.all(np.isin(U, [0.0, 1.0])) assert np.all(U.sum(1) == alloc["n_small"]), "small group size is preserved" @@ -282,7 +255,7 @@ def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): U = np.concatenate( [ prep["base_small"][None, :], - _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], 5, rng.spawn(2)), + _label_drawer(prep, np.random.default_rng(11))(5), ] ) got = _evaluate_statistic(prep, U) @@ -324,7 +297,7 @@ def test_statistic_backend_agreement(backend): U = np.concatenate( [ prep["base_small"][None, :], - _draw_labels(prep["base_small"], prep["idx_l"], prep["idx_o"], 8, rng.spawn(2)), + _label_drawer(prep, np.random.default_rng(13))(8), ] ) results[be] = _evaluate_statistic(prep, U) @@ -332,31 +305,70 @@ def test_statistic_backend_agreement(backend): assert np.allclose(results["numpy"], results[backend], rtol=1e-4, atol=1e-4) -def test_draw_labels_independent_of_batch_size(): - """The two label streams are consumed row by row, so splitting a run into - batches yields bit-identical permutations.""" - base = np.r_[np.ones(5), np.zeros(15)] - idx_l, idx_o = np.arange(12), np.arange(12, 20) - whole = _draw_labels(base, idx_l, idx_o, 10, np.random.default_rng(9).spawn(2)) - rngs = np.random.default_rng(9).spawn(2) - split = np.concatenate([_draw_labels(base, idx_l, idx_o, k, rngs) for k in (3, 3, 4)]) - assert np.array_equal(whole, split) +@pytest.mark.parametrize("backend", ["torch", "jax"]) +@pytest.mark.parametrize("n1,n2,c", [(40, 50, 30), (6, 80, 40), (1, 60, 12)]) +def test_device_draw_labels_stays_in_subgroup(backend, n1, n2, c): + """The on-device draw obeys the same subgroup constraints as the numpy + one -- the labels moved to the accelerator, the guarantees did not.""" + _require_backend(backend) + alloc, prep = _prep_for(n1, n2, c, backend=backend) + U = np.asarray(_label_drawer(prep, np.random.default_rng(23))(128)) + + assert np.all(np.isin(U, [0.0, 1.0])) + assert np.all(U.sum(1) == alloc["n_small"]) + assert np.all(U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"]) + if alloc["reference_size"] > 100: + assert len(np.unique(U, axis=0)) > 1 + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_label_draws_are_reproducible(backend): + """A fixed rng pins the draw, and a run yields exactly `permutations` valid + rows in blocks no larger than batch_size.""" + _require_backend(backend) + _, prep = _prep_for(20, 30, 50, backend=backend) + + a = np.asarray(_label_drawer(prep, np.random.default_rng(5))(40)) + b = np.asarray(_label_drawer(prep, np.random.default_rng(5))(40)) + assert np.array_equal(a, b), "the same rng must give the same draw" + + for batch in (7, 40, 1000): + rng = np.random.default_rng(5) + blocks = [np.asarray(x) for x in _sampled_label_blocks(prep, 40, batch, rng)] + assert sum(len(x) for x in blocks) == 40 + assert max(len(x) for x in blocks) <= batch + rows = np.concatenate(blocks) + assert np.all(rows.sum(1) == prep["base_small"].sum()) + + +def test_evaluate_is_independent_of_batch_size(): + """Splitting one set of labellings across batches must not change their + statistics, beyond the reordering a different matmul shape costs in + floating point. (Which labellings get drawn does depend on batch_size -- + the draw is a running stream -- but every batching is an exact test.)""" + x = np.random.default_rng(0).standard_normal((50, 4)) + y = np.random.default_rng(1).standard_normal((70, 4)) + alloc, prep = _prep_for(50, 70, 30) + U = _label_drawer(prep, np.random.default_rng(2))(64) + + whole = _evaluate_statistic(prep, U) + for batch in (1, 7, 64): + pieces = np.concatenate( + [_evaluate_statistic(prep, U[s : s + batch]) for s in range(0, len(U), batch)] + ) + assert np.allclose(pieces, whole, rtol=1e-10, atol=1e-12), f"batch_size={batch}" -def test_statistic_independent_of_batch_size(): - """Batching is purely a memory/throughput knob: with the same seed it draws - the same permutations, so the statistics agree to within the reordering - that a different matmul shape costs in floating point.""" - rng_kwargs = dict(permutations=64, n_landmarks=30, rng=12345) +def test_every_batch_size_gives_a_valid_test(): + """batch_size stays a free knob: any value returns the requested number of + null statistics and a p-value in range.""" x = np.random.default_rng(0).standard_normal((50, 4)) y = np.random.default_rng(1).standard_normal((70, 4)) - ref_stat, ref_perm = permutation_energy_test(x, y, batch_size=1, **rng_kwargs) - for batch in (7, 64, 1000): - stat, perm = permutation_energy_test(x, y, batch_size=batch, **rng_kwargs) - assert stat == ref_stat - assert np.allclose( - perm, ref_perm, rtol=1e-10, atol=1e-12 - ), f"batch_size={batch} changed the null draws" + for batch in (1, 7, 64, 1000): + stat, perm = permutation_energy_test( + x, y, permutations=64, n_landmarks=30, rng=12345, batch_size=batch + ) + assert len(perm) == 64 and np.all(np.isfinite(perm)) def test_n_landmarks_covering_the_sample_is_the_full_test(): @@ -413,3 +425,180 @@ def test_permutation_resolution_warning(): warnings.simplefilter("error", PermutationResolutionWarning) permutation_energy_test(x, y, permutations=100, n_landmarks=8) permutation_energy_test(x, y, permutations=100, n_landmarks=55) + + +# --------------------------------------------------------------------------- +# PIT plot confidence bands +# --------------------------------------------------------------------------- + + +def _band_rejects(pvals, t, lower, upper): + counts = _lattice_counts(np.asarray(pvals), t)[0] / len(pvals) + return bool(np.any((counts < lower - 1e-12) | (counts > upper + 1e-12))) + + +def _ks_rejects(pvals, d_crit): + n = len(pvals) + s = np.sort(pvals) + i = np.arange(1, n + 1) + return max(np.max(i / n - s), np.max(s - (i - 1) / n)) > d_crit + + +def test_lattice_band_is_deterministic_and_pinches_at_the_corners(): + """The band tracks the binomial variance of the ECDF count, so it is far + tighter at the ends than a constant-half-width KS band.""" + n, confidence = 100, 0.95 + t, lower, upper, _, _ = _lattice_band(n, 151, confidence) + again, lower2, _, _, _ = _lattice_band(n, 151, confidence) + assert np.array_equal(t, again) and np.array_equal(lower, lower2) + + assert np.all(lower <= upper) + half = (upper - lower) / 2 + assert half[0] < half[len(t) // 2] and half[-1] < half[len(t) // 2], "must pinch at the ends" + assert half[np.searchsorted(t, 0.05)] < kstwo.ppf(confidence, n) + + +@pytest.mark.parametrize("n,lattice", [(100, 151), (100, 1000), (60, 41), (100, None)]) +def test_lattice_band_level_is_exact(n, lattice): + """The escape probability is computed by a forward recursion over the + counting process, not estimated, so the band's level is known exactly -- + and it lands on the target rather than being conservative.""" + confidence = 0.95 + _, _, _, _, achieved = _lattice_band(n, lattice, confidence) + assert abs((1 - achieved) - (1 - confidence)) < 0.002, f"achieved {1 - achieved}" + + +def test_band_recursion_matches_brute_force(): + """The forward recursion agrees with simulating the null directly.""" + n, L, trials = 40, 61, 40000 + t, lo, hi, _, _ = _lattice_band(n, L, 0.95) + lower, upper = np.round(lo * n).astype(int), np.round(hi * n).astype(int) + exact = _band_accept_probability(t, n, lower, upper) + + rng = np.random.default_rng(0) + hits = 0 + for _ in range(0, trials, 4000): + p = rng.integers(1, L + 1, size=(4000, n)) / L + counts = _lattice_counts(p, t) + hits += int(np.sum(np.all((counts >= lower) & (counts <= upper), axis=1))) + mc = hits / trials + assert abs(exact - mc) < 4 * np.sqrt(mc * (1 - mc) / trials), f"{exact} vs {mc}" + + +def test_lattice_band_holds_level_against_simulation(): + """Sanity check the level end to end, on the coarse lattice a real + coverage test produces -- where an order-statistic band rejects ~60%.""" + n, lattice, confidence, trials = 100, 151, 0.95, 4000 + t, lower, upper, _, _ = _lattice_band(n, lattice, confidence) + rng = np.random.default_rng(3) + hits = 0 + for _ in range(trials): + p = rng.integers(1, lattice + 1, size=n) / lattice + hits += _band_rejects(p, t, lower, upper) + rate = hits / trials # se = 0.0034 + assert abs(rate - (1 - confidence)) < 0.015, f"rejected {rate}" + + +def test_lattice_band_beats_ks_on_tail_deviations(): + """More power against the p-value shapes a miscalibrated posterior makes, + on the coarse lattice a real coverage test produces.""" + n, L, trials = 100, 151, 1500 + t, lower, upper, _, _ = _lattice_band(n, L, 0.95) + d_crit = kstwo.ppf(0.95, n) + rng = np.random.default_rng(4) + for a, b in ((0.7, 1.0), (0.85, 0.85)): # skewed low, and U-shaped + draws = [np.ceil(rng.beta(a, b, n) * L) / L for _ in range(trials)] + ks = np.mean([_ks_rejects(p, d_crit) for p in draws]) + lat = np.mean([_band_rejects(p, t, lower, upper) for p in draws]) + assert lat > ks, f"Beta({a},{b}): lattice {lat:.3f} did not beat KS {ks:.3f}" + + +def test_pit_plot_writes_a_file(tmp_path): + rng = np.random.default_rng(0) + for label, pvals, lattice in ( + ("continuous", rng.random(100), None), + ("lattice", rng.integers(1, 152, size=100) / 151, 151), + ): + out = tmp_path / f"pit_{label}.pdf" + pit_plot(pvals, str(out), lattice=lattice) + assert out.exists() + + +def test_pit_plot_warns_when_discrete_pvalues_have_no_lattice(tmp_path): + """Discrete p-values need their lattice declared; without it the band is + built from the wrong null. Saying so is better than silently over-rejecting.""" + rng = np.random.default_rng(1) + coarse = rng.integers(1, 201, size=100) / 200 # 199 permutations + + with pytest.warns(UserWarning, match="no lattice was given"): + pit_plot(coarse, str(tmp_path / "coarse.pdf")) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + pit_plot(coarse, str(tmp_path / "told.pdf"), lattice=200) + pit_plot(rng.random(100), str(tmp_path / "cont.pdf")) + + +# --------------------------------------------------------------------------- +# Exhaustive enumeration of small permutation groups +# --------------------------------------------------------------------------- + + +def test_small_groups_are_enumerated_not_sampled(): + """Asking for more permutations than the subgroup has members walks the + whole group instead, returning reference_size - 1 null statistics.""" + x = np.random.default_rng(0).standard_normal((1, 3)) + y = np.random.default_rng(1).standard_normal((60, 3)) + reference = allocate_landmarks(1, 60, 61, rng=0)["reference_size"] + assert reference == 61 + + _, permute = permutation_energy_test(x, y, permutations=1000, rng=0) + assert len(permute) == reference - 1, "should have enumerated the group" + + # ask for fewer than the group holds and it samples, as before + _, permute = permutation_energy_test(x, y, permutations=30, rng=0) + assert len(permute) == 30 + + +def test_enumeration_covers_the_group_exactly_once(): + """Every assignment the subgroup reaches appears once, bar the observed.""" + n1, n2, m = 1, 12, 5 + alloc = allocate_landmarks(n1, n2, m, rng=0) + z = np.random.default_rng(0).standard_normal((n1 + n2, 2)) + prep = _prepare_statistic(cdist(z, z[alloc["landmarks"]]), alloc, "numpy") + + rows = np.concatenate(list(_enumerate_label_blocks(prep, 4))) + assert len(rows) == alloc["reference_size"] - 1 + assert len(np.unique(rows, axis=0)) == len(rows), "no assignment repeats" + assert np.all(rows.sum(1) == alloc["n_small"]), "group size preserved" + assert np.all(rows[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"]) + assert not np.any(np.all(rows == prep["base_small"], axis=1)), "observed excluded" + + +def test_enumerated_pvalues_are_exactly_lattice_uniform(): + """Enumeration removes the tie inflation that sampling a small group + causes, so P(p <= t) sits on the lattice instead of below it.""" + from pted.pted import pted as pted_api + + nsamp, trials = 40, 6000 # group of 41, so p lands on multiples of 1/41 + rng = np.random.default_rng(5) + pvals = np.empty(trials) + for t in range(trials): + loc, sd = rng.standard_normal(2) * 3, rng.uniform(1, 3, size=2) + pvals[t] = pted_api( + rng.normal(loc, sd, size=(1, 2)), + rng.normal(loc, sd, size=(nsamp, 2)), + permutations=500, + two_tailed=False, + rng=rng, + ) + R = nsamp + 1 + assert len(np.unique(pvals)) <= R + # a valid p-value on an R-point lattice has P(p <= t) = floor(tR)/R + for t in (0.1, 0.2, 0.5): + expected = np.floor(t * R) / R + observed = np.mean(pvals <= t) + assert abs(observed - expected) < 4 * np.sqrt( + expected * (1 - expected) / trials + ), f"P(p<={t}) was {observed:.4f}, lattice value is {expected:.4f}" + assert observed <= t + 1e-9, "must stay valid" From c184a44cb61cba7309b81170176b62852a6fb044 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Tue, 8 Sep 2026 21:32:12 -0400 Subject: [PATCH 5/7] Add containment test --- README.md | 113 ++++++++++---------- src/pted/__init__.py | 3 +- src/pted/pted.py | 149 +++++++++++++++++++++++++- src/pted/utils.py | 242 +++++++++++++++++++++++++++++++++++++++++-- tests/test_pted.py | 120 +++++++++++++++++++++ tests/test_utils.py | 12 +++ 6 files changed, 579 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 3ff8c0e..7dc2ffe 100644 --- a/README.md +++ b/README.md @@ -175,69 +175,78 @@ doubling procedure (``2 * min(p_right, p_left)``) to get the p-value meaning that if your posterior is underconfident or overconfident, you will get a small p-value that can be used to reject the null. -#### If you have posterior densities +## Example: Containment Test -If you have posterior densities (and you trust them), then chances are the HDP -region coverage test is a more powerful test. Essentially, instead of using a -permutation test to determine the p-value for a single simulation, you just -determine the fraction of posterior samples with higher posterior density than -the ground truth[^4]. The package has a quick tool to let you do that: +`pted` asks "are these the same distribution?". `pted_containment_test` asks the +weaker, directional question "**does y cover x?**" — a sample drawn from a +tighter distribution than `y` passes, while one that spreads beyond it, sits off +to one side, or throws a few points clear of it fails. ```python -from pted import hdp_coverage_test +from pted import pted_containment_test +import numpy as np -ground_truth = np.random.uniform(100) # Nsim -posterior_samples = np.random.uniform((200, 100)) # Nsamp, Nsim +y = np.random.normal(size = (500, 10)) +inside = np.random.normal(size = (100, 10)) * 0.5 +outside = np.random.normal(size = (100, 10)) * 1.6 -p_value = hdp_coverage_test(ground_truth, posterior_samples, two_tailed = True) -print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 +print(pted_containment_test(inside, y)) # large: contained +print(pted_containment_test(outside, y)) # small: not contained ``` -## Example: Sensitivity comparison with KS-test - -There is no single universally optimal two sample test, but a widely used method -in 1D is called the Kolmogorov-Smirnov (KS)-test. The KS-test operates -fundamentally differently from PTED and can only really work in 1D. Here I do a -super basic comparison of the two methods. Draw two samples of 100 Gaussian -distributed points, thus the null hypothesis is true for these points. Then -slowly bias one of the samples by changing the standard deviation up to 2 sigma. -By tracking how the p-value drops we can see which method is more sensitive to -this kind of mismatched sample. If you run this test a hundred times you will -find that PTED is more sensitive to this kind of bias than the KS-test. Observe -that both methods start around p=0.5 in the true null case (scale = 1), since -they are both exact tests that truly sample U(0,1) under the null. +Unlike every other test in the package this one is **not symmetric** — swapping +the arguments asks the other question and will usually give a different answer. +That asymmetry is the point, and it is why the energy distance alone cannot +answer it: the energy distance is just as large when `x` is *tighter* than `y` +as when it is broader, so a one-tailed energy test rejects 99.8% of the time on +a perfectly contained sample. + +Instead, every `x` point gets a **depth** — its mean distance to the points +labelled `y` — so peripheral points score high. This is nearly identical to +`pted_coverage_test`, and we use the same `-2 sum log p` formula to combine the +p-values for each `x`. However, instead of interpreting the result as a $\chi^2$ +distribution, we use a secondary permutation test to calibrate it. + +The test is exact where `x` and `y` share a distribution and conservative inside +the null, which is the goal for a containment test. It measures depth rather +than literal support: a tight cluster of `x` sitting in a low-density pocket +well inside `y` counts as contained. So use a heavy dose of caution when +interpreting the results. + +### Read the plot as well as the p-value + +`-2 sum log p` is a sum whose per-point floor is zero against a null mean of +two, so a bulk of `x` sitting deep inside `y` banks slack that can hide a +handful of points `y` cannot reach at all. In testing, 5% of `x` placed five +sigma outside the prior predictive went undetected, while every other failure +mode — data broader than `y`, data offset into its tail — was caught cleanly. So +pass `pit_plot`: ```python -from pted import pted -import numpy as np -from scipy.stats import kstest -import matplotlib.pyplot as plt - -np.random.seed(0) - -scale = np.linspace(1.0, 2.0, 10) -pted_p = np.zeros((10, 100)) -ks_p = np.zeros((10, 100)) -for i, s in enumerate(scale): - for trial in range(100): - x = np.random.normal(size=(100, 1)) - y = np.random.normal(scale=s, size=(100, 1)) - pted_p[i][trial] = pted(x, y, two_tailed=False) - ks_p[i][trial] = kstest(x[:, 0], y[:, 0]).pvalue - -plt.plot(scale, np.mean(pted_p, axis=1), linewidth=3, c="b", label="PTED") -plt.plot(scale, np.mean(ks_p, axis=1), linewidth=3, c="r", label="KS") -plt.legend() -plt.ylim(0, None) -plt.xlim(1, 2.0) -plt.xlabel("Out of distribution scale [*sigma]") -plt.ylabel("p-value") - -plt.savefig("pted_demo.png", bbox_inches="tight") -plt.show() +p = pted_containment_test(data, prior_predictive, pit_plot = "containment.pdf") ``` -![pted demo KS comparison](media/pted_ks.png) +The plot shows the empirical CDF of the per-point depth p-values — one step per +point of `x` — on a logarithmic p axis, because the whole diagnostic lives at +the left edge. It carries two reference marks and no diagonal. + +The **upper bound** is the one-sided simultaneous ceiling for `n1` p-values that +really are uniform: the same as the ordinary PIT plot, with the lower edge +dropped. Only the upper edge means anything here. Any part of the CDF below this +line indicates the `x` values are likely contained at the threshold level (95% +by default). + +The **vertical line** marks threshold p-value (0.05 by default). Parts of the +CDF to the right of this line are embedded in `y` at the threshold level and so +likely are contained. Points to its left are peripheral relative to `y` and so +this may suggest the values are not contained. However, many values of `x` are +being tested some some leakage to low p-values are expected, which is why the +uniform-upper-bound line is also plotted. + +Read them together rather than as a decision rule. A curve with points to the +left of the vertical line and above the uniform-upper-bound is a fair indication +that `x` is not contained in `y`. Though results either way are not definitive, +as is the nature of null hypothesis testing, and even moreso here. ## Interpreting the results diff --git a/src/pted/__init__.py b/src/pted/__init__.py index 4c7f7d9..5c6d6e6 100644 --- a/src/pted/__init__.py +++ b/src/pted/__init__.py @@ -1,4 +1,4 @@ -from .pted import pted, pted_coverage_test +from .pted import pted, pted_coverage_test, pted_containment_test from .tests import test from .utils import hdp_coverage_test from ._version import version as __version__ # noqa @@ -9,6 +9,7 @@ __all__ = [ "pted", "pted_coverage_test", + "pted_containment_test", "hdp_coverage_test", "test", "__version__", diff --git a/src/pted/pted.py b/src/pted/pted.py index 7b687c3..7c6bb3a 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -4,6 +4,7 @@ from .utils import ( permutation_energy_test as _energy_test, + containment_pit_plot as _containment_pit_plot, _as_rng, two_tailed_p, confidence_alert, @@ -11,7 +12,7 @@ pit_plot as _pit_plot, ) -__all__ = ["pted", "pted_coverage_test"] +__all__ = ["pted", "pted_coverage_test", "pted_containment_test"] def pted( @@ -180,6 +181,152 @@ def pted( return pval +def pted_containment_test( + x: Union[np.ndarray, "Tensor", "jax.Array"], + y: Union[np.ndarray, "Tensor", "jax.Array"], + permutations: int = 1000, + return_all: bool = False, + n_landmarks: Optional[int] = None, + prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, + pit_plot: Optional[str] = None, + pit_confidence: float = 0.95, + pit_threshold: float = 0.05, +) -> Union[float, tuple[float, np.ndarray, float]]: + """ + One-sided test of whether ``y`` covers ``x``. + + Where :func:`pted` asks "are these the same distribution?", this asks the + weaker, directional question "does x sit inside y?". A sample drawn from a + tighter distribution than y passes; one that spreads beyond y, sits off to + one side of it, or throws a few points clear of it, fails. The null is the + composite hypothesis that x is no more peripheral than y, so a small + p-value is evidence that x is *not* contained. + + Note this is **not** symmetric: ``pted_containment_test(x, y)`` and + ``pted_containment_test(y, x)`` ask different questions and will usually + give different answers. That is the point of it. + + How it works + ------------ + Pool the samples and give every point a *depth*: its mean distance to + the points labelled y. Peripheral points have large depth. Each point + of x then carries the p-value of the single-point energy test against + y -- exactly the per-simulation test inside + :func:`pted_coverage_test` -- namely the fraction of y points at least + as peripheral as it is. Those are aggregated with the same + ``-2 sum log p`` that the coverage test uses, and the aggregate is + calibrated by permutation rather than against a chi-squared, because + the per-point p-values share y and so are correlated. + + The energy distance itself cannot answer this question. It is symmetric + in x and y, so it is just as large when x is *tighter* than y as when x + is broader; a one-tailed energy test rejects 99.8% of the time on a + sample that is perfectly contained. Averaging the depths does not help + either -- the average collapses to the cross term, which is invariant + under swapping x and y. The comparison against y's own depth + distribution is what carries the direction. + + Example + ------- + import numpy as np + from pted import pted_containment_test + + y = np.random.normal(size=(500, 10)) + inside = np.random.normal(size=(100, 10)) * 0.5 + outside = np.random.normal(size=(100, 10)) * 1.6 + + print(pted_containment_test(inside, y)) # large: contained + print(pted_containment_test(outside, y)) # small: not contained + + Parameters + ---------- + x (Union[np.ndarray, Tensor, jax.Array]): the sample being tested for + containment. Shape (N, *D) + y (Union[np.ndarray, Tensor, jax.Array]): the sample that should + contain it. Shape (M, *D) + permutations (int): number of permutations to run. + return_all (bool): if True, return the test statistic and the permuted + statistics with the p-value. If False, just return the p-value. + n_landmarks (Optional[int]): estimate depths from a rectangular + distance matrix instead of the full pairwise one; see + :func:`pted`. Both samples must keep at least one landmark. + prog_bar (bool): if True, show a progress bar over permutations. + batch_size (Optional[int]): permutations evaluated per matrix product. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state. + pit_plot (Optional[str]): if given, the path to save the companion + diagnostic plot. Strongly recommended -- see the note below. + pit_confidence (float): simultaneous level for that plot's upper + bound. Default 0.95. + pit_threshold (float): where the plot draws its vertical marker; + x points left of it are more peripheral than all but that fraction + of y. Default 0.05. + + Note + ---- + **Read the plot as well as the p-value.** The Fisher aggregate + ``-2 sum log p`` is a sum whose per-point floor is zero against a null + mean of two, so a bulk of x that sits deep inside y banks slack that + can hide a handful of points y cannot reach at all: 5% of x placed + five sigma out went undetected in testing while every other failure + mode was caught. Those escapees are obvious in + :func:`pted.utils.containment_pit_plot`, where the curve lifts off the + floor at the smallest p-value and runs above the band for the first + decade, so pass ``pit_plot`` and look at it. + + The test is exact at the boundary of the null, where x and y share a + distribution, and conservative inside it -- a genuinely contained x + rejects far less often than the nominal rate, which is the correct + behaviour for a one-sided composite null. + + It measures depth, not literal support. A tight cluster of x sitting in + a low-density pocket well inside y's bulk counts as contained, because + every one of its points is unremarkable in depth. If you need support + containment rather than depth containment, this is the wrong tool. + """ + assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" + assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" + assert len(y.shape) >= 2, f"y must be at least 2D, not {y.shape}" + assert ( + x.shape[1:] == y.shape[1:] + ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" + if len(x.shape) > 2: + x = x.reshape(x.shape[0], -1) + if len(y.shape) > 2: + y = y.reshape(y.shape[0], -1) + + test, permute = _energy_test( + x, + y, + permutations=permutations, + n_landmarks=n_landmarks, + prog_bar=prog_bar, + batch_size=batch_size, + rng=rng, + containment=True, + ) + + # One-sided by construction: only an unusually large aggregate is evidence + # that x reaches outside y. + if pit_plot is not None: + _containment_pit_plot( + x, + y, + pit_plot, + confidence=pit_confidence, + threshold=pit_threshold, + n_landmarks=n_landmarks, + rng=rng, + ) + + pval = (1.0 + np.sum(permute >= test)) / (1.0 + len(permute)) + if return_all: + return test, permute, pval + return pval + + def pted_coverage_test( g: Union[np.ndarray, "Tensor", "jax.Array"], s: Union[np.ndarray, "Tensor", "jax.Array"], diff --git a/src/pted/utils.py b/src/pted/utils.py index f3b2e6c..71b2f20 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -104,6 +104,7 @@ class torch: "confidence_alert", "simulation_based_calibration_histogram", "pit_plot", + "containment_pit_plot", "hdp_coverage_test", ) @@ -164,6 +165,23 @@ def _asarray_like(x, ref, backend: str): return np.asarray(x, dtype=ref.dtype) +def _take_along0(a, idx, backend: str): + """Gather ``a`` along axis 0 with per-column indices.""" + if backend == "torch": + return torch.take_along_dim(a, idx, dim=0) + if backend == "jax": + return jnp.take_along_axis(a, idx, axis=0) + return np.take_along_axis(a, idx, axis=0) + + +def _log(a, backend: str): + if backend == "torch": + return torch.log(a) + if backend == "jax": + return jnp.log(a) + return np.log(a) + + def _index_like(idx, ref, backend: str): """Move an integer index array onto the device of ``ref``.""" if backend == "torch": @@ -577,6 +595,7 @@ def _prepare_statistic(D, alloc: dict, backend: str) -> dict: "const": const, "base_small": base_small, "backend": backend, + "evaluate": _evaluate_statistic, } @@ -610,6 +629,103 @@ def _evaluate_statistic(prep: dict, U) -> np.ndarray: return _to_numpy(stat, backend).astype(np.float64, copy=False) +def _containment_curve(prep: dict, U): + """``(B, n_y + 1)`` running count of x points at each step of the p lattice. + + The per-point p-values live on ``{1/(n_y+1), ..., 1}``, so their empirical + CDF is fully described by how many x points have accumulated by each lattice + step. One row per labelling. + """ + backend, D = prep["backend"], prep["D"] + Us = _asarray_like(U, D, backend) + Xs = Us if prep["x_is_small"] else 1.0 - Us + Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] + depth = (D @ (1.0 - Vx).T) / prep["m_y"] + order = (-depth).argsort(0) + ranked_x = _to_numpy(_take_along0(Xs.T, order, backend), backend) + ranked_y = _to_numpy(_take_along0((1.0 - Xs).T, order, backend), backend) + + n_y, rows = int(prep["n_y"]), ranked_x.shape[1] + # y points at least as peripheral as each ranked point; this is the count + # that turns into p = (1 + count) / (n_y + 1). + above = ranked_y.cumsum(0).astype(np.int64) + flat = above + np.arange(rows)[None, :] * (n_y + 1) + hist = np.bincount(flat.ravel(), weights=ranked_x.ravel(), minlength=rows * (n_y + 1)) + return hist.reshape(rows, n_y + 1).cumsum(1) + + +def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict: + """Precompute for the containment statistic: is x peripheral within y? + + Every pooled point gets a *depth*, its mean distance to the y-labelled + points; peripheral points have large depth. Each x point then carries the + singleton energy p-value ``p_i = (1 + #{j in y : d_j >= d_i}) / (n_y + 1)``, + and those are aggregated the way :func:`pted.pted_coverage_test` aggregates + its per-simulation p-values, ``-2 sum log p_i``. Large means x sits further + out than y does. + + Unlike the energy distance this is *not* symmetric in x and y, which is the + whole point. + """ + n_s, n_l = alloc["n_small"], alloc["n_large"] + n = n_s + n_l + m_s, m_l = alloc["n_small_landmarks"], alloc["n_large_landmarks"] + landmarks = alloc["landmarks"] + full = alloc["regime"] == "full" + m_y = m_l if x_is_small else m_s + n_y = n_l if x_is_small else n_s + if m_y < 1: + raise ValueError( + "the containing sample has no landmarks, so depth is unestimable; " + "use more landmarks or the full matrix" + ) + + # Ranks are invariant to a constant offset, but zeroing the self-pairs + # still matters: a y point counting its own zero distance would look + # artificially deep. + D = D - float(D.mean()) + D = _zero_self_pairs(D, landmarks, backend) + + base_small = np.zeros(n) + base_small[alloc["small_idx"]] = 1.0 + in_l = np.zeros(n, dtype=bool) + in_l[landmarks] = True + + return { + "D": D, + "ref": D, + "parts": ((np.flatnonzero(in_l), m_s), (np.flatnonzero(~in_l), n_s - m_s)), + "landmarks": None if full else _index_like(landmarks, D, backend), + "x_is_small": x_is_small, + "m_y": float(m_y), + "n_y": float(n_y), + "base_small": base_small, + "backend": backend, + "evaluate": _evaluate_containment, + } + + +def _evaluate_containment(prep: dict, U) -> np.ndarray: + """Fisher combination of per-point depth ranks, for a batch of labellings.""" + backend, D = prep["backend"], prep["D"] + Us = _asarray_like(U, D, backend) + Xs = Us if prep["x_is_small"] else 1.0 - Us # (B, n) indicator of the x sample + Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] + + # Depth of every pooled point against each labelling's y set. + depth = (D @ (1.0 - Vx).T) / prep["m_y"] # (n, B) + + # Rank by depth, most peripheral first; the running count of y points is + # then #{j in y : d_j >= d_i} without ever materialising a comparison. + order = (-depth).argsort(0) + ranked_x = _take_along0(Xs.T, order, backend) + ranked_y = _take_along0((1.0 - Xs).T, order, backend) + p = (1.0 + ranked_y.cumsum(0)) / (prep["n_y"] + 1.0) + + stat = -2.0 * (ranked_x * _log(p, backend)).sum(0) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + # --------------------------------------------------------------- public API @@ -621,8 +737,14 @@ def permutation_energy_test( prog_bar: bool = False, batch_size: Optional[int] = None, rng=None, + containment: bool = False, ) -> tuple[float, np.ndarray]: - """Observed energy statistic and its permutation distribution. + """Observed statistic and its permutation distribution. + + With ``containment=True`` the statistic is the depth-rank aggregate that + asks whether x sticks out of y, rather than the symmetric energy distance + that asks whether they differ at all. Everything else -- the landmark + allocation, the subgroup, the enumeration -- is identical. Without ``n_landmarks`` this is the exact full-matrix test: the ``(n, n)`` distance matrix is built once and the statistic is evaluated for batches of @@ -688,9 +810,13 @@ def permutation_energy_test( "more stable (i.e. z-score norm)." ) - prep = _prepare_statistic(dmatrix, alloc, backend) + if containment: + prep = _prepare_containment(dmatrix, alloc, backend, n1 <= n2) + else: + prep = _prepare_statistic(dmatrix, alloc, backend) + evaluate = prep["evaluate"] - test_stat = float(_evaluate_statistic(prep, prep["base_small"][None, :])[0]) + test_stat = float(evaluate(prep, prep["base_small"][None, :])[0]) assert np.isfinite(test_stat), "Observed statistic is not finite!" if batch_size is None: @@ -713,7 +839,7 @@ def permutation_energy_test( with tqdm(total=n_null, disable=not prog_bar) as bar: for U in blocks: size = len(U) - permute_stats[done : done + size] = _evaluate_statistic(prep, U) + permute_stats[done : done + size] = evaluate(prep, U) done += size bar.update(size) assert done == n_null, f"produced {done} null statistics, expected {n_null}" @@ -829,7 +955,7 @@ def _band_accept_probability(t, n, lower, upper): return float(f.sum()) -def _lattice_band(n, lattice, confidence, max_points=512): +def _lattice_band(n, lattice, confidence, max_points=512, one_sided=False): """Simultaneous ECDF band built from the known null of the p-values. The ECDF only moves where the p-values can actually land, so this works @@ -870,6 +996,9 @@ def _lattice_band(n, lattice, confidence, max_points=512): max_points (int): cap on evaluation points. A finer lattice is checked on an evenly spaced subset, which stays valid and costs a little power. + one_sided (bool): constrain only from above, spending the whole error + budget on the upper edge. For a composite null where the ECDF is + expected to sit low, a lower edge would flag the healthy case. Returns ------- @@ -896,7 +1025,7 @@ def _lattice_band(n, lattice, confidence, max_points=512): tail = np.empty_like(cdf) tail[:, 0] = 1.0 tail[:, 1:] = 1.0 - cdf[:, :-1] # P(N_k >= j) - local = np.minimum(cdf, tail) + local = tail if one_sided else np.minimum(cdf, tail) # Raising eta tightens every interval, so the acceptance probability falls # monotonically: bisect for the largest eta that still clears the target. @@ -1023,6 +1152,107 @@ def pit_plot(pvals, saveto, confidence=0.95, lattice=None): plt.close(fig) +def containment_pit_plot( + x, + y, + saveto, + confidence: float = 0.95, + threshold: float = 0.05, + n_landmarks: Optional[int] = None, + rng=None, +): + """Diagnostic plot for :func:`pted.pted_containment_test`. + + Plots the empirical CDF of the per-point depth p-values -- one step per + point of x -- against two reference marks, on a logarithmic p axis because + the whole diagnostic lives at the left edge. + + The **upper bound** is the one-sided simultaneous ceiling for ``n1`` + p-values that really are uniform, the same construction the ordinary PIT + plot uses, with the lower edge dropped. Only the upper edge is meaningful + here: the null is composite, and a genuinely contained x has *deeper* + points than y, so its p-values run large and its curve is expected to sit + well below any uniform reference. That is why the diagonal is not drawn. + + The **vertical line** marks ``threshold``: x points to its left are more + peripheral than all but that fraction of y. + + Read them together rather than as a decision rule. A curve that climbs + above the bound near the floor means x has points where y essentially + cannot reach -- the failure the Fisher aggregate in + :func:`pted.pted_containment_test` is blind to, since a deeply covered bulk + banks enough slack in ``-2 sum log p`` to hide a few escapees. But plenty of + curves cross the bound harmlessly: a tight x concentrates its p-values at + one value, so its ECDF is a step, and wherever that step lands it will + cross a bound drawn for a spread-out sample. Judgement is required, and + that is deliberate -- how much of x may sit how far out is a question about + your model, not about the arithmetic. + + Parameters + ---------- + x, y: the samples, as passed to + :func:`pted.pted_containment_test`. Shape ``(n1, d)``, ``(n2, d)``. + saveto (str): file path; the format follows the extension. + confidence (float): simultaneous level for the upper bound. + threshold (float): where to draw the vertical marker. + n_landmarks: as for :func:`pted.pted_containment_test`. + rng: seed, ``np.random.Generator``, or None. Only picks the landmarks. + """ + try: + import matplotlib.pyplot as plt + except ImportError: + warn("No containment PIT plot generated! Please install matplotlib.") + return + + backend = _backend(x) + z = _concatenate((x, y), backend) + n1, n2 = len(x), len(y) + n = n1 + n2 + m = n if n_landmarks is None else min(int(n_landmarks), n) + + alloc = allocate_landmarks(n1, n2, m, _as_rng(rng)) + landmarks = alloc["landmarks"] + zc = z if alloc["regime"] == "full" else z[_index_like(landmarks, z, backend)] + prep = _prepare_containment(_cdist(z, zc, backend), alloc, backend, n1 <= n2) + + observed = _containment_curve(prep, prep["base_small"][None, :])[0] / n1 + n_y = int(prep["n_y"]) + + # Under H0 the depth p-values are uniform on the (n_y + 1)-point lattice, + # so the ordinary lattice band applies -- upper edge only. + t, _, upper, _, achieved = _lattice_band(n1, n_y + 1, confidence, one_sided=True) + at = np.round(t * (n_y + 1)).astype(int) - 1 + + lattice = np.arange(1, n_y + 2) / (n_y + 1) + fig, ax = plt.subplots() + ax.step( + t, + upper, + where="post", + color="#40658f", + linestyle="--", + linewidth=1.4, + label=f"{achieved:.0%} upper bound, uniform p", + ) + ax.axvline( + threshold, + color="#555555", + linestyle=":", + linewidth=1.4, + label=f"p = {threshold:g}", + ) + ax.step(lattice, observed, where="post", color="#A34F4F", linewidth=2.4, label="x points") + ax.set_xscale("log") + ax.set_xlim([lattice[0] / 1.6, 1.0]) + ax.set_ylim([0, 1]) + ax.set_xlabel("depth p-value of an x point (log scale)") + ax.set_ylabel("fraction of x") + ax.set_title("Containment check") + ax.legend(loc="upper left") + fig.savefig(saveto, bbox_inches="tight") + plt.close(fig) + + def hdp_coverage_test( ground_truth: np.ndarray, posterior_samples: np.ndarray, two_tailed: bool = True ) -> float: diff --git a/tests/test_pted.py b/tests/test_pted.py index 2bfc0a1..e7122f5 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -429,3 +429,123 @@ def test_coverage_test_with_landmarks(backend): g, over, permutations=199, n_landmarks=30, warn_confidence=None ) assert p_over < 1e-2, f"overconfident posterior gave p={p_over}" + + +# --------------------------------------------------------------------------- +# Containment: does y cover x? +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_containment_direction(backend): + """A tighter sample is contained; a broader, shifted or leaky one is not.""" + _require_backend(backend) + rng = np.random.default_rng(0) + y = _to_backend(rng.standard_normal((400, 6)), backend) + + def p(x): + return pted.pted_containment_test(_to_backend(x, backend), y, rng=1) + + assert p(rng.standard_normal((80, 6)) * 0.5) > 0.1, "a tighter sample is contained" + assert p(rng.standard_normal((80, 6)) * 1.7) < 1e-2, "a broader one is not" + assert p(rng.standard_normal((80, 6)) + 1.2) < 1e-2, "a shifted one is not" + + leaky = rng.standard_normal((80, 6)) + leaky[:8] += 5.0 + assert p(leaky) < 1e-2, "a few escaped points are not contained" + + +def test_containment_is_not_symmetric(): + """The whole point: asking it the other way round must give a different + answer. The energy distance, and the mean depth it reduces to, cannot -- + both are invariant under swapping the two samples.""" + rng = np.random.default_rng(3) + broad = rng.standard_normal((100, 4)) * 1.7 + tight = rng.standard_normal((100, 4)) + + assert pted.pted_containment_test(broad, tight, rng=2) < 1e-2, "broad is not inside tight" + assert pted.pted_containment_test(tight, broad, rng=2) > 0.1, "tight is inside broad" + + # the symmetric energy test cannot separate these + assert pted.pted(broad, tight, rng=2) == pted.pted(tight, broad, rng=2) + + +def test_containment_null_is_calibrated(): + """Under x and y sharing a distribution the p-value is uniform, and a + genuinely contained x is conservative rather than merely non-significant. + Fixed seed, so this is deterministic rather than flaky.""" + trials, permutations = 400, 99 + rng = np.random.default_rng(20260908) + null, contained = np.empty(trials), np.empty(trials) + for t in range(trials): + y = rng.standard_normal((150, 4)) + null[t] = pted.pted_containment_test( + rng.standard_normal((40, 4)), y, permutations=permutations, rng=rng + ) + contained[t] = pted.pted_containment_test( + rng.standard_normal((40, 4)) * 0.5, y, permutations=permutations, rng=rng + ) + rate = float(np.mean(null <= 0.05)) # se = 0.011 + assert abs(rate - 0.05) < 0.04, f"null rejection rate {rate}" + assert float(np.mean(contained <= 0.05)) < 0.02, "a contained sample must not reject" + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_containment_with_landmarks(backend): + """Landmarks work here too, and both samples must keep at least one.""" + _require_backend(backend) + rng = np.random.default_rng(5) + y = _to_backend(rng.standard_normal((300, 5)), backend) + x_out = _to_backend(rng.standard_normal((60, 5)) * 1.8, backend) + x_in = _to_backend(rng.standard_normal((60, 5)) * 0.5, backend) + assert pted.pted_containment_test(x_out, y, n_landmarks=90, rng=1) < 1e-2 + assert pted.pted_containment_test(x_in, y, n_landmarks=90, rng=1) > 0.1 + + +def test_containment_returns_all(): + rng = np.random.default_rng(7) + x, y = rng.standard_normal((30, 3)), rng.standard_normal((90, 3)) + stat, perm, p = pted.pted_containment_test(x, y, permutations=50, return_all=True, rng=0) + assert np.isfinite(stat) and len(perm) == 50 + assert p == (1.0 + np.sum(perm >= stat)) / 51.0 + + +def test_containment_pit_plot(tmp_path): + """The plot is written, and its curve shows the failure mode the Fisher + aggregate is blind to.""" + from pted.utils import ( + allocate_landmarks, + _cdist, + _containment_curve, + _lattice_band, + _prepare_containment, + ) + + rng = np.random.default_rng(0) + y = rng.standard_normal((600, 5)) * 8.0 + leaky = rng.standard_normal((200, 5)) + leaky[:10] += np.r_[40.0, np.zeros(4)] # 5% of x where y cannot reach + + out = tmp_path / "containment.pdf" + p = pted.pted_containment_test(leaky, y, permutations=299, rng=1, pit_plot=str(out)) + assert out.exists() + assert p > 0.05, "the documented blind spot: Fisher does not see the escapees" + + def curve(x): + z = np.vstack([x, y]) + alloc = allocate_landmarks(len(x), len(y), len(z), rng=0) + prep = _prepare_containment(_cdist(z, z, "numpy"), alloc, "numpy", True) + return _containment_curve(prep, prep["base_small"][None, :])[0] / len(x) + + # ...but they lift the curve off the floor at the smallest p-value, above + # the uniform bound, where a contained sample stays pinned at zero. + n_y = len(y) + t, _, upper, _, _ = _lattice_band(len(leaky), n_y + 1, 0.95, one_sided=True) + at = np.round(t * (n_y + 1)).astype(int) - 1 + leaky_curve, tight_curve = curve(leaky), curve(rng.standard_normal((200, 5))) + + assert leaky_curve[0] > 0.0, "escapees sit at the p-value floor" + assert tight_curve[0] == 0.0, "a contained sample has nothing at the floor" + near_floor = t <= 0.01 + assert np.any(leaky_curve[at][near_floor] > upper[near_floor]), "escapees clear the bound" + assert not np.any(tight_curve[at][near_floor] > upper[near_floor]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 08914f3..1a352a1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -602,3 +602,15 @@ def test_enumerated_pvalues_are_exactly_lattice_uniform(): expected * (1 - expected) / trials ), f"P(p<={t}) was {observed:.4f}, lattice value is {expected:.4f}" assert observed <= t + 1e-9, "must stay valid" + + +def test_lattice_band_one_sided_lowers_the_ceiling(): + """Dropping the floor spends the whole error budget upward, so the ceiling + comes down -- which is what makes it worth using where only an excursion + above is evidence.""" + _, _, two_sided, _, _ = _lattice_band(100, 201, 0.95) + _, lower, one_sided, _, achieved = _lattice_band(100, 201, 0.95, one_sided=True) + assert np.all(lower == 0.0), "one-sided means no floor" + assert np.all(one_sided <= two_sided) + assert np.any(one_sided < two_sided), "and strictly lower somewhere" + assert abs((1 - achieved) - 0.05) < 0.01, f"level was {1 - achieved}" From add85749bddc640a3be52873f7bcf3bf4faebf19 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Tue, 15 Sep 2026 09:12:28 -0400 Subject: [PATCH 6/7] refactor for readability --- README.md | 24 +- src/pted/pted.py | 32 ++- src/pted/utils.py | 618 +++++++++++++++++++++++++++----------------- tests/test_pted.py | 9 +- tests/test_utils.py | 151 +++++++---- 5 files changed, 512 insertions(+), 322 deletions(-) diff --git a/README.md b/README.md index 7dc2ffe..147f8cf 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ def pted_coverage_test( * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) * **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. - Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches `max(m, n - m)` distinct label assignments. Keep `n_landmarks` well below the number of posterior samples. + Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches only `n - m` distinct label assignments. Keep `n_landmarks` well below the number of posterior samples. * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. @@ -442,7 +442,7 @@ the p-value under the `>=` convention: with a group of 151 and 199 draws, puts it at 0.0195, leaving only the unavoidable granularity of a 151-point lattice — and it costs *less* compute, since 151 evaluations beat 199. This mostly matters for `pted_coverage_test`, where the ground truth is a single -point and the group is only `max(m, n - m)` large. +point and the group is only `n - m` large. One consequence: when this kicks in, `return_all` gives back `reference_size - 1` permuted statistics rather than `permutations` of them. @@ -468,14 +468,18 @@ The detection threshold degrades as one over the square root of the compute. You also give up some p-value resolution, and this is most significant when one group holds a single point — which is exactly the per-simulation test inside -`pted_coverage_test`. There the lone point can sit inside `L`, its label roaming -over the `m` landmark positions, or outside it, roaming over the other `n - m`, -so the subgroup reaches `max(m, n - m)` distinct label assignments. PTED puts the -point on whichever side is larger, which holds the reference set at half the -pooled sample or better. The smallest attainable p-value is still about the -reciprocal of that, however many permutations you draw, so PTED raises a -`PermutationResolutionWarning` when the reachable set is too small to resolve -the p-value you asked for. +`pted_coverage_test`. The lone point is kept out of `L` — it has no within-group +pairs, so landmarking it would buy nothing — and its label then roams over the +`n - m` positions outside, which is the whole reachable set. The smallest +attainable p-value is about `1 / (n - m)` however many permutations you draw, so +PTED raises a `PermutationResolutionWarning` when that set is too small to +resolve the p-value you asked for. + +This is the sharpest case of a general rule: **landmarks are for `m << n`.** At +`m = n / 2` the rectangular matrix saves only a factor of two over the exact +test, which does not pay for the sensitivity and resolution it costs. Reach for +them when the full matrix will not fit or will not finish, not to shave a +constant factor. ### Does restricting the permutations still measure the energy distance? diff --git a/src/pted/pted.py b/src/pted/pted.py index 7c6bb3a..3d2d004 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -93,8 +93,9 @@ def pted( matrix. ``n_landmarks`` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from ``O(n^2 d)`` to - ``O(n m d)`` for ``m = n_landmarks``. A value covering the whole - pooled sample, or None, runs the exact full-matrix computation. + ``O(n m d)`` for ``m = n_landmarks``. Worth it only for + ``m << n``. A value covering the whole pooled sample, or None, + runs the exact full-matrix computation. two_tailed (bool): if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. Default is True. @@ -127,11 +128,12 @@ def pted( The one thing landmarks do cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test - inside ``pted_coverage_test``). That lone point sits on whichever side - of the landmark set leaves more room, so the subgroup reaches - ``max(m, n - m)`` distinct label assignments and the smallest - attainable p-value is about the reciprocal of that, however many - permutations are drawn. A ``PermutationResolutionWarning`` is raised + inside ``pted_coverage_test``). That lone point is kept out of the + landmark set, so the subgroup reaches only ``n - m`` distinct label + assignments and the smallest attainable p-value is about ``1 / (n - m)`` + however many permutations are drawn. Landmarks are for ``m << n``: at + ``m = n / 2`` the rectangular matrix saves a factor of two over the + exact test, which does not pay for the lost sensitivity. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. When the permutation subgroup has no more than ``permutations`` members, PTED walks the whole group instead of sampling it. That is @@ -400,8 +402,9 @@ def pted_coverage_test( matrix. ``n_landmarks`` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from ``O(n^2 d)`` to - ``O(n m d)`` for ``m = n_landmarks``. A value covering the whole - pooled sample, or None, runs the exact full-matrix computation. + ``O(n m d)`` for ``m = n_landmarks``. Worth it only for + ``m << n``. A value covering the whole pooled sample, or None, + runs the exact full-matrix computation. sbc_histogram (Optional[str]): If given, the path/filename to save a Simulation-Based-Calibration histogram. sbc_bins (Optional[int]): If given, force the histogram to have the provided @@ -445,11 +448,12 @@ def pted_coverage_test( The one thing landmarks do cost is p-value resolution, and it bites hardest when one group holds a single point (the per-simulation test - inside ``pted_coverage_test``). That lone point sits on whichever side - of the landmark set leaves more room, so the subgroup reaches - ``max(m, n - m)`` distinct label assignments and the smallest - attainable p-value is about the reciprocal of that, however many - permutations are drawn. A ``PermutationResolutionWarning`` is raised + inside ``pted_coverage_test``). That lone point is kept out of the + landmark set, so the subgroup reaches only ``n - m`` distinct label + assignments and the smallest attainable p-value is about ``1 / (n - m)`` + however many permutations are drawn. Landmarks are for ``m << n``: at + ``m = n / 2`` the rectangular matrix saves a factor of two over the + exact test, which does not pay for the lost sensitivity. A ``PermutationResolutionWarning`` is raised when the reachable set is too small to resolve the p-value requested. When the permutation subgroup has no more than ``permutations`` members, PTED walks the whole group instead of sampling it. That is diff --git a/src/pted/utils.py b/src/pted/utils.py index 71b2f20..c99a93e 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -24,29 +24,26 @@ Landmark allocation ------------------- -Dispatched on ``n_s = min(n1, n2)``, the smaller group: +Dispatched on ``n_small = min(n1, n2)``: ``full`` ``m >= n``. Every point is a landmark, ``S_L x S_{L^c}`` is the full symmetric group, and the statistic is the exact energy distance. ``singleton`` - ``n_s == 1``. Its within-group mean is identically zero (a single point - has no within-group pairs), so nothing is unestimable and the lone point - may sit on either side of ``L``. It goes on whichever side is larger: - outside, its label roams over ``L^c`` for ``n - m`` assignments; inside, - over the landmarks for ``m``. The reference set is therefore never smaller - than ``ceil(n / 2)``. + ``n_small == 1``. Its within-group mean is identically zero (a single point + has no within-group pairs), so keeping it out of ``L`` costs nothing and + lets its label roam over ``L^c``: ``n - m`` distinct assignments. ``small_group_in_L`` - ``n_s <= m // 2``. ``L`` holds ALL of the small group plus ``m - n_s`` from - the large group. Every permuted small group is then a subset of ``L``, so - every within-small-group distance appears in ``D`` and that term is - computed exactly rather than subsampled. + ``n_small <= m // 2``. ``L`` holds ALL of the small group plus + ``m - n_small`` from the large group. Every permuted small group is then a + subset of ``L``, so every within-small-group distance appears in ``D`` and + that term is computed exactly rather than subsampled. ``proportional`` - Otherwise. ``m_s ~ m * n_s / n``, and both within-group terms are - subsampled. + Otherwise. ``n_small_landmarks ~ m * n_small / n``, and both within-group + terms are subsampled. Cost ---- @@ -55,10 +52,11 @@ threshold degrades as one over the square root of the compute. See Janson (1984) on incomplete U-statistics. -In the ``singleton`` regime a single row is nearly sufficient for the -statistic, so keep ``m`` small -- roughly 5-15% of ``n``. Landmarks there come -straight out of p-value resolution until ``m`` passes ``n / 2``, beyond which -you may as well pay for the full matrix. +Landmarks come straight out of p-value resolution in the ``singleton`` regime, +where the reference set is ``n - m``. Keep ``m`` well under ``n / 2`` in any +case: at ``m = n / 2`` the rectangular matrix saves only a factor of two over +the exact test, which is not worth the lost sensitivity. Landmarks are for +``m << n``. Block means exclude the zero self-pairs, making the statistic unbiased for the population energy distance. It can therefore be negative under H0, like @@ -98,7 +96,6 @@ class torch: "is_torch_tensor", "is_jax_array", "permutation_energy_test", - "allocate_landmarks", "PermutationResolutionWarning", "two_tailed_p", "confidence_alert", @@ -243,96 +240,120 @@ class PermutationResolutionWarning(UserWarning): """The permutation group is too small to resolve the requested p-value.""" -# comb(n, k) is astronomically large for balanced samples; we only ever need to -# know whether the reference set is small, so cap it. -_REFERENCE_CAP = 10**15 - # Target element count for a batch of permutations, bounding peak memory. _BATCH_ELEMENTS = 2**30 -def _capped_comb(n: int, k: int) -> int: - if k > 50 and n - k > 50: - return _REFERENCE_CAP - return min(comb(n, k), _REFERENCE_CAP) +def _reference_size(n: int, n_landmarks: int, n_small: int, n_small_landmarks: int): + """How many distinct labellings the subgroup ``S_L x S_{L^c}`` reaches. + + It places ``n_small_landmarks`` of the small-group labels among the + landmarks and the rest among the ``n - n_landmarks`` other positions, + independently, so the count is the product of two binomials. + + Only smallness matters -- the number is either a handful, in which case the + p-value resolution is limited and the caller needs to know, or it is + astronomical. Returns ``inf`` for the latter rather than spending the + arithmetic on an integer nobody reads. + """ + total = 1 + for size, small in ( + (n_landmarks, n_small_landmarks), + (n - n_landmarks, n_small - n_small_landmarks), + ): + if min(small, size - small) > 50: + return float("inf") + total *= comb(size, small) + return total # --------------------------------------------------------------- allocation -def allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng=None) -> dict: - """Choose which pooled-sample points serve as landmarks. +def _allocate(n1: int, n2: int, n_landmarks: int, rng) -> dict: + """Pick the allocation for this landmark count. + + The two branches describe genuinely different things -- a landmark set and + its subgroup, or no landmark set at all -- so they return different keys. + ``regime`` says which, and is what everything downstream dispatches on. + """ + if n_landmarks >= n1 + n2: + return _allocate_full(n1, n2) + return _allocate_landmarks(n1, n2, n_landmarks, rng) + + +def _allocate_full(n1: int, n2: int) -> dict: + """Every point is a landmark: the classical exact permutation test. + + There is no landmark set to hold fixed, so ``S_L x S_{L^c}`` is the whole + symmetric group and the subgroup restriction is vacuous. The distance + matrix is square and symmetric, which the statistic exploits. Nothing + downstream needs a landmark index, a landmark count or a column gather, so + this allocation does not carry any. + """ + n = n1 + n2 + n_small = min(n1, n2) + return { + "regime": "full", + "small_idx": np.arange(n1) if n1 <= n2 else n1 + np.arange(n2), + "n_small": n_small, + "n_large": max(n1, n2), + # every position is free, so the group reaches every way of choosing + # which n_small of the n labels belong to the small group + "reference_size": _reference_size(n, n, n_small, n_small), + } + - The landmarks are the ``m`` points every sample is measured against: they - supply the columns of the rectangular distance matrix ``D``. +def _allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng) -> dict: + """Choose which pooled-sample points serve as landmarks. - Group 0 is x (pooled indices ``0..n1-1``), group 1 is y (``n1..n-1``). + The landmarks are the ``n_landmarks`` points every sample is measured + against: they supply the columns of the rectangular distance matrix ``D``. Selection uses only group sizes and positions, never the data values, which together with the restricted permutation group is what keeps the test exact. - Parameters - ---------- - n1 (int): number of samples in x. - n2 (int): number of samples in y. - n_landmarks (int): number of landmarks ``m``. ``m >= n1 + n2`` - requests the exact full-matrix test. - rng: seed, ``np.random.Generator``, or None (draw from the global state). + Assumes ``2 <= n_landmarks < n1 + n2`` and that both samples are non-empty; + :func:`permutation_energy_test` checks that before calling. - Returns - ------- - dict with keys ``landmarks``, ``regime``, ``small_idx`` (pooled indices - of the smaller group), ``n_small``, ``n_large``, ``n_small_landmarks``, - ``n_large_landmarks``, ``n_landmarks``, ``reference_size`` - (distinct label assignments the subgroup reaches) - and ``exact_within`` (whether the small group's within-group term is - computed exactly rather than subsampled). + Dispatched on the smaller group: + + ``singleton`` + ``n_small == 1``. A lone point has no within-group pairs, so keeping it + out of ``L`` costs nothing and lets its label roam over the + ``n - n_landmarks`` positions outside. + + ``small_group_in_L`` + ``n_small <= n_landmarks // 2``. ``L`` holds ALL of the small group, so + every permuted small group is still a subset of ``L`` and its + within-group distances are all present -- that term is exact rather + than subsampled. + + ``proportional`` + Otherwise. The landmarks are split between the groups in proportion to + their sizes, and both within-group terms are subsampled. """ - rng = _as_rng(rng) n = n1 + n2 - n_landmarks = int(n_landmarks) - if min(n1, n2) < 1: - raise ValueError(f"both samples need at least one point, got n1={n1}, n2={n2}") - if n_landmarks < 2: - raise ValueError(f"need at least 2 landmarks, got n_landmarks={n_landmarks}") - if n_landmarks > n: - raise ValueError(f"n_landmarks={n_landmarks} exceeds the pooled sample size {n}") - n_small, n_large = min(n1, n2), max(n1, n2) x_is_small = n1 <= n2 small_idx = np.arange(n1) if x_is_small else n1 + np.arange(n2) large_idx = n1 + np.arange(n2) if x_is_small else np.arange(n1) - if n_landmarks == n: - # L is everything, so S_L x S_{L^c} is the full symmetric group and - # this is the classical exact permutation test. - landmarks = np.arange(n) - regime, n_small_landmarks = "full", n_small - elif n_small == 1: - # A lone point has no within-group pairs, so it is free to sit on - # either side of L. Inside, its label roams over the m landmark - # positions; outside, over the n - m others. Take whichever side is - # larger, which holds the reference set at ceil(n / 2) or more instead - # of letting it collapse to 1 as m approaches n. - regime = "singleton" - if n_landmarks > n - n_landmarks: - landmarks = np.concatenate( - [small_idx, rng.choice(large_idx, size=n_landmarks - 1, replace=False)] - ) - n_small_landmarks = 1 - else: - landmarks = rng.choice(large_idx, size=n_landmarks, replace=False) - n_small_landmarks = 0 + if n_small == 1: + landmarks = rng.choice(large_idx, size=n_landmarks, replace=False) + regime, n_small_landmarks = "singleton", 0 elif n_small <= n_landmarks // 2: - # Put the whole small group in L so that every permuted small group is - # still a subset of L and its within-group distances are all present. landmarks = np.concatenate( [small_idx, rng.choice(large_idx, size=n_landmarks - n_small, replace=False)] ) regime, n_small_landmarks = "small_group_in_L", n_small else: - lo = max(1, n_landmarks - n_large) - hi = min(n_small, n_landmarks - 1) - n_small_landmarks = int(np.clip(round(n_landmarks * n_small / n), lo, hi)) + n_small_landmarks = int( + np.clip( + round(n_landmarks * n_small / n), + max(1, n_landmarks - n_large), + min(n_small, n_landmarks - 1), + ) + ) landmarks = np.concatenate( [ rng.choice(small_idx, size=n_small_landmarks, replace=False), @@ -341,14 +362,6 @@ def allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng=None) -> dict: ) regime = "proportional" - # The subgroup places n_small_landmarks of the small labels among the m - # landmarks and the rest among the n - m other positions, independently. - reference = min( - _capped_comb(n_landmarks, n_small_landmarks) - * _capped_comb(n - n_landmarks, n_small - n_small_landmarks), - _REFERENCE_CAP, - ) - return { "landmarks": np.sort(landmarks), "regime": regime, @@ -358,11 +371,20 @@ def allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng=None) -> dict: "n_small_landmarks": n_small_landmarks, "n_large_landmarks": n_landmarks - n_small_landmarks, "n_landmarks": n_landmarks, - "reference_size": reference, - "exact_within": n_small_landmarks == n_small or n_small == 1, + "reference_size": _reference_size(n, n_landmarks, n_small, n_small_landmarks), } +def _landmark_columns(z, alloc: dict, backend: str): + """The points every sample is measured against, i.e. the columns of ``D``. + + All of them in the full regime, the chosen landmark subset otherwise. + """ + if alloc["regime"] == "full": + return z + return z[_index_like(alloc["landmarks"], z, backend)] + + def _warn_reference_size(alloc: dict, permutations: int) -> None: """Warn when the permutation group cannot resolve the requested p-value. @@ -378,20 +400,19 @@ def _warn_reference_size(alloc: dict, permutations: int) -> None: if reference >= floor: return - hint = { - "singleton": ( + if alloc["regime"] == "full": + hint = "the samples are too small for a permutation test at this resolution" + elif alloc["regime"] == "singleton": + hint = ( f"the small group holds a single point, so the reference set is " - f"max(n_landmarks, n - n_landmarks); move n_landmarks (currently " - f"{alloc['n_landmarks']}) further from half the pooled sample size" - ), - "full": "the samples are too small for a permutation test at this resolution", - }.get( - alloc["regime"], - f"use more landmarks (currently {alloc['n_landmarks']}) or more samples", - ) + f"n - n_landmarks; use fewer landmarks (currently {alloc['n_landmarks']}), " + f"well under half the pooled sample size" + ) + else: + hint = f"use more landmarks (currently {alloc['n_landmarks']}) or more samples" warn( PermutationResolutionWarning( - f"the {alloc['regime']!r} landmark allocation reaches only {reference:,} distinct " + f"the {alloc['regime']!r} allocation reaches only {reference:,} distinct " f"label assignments, so the smallest attainable p-value is about " f"{1.0 / reference:.3g} no matter how many permutations are drawn " f"({permutations} requested): {hint}." @@ -426,7 +447,9 @@ def _label_drawer(prep: dict, rng): def draw(rows): U = torch.zeros((rows, n), dtype=ref.dtype, device=ref.device) for idx, k in parts: - keys = torch.rand((rows, idx.numel()), generator=gen, device=ref.device) + keys = torch.rand( + (rows, idx.numel()), generator=gen, device=ref.device, dtype=ref.dtype + ) pick = torch.topk(keys, k, dim=1, largest=False, sorted=False).indices U.scatter_(1, idx[pick], 1.0) return U @@ -490,142 +513,261 @@ def _enumerate_label_blocks(prep: dict, batch_size: int): # ---------------------------------------------------------------- statistic +# +# The energy statistic is a fixed combination of three block means -- the mean +# cross-group distance and the two mean within-group distances: +# +# E = (n_small * n_large / n) * (2 * cross - small - large) +# +# Each mean is a block of the distance matrix over its pair count, and every +# one of those counts is a constant: the subgroup S_L x S_{L^c} holds the +# per-group landmark counts fixed. So preparing a run means centring the +# matrix and recording the counts; evaluating a batch of labellings means +# forming the block sums and dividing. +# +# Writing u for a row of the (B, n) small-group indicator and v = u[landmarks] +# for its restriction to the landmark columns, each block sum is a bilinear +# form and one matrix product yields all four: +# +# S_ss = u @ D @ v S_sl = u @ D @ 1 - S_ss +# S_ls = v @ (1 @ D) - S_ss S_ll = total - S_ss - S_sl - S_ls +# +# The three regimes differ in which of those blocks survive, by enough that +# each gets its own prepare/evaluate pair rather than a shared one with the +# differences hidden in coefficients. def _prepare_statistic(D, alloc: dict, backend: str) -> dict: - """Precompute everything that is constant across permutations. + """Precompute everything constant across permutations, for this regime.""" + prepare = {"full": _prepare_full, "singleton": _prepare_singleton} + return prepare.get(alloc["regime"], _prepare_landmarks)(D, alloc, backend) + + +def _centre(D, self_pair_rows, backend: str): + """Centre ``D`` on its mean and put the self-pairs back to zero. + + In high dimension the pairwise distances concentrate far from zero and the + uncentred block sums spend most of their significant digits on an offset + that largely cancels, so centre before any reduction. ``self_pair_rows[k]`` + is the point that column ``k`` was measured against; those entries have to + return to zero because the pair counts exclude them. + + Returns ``(D, offset, total)``. Add ``offset`` back to any block *mean* + taken on the centred matrix to recover the mean on the real distances -- + the offset itself needs no precision, since it either cancels in + ``2 * cross - small - large`` or multiplies a block that is genuinely + absent. ``total`` is accumulated in float64 because its error does survive + into the reported statistic. + """ + offset = float(D.mean()) + D = _zero_self_pairs(D - offset, self_pair_rows, backend) + total = float(_to_numpy(D.sum(1), backend).sum(dtype=np.float64)) + return D, offset, total - Every block sum is a bilinear form in the row indicator ``u`` (length n) - and the landmark indicator ``v = u[landmarks]`` (length m): - u @ D @ v = sum_{i in A} sum_{k: landmarks[k] in A} D[i, k] +def _subgroup_parts(alloc: dict, n: int): + """``(positions, how many carry the small label)`` per half of the subgroup. + + Those two counts are exactly what ``S_L x S_{L^c}`` holds fixed. The full + regime has no ``L^c``, so it is one part spanning every position. + """ + if alloc["regime"] == "full": + return ((np.arange(n), alloc["n_small"]),) + in_l = np.zeros(n, dtype=bool) + in_l[alloc["landmarks"]] = True + n_small_landmarks = alloc["n_small_landmarks"] + return ( + (np.flatnonzero(in_l), n_small_landmarks), + (np.flatnonzero(~in_l), alloc["n_small"] - n_small_landmarks), + ) - All the denominators are constants, because the subgroup ``S_L x S_{L^c}`` - preserves the per-group landmark counts. That lets the whole statistic - collapse into a single affine function of at most three per-permutation - scalars, whose coefficients are folded down to plain Python floats here: - E = a_ss * S_ss + a_row * R + a_col * C + const +def _base_small(alloc: dict, n: int) -> np.ndarray: + """The observed labelling, as a small-group indicator.""" + base = np.zeros(n) + base[alloc["small_idx"]] = 1.0 + return base - where ``R = u @ D @ 1`` and ``C = v @ (1 @ D)``. Everything else -- the - other three block sums, the pair counts, the ``n_s n_l / n`` scaling -- is - absorbed into those four numbers, so a permutation costs one matrix - product and a handful of elementwise ops rather than a dozen small - kernels on device. + +# ----------------------------------------------------------- full matrix + + +def _prepare_full(D, alloc: dict, backend: str) -> dict: + """``D`` is square and symmetric, so the column sums are the row sums.""" + n_small, n_large = alloc["n_small"], alloc["n_large"] + n = n_small + n_large + # the columns are the points themselves, so the self-pairs are the diagonal + D, offset, total = _centre(D, np.arange(n), backend) + return { + "D": D, + "ref": D, + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), + "backend": backend, + "offset": offset, + "total": total, + "scale": n_small * n_large / n, + "n_small": n_small, + "n_large": n_large, + "evaluate": _evaluate_full, + } + + +def _evaluate_full(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, full symmetric matrix. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_full`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. """ - n_s, n_l = alloc["n_small"], alloc["n_large"] - n = n_s + n_l - m_s, m_l = alloc["n_small_landmarks"], alloc["n_large_landmarks"] - landmarks = alloc["landmarks"] - full = alloc["regime"] == "full" + backend, offset = prep["backend"], prep["offset"] + n_small, n_large = prep["n_small"], prep["n_large"] + u = _asarray_like(U, prep["D"], backend) + + W = u @ prep["D"] # (B, n) + S_ss = (W * u).sum(-1) + S_sl = W.sum(-1) - S_ss + # symmetry gives S_ls == S_sl, so there is no column-sum term to form + S_ll = prep["total"] - S_ss - 2.0 * S_sl + + cross = S_sl / (n_small * n_large) + offset + # A group of one point has no within-group pairs at all, so its mean is + # zero on the real distances -- which is exactly where `offset` matters. + small = 0.0 if n_small == 1 else S_ss / (n_small * (n_small - 1)) + offset + large = 0.0 if n_large == 1 else S_ll / (n_large * (n_large - 1)) + offset + + stat = prep["scale"] * (2.0 * cross - small - large) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +# ------------------------------------------------------------- landmarks - # allocate_landmarks guarantees m_l >= 1 in every regime, and m_s >= 1 - # unless n_s == 1 (the singleton regime, where the small group has no - # within-group pairs to estimate), so both block means below are defined. - - # The statistic is invariant to a constant offset on every non-self pair, - # so centre D before any reduction. In high dimension the pairwise - # distances concentrate far from zero, and the uncentred sums spend most of - # their significant digits on an offset that cancels anyway. Any constant - # cancels exactly, so the offset itself needs no precision -- a float32 - # mean is fine. That is not true of `total` below, whose error survives - # into the reported statistic, so that one accumulates in float64. - offset = float(D.mean()) - D = D - offset - D = _zero_self_pairs(D, landmarks, backend) - - row_sums = D.sum(1) - total = float(_to_numpy(row_sums, backend).sum(dtype=np.float64)) - - # --- fold 2*mu_sl - mu_ss - mu_ll into A*S_ss + B*S_sl + C*S_ls + const -- - a, b, c, const = 0.0, 0.0, 0.0, 0.0 - if m_s > 0: # both cross orientations are available - b += 1.0 / (n_s * m_l) - c += 1.0 / (n_l * m_s) - else: # no small landmarks, so only one orientation exists - b += 2.0 / (n_s * m_l) - if n_s < 2: # a lone point has no within-group pairs; its mean is -offset - const += offset - else: - a -= 1.0 / (m_s * (n_s - 1)) - if n_l < 2: - const += offset - else: # S_ll = total - S_ss - S_sl - S_ls - pairs_ll = m_l * (n_l - 1) - a += 1.0 / pairs_ll - b += 1.0 / pairs_ll - c += 1.0 / pairs_ll - const -= total / pairs_ll - - # S_sl = R - S_ss and S_ls = C - S_ss, so rewrite in terms of what a - # permutation actually computes, then absorb the n_s n_l / n scaling. - scale = n_s * n_l / n - a_ss, a_row, a_col = scale * (a - b - c), scale * b, scale * c - const *= scale - - # With no small landmarks both S_ss and C are identically zero, so the - # matrix product is not needed at all -- a matrix-vector product against - # the row sums is the whole statistic. - needs_matrix = m_s > 0 - if full: - # D is symmetric, so the column sums are the row sums and C == R. - a_row, a_col = a_row + a_col, 0.0 - - base_small = np.zeros(n) - base_small[alloc["small_idx"]] = 1.0 - in_l = np.zeros(n, dtype=bool) - in_l[landmarks] = True - idx_l, idx_o = np.flatnonzero(in_l), np.flatnonzero(~in_l) - # Whatever the statistic reads sets the device and dtype the labels are - # built in, so they never have to cross a bus. - ref = D if needs_matrix else row_sums +def _prepare_landmarks(D, alloc: dict, backend: str) -> dict: + """``D`` is rectangular, and all four blocks are subsampled over columns. + + Reached whenever the small group has landmarks of its own, which also + means it holds at least two points: ``n_small == 1`` goes to the singleton + pair below. ``n_large == 1`` forces ``n <= 2``, which is always full. + """ + n_small, n_large = alloc["n_small"], alloc["n_large"] + n_small_landmarks = alloc["n_small_landmarks"] + n_large_landmarks = alloc["n_large_landmarks"] + n = n_small + n_large + D, offset, total = _centre(D, alloc["landmarks"], backend) return { - "D": D if needs_matrix else None, - "ref": ref, - # (positions, how many of them carry the small label) for each half of - # the subgroup; those counts are what the subgroup holds fixed. - "parts": ((idx_l, m_s), (idx_o, n_s - m_s)), - # The landmarks cover everything in the full regime, so skip the gather. - "landmarks": None if full else _index_like(landmarks, D, backend), - "col_sums": D.sum(0) if (needs_matrix and a_col) else None, - "a_ss": a_ss, - "a_row": a_row, - "a_col": a_col, - "const": const, - "base_small": base_small, + "D": D, + "ref": D, + "col_sums": D.sum(0), + "landmarks": _index_like(alloc["landmarks"], D, backend), + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), "backend": backend, - "evaluate": _evaluate_statistic, + "offset": offset, + "total": total, + "scale": n_small * n_large / n, + # pairs behind each block: rows times landmark columns, self-pairs out + "ss_pairs": n_small_landmarks * (n_small - 1), + "sl_pairs": n_small * n_large_landmarks, + "ls_pairs": n_large * n_small_landmarks, + "ll_pairs": n_large_landmarks * (n_large - 1), + "evaluate": _evaluate_landmarks, } -def _evaluate_statistic(prep: dict, U) -> np.ndarray: - """Energy statistic for a batch of label assignments. +def _evaluate_landmarks(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, rectangular matrix. Parameters ---------- - prep (dict): output of :func:`_prepare_statistic`. + prep (dict): output of :func:`_prepare_landmarks`. U (np.ndarray): ``(B, n)`` indicator of small-group membership. Returns ------- ``(B,)`` numpy array of statistics. """ - backend, D = prep["backend"], prep["D"] - Us = _asarray_like(U, prep["ref"], backend) + backend, offset = prep["backend"], prep["offset"] + u = _asarray_like(U, prep["D"], backend) + v = u[:, prep["landmarks"]] # the same indicator, restricted to the columns + + W = u @ prep["D"] # (B, m) + S_ss = (W * v).sum(-1) + S_sl = W.sum(-1) - S_ss + S_ls = v @ prep["col_sums"] - S_ss + S_ll = prep["total"] - S_ss - S_sl - S_ls + + # The cross block is sampled in both orientations here, small rows against + # large columns and large rows against small columns. They estimate the + # same mean over different pairs, so average them. + cross = 0.5 * (S_sl / prep["sl_pairs"] + S_ls / prep["ls_pairs"]) + offset + small = S_ss / prep["ss_pairs"] + offset + large = S_ll / prep["ll_pairs"] + offset + + stat = prep["scale"] * (2.0 * cross - small - large) + return _to_numpy(stat, backend).astype(np.float64, copy=False) - if D is None: # no small landmarks, so ref is the row sums and they are the whole statistic - stat = prep["a_row"] * (Us @ prep["ref"]) + prep["const"] - return _to_numpy(stat, backend).astype(np.float64, copy=False) - Vs = Us if prep["landmarks"] is None else Us[:, prep["landmarks"]] +# ------------------------------------------------------------- singleton - # One matrix product carries both scalars: W @ 1 is the row-sum term and - # (W * Vs) summed is the small-by-small block. - W = Us @ D - stat = prep["a_ss"] * (W * Vs).sum(-1) + prep["a_row"] * W.sum(-1) + prep["const"] - if prep["col_sums"] is not None: - stat = stat + prep["a_col"] * (Vs @ prep["col_sums"]) + +def _prepare_singleton(D, alloc: dict, backend: str) -> dict: + """The small group is one point, deliberately kept out of the landmarks. + + With no small-group landmarks two of the four blocks vanish: ``S_ss`` and + ``S_ls`` have no columns to sum over. The only quantity left that varies + with the labelling is the row sum of whichever point carries the small + label, so the matrix is reduced to its row sums here and never touched + again -- there is no matrix product per permutation, only a dot product. + """ + n_large = alloc["n_large"] + n_large_landmarks = alloc["n_large_landmarks"] + n = 1 + n_large + D, offset, total = _centre(D, alloc["landmarks"], backend) + return { + "ref": D, + "row_sums": D.sum(1), + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), + "backend": backend, + "offset": offset, + "total": total, + "scale": n_large / n, + # the one small point against every large landmark + "sl_pairs": n_large_landmarks, + "ll_pairs": n_large_landmarks * (n_large - 1), + "evaluate": _evaluate_singleton, + } + + +def _evaluate_singleton(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, one-point small group. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_singleton`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. + """ + backend, offset = prep["backend"], prep["offset"] + u = _asarray_like(U, prep["row_sums"], backend) + + S_sl = u @ prep["row_sums"] # S_ss and S_ls are empty blocks + S_ll = prep["total"] - S_sl + + cross = S_sl / prep["sl_pairs"] + offset + small = 0.0 # one point, no within-group pairs: zero on the real distances + large = S_ll / prep["ll_pairs"] + offset + + stat = prep["scale"] * (2.0 * cross - small - large) return _to_numpy(stat, backend).astype(np.float64, copy=False) @@ -640,7 +782,7 @@ def _containment_curve(prep: dict, U): Us = _asarray_like(U, D, backend) Xs = Us if prep["x_is_small"] else 1.0 - Us Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] - depth = (D @ (1.0 - Vx).T) / prep["m_y"] + depth = (D @ (1.0 - Vx).T) / prep["n_y_landmarks"] order = (-depth).argsort(0) ranked_x = _to_numpy(_take_along0(Xs.T, order, backend), backend) ranked_y = _to_numpy(_take_along0((1.0 - Xs).T, order, backend), backend) @@ -667,14 +809,16 @@ def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict Unlike the energy distance this is *not* symmetric in x and y, which is the whole point. """ - n_s, n_l = alloc["n_small"], alloc["n_large"] - n = n_s + n_l - m_s, m_l = alloc["n_small_landmarks"], alloc["n_large_landmarks"] - landmarks = alloc["landmarks"] + n = alloc["n_small"] + alloc["n_large"] full = alloc["regime"] == "full" - m_y = m_l if x_is_small else m_s - n_y = n_l if x_is_small else n_s - if m_y < 1: + # y is whichever group x is not. + n_y = alloc["n_large"] if x_is_small else alloc["n_small"] + if full: # every point is a column, so every y point is one + self_pair_rows, n_y_landmarks = np.arange(n), n_y + else: + self_pair_rows = alloc["landmarks"] + n_y_landmarks = alloc["n_large_landmarks"] if x_is_small else alloc["n_small_landmarks"] + if n_y_landmarks < 1: raise ValueError( "the containing sample has no landmarks, so depth is unestimable; " "use more landmarks or the full matrix" @@ -683,23 +827,17 @@ def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict # Ranks are invariant to a constant offset, but zeroing the self-pairs # still matters: a y point counting its own zero distance would look # artificially deep. - D = D - float(D.mean()) - D = _zero_self_pairs(D, landmarks, backend) - - base_small = np.zeros(n) - base_small[alloc["small_idx"]] = 1.0 - in_l = np.zeros(n, dtype=bool) - in_l[landmarks] = True + D, _, _ = _centre(D, self_pair_rows, backend) return { "D": D, "ref": D, - "parts": ((np.flatnonzero(in_l), m_s), (np.flatnonzero(~in_l), n_s - m_s)), - "landmarks": None if full else _index_like(landmarks, D, backend), + "parts": _subgroup_parts(alloc, n), + "landmarks": None if full else _index_like(alloc["landmarks"], D, backend), "x_is_small": x_is_small, - "m_y": float(m_y), + "n_y_landmarks": float(n_y_landmarks), "n_y": float(n_y), - "base_small": base_small, + "base_small": _base_small(alloc, n), "backend": backend, "evaluate": _evaluate_containment, } @@ -713,7 +851,7 @@ def _evaluate_containment(prep: dict, U) -> np.ndarray: Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] # Depth of every pooled point against each labelling's y set. - depth = (D @ (1.0 - Vx).T) / prep["m_y"] # (n, B) + depth = (D @ (1.0 - Vx).T) / prep["n_y_landmarks"] # (n, B) # Rank by depth, most peripheral first; the running count of y points is # then #{j in y : d_j >= d_i} without ever materialising a comparison. @@ -794,17 +932,18 @@ def permutation_energy_test( n1, n2 = len(x), len(y) n = n1 + n2 - # More landmarks than points is just the full matrix; anything else - # nonsensical is rejected by allocate_landmarks, which owns that check. + if min(n1, n2) < 1: + raise ValueError(f"both samples need at least one point, got n1={n1}, n2={n2}") + # More landmarks than points is just the full matrix. m = n if n_landmarks is None else min(int(n_landmarks), n) + if m < 2: + raise ValueError(f"need at least 2 landmarks, got n_landmarks={n_landmarks}") rng = _as_rng(rng) - alloc = allocate_landmarks(n1, n2, m, rng) + alloc = _allocate(n1, n2, m, rng) _warn_reference_size(alloc, permutations) - landmarks = alloc["landmarks"] - zc = z if alloc["regime"] == "full" else z[_index_like(landmarks, z, backend)] - dmatrix = _cdist(z, zc, backend) + dmatrix = _cdist(z, _landmark_columns(z, alloc, backend), backend) assert _all_finite(dmatrix, backend), ( "Distance matrix contains NaN or Inf! Consider normalizing values to be " "more stable (i.e. z-score norm)." @@ -1210,10 +1349,9 @@ def containment_pit_plot( n = n1 + n2 m = n if n_landmarks is None else min(int(n_landmarks), n) - alloc = allocate_landmarks(n1, n2, m, _as_rng(rng)) - landmarks = alloc["landmarks"] - zc = z if alloc["regime"] == "full" else z[_index_like(landmarks, z, backend)] - prep = _prepare_containment(_cdist(z, zc, backend), alloc, backend, n1 <= n2) + alloc = _allocate(n1, n2, m, _as_rng(rng)) + D = _cdist(z, _landmark_columns(z, alloc, backend), backend) + prep = _prepare_containment(D, alloc, backend, n1 <= n2) observed = _containment_curve(prep, prep["base_small"][None, :])[0] / n1 n_y = int(prep["n_y"]) diff --git a/tests/test_pted.py b/tests/test_pted.py index e7122f5..773078b 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -368,9 +368,9 @@ def test_landmark_pvalues_are_calibrated(n1, n2, n_landmarks, regime): permuted ones. It rejected at 29% (proportional) and 84% (singleton) for a nominal 5%. Fixed seed, so this is deterministic rather than flaky. """ - from pted.utils import allocate_landmarks + from pted.utils import _allocate, _as_rng - assert allocate_landmarks(n1, n2, n_landmarks, rng=0)["regime"] == regime + assert _allocate(n1, n2, n_landmarks, _as_rng(0))["regime"] == regime trials, permutations = 600, 49 rng = np.random.default_rng(20240904) @@ -514,7 +514,8 @@ def test_containment_pit_plot(tmp_path): """The plot is written, and its curve shows the failure mode the Fisher aggregate is blind to.""" from pted.utils import ( - allocate_landmarks, + _allocate, + _as_rng, _cdist, _containment_curve, _lattice_band, @@ -533,7 +534,7 @@ def test_containment_pit_plot(tmp_path): def curve(x): z = np.vstack([x, y]) - alloc = allocate_landmarks(len(x), len(y), len(z), rng=0) + alloc = _allocate(len(x), len(y), len(z), _as_rng(0)) prep = _prepare_containment(_cdist(z, z, "numpy"), alloc, "numpy", True) return _containment_curve(prep, prep["base_small"][None, :])[0] / len(x) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1a352a1..6cb6904 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,13 +8,14 @@ from scipy.stats import kstwo from pted.utils import ( + _as_rng, _band_accept_probability, _lattice_band, _sampled_label_blocks, _lattice_counts, _enumerate_label_blocks, PermutationResolutionWarning, - allocate_landmarks, + _allocate, two_tailed_p, simulation_based_calibration_histogram, pit_plot, @@ -22,11 +23,17 @@ permutation_energy_test, _cdist, _label_drawer, - _evaluate_statistic, _index_like, + _landmark_columns, _prepare_statistic, ) + +def _evaluate_statistic(prep, U): + """The statistic for a batch of labellings, through this prep's evaluator.""" + return prep["evaluate"](prep, U) + + try: import torch except ImportError: @@ -152,19 +159,21 @@ def block(rows, kk, drop_self): # n1, n2, n_landmarks, expected regime (100, 100, 200, "full"), (3, 40, 43, "full"), - (1, 60, 12, "singleton"), # c < n/2 -> the lone point stays out of C - (1, 20, 15, "singleton"), # c > n/2 -> it moves into C instead + (1, 60, 12, "singleton"), + (1, 200, 20, "singleton"), (6, 80, 40, "small_group_in_L"), (40, 50, 30, "proportional"), (40, 50, 3, "proportional"), ] +LANDMARK_ALLOCATIONS = [a for a in ALLOCATIONS if a[3] != "full"] -@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) + +@pytest.mark.parametrize("n1,n2,c,regime", LANDMARK_ALLOCATIONS) def test_allocate_landmarks_regimes(n1, n2, c, regime): """Each size combination lands in the intended regime with a consistent set of columns.""" - alloc = allocate_landmarks(n1, n2, c, rng=0) + alloc = _allocate(n1, n2, c, _as_rng(0)) landmarks = alloc["landmarks"] n = n1 + n2 @@ -178,45 +187,56 @@ def test_allocate_landmarks_regimes(n1, n2, c, regime): assert np.sum(np.isin(landmarks, alloc["small_idx"])) == alloc["n_small_landmarks"] assert alloc["n_small_landmarks"] + alloc["n_large_landmarks"] == c assert alloc["n_large_landmarks"] >= 1, "the large group must keep at least one column" - assert alloc["exact_within"] == ( - alloc["n_small_landmarks"] == alloc["n_small"] or alloc["n_small"] == 1 - ) -def test_allocate_landmarks_reference_size(): - """reference_size counts the label assignments the subgroup can reach.""" - for n1, n2, c, _ in ALLOCATIONS: - alloc = allocate_landmarks(n1, n2, c, rng=1) - n, n_s, c_s = n1 + n2, alloc["n_small"], alloc["n_small_landmarks"] - expected = comb(c, c_s) * comb(n - c, n_s - c_s) - assert alloc["reference_size"] == min(expected, 10**15) +@pytest.mark.parametrize("n1,n2,c,regime", [a for a in ALLOCATIONS if a[3] == "full"]) +def test_full_allocation_carries_no_landmark_fields(n1, n2, c, regime): + """The full regime has no landmark set, so it does not describe one: the + statistic, the subgroup and the distance matrix are all built without.""" + alloc = _allocate(n1, n2, c, _as_rng(0)) + assert alloc["regime"] == "full" + assert not [k for k in alloc if "landmark" in k] + exact = comb(n1 + n2, min(n1, n2)) + assert alloc["reference_size"] in (exact, float("inf")) + z = np.random.default_rng(0).standard_normal((n1 + n2, 3)) + assert _landmark_columns(z, alloc, "numpy") is z, "no column gather" -def test_allocate_landmarks_errors(): - """allocate_landmarks is the single gate on sample and column counts.""" - with pytest.raises(ValueError, match="at least 2 landmarks"): - allocate_landmarks(10, 10, 1) - with pytest.raises(ValueError, match="exceeds the pooled sample size"): - allocate_landmarks(10, 10, 21) - with pytest.raises(ValueError, match="both samples need at least one point"): - allocate_landmarks(0, 10, 5) - # and it is reached through the public API, not bypassed by the mapping + +def test_allocate_landmarks_reference_size(): + """reference_size counts the label assignments the subgroup can reach.""" + for n1, n2, m, _ in LANDMARK_ALLOCATIONS: + alloc = _allocate(n1, n2, m, _as_rng(1)) + n = n1 + n2 + n_small, n_small_landmarks = alloc["n_small"], alloc["n_small_landmarks"] + expected = comb(m, n_small_landmarks) * comb(n - m, n_small - n_small_landmarks) + got = alloc["reference_size"] + # counted exactly while it is small enough to matter, inf beyond that + assert got == expected or (got == float("inf") and expected > 10**15) + + +def test_sample_and_landmark_counts_are_validated(): + """The allocators assume valid input; permutation_energy_test is the gate.""" + x = np.zeros((5, 2)) with pytest.raises(ValueError, match="at least 2 landmarks"): - permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=1, n_landmarks=1) + permutation_energy_test(x, x, permutations=1, n_landmarks=1) with pytest.raises(ValueError, match="both samples need at least one point"): - permutation_energy_test(np.zeros((0, 2)), np.zeros((5, 2)), permutations=1) + permutation_energy_test(np.zeros((0, 2)), x, permutations=1) + # more landmarks than points is the full matrix, not an error + stat, perm = permutation_energy_test(x, x, permutations=4, n_landmarks=10_000, rng=0) + assert np.isfinite(stat) def _prep_for(n1, n2, c, seed=3, backend="numpy"): """An allocation plus its prepared statistic, on the given backend.""" rng = np.random.default_rng(seed) - alloc = allocate_landmarks(n1, n2, c, rng) + alloc = _allocate(n1, n2, c, rng) z = rng.standard_normal((n1 + n2, 4)) if backend == "torch": z = torch.tensor(z) elif backend == "jax": z = jnp.array(z) - zc = z if alloc["regime"] == "full" else z[_index_like(alloc["landmarks"], z, backend)] + zc = _landmark_columns(z, alloc, backend) return alloc, _prepare_statistic(_cdist(z, zc, backend), alloc, backend) @@ -231,9 +251,10 @@ def test_draw_labels_stays_in_subgroup(n1, n2, c, regime): assert np.all(np.isin(U, [0.0, 1.0])) assert np.all(U.sum(1) == alloc["n_small"]), "small group size is preserved" - assert np.all( - U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"] - ), "column counts are preserved" + if alloc["regime"] != "full": + assert np.all( + U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"] + ), "column counts are preserved" # and the labels really do move around within each part if alloc["reference_size"] > 100: assert len(np.unique(U, axis=0)) > 1 @@ -248,9 +269,10 @@ def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): y = rng.standard_normal((n2, 4)) + 0.4 z = np.vstack([x, y]) - alloc = allocate_landmarks(n1, n2, c, rng) - D = cdist(z, z[alloc["landmarks"]]) - prep = _prepare_statistic(D, alloc, "numpy") + alloc = _allocate(n1, n2, c, rng) + # the full regime keeps no landmark index; its columns are every point + columns = np.arange(n1 + n2) if regime == "full" else alloc["landmarks"] + prep = _prepare_statistic(cdist(z, z[columns]), alloc, "numpy") U = np.concatenate( [ @@ -259,7 +281,7 @@ def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): ] ) got = _evaluate_statistic(prep, U) - expect = [_brute_block_means(z, U[b], alloc["landmarks"], n1 + n2) for b in range(len(U))] + expect = [_brute_block_means(z, U[b], columns, n1 + n2) for b in range(len(U))] assert np.allclose(got, expect, atol=1e-9) @@ -284,7 +306,7 @@ def test_statistic_backend_agreement(backend): y = rng.standard_normal((n2, 5)) + 0.3 z = np.vstack([x, y]) - alloc = allocate_landmarks(n1, n2, c, rng) + alloc = _allocate(n1, n2, c, rng) U = None results = {} for be in ("numpy", backend): @@ -381,20 +403,29 @@ def test_n_landmarks_covering_the_sample_is_the_full_test(): assert stat == ref[0] and np.array_equal(perm, ref[1]) -def test_singleton_picks_the_larger_side_of_C(): - """A lone point sits inside or outside C, whichever leaves more label - assignments reachable. Without this the reference set collapses to 1 as c - approaches n, and the test can only ever return p = 1.""" +def test_singleton_keeps_its_point_out_of_the_landmarks(): + """A lone point has no within-group pairs, so leaving it out of L costs + nothing and its label roams over the n - m positions outside. The reference + set therefore shrinks as m grows -- which is why landmarks are for m << n, + and why the resolution warning exists.""" n2 = 200 n = 1 + n2 - for c in range(2, n): - alloc = allocate_landmarks(1, n2, c, rng=0) + for m in range(2, n): + alloc = _allocate(1, n2, m, _as_rng(0)) assert alloc["regime"] == "singleton" - assert alloc["n_small_landmarks"] == (1 if c > n - c else 0) - assert alloc["reference_size"] == max(c, n - c) - # so the reference set never drops below half the pooled sample - worst = min(allocate_landmarks(1, n2, c, rng=0)["reference_size"] for c in range(2, n)) - assert worst >= n // 2 + assert alloc["n_small_landmarks"] == 0 + assert alloc["reference_size"] == n - m + assert 0 not in alloc["landmarks"], "the lone point is never a landmark" + + +def test_reference_size_is_inf_when_astronomical(): + """Only smallness matters, so the count is not carried past what anyone + would read.""" + from pted.utils import _reference_size + + assert _reference_size(20, 20, 10, 10) == comb(20, 10) + assert _reference_size(201, 20, 1, 0) == 181 + assert _reference_size(2000, 1000, 500, 250) == float("inf") def test_permutation_resolution_warning(): @@ -419,12 +450,24 @@ def test_permutation_resolution_warning(): n_landmarks=5, ) - # sensible column counts are quiet, at either end of the singleton range - x, y = rng.standard_normal((1, 3)), rng.standard_normal((60, 3)) + # a singleton with too many landmarks now warns rather than flipping sides + with pytest.warns(PermutationResolutionWarning, match="use fewer landmarks"): + permutation_energy_test( + rng.standard_normal((1, 3)), + rng.standard_normal((60, 3)), + permutations=100, + n_landmarks=55, + ) + + # a sensible landmark count is quiet with warnings.catch_warnings(): warnings.simplefilter("error", PermutationResolutionWarning) - permutation_energy_test(x, y, permutations=100, n_landmarks=8) - permutation_energy_test(x, y, permutations=100, n_landmarks=55) + permutation_energy_test( + rng.standard_normal((1, 3)), + rng.standard_normal((60, 3)), + permutations=100, + n_landmarks=8, + ) # --------------------------------------------------------------------------- @@ -549,7 +592,7 @@ def test_small_groups_are_enumerated_not_sampled(): whole group instead, returning reference_size - 1 null statistics.""" x = np.random.default_rng(0).standard_normal((1, 3)) y = np.random.default_rng(1).standard_normal((60, 3)) - reference = allocate_landmarks(1, 60, 61, rng=0)["reference_size"] + reference = _allocate(1, 60, 61, _as_rng(0))["reference_size"] assert reference == 61 _, permute = permutation_energy_test(x, y, permutations=1000, rng=0) @@ -563,7 +606,7 @@ def test_small_groups_are_enumerated_not_sampled(): def test_enumeration_covers_the_group_exactly_once(): """Every assignment the subgroup reaches appears once, bar the observed.""" n1, n2, m = 1, 12, 5 - alloc = allocate_landmarks(n1, n2, m, rng=0) + alloc = _allocate(n1, n2, m, _as_rng(0)) z = np.random.default_rng(0).standard_normal((n1 + n2, 2)) prep = _prepare_statistic(cdist(z, z[alloc["landmarks"]]), alloc, "numpy") From 3dfc1a1f888720ca52dbcaa0c975f9f7a92fa351 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Tue, 15 Sep 2026 10:25:51 -0400 Subject: [PATCH 7/7] review comments addressed --- README.md | 8 +-- src/pted/pted.py | 59 ++++++++-------------- src/pted/utils.py | 116 ++++++++++++++++++++++++++++++++++---------- tests/test_utils.py | 62 +++++++++++++++++++++++ 4 files changed, 178 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 147f8cf..201b87d 100644 --- a/README.md +++ b/README.md @@ -239,8 +239,8 @@ by default). The **vertical line** marks threshold p-value (0.05 by default). Parts of the CDF to the right of this line are embedded in `y` at the threshold level and so likely are contained. Points to its left are peripheral relative to `y` and so -this may suggest the values are not contained. However, many values of `x` are -being tested some some leakage to low p-values are expected, which is why the +this may suggest the values are not contained. However, when many values of `x` +are being tested some leakage to low p-values are expected, which is why the uniform-upper-bound line is also plotted. Read them together rather than as a decision rule. A curve with points to the @@ -333,7 +333,7 @@ def pted( * **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. * **two_tailed** *(bool)*: if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. If False, only checks for dissimilarity but is more sensitive. Default is True. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of permutation tests. Default is False. -* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks the largest size that keeps the batch-scaling tensors within 1 GiB and bounds one batch's matrix product, so a run cannot allocate unboundedly whatever the sample shape. * **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ### Coverage test @@ -368,7 +368,7 @@ def pted_coverage_test( * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. * **pit_confidence** *(float)*: Confidence level for the PIT plot's simultaneous band. Default is 0.95 (95%). Only used when `pit_plot` is not None. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of simulations. Default is False. -* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks a size that keeps a batch to a few million elements. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks the largest size that keeps the batch-scaling tensors within 1 GiB and bounds one batch's matrix product, so a run cannot allocate unboundedly whatever the sample shape. * **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ## GPU Compatibility diff --git a/src/pted/pted.py b/src/pted/pted.py index 3d2d004..4e92ba4 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -4,7 +4,9 @@ from .utils import ( permutation_energy_test as _energy_test, - containment_pit_plot as _containment_pit_plot, + _prepare_test, + _permutation_distribution, + _draw_containment_pit, _as_rng, two_tailed_p, confidence_alert, @@ -16,8 +18,8 @@ def pted( - x: Union[np.ndarray, "Tensor", "jax.Array"], - y: Union[np.ndarray, "Tensor", "jax.Array"], + x, + y, permutations: int = 1000, return_all: bool = False, n_landmarks: Optional[int] = None, @@ -149,10 +151,8 @@ def pted( assert ( x.shape[1:] == y.shape[1:] ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" - if len(x.shape) > 2: - x = x.reshape(x.shape[0], -1) - if len(y.shape) > 2: - y = y.reshape(y.shape[0], -1) + x = x.reshape(x.shape[0], -1) + y = y.reshape(y.shape[0], -1) # A landmark count covering the whole pooled sample lands in the exact # full-matrix regime, so there is no separate branch for it here. @@ -184,8 +184,8 @@ def pted( def pted_containment_test( - x: Union[np.ndarray, "Tensor", "jax.Array"], - y: Union[np.ndarray, "Tensor", "jax.Array"], + x, + y, permutations: int = 1000, return_all: bool = False, n_landmarks: Optional[int] = None, @@ -294,35 +294,19 @@ def pted_containment_test( assert ( x.shape[1:] == y.shape[1:] ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" - if len(x.shape) > 2: - x = x.reshape(x.shape[0], -1) - if len(y.shape) > 2: - y = y.reshape(y.shape[0], -1) + x = x.reshape(x.shape[0], -1) + y = y.reshape(y.shape[0], -1) - test, permute = _energy_test( - x, - y, - permutations=permutations, - n_landmarks=n_landmarks, - prog_bar=prog_bar, - batch_size=batch_size, - rng=rng, - containment=True, - ) + # Prepared once and used twice: the plot describes the same landmarks and + # the same distance matrix the statistic was computed from. + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=True) + test, permute = _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng) - # One-sided by construction: only an unusually large aggregate is evidence - # that x reaches outside y. if pit_plot is not None: - _containment_pit_plot( - x, - y, - pit_plot, - confidence=pit_confidence, - threshold=pit_threshold, - n_landmarks=n_landmarks, - rng=rng, - ) + _draw_containment_pit(prep, pit_plot, pit_confidence, pit_threshold) + # One-sided by construction: only an unusually large aggregate is evidence + # that x reaches outside y. pval = (1.0 + np.sum(permute >= test)) / (1.0 + len(permute)) if return_all: return test, permute, pval @@ -330,8 +314,8 @@ def pted_containment_test( def pted_coverage_test( - g: Union[np.ndarray, "Tensor", "jax.Array"], - s: Union[np.ndarray, "Tensor", "jax.Array"], + g, + s, permutations: int = 1000, warn_confidence: Optional[float] = 1e-3, return_all: bool = False, @@ -468,8 +452,7 @@ def pted_coverage_test( assert ( g.shape == s.shape[1:] ), f"g and s must have the same shape (past first dim of s), not {g.shape} and {s.shape}" - if len(s.shape) > 3: - s = s.reshape(nsamp, nsim, -1) + s = s.reshape(nsamp, nsim, -1) g = g.reshape(1, nsim, -1) # Coerce once, so the generator's state advances across simulations. Passing diff --git a/src/pted/utils.py b/src/pted/utils.py index c99a93e..80e913c 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -232,7 +232,7 @@ def _as_rng(rng) -> np.random.Generator: if isinstance(rng, np.random.Generator): return rng if rng is None: - return np.random.default_rng(np.random.randint(0, 2**32)) + return np.random.default_rng(np.random.randint(0, 2**32, dtype=np.uint32)) return np.random.default_rng(rng) @@ -240,7 +240,10 @@ class PermutationResolutionWarning(UserWarning): """The permutation group is too small to resolve the requested p-value.""" -# Target element count for a batch of permutations, bounding peak memory. +# Budgets for the automatic batch size, each bounding a different resource: +# the memory the batch-scaling tensors hold at once, and the work one batch +# does between progress updates. +_BATCH_BYTES = 2**30 _BATCH_ELEMENTS = 2**30 @@ -565,6 +568,29 @@ def _centre(D, self_pair_rows, backend: str): return D, offset, total +def _itemsize(a, backend: str) -> int: + """Bytes per element of ``a``.""" + return a.element_size() if backend == "torch" else a.dtype.itemsize + + +def _auto_batch_size(ref, backend: str, row_elements: int, row_work: int) -> int: + """Permutations per batch when the caller names none. Two caps, smaller wins. + + ``row_elements`` is how many array elements one permutation needs across + the tensors an evaluator holds live at once, so the first cap is a direct + byte budget on them. Sizing off the matrix product alone would miss it: a + large sample against few landmarks -- which is exactly what landmarks are + for -- looks cheap as ``n * m`` while the batch really carries ``rows * n``. + + ``row_work`` is the matrix-product cost of one permutation. Capping that + keeps a single batch from running for minutes without touching the + progress bar or giving the caller a chance to interrupt. + """ + by_memory = _BATCH_BYTES // (row_elements * _itemsize(ref, backend)) + by_work = _BATCH_ELEMENTS // row_work + return max(1, min(by_memory, by_work)) + + def _subgroup_parts(alloc: dict, n: int): """``(positions, how many carry the small label)`` per half of the subgroup. @@ -609,6 +635,9 @@ def _prepare_full(D, alloc: dict, backend: str) -> dict: "scale": n_small * n_large / n, "n_small": n_small, "n_large": n_large, + # per permutation: the drawn labels, u, W = u @ D, and the W * u + # temporary, all (rows, n); the block sums that follow are scalars + "batch_size": _auto_batch_size(D, backend, 4 * n, n * n), "evaluate": _evaluate_full, } @@ -658,7 +687,7 @@ def _prepare_landmarks(D, alloc: dict, backend: str) -> dict: n_small, n_large = alloc["n_small"], alloc["n_large"] n_small_landmarks = alloc["n_small_landmarks"] n_large_landmarks = alloc["n_large_landmarks"] - n = n_small + n_large + n, m = n_small + n_large, n_small_landmarks + n_large_landmarks D, offset, total = _centre(D, alloc["landmarks"], backend) return { "D": D, @@ -676,6 +705,9 @@ def _prepare_landmarks(D, alloc: dict, backend: str) -> dict: "sl_pairs": n_small * n_large_landmarks, "ls_pairs": n_large * n_small_landmarks, "ll_pairs": n_large_landmarks * (n_large - 1), + # per permutation: the drawn labels and u at (rows, n), then v, W and + # the W * v temporary at (rows, m) + "batch_size": _auto_batch_size(D, backend, 2 * n + 3 * m, n * m), "evaluate": _evaluate_landmarks, } @@ -729,9 +761,10 @@ def _prepare_singleton(D, alloc: dict, backend: str) -> dict: n_large_landmarks = alloc["n_large_landmarks"] n = 1 + n_large D, offset, total = _centre(D, alloc["landmarks"], backend) + row_sums = D.sum(1) return { - "ref": D, - "row_sums": D.sum(1), + "ref": row_sums, + "row_sums": row_sums, "parts": _subgroup_parts(alloc, n), "base_small": _base_small(alloc, n), "backend": backend, @@ -741,6 +774,9 @@ def _prepare_singleton(D, alloc: dict, backend: str) -> dict: # the one small point against every large landmark "sl_pairs": n_large_landmarks, "ll_pairs": n_large_landmarks * (n_large - 1), + # per permutation: the drawn labels and u, nothing wider -- and a dot + # product against the row sums rather than a matrix product + "batch_size": _auto_batch_size(row_sums, backend, 2 * n, n), "evaluate": _evaluate_singleton, } @@ -811,6 +847,7 @@ def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict """ n = alloc["n_small"] + alloc["n_large"] full = alloc["regime"] == "full" + m = n if full else len(alloc["landmarks"]) # y is whichever group x is not. n_y = alloc["n_large"] if x_is_small else alloc["n_small"] if full: # every point is a column, so every y point is one @@ -839,6 +876,10 @@ def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict "n_y": float(n_y), "base_small": _base_small(alloc, n), "backend": backend, + # The widest path of the three: the depth ranking holds the labels, + # the depths, the sort order, both ranked indicators and the p-values + # at (rows, n) simultaneously, so it needs the tightest batches. + "batch_size": _auto_batch_size(D, backend, 12 * n + 2 * m, n * m), "evaluate": _evaluate_containment, } @@ -908,7 +949,9 @@ def permutation_energy_test( prog_bar (bool): show a progress bar over permutations. batch_size (Optional[int]): permutations evaluated per matrix product. Larger is faster on GPU, at ``O(batch_size * n)`` extra memory. - None picks a size that bounds the batch at a few million elements. + None picks the largest size that keeps the batch-scaling tensors + within 1 GiB and one batch's matrix product bounded, so a run + cannot allocate unboundedly whatever the sample shape. rng: seed, ``np.random.Generator``, or None to draw from the global numpy state (so ``np.random.seed`` still applies). @@ -921,6 +964,22 @@ def permutation_energy_test( (``reference_size - 1`` entries) and the resulting p-value is exact rather than sampled. """ + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment) + return _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng) + + +def _prepare_test(x, y, n_landmarks: Optional[int], rng, containment: bool): + """Everything that does not depend on the permutations. + + Validates the samples, picks the landmarks, builds the distance matrix and + precomputes the statistic -- which is all of the expensive work, since the + distance matrix is ``O(n * m * d)``. Split out from the permutation loop so + that a caller wanting both a test and a diagnostic built from the *same* + landmarks pays for it once; see :func:`containment_pit_plot`. + + Returns ``(prep, alloc, rng)``. The generator is the coerced one, already + advanced past the landmark draw, so passing it on continues the stream. + """ if is_torch_tensor(x): assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" if is_jax_array(x): @@ -941,7 +1000,6 @@ def permutation_energy_test( rng = _as_rng(rng) alloc = _allocate(n1, n2, m, rng) - _warn_reference_size(alloc, permutations) dmatrix = _cdist(z, _landmark_columns(z, alloc, backend), backend) assert _all_finite(dmatrix, backend), ( @@ -950,17 +1008,22 @@ def permutation_energy_test( ) if containment: - prep = _prepare_containment(dmatrix, alloc, backend, n1 <= n2) - else: - prep = _prepare_statistic(dmatrix, alloc, backend) - evaluate = prep["evaluate"] + return _prepare_containment(dmatrix, alloc, backend, n1 <= n2), alloc, rng + return _prepare_statistic(dmatrix, alloc, backend), alloc, rng + +def _permutation_distribution( + prep: dict, alloc: dict, permutations: int, prog_bar: bool, batch_size, rng +) -> tuple[float, np.ndarray]: + """Evaluate the observed labelling and the permuted ones.""" + _warn_reference_size(alloc, permutations) + evaluate = prep["evaluate"] test_stat = float(evaluate(prep, prep["base_small"][None, :])[0]) assert np.isfinite(test_stat), "Observed statistic is not finite!" - if batch_size is None: - batch_size = max(1, _BATCH_ELEMENTS // (n * m)) - batch_size = max(1, int(batch_size)) + # Each preparer sized its own batches against the tensors its evaluator + # holds live, so an unset batch_size is already bounded. + batch_size = max(1, int(prep["batch_size"] if batch_size is None else batch_size)) # Sampling more permutations than the subgroup has members just redraws the # observed labelling over and over, and every such tie inflates q under the @@ -1337,24 +1400,27 @@ def containment_pit_plot( n_landmarks: as for :func:`pted.pted_containment_test`. rng: seed, ``np.random.Generator``, or None. Only picks the landmarks. """ + prep, _, _ = _prepare_test(x, y, n_landmarks, rng, containment=True) + _draw_containment_pit(prep, saveto, confidence, threshold) + + +def _draw_containment_pit(prep: dict, saveto, confidence: float, threshold: float): + """Draw the containment PIT plot from an already-prepared statistic. + + Takes ``prep`` rather than the samples so that a caller which has just run + the test can reuse it: the distance matrix and the landmark choice are then + literally the same objects the statistic was computed from, rather than an + independent redraw that would describe a different landmark set. + """ try: import matplotlib.pyplot as plt except ImportError: warn("No containment PIT plot generated! Please install matplotlib.") return - backend = _backend(x) - z = _concatenate((x, y), backend) - n1, n2 = len(x), len(y) - n = n1 + n2 - m = n if n_landmarks is None else min(int(n_landmarks), n) - - alloc = _allocate(n1, n2, m, _as_rng(rng)) - D = _cdist(z, _landmark_columns(z, alloc, backend), backend) - prep = _prepare_containment(D, alloc, backend, n1 <= n2) - - observed = _containment_curve(prep, prep["base_small"][None, :])[0] / n1 n_y = int(prep["n_y"]) + n1 = prep["base_small"].size - n_y # x is whichever group y is not + observed = _containment_curve(prep, prep["base_small"][None, :])[0] / n1 # Under H0 the depth p-values are uniform on the (n_y + 1)-point lattice, # so the ordinary lattice band applies -- upper edge only. diff --git a/tests/test_utils.py b/tests/test_utils.py index 6cb6904..e571f0e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -23,8 +23,10 @@ permutation_energy_test, _cdist, _label_drawer, + _BATCH_BYTES, _index_like, _landmark_columns, + _prepare_test, _prepare_statistic, ) @@ -657,3 +659,63 @@ def test_lattice_band_one_sided_lowers_the_ceiling(): assert np.all(one_sided <= two_sided) assert np.any(one_sided < two_sided), "and strictly lower somewhere" assert abs((1 - achieved) - 0.05) < 0.01, f"level was {1 - achieved}" + + +# --------------------------------------------------------------------------- +# Batch sizing and setup reuse +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "n1,n2,m", + [ + (1, 100_000, 10), # huge sample, few landmarks: rows x n is the cost + (2_000, 3_000, 50), + (200, 300, None), # full matrix + (50, 50, 8), + ], +) +@pytest.mark.parametrize("containment", [False, True]) +def test_auto_batch_size_stays_within_the_memory_budget(n1, n2, m, containment): + """No batch-scaling tensor may exceed the byte budget, whatever the shape. + + Every regime holds at least one (rows, n) tensor, so bounding that one is + a necessary condition and does not restate the per-regime element counts. + Sizing off the matrix product alone misses this entirely: n * m is small + exactly when n is large and m is tiny.""" + prep, _, _ = _prepare_test(np.zeros((n1, 2)), np.zeros((n2, 2)), m, 0, containment) + n = n1 + n2 + held = prep["batch_size"] * n * prep["ref"].dtype.itemsize + assert held <= _BATCH_BYTES, f"a batch would hold {held / 2**30:.2f} GiB of (rows, n)" + assert prep["batch_size"] >= 1 + + +def test_containment_plot_reuses_the_test_setup(tmp_path, monkeypatch): + """Asking for the plot must not re-run the allocation or the distance + matrix. Doing so cost a second O(n*m*d) build, and -- because _as_rng + passes a Generator through untouched -- drew landmarks from an already + advanced stream, so the figure described a different landmark set than the + statistic it was meant to explain.""" + import pted.utils as U + from pted import pted_containment_test + + seen = [] + original = U._allocate + monkeypatch.setattr(U, "_allocate", lambda *a, **k: seen.append(original(*a, **k)) or seen[-1]) + + rng = np.random.default_rng(0) + x, y = rng.standard_normal((30, 5)) * 0.5, rng.standard_normal((80, 5)) + out = str(tmp_path / "containment.png") + + for label in ("Generator", "None"): + seen.clear() + pted_containment_test( + x, + y, + permutations=50, + n_landmarks=40, + rng=np.random.default_rng(3) if label == "Generator" else None, + pit_plot=out, + ) + assert len(seen) == 1, f"rng={label}: allocated landmarks {len(seen)} times" + assert os.path.exists(out)