diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1dc49b6..3fa8277 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,12 +15,17 @@ concurrency: env: FORCE_COLOR: 3 PROJECT_NAME: "pted" + # Keep matplotlib off any GUI backend. The runners ship a tkinter that + # matplotlib will happily select and that then fails to initialise Tcl. + # tests/conftest.py sets this too; belt and braces, and it also covers any + # plotting reached outside pytest. + MPLBACKEND: Agg jobs: build: runs-on: ${{matrix.os}} strategy: - fail-fast: true + fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] os: [ubuntu-latest, windows-latest, macOS-latest] @@ -51,11 +56,6 @@ jobs: python -m pip install --upgrade pip pip install pytest pytest-cov wheel - - name: Install torch # no torch on 3.13 to test no-torch scenario - if: ${{ matrix.python-version != '3.13' }} - run: | - pip install torch - # We only want to install this on one run, because otherwise we'll have # duplicate annotations. - name: Install error reporter @@ -68,6 +68,13 @@ jobs: pip install -e .[dev] pip show ${{ env.PROJECT_NAME }} + # The dev extra deliberately omits torch, so skipping this leaves 3.13 + # genuinely torch-free and the no-torch code path gets exercised. + - name: Install torch # no torch on 3.13 to test no-torch scenario + if: ${{ matrix.python-version != '3.13' }} + run: | + pip install -e .[torch] + - name: Test with pytest run: | pytest -vvv --cov=${{ env.PROJECT_NAME }} --cov-report=xml --cov-report=term tests/ diff --git a/README.md b/README.md index fca8df8..201b87d 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,12 @@ print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 Note, you can also provide a filename via a parameter: `sbc_histogram = "sbc_hist.pdf"` and this will generate an SBC histogram from the test[^1]. -You can also generate a Probability Integral Transform (PIT) plot via `pit_plot = "pit.pdf"` for `pted_coverage_test`. The PIT plot shows the empirical CDF of the p-values against the expected uniform CDF (1:1 diagonal), along with a shaded KS confidence band determined by `pit_confidence` (95% by default). Any portion of the curve that deviates outside the band is evidence that the p-values are not uniformly distributed, at a significance level of `1 - pit_confidence` (5% by default), indicating a potentially miscalibrated posterior. +You can also generate a Probability Integral Transform (PIT) plot via `pit_plot = "pit.pdf"` for `pted_coverage_test`. The PIT plot shows the empirical CDF of the p-values against the expected uniform CDF (1:1 diagonal), along with a shaded band determined by `pit_confidence` (95% by default). The band is simultaneous: under the null the whole curve stays inside it with probability `pit_confidence`, so any portion of the curve that pokes out is evidence of non-uniform p-values at significance `1 - pit_confidence`, indicating a potentially miscalibrated posterior. + +The band is built from the exact null distribution of the p-values. The ECDF can only move where the p-values can actually land, so rather than constraining order statistics the band constrains the counts there. +Working with counts rather than order statistics is what makes this usable here. Permutation p-values are discrete — after enumeration they are exactly uniform on a grid of `L = len(permute) + 1` points — and ties are precisely what a binomial count expects. +`pted_coverage_test` works out the lattice size and passes it in for you. +Ultimately, this means that the fact we only use a finite number of simulations to compute the coverage test does not affect the validity of the bands in the PIT plot; any point that sticks out of the grey shaded area indicates possible miscalibration at the given significance level (default 95%). ## How it works @@ -97,7 +102,16 @@ You can also generate a Probability Integral Transform (PIT) plot via `pit_plot PTED uses the energy distance of the two samples `x` and `y`, this is computed as: -$$d = \frac{2}{n_xn_y}\sum_{i,j}||x_i - y_j|| - \frac{1}{n_x^2}\sum_{i,j}||x_i - x_j|| - \frac{1}{n_y^2}\sum_{i,j}||y_i - y_j||$$ +$$d = \frac{2}{n_xn_y}\sum_{i,j}||x_i - y_j|| - \frac{1}{n_x(n_x-1)}\sum_{i\neq j}||x_i - x_j|| - \frac{1}{n_y(n_y-1)}\sum_{i\neq j}||y_i - y_j||$$ + +The within-group sums run over distinct pairs, skipping the `i = j` terms which +are zero by construction. That makes `d` an unbiased estimator of the population +energy distance, so under the null it scatters around zero and can come out +slightly negative (and strongly negative if `x` and `y` are *more* alike than +chance allows, e.g. if they share samples). Only its rank among the permuted +values matters. For equal sample sizes this ranks the permutations identically +to the `1/n^2` normalisation of Székely & Rizzo, so the p-value is unchanged; for +unequal sizes the two differ slightly. The energy distance measures distances between pairs of points[^2]. It becomes more positive if the `x` and `y` samples tend to be further from each other than from @@ -161,69 +175,78 @@ doubling procedure (``2 * min(p_right, p_left)``) to get the p-value meaning that if your posterior is underconfident or overconfident, you will get a small p-value that can be used to reject the null. -#### If you have posterior densities +## Example: Containment Test -If you have posterior densities (and you trust them), then chances are the HDP -region coverage test is a more powerful test. Essentially, instead of using a -permutation test to determine the p-value for a single simulation, you just -determine the fraction of posterior samples with higher posterior density than -the ground truth[^4]. The package has a quick tool to let you do that: +`pted` asks "are these the same distribution?". `pted_containment_test` asks the +weaker, directional question "**does y cover x?**" — a sample drawn from a +tighter distribution than `y` passes, while one that spreads beyond it, sits off +to one side, or throws a few points clear of it fails. ```python -from pted import hdp_coverage_test +from pted import pted_containment_test +import numpy as np -ground_truth = np.random.uniform(100) # Nsim -posterior_samples = np.random.uniform((200, 100)) # Nsamp, Nsim +y = np.random.normal(size = (500, 10)) +inside = np.random.normal(size = (100, 10)) * 0.5 +outside = np.random.normal(size = (100, 10)) * 1.6 -p_value = hdp_coverage_test(ground_truth, posterior_samples, two_tailed = True) -print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 +print(pted_containment_test(inside, y)) # large: contained +print(pted_containment_test(outside, y)) # small: not contained ``` -## Example: Sensitivity comparison with KS-test - -There is no single universally optimal two sample test, but a widely used method -in 1D is called the Kolmogorov-Smirnov (KS)-test. The KS-test operates -fundamentally differently from PTED and can only really work in 1D. Here I do a -super basic comparison of the two methods. Draw two samples of 100 Gaussian -distributed points, thus the null hypothesis is true for these points. Then -slowly bias one of the samples by changing the standard deviation up to 2 sigma. -By tracking how the p-value drops we can see which method is more sensitive to -this kind of mismatched sample. If you run this test a hundred times you will -find that PTED is more sensitive to this kind of bias than the KS-test. Observe -that both methods start around p=0.5 in the true null case (scale = 1), since -they are both exact tests that truly sample U(0,1) under the null. +Unlike every other test in the package this one is **not symmetric** — swapping +the arguments asks the other question and will usually give a different answer. +That asymmetry is the point, and it is why the energy distance alone cannot +answer it: the energy distance is just as large when `x` is *tighter* than `y` +as when it is broader, so a one-tailed energy test rejects 99.8% of the time on +a perfectly contained sample. + +Instead, every `x` point gets a **depth** — its mean distance to the points +labelled `y` — so peripheral points score high. This is nearly identical to +`pted_coverage_test`, and we use the same `-2 sum log p` formula to combine the +p-values for each `x`. However, instead of interpreting the result as a $\chi^2$ +distribution, we use a secondary permutation test to calibrate it. + +The test is exact where `x` and `y` share a distribution and conservative inside +the null, which is the goal for a containment test. It measures depth rather +than literal support: a tight cluster of `x` sitting in a low-density pocket +well inside `y` counts as contained. So use a heavy dose of caution when +interpreting the results. + +### Read the plot as well as the p-value + +`-2 sum log p` is a sum whose per-point floor is zero against a null mean of +two, so a bulk of `x` sitting deep inside `y` banks slack that can hide a +handful of points `y` cannot reach at all. In testing, 5% of `x` placed five +sigma outside the prior predictive went undetected, while every other failure +mode — data broader than `y`, data offset into its tail — was caught cleanly. So +pass `pit_plot`: ```python -from pted import pted -import numpy as np -from scipy.stats import kstest -import matplotlib.pyplot as plt - -np.random.seed(0) - -scale = np.linspace(1.0, 2.0, 10) -pted_p = np.zeros((10, 100)) -ks_p = np.zeros((10, 100)) -for i, s in enumerate(scale): - for trial in range(100): - x = np.random.normal(size=(100, 1)) - y = np.random.normal(scale=s, size=(100, 1)) - pted_p[i][trial] = pted(x, y, two_tailed=False) - ks_p[i][trial] = kstest(x[:, 0], y[:, 0]).pvalue - -plt.plot(scale, np.mean(pted_p, axis=1), linewidth=3, c="b", label="PTED") -plt.plot(scale, np.mean(ks_p, axis=1), linewidth=3, c="r", label="KS") -plt.legend() -plt.ylim(0, None) -plt.xlim(1, 2.0) -plt.xlabel("Out of distribution scale [*sigma]") -plt.ylabel("p-value") - -plt.savefig("pted_demo.png", bbox_inches="tight") -plt.show() +p = pted_containment_test(data, prior_predictive, pit_plot = "containment.pdf") ``` -![pted demo KS comparison](media/pted_ks.png) +The plot shows the empirical CDF of the per-point depth p-values — one step per +point of `x` — on a logarithmic p axis, because the whole diagnostic lives at +the left edge. It carries two reference marks and no diagonal. + +The **upper bound** is the one-sided simultaneous ceiling for `n1` p-values that +really are uniform: the same as the ordinary PIT plot, with the lower edge +dropped. Only the upper edge means anything here. Any part of the CDF below this +line indicates the `x` values are likely contained at the threshold level (95% +by default). + +The **vertical line** marks threshold p-value (0.05 by default). Parts of the +CDF to the right of this line are embedded in `y` at the threshold level and so +likely are contained. Points to its left are peripheral relative to `y` and so +this may suggest the values are not contained. However, when many values of `x` +are being tested some leakage to low p-values are expected, which is why the +uniform-upper-bound line is also plotted. + +Read them together rather than as a decision rule. A curve with points to the +left of the vertical line and above the uniform-upper-bound is a fair indication +that `x` is not contained in `y`. Though results either way are not definitive, +as is the nature of null hypothesis testing, and even moreso here. ## Interpreting the results @@ -295,9 +318,11 @@ def pted( y: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: ``` @@ -305,9 +330,11 @@ def pted( * **y** *(Union[np.ndarray, Tensor, jax.Array])*: second set of samples. Shape (M, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If `chunk_size >= len(x)` and `chunk_size >= len(y)`, PTED falls back to the full (non-chunked) computation. If None, use the full dataset. +* **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. * **two_tailed** *(bool)*: if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. If False, only checks for dissimilarity but is more sensitive. Default is True. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of permutation tests. Default is False. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks the largest size that keeps the batch-scaling tensors within 1 GiB and bounds one batch's matrix product, so a run cannot allocate unboundedly whatever the sample shape. +* **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ### Coverage test @@ -318,12 +345,14 @@ def pted_coverage_test( permutations: int = 1000, warn_confidence: Optional[float] = 1e-3, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, sbc_histogram: Optional[str] = None, sbc_bins: Optional[int] = None, pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: ``` @@ -331,12 +360,16 @@ def pted_coverage_test( * **s** *(Union[np.ndarray, Tensor, jax.Array])*: Posterior samples. Shape (n_samples, n_sims, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If None, use the full dataset. +* **n_landmarks** *(Optional[int])*: if not None, estimate the energy distance from a rectangular distance matrix instead of the full pairwise matrix. `n_landmarks` points are drawn from the pooled sample as "landmarks", and only the distance from every sample to each landmark is computed, so the cost drops from `O(n^2 d)` to `O(n m d)` for `m = n_landmarks`. A value covering the whole pooled sample, or None, runs the exact full-matrix computation. + + Because the ground truth is a single point, the per-simulation test runs in the `singleton` regime, where the permutation subgroup reaches only `n - m` distinct label assignments. Keep `n_landmarks` well below the number of posterior samples. * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. -* **pit_confidence** *(float)*: Confidence level for the KS confidence band in the PIT plot. Default is 0.95 (95%). Only used when `pit_plot` is not None. +* **pit_confidence** *(float)*: Confidence level for the PIT plot's simultaneous band. Default is 0.95 (95%). Only used when `pit_plot` is not None. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of simulations. Default is False. +* **batch_size** *(Optional[int])*: number of permutations evaluated per matrix product. Larger values are faster (especially on GPU) at `O(batch_size * n)` extra memory. None picks the largest size that keeps the batch-scaling tensors within 1 GiB and bounds one batch's matrix product, so a run cannot allocate unboundedly whatever the sample shape. +* **rng**: seed, `np.random.Generator`, or None to draw from the global numpy state, so `np.random.seed` still controls reproducibility. ## GPU Compatibility @@ -373,38 +406,110 @@ print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 If a GPU isn't enough to get PTED running fast enough for you, or if you are running into memory limitations, there are still options! We can use an -approximation of the energy distance, in this case the test is still exact but -less sensitive than it would be otherwise. Instead of building the full -`(n_samp_x + n_samp_y) x (n_samp_x + n_samp_y)` pairwise distance matrix, PTED -can build a smaller rectangular `(n_samp_x + n_samp_y) x (nxc + nyc)` matrix of -distances from every sample to a fixed set of "landmark" samples, where `nxc = -min(chunk_size, n_samp_x)` landmarks are drawn from x and `nyc = min(chunk_size, -n_samp_y)` from y. Just set the `chunk_size` parameter to control how many -landmarks are used. This rectangular matrix is computed only once; the -permutation distribution is then obtained purely by re-indexing (permuting) its -rows, which reassigns samples to the x/y groups without recomputing any -distances. The larger the chunk size, the closer the estimate will be to the -true energy distance, but it will take more compute. - -Note that the computational complexity for standard PTED goes as `O((n_samp_x + -n_samp_y)^2)` to build the full distance matrix, while the chunked version goes -as `O((n_samp_x + n_samp_y) * (nxc + nyc))`, so plan your chunking accordingly. -For a given chunk size, the computational complexity of PTED grows linearly -with dataset size, much like other large scale (machine learning oriented) two -sample tests. +approximation of the energy distance; the test stays exact, it just becomes less +sensitive than it would otherwise be. Instead of building the full `n x n` +pairwise distance matrix (`n = n_samp_x + n_samp_y`), PTED builds a smaller +rectangular `n x m` matrix of distances from every sample to `m` "landmark" +points drawn from the pooled sample — the Nyström-style subsampling familiar +from kernel methods. Set `n_landmarks = m` to choose how many. + +Building the full matrix costs `O(n^2 d)` and each permutation `O(n^2)`; the +rectangular matrix costs `O(n m d)` and each permutation `O(n m)`. So for a +fixed number of landmarks PTED grows linearly with dataset size, much like other +large scale (machine learning oriented) two sample tests. Permutations are +evaluated in batches as a single matrix product rather than one at a time, and +everything heavy stays on whichever backend your arrays live on. Example: ```python from pted import pted import numpy as np -x = np.random.normal(size = (500, 10)) # (n_samples_x, n_dimensions) -y = np.random.normal(size = (400, 10)) # (n_samples_y, n_dimensions) +x = np.random.normal(size = (5000, 10)) # (n_samples_x, n_dimensions) +y = np.random.normal(size = (4000, 10)) # (n_samples_y, n_dimensions) -p_value = pted(x, y, chunk_size = 50) +p_value = pted(x, y, n_landmarks = 200) print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 ``` +### Enumerating small permutation groups + +If the permutation subgroup turns out to have no more than `permutations` +members, PTED evaluates *all* of it rather than sampling. Sampling a small +group keeps redrawing the observed labelling, and each of those ties inflates +the p-value under the `>=` convention: with a group of 151 and 199 draws, +`P(p <= 0.02)` came out at 0.0169 instead of 0.0200. Walking the group instead +puts it at 0.0195, leaving only the unavoidable granularity of a 151-point +lattice — and it costs *less* compute, since 151 evaluations beat 199. This +mostly matters for `pted_coverage_test`, where the ground truth is a single +point and the group is only `n - m` large. + +One consequence: when this kicks in, `return_all` gives back +`reference_size - 1` permuted statistics rather than `permutations` of them. +The p-value is then exact rather than sampled. + +### Why using landmarks is still exact + +The subtlety is that the landmarks cannot simply be re-split between the two +groups after each shuffle. The landmark set `L` is chosen using the group labels +(so that both groups are represented), which means the labels are no longer +uniformly distributed once you condition on `L`. PTED handles this by confining +permutations to the subgroup that fixes `L`: labels are shuffled within the +landmark positions and within their complement, never between. `L` then stays put +under every permutation and the observed labelling is exchangeable with the permuted +ones. That is what makes the p-value exact. (`L` is chosen from sample positions +and group labels only, never from the data values — picking landmarks by maximin, +k-means or leverage would break the argument. "Landmark" here means only "a point +everything is measured against", never "a point chosen for its importance".) + +What you give up is sensitivity: the null spread grows like `sqrt(n / m)`, so the +smallest detectable energy distance scales as `(n m)^-0.5` rather than `n^-1`. +The detection threshold degrades as one over the square root of the compute. + +You also give up some p-value resolution, and this is most significant when one +group holds a single point — which is exactly the per-simulation test inside +`pted_coverage_test`. The lone point is kept out of `L` — it has no within-group +pairs, so landmarking it would buy nothing — and its label then roams over the +`n - m` positions outside, which is the whole reachable set. The smallest +attainable p-value is about `1 / (n - m)` however many permutations you draw, so +PTED raises a `PermutationResolutionWarning` when that set is too small to +resolve the p-value you asked for. + +This is the sharpest case of a general rule: **landmarks are for `m << n`.** At +`m = n / 2` the rectangular matrix saves only a factor of two over the exact +test, which does not pay for the sensitivity and resolution it costs. Reach for +them when the full matrix will not fit or will not finish, not to shave a +constant factor. + +### Does restricting the permutations still measure the energy distance? + +Yes — the restriction is on how the statistic is *calibrated*, not on what it +measures. Every block mean still averages genuine distances between genuine +members of the two groups: the cross term pairs **all** `n_x` samples against the +large-group landmarks and **all** `n_y` samples against the small-group +landmarks, so no sample is demoted to a mere marker. Each block mean is an unbiased estimate of +exactly the population quantity the full test estimates, which makes the whole +statistic an unbiased estimator of the population energy distance — an +*incomplete U-statistic* in the sense of Janson (1984), averaging over a subset +of the pairs rather than computing a different function of them. + +Freezing the per-group landmark counts is a restricted-randomisation device, the +same idea as conditioning on the margins in Fisher's exact test: it removes a +nuisance source of variability from the null rather than changing the estimand. +Empirically it is power-neutral. Compare it against the obvious alternative — +draw `L` uniformly by position, so that `L` is independent of the labels and the +full permutation group is legal — and both reject at the same rate (`n = 200`, +`m = 40`, 500 trials: 0.16 vs 0.16 at a 0.25σ shift, 0.31 vs 0.35 at 0.40σ). The +power lost relative to the full test comes from subsampling, not from the +subgroup. + +What the subgroup buys is the freedom to choose `L` *by design*. Drawing `L` +blind to the labels leaves the smaller group with no landmarks at all — and its +within-group term unestimable — distressingly often once the samples are +unbalanced: with `n_x = 3`, `n_y = 200` and `m = 40`, 52% of permutations have no +x-landmarks. Choosing `L` so that it covers the small group fixes that, and the +subgroup is what keeps the test exact when you do. + ## Citation If you use PTED in your work, please include a citation to the [zenodo diff --git a/pyproject.toml b/pyproject.toml index c96f676..fe9959a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,11 +36,14 @@ Repository = "https://github.com/ConnorStoneAstro/pted" Issues = "https://github.com/ConnorStoneAstro/pted/issues" [project.optional-dependencies] +# Deliberately no torch: CI installs it per matrix entry via the `torch` +# extra below, leaving one entry without it so the no-torch code path is +# actually exercised. Pulling it in here would defeat that. For a local +# environment that runs the whole suite: pip install -e '.[dev,torch]' dev = [ "pytest>=8.0,<9", "pytest-cov>=4.1,<5", "pytest-mock>=3.12,<4", - "torch>=2.0,<3", "jax>=0.7,<1", "matplotlib", ] diff --git a/src/pted/__init__.py b/src/pted/__init__.py index 4c7f7d9..5c6d6e6 100644 --- a/src/pted/__init__.py +++ b/src/pted/__init__.py @@ -1,4 +1,4 @@ -from .pted import pted, pted_coverage_test +from .pted import pted, pted_coverage_test, pted_containment_test from .tests import test from .utils import hdp_coverage_test from ._version import version as __version__ # noqa @@ -9,6 +9,7 @@ __all__ = [ "pted", "pted_coverage_test", + "pted_containment_test", "hdp_coverage_test", "test", "__version__", diff --git a/src/pted/pted.py b/src/pted/pted.py index 06f33a5..4e92ba4 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -3,25 +3,30 @@ import numpy as np from .utils import ( - pted as _pted_impl, - pted_chunk as _pted_chunk_impl, + permutation_energy_test as _energy_test, + _prepare_test, + _permutation_distribution, + _draw_containment_pit, + _as_rng, two_tailed_p, confidence_alert, simulation_based_calibration_histogram, pit_plot as _pit_plot, ) -__all__ = ["pted", "pted_coverage_test"] +__all__ = ["pted", "pted_coverage_test", "pted_containment_test"] def pted( - x: Union[np.ndarray, "Tensor", "jax.Array"], - y: Union[np.ndarray, "Tensor", "jax.Array"], + x, + y, permutations: int = 1000, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, two_tailed: bool = True, prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[float, np.ndarray, float]]: """ Two sample null hypothesis test using a permutation test on the energy @@ -53,6 +58,13 @@ def pted( p = sum(permute_stats > test_stat) return (1 + p) / (1 + permutations) + The energy distance is computed with the within-group means taken over + distinct pairs, excluding the zero self-pairs. This makes the statistic an + unbiased estimator of the population energy distance, so it may come out + slightly negative when x and y are drawn from the same distribution (and + strongly negative when they are more alike than chance allows, e.g. shared + samples). Only its rank among the permuted statistics matters. + Example ------- import numpy as np @@ -74,94 +86,247 @@ def pted( accurately the p-value is computed. return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. + The permuted statistics are ``permutations`` random draws, or the + whole permutation subgroup when that is smaller (see the note + below), so check its length rather than assuming. bool (default: False) - chunk_size (Optional[int]): if not None, use a landmark-based - rectangular distance matrix to estimate the energy distance - instead of the full pairwise matrix. ``nxc = min(chunk_size, - len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` - samples from y are used as fixed "landmarks"; only distances from - every sample to these landmarks are computed. If ``chunk_size`` - covers both full datasets, this falls back to the exact - (non-chunked) computation. If None, use the full dataset. + n_landmarks (Optional[int]): if not None, estimate the energy distance + from a rectangular distance matrix instead of the full pairwise + matrix. ``n_landmarks`` points are drawn from the pooled sample as + "landmarks", and only the distance from every sample to each + landmark is computed, so the cost drops from ``O(n^2 d)`` to + ``O(n m d)`` for ``m = n_landmarks``. Worth it only for + ``m << n``. A value covering the whole pooled sample, or None, + runs the exact full-matrix computation. two_tailed (bool): if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. Default is True. prog_bar (bool): if True, show a progress bar to track the progress of permutation tests. Default is False. - + batch_size (Optional[int]): number of permutations evaluated per matrix + product. Larger values are faster (especially on GPU) at + ``O(batch_size * n)`` extra memory. None picks a size that keeps a + batch to a few million elements. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state, so ``np.random.seed`` still controls reproducibility. Note ---- - PTED has O(n^2 * D) time complexity to build the distance matrix, - where n is the number of samples in x and y and D is the number of - dimensions; each of the P permutations then costs O(n^2) to - re-evaluate the test statistic from the re-indexed matrix. For large - datasets this can get unwieldy, so chunking is recommended. When - chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances - from every sample to a fixed set of landmark samples is computed, so - this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. - The permutation distribution is then obtained by re-indexing (permuting) - the rows of this matrix, which costs O(n * nc) per permutation and - never recomputes distances. PTED remains an exact p-value test even - when chunking, it simply becomes less sensitive to the difference - between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and - ``len(y)``. + The full test builds the ``(n, n)`` distance matrix in ``O(n^2 d)`` + time, where n is the pooled sample size and d the dimension. Each + permutation then costs ``O(n^2)``, though permutations are evaluated in + batches as a single matrix product rather than one at a time. For large + datasets this gets unwieldy, so landmarks are recommended: only a + rectangular ``(n, m)`` matrix of distances to ``m`` landmark points is + built, in ``O(n m d)`` time, and each permutation costs ``O(n m)``. + + Landmarks keep the p-value exact. Permutations are drawn from the + subgroup that holds the landmark set fixed, shuffling labels within + the landmarks and within their complement but never between, so the + observed labelling is exchangeable with the permuted ones. The test + simply becomes less sensitive as ``m`` shrinks: the null spread grows + like ``sqrt(n / m)``, so the detectable energy distance scales as + ``(n m)^-0.5`` rather than ``n^-1``. + + The one thing landmarks do cost is p-value resolution, and it bites + hardest when one group holds a single point (the per-simulation test + inside ``pted_coverage_test``). That lone point is kept out of the + landmark set, so the subgroup reaches only ``n - m`` distinct label + assignments and the smallest attainable p-value is about ``1 / (n - m)`` + however many permutations are drawn. Landmarks are for ``m << n``: at + ``m = n / 2`` the rectangular matrix saves a factor of two over the + exact test, which does not pay for the lost sensitivity. A ``PermutationResolutionWarning`` is raised + when the reachable set is too small to resolve the p-value requested. + When the permutation subgroup has no more than ``permutations`` + members, PTED walks the whole group instead of sampling it. That is + both cheaper and exact: sampling a small group keeps redrawing the + observed labelling, and every such tie inflates the p-value under the + ``>=`` convention. In that case the returned array of permuted + statistics holds ``reference_size - 1`` entries rather than + ``permutations``. """ assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" assert len(y.shape) >= 2, f"y must be at least 2D, not {y.shape}" - if chunk_size is not None: - assert chunk_size > 0, "chunk_size must be > 0" - # If chunk_size covers both full datasets, chunking adds no benefit - if chunk_size >= len(x) and chunk_size >= len(y): - chunk_size = None assert ( x.shape[1:] == y.shape[1:] ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" - if len(x.shape) > 2: - x = x.reshape(x.shape[0], -1) - if len(y.shape) > 2: - y = y.reshape(y.shape[0], -1) - - if chunk_size is not None: - test, permute = _pted_chunk_impl( - x, - y, - permutations=permutations, - chunk_size=int(chunk_size), - prog_bar=prog_bar, - ) - else: - test, permute = _pted_impl(x, y, permutations=permutations, prog_bar=prog_bar) - - permute = np.array(permute) - - # Compute p-value + x = x.reshape(x.shape[0], -1) + y = y.reshape(y.shape[0], -1) + + # A landmark count covering the whole pooled sample lands in the exact + # full-matrix regime, so there is no separate branch for it here. + test, permute = _energy_test( + x, + y, + permutations=permutations, + n_landmarks=n_landmarks, + prog_bar=prog_bar, + batch_size=batch_size, + rng=rng, + ) + + # Compute p-value. The null set is `permutations` random draws, or the + # whole permutation subgroup when that is smaller, so count what came back + # rather than what was asked for. + n_null = len(permute) if two_tailed: q = 2 * min(np.sum(permute >= test), np.sum(permute <= test)) - q = min(q, permutations) + q = min(q, n_null) else: q = np.sum(permute >= test) - pval = (1.0 + q) / (1.0 + permutations) + pval = (1.0 + q) / (1.0 + n_null) if return_all: return test, permute, pval return pval +def pted_containment_test( + x, + y, + permutations: int = 1000, + return_all: bool = False, + n_landmarks: Optional[int] = None, + prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, + pit_plot: Optional[str] = None, + pit_confidence: float = 0.95, + pit_threshold: float = 0.05, +) -> Union[float, tuple[float, np.ndarray, float]]: + """ + One-sided test of whether ``y`` covers ``x``. + + Where :func:`pted` asks "are these the same distribution?", this asks the + weaker, directional question "does x sit inside y?". A sample drawn from a + tighter distribution than y passes; one that spreads beyond y, sits off to + one side of it, or throws a few points clear of it, fails. The null is the + composite hypothesis that x is no more peripheral than y, so a small + p-value is evidence that x is *not* contained. + + Note this is **not** symmetric: ``pted_containment_test(x, y)`` and + ``pted_containment_test(y, x)`` ask different questions and will usually + give different answers. That is the point of it. + + How it works + ------------ + Pool the samples and give every point a *depth*: its mean distance to + the points labelled y. Peripheral points have large depth. Each point + of x then carries the p-value of the single-point energy test against + y -- exactly the per-simulation test inside + :func:`pted_coverage_test` -- namely the fraction of y points at least + as peripheral as it is. Those are aggregated with the same + ``-2 sum log p`` that the coverage test uses, and the aggregate is + calibrated by permutation rather than against a chi-squared, because + the per-point p-values share y and so are correlated. + + The energy distance itself cannot answer this question. It is symmetric + in x and y, so it is just as large when x is *tighter* than y as when x + is broader; a one-tailed energy test rejects 99.8% of the time on a + sample that is perfectly contained. Averaging the depths does not help + either -- the average collapses to the cross term, which is invariant + under swapping x and y. The comparison against y's own depth + distribution is what carries the direction. + + Example + ------- + import numpy as np + from pted import pted_containment_test + + y = np.random.normal(size=(500, 10)) + inside = np.random.normal(size=(100, 10)) * 0.5 + outside = np.random.normal(size=(100, 10)) * 1.6 + + print(pted_containment_test(inside, y)) # large: contained + print(pted_containment_test(outside, y)) # small: not contained + + Parameters + ---------- + x (Union[np.ndarray, Tensor, jax.Array]): the sample being tested for + containment. Shape (N, *D) + y (Union[np.ndarray, Tensor, jax.Array]): the sample that should + contain it. Shape (M, *D) + permutations (int): number of permutations to run. + return_all (bool): if True, return the test statistic and the permuted + statistics with the p-value. If False, just return the p-value. + n_landmarks (Optional[int]): estimate depths from a rectangular + distance matrix instead of the full pairwise one; see + :func:`pted`. Both samples must keep at least one landmark. + prog_bar (bool): if True, show a progress bar over permutations. + batch_size (Optional[int]): permutations evaluated per matrix product. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state. + pit_plot (Optional[str]): if given, the path to save the companion + diagnostic plot. Strongly recommended -- see the note below. + pit_confidence (float): simultaneous level for that plot's upper + bound. Default 0.95. + pit_threshold (float): where the plot draws its vertical marker; + x points left of it are more peripheral than all but that fraction + of y. Default 0.05. + + Note + ---- + **Read the plot as well as the p-value.** The Fisher aggregate + ``-2 sum log p`` is a sum whose per-point floor is zero against a null + mean of two, so a bulk of x that sits deep inside y banks slack that + can hide a handful of points y cannot reach at all: 5% of x placed + five sigma out went undetected in testing while every other failure + mode was caught. Those escapees are obvious in + :func:`pted.utils.containment_pit_plot`, where the curve lifts off the + floor at the smallest p-value and runs above the band for the first + decade, so pass ``pit_plot`` and look at it. + + The test is exact at the boundary of the null, where x and y share a + distribution, and conservative inside it -- a genuinely contained x + rejects far less often than the nominal rate, which is the correct + behaviour for a one-sided composite null. + + It measures depth, not literal support. A tight cluster of x sitting in + a low-density pocket well inside y's bulk counts as contained, because + every one of its points is unremarkable in depth. If you need support + containment rather than depth containment, this is the wrong tool. + """ + assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" + assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" + assert len(y.shape) >= 2, f"y must be at least 2D, not {y.shape}" + assert ( + x.shape[1:] == y.shape[1:] + ), f"x and y samples must have the same shape (past first dim), not {x.shape} and {y.shape}" + x = x.reshape(x.shape[0], -1) + y = y.reshape(y.shape[0], -1) + + # Prepared once and used twice: the plot describes the same landmarks and + # the same distance matrix the statistic was computed from. + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=True) + test, permute = _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng) + + if pit_plot is not None: + _draw_containment_pit(prep, pit_plot, pit_confidence, pit_threshold) + + # One-sided by construction: only an unusually large aggregate is evidence + # that x reaches outside y. + pval = (1.0 + np.sum(permute >= test)) / (1.0 + len(permute)) + if return_all: + return test, permute, pval + return pval + + def pted_coverage_test( - g: Union[np.ndarray, "Tensor", "jax.Array"], - s: Union[np.ndarray, "Tensor", "jax.Array"], + g, + s, permutations: int = 1000, warn_confidence: Optional[float] = 1e-3, return_all: bool = False, - chunk_size: Optional[int] = None, + n_landmarks: Optional[int] = None, sbc_histogram: Optional[str] = None, sbc_bins: Optional[int] = None, pit_plot: Optional[str] = None, pit_confidence: float = 0.95, prog_bar: bool = False, + batch_size: Optional[int] = None, + rng=None, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: """ Coverage test using a permutation test on the energy distance (Euclidean). @@ -216,14 +381,14 @@ def pted_coverage_test( return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): If not None, use a landmark-based - rectangular distance matrix to estimate the energy distance - instead of the full pairwise matrix. ``nxc = min(chunk_size, - len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` - samples from y are used as fixed "landmarks"; only distances from - every sample to these landmarks are computed. If ``chunk_size`` - covers both full datasets, this falls back to the exact - (non-chunked) computation. If None, use the full dataset. + n_landmarks (Optional[int]): if not None, estimate the energy distance + from a rectangular distance matrix instead of the full pairwise + matrix. ``n_landmarks`` points are drawn from the pooled sample as + "landmarks", and only the distance from every sample to each + landmark is computed, so the cost drops from ``O(n^2 d)`` to + ``O(n m d)`` for ``m = n_landmarks``. Worth it only for + ``m << n``. A value covering the whole pooled sample, or None, + runs the exact full-matrix computation. sbc_histogram (Optional[str]): If given, the path/filename to save a Simulation-Based-Calibration histogram. sbc_bins (Optional[int]): If given, force the histogram to have the provided @@ -232,40 +397,69 @@ def pted_coverage_test( Probability Integral Transform (PIT) plot. The plot shows the empirical CDF of the per-simulation p-values against the expected uniform CDF (1:1 diagonal), together with a shaded KS confidence - band. Deviations outside the band indicate that the p-values are - not uniformly distributed. Default is None (no plot saved). - pit_confidence (float): Confidence level for the KS confidence band - in the PIT plot. Default is 0.95 (95%). Only used when - ``pit_plot`` is not None. + band built from the exact null of the p-values. Deviations + outside the band indicate that the p-values are not uniformly + distributed. Default is None (no plot saved). + pit_confidence (float): Confidence level for the PIT plot's band. + Default is 0.95 (95%). Only used when ``pit_plot`` is not None. prog_bar (bool): If True, show a progress bar to track the progress of simulations. Default is False. + batch_size (Optional[int]): number of permutations evaluated per matrix + product. Larger values are faster (especially on GPU) at + ``O(batch_size * n)`` extra memory. None picks a size that keeps a + batch to a few million elements. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state, so ``np.random.seed`` still controls reproducibility. Note ---- - PTED has O(n^2 * D) time complexity to build the distance matrix, - where n is the number of samples in x and y and D is the number of - dimensions; each of the P permutations then costs O(n^2) to - re-evaluate the test statistic from the re-indexed matrix. For large - datasets this can get unwieldy, so chunking is recommended. When - chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances - from every sample to a fixed set of landmark samples is computed, so - this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. - The permutation distribution is then obtained by re-indexing (permuting) - the rows of this matrix, which costs O(n * nc) per permutation and - never recomputes distances. PTED remains an exact p-value test even - when chunking, it simply becomes less sensitive to the difference - between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and - ``len(y)``. + The full test builds the ``(n, n)`` distance matrix in ``O(n^2 d)`` + time, where n is the pooled sample size and d the dimension. Each + permutation then costs ``O(n^2)``, though permutations are evaluated in + batches as a single matrix product rather than one at a time. For large + datasets this gets unwieldy, so landmarks are recommended: only a + rectangular ``(n, m)`` matrix of distances to ``m`` landmark points is + built, in ``O(n m d)`` time, and each permutation costs ``O(n m)``. + + Landmarks keep the p-value exact. Permutations are drawn from the + subgroup that holds the landmark set fixed, shuffling labels within + the landmarks and within their complement but never between, so every + normalising constant in the statistic is a constant and the observed + labelling is exchangeable with the permuted ones. The test simply + becomes less sensitive as ``m`` shrinks: the null spread grows like + ``sqrt(n / m)``, so the detectable energy distance scales as + ``(n m)^-0.5`` rather than ``n^-1``. + + The one thing landmarks do cost is p-value resolution, and it bites + hardest when one group holds a single point (the per-simulation test + inside ``pted_coverage_test``). That lone point is kept out of the + landmark set, so the subgroup reaches only ``n - m`` distinct label + assignments and the smallest attainable p-value is about ``1 / (n - m)`` + however many permutations are drawn. Landmarks are for ``m << n``: at + ``m = n / 2`` the rectangular matrix saves a factor of two over the + exact test, which does not pay for the lost sensitivity. A ``PermutationResolutionWarning`` is raised + when the reachable set is too small to resolve the p-value requested. + When the permutation subgroup has no more than ``permutations`` + members, PTED walks the whole group instead of sampling it. That is + both cheaper and exact: sampling a small group keeps redrawing the + observed labelling, and every such tie inflates the p-value under the + ``>=`` convention. In that case the returned array of permuted + statistics holds ``reference_size - 1`` entries rather than + ``permutations``. """ nsamp, nsim, *_ = s.shape assert nsim > 0, "need some simulations to run test, got 0 simulations" assert ( g.shape == s.shape[1:] ), f"g and s must have the same shape (past first dim of s), not {g.shape} and {s.shape}" - if len(s.shape) > 3: - s = s.reshape(nsamp, nsim, -1) + s = s.reshape(nsamp, nsim, -1) g = g.reshape(1, nsim, -1) + # Coerce once, so the generator's state advances across simulations. Passing + # a bare seed straight through would hand every simulation the same + # permutation draws. + rng = _as_rng(rng) + test_stats = [] permute_stats = [] pvals = [] @@ -276,7 +470,9 @@ def pted_coverage_test( permutations=permutations, return_all=True, two_tailed=False, - chunk_size=chunk_size, + n_landmarks=n_landmarks, + batch_size=batch_size, + rng=rng, ) test_stats.append(test) permute_stats.append(permute) @@ -287,12 +483,19 @@ def pted_coverage_test( # Simulation-Based-Calibration histogram if sbc_histogram is not None: - ranks = np.sum(test_stats[:, None] >= permute_stats, axis=1) / permutations + ranks = np.sum(test_stats[:, None] >= permute_stats, axis=1) / permute_stats.shape[1] simulation_based_calibration_histogram(ranks, sbc_histogram, bins=sbc_bins) # Probability Integral Transform (PIT) plot if pit_plot is not None: - _pit_plot(pvals, pit_plot, confidence=pit_confidence) + # Under H0 a p-value of (1 + q) / (1 + n_null) is uniform on that many + # lattice points, so the band can be built from the exact null. + _pit_plot( + pvals, + pit_plot, + confidence=pit_confidence, + lattice=permute_stats.shape[1] + 1, + ) # Compute p-value if nsim == 1: diff --git a/src/pted/tests.py b/src/pted/tests.py index 3ea834f..ad3062f 100644 --- a/src/pted/tests.py +++ b/src/pted/tests.py @@ -6,11 +6,14 @@ def test(): np.random.seed(42) # example 2 sample test D = 300 - for _ in range(20): - x = np.random.normal(size=(100, D)) - y = np.random.normal(size=(100, D)) - p = pted(x, y) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" + # Under the null these are U(0,1), so any single draw can land anywhere: + # check the batch, not each p-value. Asserting 0.01 < p < 0.99 on each of + # 20 uniform draws fails a third of the time on its own. + pvals = [ + pted(np.random.normal(size=(100, D)), np.random.normal(size=(100, D))) for _ in range(20) + ] + median = float(np.median(pvals)) + assert 0.1 < median < 0.9, f"null p-values are not U(0,1): median {median}, {pvals}" x = np.random.normal(size=(100, D)) y = np.random.uniform(size=(100, D)) diff --git a/src/pted/utils.py b/src/pted/utils.py index a5be282..80e913c 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -1,10 +1,77 @@ -from typing import Optional, Union +"""Backend-agnostic machinery for the permutation test on the energy distance. + +The pooled sample is ``z = [x; y]`` of size ``n = n1 + n2``. Rather than the +full ``n x n`` distance matrix of Szekely & Rizzo (2004), the test may be run +on a rectangular ``n x m`` matrix ``D = cdist(z, z[landmarks])``, where +``landmarks`` indexes ``m`` points drawn from the pooled sample. Every sample +is measured against those ``m`` landmarks and no other distance is computed: +``O(n * m)`` distances and ``O(n * m)`` work per permutation rather than +``O(n^2)``. Setting ``m = n`` recovers the exact full-matrix test, so both +live in one code path. This is the Nystrom-style subsampling familiar from +kernel methods, applied to the energy distance. + +Validity +-------- +Permutations are always confined to the subgroup ``S_L x S_{L^c}``, where ``L`` +is the landmark set: labels are shuffled within the landmark positions and +within their complement, never across. Since ``sigma(L) = L`` identically, the +landmark set is invariant and the test is exact conditional on ``L``. This is +what licenses choosing ``L`` by design (balanced, or aligned with the group +structure). The one requirement is that ``L`` must not be chosen using the data +values -- sample positions and group labels are fine, but maximin, k-means or +leverage selection would break the argument. "Landmark" here means only "a +point everything is measured against", never "a point picked for importance". + +Landmark allocation +------------------- +Dispatched on ``n_small = min(n1, n2)``: + +``full`` + ``m >= n``. Every point is a landmark, ``S_L x S_{L^c}`` is the full + symmetric group, and the statistic is the exact energy distance. + +``singleton`` + ``n_small == 1``. Its within-group mean is identically zero (a single point + has no within-group pairs), so keeping it out of ``L`` costs nothing and + lets its label roam over ``L^c``: ``n - m`` distinct assignments. + +``small_group_in_L`` + ``n_small <= m // 2``. ``L`` holds ALL of the small group plus + ``m - n_small`` from the large group. Every permuted small group is then a + subset of ``L``, so every within-small-group distance appears in ``D`` and + that term is computed exactly rather than subsampled. + +``proportional`` + Otherwise. ``n_small_landmarks ~ m * n_small / n``, and both within-group + terms are subsampled. + +Cost +---- +Null spread grows like ``sqrt(n / m)``, so the detectable energy distance +scales as ``(n * m)^{-1/2}`` versus ``n^{-1}`` for the full test: the detection +threshold degrades as one over the square root of the compute. See Janson +(1984) on incomplete U-statistics. + +Landmarks come straight out of p-value resolution in the ``singleton`` regime, +where the reference set is ``n - m``. Keep ``m`` well under ``n / 2`` in any +case: at ``m = n / 2`` the rectangular matrix saves only a factor of two over +the exact test, which is not worth the lost sensitivity. Landmarks are for +``m << n``. + +Block means exclude the zero self-pairs, making the statistic unbiased for the +population energy distance. It can therefore be negative under H0, like +unbiased distance covariance. Only its rank among the permutations matters. +""" + +from itertools import combinations, islice, product +from math import comb +from typing import Optional from warnings import warn import numpy as np from scipy.spatial.distance import cdist -from scipy.stats import chi2 as chi2_dist, binom, kstwo, kstest -from tqdm.auto import trange +from scipy.stats import chi2 as chi2_dist, binom +from tqdm.auto import tqdm try: import torch @@ -28,12 +95,13 @@ class torch: __all__ = ( "is_torch_tensor", "is_jax_array", - "pted", - "pted_chunk", + "permutation_energy_test", + "PermutationResolutionWarning", "two_tailed_p", "confidence_alert", "simulation_based_calibration_histogram", "pit_plot", + "containment_pit_plot", "hdp_coverage_test", ) @@ -79,30 +147,45 @@ def _all_finite(z, backend: str) -> bool: return bool(np.all(np.isfinite(z))) -def _to_scalar(x, backend: str) -> float: +def _to_numpy(x, backend: str) -> np.ndarray: if backend == "torch": - return float(x.item()) + return x.detach().cpu().numpy() + return np.asarray(x) + + +def _asarray_like(x, ref, backend: str): + """Move ``x`` onto the device and dtype of ``ref``.""" + if backend == "torch": + return torch.as_tensor(x, dtype=ref.dtype, device=ref.device) if backend == "jax": - return float(x.item()) - return float(x) + return jnp.asarray(x, dtype=ref.dtype) + return np.asarray(x, dtype=ref.dtype) -def _to_backend(x, backend: str): +def _take_along0(a, idx, backend: str): + """Gather ``a`` along axis 0 with per-column indices.""" if backend == "torch": - return torch.as_tensor(x) + return torch.take_along_dim(a, idx, dim=0) if backend == "jax": - return jnp.asarray(x) - return np.asarray(x) + return jnp.take_along_axis(a, idx, axis=0) + return np.take_along_axis(a, idx, axis=0) -def _random_permutation(D, backend: str = "numpy"): +def _log(a, backend: str): if backend == "torch": - I = torch.randperm(D.shape[0], device=D.device) - elif backend == "jax": - I = jax.random.permutation(jax.random.PRNGKey(np.random.randint(0, 1e6)), D.shape[0]) - else: - I = np.random.permutation(D.shape[0]) - return D[I][:, I] + return torch.log(a) + if backend == "jax": + return jnp.log(a) + return np.log(a) + + +def _index_like(idx, ref, backend: str): + """Move an integer index array onto the device of ``ref``.""" + if backend == "torch": + return torch.as_tensor(idx, dtype=torch.long, device=ref.device) + if backend == "jax": + return jnp.asarray(idx) + return np.asarray(idx) @jit @@ -118,93 +201,784 @@ def _cdist(a, b, backend: str): return cdist(a, b, metric="euclidean") -def _energy_distance_precompute( - D: Union[np.ndarray, torch.Tensor], - nx: int, - ny: int, - nxc: Optional[int] = None, - nyc: Optional[int] = None, -) -> Union[float, torch.Tensor]: - """Energy distance from a rectangular (nx+ny, nxc+nyc) distance matrix. - - If ``D`` is a full (nx+ny, nx+ny) distance matrix, then ``nxc=nx`` and - ``nyc=ny`` and this is the standard energy distance formula. Otherwise, if - ``D`` is a rectangular (nx+ny, nxc+nyc) distance matrix, then ``D`` holds - distances from every sample (rows: ``nx`` of x followed by ``ny`` of y) to a - set of landmark samples (columns: ``nxc`` from x followed by ``nyc`` from - y). Exx/Eyy are estimated from each group against its own landmarks, and Exy - is estimated by averaging the two cross blocks (x-rows vs y-landmarks, and - y-rows vs x-landmarks). +def _zero_self_pairs(D, landmarks, backend: str): + """Set ``D[landmarks[k], k] = 0``, the entries pairing a point with itself. + + Called after ``D`` has been shifted by a constant, to restore the property + that a point is at distance zero from itself. Self-pairs are excluded from + every pair count, so leaving them shifted would bias the block means. + + Only positional self-pairs are touched. A genuine duplicate point elsewhere + in the sample is a real pair at distance zero and stays counted. This also + scrubs the ~1e-7 that ``torch.cdist`` leaves on the diagonal once it + switches to its matmul-based algorithm. """ - nxc = nx if nxc is None else nxc - nyc = ny if nyc is None else nyc - Exx = D[:nx, :nxc].sum() / (nx * nxc) - Eyy = D[nx:, nxc:].sum() / (ny * nyc) - Exy = (D[:nx, nxc:].sum() + D[nx:, :nxc].sum()) / (nx * nyc + ny * nxc) - return 2 * Exy - Exx - Eyy + k = np.arange(len(landmarks)) + if backend == "jax": + return D.at[jnp.asarray(landmarks), jnp.asarray(k)].set(0.0) + D[_index_like(landmarks, D, backend), _index_like(k, D, backend)] = 0.0 + return D -def pted( - x, - y, - permutations: int, - prog_bar: bool = False, -) -> tuple[float, list[float]]: - """Permutation-based energy distance test statistic and its permutation distribution. +def _as_rng(rng) -> np.random.Generator: + """Coerce ``rng`` to a numpy ``Generator``. - Works transparently with numpy arrays, torch tensors, and jax arrays. + ``None`` seeds a fresh generator from the legacy global state, so that + ``np.random.seed(...)`` still controls reproducibility for callers who + rely on it. An existing ``Generator`` is returned untouched, so a caller + that has already coerced its own argument can pass the result down without + reseeding anything. """ - if is_torch_tensor(x): - assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" - if is_jax_array(x): - assert jax is not None, "JAX is not installed! try: `pip install jax`" + if isinstance(rng, np.random.Generator): + return rng + if rng is None: + return np.random.default_rng(np.random.randint(0, 2**32, dtype=np.uint32)) + return np.random.default_rng(rng) - backend = _backend(x) - z = _concatenate((x, y), backend) - assert _all_finite(z, backend), "Input contains NaN or Inf!" - dmatrix = _cdist(z, z, backend) - assert _all_finite( - dmatrix, backend - ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." - if backend == "jax": # numpy is faster for eager stuff - dmatrix = np.asarray(dmatrix) - backend = "numpy" - nx = len(x) - ny = len(y) - - test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend) - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - dmatrix = _random_permutation(dmatrix, backend) - permute_stats.append(_to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend)) - return test_stat, permute_stats +class PermutationResolutionWarning(UserWarning): + """The permutation group is too small to resolve the requested p-value.""" + + +# Budgets for the automatic batch size, each bounding a different resource: +# the memory the batch-scaling tensors hold at once, and the work one batch +# does between progress updates. +_BATCH_BYTES = 2**30 +_BATCH_ELEMENTS = 2**30 + + +def _reference_size(n: int, n_landmarks: int, n_small: int, n_small_landmarks: int): + """How many distinct labellings the subgroup ``S_L x S_{L^c}`` reaches. + + It places ``n_small_landmarks`` of the small-group labels among the + landmarks and the rest among the ``n - n_landmarks`` other positions, + independently, so the count is the product of two binomials. + + Only smallness matters -- the number is either a handful, in which case the + p-value resolution is limited and the caller needs to know, or it is + astronomical. Returns ``inf`` for the latter rather than spending the + arithmetic on an integer nobody reads. + """ + total = 1 + for size, small in ( + (n_landmarks, n_small_landmarks), + (n - n_landmarks, n_small - n_small_landmarks), + ): + if min(small, size - small) > 50: + return float("inf") + total *= comb(size, small) + return total + + +# --------------------------------------------------------------- allocation + + +def _allocate(n1: int, n2: int, n_landmarks: int, rng) -> dict: + """Pick the allocation for this landmark count. + + The two branches describe genuinely different things -- a landmark set and + its subgroup, or no landmark set at all -- so they return different keys. + ``regime`` says which, and is what everything downstream dispatches on. + """ + if n_landmarks >= n1 + n2: + return _allocate_full(n1, n2) + return _allocate_landmarks(n1, n2, n_landmarks, rng) + + +def _allocate_full(n1: int, n2: int) -> dict: + """Every point is a landmark: the classical exact permutation test. + + There is no landmark set to hold fixed, so ``S_L x S_{L^c}`` is the whole + symmetric group and the subgroup restriction is vacuous. The distance + matrix is square and symmetric, which the statistic exploits. Nothing + downstream needs a landmark index, a landmark count or a column gather, so + this allocation does not carry any. + """ + n = n1 + n2 + n_small = min(n1, n2) + return { + "regime": "full", + "small_idx": np.arange(n1) if n1 <= n2 else n1 + np.arange(n2), + "n_small": n_small, + "n_large": max(n1, n2), + # every position is free, so the group reaches every way of choosing + # which n_small of the n labels belong to the small group + "reference_size": _reference_size(n, n, n_small, n_small), + } + + +def _allocate_landmarks(n1: int, n2: int, n_landmarks: int, rng) -> dict: + """Choose which pooled-sample points serve as landmarks. + + The landmarks are the ``n_landmarks`` points every sample is measured + against: they supply the columns of the rectangular distance matrix ``D``. + Selection uses only group sizes and positions, never the data values, which + together with the restricted permutation group is what keeps the test exact. + + Assumes ``2 <= n_landmarks < n1 + n2`` and that both samples are non-empty; + :func:`permutation_energy_test` checks that before calling. + + Dispatched on the smaller group: + + ``singleton`` + ``n_small == 1``. A lone point has no within-group pairs, so keeping it + out of ``L`` costs nothing and lets its label roam over the + ``n - n_landmarks`` positions outside. + + ``small_group_in_L`` + ``n_small <= n_landmarks // 2``. ``L`` holds ALL of the small group, so + every permuted small group is still a subset of ``L`` and its + within-group distances are all present -- that term is exact rather + than subsampled. + + ``proportional`` + Otherwise. The landmarks are split between the groups in proportion to + their sizes, and both within-group terms are subsampled. + """ + n = n1 + n2 + n_small, n_large = min(n1, n2), max(n1, n2) + x_is_small = n1 <= n2 + small_idx = np.arange(n1) if x_is_small else n1 + np.arange(n2) + large_idx = n1 + np.arange(n2) if x_is_small else np.arange(n1) + + if n_small == 1: + landmarks = rng.choice(large_idx, size=n_landmarks, replace=False) + regime, n_small_landmarks = "singleton", 0 + elif n_small <= n_landmarks // 2: + landmarks = np.concatenate( + [small_idx, rng.choice(large_idx, size=n_landmarks - n_small, replace=False)] + ) + regime, n_small_landmarks = "small_group_in_L", n_small + else: + n_small_landmarks = int( + np.clip( + round(n_landmarks * n_small / n), + max(1, n_landmarks - n_large), + min(n_small, n_landmarks - 1), + ) + ) + landmarks = np.concatenate( + [ + rng.choice(small_idx, size=n_small_landmarks, replace=False), + rng.choice(large_idx, size=n_landmarks - n_small_landmarks, replace=False), + ] + ) + regime = "proportional" + + return { + "landmarks": np.sort(landmarks), + "regime": regime, + "small_idx": small_idx, + "n_small": n_small, + "n_large": n_large, + "n_small_landmarks": n_small_landmarks, + "n_large_landmarks": n_landmarks - n_small_landmarks, + "n_landmarks": n_landmarks, + "reference_size": _reference_size(n, n_landmarks, n_small, n_small_landmarks), + } + + +def _landmark_columns(z, alloc: dict, backend: str): + """The points every sample is measured against, i.e. the columns of ``D``. + + All of them in the full regime, the chosen landmark subset otherwise. + """ + if alloc["regime"] == "full": + return z + return z[_index_like(alloc["landmarks"], z, backend)] + + +def _warn_reference_size(alloc: dict, permutations: int) -> None: + """Warn when the permutation group cannot resolve the requested p-value. + + The observed labelling is itself a member of the group, so with only ``R`` + reachable assignments roughly one draw in ``R`` reproduces it exactly and + the p-value is floored near ``1/R`` however many permutations are drawn. + """ + reference = alloc["reference_size"] + # In the full regime the reference set is fixed by the sample sizes alone, + # so only flag it when it is degenerate. Everywhere else it is a + # consequence of the landmark count, which the caller can act on. + floor = 20 if alloc["regime"] == "full" else max(20, (permutations + 1) // 10) + if reference >= floor: + return + + if alloc["regime"] == "full": + hint = "the samples are too small for a permutation test at this resolution" + elif alloc["regime"] == "singleton": + hint = ( + f"the small group holds a single point, so the reference set is " + f"n - n_landmarks; use fewer landmarks (currently {alloc['n_landmarks']}), " + f"well under half the pooled sample size" + ) + else: + hint = f"use more landmarks (currently {alloc['n_landmarks']}) or more samples" + warn( + PermutationResolutionWarning( + f"the {alloc['regime']!r} allocation reaches only {reference:,} distinct " + f"label assignments, so the smallest attainable p-value is about " + f"{1.0 / reference:.3g} no matter how many permutations are drawn " + f"({permutations} requested): {hint}." + ) + ) + + +def _label_drawer(prep: dict, rng): + """Return ``draw(rows) -> (rows, n)`` small-group indicator matrices. + + Each part -- the landmark positions and their complement -- gets a + uniformly random subset of the size the subgroup fixes, found by ranking + uniform keys: the ``k`` smallest of ``|part|`` iid uniforms are a uniform + random ``k``-subset. Ranking is what makes this a device operation, so the + labels are built wherever the distance matrix lives rather than shuffled on + the host and shipped over -- for large ``n`` that shuffle costs more than + the matrix product it feeds, and no accelerator can help with it. + + The returned function carries the generator state, so successive calls + continue the same stream. Which permutations that yields depends on how the + run is batched; the test is exact either way, and a fixed ``rng`` with a + fixed ``batch_size`` reproduces a run exactly. + """ + backend, ref = prep["backend"], prep["ref"] + n = prep["base_small"].size + parts = [(_index_like(idx, ref, backend), k) for idx, k in prep["parts"] if k] + + if backend == "torch": + gen = torch.Generator(device=ref.device) + gen.manual_seed(int(rng.integers(0, 2**62))) + + def draw(rows): + U = torch.zeros((rows, n), dtype=ref.dtype, device=ref.device) + for idx, k in parts: + keys = torch.rand( + (rows, idx.numel()), generator=gen, device=ref.device, dtype=ref.dtype + ) + pick = torch.topk(keys, k, dim=1, largest=False, sorted=False).indices + U.scatter_(1, idx[pick], 1.0) + return U + + elif backend == "jax": + key = jax.random.PRNGKey(int(rng.integers(0, 2**62))) + + def draw(rows): + nonlocal key + U = jnp.zeros((rows, n), dtype=ref.dtype) + index = jnp.arange(rows)[:, None] + for idx, k in parts: + key, subkey = jax.random.split(key) + keys = jax.random.uniform(subkey, (rows, int(idx.size))) + U = U.at[index, idx[jax.lax.top_k(-keys, k)[1]]].set(1.0) + return U + + else: + + def draw(rows): + U = np.zeros((rows, n), dtype=ref.dtype) + for idx, k in parts: + keys = rng.random((rows, idx.size)) + pick = np.argpartition(keys, k - 1, axis=1)[:, :k] + np.put_along_axis(U, idx[pick], U.dtype.type(1.0), axis=1) + return U + + return draw + + +def _sampled_label_blocks(prep: dict, permutations: int, batch_size: int, rng): + """Yield blocks of randomly drawn labels, ``permutations`` rows in total.""" + draw = _label_drawer(prep, rng) + for start in range(0, permutations, batch_size): + yield draw(min(batch_size, permutations - start)) + + +def _enumerate_label_blocks(prep: dict, batch_size: int): + """Yield blocks covering the whole subgroup, minus the observed labelling. + + The subgroup fixes how many small labels sit among the landmarks, so it is + exactly: which of the landmark positions carry a small label, crossed with + which of the remaining positions carry the rest. Dropping the observed + assignment leaves ``reference_size - 1`` rows, so the caller's + ``(1 + q) / (1 + len)`` becomes the exact rank of the observed statistic + within the group -- no sampling, and no ties from redrawing the observed + labelling. + + Only ever called when ``reference_size`` is small, so materialising the + group a block at a time is cheap. + """ + base = prep["base_small"] + assignments = product(*(combinations(idx, k) for idx, k in prep["parts"])) + while chunk := list(islice(assignments, batch_size)): + U = np.zeros((len(chunk), base.size), dtype=base.dtype) + for j, picks in enumerate(chunk): + U[j, [i for pick in picks for i in pick]] = 1.0 + keep = np.any(U != base, axis=1) + if keep.any(): + yield U[keep] + + +# ---------------------------------------------------------------- statistic +# +# The energy statistic is a fixed combination of three block means -- the mean +# cross-group distance and the two mean within-group distances: +# +# E = (n_small * n_large / n) * (2 * cross - small - large) +# +# Each mean is a block of the distance matrix over its pair count, and every +# one of those counts is a constant: the subgroup S_L x S_{L^c} holds the +# per-group landmark counts fixed. So preparing a run means centring the +# matrix and recording the counts; evaluating a batch of labellings means +# forming the block sums and dividing. +# +# Writing u for a row of the (B, n) small-group indicator and v = u[landmarks] +# for its restriction to the landmark columns, each block sum is a bilinear +# form and one matrix product yields all four: +# +# S_ss = u @ D @ v S_sl = u @ D @ 1 - S_ss +# S_ls = v @ (1 @ D) - S_ss S_ll = total - S_ss - S_sl - S_ls +# +# The three regimes differ in which of those blocks survive, by enough that +# each gets its own prepare/evaluate pair rather than a shared one with the +# differences hidden in coefficients. + + +def _prepare_statistic(D, alloc: dict, backend: str) -> dict: + """Precompute everything constant across permutations, for this regime.""" + prepare = {"full": _prepare_full, "singleton": _prepare_singleton} + return prepare.get(alloc["regime"], _prepare_landmarks)(D, alloc, backend) + + +def _centre(D, self_pair_rows, backend: str): + """Centre ``D`` on its mean and put the self-pairs back to zero. + + In high dimension the pairwise distances concentrate far from zero and the + uncentred block sums spend most of their significant digits on an offset + that largely cancels, so centre before any reduction. ``self_pair_rows[k]`` + is the point that column ``k`` was measured against; those entries have to + return to zero because the pair counts exclude them. + + Returns ``(D, offset, total)``. Add ``offset`` back to any block *mean* + taken on the centred matrix to recover the mean on the real distances -- + the offset itself needs no precision, since it either cancels in + ``2 * cross - small - large`` or multiplies a block that is genuinely + absent. ``total`` is accumulated in float64 because its error does survive + into the reported statistic. + """ + offset = float(D.mean()) + D = _zero_self_pairs(D - offset, self_pair_rows, backend) + total = float(_to_numpy(D.sum(1), backend).sum(dtype=np.float64)) + return D, offset, total + + +def _itemsize(a, backend: str) -> int: + """Bytes per element of ``a``.""" + return a.element_size() if backend == "torch" else a.dtype.itemsize + + +def _auto_batch_size(ref, backend: str, row_elements: int, row_work: int) -> int: + """Permutations per batch when the caller names none. Two caps, smaller wins. + + ``row_elements`` is how many array elements one permutation needs across + the tensors an evaluator holds live at once, so the first cap is a direct + byte budget on them. Sizing off the matrix product alone would miss it: a + large sample against few landmarks -- which is exactly what landmarks are + for -- looks cheap as ``n * m`` while the batch really carries ``rows * n``. + + ``row_work`` is the matrix-product cost of one permutation. Capping that + keeps a single batch from running for minutes without touching the + progress bar or giving the caller a chance to interrupt. + """ + by_memory = _BATCH_BYTES // (row_elements * _itemsize(ref, backend)) + by_work = _BATCH_ELEMENTS // row_work + return max(1, min(by_memory, by_work)) + + +def _subgroup_parts(alloc: dict, n: int): + """``(positions, how many carry the small label)`` per half of the subgroup. + + Those two counts are exactly what ``S_L x S_{L^c}`` holds fixed. The full + regime has no ``L^c``, so it is one part spanning every position. + """ + if alloc["regime"] == "full": + return ((np.arange(n), alloc["n_small"]),) + in_l = np.zeros(n, dtype=bool) + in_l[alloc["landmarks"]] = True + n_small_landmarks = alloc["n_small_landmarks"] + return ( + (np.flatnonzero(in_l), n_small_landmarks), + (np.flatnonzero(~in_l), alloc["n_small"] - n_small_landmarks), + ) + + +def _base_small(alloc: dict, n: int) -> np.ndarray: + """The observed labelling, as a small-group indicator.""" + base = np.zeros(n) + base[alloc["small_idx"]] = 1.0 + return base + + +# ----------------------------------------------------------- full matrix + +def _prepare_full(D, alloc: dict, backend: str) -> dict: + """``D`` is square and symmetric, so the column sums are the row sums.""" + n_small, n_large = alloc["n_small"], alloc["n_large"] + n = n_small + n_large + # the columns are the points themselves, so the self-pairs are the diagonal + D, offset, total = _centre(D, np.arange(n), backend) + return { + "D": D, + "ref": D, + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), + "backend": backend, + "offset": offset, + "total": total, + "scale": n_small * n_large / n, + "n_small": n_small, + "n_large": n_large, + # per permutation: the drawn labels, u, W = u @ D, and the W * u + # temporary, all (rows, n); the block sums that follow are scalars + "batch_size": _auto_batch_size(D, backend, 4 * n, n * n), + "evaluate": _evaluate_full, + } -def pted_chunk( + +def _evaluate_full(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, full symmetric matrix. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_full`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. + """ + backend, offset = prep["backend"], prep["offset"] + n_small, n_large = prep["n_small"], prep["n_large"] + u = _asarray_like(U, prep["D"], backend) + + W = u @ prep["D"] # (B, n) + S_ss = (W * u).sum(-1) + S_sl = W.sum(-1) - S_ss + # symmetry gives S_ls == S_sl, so there is no column-sum term to form + S_ll = prep["total"] - S_ss - 2.0 * S_sl + + cross = S_sl / (n_small * n_large) + offset + # A group of one point has no within-group pairs at all, so its mean is + # zero on the real distances -- which is exactly where `offset` matters. + small = 0.0 if n_small == 1 else S_ss / (n_small * (n_small - 1)) + offset + large = 0.0 if n_large == 1 else S_ll / (n_large * (n_large - 1)) + offset + + stat = prep["scale"] * (2.0 * cross - small - large) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +# ------------------------------------------------------------- landmarks + + +def _prepare_landmarks(D, alloc: dict, backend: str) -> dict: + """``D`` is rectangular, and all four blocks are subsampled over columns. + + Reached whenever the small group has landmarks of its own, which also + means it holds at least two points: ``n_small == 1`` goes to the singleton + pair below. ``n_large == 1`` forces ``n <= 2``, which is always full. + """ + n_small, n_large = alloc["n_small"], alloc["n_large"] + n_small_landmarks = alloc["n_small_landmarks"] + n_large_landmarks = alloc["n_large_landmarks"] + n, m = n_small + n_large, n_small_landmarks + n_large_landmarks + D, offset, total = _centre(D, alloc["landmarks"], backend) + return { + "D": D, + "ref": D, + "col_sums": D.sum(0), + "landmarks": _index_like(alloc["landmarks"], D, backend), + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), + "backend": backend, + "offset": offset, + "total": total, + "scale": n_small * n_large / n, + # pairs behind each block: rows times landmark columns, self-pairs out + "ss_pairs": n_small_landmarks * (n_small - 1), + "sl_pairs": n_small * n_large_landmarks, + "ls_pairs": n_large * n_small_landmarks, + "ll_pairs": n_large_landmarks * (n_large - 1), + # per permutation: the drawn labels and u at (rows, n), then v, W and + # the W * v temporary at (rows, m) + "batch_size": _auto_batch_size(D, backend, 2 * n + 3 * m, n * m), + "evaluate": _evaluate_landmarks, + } + + +def _evaluate_landmarks(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, rectangular matrix. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_landmarks`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. + """ + backend, offset = prep["backend"], prep["offset"] + u = _asarray_like(U, prep["D"], backend) + v = u[:, prep["landmarks"]] # the same indicator, restricted to the columns + + W = u @ prep["D"] # (B, m) + S_ss = (W * v).sum(-1) + S_sl = W.sum(-1) - S_ss + S_ls = v @ prep["col_sums"] - S_ss + S_ll = prep["total"] - S_ss - S_sl - S_ls + + # The cross block is sampled in both orientations here, small rows against + # large columns and large rows against small columns. They estimate the + # same mean over different pairs, so average them. + cross = 0.5 * (S_sl / prep["sl_pairs"] + S_ls / prep["ls_pairs"]) + offset + small = S_ss / prep["ss_pairs"] + offset + large = S_ll / prep["ll_pairs"] + offset + + stat = prep["scale"] * (2.0 * cross - small - large) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +# ------------------------------------------------------------- singleton + + +def _prepare_singleton(D, alloc: dict, backend: str) -> dict: + """The small group is one point, deliberately kept out of the landmarks. + + With no small-group landmarks two of the four blocks vanish: ``S_ss`` and + ``S_ls`` have no columns to sum over. The only quantity left that varies + with the labelling is the row sum of whichever point carries the small + label, so the matrix is reduced to its row sums here and never touched + again -- there is no matrix product per permutation, only a dot product. + """ + n_large = alloc["n_large"] + n_large_landmarks = alloc["n_large_landmarks"] + n = 1 + n_large + D, offset, total = _centre(D, alloc["landmarks"], backend) + row_sums = D.sum(1) + return { + "ref": row_sums, + "row_sums": row_sums, + "parts": _subgroup_parts(alloc, n), + "base_small": _base_small(alloc, n), + "backend": backend, + "offset": offset, + "total": total, + "scale": n_large / n, + # the one small point against every large landmark + "sl_pairs": n_large_landmarks, + "ll_pairs": n_large_landmarks * (n_large - 1), + # per permutation: the drawn labels and u, nothing wider -- and a dot + # product against the row sums rather than a matrix product + "batch_size": _auto_batch_size(row_sums, backend, 2 * n, n), + "evaluate": _evaluate_singleton, + } + + +def _evaluate_singleton(prep: dict, U) -> np.ndarray: + """Energy statistic for a batch of labellings, one-point small group. + + Parameters + ---------- + prep (dict): output of :func:`_prepare_singleton`. + U (np.ndarray): ``(B, n)`` indicator of small-group membership. + + Returns + ------- + ``(B,)`` numpy array of statistics. + """ + backend, offset = prep["backend"], prep["offset"] + u = _asarray_like(U, prep["row_sums"], backend) + + S_sl = u @ prep["row_sums"] # S_ss and S_ls are empty blocks + S_ll = prep["total"] - S_sl + + cross = S_sl / prep["sl_pairs"] + offset + small = 0.0 # one point, no within-group pairs: zero on the real distances + large = S_ll / prep["ll_pairs"] + offset + + stat = prep["scale"] * (2.0 * cross - small - large) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +def _containment_curve(prep: dict, U): + """``(B, n_y + 1)`` running count of x points at each step of the p lattice. + + The per-point p-values live on ``{1/(n_y+1), ..., 1}``, so their empirical + CDF is fully described by how many x points have accumulated by each lattice + step. One row per labelling. + """ + backend, D = prep["backend"], prep["D"] + Us = _asarray_like(U, D, backend) + Xs = Us if prep["x_is_small"] else 1.0 - Us + Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] + depth = (D @ (1.0 - Vx).T) / prep["n_y_landmarks"] + order = (-depth).argsort(0) + ranked_x = _to_numpy(_take_along0(Xs.T, order, backend), backend) + ranked_y = _to_numpy(_take_along0((1.0 - Xs).T, order, backend), backend) + + n_y, rows = int(prep["n_y"]), ranked_x.shape[1] + # y points at least as peripheral as each ranked point; this is the count + # that turns into p = (1 + count) / (n_y + 1). + above = ranked_y.cumsum(0).astype(np.int64) + flat = above + np.arange(rows)[None, :] * (n_y + 1) + hist = np.bincount(flat.ravel(), weights=ranked_x.ravel(), minlength=rows * (n_y + 1)) + return hist.reshape(rows, n_y + 1).cumsum(1) + + +def _prepare_containment(D, alloc: dict, backend: str, x_is_small: bool) -> dict: + """Precompute for the containment statistic: is x peripheral within y? + + Every pooled point gets a *depth*, its mean distance to the y-labelled + points; peripheral points have large depth. Each x point then carries the + singleton energy p-value ``p_i = (1 + #{j in y : d_j >= d_i}) / (n_y + 1)``, + and those are aggregated the way :func:`pted.pted_coverage_test` aggregates + its per-simulation p-values, ``-2 sum log p_i``. Large means x sits further + out than y does. + + Unlike the energy distance this is *not* symmetric in x and y, which is the + whole point. + """ + n = alloc["n_small"] + alloc["n_large"] + full = alloc["regime"] == "full" + m = n if full else len(alloc["landmarks"]) + # y is whichever group x is not. + n_y = alloc["n_large"] if x_is_small else alloc["n_small"] + if full: # every point is a column, so every y point is one + self_pair_rows, n_y_landmarks = np.arange(n), n_y + else: + self_pair_rows = alloc["landmarks"] + n_y_landmarks = alloc["n_large_landmarks"] if x_is_small else alloc["n_small_landmarks"] + if n_y_landmarks < 1: + raise ValueError( + "the containing sample has no landmarks, so depth is unestimable; " + "use more landmarks or the full matrix" + ) + + # Ranks are invariant to a constant offset, but zeroing the self-pairs + # still matters: a y point counting its own zero distance would look + # artificially deep. + D, _, _ = _centre(D, self_pair_rows, backend) + + return { + "D": D, + "ref": D, + "parts": _subgroup_parts(alloc, n), + "landmarks": None if full else _index_like(alloc["landmarks"], D, backend), + "x_is_small": x_is_small, + "n_y_landmarks": float(n_y_landmarks), + "n_y": float(n_y), + "base_small": _base_small(alloc, n), + "backend": backend, + # The widest path of the three: the depth ranking holds the labels, + # the depths, the sort order, both ranked indicators and the p-values + # at (rows, n) simultaneously, so it needs the tightest batches. + "batch_size": _auto_batch_size(D, backend, 12 * n + 2 * m, n * m), + "evaluate": _evaluate_containment, + } + + +def _evaluate_containment(prep: dict, U) -> np.ndarray: + """Fisher combination of per-point depth ranks, for a batch of labellings.""" + backend, D = prep["backend"], prep["D"] + Us = _asarray_like(U, D, backend) + Xs = Us if prep["x_is_small"] else 1.0 - Us # (B, n) indicator of the x sample + Vx = Xs if prep["landmarks"] is None else Xs[:, prep["landmarks"]] + + # Depth of every pooled point against each labelling's y set. + depth = (D @ (1.0 - Vx).T) / prep["n_y_landmarks"] # (n, B) + + # Rank by depth, most peripheral first; the running count of y points is + # then #{j in y : d_j >= d_i} without ever materialising a comparison. + order = (-depth).argsort(0) + ranked_x = _take_along0(Xs.T, order, backend) + ranked_y = _take_along0((1.0 - Xs).T, order, backend) + p = (1.0 + ranked_y.cumsum(0)) / (prep["n_y"] + 1.0) + + stat = -2.0 * (ranked_x * _log(p, backend)).sum(0) + return _to_numpy(stat, backend).astype(np.float64, copy=False) + + +# --------------------------------------------------------------- public API + + +def permutation_energy_test( x, y, permutations: int, - chunk_size: int, + n_landmarks: Optional[int] = None, prog_bar: bool = False, -) -> tuple[float, list[float]]: - """Chunked variant of `pted` for large datasets. - - Rather than the full ``(nx+ny, nx+ny)`` pairwise distance matrix, this - builds a rectangular ``(nx+ny, nxc+nyc)`` matrix of distances from every - sample to a set of randomly chosen "landmark" samples, where ``nxc = - min(chunk_size, nx)`` landmarks are drawn from ``x`` and ``nyc = - min(chunk_size, ny)`` from ``y``. This matrix is computed only once; the - permutation distribution is then obtained by re-indexing (permuting) its - rows, which reassigns samples to the x/y groups without recomputing any - distances. Each landmark's row position is tracked and carried through the - same row permutation, so after every shuffle the landmarks are re-split into - "x-side"/"y-side" columns according to where they now fall, keeping rows and - columns consistent. In pathological cases where too few landmarks are - permuted onto either side (zero or one tenth of the original number of - landmarks), a new permutation is drawn. + batch_size: Optional[int] = None, + rng=None, + containment: bool = False, +) -> tuple[float, np.ndarray]: + """Observed statistic and its permutation distribution. + + With ``containment=True`` the statistic is the depth-rank aggregate that + asks whether x sticks out of y, rather than the symmetric energy distance + that asks whether they differ at all. Everything else -- the landmark + allocation, the subgroup, the enumeration -- is identical. + + Without ``n_landmarks`` this is the exact full-matrix test: the ``(n, n)`` + distance matrix is built once and the statistic is evaluated for batches of + permutations as a bilinear form in the group indicator, so no distance is + recomputed and no matrix is re-indexed. Asking for fewer landmarks switches to the rectangular + ``(n, m)`` matrix, cutting the cost from ``O(n^2 d)`` to ``O(n m d)``; + permutations are then confined to the subgroup that fixes the landmark + set, which keeps the p-value exact. See the module docstring for the + landmark allocation regimes and what they cost in p-value resolution. Works transparently with numpy arrays, torch tensors, and jax arrays. + Distances and all matrix algebra stay on the backend that ``x`` and ``y`` + live on; only the label bookkeeping and the resulting statistics touch + numpy. + + Parameters + ---------- + x, y: samples, shape ``(n1, d)`` and ``(n2, d)``. + permutations (int): number of permutations to draw. + n_landmarks (Optional[int]): number of landmarks ``m``, the points + every sample is measured against. None, or any value covering the + whole pooled sample, runs the exact full-matrix test. + prog_bar (bool): show a progress bar over permutations. + batch_size (Optional[int]): permutations evaluated per matrix product. + Larger is faster on GPU, at ``O(batch_size * n)`` extra memory. + None picks the largest size that keeps the batch-scaling tensors + within 1 GiB and one batch's matrix product bounded, so a run + cannot allocate unboundedly whatever the sample shape. + rng: seed, ``np.random.Generator``, or None to draw from the global + numpy state (so ``np.random.seed`` still applies). + + Returns + ------- + ``(test_stat, permute_stats)``, the observed statistic and an array of + permuted statistics. That array holds ``permutations`` random draws -- + unless the subgroup has no more than ``permutations`` members, in which + case it holds every one of them except the observed labelling + (``reference_size - 1`` entries) and the resulting p-value is exact + rather than sampled. + """ + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment) + return _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng) + + +def _prepare_test(x, y, n_landmarks: Optional[int], rng, containment: bool): + """Everything that does not depend on the permutations. + + Validates the samples, picks the landmarks, builds the distance matrix and + precomputes the statistic -- which is all of the expensive work, since the + distance matrix is ``O(n * m * d)``. Split out from the permutation loop so + that a caller wanting both a test and a diagnostic built from the *same* + landmarks pays for it once; see :func:`containment_pit_plot`. + + Returns ``(prep, alloc, rng)``. The generator is the coerced one, already + advanced past the landmark draw, so passing it on continues the stream. """ if is_torch_tensor(x): assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" @@ -214,45 +988,64 @@ def pted_chunk( backend = _backend(x) z = _concatenate((x, y), backend) assert _all_finite(z, backend), "Input contains NaN or Inf!" - nx = len(x) - ny = len(y) - nxc = min(chunk_size, nx) - nyc = min(chunk_size, ny) - landmark_pos = np.sort( - np.concatenate( - [ - np.random.choice(nx, size=nxc, replace=False), - nx + np.random.choice(ny, size=nyc, replace=False), - ] - ) + n1, n2 = len(x), len(y) + n = n1 + n2 + if min(n1, n2) < 1: + raise ValueError(f"both samples need at least one point, got n1={n1}, n2={n2}") + # More landmarks than points is just the full matrix. + m = n if n_landmarks is None else min(int(n_landmarks), n) + if m < 2: + raise ValueError(f"need at least 2 landmarks, got n_landmarks={n_landmarks}") + + rng = _as_rng(rng) + alloc = _allocate(n1, n2, m, rng) + + dmatrix = _cdist(z, _landmark_columns(z, alloc, backend), backend) + assert _all_finite(dmatrix, backend), ( + "Distance matrix contains NaN or Inf! Consider normalizing values to be " + "more stable (i.e. z-score norm)." ) - dmatrix = _cdist(z, z[_to_backend(landmark_pos, backend)], backend) - assert _all_finite( - dmatrix, backend - ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." - if backend == "jax": # numpy is faster for eager stuff - dmatrix = np.asarray(dmatrix) - backend = "numpy" - - test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny, nxc, nyc), backend) - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - while True: - I = np.random.permutation(len(z)) - # Track where each landmark moved to, then re-split columns by their new x/y side. - landmark_pos = np.argsort(I)[landmark_pos] - order = np.argsort(landmark_pos >= nx, kind="stable") - dmatrix_i = dmatrix[_to_backend(I, backend)][:, _to_backend(order, backend)] - landmark_pos = landmark_pos[order] - nxc_i = int(np.sum(landmark_pos < nx)) - nyc_i = nxc + nyc - nxc_i - if nxc_i < max(1, (nxc * 0.1)) or nyc_i < max(1, (nyc * 0.1)): - continue # Skip this permutation if too few landmarks remain on either side, to avoid unstable estimates. - permute_stats.append( - _to_scalar(_energy_distance_precompute(dmatrix_i, nx, ny, nxc_i, nyc_i), backend) - ) - break + + if containment: + return _prepare_containment(dmatrix, alloc, backend, n1 <= n2), alloc, rng + return _prepare_statistic(dmatrix, alloc, backend), alloc, rng + + +def _permutation_distribution( + prep: dict, alloc: dict, permutations: int, prog_bar: bool, batch_size, rng +) -> tuple[float, np.ndarray]: + """Evaluate the observed labelling and the permuted ones.""" + _warn_reference_size(alloc, permutations) + evaluate = prep["evaluate"] + test_stat = float(evaluate(prep, prep["base_small"][None, :])[0]) + assert np.isfinite(test_stat), "Observed statistic is not finite!" + + # Each preparer sized its own batches against the tensors its evaluator + # holds live, so an unset batch_size is already bounded. + batch_size = max(1, int(prep["batch_size"] if batch_size is None else batch_size)) + + # Sampling more permutations than the subgroup has members just redraws the + # observed labelling over and over, and every such tie inflates q under the + # >= convention. Below that point, walking the whole group is both exact + # and cheaper. + if alloc["reference_size"] <= permutations: + n_null = int(alloc["reference_size"]) - 1 + blocks = _enumerate_label_blocks(prep, batch_size) + else: + n_null = permutations + blocks = _sampled_label_blocks(prep, permutations, batch_size, rng) + + permute_stats = np.empty(n_null, dtype=np.float64) + done = 0 + with tqdm(total=n_null, disable=not prog_bar) as bar: + for U in blocks: + size = len(U) + permute_stats[done : done + size] = evaluate(prep, U) + done += size + bar.update(size) + assert done == n_null, f"produced {done} null statistics, expected {n_null}" + return test_stat, permute_stats @@ -321,27 +1114,182 @@ def simulation_based_calibration_histogram(ranks, saveto, bins=None): plt.close() -def pit_plot(pvals, saveto, confidence=0.95): - """Create a Probability Integral Transform (PIT) plot. +def _lattice_counts(pvals, t): + """``N_k = #{p <= t_k}`` for each evaluation point, for one or many rows.""" + p = np.atleast_2d(pvals) + rows, k = p.shape[0], len(t) + flat = np.searchsorted(t, p, side="left") + np.arange(rows)[:, None] * k + return np.cumsum(np.bincount(flat.ravel(), minlength=rows * k).reshape(rows, k), axis=1) + - Plots the empirical CDF of the provided p-values against the expected - CDF for a uniform distribution (the 1:1 diagonal). A shaded confidence - region is drawn showing the range within which the empirical CDF should - fall with probability ``confidence`` if the p-values are truly uniform. - The confidence band is derived from the two-sided Kolmogorov-Smirnov - statistic. Any portion of the empirical CDF that lies outside this band - constitutes evidence that the p-values are not uniformly distributed. +def _band_bounds(local, eta): + """Counts whose local level clears ``eta``, per evaluation point. + + ``local[k]`` is a lower tail min'd with an upper tail, so it rises then + falls and the set is a contiguous interval of counts. + """ + inside = local > eta + lower = np.argmax(inside, axis=1) + return lower, lower + inside.sum(axis=1) - 1 - The KS statistic and its p-value are annotated on the plot to quantify - the maximum deviation from the diagonal. + +def _band_accept_probability(t, n, lower, upper): + """``P(lower_k <= N_k <= upper_k for every k)`` under the null, exactly. + + The counting process is Markov: given ``N_k = j`` the other ``n - j`` + p-values are iid on ``(t_k, 1]``, so the increment is + ``Binomial(n - j, rho_k)`` with ``rho_k = (F0(t_k+1) - F0(t_k)) / (1 - + F0(t_k))``. Pushing the distribution of ``N_k`` forward through those + transitions, zeroing everything outside the interval at each step, leaves + exactly the probability of never having escaped. No simulation needed. + """ + f = binom.pmf(np.arange(n + 1), n, t[0]) + f[: lower[0]] = 0.0 + f[upper[0] + 1 :] = 0.0 + for k in range(len(t) - 1): + rho = 1.0 if t[k] >= 1.0 else (t[k + 1] - t[k]) / (1.0 - t[k]) + j = np.arange(lower[k], upper[k] + 1) + j_next = np.arange(lower[k + 1], upper[k + 1] + 1) + step = binom.pmf(j_next[None, :] - j[:, None], (n - j)[:, None], rho) + nxt = np.zeros(n + 1) + nxt[lower[k + 1] : upper[k + 1] + 1] = f[lower[k] : upper[k] + 1] @ step + f = nxt + return float(f.sum()) + + +def _lattice_band(n, lattice, confidence, max_points=512, one_sided=False): + """Simultaneous ECDF band built from the known null of the p-values. + + The ECDF only moves where the p-values can actually land, so this works + with the counts there rather than with order statistics: + + N_k = #{i : p_i <= t_k} ~ Binomial(n, F0(t_k)) exactly + + Every evaluation point gets the same two-sided *local* level ``eta``, + chosen as large as it can be while the probability of *any* count escaping + its interval stays within ``1 - confidence``. The ECDF then leaves the band + with at most that probability under H0, so "pokes out of the band" reads as + "reject", simultaneously over the whole curve. The boundary follows the + real ``n F0(t) (1 - F0(t))`` variance of the count and pinches to nothing + at both corners, rather than holding one constant half-width across the + plot. + + Working with counts rather than order statistics is what lets this handle + *discrete* p-values: ties are exactly what a binomial count expects, + whereas the Beta distribution of an order statistic assumes they never + happen. + + Everything here is exact -- both the escape probability, via + :func:`_band_accept_probability`, and the search for ``eta``, which is a + bisection over the finitely many local levels the table can take. There is + no simulation and no seed. + + Because the null is discrete the attainable levels are discrete too, so the + band is generally a little conservative; the level it actually achieves is + returned rather than assumed. + + Parameters + ---------- + n (int): number of p-values. + lattice (Optional[int]): ``L`` if the p-values are uniform on + ``{1/L, 2/L, ..., 1}`` under H0, as permutation p-values are. None + for a continuous null. + confidence (float): target simultaneous confidence level. + max_points (int): cap on evaluation points. A finer lattice is checked + on an evenly spaced subset, which stays valid and costs a little + power. + one_sided (bool): constrain only from above, spending the whole error + budget on the upper edge. For a composite null where the ECDF is + expected to sit low, a lower edge would flag the healthy case. + + Returns + ------- + ``(t, lower, upper, local, achieved)``: evaluation points, the allowed + ECDF range at each as a fraction of ``n``, the table of local levels + ``local[k, j]`` for count ``j`` at point ``k``, and the exact + simultaneous level the band achieves. + """ + if lattice is None: + points = min(max_points, max(200, 20 * n)) + t = np.arange(1, points + 1) / points + else: + lattice = int(lattice) + k = ( + np.arange(1, lattice + 1) + if lattice <= max_points + else np.unique(np.round(np.linspace(1, lattice, max_points)).astype(int)) + ) + t = k / lattice + + # F0(t_k) = t_k for both nulls; the lattice only changes where we may look. + j = np.arange(n + 1) + cdf = binom.cdf(j[None, :], n, t[:, None]) # P(N_k <= j) + tail = np.empty_like(cdf) + tail[:, 0] = 1.0 + tail[:, 1:] = 1.0 - cdf[:, :-1] # P(N_k >= j) + local = tail if one_sided else np.minimum(cdf, tail) + + # Raising eta tightens every interval, so the acceptance probability falls + # monotonically: bisect for the largest eta that still clears the target. + candidates = np.unique(local) + lo, hi = -1, len(candidates) # lo = -1 means "eta below every level" + while hi - lo > 1: + mid = (lo + hi) // 2 + if _band_accept_probability(t, n, *_band_bounds(local, candidates[mid])) >= confidence: + lo = mid + else: + hi = mid + eta = float(candidates[lo]) if lo >= 0 else -np.inf + + lower, upper = _band_bounds(local, eta) + return t, lower / n, upper / n, local, _band_accept_probability(t, n, lower, upper) + + +def pit_plot(pvals, saveto, confidence=0.95, lattice=None): + """Create a Probability Integral Transform (PIT) plot. + + Plots the empirical CDF of the provided p-values against the expected CDF + for a uniform distribution (the 1:1 diagonal), inside a shaded band that + the ECDF stays wholly within with probability ``confidence`` if the + p-values really are uniform. Any excursion outside the band rejects that + null at significance ``1 - confidence`` -- and the band is *simultaneous*, + so it is the whole curve being read at once, not each point separately. + + The band is built from the exact null distribution of the p-values. The + ECDF only moves where the p-values can land, so rather than constraining + order statistics it constrains the counts there, + ``N_k = #{p_i <= t_k} ~ Binomial(n, F0(t_k))``, giving every point the same + local level and taking that level as large as the target simultaneous level + allows. The boundary follows the real ``n F0(t) (1 - F0(t))`` variance of + the count and pinches to nothing at both corners, so it is far tighter + where miscalibrated p-values pile up -- at n = 100 it is half the width of + a Kolmogorov-Smirnov band at t = 0.05 -- while giving up about 12% near the + median. Worth roughly 1.2x the power of KS against skewed p-values and 2x + against U-shaped or over-central ones. + + Pass ``lattice=L`` when the p-values are uniform on ``{1/L, ..., 1}`` under + H0, as permutation p-values are; ``pted_coverage_test`` supplies it + automatically. Because the band constrains counts rather than order + statistics, ties are exactly what it expects, and it holds its level on + lattices as coarse as ``L = 151`` where an order-statistic band would + reject 60% of valid p-value sets. + + Both the escape probability and the search for the local level are exact, + so the band is deterministic and its achieved level is known rather than + estimated -- it is annotated on the plot alongside the worst local level + the data reached and its simultaneous p-value. Parameters ---------- pvals (array-like): Array of p-values in [0, 1]. saveto (str): File path where the plot will be saved. The format is inferred from the file extension (e.g. ".pdf", ".png"). - confidence (float): Confidence level for the KS confidence band. - Default is 0.95 (95%). + confidence (float): Confidence level for the band. Default is 0.95 + (95%), i.e. the ECDF escapes it 5% of the time under the null. + lattice (Optional[int]): ``L`` if the p-values are uniform on + ``{1/L, ..., 1}`` under H0. None treats them as continuous, which + over-rejects if they are in fact discrete -- a warning fires if + they look it. """ try: import matplotlib.pyplot as plt @@ -358,31 +1306,43 @@ def pit_plot(pvals, saveto, confidence=0.95): sorted_pvals = np.sort(pvals) ecdf = np.arange(1, n + 1) / n - # Critical value for the two-sided KS statistic at the given confidence level. - d_crit = kstwo.ppf(confidence, n) - - # One-sample KS test against U(0,1) for annotation - ks_stat, ks_pval = kstest(pvals, "uniform") + if lattice is None and n - len(np.unique(sorted_pvals)) >= 3: + warn( + f"{n - len(np.unique(sorted_pvals))} of the {n} p-values are duplicates, so they " + f"are discrete, but no lattice was given. Pass lattice=L (permutation p-values " + f"from B permutations are uniform on L = B + 1 points) so the band is built from " + f"the right null; treating them as continuous over-rejects." + ) - x = np.linspace(0, 1, 500) + t, lower, upper, local, achieved = _lattice_band(n, lattice, confidence) + counts = _lattice_counts(sorted_pvals, t)[0] + worst = float(local[np.arange(len(t)), counts].min()) + # The simultaneous p-value is the probability of any point reaching a local + # level this small, which is one minus the acceptance probability at that + # threshold -- the same exact recursion that calibrated the band. + band_pval = 1.0 - _band_accept_probability(t, n, *_band_bounds(local, worst)) fig, ax = plt.subplots() ax.fill_between( - x, - np.maximum(x - d_crit, 0), - np.minimum(x + d_crit, 1), + t, + lower, + upper, + step="post", color="grey", alpha=0.3, linewidth=0, - label=f"{int(confidence * 100)}% KS confidence band", + label=f"{achieved:.1%} simultaneous band" + + ("" if lattice is None else f", lattice of {int(lattice)}"), ) + curve_label = f"Empirical CDF (worst local level={worst:.2g}, p={band_pval:.3f})" + ax.plot([0, 1], [0, 1], "k--", alpha=0.8, label="Expected (Uniform)") ax.step( np.concatenate([[0], sorted_pvals, [1]]), np.concatenate([[0], ecdf, [1]]), where="post", color="#A34F4F", - label=f"Empirical CDF (KS={ks_stat:.3f}, p={ks_pval:.3f})", + label=curve_label, ) ax.set_xlabel("p-value") ax.set_ylabel("Empirical CDF") @@ -394,6 +1354,109 @@ def pit_plot(pvals, saveto, confidence=0.95): plt.close(fig) +def containment_pit_plot( + x, + y, + saveto, + confidence: float = 0.95, + threshold: float = 0.05, + n_landmarks: Optional[int] = None, + rng=None, +): + """Diagnostic plot for :func:`pted.pted_containment_test`. + + Plots the empirical CDF of the per-point depth p-values -- one step per + point of x -- against two reference marks, on a logarithmic p axis because + the whole diagnostic lives at the left edge. + + The **upper bound** is the one-sided simultaneous ceiling for ``n1`` + p-values that really are uniform, the same construction the ordinary PIT + plot uses, with the lower edge dropped. Only the upper edge is meaningful + here: the null is composite, and a genuinely contained x has *deeper* + points than y, so its p-values run large and its curve is expected to sit + well below any uniform reference. That is why the diagonal is not drawn. + + The **vertical line** marks ``threshold``: x points to its left are more + peripheral than all but that fraction of y. + + Read them together rather than as a decision rule. A curve that climbs + above the bound near the floor means x has points where y essentially + cannot reach -- the failure the Fisher aggregate in + :func:`pted.pted_containment_test` is blind to, since a deeply covered bulk + banks enough slack in ``-2 sum log p`` to hide a few escapees. But plenty of + curves cross the bound harmlessly: a tight x concentrates its p-values at + one value, so its ECDF is a step, and wherever that step lands it will + cross a bound drawn for a spread-out sample. Judgement is required, and + that is deliberate -- how much of x may sit how far out is a question about + your model, not about the arithmetic. + + Parameters + ---------- + x, y: the samples, as passed to + :func:`pted.pted_containment_test`. Shape ``(n1, d)``, ``(n2, d)``. + saveto (str): file path; the format follows the extension. + confidence (float): simultaneous level for the upper bound. + threshold (float): where to draw the vertical marker. + n_landmarks: as for :func:`pted.pted_containment_test`. + rng: seed, ``np.random.Generator``, or None. Only picks the landmarks. + """ + prep, _, _ = _prepare_test(x, y, n_landmarks, rng, containment=True) + _draw_containment_pit(prep, saveto, confidence, threshold) + + +def _draw_containment_pit(prep: dict, saveto, confidence: float, threshold: float): + """Draw the containment PIT plot from an already-prepared statistic. + + Takes ``prep`` rather than the samples so that a caller which has just run + the test can reuse it: the distance matrix and the landmark choice are then + literally the same objects the statistic was computed from, rather than an + independent redraw that would describe a different landmark set. + """ + try: + import matplotlib.pyplot as plt + except ImportError: + warn("No containment PIT plot generated! Please install matplotlib.") + return + + n_y = int(prep["n_y"]) + n1 = prep["base_small"].size - n_y # x is whichever group y is not + observed = _containment_curve(prep, prep["base_small"][None, :])[0] / n1 + + # Under H0 the depth p-values are uniform on the (n_y + 1)-point lattice, + # so the ordinary lattice band applies -- upper edge only. + t, _, upper, _, achieved = _lattice_band(n1, n_y + 1, confidence, one_sided=True) + at = np.round(t * (n_y + 1)).astype(int) - 1 + + lattice = np.arange(1, n_y + 2) / (n_y + 1) + fig, ax = plt.subplots() + ax.step( + t, + upper, + where="post", + color="#40658f", + linestyle="--", + linewidth=1.4, + label=f"{achieved:.0%} upper bound, uniform p", + ) + ax.axvline( + threshold, + color="#555555", + linestyle=":", + linewidth=1.4, + label=f"p = {threshold:g}", + ) + ax.step(lattice, observed, where="post", color="#A34F4F", linewidth=2.4, label="x points") + ax.set_xscale("log") + ax.set_xlim([lattice[0] / 1.6, 1.0]) + ax.set_ylim([0, 1]) + ax.set_xlabel("depth p-value of an x point (log scale)") + ax.set_ylabel("fraction of x") + ax.set_title("Containment check") + ax.legend(loc="upper left") + fig.savefig(saveto, bbox_inches="tight") + plt.close(fig) + + def hdp_coverage_test( ground_truth: np.ndarray, posterior_samples: np.ndarray, two_tailed: bool = True ) -> float: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c963c3a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +"""Force a non-interactive matplotlib backend for the whole test suite. + +Several tests call the plotting helpers, and importing ``pyplot`` picks a +backend the first time. Left alone matplotlib prefers an interactive one +wherever it can find a GUI toolkit, so on a machine with tkinter installed but +no usable display -- a Windows CI runner, a headless workstation, an ssh +session -- ``pit_plot`` dies inside Tk rather than writing its file: + + _tkinter.TclError: Can't find a usable init.tcl + +Agg writes files and touches no GUI, which is all these tests want. Set here +rather than in the library, which has no business overriding a backend the +caller chose deliberately. +""" + +try: + import matplotlib + + matplotlib.use("Agg") +except ImportError: # the suite covers the no-matplotlib path too + pass diff --git a/tests/test_pted.py b/tests/test_pted.py index 5acb823..773078b 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -62,15 +62,19 @@ def test_pted_main(): def test_pted_progress_bar(capsys): - pted.pted(np.array([[1, 2], [3, 4]]), np.array([[3, 2], [1, 4]]), permutations=42) + # Large enough that the permutation group dwarfs 42, so 42 draws really are + # taken; a tiny sample would be enumerated instead and the bar would count + # the group rather than the request. + np.random.seed(0) + x = np.random.normal(size=(10, 2)) + y = np.random.normal(size=(10, 2)) + pted.pted(x, y, permutations=42) captured = capsys.readouterr().err assert ( "42/42" not in captured ), "progress bar showed up when prog_bar is set to False by default" - pted.pted( - np.array([[1, 2], [3, 4]]), np.array([[3, 2], [1, 4]]), permutations=42, prog_bar=True - ) + pted.pted(x, y, permutations=42, prog_bar=True) captured = capsys.readouterr().err assert "42/42" in captured, "progress bar did not show when prog_bar is set to True" @@ -121,7 +125,7 @@ def test_pted_coverage_full(backend): @pytest.mark.parametrize("backend", BACKENDS) -def test_pted_chunk(backend): +def test_landmarks(backend): _require_backend(backend) np.random.seed(42) @@ -129,29 +133,31 @@ def test_pted_chunk(backend): D = 10 x = _to_backend(np.random.normal(size=(1000, D)), backend) y = _to_backend(np.random.normal(size=(1000, D)), backend) - p = pted.pted(x, y, chunk_size=100) + p = pted.pted(x, y, n_landmarks=200) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" y = _to_backend(np.random.uniform(size=(1000, D)), backend) - p = pted.pted(x, y, chunk_size=100) + p = pted.pted(x, y, n_landmarks=200) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" @pytest.mark.parametrize("backend", BACKENDS) -def test_pted_chunk_mismatched_sizes(backend): +def test_landmarks_mismatched_sizes(backend): """Chunked PTED works correctly when x and y have different sizes.""" _require_backend(backend) np.random.seed(0) D = 5 - # x has 200 samples, y has 30 samples; chunk_size=50 → nxc=50, nyc=30 landmarks + # x has 200 samples, y has 30 samples; 80 landmarks puts all 30 y in L x = _to_backend(np.random.normal(size=(200, D)), backend) y = _to_backend(np.random.normal(size=(30, D)), backend) - p = pted.pted(x, y, chunk_size=50) + # Pinned: under the null a two-tailed p-value is uniform, so an unpinned + # `p < 0.99` bound fails once in a hundred runs on its own. + p = pted.pted(x, y, n_landmarks=80, rng=0) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" # Different distributions should give small p-value even with mismatched sizes y_diff = _to_backend(np.random.uniform(size=(30, D)), backend) - p = pted.pted(x, y_diff, chunk_size=50) + p = pted.pted(x, y_diff, n_landmarks=80, rng=0) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" @@ -249,37 +255,41 @@ def test_is_jax_array_no_jax(monkeypatch): def test_pted_jax_no_jax(monkeypatch): - """pted raises AssertionError when a jax array is passed but JAX is not installed.""" + """permutation_energy_test raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) + pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) -def test_pted_chunk_jax_no_jax(monkeypatch): - """pted_chunk raises AssertionError when a jax array is passed but JAX is not installed.""" +def test_landmarks_jax_no_jax(monkeypatch): + """permutation_energy_test raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) + pted.utils.permutation_energy_test( + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, n_landmarks=2 + ) def test_pted_torch_no_torch(monkeypatch): - """pted raises AssertionError when a torch tensor is passed but torch is not installed.""" + """permutation_energy_test raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) + pted.utils.permutation_energy_test(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) -def test_pted_chunk_torch_no_torch(monkeypatch): - """pted_chunk raises AssertionError when a torch tensor is passed but torch is not installed.""" +def test_landmarks_torch_no_torch(monkeypatch): + """permutation_energy_test raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) + pted.utils.permutation_energy_test( + np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, n_landmarks=2 + ) @pytest.mark.parametrize("backend", ["torch", "jax"]) @@ -297,3 +307,246 @@ def test_cdist_matches_scipy(backend): pted.utils._cdist(_to_backend(x_np, backend), _to_backend(y_np, backend), backend=backend) ) assert np.allclose(got, expected, rtol=1e-5) + + +# --------------------------------------------------------------------------- +# The new column/batching controls +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_n_landmarks(backend): + """n_landmarks selects how many points every sample is measured against.""" + _require_backend(backend) + np.random.seed(11) + x = _to_backend(np.random.normal(size=(300, 6)), backend) + y = _to_backend(np.random.normal(size=(300, 6)), backend) + + p = pted.pted(x, y, n_landmarks=80) + assert 1e-2 < p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" + + y_diff = _to_backend(np.random.uniform(size=(300, 6)), backend) + assert pted.pted(x, y_diff, n_landmarks=80) < 1e-2 + + # a landmark count covering the pooled sample is just the full test + assert pted.pted(x, y, n_landmarks=10_000, rng=3) == pted.pted(x, y, rng=3) + assert pted.pted(x, y, n_landmarks=600, rng=3) == pted.pted(x, y, rng=3) + + +def test_pted_rng_is_reproducible(): + """An explicit rng pins the permutations; the global seed still works when + none is given.""" + x = np.random.normal(size=(60, 4)) + y = np.random.normal(size=(60, 4)) + + a = pted.pted(x, y, permutations=200, n_landmarks=30, rng=7, return_all=True) + b = pted.pted(x, y, permutations=200, n_landmarks=30, rng=7, return_all=True) + assert a[0] == b[0] and np.array_equal(a[1], b[1]) and a[2] == b[2] + + np.random.seed(4) + c = pted.pted(x, y, permutations=200, n_landmarks=30, return_all=True) + np.random.seed(4) + d = pted.pted(x, y, permutations=200, n_landmarks=30, return_all=True) + assert np.array_equal(c[1], d[1]), "np.random.seed should still pin the permutations" + + +@pytest.mark.parametrize( + "n1,n2,n_landmarks,regime", + [ + (60, 60, 24, "proportional"), + (1, 100, 11, "singleton"), + (1, 40, 30, "singleton"), # c > n/2, so the lone point sits inside C + ], +) +def test_landmark_pvalues_are_calibrated(n1, n2, n_landmarks, regime): + """Type-I error under H0 must match the nominal rate. + + This is the property the landmark test exists to preserve, and the one the + earlier landmark-reshuffling scheme lost: because it re-split the columns + by group after each permutation while the observed statistic always used a + perfectly balanced split, the observed value was not exchangeable with the + permuted ones. It rejected at 29% (proportional) and 84% (singleton) for a + nominal 5%. Fixed seed, so this is deterministic rather than flaky. + """ + from pted.utils import _allocate, _as_rng + + assert _allocate(n1, n2, n_landmarks, _as_rng(0))["regime"] == regime + + trials, permutations = 600, 49 + rng = np.random.default_rng(20240904) + pvals = np.empty(trials) + for t in range(trials): + # H0 is true: both groups come from the same distribution + x = rng.standard_normal((n1, 4)) + y = rng.standard_normal((n2, 4)) + pvals[t] = pted.pted( + x, y, permutations=permutations, n_landmarks=n_landmarks, two_tailed=False, rng=rng + ) + + for nominal, tol in ((0.10, 0.05), (0.20, 0.06)): + rate = float(np.mean(pvals <= nominal)) + assert ( + abs(rate - nominal) < tol + ), f"{regime}: rejected {rate:.3f} of {trials} null trials at nominal {nominal}" + + +def test_coverage_test_rng_is_not_shared_across_simulations(): + """A bare seed must not hand every simulation the same permutation draws, + while the run as a whole stays reproducible from that seed.""" + g = np.random.default_rng(0).standard_normal((6, 3)) + s = np.random.default_rng(1).standard_normal((40, 6, 3)) + + _, permute, _ = pted.pted_coverage_test(g, s, permutations=50, rng=7, return_all=True) + assert not any(np.array_equal(permute[0], permute[i]) for i in range(1, len(permute))) + + _, again, _ = pted.pted_coverage_test(g, s, permutations=50, rng=7, return_all=True) + assert np.array_equal(permute, again) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_coverage_test_with_landmarks(backend): + """Landmark coverage runs end to end and still separates a calibrated + posterior from an over/under-confident one.""" + _require_backend(backend) + rng = np.random.default_rng(3) + nsim, nsamp, d = 40, 300, 3 + g, draws = [], [] + for _ in range(nsim): + loc = rng.standard_normal(d) * 5 + scale = rng.uniform(1, 4, size=d) + g.append(rng.normal(loc, scale, size=d)) + draws.append(rng.normal(loc, scale, size=(nsamp, d))) + g = _to_backend(np.array(g), backend) + ok = _to_backend(np.stack(draws, axis=1), backend) + over = _to_backend( + np.stack([v.mean(0) + (v - v.mean(0)) * 0.4 for v in draws], axis=1), backend + ) + + p_ok = pted.pted_coverage_test(g, ok, permutations=199, n_landmarks=30) + assert 1e-2 < p_ok < 0.99, f"calibrated posterior gave p={p_ok}" + + p_over = pted.pted_coverage_test( + g, over, permutations=199, n_landmarks=30, warn_confidence=None + ) + assert p_over < 1e-2, f"overconfident posterior gave p={p_over}" + + +# --------------------------------------------------------------------------- +# Containment: does y cover x? +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_containment_direction(backend): + """A tighter sample is contained; a broader, shifted or leaky one is not.""" + _require_backend(backend) + rng = np.random.default_rng(0) + y = _to_backend(rng.standard_normal((400, 6)), backend) + + def p(x): + return pted.pted_containment_test(_to_backend(x, backend), y, rng=1) + + assert p(rng.standard_normal((80, 6)) * 0.5) > 0.1, "a tighter sample is contained" + assert p(rng.standard_normal((80, 6)) * 1.7) < 1e-2, "a broader one is not" + assert p(rng.standard_normal((80, 6)) + 1.2) < 1e-2, "a shifted one is not" + + leaky = rng.standard_normal((80, 6)) + leaky[:8] += 5.0 + assert p(leaky) < 1e-2, "a few escaped points are not contained" + + +def test_containment_is_not_symmetric(): + """The whole point: asking it the other way round must give a different + answer. The energy distance, and the mean depth it reduces to, cannot -- + both are invariant under swapping the two samples.""" + rng = np.random.default_rng(3) + broad = rng.standard_normal((100, 4)) * 1.7 + tight = rng.standard_normal((100, 4)) + + assert pted.pted_containment_test(broad, tight, rng=2) < 1e-2, "broad is not inside tight" + assert pted.pted_containment_test(tight, broad, rng=2) > 0.1, "tight is inside broad" + + # the symmetric energy test cannot separate these + assert pted.pted(broad, tight, rng=2) == pted.pted(tight, broad, rng=2) + + +def test_containment_null_is_calibrated(): + """Under x and y sharing a distribution the p-value is uniform, and a + genuinely contained x is conservative rather than merely non-significant. + Fixed seed, so this is deterministic rather than flaky.""" + trials, permutations = 400, 99 + rng = np.random.default_rng(20260908) + null, contained = np.empty(trials), np.empty(trials) + for t in range(trials): + y = rng.standard_normal((150, 4)) + null[t] = pted.pted_containment_test( + rng.standard_normal((40, 4)), y, permutations=permutations, rng=rng + ) + contained[t] = pted.pted_containment_test( + rng.standard_normal((40, 4)) * 0.5, y, permutations=permutations, rng=rng + ) + rate = float(np.mean(null <= 0.05)) # se = 0.011 + assert abs(rate - 0.05) < 0.04, f"null rejection rate {rate}" + assert float(np.mean(contained <= 0.05)) < 0.02, "a contained sample must not reject" + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_containment_with_landmarks(backend): + """Landmarks work here too, and both samples must keep at least one.""" + _require_backend(backend) + rng = np.random.default_rng(5) + y = _to_backend(rng.standard_normal((300, 5)), backend) + x_out = _to_backend(rng.standard_normal((60, 5)) * 1.8, backend) + x_in = _to_backend(rng.standard_normal((60, 5)) * 0.5, backend) + assert pted.pted_containment_test(x_out, y, n_landmarks=90, rng=1) < 1e-2 + assert pted.pted_containment_test(x_in, y, n_landmarks=90, rng=1) > 0.1 + + +def test_containment_returns_all(): + rng = np.random.default_rng(7) + x, y = rng.standard_normal((30, 3)), rng.standard_normal((90, 3)) + stat, perm, p = pted.pted_containment_test(x, y, permutations=50, return_all=True, rng=0) + assert np.isfinite(stat) and len(perm) == 50 + assert p == (1.0 + np.sum(perm >= stat)) / 51.0 + + +def test_containment_pit_plot(tmp_path): + """The plot is written, and its curve shows the failure mode the Fisher + aggregate is blind to.""" + from pted.utils import ( + _allocate, + _as_rng, + _cdist, + _containment_curve, + _lattice_band, + _prepare_containment, + ) + + rng = np.random.default_rng(0) + y = rng.standard_normal((600, 5)) * 8.0 + leaky = rng.standard_normal((200, 5)) + leaky[:10] += np.r_[40.0, np.zeros(4)] # 5% of x where y cannot reach + + out = tmp_path / "containment.pdf" + p = pted.pted_containment_test(leaky, y, permutations=299, rng=1, pit_plot=str(out)) + assert out.exists() + assert p > 0.05, "the documented blind spot: Fisher does not see the escapees" + + def curve(x): + z = np.vstack([x, y]) + alloc = _allocate(len(x), len(y), len(z), _as_rng(0)) + prep = _prepare_containment(_cdist(z, z, "numpy"), alloc, "numpy", True) + return _containment_curve(prep, prep["base_small"][None, :])[0] / len(x) + + # ...but they lift the curve off the floor at the smallest p-value, above + # the uniform bound, where a contained sample stays pinned at zero. + n_y = len(y) + t, _, upper, _, _ = _lattice_band(len(leaky), n_y + 1, 0.95, one_sided=True) + at = np.round(t * (n_y + 1)).astype(int) - 1 + leaky_curve, tight_curve = curve(leaky), curve(rng.standard_normal((200, 5))) + + assert leaky_curve[0] > 0.0, "escapees sit at the p-value floor" + assert tight_curve[0] == 0.0, "a contained sample has nothing at the floor" + near_floor = t <= 0.01 + assert np.any(leaky_curve[at][near_floor] > upper[near_floor]), "escapees clear the bound" + assert not np.any(tight_curve[at][near_floor] > upper[near_floor]) diff --git a/tests/test_utils.py b/tests/test_utils.py index aac3f3a..e571f0e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,16 +1,41 @@ import sys, os +import warnings +from math import comb import numpy as np +from scipy.spatial.distance import cdist + +from scipy.stats import kstwo from pted.utils import ( + _as_rng, + _band_accept_probability, + _lattice_band, + _sampled_label_blocks, + _lattice_counts, + _enumerate_label_blocks, + PermutationResolutionWarning, + _allocate, two_tailed_p, simulation_based_calibration_histogram, pit_plot, hdp_coverage_test, - _to_scalar, - _random_permutation, + permutation_energy_test, + _cdist, + _label_drawer, + _BATCH_BYTES, + _index_like, + _landmark_columns, + _prepare_test, + _prepare_statistic, ) + +def _evaluate_statistic(prep, U): + """The statistic for a batch of labellings, through this prep's evaluator.""" + return prep["evaluate"](prep, U) + + try: import torch except ImportError: @@ -25,6 +50,15 @@ import pytest +BACKENDS = ["numpy", "torch", "jax"] + + +def _require_backend(backend): + if backend == "torch" and torch is None: + pytest.skip("torch not installed") + if backend == "jax" and jax is None: + pytest.skip("jax not installed") + def test_two_tailed_p(): @@ -80,41 +114,608 @@ def test_hdp_coverage_test(): assert pvalue > 0.01, "p-value should not be small for underconfident and one_tailed test" -@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) -def test_to_scalar(backend): - if backend == "torch" and torch is None: - pytest.skip("torch not installed") - if backend == "jax" and jax is None: - pytest.skip("jax not installed") - - if backend == "numpy": - x = np.array(5) - - elif backend == "torch": - x = torch.tensor(5) - - elif backend == "jax": - x = jnp.array(5) - - # Test with scalar - assert _to_scalar(x, backend) == 5 - - -@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) -def test_random_permutation(backend): - if backend == "torch" and torch is None: - pytest.skip("torch not installed") - if backend == "jax" and jax is None: - pytest.skip("jax not installed") - - D = np.arange(16).reshape(4, 4) +# --------------------------------------------------------------------------- +# Rectangular / batched permutation backend +# --------------------------------------------------------------------------- + + +def _unbiased_energy(x, y): + """Energy statistic from first principles, within-group means over + distinct pairs only.""" + n1, n2 = len(x), len(y) + dxx, dyy = cdist(x, x), cdist(y, y) + exx = (dxx.sum() - np.trace(dxx)) / (n1 * (n1 - 1)) if n1 > 1 else 0.0 + eyy = (dyy.sum() - np.trace(dyy)) / (n2 * (n2 - 1)) if n2 > 1 else 0.0 + return (n1 * n2 / (n1 + n2)) * (2 * cdist(x, y).mean() - exx - eyy) + + +def _brute_block_means(z, small_indicator, landmarks, n): + """Energy statistic by explicit block sums over the rectangular matrix.""" + D = cdist(z, z[landmarks]) + is_s = small_indicator.astype(bool) + rows_s, rows_l = np.flatnonzero(is_s), np.flatnonzero(~is_s) + ks, kl = np.flatnonzero(is_s[landmarks]), np.flatnonzero(~is_s[landmarks]) + n_s, n_l = len(rows_s), len(rows_l) + + def block(rows, kk, drop_self): + tot = cnt = 0 + for i in rows: + for k in kk: + if drop_self and landmarks[k] == i: + continue + tot += D[i, k] + cnt += 1 + return tot, cnt + + s_ss, c_ss = block(rows_s, ks, True) + s_ll, c_ll = block(rows_l, kl, True) + s_sl, c_sl = block(rows_s, kl, False) + s_ls, c_ls = block(rows_l, ks, False) + mu_ss = 0.0 if c_ss == 0 else s_ss / c_ss + mu_ll = 0.0 if c_ll == 0 else s_ll / c_ll + parts = [s_sl / c_sl] + ([s_ls / c_ls] if c_ls else []) + return (n_s * n_l / n) * (2 * sum(parts) / len(parts) - mu_ss - mu_ll) + + +ALLOCATIONS = [ + # n1, n2, n_landmarks, expected regime + (100, 100, 200, "full"), + (3, 40, 43, "full"), + (1, 60, 12, "singleton"), + (1, 200, 20, "singleton"), + (6, 80, 40, "small_group_in_L"), + (40, 50, 30, "proportional"), + (40, 50, 3, "proportional"), +] + +LANDMARK_ALLOCATIONS = [a for a in ALLOCATIONS if a[3] != "full"] + + +@pytest.mark.parametrize("n1,n2,c,regime", LANDMARK_ALLOCATIONS) +def test_allocate_landmarks_regimes(n1, n2, c, regime): + """Each size combination lands in the intended regime with a consistent + set of columns.""" + alloc = _allocate(n1, n2, c, _as_rng(0)) + landmarks = alloc["landmarks"] + n = n1 + n2 + + assert alloc["regime"] == regime + assert landmarks.shape == (c,) + assert len(np.unique(landmarks)) == c, "columns must be distinct points" + assert landmarks.min() >= 0 and landmarks.max() < n + assert np.all(np.diff(landmarks) > 0), "columns are returned sorted" + + # n_small_landmarks must be the number of columns actually belonging to the small group + assert np.sum(np.isin(landmarks, alloc["small_idx"])) == alloc["n_small_landmarks"] + assert alloc["n_small_landmarks"] + alloc["n_large_landmarks"] == c + assert alloc["n_large_landmarks"] >= 1, "the large group must keep at least one column" + + +@pytest.mark.parametrize("n1,n2,c,regime", [a for a in ALLOCATIONS if a[3] == "full"]) +def test_full_allocation_carries_no_landmark_fields(n1, n2, c, regime): + """The full regime has no landmark set, so it does not describe one: the + statistic, the subgroup and the distance matrix are all built without.""" + alloc = _allocate(n1, n2, c, _as_rng(0)) + assert alloc["regime"] == "full" + assert not [k for k in alloc if "landmark" in k] + exact = comb(n1 + n2, min(n1, n2)) + assert alloc["reference_size"] in (exact, float("inf")) + + z = np.random.default_rng(0).standard_normal((n1 + n2, 3)) + assert _landmark_columns(z, alloc, "numpy") is z, "no column gather" + + +def test_allocate_landmarks_reference_size(): + """reference_size counts the label assignments the subgroup can reach.""" + for n1, n2, m, _ in LANDMARK_ALLOCATIONS: + alloc = _allocate(n1, n2, m, _as_rng(1)) + n = n1 + n2 + n_small, n_small_landmarks = alloc["n_small"], alloc["n_small_landmarks"] + expected = comb(m, n_small_landmarks) * comb(n - m, n_small - n_small_landmarks) + got = alloc["reference_size"] + # counted exactly while it is small enough to matter, inf beyond that + assert got == expected or (got == float("inf") and expected > 10**15) + + +def test_sample_and_landmark_counts_are_validated(): + """The allocators assume valid input; permutation_energy_test is the gate.""" + x = np.zeros((5, 2)) + with pytest.raises(ValueError, match="at least 2 landmarks"): + permutation_energy_test(x, x, permutations=1, n_landmarks=1) + with pytest.raises(ValueError, match="both samples need at least one point"): + permutation_energy_test(np.zeros((0, 2)), x, permutations=1) + # more landmarks than points is the full matrix, not an error + stat, perm = permutation_energy_test(x, x, permutations=4, n_landmarks=10_000, rng=0) + assert np.isfinite(stat) + + +def _prep_for(n1, n2, c, seed=3, backend="numpy"): + """An allocation plus its prepared statistic, on the given backend.""" + rng = np.random.default_rng(seed) + alloc = _allocate(n1, n2, c, rng) + z = rng.standard_normal((n1 + n2, 4)) if backend == "torch": - D = torch.tensor(D) + z = torch.tensor(z) elif backend == "jax": - D = jnp.array(D) - - permuted_D = _random_permutation(D, backend=backend) - - assert set(np.array(permuted_D.flatten()).tolist()) == set( - np.array(D.flatten()).tolist() - ), "Permutation should contain all original elements" + z = jnp.array(z) + zc = _landmark_columns(z, alloc, backend) + return alloc, _prepare_statistic(_cdist(z, zc, backend), alloc, backend) + + +@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) +def test_draw_labels_stays_in_subgroup(n1, n2, c, regime): + """Permutations never move a label across the L / L^c boundary, so the + per-group landmark counts -- and hence every normalising constant -- are + fixed. This is what makes the observed labelling exchangeable with the + permuted ones.""" + alloc, prep = _prep_for(n1, n2, c) + U = np.asarray(_label_drawer(prep, np.random.default_rng(17))(200)) + + assert np.all(np.isin(U, [0.0, 1.0])) + assert np.all(U.sum(1) == alloc["n_small"]), "small group size is preserved" + if alloc["regime"] != "full": + assert np.all( + U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"] + ), "column counts are preserved" + # and the labels really do move around within each part + if alloc["reference_size"] > 100: + assert len(np.unique(U, axis=0)) > 1 + + +@pytest.mark.parametrize("n1,n2,c,regime", ALLOCATIONS) +def test_statistic_matches_brute_force_block_means(n1, n2, c, regime): + """The bilinear-form shortcut agrees with explicit block sums, for the + observed labelling and for permuted ones.""" + rng = np.random.default_rng(7) + x = rng.standard_normal((n1, 4)) + y = rng.standard_normal((n2, 4)) + 0.4 + z = np.vstack([x, y]) + + alloc = _allocate(n1, n2, c, rng) + # the full regime keeps no landmark index; its columns are every point + columns = np.arange(n1 + n2) if regime == "full" else alloc["landmarks"] + prep = _prepare_statistic(cdist(z, z[columns]), alloc, "numpy") + + U = np.concatenate( + [ + prep["base_small"][None, :], + _label_drawer(prep, np.random.default_rng(11))(5), + ] + ) + got = _evaluate_statistic(prep, U) + expect = [_brute_block_means(z, U[b], columns, n1 + n2) for b in range(len(U))] + assert np.allclose(got, expect, atol=1e-9) + + +@pytest.mark.parametrize("n1,n2", [(30, 30), (7, 23), (1, 40), (2, 2)]) +def test_full_matrix_is_the_energy_distance(n1, n2): + """With every point a column, the statistic is exactly the energy + distance with within-group means over distinct pairs.""" + rng = np.random.default_rng(11) + x = rng.standard_normal((n1, 3)) + y = rng.standard_normal((n2, 3)) + 0.6 + test_stat, _ = permutation_energy_test(x, y, permutations=0) + assert np.isclose(test_stat, _unbiased_energy(x, y), atol=1e-9) + + +@pytest.mark.parametrize("backend", ["torch", "jax"]) +def test_statistic_backend_agreement(backend): + """numpy, torch and jax evaluate the same statistic for the same labels.""" + _require_backend(backend) + rng = np.random.default_rng(5) + n1, n2, c = 40, 60, 30 + x = rng.standard_normal((n1, 5)) + y = rng.standard_normal((n2, 5)) + 0.3 + z = np.vstack([x, y]) + + alloc = _allocate(n1, n2, c, rng) + U = None + results = {} + for be in ("numpy", backend): + zb = {"numpy": lambda: z, "torch": lambda: torch.tensor(z), "jax": lambda: jnp.array(z)}[ + be + ]() + D = _cdist(zb, zb[_index_like(alloc["landmarks"], zb, be)], be) + prep = _prepare_statistic(D, alloc, be) + if U is None: + U = np.concatenate( + [ + prep["base_small"][None, :], + _label_drawer(prep, np.random.default_rng(13))(8), + ] + ) + results[be] = _evaluate_statistic(prep, U) + # jax defaults to float32, so compare with a float32-appropriate tolerance + assert np.allclose(results["numpy"], results[backend], rtol=1e-4, atol=1e-4) + + +@pytest.mark.parametrize("backend", ["torch", "jax"]) +@pytest.mark.parametrize("n1,n2,c", [(40, 50, 30), (6, 80, 40), (1, 60, 12)]) +def test_device_draw_labels_stays_in_subgroup(backend, n1, n2, c): + """The on-device draw obeys the same subgroup constraints as the numpy + one -- the labels moved to the accelerator, the guarantees did not.""" + _require_backend(backend) + alloc, prep = _prep_for(n1, n2, c, backend=backend) + U = np.asarray(_label_drawer(prep, np.random.default_rng(23))(128)) + + assert np.all(np.isin(U, [0.0, 1.0])) + assert np.all(U.sum(1) == alloc["n_small"]) + assert np.all(U[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"]) + if alloc["reference_size"] > 100: + assert len(np.unique(U, axis=0)) > 1 + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_label_draws_are_reproducible(backend): + """A fixed rng pins the draw, and a run yields exactly `permutations` valid + rows in blocks no larger than batch_size.""" + _require_backend(backend) + _, prep = _prep_for(20, 30, 50, backend=backend) + + a = np.asarray(_label_drawer(prep, np.random.default_rng(5))(40)) + b = np.asarray(_label_drawer(prep, np.random.default_rng(5))(40)) + assert np.array_equal(a, b), "the same rng must give the same draw" + + for batch in (7, 40, 1000): + rng = np.random.default_rng(5) + blocks = [np.asarray(x) for x in _sampled_label_blocks(prep, 40, batch, rng)] + assert sum(len(x) for x in blocks) == 40 + assert max(len(x) for x in blocks) <= batch + rows = np.concatenate(blocks) + assert np.all(rows.sum(1) == prep["base_small"].sum()) + + +def test_evaluate_is_independent_of_batch_size(): + """Splitting one set of labellings across batches must not change their + statistics, beyond the reordering a different matmul shape costs in + floating point. (Which labellings get drawn does depend on batch_size -- + the draw is a running stream -- but every batching is an exact test.)""" + x = np.random.default_rng(0).standard_normal((50, 4)) + y = np.random.default_rng(1).standard_normal((70, 4)) + alloc, prep = _prep_for(50, 70, 30) + U = _label_drawer(prep, np.random.default_rng(2))(64) + + whole = _evaluate_statistic(prep, U) + for batch in (1, 7, 64): + pieces = np.concatenate( + [_evaluate_statistic(prep, U[s : s + batch]) for s in range(0, len(U), batch)] + ) + assert np.allclose(pieces, whole, rtol=1e-10, atol=1e-12), f"batch_size={batch}" + + +def test_every_batch_size_gives_a_valid_test(): + """batch_size stays a free knob: any value returns the requested number of + null statistics and a p-value in range.""" + x = np.random.default_rng(0).standard_normal((50, 4)) + y = np.random.default_rng(1).standard_normal((70, 4)) + for batch in (1, 7, 64, 1000): + stat, perm = permutation_energy_test( + x, y, permutations=64, n_landmarks=30, rng=12345, batch_size=batch + ) + assert len(perm) == 64 and np.all(np.isfinite(perm)) + + +def test_n_landmarks_covering_the_sample_is_the_full_test(): + """None, or any count at or above the pooled size, is the exact test.""" + x = np.random.default_rng(0).standard_normal((20, 3)) + y = np.random.default_rng(1).standard_normal((30, 3)) + ref = permutation_energy_test(x, y, permutations=32, rng=5) + for m in (50, 500): + stat, perm = permutation_energy_test(x, y, permutations=32, n_landmarks=m, rng=5) + assert stat == ref[0] and np.array_equal(perm, ref[1]) + + +def test_singleton_keeps_its_point_out_of_the_landmarks(): + """A lone point has no within-group pairs, so leaving it out of L costs + nothing and its label roams over the n - m positions outside. The reference + set therefore shrinks as m grows -- which is why landmarks are for m << n, + and why the resolution warning exists.""" + n2 = 200 + n = 1 + n2 + for m in range(2, n): + alloc = _allocate(1, n2, m, _as_rng(0)) + assert alloc["regime"] == "singleton" + assert alloc["n_small_landmarks"] == 0 + assert alloc["reference_size"] == n - m + assert 0 not in alloc["landmarks"], "the lone point is never a landmark" + + +def test_reference_size_is_inf_when_astronomical(): + """Only smallness matters, so the count is not carried past what anyone + would read.""" + from pted.utils import _reference_size + + assert _reference_size(20, 20, 10, 10) == comb(20, 10) + assert _reference_size(201, 20, 1, 0) == 181 + assert _reference_size(2000, 1000, 500, 250) == float("inf") + + +def test_permutation_resolution_warning(): + """Column counts that starve the permutation group are flagged.""" + rng = np.random.default_rng(2) + + # singleton: only a genuinely tiny pooled sample is short of assignments now + with pytest.warns(PermutationResolutionWarning, match="single point"): + permutation_energy_test( + rng.standard_normal((1, 3)), + rng.standard_normal((10, 3)), + permutations=100, + n_landmarks=5, + ) + + # small_group_in_L: too FEW columns is the failure mode here + with pytest.warns(PermutationResolutionWarning, match="use more landmarks"): + permutation_energy_test( + rng.standard_normal((2, 3)), + rng.standard_normal((30, 3)), + permutations=100, + n_landmarks=5, + ) + + # a singleton with too many landmarks now warns rather than flipping sides + with pytest.warns(PermutationResolutionWarning, match="use fewer landmarks"): + permutation_energy_test( + rng.standard_normal((1, 3)), + rng.standard_normal((60, 3)), + permutations=100, + n_landmarks=55, + ) + + # a sensible landmark count is quiet + with warnings.catch_warnings(): + warnings.simplefilter("error", PermutationResolutionWarning) + permutation_energy_test( + rng.standard_normal((1, 3)), + rng.standard_normal((60, 3)), + permutations=100, + n_landmarks=8, + ) + + +# --------------------------------------------------------------------------- +# PIT plot confidence bands +# --------------------------------------------------------------------------- + + +def _band_rejects(pvals, t, lower, upper): + counts = _lattice_counts(np.asarray(pvals), t)[0] / len(pvals) + return bool(np.any((counts < lower - 1e-12) | (counts > upper + 1e-12))) + + +def _ks_rejects(pvals, d_crit): + n = len(pvals) + s = np.sort(pvals) + i = np.arange(1, n + 1) + return max(np.max(i / n - s), np.max(s - (i - 1) / n)) > d_crit + + +def test_lattice_band_is_deterministic_and_pinches_at_the_corners(): + """The band tracks the binomial variance of the ECDF count, so it is far + tighter at the ends than a constant-half-width KS band.""" + n, confidence = 100, 0.95 + t, lower, upper, _, _ = _lattice_band(n, 151, confidence) + again, lower2, _, _, _ = _lattice_band(n, 151, confidence) + assert np.array_equal(t, again) and np.array_equal(lower, lower2) + + assert np.all(lower <= upper) + half = (upper - lower) / 2 + assert half[0] < half[len(t) // 2] and half[-1] < half[len(t) // 2], "must pinch at the ends" + assert half[np.searchsorted(t, 0.05)] < kstwo.ppf(confidence, n) + + +@pytest.mark.parametrize("n,lattice", [(100, 151), (100, 1000), (60, 41), (100, None)]) +def test_lattice_band_level_is_exact(n, lattice): + """The escape probability is computed by a forward recursion over the + counting process, not estimated, so the band's level is known exactly -- + and it lands on the target rather than being conservative.""" + confidence = 0.95 + _, _, _, _, achieved = _lattice_band(n, lattice, confidence) + assert abs((1 - achieved) - (1 - confidence)) < 0.002, f"achieved {1 - achieved}" + + +def test_band_recursion_matches_brute_force(): + """The forward recursion agrees with simulating the null directly.""" + n, L, trials = 40, 61, 40000 + t, lo, hi, _, _ = _lattice_band(n, L, 0.95) + lower, upper = np.round(lo * n).astype(int), np.round(hi * n).astype(int) + exact = _band_accept_probability(t, n, lower, upper) + + rng = np.random.default_rng(0) + hits = 0 + for _ in range(0, trials, 4000): + p = rng.integers(1, L + 1, size=(4000, n)) / L + counts = _lattice_counts(p, t) + hits += int(np.sum(np.all((counts >= lower) & (counts <= upper), axis=1))) + mc = hits / trials + assert abs(exact - mc) < 4 * np.sqrt(mc * (1 - mc) / trials), f"{exact} vs {mc}" + + +def test_lattice_band_holds_level_against_simulation(): + """Sanity check the level end to end, on the coarse lattice a real + coverage test produces -- where an order-statistic band rejects ~60%.""" + n, lattice, confidence, trials = 100, 151, 0.95, 4000 + t, lower, upper, _, _ = _lattice_band(n, lattice, confidence) + rng = np.random.default_rng(3) + hits = 0 + for _ in range(trials): + p = rng.integers(1, lattice + 1, size=n) / lattice + hits += _band_rejects(p, t, lower, upper) + rate = hits / trials # se = 0.0034 + assert abs(rate - (1 - confidence)) < 0.015, f"rejected {rate}" + + +def test_lattice_band_beats_ks_on_tail_deviations(): + """More power against the p-value shapes a miscalibrated posterior makes, + on the coarse lattice a real coverage test produces.""" + n, L, trials = 100, 151, 1500 + t, lower, upper, _, _ = _lattice_band(n, L, 0.95) + d_crit = kstwo.ppf(0.95, n) + rng = np.random.default_rng(4) + for a, b in ((0.7, 1.0), (0.85, 0.85)): # skewed low, and U-shaped + draws = [np.ceil(rng.beta(a, b, n) * L) / L for _ in range(trials)] + ks = np.mean([_ks_rejects(p, d_crit) for p in draws]) + lat = np.mean([_band_rejects(p, t, lower, upper) for p in draws]) + assert lat > ks, f"Beta({a},{b}): lattice {lat:.3f} did not beat KS {ks:.3f}" + + +def test_pit_plot_writes_a_file(tmp_path): + rng = np.random.default_rng(0) + for label, pvals, lattice in ( + ("continuous", rng.random(100), None), + ("lattice", rng.integers(1, 152, size=100) / 151, 151), + ): + out = tmp_path / f"pit_{label}.pdf" + pit_plot(pvals, str(out), lattice=lattice) + assert out.exists() + + +def test_pit_plot_warns_when_discrete_pvalues_have_no_lattice(tmp_path): + """Discrete p-values need their lattice declared; without it the band is + built from the wrong null. Saying so is better than silently over-rejecting.""" + rng = np.random.default_rng(1) + coarse = rng.integers(1, 201, size=100) / 200 # 199 permutations + + with pytest.warns(UserWarning, match="no lattice was given"): + pit_plot(coarse, str(tmp_path / "coarse.pdf")) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + pit_plot(coarse, str(tmp_path / "told.pdf"), lattice=200) + pit_plot(rng.random(100), str(tmp_path / "cont.pdf")) + + +# --------------------------------------------------------------------------- +# Exhaustive enumeration of small permutation groups +# --------------------------------------------------------------------------- + + +def test_small_groups_are_enumerated_not_sampled(): + """Asking for more permutations than the subgroup has members walks the + whole group instead, returning reference_size - 1 null statistics.""" + x = np.random.default_rng(0).standard_normal((1, 3)) + y = np.random.default_rng(1).standard_normal((60, 3)) + reference = _allocate(1, 60, 61, _as_rng(0))["reference_size"] + assert reference == 61 + + _, permute = permutation_energy_test(x, y, permutations=1000, rng=0) + assert len(permute) == reference - 1, "should have enumerated the group" + + # ask for fewer than the group holds and it samples, as before + _, permute = permutation_energy_test(x, y, permutations=30, rng=0) + assert len(permute) == 30 + + +def test_enumeration_covers_the_group_exactly_once(): + """Every assignment the subgroup reaches appears once, bar the observed.""" + n1, n2, m = 1, 12, 5 + alloc = _allocate(n1, n2, m, _as_rng(0)) + z = np.random.default_rng(0).standard_normal((n1 + n2, 2)) + prep = _prepare_statistic(cdist(z, z[alloc["landmarks"]]), alloc, "numpy") + + rows = np.concatenate(list(_enumerate_label_blocks(prep, 4))) + assert len(rows) == alloc["reference_size"] - 1 + assert len(np.unique(rows, axis=0)) == len(rows), "no assignment repeats" + assert np.all(rows.sum(1) == alloc["n_small"]), "group size preserved" + assert np.all(rows[:, alloc["landmarks"]].sum(1) == alloc["n_small_landmarks"]) + assert not np.any(np.all(rows == prep["base_small"], axis=1)), "observed excluded" + + +def test_enumerated_pvalues_are_exactly_lattice_uniform(): + """Enumeration removes the tie inflation that sampling a small group + causes, so P(p <= t) sits on the lattice instead of below it.""" + from pted.pted import pted as pted_api + + nsamp, trials = 40, 6000 # group of 41, so p lands on multiples of 1/41 + rng = np.random.default_rng(5) + pvals = np.empty(trials) + for t in range(trials): + loc, sd = rng.standard_normal(2) * 3, rng.uniform(1, 3, size=2) + pvals[t] = pted_api( + rng.normal(loc, sd, size=(1, 2)), + rng.normal(loc, sd, size=(nsamp, 2)), + permutations=500, + two_tailed=False, + rng=rng, + ) + R = nsamp + 1 + assert len(np.unique(pvals)) <= R + # a valid p-value on an R-point lattice has P(p <= t) = floor(tR)/R + for t in (0.1, 0.2, 0.5): + expected = np.floor(t * R) / R + observed = np.mean(pvals <= t) + assert abs(observed - expected) < 4 * np.sqrt( + expected * (1 - expected) / trials + ), f"P(p<={t}) was {observed:.4f}, lattice value is {expected:.4f}" + assert observed <= t + 1e-9, "must stay valid" + + +def test_lattice_band_one_sided_lowers_the_ceiling(): + """Dropping the floor spends the whole error budget upward, so the ceiling + comes down -- which is what makes it worth using where only an excursion + above is evidence.""" + _, _, two_sided, _, _ = _lattice_band(100, 201, 0.95) + _, lower, one_sided, _, achieved = _lattice_band(100, 201, 0.95, one_sided=True) + assert np.all(lower == 0.0), "one-sided means no floor" + assert np.all(one_sided <= two_sided) + assert np.any(one_sided < two_sided), "and strictly lower somewhere" + assert abs((1 - achieved) - 0.05) < 0.01, f"level was {1 - achieved}" + + +# --------------------------------------------------------------------------- +# Batch sizing and setup reuse +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "n1,n2,m", + [ + (1, 100_000, 10), # huge sample, few landmarks: rows x n is the cost + (2_000, 3_000, 50), + (200, 300, None), # full matrix + (50, 50, 8), + ], +) +@pytest.mark.parametrize("containment", [False, True]) +def test_auto_batch_size_stays_within_the_memory_budget(n1, n2, m, containment): + """No batch-scaling tensor may exceed the byte budget, whatever the shape. + + Every regime holds at least one (rows, n) tensor, so bounding that one is + a necessary condition and does not restate the per-regime element counts. + Sizing off the matrix product alone misses this entirely: n * m is small + exactly when n is large and m is tiny.""" + prep, _, _ = _prepare_test(np.zeros((n1, 2)), np.zeros((n2, 2)), m, 0, containment) + n = n1 + n2 + held = prep["batch_size"] * n * prep["ref"].dtype.itemsize + assert held <= _BATCH_BYTES, f"a batch would hold {held / 2**30:.2f} GiB of (rows, n)" + assert prep["batch_size"] >= 1 + + +def test_containment_plot_reuses_the_test_setup(tmp_path, monkeypatch): + """Asking for the plot must not re-run the allocation or the distance + matrix. Doing so cost a second O(n*m*d) build, and -- because _as_rng + passes a Generator through untouched -- drew landmarks from an already + advanced stream, so the figure described a different landmark set than the + statistic it was meant to explain.""" + import pted.utils as U + from pted import pted_containment_test + + seen = [] + original = U._allocate + monkeypatch.setattr(U, "_allocate", lambda *a, **k: seen.append(original(*a, **k)) or seen[-1]) + + rng = np.random.default_rng(0) + x, y = rng.standard_normal((30, 5)) * 0.5, rng.standard_normal((80, 5)) + out = str(tmp_path / "containment.png") + + for label in ("Generator", "None"): + seen.clear() + pted_containment_test( + x, + y, + permutations=50, + n_landmarks=40, + rng=np.random.default_rng(3) if label == "Generator" else None, + pit_plot=out, + ) + assert len(seen) == 1, f"rng={label}: allocated landmarks {len(seen)} times" + assert os.path.exists(out)