diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 21e2290..3e4803b 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -61,7 +61,7 @@ jobs: ls -ltrh ls -ltrh dist - name: Publish to Test PyPI - uses: pypa/gh-action-pypi-publish@v1.12.4 + uses: pypa/gh-action-pypi-publish@v1.14.2 with: repository-url: https://test.pypi.org/legacy/ verbose: true @@ -99,5 +99,5 @@ jobs: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@v1.12.4 + - uses: pypa/gh-action-pypi-publish@v1.14.2 if: startsWith(github.ref, 'refs/tags') \ No newline at end of file diff --git a/README.md b/README.md index 4043018..fca8df8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ correctly calibrated type I error rate regardless of the data distribution. ![pted logo](media/pted_logo.png) +> **NOTE:** PTED is basically just the Maximum Mean Discrepancy (MMD) except using the Euclidean distance as the kernel. It took me a while to realize this, and in the meantime I made a nice python package. So I consider it a win for everyone. + ## Install To install PTED, run the following: @@ -55,6 +57,7 @@ PTED is useful for: * Evaluate if a simulator generates true "data-like" samples * PTED (or just the energy distance) can be a distance metric for Approximate Bayesian Computing posteriors * Check for drift in a time series, comparing samples before/after some cutoff time +* The energy distance is differentiable, so it can be used to train machine learning algorithms And much more! @@ -96,7 +99,7 @@ PTED uses the energy distance of the two samples `x` and `y`, this is computed a $$d = \frac{2}{n_xn_y}\sum_{i,j}||x_i - y_j|| - \frac{1}{n_x^2}\sum_{i,j}||x_i - x_j|| - \frac{1}{n_y^2}\sum_{i,j}||y_i - y_j||$$ -The energy distance measures distances between pairs of points. It becomes more +The energy distance measures distances between pairs of points[^2]. It becomes more positive if the `x` and `y` samples tend to be further from each other than from themselves. We demonstrate this in the figure below, where the `x` samples are drawn from a (thick) circle, while the `y` samples are drawn from a (thick) @@ -149,7 +152,7 @@ Internally, for each simulation separately we use PTED to compute a p-value, essentially asking the question "was ``g`` drawn from the distribution that generated ``s``?". Individually, these tests are possibly not especially informative (unless the sampler is really bad), however their p-values must have -been drawn from ``U(0,1)`` under the null-hypothesis[^2]. Thus we just need a +been drawn from ``U(0,1)`` under the null-hypothesis[^3]. Thus we just need a way to combine their statistical power. It turns out that for some ``p ~ U(0,1)``, we have that ``- 2 ln(p)`` is chi2 distributed with ``dof = 2``. This means that we can sum the chi2 values for the PTED test on each simulation and @@ -164,7 +167,7 @@ If you have posterior densities (and you trust them), then chances are the HDP region coverage test is a more powerful test. Essentially, instead of using a permutation test to determine the p-value for a single simulation, you just determine the fraction of posterior samples with higher posterior density than -the ground truth[^3]. The package has a quick tool to let you do that: +the ground truth[^4]. The package has a quick tool to let you do that: ```python from pted import hdp_coverage_test @@ -278,7 +281,7 @@ could not find any significant discrepancies. The samples could have been drawn from the same distribution, or PTED could be insensitive to the deviation, or maybe the test needs more samples. In some sense PTED (like all null hypothesis tests) is "necessary but not sufficient" in that failing the test is bad news -for the null, but passing the test is possibly inconclusive[^4]. Use your judgement, +for the null, but passing the test is possibly inconclusive[^5]. Use your judgement, and contact me or some smarter stat-oriented person if you are unsure about the results you are getting! @@ -291,7 +294,6 @@ def pted( x: Union[np.ndarray, "Tensor", "jax.Array"], y: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, - metric: Union[str, float] = "euclidean", return_all: bool = False, chunk_size: Optional[int] = None, two_tailed: bool = True, @@ -302,9 +304,8 @@ def pted( * **x** *(Union[np.ndarray, Tensor, jax.Array])*: first set of samples. Shape (N, *D) * **y** *(Union[np.ndarray, Tensor, jax.Array])*: second set of samples. Shape (M, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. -* **metric** *(Union[str, float])*: distance metric to use. See scipy.spatial.distance.cdist for the list of available metrics with numpy. See torch.cdist when using PyTorch, note that the metric is passed as the "p" for torch.cdist and therefore is a float from 0 to inf. When using JAX arrays, the metric is passed as the "ord" for jnp.linalg.norm and therefore is also a float. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use chunked energy distance estimation. The chunk size is the number of samples per chunk. The number of chunks is determined automatically as `max(len(x), len(y)) // chunk_size`, iterating over the larger dataset once and cycling through the smaller one. If `chunk_size >= len(x)` and `chunk_size >= len(y)`, PTED falls back to the full (non-chunked) computation. If None, use the full dataset. +* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If `chunk_size >= len(x)` and `chunk_size >= len(y)`, PTED falls back to the full (non-chunked) computation. If None, use the full dataset. * **two_tailed** *(bool)*: if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. If False, only checks for dissimilarity but is more sensitive. Default is True. * **prog_bar** *(bool)*: if True, show a progress bar to track the progress of permutation tests. Default is False. @@ -315,7 +316,6 @@ def pted_coverage_test( g: Union[np.ndarray, "Tensor", "jax.Array"], s: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, - metric: Union[str, float] = "euclidean", warn_confidence: Optional[float] = 1e-3, return_all: bool = False, chunk_size: Optional[int] = None, @@ -330,9 +330,8 @@ def pted_coverage_test( * **g** *(Union[np.ndarray, Tensor, jax.Array])*: Ground truth samples. Shape (n_sims, *D) * **s** *(Union[np.ndarray, Tensor, jax.Array])*: Posterior samples. Shape (n_samples, n_sims, *D) * **permutations** *(int)*: number of permutations to run. This determines how accurately the p-value is computed. -* **metric** *(Union[str, float])*: distance metric to use. See scipy.spatial.distance.cdist for the list of available metrics with numpy. See torch.cdist when using PyTorch, note that the metric is passed as the "p" for torch.cdist and therefore is a float from 0 to inf. When using JAX arrays, the metric is passed as the "ord" for jnp.linalg.norm and therefore is also a float. * **return_all** *(bool)*: if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) -* **chunk_size** *(Optional[int])*: if not None, use chunked energy distance estimation. The chunk size is the number of samples per chunk. The number of chunks is determined automatically as `max(len(x), len(y)) // chunk_size`, iterating over the larger dataset once and cycling through the smaller one. If None, use the full dataset. +* **chunk_size** *(Optional[int])*: if not None, use a landmark-based rectangular distance matrix to estimate the energy distance instead of the full pairwise matrix. `nxc = min(chunk_size, len(x))` samples from x and `nyc = min(chunk_size, len(y))` samples from y are used as fixed "landmarks"; only distances from every sample to these landmarks are computed. If None, use the full dataset. * **sbc_histogram** *(Optional[str])*: If given, the path/filename to save a Simulation-Based-Calibration histogram. * **sbc_bins** *(Optional[int])*: If given, force the histogram to have the provided number of bins. Otherwise, select an appropriate size: ~sqrt(N). * **pit_plot** *(Optional[str])*: If given, the path/filename to save a Probability Integral Transform (PIT) plot of the per-simulation p-values against the expected uniform distribution, with a shaded KS confidence band. @@ -375,21 +374,24 @@ print(f"p-value: {p_value:.3f}") # expect uniform random from 0-1 If a GPU isn't enough to get PTED running fast enough for you, or if you are running into memory limitations, there are still options! We can use an approximation of the energy distance, in this case the test is still exact but -less sensitive than it would be otherwise. We can approximate the energy -distance by iterating through sequential chunks of the full dataset, computing -the energy distance on each chunk, then averaging. Just set the `chunk_size` -parameter for the number of samples you can manage at once. The number of -iterations is determined automatically as `max(len(x), len(y)) // chunk_size`, -iterating over the larger dataset once and cycling through the smaller one if -their sizes differ. The larger the chunk size, the closer the estimate will be -to the true energy distance, but it will take more compute. +less sensitive than it would be otherwise. Instead of building the full +`(n_samp_x + n_samp_y) x (n_samp_x + n_samp_y)` pairwise distance matrix, PTED +can build a smaller rectangular `(n_samp_x + n_samp_y) x (nxc + nyc)` matrix of +distances from every sample to a fixed set of "landmark" samples, where `nxc = +min(chunk_size, n_samp_x)` landmarks are drawn from x and `nyc = min(chunk_size, +n_samp_y)` from y. Just set the `chunk_size` parameter to control how many +landmarks are used. This rectangular matrix is computed only once; the +permutation distribution is then obtained purely by re-indexing (permuting) its +rows, which reassigns samples to the x/y groups without recomputing any +distances. The larger the chunk size, the closer the estimate will be to the +true energy distance, but it will take more compute. Note that the computational complexity for standard PTED goes as `O((n_samp_x + -n_samp_y)^2)` while the chunked version goes as `O(n_iter * (2 * chunk_size)^2)` -where `n_iter = max(n_samp_x, n_samp_y) // chunk_size`, so plan your chunking -accordingly. For a given chunk size, the computational complexity of PTED grows -linearly with dataset size, much like other large scale (machine learning -oriented) two sample tests. +n_samp_y)^2)` to build the full distance matrix, while the chunked version goes +as `O((n_samp_x + n_samp_y) * (nxc + nyc))`, so plan your chunking accordingly. +For a given chunk size, the computational complexity of PTED grows linearly +with dataset size, much like other large scale (machine learning oriented) two +sample tests. Example: ```python @@ -509,6 +511,7 @@ If you think those are neat, then you'll probably also like this paper, which us ``` [^1]: See the Simulation-Based Calibration paper by Talts et al. 2018 for what "SBC" is. -[^2]: Since PTED works by a permutation test, we only get the p-value from a discrete uniform distribution. By default we use 1000 permutations, if you are running an especially sensitive test you may need more permutations, but for most purposes this is sufficient. -[^3]: Actually, we take (q + 1) / (Nsamp + 1) rather than q/Nsamp where q is the number of posterior samples with posterior density greater than the ground truth. This just turns out to be a better estimator for finite Nsamp. -[^4]: actual "necessary but not sufficient" conditions are a different thing than null hypothesis tests, but they have a similar intuitive meaning. \ No newline at end of file +[^2]: Yes, I see it now. That formula is the same as for MMD. Just think of it as a special case of MMD that uses the Euclidean distance kernel. In fact this way it is often more sensitive than the RBF kernel typically used for MMD. +[^3]: Since PTED works by a permutation test, we only get the p-value from a discrete uniform distribution. By default we use 1000 permutations, if you are running an especially sensitive test you may need more permutations, but for most purposes this is sufficient. +[^4]: Actually, we take (q + 1) / (Nsamp + 1) rather than q/Nsamp where q is the number of posterior samples with posterior density greater than the ground truth. This just turns out to be a better estimator for finite Nsamp. +[^5]: actual "necessary but not sufficient" conditions are a different thing than null hypothesis tests, but they have a similar intuitive meaning. \ No newline at end of file diff --git a/src/pted/pted.py b/src/pted/pted.py index 590941c..06f33a5 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -3,14 +3,8 @@ import numpy as np from .utils import ( - is_torch_tensor, - is_jax_array, - pted_torch, - pted_numpy, - pted_chunk_torch, - pted_chunk_numpy, - pted_jax, - pted_chunk_jax, + pted as _pted_impl, + pted_chunk as _pted_chunk_impl, two_tailed_p, confidence_alert, simulation_based_calibration_histogram, @@ -24,7 +18,6 @@ def pted( x: Union[np.ndarray, "Tensor", "jax.Array"], y: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, - metric: Union[str, float] = "euclidean", return_all: bool = False, chunk_size: Optional[int] = None, two_tailed: bool = True, @@ -32,7 +25,7 @@ def pted( ) -> Union[float, tuple[float, np.ndarray, float]]: """ Two sample null hypothesis test using a permutation test on the energy - distance. + distance (Euclidean). A "two sample test" is a statistical test that compares two samples to determine if they come from the same distribution. The null hypothesis is @@ -79,22 +72,17 @@ def pted( y (Union[np.ndarray, Tensor, jax.Array]): second set of samples. Shape (M, *D) permutations (int): number of permutations to run. This determines how accurately the p-value is computed. - metric (Union[str, float]): distance metric to use. For NumPy inputs, - see scipy.spatial.distance.cdist for available metrics. For PyTorch - inputs, the metric is passed as the "p" argument to torch.cdist and - therefore is a float from 0 to inf. For JAX inputs, "euclidean" uses - the squared-norm identity (p=2), and any float p uses - jnp.linalg.norm with ord=p; string metrics other than "euclidean" - are not supported for JAX. return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): if not None, use chunked energy distance - estimation. This is useful for large datasets. The chunk size is the - number of samples per chunk. The number of chunks is determined - automatically as ``max(len(x), len(y)) // chunk_size``, iterating - over the larger dataset once and cycling through the smaller one. - If None, use the full dataset. + chunk_size (Optional[int]): if not None, use a landmark-based + rectangular distance matrix to estimate the energy distance + instead of the full pairwise matrix. ``nxc = min(chunk_size, + len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` + samples from y are used as fixed "landmarks"; only distances from + every sample to these landmarks are computed. If ``chunk_size`` + covers both full datasets, this falls back to the exact + (non-chunked) computation. If None, use the full dataset. two_tailed (bool): if True, compute a two-tailed p-value. This is useful if you want to reject the null hypothesis when x and y are either too similar or too different. Default is True. @@ -104,20 +92,20 @@ def pted( Note ---- - PTED has O(n^2 * D * P) time complexity, where n is the number of - samples in x and y, D is the number of dimensions, and P is the number - of permutations. For large datasets this can get unwieldy, so chunking - is recommended. For chunking, the energy distance will be estimated at - each iteration rather than fully computed. The dataset is divided into - sequential chunks of size `chunk_size`; the number of chunks (iterations) - is ``max(len(x), len(y)) // chunk_size``, iterating over the larger - dataset once and cycling through the smaller one. The average energy - distance over all chunks is the final estimate. PTED remains an exact - p-value test even when chunking, it simply becomes less sensitive to - the difference between x and y. The chunked pted test has time - complexity O(c^2 * I * D * P), where c is the chunk size, I is the - number of iterations, D is the number of dimensions, and P is the number - of permutations. For chunking to be worth it you should have c^2 * I << n^2. + PTED has O(n^2 * D) time complexity to build the distance matrix, + where n is the number of samples in x and y and D is the number of + dimensions; each of the P permutations then costs O(n^2) to + re-evaluate the test statistic from the re-indexed matrix. For large + datasets this can get unwieldy, so chunking is recommended. When + chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances + from every sample to a fixed set of landmark samples is computed, so + this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. + The permutation distribution is then obtained by re-indexing (permuting) + the rows of this matrix, which costs O(n * nc) per permutation and + never recomputes distances. PTED remains an exact p-value test even + when chunking, it simply becomes less sensitive to the difference + between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and + ``len(y)``. """ assert type(x) == type(y), f"x and y must be of the same type, not {type(x)} and {type(y)}" assert len(x.shape) >= 2, f"x must be at least 2D, not {x.shape}" @@ -135,43 +123,16 @@ def pted( if len(y.shape) > 2: y = y.reshape(y.shape[0], -1) - if is_torch_tensor(x) and chunk_size is not None: - test, permute = pted_chunk_torch( - x, - y, - permutations=permutations, - metric=metric, - chunk_size=int(chunk_size), - prog_bar=prog_bar, - ) - elif is_torch_tensor(x): - test, permute = pted_torch( - x, y, permutations=permutations, metric=metric, prog_bar=prog_bar - ) - elif is_jax_array(x) and chunk_size is not None: - test, permute = pted_chunk_jax( - x, - y, - permutations=permutations, - metric=metric, - chunk_size=int(chunk_size), - prog_bar=prog_bar, - ) - elif is_jax_array(x): - test, permute = pted_jax(x, y, permutations=permutations, metric=metric, prog_bar=prog_bar) - elif chunk_size is not None: - test, permute = pted_chunk_numpy( + if chunk_size is not None: + test, permute = _pted_chunk_impl( x, y, permutations=permutations, - metric=metric, chunk_size=int(chunk_size), prog_bar=prog_bar, ) else: - test, permute = pted_numpy( - x, y, permutations=permutations, metric=metric, prog_bar=prog_bar - ) + test, permute = _pted_impl(x, y, permutations=permutations, prog_bar=prog_bar) permute = np.array(permute) @@ -193,7 +154,6 @@ def pted_coverage_test( g: Union[np.ndarray, "Tensor", "jax.Array"], s: Union[np.ndarray, "Tensor", "jax.Array"], permutations: int = 1000, - metric: Union[str, float] = "euclidean", warn_confidence: Optional[float] = 1e-3, return_all: bool = False, chunk_size: Optional[int] = None, @@ -204,7 +164,7 @@ def pted_coverage_test( prog_bar: bool = False, ) -> Union[float, tuple[np.ndarray, np.ndarray, float]]: """ - Coverage test using a permutation test on the energy distance. + Coverage test using a permutation test on the energy distance (Euclidean). A "coverage test" is a statistical test that determines if the posterior samples (s) cover the ground truth samples (g) with the correct uncertainty. @@ -253,22 +213,17 @@ def pted_coverage_test( s (Union[np.ndarray, Tensor, jax.Array]): Posterior samples. Shape (n_samples, n_sims, *D) permutations (int): number of permutations to run. This determines how accurately the p-value is computed. - metric (Union[str, float]): distance metric to use. For NumPy inputs, - see scipy.spatial.distance.cdist for available metrics. For PyTorch - inputs, the metric is passed as the "p" argument to torch.cdist and - therefore is a float from 0 to inf. For JAX inputs, "euclidean" uses - the squared-norm identity (p=2), and any float p uses - jnp.linalg.norm with ord=p; string metrics other than "euclidean" - are not supported for JAX. return_all (bool): if True, return the test statistic and the permuted statistics with the p-value. If False, just return the p-value. bool (default: False) - chunk_size (Optional[int]): If not None, use chunked energy distance - estimation. This is useful for large datasets. The chunk size is the - number of samples per chunk. The number of chunks is determined - automatically as ``max(len(x), len(y)) // chunk_size``, iterating - over the larger dataset once and cycling through the smaller one. - If None, use the full dataset. + chunk_size (Optional[int]): If not None, use a landmark-based + rectangular distance matrix to estimate the energy distance + instead of the full pairwise matrix. ``nxc = min(chunk_size, + len(x))`` samples from x and ``nyc = min(chunk_size, len(y))`` + samples from y are used as fixed "landmarks"; only distances from + every sample to these landmarks are computed. If ``chunk_size`` + covers both full datasets, this falls back to the exact + (non-chunked) computation. If None, use the full dataset. sbc_histogram (Optional[str]): If given, the path/filename to save a Simulation-Based-Calibration histogram. sbc_bins (Optional[int]): If given, force the histogram to have the provided @@ -287,20 +242,20 @@ def pted_coverage_test( Note ---- - PTED has O(n^2 * D * P) time complexity, where n is the number of - samples in x and y, D is the number of dimensions, and P is the number - of permutations. For large datasets this can get unwieldy, so chunking - is recommended. For chunking, the energy distance will be estimated at - each iteration rather than fully computed. The dataset is divided into - sequential chunks of size `chunk_size`; the number of chunks (iterations) - is ``max(len(x), len(y)) // chunk_size``, iterating over the larger - dataset once and cycling through the smaller one. The average energy - distance over all chunks is the final estimate. PTED remains an exact - p-value test even when chunking, it simply becomes less sensitive to - the difference between x and y. The chunked pted test has time - complexity O(c^2 * I * D * P), where c is the chunk size, I is the - number of iterations, D is the number of dimensions, and P is the number - of permutations. For chunking to be worth it you should have c^2 * I << n^2. + PTED has O(n^2 * D) time complexity to build the distance matrix, + where n is the number of samples in x and y and D is the number of + dimensions; each of the P permutations then costs O(n^2) to + re-evaluate the test statistic from the re-indexed matrix. For large + datasets this can get unwieldy, so chunking is recommended. When + chunking, only a rectangular ``(n, nxc + nyc)`` matrix of distances + from every sample to a fixed set of landmark samples is computed, so + this matrix is built in O(n * nc * D) time, where ``nc = nxc + nyc``. + The permutation distribution is then obtained by re-indexing (permuting) + the rows of this matrix, which costs O(n * nc) per permutation and + never recomputes distances. PTED remains an exact p-value test even + when chunking, it simply becomes less sensitive to the difference + between x and y as ``chunk_size`` shrinks relative to ``len(x)`` and + ``len(y)``. """ nsamp, nsim, *_ = s.shape assert nsim > 0, "need some simulations to run test, got 0 simulations" @@ -319,7 +274,6 @@ def pted_coverage_test( g[:, i], s[:, i], permutations=permutations, - metric=metric, return_all=True, two_tailed=False, chunk_size=chunk_size, diff --git a/src/pted/utils.py b/src/pted/utils.py index 21a3a8e..a5be282 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -1,10 +1,9 @@ -from typing import Union +from typing import Optional, Union from warnings import warn import numpy as np from scipy.spatial.distance import cdist from scipy.stats import chi2 as chi2_dist, binom, kstwo, kstest -from scipy.optimize import root_scalar from tqdm.auto import trange try: @@ -29,16 +28,13 @@ class torch: __all__ = ( "is_torch_tensor", "is_jax_array", - "pted_numpy", - "pted_chunk_numpy", - "pted_torch", - "pted_chunk_torch", - "pted_jax", - "pted_chunk_jax", + "pted", + "pted_chunk", "two_tailed_p", "confidence_alert", "simulation_based_calibration_histogram", "pit_plot", + "hdp_coverage_test", ) @@ -59,287 +55,213 @@ def is_jax_array(o): return isinstance(o, jax.Array) -def _energy_distance_precompute( - D: Union[np.ndarray, torch.Tensor], nx: int, ny: int -) -> Union[float, torch.Tensor]: - Exx = D[:nx, :nx].sum() / nx**2 - Eyy = D[nx:, nx:].sum() / ny**2 - Exy = D[:nx, nx:].sum() / (nx * ny) - return 2 * Exy - Exx - Eyy +def _backend(x) -> str: + if is_torch_tensor(x): + return "torch" + if is_jax_array(x): + return "jax" + return "numpy" -def _energy_distance_numpy(x: np.ndarray, y: np.ndarray, metric: str = "euclidean") -> float: - nx = len(x) - ny = len(y) - z = np.concatenate((x, y), axis=0) - D = cdist(z, z, metric=metric) - return _energy_distance_precompute(D, nx, ny) +def _concatenate(arrays, backend: str): + if backend == "torch": + return torch.cat(arrays, dim=0) + if backend == "jax": + return jnp.concatenate(arrays, axis=0) + return np.concatenate(arrays, axis=0) -def _energy_distance_torch( - x: torch.Tensor, y: torch.Tensor, metric: Union[str, float] = "euclidean" -) -> float: - nx = len(x) - ny = len(y) - z = torch.cat((x, y), dim=0) - if metric == "euclidean": - metric = 2.0 - D = torch.cdist(z, z, p=metric) - return _energy_distance_precompute(D, nx, ny).item() +def _all_finite(z, backend: str) -> bool: + if backend == "torch": + return bool(torch.all(torch.isfinite(z))) + if backend == "jax": + return bool(jnp.all(jnp.isfinite(z))) + return bool(np.all(np.isfinite(z))) -def _chunk_slices(lenx: int, leny: int, chunk_size: int): - """Yield slices for chunking two arrays of lengths lenx and leny. +def _to_scalar(x, backend: str) -> float: + if backend == "torch": + return float(x.item()) + if backend == "jax": + return float(x.item()) + return float(x) - The smaller of the two is cycled while the larger is iterated through - completely (minus the last incomplete chunk if any). - """ - nx = max(1, lenx // chunk_size) - ny = max(1, leny // chunk_size) - - for i in range(max(nx, ny)): - ix = i % nx - iy = i % ny - yield slice(ix * chunk_size, (ix + 1) * chunk_size), slice( - iy * chunk_size, (iy + 1) * chunk_size - ) +def _to_backend(x, backend: str): + if backend == "torch": + return torch.as_tensor(x) + if backend == "jax": + return jnp.asarray(x) + return np.asarray(x) -def _energy_distance_estimate( - x, - y, - chunk_size: int, - metric: Union[str, float], - energy_distance_fn, -) -> float: - """Estimate energy distance by averaging over sequential sliced chunks. - Iterates ``max(len(x), len(y)) // chunk_size`` times, using plain slicing - on both arrays. The smaller of the two is tiled along axis 0 as needed so - that both arrays are at least ``n_iter * chunk_size`` rows long before the - loop begins. - """ - E_est = [] - for cx, cy in _chunk_slices(len(x), len(y), chunk_size): - E_est.append(energy_distance_fn(x[cx], y[cy], metric=metric)) - return np.mean(E_est) +def _random_permutation(D, backend: str = "numpy"): + if backend == "torch": + I = torch.randperm(D.shape[0], device=D.device) + elif backend == "jax": + I = jax.random.permutation(jax.random.PRNGKey(np.random.randint(0, 1e6)), D.shape[0]) + else: + I = np.random.permutation(D.shape[0]) + return D[I][:, I] -@jit(static_argnames=["p"]) -def _jax_cdist(x, y, p: float = 2.0): - # For general p-norms use vmap to avoid the (nx, ny, d) intermediate. - return jax.vmap(lambda xi: jnp.linalg.norm(xi - y, ord=p, axis=-1))(x) +@jit +def _jax_cdist(x, y): + return jax.vmap(lambda xi: jnp.linalg.norm(xi - y, ord=2.0, axis=-1))(x) -def _energy_distance_jax(x, y, metric: Union[str, float] = "euclidean") -> float: - nx = len(x) - ny = len(y) - z = jnp.concatenate([x, y], axis=0) - if metric == "euclidean": - metric = 2.0 - D = _jax_cdist(z, z, p=metric) - return float(_energy_distance_precompute(D, nx, ny)) - - -def pted_chunk_numpy( - x: np.ndarray, - y: np.ndarray, - permutations: int = 100, - metric: str = "euclidean", - chunk_size: int = 100, - prog_bar: bool = False, -) -> tuple[float, list[float]]: - assert np.all(np.isfinite(x)) and np.all(np.isfinite(y)), "Input contains NaN or Inf!" - nx = len(x) - - test_stat = _energy_distance_estimate( - x, y, chunk_size, metric=metric, energy_distance_fn=_energy_distance_numpy - ) - permute_stats = [] - z = np.concatenate((x, y), axis=0) - for _ in trange(permutations, disable=not prog_bar): - z = z[np.random.permutation(len(z))] - permute_stats.append( - _energy_distance_estimate( - z[:nx], z[nx:], chunk_size, metric=metric, energy_distance_fn=_energy_distance_numpy - ) - ) - return test_stat, permute_stats - - -def pted_chunk_torch( - x: torch.Tensor, - y: torch.Tensor, - permutations: int = 100, - metric: Union[str, float] = "euclidean", - chunk_size: int = 100, - prog_bar: bool = False, -) -> tuple[float, list[float]]: - assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" - assert torch.all(torch.isfinite(x)) and torch.all( - torch.isfinite(y) - ), "Input contains NaN or Inf!" - nx = len(x) - - test_stat = _energy_distance_estimate( - x, y, chunk_size, metric=metric, energy_distance_fn=_energy_distance_torch - ) - permute_stats = [] - z = torch.cat((x, y), dim=0) - for _ in trange(permutations, disable=not prog_bar): - z = z[torch.randperm(len(z))] - permute_stats.append( - _energy_distance_estimate( - z[:nx], z[nx:], chunk_size, metric=metric, energy_distance_fn=_energy_distance_torch - ) - ) - return test_stat, permute_stats +def _cdist(a, b, backend: str): + if backend == "torch": + return torch.cdist(a, b, p=2.0) + if backend == "jax": + return _jax_cdist(a, b) + return cdist(a, b, metric="euclidean") -def pted_numpy( - x: np.ndarray, - y: np.ndarray, - permutations: int = 100, - metric: str = "euclidean", - prog_bar: bool = False, -) -> tuple[float, list[float]]: - z = np.concatenate((x, y), axis=0) - assert np.all(np.isfinite(z)), "Input contains NaN or Inf!" - dmatrix = cdist(z, z, metric=metric) - assert np.all( - np.isfinite(dmatrix) - ), "Distance matrix contains NaN or Inf! Consider using a different metric or normalizing values to be more stable (i.e. z-score norm)." - nx = len(x) - ny = len(y) - - test_stat = _energy_distance_precompute(dmatrix, nx, ny) - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - I = np.random.permutation(len(z)) - dmatrix = dmatrix[I][:, I] - permute_stats.append(_energy_distance_precompute(dmatrix, nx, ny)) - return test_stat, permute_stats - - -def pted_torch( - x: torch.Tensor, - y: torch.Tensor, - permutations: int = 100, - metric: Union[str, float] = "euclidean", - prog_bar: bool = False, -) -> tuple[float, list[float]]: - assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" - z = torch.cat((x, y), dim=0) - assert torch.all(torch.isfinite(z)), "Input contains NaN or Inf!" - if metric == "euclidean": - metric = 2.0 - dmatrix = torch.cdist(z, z, p=metric) - assert torch.all( - torch.isfinite(dmatrix) - ), "Distance matrix contains NaN or Inf! Consider using a different metric or normalizing values to be more stable (i.e. z-score norm)." - nx = len(x) - ny = len(y) - - test_stat = _energy_distance_precompute(dmatrix, nx, ny).item() - permute_stats = [] - for _ in trange(permutations, disable=not prog_bar): - I = torch.randperm(len(z)) - dmatrix = dmatrix[I][:, I] - permute_stats.append(_energy_distance_precompute(dmatrix, nx, ny).item()) - return test_stat, permute_stats +def _energy_distance_precompute( + D: Union[np.ndarray, torch.Tensor], + nx: int, + ny: int, + nxc: Optional[int] = None, + nyc: Optional[int] = None, +) -> Union[float, torch.Tensor]: + """Energy distance from a rectangular (nx+ny, nxc+nyc) distance matrix. + + If ``D`` is a full (nx+ny, nx+ny) distance matrix, then ``nxc=nx`` and + ``nyc=ny`` and this is the standard energy distance formula. Otherwise, if + ``D`` is a rectangular (nx+ny, nxc+nyc) distance matrix, then ``D`` holds + distances from every sample (rows: ``nx`` of x followed by ``ny`` of y) to a + set of landmark samples (columns: ``nxc`` from x followed by ``nyc`` from + y). Exx/Eyy are estimated from each group against its own landmarks, and Exy + is estimated by averaging the two cross blocks (x-rows vs y-landmarks, and + y-rows vs x-landmarks). + """ + nxc = nx if nxc is None else nxc + nyc = ny if nyc is None else nyc + Exx = D[:nx, :nxc].sum() / (nx * nxc) + Eyy = D[nx:, nxc:].sum() / (ny * nyc) + Exy = (D[:nx, nxc:].sum() + D[nx:, :nxc].sum()) / (nx * nyc + ny * nxc) + return 2 * Exy - Exx - Eyy -def pted_jax( +def pted( x, y, - permutations: int = 100, - metric: Union[str, float] = "euclidean", + permutations: int, prog_bar: bool = False, ) -> tuple[float, list[float]]: - assert jax is not None, "JAX is not installed! try: `pip install jax`" - z = jnp.concatenate([x, y], axis=0) - assert jnp.all(jnp.isfinite(z)), "Input contains NaN or Inf!" - if metric == "euclidean": - metric = 2.0 - dmatrix = _jax_cdist(z, z, p=metric) - assert jnp.all( - jnp.isfinite(dmatrix) - ), "Distance matrix contains NaN or Inf! Consider using a different metric or normalizing values to be more stable (i.e. z-score norm)." + """Permutation-based energy distance test statistic and its permutation distribution. + + Works transparently with numpy arrays, torch tensors, and jax arrays. + """ + if is_torch_tensor(x): + assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" + if is_jax_array(x): + assert jax is not None, "JAX is not installed! try: `pip install jax`" + + backend = _backend(x) + z = _concatenate((x, y), backend) + assert _all_finite(z, backend), "Input contains NaN or Inf!" + dmatrix = _cdist(z, z, backend) + assert _all_finite( + dmatrix, backend + ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." + if backend == "jax": # numpy is faster for eager stuff + dmatrix = np.asarray(dmatrix) + backend = "numpy" nx = len(x) ny = len(y) - test_stat = float(_energy_distance_precompute(dmatrix, nx, ny)) + test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend) permute_stats = [] for _ in trange(permutations, disable=not prog_bar): - I = np.random.permutation(len(z)) - dmatrix = dmatrix[I][:, I] - permute_stats.append(float(_energy_distance_precompute(dmatrix, nx, ny))) + dmatrix = _random_permutation(dmatrix, backend) + permute_stats.append(_to_scalar(_energy_distance_precompute(dmatrix, nx, ny), backend)) + return test_stat, permute_stats -def pted_chunk_jax( +def pted_chunk( x, y, - permutations: int = 100, - metric: Union[str, float] = "euclidean", - chunk_size: int = 100, + permutations: int, + chunk_size: int, prog_bar: bool = False, ) -> tuple[float, list[float]]: - assert jax is not None, "JAX is not installed! try: `pip install jax`" - assert jnp.all(jnp.isfinite(x)) and jnp.all(jnp.isfinite(y)), "Input contains NaN or Inf!" + """Chunked variant of `pted` for large datasets. + + Rather than the full ``(nx+ny, nx+ny)`` pairwise distance matrix, this + builds a rectangular ``(nx+ny, nxc+nyc)`` matrix of distances from every + sample to a set of randomly chosen "landmark" samples, where ``nxc = + min(chunk_size, nx)`` landmarks are drawn from ``x`` and ``nyc = + min(chunk_size, ny)`` from ``y``. This matrix is computed only once; the + permutation distribution is then obtained by re-indexing (permuting) its + rows, which reassigns samples to the x/y groups without recomputing any + distances. Each landmark's row position is tracked and carried through the + same row permutation, so after every shuffle the landmarks are re-split into + "x-side"/"y-side" columns according to where they now fall, keeping rows and + columns consistent. In pathological cases where too few landmarks are + permuted onto either side (zero or one tenth of the original number of + landmarks), a new permutation is drawn. + + Works transparently with numpy arrays, torch tensors, and jax arrays. + """ + if is_torch_tensor(x): + assert torch.__version__ != "null", "PyTorch is not installed! try: `pip install torch`" + if is_jax_array(x): + assert jax is not None, "JAX is not installed! try: `pip install jax`" + + backend = _backend(x) + z = _concatenate((x, y), backend) + assert _all_finite(z, backend), "Input contains NaN or Inf!" nx = len(x) - - test_stat = _energy_distance_estimate( - x, y, chunk_size, metric=metric, energy_distance_fn=_energy_distance_jax + ny = len(y) + nxc = min(chunk_size, nx) + nyc = min(chunk_size, ny) + + landmark_pos = np.sort( + np.concatenate( + [ + np.random.choice(nx, size=nxc, replace=False), + nx + np.random.choice(ny, size=nyc, replace=False), + ] + ) ) + dmatrix = _cdist(z, z[_to_backend(landmark_pos, backend)], backend) + assert _all_finite( + dmatrix, backend + ), "Distance matrix contains NaN or Inf! Consider normalizing values to be more stable (i.e. z-score norm)." + if backend == "jax": # numpy is faster for eager stuff + dmatrix = np.asarray(dmatrix) + backend = "numpy" + + test_stat = _to_scalar(_energy_distance_precompute(dmatrix, nx, ny, nxc, nyc), backend) permute_stats = [] - z = jnp.concatenate([x, y], axis=0) for _ in trange(permutations, disable=not prog_bar): - z = z[np.random.permutation(len(z))] - permute_stats.append( - _energy_distance_estimate( - z[:nx], z[nx:], chunk_size, metric=metric, energy_distance_fn=_energy_distance_jax + while True: + I = np.random.permutation(len(z)) + # Track where each landmark moved to, then re-split columns by their new x/y side. + landmark_pos = np.argsort(I)[landmark_pos] + order = np.argsort(landmark_pos >= nx, kind="stable") + dmatrix_i = dmatrix[_to_backend(I, backend)][:, _to_backend(order, backend)] + landmark_pos = landmark_pos[order] + nxc_i = int(np.sum(landmark_pos < nx)) + nyc_i = nxc + nyc - nxc_i + if nxc_i < max(1, (nxc * 0.1)) or nyc_i < max(1, (nyc * 0.1)): + continue # Skip this permutation if too few landmarks remain on either side, to avoid unstable estimates. + permute_stats.append( + _to_scalar(_energy_distance_precompute(dmatrix_i, nx, ny, nxc_i, nyc_i), backend) ) - ) + break return test_stat, permute_stats def two_tailed_p(chi2, df): - assert df > 2, "Degrees of freedom must be greater than 2 for two-tailed p-value calculation." p_left = chi2_dist.cdf(chi2, df) p_right = chi2_dist.sf(chi2, df) return 2 * min(p_left, p_right) -###### This is a density based two tailed p-value, it is kept for reference but not used ####### -# def two_tailed_p(chi2, df): -# assert df > 2, "Degrees of freedom must be greater than 2 for two-tailed p-value calculation." -# alpha = chi2_dist.pdf(chi2, df) -# mode = df - 2 - -# if np.isclose(chi2, mode): -# return 1.0 - -# def root_eq(x): -# return chi2_dist.pdf(x, df) - alpha - -# # Find left root -# if chi2 < mode: -# left = chi2_dist.cdf(chi2, df) -# else: -# res_left = root_scalar(root_eq, bracket=[0, mode], method="brentq") -# left = chi2_dist.cdf(res_left.root, df) - -# # Find right root -# if chi2 > mode: -# right = chi2_dist.sf(chi2, df) -# else: -# res_right = root_scalar(root_eq, bracket=[mode, 10000 * df], method="brentq") -# right = chi2_dist.sf(res_right.root, df) - -# return left + right - - class OverconfidenceWarning(UserWarning): """Warning for overconfidence in chi-squared test results.""" diff --git a/tests/test_pted.py b/tests/test_pted.py index 7d3d166..5acb823 100644 --- a/tests/test_pted.py +++ b/tests/test_pted.py @@ -19,21 +19,40 @@ import pytest +BACKENDS = ["numpy", "torch", "jax"] -def test_inputs_extra_dims(): + +def _require_backend(backend): + if backend == "torch" and torch is None: + pytest.skip("torch not installed") + if backend == "jax" and jax is None: + pytest.skip("jax not installed") + + +def _to_backend(arr, backend): + if backend == "torch": + return torch.tensor(arr) + if backend == "jax": + return jnp.array(arr) + return arr + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_inputs_extra_dims_two_sample(backend): + _require_backend(backend) np.random.seed(42) - # Test with numpy arrays - x = np.random.normal(size=(100, 30, 30)) - y = np.random.normal(size=(100, 30, 30)) + x = _to_backend(np.random.normal(size=(100, 3, 3)), backend) + y = _to_backend(np.random.normal(size=(100, 3, 3)), backend) p = pted.pted(x, y) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - if torch is None: - pytest.skip("torch not installed") - # Test with torch tensors - torch.manual_seed(43) - g = torch.randn(100, 30, 30) - s = torch.randn(50, 100, 30, 30) + +@pytest.mark.parametrize("backend", BACKENDS) +def test_inputs_extra_dims_coverage(backend): + _require_backend(backend) + np.random.seed(43) + g = _to_backend(np.random.normal(size=(100, 3, 3)), backend) + s = _to_backend(np.random.normal(size=(50, 100, 3, 3)), backend) p = pted.pted_coverage_test(g, s) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" @@ -56,117 +75,93 @@ def test_pted_progress_bar(capsys): assert "42/42" in captured, "progress bar did not show when prog_bar is set to True" -def test_pted_torch(): - if torch is None: - pytest.skip("torch not installed") - - # Set the random seed for reproducibility - torch.manual_seed(41) +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_two_sample(backend): + _require_backend(backend) np.random.seed(42) # example 2 sample test - D = 300 - for _ in range(20): - x = torch.randn(100, D) - y = torch.randn(100, D) - p = pted.pted(x, y) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - - x = torch.randn(100, D) - y = torch.rand(100, D) - p = pted.pted(x, y, two_tailed=False) + D = 10 + x = _to_backend(np.random.normal(size=(100, D)), backend) + y = _to_backend(np.random.normal(size=(100, D)), backend) + p = pted.pted(x, y, two_tailed=True) # regular two tailed, in null + assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" + p = pted.pted(x, x, two_tailed=True) # exact replication, two tailed + assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" + p = pted.pted(x, x, two_tailed=False) # exact replication, one tailed + assert p > 1e-2, f"p-value {p} is not in the expected range (~1)" + + x = _to_backend(np.random.normal(size=(100, D)), backend) + y = _to_backend(np.random.uniform(size=(100, D)), backend) + p = pted.pted(x, y, two_tailed=False) # one tailed, different distributions assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" - x = torch.randn(100, D) + x = _to_backend(np.random.normal(size=(100, D)), backend) t, p, _ = pted.pted(x, x, return_all=True) q = 2 * min(np.sum(p > t), np.sum(p < t)) p = (1 + q) / (len(p) + 1) # add one to numerator and denominator to avoid p=0 assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" -def test_pted_coverage_full(): +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_coverage_full(backend): + _require_backend(backend) np.random.seed(42) - g = np.random.normal(size=(100, 10)) # ground truth (n_simulations, n_dimensions) - s = np.random.normal( - size=(200, 100, 10) + g = _to_backend( + np.random.normal(size=(32, 4)), backend + ) # ground truth (n_simulations, n_dimensions) + s = _to_backend( + np.random.normal(size=(128, 32, 4)), backend ) # posterior samples (n_samples, n_simulations, n_dimensions) - test, permute, _ = pted.pted_coverage_test(g, s, permutations=100, return_all=True) - assert test.shape == (100,) - assert permute.shape == (100, 100) - - -def test_pted_chunk_torch(): - if torch is None: - pytest.skip("torch not installed") - np.random.seed(42) - torch.manual_seed(42) - - # example 2 sample test - D = 10 - x = torch.randn(1000, D) - y = torch.randn(1000, D) - p = pted.pted(x, y, chunk_size=100) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - - y = torch.rand(1000, D) - p = pted.pted(x, y, chunk_size=100) - assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" + test, permute, pval = pted.pted_coverage_test(g, s, permutations=100, return_all=True) + assert test.shape == (32,) + assert permute.shape == (32, 100) + assert pval > 1e-2 and pval < 0.99, f"p-value {pval} is not in the expected range (U(0,1))" -def test_pted_chunk_numpy(): +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_chunk(backend): + _require_backend(backend) np.random.seed(42) # example 2 sample test D = 10 - x = np.random.normal(size=(1000, D)) - y = np.random.normal(size=(1000, D)) + x = _to_backend(np.random.normal(size=(1000, D)), backend) + y = _to_backend(np.random.normal(size=(1000, D)), backend) p = pted.pted(x, y, chunk_size=100) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - y = np.random.uniform(size=(1000, D)) + y = _to_backend(np.random.uniform(size=(1000, D)), backend) p = pted.pted(x, y, chunk_size=100) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" -def test_pted_chunk_mismatched_sizes(): - """Chunked PTED correctly cycles the smaller dataset for mismatched x/y sizes.""" +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_chunk_mismatched_sizes(backend): + """Chunked PTED works correctly when x and y have different sizes.""" + _require_backend(backend) np.random.seed(0) D = 5 - # x has 800 samples, y has 300 samples; chunk_size=100 → 8 iterations, cycling y - x = np.random.normal(size=(800, D)) - y = np.random.normal(size=(300, D)) - p = pted.pted(x, y, chunk_size=100) + # x has 200 samples, y has 30 samples; chunk_size=50 → nxc=50, nyc=30 landmarks + x = _to_backend(np.random.normal(size=(200, D)), backend) + y = _to_backend(np.random.normal(size=(30, D)), backend) + p = pted.pted(x, y, chunk_size=50) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" # Different distributions should give small p-value even with mismatched sizes - y_diff = np.random.uniform(size=(300, D)) - p = pted.pted(x, y_diff, chunk_size=100) + y_diff = _to_backend(np.random.uniform(size=(30, D)), backend) + p = pted.pted(x, y_diff, chunk_size=50) assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" -def test_pted_chunk_size_fallback(): - """chunk_size >= both dataset sizes falls back to regular (non-chunked) PTED.""" - np.random.seed(7) - D = 5 - x = np.random.normal(size=(50, D)) - y = np.random.normal(size=(50, D)) - - # chunk_size equals both lengths → should silently fall back to regular PTED - p_chunk = pted.pted(x, y, chunk_size=50) - # Rerun with explicit non-chunked PTED using same seed for reference - np.random.seed(7) - p_plain = pted.pted(x, y) - # Both should give a valid p-value in U(0,1) - assert p_chunk > 1e-2 and p_chunk < 0.99, f"p-value {p_chunk} outside expected range" - assert p_plain > 1e-2 and p_plain < 0.99, f"p-value {p_plain} outside expected range" - - -def test_pted_coverage_edgecase(): +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_coverage_edgecase(backend): # Test with single simulation + _require_backend(backend) np.random.seed(42) - g = np.random.normal(size=(1, 10)) - s = np.random.normal(size=(100, 1, 10)) + g = _to_backend(np.random.normal(size=(1, 10)), backend) + s = _to_backend(np.random.normal(size=(100, 1, 10)), backend) p = pted.pted_coverage_test(g, s) assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" @@ -186,16 +181,17 @@ def test_pted_coverage_progress_bar(capsys): assert "42/42" in captured, "progress bar did not show when prog_bar is set to True" -def test_pted_coverage_overunder(): - if torch is None: - pytest.skip("torch not installed") - torch.manual_seed(42) - g = torch.randn(100, 3) - s = torch.randn(50, 100, 3) +@pytest.mark.parametrize("backend", BACKENDS) +def test_pted_coverage_overunder(backend): + _require_backend(backend) + np.random.seed(42) + g_np = np.random.normal(size=(100, 3)) + s_np = np.random.normal(size=(50, 100, 3)) + g = _to_backend(g_np, backend) with pytest.warns(pted.utils.OverconfidenceWarning): - pted.pted_coverage_test(g, s * 0.5) + pted.pted_coverage_test(g, _to_backend(s_np * 0.5, backend)) with pytest.warns(pted.utils.UnderconfidenceWarning): - pted.pted_coverage_test(g, s * 2) + pted.pted_coverage_test(g, _to_backend(s_np * 2, backend)) def test_sbc_histogram(): @@ -204,6 +200,7 @@ def test_sbc_histogram(): s = np.random.normal(size=(150, 100, 10)) # posterior samples (nsamp, nsim, ndim) pted.pted_coverage_test(g, s, permutations=100, sbc_histogram="sbc_hist.pdf") + assert os.path.exists("sbc_hist.pdf"), "SBC histogram file was not created" os.remove("sbc_hist.pdf") @@ -236,61 +233,6 @@ def test_pit_plot_utility_direct(): assert not os.path.exists("pit_single.pdf") -def test_pted_jax(): - if jax is None: - pytest.skip("jax not installed") - - # Set the random seed for reproducibility - np.random.seed(42) - - # example 2 sample test - D = 300 - for _ in range(20): - x = jnp.array(np.random.normal(size=(100, D))) - y = jnp.array(np.random.normal(size=(100, D))) - p = pted.pted(x, y) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - - x = jnp.array(np.random.normal(size=(100, D))) - y = jnp.array(np.random.uniform(size=(100, D))) - p = pted.pted(x, y, two_tailed=False) - assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" - - x = jnp.array(np.random.normal(size=(100, D))) - t, p, _ = pted.pted(x, x, return_all=True) - q = 2 * min(np.sum(p > t), np.sum(p < t)) - p = (1 + q) / (len(p) + 1) - assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" - - -def test_pted_chunk_jax(): - if jax is None: - pytest.skip("jax not installed") - np.random.seed(0) - - # example 2 sample test - D = 3 - x = jnp.array(np.random.normal(size=(200, D))) - y = jnp.array(np.random.normal(size=(200, D))) - p = pted.pted(x, y, chunk_size=50) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - - y = jnp.array(np.random.uniform(size=(300, D))) - p = pted.pted(x, y, chunk_size=50) - assert p < 1e-2, f"p-value {p} is not in the expected range (~0)" - - -def test_pted_coverage_jax(): - if jax is None: - pytest.skip("jax not installed") - - np.random.seed(42) - g = jnp.array(np.random.normal(size=(75, 5))) - s = jnp.array(np.random.normal(size=(50, 75, 5))) - p = pted.pted_coverage_test(g, s) - assert p > 1e-2 and p < 0.99, f"p-value {p} is not in the expected range (U(0,1))" - - def test_is_jax_array_with_jax(): """is_jax_array returns True for a real JAX array and False for other types.""" if jax is None: @@ -306,76 +248,44 @@ def test_is_jax_array_no_jax(monkeypatch): assert pted.utils.is_jax_array(42) is False -def test_jax_cdist(): - """_jax_cdist produces correct pairwise Euclidean distances.""" - if jax is None: - pytest.skip("jax not installed") - x = jnp.array([[0.0, 0.0], [3.0, 4.0]]) - y = jnp.array([[0.0, 0.0], [1.0, 0.0]]) - D = pted.utils._jax_cdist(x, y) - assert D.shape == (2, 2) - assert float(D[0, 1]) == pytest.approx(1.0) - assert float(D[1, 0]) == pytest.approx(5.0) - - -def test_jax_cdist_non_euclidean(): - """_jax_cdist produces correct pairwise distances for p != 2 (vmap path).""" - if jax is None: - pytest.skip("jax not installed") - x = jnp.array([[0.0, 0.0], [3.0, 4.0]]) - y = jnp.array([[0.0, 0.0], [1.0, 0.0]]) - # L1: d(x[0], y[1]) = |0-1| + |0-0| = 1; d(x[1], y[0]) = |3-0| + |4-0| = 7 - D = pted.utils._jax_cdist(x, y, p=1.0) - assert D.shape == (2, 2) - assert float(D[0, 1]) == pytest.approx(1.0) - assert float(D[1, 0]) == pytest.approx(7.0) - - -def test_energy_distance_jax(): - """_energy_distance_jax returns 0 when x and y are identical.""" - if jax is None: - pytest.skip("jax not installed") - np.random.seed(42) - x = jnp.array(np.random.normal(size=(50, 5))) - # Identical samples means energy distance should be ~0 - ed = pted.utils._energy_distance_jax(x, x) - assert abs(ed) < 1e-6 - - def test_pted_jax_no_jax(monkeypatch): - """pted_jax raises AssertionError when JAX is not installed.""" + """pted raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) + monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted_jax(np.zeros((5, 2)), np.zeros((5, 2))) + pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) def test_pted_chunk_jax_no_jax(monkeypatch): - """pted_chunk_jax raises AssertionError when JAX is not installed.""" + """pted_chunk raises AssertionError when a jax array is passed but JAX is not installed.""" monkeypatch.setattr("pted.utils.jax", None) + monkeypatch.setattr("pted.utils.is_jax_array", lambda o: True) with pytest.raises(AssertionError, match="JAX is not installed"): - pted.utils.pted_chunk_jax(np.zeros((5, 2)), np.zeros((5, 2))) + pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) def test_pted_torch_no_torch(monkeypatch): - """pted_torch raises AssertionError when torch is not installed.""" + """pted raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) + monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted_torch(np.zeros((5, 2)), np.zeros((5, 2))) + pted.utils.pted(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10) def test_pted_chunk_torch_no_torch(monkeypatch): - """pted_chunk_torch raises AssertionError when torch is not installed.""" + """pted_chunk raises AssertionError when a torch tensor is passed but torch is not installed.""" fake_torch = types.SimpleNamespace(__version__="null") monkeypatch.setattr("pted.utils.torch", fake_torch) + monkeypatch.setattr("pted.utils.is_torch_tensor", lambda o: True) with pytest.raises(AssertionError, match="PyTorch is not installed"): - pted.utils.pted_chunk_torch(np.zeros((5, 2)), np.zeros((5, 2))) + pted.utils.pted_chunk(np.zeros((5, 2)), np.zeros((5, 2)), permutations=10, chunk_size=2) -def test_jax_cdist_matches_scipy(): +@pytest.mark.parametrize("backend", ["torch", "jax"]) +def test_cdist_matches_scipy(backend): """_jax_cdist (L2) and scipy cdist produce the same pairwise distances.""" - if jax is None: - pytest.skip("jax not installed") + _require_backend(backend) from scipy.spatial.distance import cdist as scipy_cdist np.random.seed(7) @@ -383,50 +293,7 @@ def test_jax_cdist_matches_scipy(): y_np = np.random.normal(size=(8, 4)).astype(np.float32) expected = scipy_cdist(x_np, y_np, metric="euclidean") - got = np.array(pted.utils._jax_cdist(jnp.array(x_np), jnp.array(y_np))) - np.testing.assert_allclose(got, expected, rtol=1e-5) - - -def test_energy_distance_numpy_torch_jax_agree(): - """_energy_distance_{numpy,torch,jax} return the same value for identical inputs.""" - if torch is None: - pytest.skip("torch not installed") - if jax is None: - pytest.skip("jax not installed") - - np.random.seed(99) - # Use float32 so all backends operate at the same precision - # (JAX uses float32 by default) - x_np = np.random.normal(size=(30, 5)).astype(np.float32) - y_np = np.random.normal(size=(30, 5)).astype(np.float32) - - ed_numpy = pted.utils._energy_distance_numpy(x_np, y_np) - ed_torch = pted.utils._energy_distance_torch(torch.tensor(x_np), torch.tensor(y_np)) - ed_jax = pted.utils._energy_distance_jax(jnp.array(x_np), jnp.array(y_np)) - - assert ed_numpy == pytest.approx( - ed_torch, rel=1e-4 - ), f"numpy ({ed_numpy}) and torch ({ed_torch}) energy distances differ" - assert ed_numpy == pytest.approx( - ed_jax, rel=1e-4 - ), f"numpy ({ed_numpy}) and jax ({ed_jax}) energy distances differ" - - -def test_energy_distance_estimate_matches_energy_distance(): - """_energy_distance_estimate returns a value close to _energy_distance for small inputs.""" - np.random.seed(123) - x = np.random.normal(size=(100, 5)) - y = np.random.uniform(size=(100, 5)) - - ed_direct = pted.utils._energy_distance_numpy(x, y) - ed_estimate = pted.utils._energy_distance_estimate( - x, - y, - chunk_size=25, - metric="euclidean", - energy_distance_fn=pted.utils._energy_distance_numpy, + got = np.asarray( + pted.utils._cdist(_to_backend(x_np, backend), _to_backend(y_np, backend), backend=backend) ) - - assert ed_direct == pytest.approx( - ed_estimate, rel=1.5e-1 - ), f"direct ({ed_direct}) and estimate ({ed_estimate}) energy distances differ" + assert np.allclose(got, expected, rtol=1e-5) diff --git a/tests/test_utils.py b/tests/test_utils.py index f257ac3..aac3f3a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,8 +7,22 @@ simulation_based_calibration_histogram, pit_plot, hdp_coverage_test, + _to_scalar, + _random_permutation, ) +try: + import torch +except ImportError: + torch = None + +try: + import jax + import jax.numpy as jnp +except ImportError: + jax = None + jnp = None + import pytest @@ -23,9 +37,6 @@ def test_two_tailed_p(): assert two_tailed_p(0, 10) < 0.01 assert two_tailed_p(1e-25, 1000) < 0.01 - with pytest.raises(AssertionError): - two_tailed_p(4, 2) - def test_sbc_histogram(monkeypatch): @@ -67,3 +78,43 @@ def test_hdp_coverage_test(): assert pvalue < 0.01, "p-value should be small for poorly calibrated posterior samples" pvalue = hdp_coverage_test(ground_truth, posterior_samples, two_tailed=False) assert pvalue > 0.01, "p-value should not be small for underconfident and one_tailed test" + + +@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) +def test_to_scalar(backend): + if backend == "torch" and torch is None: + pytest.skip("torch not installed") + if backend == "jax" and jax is None: + pytest.skip("jax not installed") + + if backend == "numpy": + x = np.array(5) + + elif backend == "torch": + x = torch.tensor(5) + + elif backend == "jax": + x = jnp.array(5) + + # Test with scalar + assert _to_scalar(x, backend) == 5 + + +@pytest.mark.parametrize("backend", ["numpy", "torch", "jax"]) +def test_random_permutation(backend): + if backend == "torch" and torch is None: + pytest.skip("torch not installed") + if backend == "jax" and jax is None: + pytest.skip("jax not installed") + + D = np.arange(16).reshape(4, 4) + if backend == "torch": + D = torch.tensor(D) + elif backend == "jax": + D = jnp.array(D) + + permuted_D = _random_permutation(D, backend=backend) + + assert set(np.array(permuted_D.flatten()).tolist()) == set( + np.array(D.flatten()).tolist() + ), "Permutation should contain all original elements"