diff --git a/README.md b/README.md index 201b87d..8714190 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,16 @@ line. ![pted demo test](media/test_PTED.png) +You can produce this centre panel for your own data by passing a filename to +`pted`: `distance_histogram = "components.pdf"`. It plots the three +pairwise-distance densities the energy distance is built from, with each one's +null band from `distance_permutations` relabellings behind it. A significant +p-value tells you the samples differ; this tells you *how* -- one sample +tighter than the other, offset from it, or multimodal. The bands are per +component because the three rest on very different numbers of pairs, so they +are not equally noisy; they are pointwise and meant for reading shape, not as +a decision rule. + In the left figure, we show the two distributions, which by eye are clearly not drawn from the same distribution (circle and line). In the center figure we show the individual distance measurements as histograms. To compute the energy diff --git a/media/plot_pted_demo.py b/media/plot_pted_demo.py index 094aa10..eb62688 100644 --- a/media/plot_pted_demo.py +++ b/media/plot_pted_demo.py @@ -45,11 +45,11 @@ def plot_demo(x, y, name, E_permute=None): D = cdist(z, z) hist, bins = np.histogram(D[:N, :N].ravel(), bins = 50, density=True) - axarr[1].step(bins[:-1], hist, where="post", color = "b", linewidth = 3) + axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "b", linewidth = 3) hist, bins = np.histogram(D[N:, N:].ravel(), bins = 50, density=True) - axarr[1].step(bins[:-1], hist, where="post", color = "r", linewidth = 3) + axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "r", linewidth = 3) hist, bins = np.histogram(D[:N, N:].ravel(), bins = 50, density=True) - axarr[1].step(bins[:-1], hist, where="post", color = "purple", linewidth = 3) + axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "purple", linewidth = 3) bx, by = 0.7, 0.7 dx, dy = 0.13, 0.08 @@ -67,7 +67,7 @@ def plot_demo(x, y, name, E_permute=None): E_test = (2 * np.sum(D[:N,N:]) - np.sum(D[:N,:N]) - np.sum(D[N:,N:])) / N**2 if E_permute is not None: hist, bins = np.histogram(E_permute, bins = 15) - axarr[2].step(bins[:-1], hist, where="post", color = "grey", linewidth = 3, label="permutations") + axarr[2].step(bins, np.append(hist, hist[-1]), where="post", color = "grey", linewidth = 3, label="permutations") axarr[2].axvline(E_test, linewidth = 3, color="k", label="test") axarr[2].legend() axarr[2].set_xlabel("Energy [2 * Exy - Exx - Eyy]") diff --git a/src/pted/pted.py b/src/pted/pted.py index 4e92ba4..7987680 100644 --- a/src/pted/pted.py +++ b/src/pted/pted.py @@ -7,6 +7,7 @@ _prepare_test, _permutation_distribution, _draw_containment_pit, + _draw_distance_histogram, _as_rng, two_tailed_p, confidence_alert, @@ -27,6 +28,9 @@ def pted( prog_bar: bool = False, batch_size: Optional[int] = None, rng=None, + distance_histogram: Optional[str] = None, + distance_permutations: int = 200, + distance_bins: int = 50, ) -> Union[float, tuple[float, np.ndarray, float]]: """ Two sample null hypothesis test using a permutation test on the energy @@ -109,6 +113,15 @@ def pted( batch to a few million elements. rng: seed, ``np.random.Generator``, or None to draw from the global numpy state, so ``np.random.seed`` still controls reproducibility. + distance_histogram (Optional[str]): if given, the path to save a + diagnostic plot of the three pairwise-distance distributions the + energy distance is built from, against a permuted null band. Shows + *where* two samples differ, not just that they do. See + :func:`pted.utils.distance_histogram`. + distance_permutations (int): relabellings behind that plot's null + band; 0 omits the band. Each costs a pass over the distance + matrix, so this is far smaller than ``permutations``. + distance_bins (int): histogram bins for that plot. Note ---- @@ -156,15 +169,22 @@ def pted( # A landmark count covering the whole pooled sample lands in the exact # full-matrix regime, so there is no separate branch for it here. - test, permute = _energy_test( - x, - y, - permutations=permutations, - n_landmarks=n_landmarks, - prog_bar=prog_bar, - batch_size=batch_size, - rng=rng, - ) + # Prepared once and used twice, so the diagnostic shows the same landmarks + # and the same distance matrix the statistic was computed from. + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=False) + test, permute = _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng) + + if distance_histogram is not None: + _draw_distance_histogram( + prep, + alloc, + len(x), + distance_histogram, + distance_permutations, + distance_bins, + batch_size, + rng, + ) # Compute p-value. The null set is `permutations` random draws, or the # whole permutation subgroup when that is smaller, so count what came back diff --git a/src/pted/utils.py b/src/pted/utils.py index 80e913c..5881c29 100644 --- a/src/pted/utils.py +++ b/src/pted/utils.py @@ -102,6 +102,7 @@ class torch: "simulation_based_calibration_histogram", "pit_plot", "containment_pit_plot", + "distance_histogram", "hdp_coverage_test", ) @@ -1457,6 +1458,197 @@ def _draw_containment_pit(prep: dict, saveto, confidence: float, threshold: floa plt.close(fig) +def _component_bin_counts(bidx, bins, columns, u, rows_per_chunk): + """Bin counts of the three energy-distance components for one labelling. + + ``u`` is the 0/1 x-membership of every pooled point and ``u[columns]`` is + the same membership for the matrix columns, so every entry of the matrix + falls into one of four classes. Encoding the class into the bin index lets + a single ``bincount`` pass count all four at once, which is far cheaper + than taking four boolean-mask copies of the matrix per permutation. + """ + v = u[columns] + stride = bins + 1 # the extra slot swallows the self-pairs + counts = np.zeros(4 * stride, dtype=np.int64) + for start in range(0, len(u), rows_per_chunk): + rows = slice(start, start + rows_per_chunk) + cls = u[rows, None] + 2 * v[None, :] + counts += np.bincount((bidx[rows] + stride * cls).ravel(), minlength=4 * stride) + counts = counts.reshape(4, stride)[:, :bins] + # class 3 = x row and x column, 0 = y row and y column, 1 | 2 = cross + return counts[3], counts[0], counts[1] + counts[2] + + +def _as_density(counts, width): + """Normalise bin counts to a density, so unequal sample sizes compare.""" + total = counts.sum() + return counts / (total * width) if total else np.zeros(len(counts)) + + +def _draw_distance_histogram(prep, alloc, n1, saveto, permutations, bins, batch_size, rng): + """Draw the distance histogram from an already-prepared statistic.""" + try: + import matplotlib.pyplot as plt + except ImportError: + warn("No distance histogram generated! Please install matplotlib.") + return + if "D" not in prep: + # The singleton preparer keeps only the row sums, and rightly so: a + # one-point sample has no within-group distances to show anyway. + warn( + "No distance histogram generated! The 'singleton' landmark allocation " + "reduces the distance matrix to its row sums, and a one-point sample " + "has no within-group pairs to histogram." + ) + return + + backend = prep["backend"] + # The preparer centred the matrix and zeroed the self-pairs. Undo the + # centring to recover the distances themselves; the self-pairs are then + # dropped rather than restored, exactly as the statistic drops them. + D = _to_numpy(prep["D"], backend) + prep["offset"] + n, m = D.shape + columns = np.arange(n) if alloc["regime"] == "full" else alloc["landmarks"] + + low, high = float(D.min()), float(D.max()) + if not high > low: # every pair at the same distance + high = low + 1.0 + edges = np.linspace(low, high, bins + 1) + width = edges[1] - edges[0] + + # One pass holds a (rows, m) temporary, so chunk the rows to bound it. + rows_per_chunk = max(1, _BATCH_BYTES // (16 * max(m, 1))) + bidx = np.empty((n, m), dtype=np.int32) + for start in range(0, n, rows_per_chunk): + rows = slice(start, start + rows_per_chunk) + bidx[rows] = np.digitize(D[rows], edges[1:-1]) + bidx[columns, np.arange(m)] = bins # self-pairs land in the discarded slot + del D + + observed = (np.arange(n) < n1).astype(np.int32) + seen = np.array( + [ + _as_density(c, width) + for c in _component_bin_counts(bidx, bins, columns, observed, rows_per_chunk) + ] + ) + + # Under the null all three components estimate the same density, so they + # share one median curve -- but NOT one band. They rest on n1 * (n1 - 1), + # n2 * (n2 - 1) and 2 * n1 * n2 pairs respectively, so a relabelling moves + # them by quite different amounts and each needs its own spread. Pooling + # the three into a single band leaves the smallest group's curve escaping + # a 16-84 band far more than 32% of the time and the cross term far less. + # + # Only the curves are kept, which are tiny; the labels behind them are a + # (rows, n) block, so they are drawn in batches like the test's own. + null = None + if permutations > 0: + draw = _label_drawer(prep, rng) + batch_size = max(1, int(prep["batch_size"] if batch_size is None else batch_size)) + curves = [] + for start in range(0, permutations, batch_size): + U = _to_numpy(draw(min(batch_size, permutations - start)), backend).astype(np.int32) + if n1 > n - n1: # the drawer marks the smaller group, which here is y + U = 1 - U + curves.extend( + [ + _as_density(c, width) + for c in _component_bin_counts(bidx, bins, columns, labels, rows_per_chunk) + ] + for labels in U + ) + null = np.array(curves) # (permutations, component, bin) + + # A `where="post"` step draws from x[i] to x[i+1], so a histogram has to + # carry the right-hand edge and repeat the final height -- otherwise the + # last bin spans nothing and is silently never drawn. + stair = lambda heights: np.append(heights, heights[-1]) + colours = ("b", "r", "purple") + labels = ("x to x", "y to y", "x to y") + fig, ax = plt.subplots() + if null is not None: + for k, colour in enumerate(colours): + low_k, high_k = np.percentile(null[:, k], [16, 84], axis=0) + ax.fill_between( + edges, stair(low_k), stair(high_k), step="post", + color=colour, alpha=0.2, zorder=1, + ) # fmt: skip + ax.fill_between([], [], [], color="0.6", label="permuted 16-84%") # one legend entry + ax.step( + edges, stair(np.median(null.reshape(-1, bins), 0)), where="post", + color="k", linewidth=1.4, zorder=2, label="permuted median", + ) # fmt: skip + for k, (colour, label) in enumerate(zip(colours, labels)): + ax.step( + edges, stair(seen[k]), where="post", + color=colour, linewidth=2.5, zorder=3 + k, label=label, + ) # fmt: skip + ax.set_xlabel("Distance [sample pairs]") + ax.set_ylabel("density") + ax.set_title("Energy distance components") + ax.legend() + fig.savefig(saveto, bbox_inches="tight") + plt.close(fig) + + +def distance_histogram( + x, + y, + saveto, + permutations: int = 200, + bins: int = 50, + n_landmarks: Optional[int] = None, + batch_size: Optional[int] = None, + rng=None, +): + """Diagnostic plot for :func:`pted.pted`: *where* the two samples differ. + + The energy distance is built from three mean pairwise distances -- within + x, within y, and across the two. This plots the whole distribution behind + each of those means rather than the mean alone, so a significant p-value + can be read as a shape: x more tightly clustered than y, the two centred + in different places, one of them bimodal. The histograms are densities, so + the three stay comparable when the samples are different sizes. + + Behind them is the null: the same three components recomputed over + ``permutations`` relabellings, drawn from the same subgroup the test + itself uses. Under the null all three estimate one common density, which + is the black median line, so any separation between the coloured curves is + the signal the energy distance is responding to. + + Each component gets its own shaded band, in its own colour, because they + rest on very different numbers of pairs -- ``n1 * (n1 - 1)``, + ``n2 * (n2 - 1)`` and ``2 * n1 * n2``. A relabelling therefore moves them + by different amounts, and one shared band would be far too tight for the + smaller sample and far too loose for the cross term. Where the samples are + the same size the three bands nearly coincide and read as one. + + The bands are pointwise, not simultaneous: roughly a third of the bins of + a perfectly null curve stray outside, and neighbouring bins stray together + because the curve is smooth. Read it for shape, not as a decision rule -- + the p-value is the test. + + Parameters + ---------- + x, y: the samples, as passed to :func:`pted.pted`. Shape ``(n1, d)``, + ``(n2, d)``. + saveto (str): file path; the format follows the extension. + permutations (int): relabellings behind the null band; 0 omits it. + Each one costs a pass over the distance matrix, so this is + typically far smaller than the test's own permutation count. + bins (int): histogram bins, shared by all three components. + n_landmarks: as for :func:`pted.pted`. With landmarks the histogram + covers the pairs the statistic actually used, not every pair. + batch_size (Optional[int]): relabellings drawn per batch, bounding the + ``O(batch_size * n)`` label block. None reuses the size the + statistic chose for itself. + rng: seed, ``np.random.Generator``, or None. + """ + prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=False) + _draw_distance_histogram(prep, alloc, len(x), saveto, permutations, bins, batch_size, rng) + + def hdp_coverage_test( ground_truth: np.ndarray, posterior_samples: np.ndarray, two_tailed: bool = True ) -> float: diff --git a/tests/test_utils.py b/tests/test_utils.py index e571f0e..cb60870 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -719,3 +719,170 @@ def test_containment_plot_reuses_the_test_setup(tmp_path, monkeypatch): ) assert len(seen) == 1, f"rng={label}: allocated landmarks {len(seen)} times" assert os.path.exists(out) + + +# --------------------------------------------------------------------------- +# Distance histogram diagnostic +# --------------------------------------------------------------------------- + + +def test_distance_histogram_components_match_brute_force(): + """The bincount-by-class shortcut must agree with explicit block slicing, + including dropping the self-pairs the statistic also drops.""" + from pted.utils import _component_bin_counts + + rng = np.random.default_rng(3) + n, m, bins = 40, 12, 7 + columns = np.sort(rng.choice(n, size=m, replace=False)) + bidx = rng.integers(0, bins, size=(n, m)).astype(np.int32) + bidx[columns, np.arange(m)] = bins # self-pairs, excluded from every block + u = (rng.random(n) < 0.4).astype(np.int32) + + xx, yy, xy = _component_bin_counts(bidx, bins, columns, u, rows_per_chunk=7) + + rows_x, cols_x = u.astype(bool), u[columns].astype(bool) + want = lambda r, c: np.bincount(bidx[np.ix_(r, c)].ravel(), minlength=bins + 1)[:bins] + assert np.array_equal(xx, want(rows_x, cols_x)) + assert np.array_equal(yy, want(~rows_x, ~cols_x)) + assert np.array_equal(xy, want(rows_x, ~cols_x) + want(~rows_x, cols_x)) + # the self-pairs went nowhere: every real entry is counted exactly once + assert xx.sum() + yy.sum() + xy.sum() == n * m - m + + # chunking is an implementation detail, not a result + for chunk in (1, 3, n, 2 * n): + assert all( + np.array_equal(a, b) + for a, b in zip((xx, yy, xy), _component_bin_counts(bidx, bins, columns, u, chunk)) + ) + + +@pytest.mark.parametrize("n1,n2,m", [(60, 90, None), (60, 90, 40)]) +def test_distance_histogram_writes_a_file(tmp_path, n1, n2, m): + """Both the pted trigger and the standalone entry point produce a figure, + for the full matrix and for a landmark allocation.""" + import pted + from pted.utils import distance_histogram + + rng = np.random.default_rng(0) + x, y = rng.standard_normal((n1, 3)), rng.standard_normal((n2, 3)) + 0.5 + + direct = tmp_path / "direct.png" + distance_histogram(x, y, str(direct), permutations=5, bins=12, n_landmarks=m, rng=0) + assert direct.exists() + + triggered = tmp_path / "triggered.png" + pval = pted.pted( + x, + y, + permutations=20, + n_landmarks=m, + rng=0, + distance_histogram=str(triggered), + distance_permutations=5, + distance_bins=12, + ) + assert triggered.exists() + assert 0.0 < pval <= 1.0 + + # asking for the plot must not perturb the test it came from + assert pval == pted.pted(x, y, permutations=20, n_landmarks=m, rng=0) + + +def test_distance_histogram_skips_when_the_matrix_is_gone(tmp_path): + """The singleton allocation keeps only row sums, so the histogram has + nothing to bin and says so rather than failing late or writing junk.""" + from pted.utils import distance_histogram + + rng = np.random.default_rng(0) + out = tmp_path / "singleton.png" + with pytest.warns(UserWarning, match="singleton"): + distance_histogram( + rng.standard_normal((1, 3)), + rng.standard_normal((60, 3)), + str(out), + permutations=2, + n_landmarks=12, + rng=0, + ) + assert not out.exists() + + +def test_distance_histogram_draws_the_final_bin(tmp_path, monkeypatch): + """A `where="post"` step draws from x[i] to x[i+1], so a histogram must + carry the right-hand edge and repeat the last height. Without that the + final bin spans nothing and is silently dropped from the figure.""" + import matplotlib.pyplot as plt + from pted.utils import distance_histogram + + captured = {} + real_subplots = plt.subplots + + def spy(*args, **kwargs): + fig, ax = real_subplots(*args, **kwargs) + captured["ax"] = ax + return fig, ax + + monkeypatch.setattr(plt, "subplots", spy) + + rng = np.random.default_rng(0) + bins = 9 + distance_histogram( + rng.standard_normal((40, 2)), + rng.standard_normal((50, 2)), + str(tmp_path / "bins.png"), + permutations=3, + bins=bins, + rng=0, + ) + + lines = captured["ax"].lines + assert len(lines) == 4, "three components plus the permuted median" + for line in lines: + xs, ys = line.get_xdata(), line.get_ydata() + assert len(xs) == bins + 1, "the right-hand edge must be carried" + assert len(ys) == bins + 1 + assert ys[-1] == ys[-2], "the final height repeats, or the last bin is not drawn" + assert np.allclose(np.diff(xs), xs[1] - xs[0]), "edges are uniform" + + +def test_distance_histogram_batches_the_labels(tmp_path, monkeypatch): + """Relabellings are drawn in batches summing to `permutations`, so the + (rows, n) label block stays bounded however many are asked for.""" + import pted.utils as U + + drawn = [] + real_drawer = U._label_drawer + + def spy(prep, rng): + draw = real_drawer(prep, rng) + + def wrapped(rows): + drawn.append(rows) + return draw(rows) + + return wrapped + + monkeypatch.setattr(U, "_label_drawer", spy) + + rng = np.random.default_rng(0) + U.distance_histogram( + rng.standard_normal((30, 2)), + rng.standard_normal((40, 2)), + str(tmp_path / "batched.png"), + permutations=7, + bins=6, + batch_size=3, + rng=0, + ) + assert drawn == [3, 3, 1], f"expected an uneven 3+3+1 split, got {drawn}" + + drawn.clear() + U.distance_histogram( + rng.standard_normal((30, 2)), + rng.standard_normal((40, 2)), + str(tmp_path / "unbatched.png"), + permutations=7, + bins=6, + rng=0, + ) + assert sum(drawn) == 7, "an unset batch_size still draws exactly `permutations` rows"