Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,16 @@ line.

![pted demo test](media/test_PTED.png)

You can produce this centre panel for your own data by passing a filename to
`pted`: `distance_histogram = "components.pdf"`. It plots the three
pairwise-distance densities the energy distance is built from, with each one's
null band from `distance_permutations` relabellings behind it. A significant
p-value tells you the samples differ; this tells you *how* -- one sample
tighter than the other, offset from it, or multimodal. The bands are per
component because the three rest on very different numbers of pairs, so they
are not equally noisy; they are pointwise and meant for reading shape, not as
a decision rule.

In the left figure, we show the two distributions, which by eye are clearly not
drawn from the same distribution (circle and line). In the center figure we show
the individual distance measurements as histograms. To compute the energy
Expand Down
8 changes: 4 additions & 4 deletions media/plot_pted_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,11 @@ def plot_demo(x, y, name, E_permute=None):
D = cdist(z, z)

hist, bins = np.histogram(D[:N, :N].ravel(), bins = 50, density=True)
axarr[1].step(bins[:-1], hist, where="post", color = "b", linewidth = 3)
axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "b", linewidth = 3)
hist, bins = np.histogram(D[N:, N:].ravel(), bins = 50, density=True)
axarr[1].step(bins[:-1], hist, where="post", color = "r", linewidth = 3)
axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "r", linewidth = 3)
hist, bins = np.histogram(D[:N, N:].ravel(), bins = 50, density=True)
axarr[1].step(bins[:-1], hist, where="post", color = "purple", linewidth = 3)
axarr[1].step(bins, np.append(hist, hist[-1]), where="post", color = "purple", linewidth = 3)

bx, by = 0.7, 0.7
dx, dy = 0.13, 0.08
Expand All @@ -67,7 +67,7 @@ def plot_demo(x, y, name, E_permute=None):
E_test = (2 * np.sum(D[:N,N:]) - np.sum(D[:N,:N]) - np.sum(D[N:,N:])) / N**2
if E_permute is not None:
hist, bins = np.histogram(E_permute, bins = 15)
axarr[2].step(bins[:-1], hist, where="post", color = "grey", linewidth = 3, label="permutations")
axarr[2].step(bins, np.append(hist, hist[-1]), where="post", color = "grey", linewidth = 3, label="permutations")
axarr[2].axvline(E_test, linewidth = 3, color="k", label="test")
axarr[2].legend()
axarr[2].set_xlabel("Energy [2 * Exy - Exx - Eyy]")
Expand Down
38 changes: 29 additions & 9 deletions src/pted/pted.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
_prepare_test,
_permutation_distribution,
_draw_containment_pit,
_draw_distance_histogram,
_as_rng,
two_tailed_p,
confidence_alert,
Expand All @@ -27,6 +28,9 @@ def pted(
prog_bar: bool = False,
batch_size: Optional[int] = None,
rng=None,
distance_histogram: Optional[str] = None,
distance_permutations: int = 200,
distance_bins: int = 50,
) -> Union[float, tuple[float, np.ndarray, float]]:
"""
Two sample null hypothesis test using a permutation test on the energy
Expand Down Expand Up @@ -109,6 +113,15 @@ def pted(
batch to a few million elements.
rng: seed, ``np.random.Generator``, or None to draw from the global
numpy state, so ``np.random.seed`` still controls reproducibility.
distance_histogram (Optional[str]): if given, the path to save a
diagnostic plot of the three pairwise-distance distributions the
energy distance is built from, against a permuted null band. Shows
*where* two samples differ, not just that they do. See
:func:`pted.utils.distance_histogram`.
distance_permutations (int): relabellings behind that plot's null
band; 0 omits the band. Each costs a pass over the distance
matrix, so this is far smaller than ``permutations``.
distance_bins (int): histogram bins for that plot.

Note
----
Expand Down Expand Up @@ -156,15 +169,22 @@ def pted(

# A landmark count covering the whole pooled sample lands in the exact
# full-matrix regime, so there is no separate branch for it here.
test, permute = _energy_test(
x,
y,
permutations=permutations,
n_landmarks=n_landmarks,
prog_bar=prog_bar,
batch_size=batch_size,
rng=rng,
)
# Prepared once and used twice, so the diagnostic shows the same landmarks
# and the same distance matrix the statistic was computed from.
prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=False)
test, permute = _permutation_distribution(prep, alloc, permutations, prog_bar, batch_size, rng)

if distance_histogram is not None:
_draw_distance_histogram(
prep,
alloc,
len(x),
distance_histogram,
distance_permutations,
distance_bins,
batch_size,
rng,
)

# Compute p-value. The null set is `permutations` random draws, or the
# whole permutation subgroup when that is smaller, so count what came back
Expand Down
192 changes: 192 additions & 0 deletions src/pted/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@
"simulation_based_calibration_histogram",
"pit_plot",
"containment_pit_plot",
"distance_histogram",
"hdp_coverage_test",
)

Expand Down Expand Up @@ -413,7 +414,7 @@
)
else:
hint = f"use more landmarks (currently {alloc['n_landmarks']}) or more samples"
warn(

Check warning on line 417 in src/pted/utils.py

View workflow job for this annotation

GitHub Actions / build (3.13, ubuntu-latest)

the 'full' allocation reaches only 6 distinct label assignments, so the smallest attainable p-value is about 0.167 no matter how many permutations are drawn (0 requested): the samples are too small for a permutation test at this resolution.

Check warning on line 417 in src/pted/utils.py

View workflow job for this annotation

GitHub Actions / build (3.13, ubuntu-latest)

the 'singleton' allocation reaches only 11 distinct label assignments, so the smallest attainable p-value is about 0.0909 no matter how many permutations are drawn (49 requested): the small group holds a single point, so the reference set is n - n_landmarks; use fewer landmarks (currently 30), well under half the pooled sample size.

Check warning on line 417 in src/pted/utils.py

View workflow job for this annotation

GitHub Actions / build (3.13, macOS-latest)

the 'full' allocation reaches only 6 distinct label assignments, so the smallest attainable p-value is about 0.167 no matter how many permutations are drawn (0 requested): the samples are too small for a permutation test at this resolution.

Check warning on line 417 in src/pted/utils.py

View workflow job for this annotation

GitHub Actions / build (3.13, macOS-latest)

the 'singleton' allocation reaches only 11 distinct label assignments, so the smallest attainable p-value is about 0.0909 no matter how many permutations are drawn (49 requested): the small group holds a single point, so the reference set is n - n_landmarks; use fewer landmarks (currently 30), well under half the pooled sample size.
PermutationResolutionWarning(
f"the {alloc['regime']!r} allocation reaches only {reference:,} distinct "
f"label assignments, so the smallest attainable p-value is about "
Expand Down Expand Up @@ -1457,6 +1458,197 @@
plt.close(fig)


def _component_bin_counts(bidx, bins, columns, u, rows_per_chunk):
"""Bin counts of the three energy-distance components for one labelling.

``u`` is the 0/1 x-membership of every pooled point and ``u[columns]`` is
the same membership for the matrix columns, so every entry of the matrix
falls into one of four classes. Encoding the class into the bin index lets
a single ``bincount`` pass count all four at once, which is far cheaper
than taking four boolean-mask copies of the matrix per permutation.
"""
v = u[columns]
stride = bins + 1 # the extra slot swallows the self-pairs
counts = np.zeros(4 * stride, dtype=np.int64)
for start in range(0, len(u), rows_per_chunk):
rows = slice(start, start + rows_per_chunk)
cls = u[rows, None] + 2 * v[None, :]
counts += np.bincount((bidx[rows] + stride * cls).ravel(), minlength=4 * stride)
counts = counts.reshape(4, stride)[:, :bins]
# class 3 = x row and x column, 0 = y row and y column, 1 | 2 = cross
return counts[3], counts[0], counts[1] + counts[2]
Comment thread
ConnorStoneAstro marked this conversation as resolved.
Resolved


def _as_density(counts, width):
"""Normalise bin counts to a density, so unequal sample sizes compare."""
total = counts.sum()
return counts / (total * width) if total else np.zeros(len(counts))


def _draw_distance_histogram(prep, alloc, n1, saveto, permutations, bins, batch_size, rng):
"""Draw the distance histogram from an already-prepared statistic."""
try:
import matplotlib.pyplot as plt
except ImportError:
warn("No distance histogram generated! Please install matplotlib.")
return
if "D" not in prep:
# The singleton preparer keeps only the row sums, and rightly so: a
# one-point sample has no within-group distances to show anyway.
warn(
"No distance histogram generated! The 'singleton' landmark allocation "
"reduces the distance matrix to its row sums, and a one-point sample "
"has no within-group pairs to histogram."
)
return

backend = prep["backend"]
# The preparer centred the matrix and zeroed the self-pairs. Undo the
# centring to recover the distances themselves; the self-pairs are then
# dropped rather than restored, exactly as the statistic drops them.
D = _to_numpy(prep["D"], backend) + prep["offset"]
n, m = D.shape
columns = np.arange(n) if alloc["regime"] == "full" else alloc["landmarks"]

low, high = float(D.min()), float(D.max())
if not high > low: # every pair at the same distance
high = low + 1.0
edges = np.linspace(low, high, bins + 1)
width = edges[1] - edges[0]

# One pass holds a (rows, m) temporary, so chunk the rows to bound it.
rows_per_chunk = max(1, _BATCH_BYTES // (16 * max(m, 1)))
bidx = np.empty((n, m), dtype=np.int32)
for start in range(0, n, rows_per_chunk):
rows = slice(start, start + rows_per_chunk)
bidx[rows] = np.digitize(D[rows], edges[1:-1])
bidx[columns, np.arange(m)] = bins # self-pairs land in the discarded slot
del D

observed = (np.arange(n) < n1).astype(np.int32)
seen = np.array(
[
_as_density(c, width)
for c in _component_bin_counts(bidx, bins, columns, observed, rows_per_chunk)
]
)

# Under the null all three components estimate the same density, so they
# share one median curve -- but NOT one band. They rest on n1 * (n1 - 1),
# n2 * (n2 - 1) and 2 * n1 * n2 pairs respectively, so a relabelling moves
# them by quite different amounts and each needs its own spread. Pooling
# the three into a single band leaves the smallest group's curve escaping
# a 16-84 band far more than 32% of the time and the cross term far less.
#
# Only the curves are kept, which are tiny; the labels behind them are a
# (rows, n) block, so they are drawn in batches like the test's own.
null = None
if permutations > 0:
draw = _label_drawer(prep, rng)
batch_size = max(1, int(prep["batch_size"] if batch_size is None else batch_size))
curves = []
for start in range(0, permutations, batch_size):
U = _to_numpy(draw(min(batch_size, permutations - start)), backend).astype(np.int32)
if n1 > n - n1: # the drawer marks the smaller group, which here is y
U = 1 - U
curves.extend(
[
_as_density(c, width)
for c in _component_bin_counts(bidx, bins, columns, labels, rows_per_chunk)
]
for labels in U
)
null = np.array(curves) # (permutations, component, bin)

# A `where="post"` step draws from x[i] to x[i+1], so a histogram has to
# carry the right-hand edge and repeat the final height -- otherwise the
# last bin spans nothing and is silently never drawn.
stair = lambda heights: np.append(heights, heights[-1])
colours = ("b", "r", "purple")
labels = ("x to x", "y to y", "x to y")
fig, ax = plt.subplots()
if null is not None:
for k, colour in enumerate(colours):
low_k, high_k = np.percentile(null[:, k], [16, 84], axis=0)
ax.fill_between(
edges, stair(low_k), stair(high_k), step="post",
color=colour, alpha=0.2, zorder=1,
) # fmt: skip
ax.fill_between([], [], [], color="0.6", label="permuted 16-84%") # one legend entry
ax.step(
edges, stair(np.median(null.reshape(-1, bins), 0)), where="post",
color="k", linewidth=1.4, zorder=2, label="permuted median",
) # fmt: skip
for k, (colour, label) in enumerate(zip(colours, labels)):
ax.step(
edges, stair(seen[k]), where="post",
color=colour, linewidth=2.5, zorder=3 + k, label=label,
) # fmt: skip
ax.set_xlabel("Distance [sample pairs]")
ax.set_ylabel("density")
ax.set_title("Energy distance components")
ax.legend()
fig.savefig(saveto, bbox_inches="tight")
plt.close(fig)


def distance_histogram(
x,
y,
saveto,
permutations: int = 200,
bins: int = 50,
n_landmarks: Optional[int] = None,
batch_size: Optional[int] = None,
rng=None,
):
"""Diagnostic plot for :func:`pted.pted`: *where* the two samples differ.

The energy distance is built from three mean pairwise distances -- within
x, within y, and across the two. This plots the whole distribution behind
each of those means rather than the mean alone, so a significant p-value
can be read as a shape: x more tightly clustered than y, the two centred
in different places, one of them bimodal. The histograms are densities, so
the three stay comparable when the samples are different sizes.

Behind them is the null: the same three components recomputed over
``permutations`` relabellings, drawn from the same subgroup the test
itself uses. Under the null all three estimate one common density, which
is the black median line, so any separation between the coloured curves is
the signal the energy distance is responding to.

Each component gets its own shaded band, in its own colour, because they
rest on very different numbers of pairs -- ``n1 * (n1 - 1)``,
``n2 * (n2 - 1)`` and ``2 * n1 * n2``. A relabelling therefore moves them
by different amounts, and one shared band would be far too tight for the
smaller sample and far too loose for the cross term. Where the samples are
the same size the three bands nearly coincide and read as one.

The bands are pointwise, not simultaneous: roughly a third of the bins of
a perfectly null curve stray outside, and neighbouring bins stray together
because the curve is smooth. Read it for shape, not as a decision rule --
the p-value is the test.

Parameters
----------
x, y: the samples, as passed to :func:`pted.pted`. Shape ``(n1, d)``,
``(n2, d)``.
saveto (str): file path; the format follows the extension.
permutations (int): relabellings behind the null band; 0 omits it.
Each one costs a pass over the distance matrix, so this is
typically far smaller than the test's own permutation count.
bins (int): histogram bins, shared by all three components.
n_landmarks: as for :func:`pted.pted`. With landmarks the histogram
covers the pairs the statistic actually used, not every pair.
batch_size (Optional[int]): relabellings drawn per batch, bounding the
``O(batch_size * n)`` label block. None reuses the size the
statistic chose for itself.
rng: seed, ``np.random.Generator``, or None.
"""
prep, alloc, rng = _prepare_test(x, y, n_landmarks, rng, containment=False)
_draw_distance_histogram(prep, alloc, len(x), saveto, permutations, bins, batch_size, rng)


def hdp_coverage_test(
ground_truth: np.ndarray, posterior_samples: np.ndarray, two_tailed: bool = True
) -> float:
Expand Down
Loading
Loading