diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc9c5e..efb85bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.13.0] - 2026-11-06 + +### Added + +- `ForcedChoiceModelConfig.scoring_mode` selects `joint` classification (the + existing default) or `independent` scoring. Independent scoring requires + `dual_encoder` and applies one shared scalar head to each option before + softmax, giving a Bradley-Terry choice rule for two alternatives. +- Independent scoring supports participant-specific position intercepts and + scalar scoring heads, retains unselected alternatives in the choice set, and + validates consistent option names across training and prediction. +- The argument-structure acceptability training script reads `encoder_mode` and + `scoring_mode` from its configuration. Checkpoints persist the scoring mode; + checkpoints without the field retain joint classification. + +### Fixed + +- Forced-choice dual encoders train through the same per-option encoding path + used for prediction, avoiding the single-embedding trainer's dimension mismatch. +- The forced-choice custom training loop creates participant parameters before + constructing its optimizer and restores the best encoder, head, and participant + effects when early stopping on held-out cross-entropy. +- Random-effect regularization accumulates penalties on the parameter device. +- Forced-choice metrics use installed scikit-learn functions without downloading + metric scripts or writing a Hugging Face metric cache. + ## [0.12.0] - 2026-11-03 ### Added @@ -766,7 +792,8 @@ guards as type-checkers. - CI/CD: GitHub Actions for testing, docs, PyPI publishing - Read the Docs integration -[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.12.0...HEAD +[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.13.0...HEAD +[0.13.0]: https://github.com/FACTSlab/bead/compare/v0.12.0...v0.13.0 [0.12.0]: https://github.com/FACTSlab/bead/compare/v0.11.0...v0.12.0 [0.11.0]: https://github.com/FACTSlab/bead/compare/v0.10.0...v0.11.0 [0.10.0]: https://github.com/FACTSlab/bead/compare/v0.9.0...v0.10.0 diff --git a/bead/active_learning/models/forced_choice.py b/bead/active_learning/models/forced_choice.py index 8b147d1..d364dd0 100644 --- a/bead/active_learning/models/forced_choice.py +++ b/bead/active_learning/models/forced_choice.py @@ -4,11 +4,13 @@ import tempfile from collections.abc import Callable +from copy import deepcopy from pathlib import Path import numpy as np import torch import torch.nn as nn +from sklearn.metrics import accuracy_score, precision_recall_fscore_support from transformers import ( AutoModel, AutoTokenizer, @@ -23,7 +25,6 @@ from bead.active_learning.models.random_effects import RandomEffectsManager from bead.active_learning.trainers.data_collator import MixedEffectsDataCollator from bead.active_learning.trainers.dataset_utils import items_to_dataset -from bead.active_learning.trainers.metrics import compute_multiclass_metrics from bead.active_learning.trainers.model_wrapper import EncoderClassifierWrapper from bead.config.active_learning import ForcedChoiceModelConfig from bead.items.item import Item @@ -116,6 +117,10 @@ class ForcedChoiceModel(ActiveLearningModel): Supports 2AFC, 3AFC, 4AFC, and general nAFC tasks using any HuggingFace transformer model. Provides two encoding strategies: single encoder (concatenate options) or dual encoder (separate embeddings). + With ``scoring_mode="independent"``, the dual encoder retains the option + dimension and one shared head assigns a scalar to each option. Softmax over + these scores gives choice probabilities. All rendered elements are options + in this mode; their names must match across training and prediction items. Parameters ---------- @@ -253,7 +258,9 @@ def _initialize_classifier(self, num_classes: int) -> None: """ hidden_size = self.encoder.config.hidden_size - if self.config.encoder_mode == "dual_encoder": + if self.config.scoring_mode == "independent": + input_size = hidden_size + elif self.config.encoder_mode == "dual_encoder": input_size = hidden_size * num_classes else: input_size = hidden_size @@ -262,24 +269,27 @@ def _initialize_classifier(self, num_classes: int) -> None: nn.Linear(input_size, 256), nn.ReLU(), nn.Dropout(0.1), - nn.Linear(256, num_classes), + nn.Linear( + 256, 1 if self.config.scoring_mode == "independent" else num_classes + ), ) self.classifier_head.to(self.config.device) def _encode_single(self, texts: list[str]) -> torch.Tensor: - """Encode texts using single encoder strategy. + """Encode a batch of texts with the shared transformer. - Concatenates all option texts with [SEP] tokens and encodes once. + Return the first-token representation of each text. Callers provide + either concatenated alternatives or independently encoded options. Parameters ---------- texts : list[str] - List of concatenated option texts for each item. + Texts to encode, one per tokenized sequence. Returns ------- torch.Tensor - Encoded representations of shape (batch_size, hidden_size). + Encoded representations of shape (len(texts), hidden_size). """ encodings = self.tokenizer( texts, @@ -296,7 +306,9 @@ def _encode_single(self, texts: list[str]) -> torch.Tensor: def _encode_dual(self, options_per_item: list[list[str]]) -> torch.Tensor: """Encode texts using dual encoder strategy. - Encodes each option separately and concatenates embeddings. + Encode options independently with shared transformer weights. + Shared scalar scoring retains the option dimension; joint scoring + concatenates the option embeddings. Parameters ---------- @@ -306,30 +318,14 @@ def _encode_dual(self, options_per_item: list[list[str]]) -> torch.Tensor: Returns ------- torch.Tensor - Concatenated encodings of shape (batch_size, hidden_size * num_options). + Encodings of shape (batch_size, num_options, hidden_size) for shared + scalar scoring, or (batch_size, num_options * hidden_size) otherwise. """ - all_embeddings = [] - - for options in options_per_item: - option_embeddings = [] - for option_text in options: - encodings = self.tokenizer( - [option_text], - padding=True, - truncation=True, - max_length=self.config.max_length, - return_tensors="pt", - ) - encodings = {k: v.to(self.config.device) for k, v in encodings.items()} - - outputs = self.encoder(**encodings) - cls_embedding = outputs.last_hidden_state[0, 0, :] - option_embeddings.append(cls_embedding) - - concatenated = torch.cat(option_embeddings, dim=0) - all_embeddings.append(concatenated) - - return torch.stack(all_embeddings) + texts = [text for options in options_per_item for text in options] + embeddings = self._encode_single(texts) + if self.config.scoring_mode == "independent": + return embeddings.reshape(len(options_per_item), -1, embeddings.shape[-1]) + return embeddings.reshape(len(options_per_item), -1) def _prepare_inputs(self, items: list[Item]) -> torch.Tensor: """Prepare inputs for encoding based on encoder mode. @@ -347,6 +343,14 @@ def _prepare_inputs(self, items: list[Item]) -> torch.Tensor: if self.option_names is None: raise ValueError("Model not initialized. Call train() first.") + if self.config.scoring_mode == "independent": + for item in items: + if set(item.rendered_elements) != set(self.option_names): + raise ValueError( + "independent items must have exactly the fitted option " + f"names {self.option_names} in rendered_elements" + ) + if self.config.encoder_mode == "single_encoder": texts = [] for item in items: @@ -418,7 +422,21 @@ def _prepare_training_data( Prepared items, numeric labels, participant_ids, validation_items, numeric validation_labels. """ - unique_labels = sorted(set(labels)) + if self.config.scoring_mode == "independent": + if not items: + raise ValueError("independent scoring requires nonempty training items") + # include options that were never selected in the training responses + unique_labels = sorted(items[0].rendered_elements) + if len(unique_labels) < 2: + raise ValueError("independent scoring requires at least two options") + for item in items + (validation_items or []): + if sorted(item.rendered_elements) != unique_labels: + raise ValueError( + "independent items must have identical option names " + "in rendered_elements" + ) + else: + unique_labels = sorted(set(labels)) self.num_classes = len(unique_labels) self.option_names = unique_labels @@ -488,8 +506,10 @@ def _do_training( self.option_names[label_idx] for label_idx in validation_labels_numeric ] - # Use HuggingFace Trainer for fixed and random_intercepts modes - if self.config.mixed_effects.mode in ("fixed", "random_intercepts"): + # the generic Trainer wrapper only handles one embedding per item + if self.config.encoder_mode == "single_encoder" and ( + self.config.mixed_effects.mode in ("fixed", "random_intercepts") + ): metrics = self._train_with_huggingface_trainer( items=items, y_numeric=labels_numeric, @@ -498,7 +518,7 @@ def _do_training( validation_labels=validation_labels, ) else: - # Use custom loop for random_slopes mode + # dual encoders and participant heads use the prediction encoding path metrics = self._train_with_custom_loop( items=items, y_numeric=labels_numeric, @@ -703,11 +723,62 @@ def _classification_metrics( dict[str, float] Keys ``accuracy``, ``precision``, ``recall``, ``f1``. """ - from transformers import EvalPrediction # noqa: PLC0415 - proba = self._do_predict_proba(items, participant_ids) - eval_pred = EvalPrediction(predictions=proba, label_ids=np.asarray(y_numeric)) - return compute_multiclass_metrics(eval_pred, num_labels=self.num_classes) + predictions = np.argmax(proba, axis=1) + precision, recall, f1, _ = precision_recall_fscore_support( + y_numeric, predictions, average="macro", zero_division=0 + ) + return { + "accuracy": float(accuracy_score(y_numeric, predictions)), + "precision": float(precision), + "recall": float(recall), + "f1": float(f1), + } + + def _head_logits(self, head: nn.Module, embeddings: torch.Tensor) -> torch.Tensor: + """Apply a head, removing only the scalar output dimension when present.""" + logits = head(embeddings) + if self.config.scoring_mode == "independent": + return logits.squeeze(-1) + return logits + + def _compute_logits( + self, + embeddings: torch.Tensor, + participant_ids: list[str], + *, + training: bool = False, + ) -> torch.Tensor: + """Compute option logits with the configured participant effects.""" + assert self.classifier_head is not None + if self.config.mixed_effects.mode == "fixed": + return self._head_logits(self.classifier_head, embeddings) + + assert self.random_effects is not None + assert self.num_classes is not None + if self.config.mixed_effects.mode == "random_intercepts": + logits = self._head_logits(self.classifier_head, embeddings) + biases = torch.stack( + [ + self.random_effects.get_intercepts( + pid, + n_classes=self.num_classes, + param_name="mu", + create_if_missing=training, + ).to(embeddings.device) + for pid in participant_ids + ] + ) + return logits + biases + + logits_list: list[torch.Tensor] = [] + for i, pid in enumerate(participant_ids): + head = self.random_effects.get_slopes( + pid, fixed_head=self.classifier_head, create_if_missing=training + ).to(embeddings.device) + head.train(training) + logits_list.append(self._head_logits(head, embeddings[i : i + 1])) + return torch.cat(logits_list, dim=0) def _train_with_custom_loop( self, @@ -717,7 +788,7 @@ def _train_with_custom_loop( validation_items: list[Item] | None, validation_labels: list[str] | None, ) -> dict[str, float]: - """Train using custom training loop (for random_slopes mode). + """Train dual encoders or participant heads through the prediction encoder. Parameters ---------- @@ -728,105 +799,98 @@ def _train_with_custom_loop( participant_ids : list[str] Participant IDs. validation_items : list[Item] | None - Validation items. + Held-out items for early stopping. validation_labels : list[str] | None - Validation labels. + Held-out option names. Returns ------- dict[str, float] - Training metrics. + Training metrics and, when provided, validation metrics. """ - # Convert to tensor + assert self.classifier_head is not None + assert self.random_effects is not None + assert self.num_classes is not None + assert self.option_names is not None y = torch.tensor(y_numeric, dtype=torch.long, device=self.config.device) - # Build optimizer parameters + # create participant parameters before collecting optimizer parameters params_to_optimize = list(self.encoder.parameters()) + list( self.classifier_head.parameters() ) - - # Add random effects parameters (for random_slopes) - if self.config.mixed_effects.mode == "random_slopes": - for head in self.random_effects.slopes.values(): + for pid in dict.fromkeys(participant_ids): + if self.config.mixed_effects.mode == "random_intercepts": + bias = self.random_effects.get_intercepts( + pid, n_classes=self.num_classes, param_name="mu" + ) + bias = bias.to(self.config.device).detach().requires_grad_() + self.random_effects.intercepts["mu"][pid] = bias + params_to_optimize.append(bias) + elif self.config.mixed_effects.mode == "random_slopes": + head = self.random_effects.get_slopes( + pid, fixed_head=self.classifier_head + ).to(self.config.device) params_to_optimize.extend(head.parameters()) optimizer = torch.optim.AdamW(params_to_optimize, lr=self.config.learning_rate) criterion = nn.CrossEntropyLoss() - - self.encoder.train() - self.classifier_head.train() + val_ids = ["_validation_"] * len(validation_items or []) + val_y = [self.option_names.index(label) for label in validation_labels or []] + best_loss = float("inf") + best_encoder: dict[str, torch.Tensor] | None = None + best_head: dict[str, torch.Tensor] | None = None + best_effects: RandomEffectsManager | None = None + wait = 0 + epoch_loss = 0.0 for _epoch in range(self.config.num_epochs): - n_batches = ( - len(items) + self.config.batch_size - 1 - ) // self.config.batch_size + # validation switches modules to eval mode at the end of each epoch + self.encoder.train() + self.classifier_head.train() epoch_loss = 0.0 - epoch_correct = 0 - - for i in range(n_batches): - start_idx = i * self.config.batch_size + for start_idx in range(0, len(items), self.config.batch_size): end_idx = min(start_idx + self.config.batch_size, len(items)) - - batch_items = items[start_idx:end_idx] - batch_labels = y[start_idx:end_idx] - batch_participant_ids = participant_ids[start_idx:end_idx] - - embeddings = self._prepare_inputs(batch_items) - - # Forward pass depends on mixed effects mode - if self.config.mixed_effects.mode == "fixed": - # Standard forward pass - logits = self.classifier_head(embeddings) - - elif self.config.mixed_effects.mode == "random_intercepts": - # Fixed head + per-participant bias - logits = self.classifier_head(embeddings) - for j, pid in enumerate(batch_participant_ids): - bias = self.random_effects.get_intercepts( - pid, - n_classes=self.num_classes, - param_name="mu", - create_if_missing=True, - ) - logits[j] = logits[j] + bias - - elif self.config.mixed_effects.mode == "random_slopes": - # Per-participant head - logits_list = [] - for j, pid in enumerate(batch_participant_ids): - participant_head = self.random_effects.get_slopes( - pid, - fixed_head=self.classifier_head, - create_if_missing=True, - ) - logits_j = participant_head(embeddings[j : j + 1]) - logits_list.append(logits_j) - logits = torch.cat(logits_list, dim=0) - - # Data loss + prior regularization - loss_ce = criterion(logits, batch_labels) - loss_prior = self.random_effects.compute_prior_loss() + embeddings = self._prepare_inputs(items[start_idx:end_idx]) + logits = self._compute_logits( + embeddings, participant_ids[start_idx:end_idx], training=True + ) + loss_ce = criterion(logits, y[start_idx:end_idx]) + loss_prior = self.random_effects.compute_prior_loss().to(logits.device) loss = loss_ce + loss_prior - optimizer.zero_grad() loss.backward() optimizer.step() + epoch_loss += loss.item() * (end_idx - start_idx) + epoch_loss /= len(items) + + if validation_items: + proba = self._do_predict_proba(validation_items, val_ids) + chosen = np.clip(proba[np.arange(len(val_y)), val_y], 1e-9, 1.0) + val_loss = float(-np.mean(np.log(chosen))) + if val_loss < best_loss - 1e-4: + best_loss = val_loss + best_encoder = deepcopy(self.encoder.state_dict()) + best_head = deepcopy(self.classifier_head.state_dict()) + best_effects = deepcopy(self.random_effects) + wait = 0 + else: + wait += 1 + if wait >= self.config.early_stopping_patience: + break + + if best_encoder is not None and best_head is not None: + self.encoder.load_state_dict(best_encoder) + self.classifier_head.load_state_dict(best_head) + assert best_effects is not None + self.random_effects = best_effects - epoch_loss += loss.item() - predictions = torch.argmax(logits, dim=1) - epoch_correct += (predictions == batch_labels).sum().item() - - epoch_acc = epoch_correct / len(items) - epoch_loss = epoch_loss / n_batches - - self._is_fitted = True - - metrics: dict[str, float] = { - "train_accuracy": epoch_acc, - "train_loss": epoch_loss, - } + train_scores = self._classification_metrics(items, participant_ids, y_numeric) + metrics = {f"train_{key}": value for key, value in train_scores.items()} + metrics["train_loss"] = epoch_loss + if validation_items: + val_scores = self._classification_metrics(validation_items, val_ids, val_y) + metrics.update({f"val_{key}": value for key, value in val_scores.items()}) - # Estimate variance components if self.config.mixed_effects.estimate_variance_components: var_comps = self.random_effects.estimate_variance_components() if var_comps: @@ -835,31 +899,6 @@ def _train_with_custom_loop( self.variance_history.append(var_comp) metrics["participant_variance"] = var_comp.variance metrics["n_participants"] = var_comp.n_groups - - if validation_items is not None and validation_labels is not None: - self._validate_labels(validation_labels) - - if len(validation_items) != len(validation_labels): - raise ValueError( - f"Number of validation items ({len(validation_items)}) " - f"must match number of validation labels ({len(validation_labels)})" - ) - - # Validation with placeholder participant_ids for mixed effects - if self.config.mixed_effects.mode == "fixed": - val_predictions = self.predict(validation_items, participant_ids=None) - else: - val_participant_ids = ["_validation_"] * len(validation_items) - val_predictions = self.predict( - validation_items, participant_ids=val_participant_ids - ) - val_pred_labels = [p.predicted_class for p in val_predictions] - val_acc = sum( - pred == true - for pred, true in zip(val_pred_labels, validation_labels, strict=True) - ) / len(validation_labels) - metrics["val_accuracy"] = val_acc - return metrics def _do_predict( @@ -879,41 +918,8 @@ def _do_predict( list[ModelPrediction] Predictions. """ - self.encoder.eval() - self.classifier_head.eval() - - with torch.no_grad(): - embeddings = self._prepare_inputs(items) - - # Forward pass depends on mixed effects mode - if self.config.mixed_effects.mode == "fixed": - logits = self.classifier_head(embeddings) - - elif self.config.mixed_effects.mode == "random_intercepts": - logits = self.classifier_head(embeddings) - for i, pid in enumerate(participant_ids): - # Unknown participants: use prior mean (zero bias) - bias = self.random_effects.get_intercepts( - pid, - n_classes=self.num_classes, - param_name="mu", - create_if_missing=False, - ) - logits[i] = logits[i] + bias - - elif self.config.mixed_effects.mode == "random_slopes": - logits_list = [] - for i, pid in enumerate(participant_ids): - # Unknown participants: use fixed head - participant_head = self.random_effects.get_slopes( - pid, fixed_head=self.classifier_head, create_if_missing=False - ) - logits_i = participant_head(embeddings[i : i + 1]) - logits_list.append(logits_i) - logits = torch.cat(logits_list, dim=0) - - proba = torch.softmax(logits, dim=1).cpu().numpy() - pred_classes = torch.argmax(logits, dim=1).cpu().numpy() + proba = self._do_predict_proba(items, participant_ids) + pred_classes = np.argmax(proba, axis=1) predictions = [] for i, item in enumerate(items): @@ -949,42 +955,20 @@ def _do_predict_proba( np.ndarray Probability array of shape (n_items, n_classes). """ + assert self.classifier_head is not None + assert self.num_classes is not None self.encoder.eval() self.classifier_head.eval() - + batches: list[np.ndarray] = [] with torch.no_grad(): - embeddings = self._prepare_inputs(items) - - # Forward pass depends on mixed effects mode - if self.config.mixed_effects.mode == "fixed": - logits = self.classifier_head(embeddings) - - elif self.config.mixed_effects.mode == "random_intercepts": - logits = self.classifier_head(embeddings) - for i, pid in enumerate(participant_ids): - bias = self.random_effects.get_intercepts( - pid, - n_classes=self.num_classes, - param_name="mu", - create_if_missing=False, - ) - # Random-effects params live on CPU; move to the logits - # device (matters on mps/cuda). - logits[i] = logits[i] + bias.to(logits.device) - - elif self.config.mixed_effects.mode == "random_slopes": - logits_list = [] - for i, pid in enumerate(participant_ids): - participant_head = self.random_effects.get_slopes( - pid, fixed_head=self.classifier_head, create_if_missing=False - ).to(embeddings.device) - logits_i = participant_head(embeddings[i : i + 1]) - logits_list.append(logits_i) - logits = torch.cat(logits_list, dim=0) - - proba = torch.softmax(logits, dim=1).cpu().numpy() - - return proba + for start in range(0, len(items), self.config.batch_size): + end = start + self.config.batch_size + embeddings = self._prepare_inputs(items[start:end]) + logits = self._compute_logits(embeddings, participant_ids[start:end]) + batches.append(torch.softmax(logits, dim=1).cpu().numpy()) + if not batches: + return np.empty((0, self.num_classes)) + return np.concatenate(batches, axis=0) def _save_model_components(self, save_path: Path) -> None: """Save model-specific components. diff --git a/bead/active_learning/models/random_effects.py b/bead/active_learning/models/random_effects.py index a40b8e8..07b3b3a 100644 --- a/bead/active_learning/models/random_effects.py +++ b/bead/active_learning/models/random_effects.py @@ -507,7 +507,7 @@ def compute_prior_loss(self) -> torch.Tensor: else: weight = 1.0 - loss += weight * squared_dev + loss = loss.to(squared_dev.device) + weight * squared_dev elif self.config.mode == "random_slopes": for participant_id, head in self.slopes.items(): @@ -523,7 +523,7 @@ def compute_prior_loss(self) -> torch.Tensor: else: weight = 1.0 - loss += weight * squared_dev + loss = loss.to(squared_dev.device) + weight * squared_dev return self.config.regularization_strength * loss diff --git a/bead/config/active_learning.py b/bead/config/active_learning.py index 874cab8..ff6560b 100644 --- a/bead/config/active_learning.py +++ b/bead/config/active_learning.py @@ -55,7 +55,34 @@ class BaseEncoderModelConfig(dx.Model): class ForcedChoiceModelConfig(BaseEncoderModelConfig): - """Forced-choice active-learning model configuration.""" + """Forced-choice active-learning model configuration. + + Attributes + ---------- + scoring_mode : Literal["joint", "independent"] + ``joint`` predicts option logits from a combined representation. + ``independent`` applies one scalar head to each independently encoded + option, then normalizes the scores with softmax. Requires + ``encoder_mode="dual_encoder"``. Defaults to ``joint`` for compatibility + with existing configurations and checkpoints. + + Examples + -------- + >>> config = ForcedChoiceModelConfig( + ... encoder_mode="dual_encoder", scoring_mode="independent" + ... ) + >>> config.scoring_mode + 'independent' + """ + + scoring_mode: Literal["joint", "independent"] = "joint" + + __axioms__ = ( + dx.axiom( + 'scoring_mode != "independent" or encoder_mode == "dual_encoder"', + message="independent scoring requires encoder_mode='dual_encoder'", + ), + ) class CategoricalModelConfig(BaseEncoderModelConfig): diff --git a/docs/api/active_learning.md b/docs/api/active_learning.md index ae57ba8..9a5202c 100644 --- a/docs/api/active_learning.md +++ b/docs/api/active_learning.md @@ -46,6 +46,87 @@ factory. ## Task-Specific Models +### Forced-choice scoring + +`ForcedChoiceModelConfig` separates encoding from scoring. The default +`scoring_mode="joint"` preserves the existing classifier: it predicts all option +logits from either one encoding of the concatenated texts (`single_encoder`) or +concatenated per-option embeddings (`dual_encoder`). + +With `scoring_mode="independent"`, each option is encoded separately and passed +through the same scalar head. For option texts \(s_1,\ldots,s_K\), the model uses + +\[ +a_j=f_\phi(E_\theta(s_j)),\qquad +P(y=j)=\frac{\exp(a_j)}{\sum_{k=1}^{K}\exp(a_k)}. +\] + +The head has a 256-unit ReLU hidden layer, dropout of 0.1 during training, and one +scalar output. For two options, the first option's probability is +\(\sigma(a_1-a_2)\). Scores are defined up to a common additive constant; the +logistic noise scale is fixed to one. Independent scoring requires +`encoder_mode="dual_encoder"`; combining it with `single_encoder` raises a +configuration validation error. + +```python +from uuid import uuid4 + +from bead.active_learning.models.forced_choice import ForcedChoiceModel +from bead.config.active_learning import ForcedChoiceModelConfig +from bead.items.item import Item + +config = ForcedChoiceModelConfig( + model_name="bert-base-uncased", + encoder_mode="dual_encoder", + scoring_mode="independent", + num_epochs=1, +) +items = [ + Item( + item_template_id=uuid4(), + rendered_elements={"option_a": "The cat sleeps.", "option_b": "Cat the sleeps."}, + ), + Item( + item_template_id=uuid4(), + rendered_elements={"option_a": "Dogs the bark.", "option_b": "The dogs bark."}, + ), +] +model = ForcedChoiceModel(config) +metrics = model.train(items, ["option_a", "option_b"]) +probabilities = model.predict_proba(items) +``` + +In independent mode, every `rendered_elements` entry is an alternative. The +model infers option names from the first training item, sorts them for probability +column order, and requires the same names on subsequent training, validation, +and prediction items. An option remains in the denominator even when no training +response selects it. Each fitted model has a fixed option vocabulary; the scalar +head's parameter shapes do not depend on the number of alternatives. + +At prediction time, swapping option texts swaps their probabilities, identical +texts receive equal scores, and changing a third option leaves the odds between +the first two unchanged. These properties hold for the shared head and for each +participant-specific scalar head. Position-specific intercepts can break the +swap and tie properties. + +The existing `mixed_effects.mode` options also apply: + +- `fixed`: one scalar head shared across participants. +- `random_intercepts`: a participant-specific bias for each option position is + added to the scalar scores. A common scalar bias would cancel under softmax. +- `random_slopes`: a participant-specific copy of the scalar head scores every + option. The existing head initialization and regularization rules apply. + +Unseen participants use the shared head, with equal prior-mean biases in +intercept mode. These are fallback predictions, not averages over participant +effects. Dual-encoder training uses the same input preparation as prediction, +includes participant parameters in the optimizer, and supports held-out +cross-entropy early stopping. Checkpoints retain both architecture settings; +older checkpoints without `scoring_mode` load as `joint`. + +The argument-structure gallery accepts `encoder_mode: dual_encoder` and +`scoring_mode: independent` in its `acceptability_model` configuration section. + ::: bead.active_learning.models.forced_choice options: show_root_heading: true diff --git a/gallery/eng/argument_structure/train_acceptability_model.py b/gallery/eng/argument_structure/train_acceptability_model.py index 709d586..cb9f7fb 100644 --- a/gallery/eng/argument_structure/train_acceptability_model.py +++ b/gallery/eng/argument_structure/train_acceptability_model.py @@ -123,8 +123,8 @@ def build_model_config(section: dict[str, object]) -> ForcedChoiceModelConfig: Returns ------- ForcedChoiceModelConfig - Config wired for single-encoder 2AFC training with participant random - effects. + Config for 2AFC training with the selected encoder, scoring head, and + participant effects. Independent scoring requires dual encoding. """ from bead.active_learning.config import MixedEffectsConfig # noqa: PLC0415 from bead.config.active_learning import ForcedChoiceModelConfig # noqa: PLC0415 @@ -132,7 +132,8 @@ def build_model_config(section: dict[str, object]) -> ForcedChoiceModelConfig: mode = str(section.get("mixed_effects_mode", DEFAULT_MIXED_EFFECTS_MODE)) return ForcedChoiceModelConfig( model_name=str(section.get("model_name", DEFAULT_MODEL_NAME)), - encoder_mode="single_encoder", + encoder_mode=str(section.get("encoder_mode", "single_encoder")), + scoring_mode=str(section.get("scoring_mode", "joint")), learning_rate=float(section.get("learning_rate", DEFAULT_LEARNING_RATE)), batch_size=int(section.get("batch_size", DEFAULT_BATCH_SIZE)), num_epochs=int(section.get("epochs", DEFAULT_EPOCHS)), @@ -160,6 +161,7 @@ def run_self_test(config_path: Path) -> int: { "model_name": model_config.model_name, "encoder_mode": model_config.encoder_mode, + "scoring_mode": model_config.scoring_mode, "learning_rate": str(model_config.learning_rate), "batch_size": str(model_config.batch_size), "num_epochs": str(model_config.num_epochs), diff --git a/pyproject.toml b/pyproject.toml index a20466b..68a1616 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bead" -version = "0.12.0" +version = "0.13.0" description = "Lexicon and Template Collection Construction Pipeline for Acceptability and Inference Judgment Data" authors = [{name = "Aaron Steven White", email = "aaron.white@rochester.edu"}] readme = "README.md" diff --git a/tests/active_learning/models/forced_choice/test_independent_scoring.py b/tests/active_learning/models/forced_choice/test_independent_scoring.py new file mode 100644 index 0000000..df7fd51 --- /dev/null +++ b/tests/active_learning/models/forced_choice/test_independent_scoring.py @@ -0,0 +1,521 @@ +"""Behavioral tests for independent forced-choice scoring with a local encoder.""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import numpy as np +import pytest +import torch +from torch import nn +from transformers import BertConfig, BertModel, BertTokenizer + +from bead.active_learning.config import MixedEffectsConfig +from bead.active_learning.models.forced_choice import ForcedChoiceModel +from bead.active_learning.models.random_effects import RandomEffectsManager +from bead.config.active_learning import ForcedChoiceModelConfig +from bead.items.item import Item +from bead.items.scoring import AcceptabilityScorer + + +@pytest.fixture(scope="module") +def tiny_encoder(tmp_path_factory): + """Save an actual small transformer and tokenizer, requiring no downloads.""" + directory = tmp_path_factory.mktemp("forced-choice-encoder") + vocabulary = [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "good", + "bad", + "other", + "the", + "cat", + "sat", + "on", + "mat", + ".", + ] + vocab_path = directory / "vocab.txt" + vocab_path.write_text("\n".join(vocabulary) + "\n") + # Transformers 5 renamed vocab_file to vocab; both accept the first argument. + BertTokenizer(str(vocab_path)).save_pretrained(directory) + with torch.random.fork_rng(): + torch.manual_seed(17) + BertModel( + BertConfig( + vocab_size=len(vocabulary), + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=24, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=64, + ) + ).save_pretrained(directory) + return str(directory) + + +@pytest.fixture(autouse=True) +def deterministic_torch(): + threads = torch.get_num_threads() + torch.set_num_threads(1) + with torch.random.fork_rng(): + torch.manual_seed(29) + yield + torch.set_num_threads(threads) + + +def item(*texts: str) -> Item: + return Item( + item_template_id=uuid4(), + rendered_elements={f"option_{i}": text for i, text in enumerate(texts)}, + ) + + +def make_model(tiny_encoder, mode="fixed", **overrides): + values = { + "model_name": tiny_encoder, + "encoder_mode": "dual_encoder", + "scoring_mode": "independent", + "max_length": 32, + "batch_size": 3, + "num_epochs": 2, + "learning_rate": 0.01, + "device": "cpu", + "mixed_effects": MixedEffectsConfig( + mode=mode, prior_variance=0.0, regularization_strength=0.001 + ), + } + values.update(overrides) + return ForcedChoiceModel(ForcedChoiceModelConfig(**values)) + + +def initialize(model, n_options=2): + model.option_names = [f"option_{i}" for i in range(n_options)] + model.num_classes = n_options + model._initialize_classifier(n_options) + model._initialize_random_effects(n_options) + model._is_fitted = True + return model + + +def test_local_encoder_preserves_alternative_tokens(tiny_encoder): + """The saved tokenizer keeps content words distinct after model loading.""" + model = make_model(tiny_encoder) + tokens = model.tokenizer(["good", "bad", "other"], add_special_tokens=False) + assert tokens["input_ids"] == [[5], [6], [7]] + + +@pytest.mark.parametrize("n_options", [2, 3, 5]) +@pytest.mark.parametrize("batch_size", [1, 3]) +def test_probabilities_equal_softmax_of_independent_scores( + tiny_encoder, monkeypatch, n_options, batch_size +): + model = initialize(make_model(tiny_encoder, batch_size=batch_size), n_options) + scores = {"good": 2.0, "bad": -1.0, "other": 0.5} + + def encode(texts): + return torch.tensor([[scores[text]] + [0.0] * 15 for text in texts]) + + monkeypatch.setattr(model, "_encode_single", encode) + model.classifier_head = nn.Linear(16, 1, bias=False) + with torch.no_grad(): + model.classifier_head.weight.zero_() + model.classifier_head.weight[0, 0] = 1.0 + texts = ["good", "bad"] + ["other"] * (n_options - 2) + items = [item(*texts), item(*reversed(texts)), item(*texts)] + expected_logits = torch.tensor( + [ + [scores[text] for text in example.rendered_elements.values()] + for example in items + ] + ) + expected = expected_logits.softmax(dim=1).numpy() + np.testing.assert_allclose(model.predict_proba(items), expected, atol=1e-7) + if n_options == 2: + assert expected[0, 0] == pytest.approx(torch.sigmoid(torch.tensor(3.0)).item()) + predictions = model.predict(items) + for row, prediction in zip(expected, predictions, strict=True): + assert prediction.predicted_class == model.option_names[int(row.argmax())] + assert prediction.confidence == pytest.approx(float(row.max())) + np.testing.assert_allclose(list(prediction.probabilities.values()), row) + + +@pytest.mark.parametrize("mode", ["fixed", "random_slopes"]) +def test_swap_equivariance_and_identical_option_ties(tiny_encoder, mode): + model = initialize(make_model(tiny_encoder, mode)) + if mode == "random_slopes": + model.random_effects.get_slopes("rater", model.classifier_head) + items = [item("good", "bad"), item("bad", "good"), item("good", "good")] + ids = None if mode == "fixed" else ["rater"] * len(items) + proba = model.predict_proba(items, ids) + np.testing.assert_allclose(proba[0], proba[1, ::-1], atol=1e-7) + np.testing.assert_allclose(proba[2], [0.5, 0.5], atol=1e-7) + np.testing.assert_array_equal(proba, model.predict_proba(items, ids)) + + +def test_scores_and_pairwise_odds_do_not_depend_on_other_options(tiny_encoder): + model = initialize(make_model(tiny_encoder), 3) + items = [ + item("good", "bad", "other"), + item("good", "bad", "the cat sat on the mat ."), + ] + model.encoder.eval() + model.classifier_head.eval() + with torch.no_grad(): + embeddings = model._prepare_inputs(items) + logits = model._compute_logits(embeddings, ["p", "p"]) + torch.testing.assert_close(logits[0, :2], logits[1, :2], atol=1e-7, rtol=0) + proba = model.predict_proba(items) + # Softmax normalization and division each round in the probability dtype. + odds = proba[:, 0] / proba[:, 1] + np.testing.assert_allclose( + odds, odds[0], rtol=4 * np.finfo(proba.dtype).eps, atol=0 + ) + + +def test_each_alternative_receives_the_choice_loss_gradient(tiny_encoder): + model = initialize(make_model(tiny_encoder)) + model.classifier_head = nn.Linear(16, 1, bias=False) + with torch.no_grad(): + model.classifier_head.weight.fill_(0.1) + embeddings = torch.arange(32, dtype=torch.float).reshape(1, 2, 16).requires_grad_() + logits = model._compute_logits(embeddings, ["p"]) + loss = nn.functional.cross_entropy(logits, torch.tensor([0])) + loss.backward() + expected = (logits.detach().softmax(1) - torch.tensor([[1.0, 0.0]])).unsqueeze( + -1 + ) * 0.1 + torch.testing.assert_close(embeddings.grad, expected.expand_as(embeddings)) + assert model.classifier_head.weight.grad.abs().sum() > 0 + + +def test_position_intercepts_change_log_odds_and_unknown_raters_use_shared_scores( + tiny_encoder, +): + model = initialize(make_model(tiny_encoder, "random_intercepts")) + bias = model.random_effects.get_intercepts("rater", 2, "mu") + with torch.no_grad(): + bias.copy_(torch.tensor([1.5, -0.5])) + items = [item("good", "good")] + known = model.predict_proba(items, ["rater"])[0] + unknown = model.predict_proba(items, ["new-rater"])[0] + assert np.log(known[0] / known[1]) == pytest.approx(2.0) + np.testing.assert_allclose(unknown, [0.5, 0.5], atol=1e-7) + assert "new-rater" not in model.random_effects.intercepts["mu"] + + +@pytest.mark.parametrize("mode", ["fixed", "random_intercepts", "random_slopes"]) +@pytest.mark.parametrize("n_options", [2, 3]) +def test_training_updates_encoder_head_and_participant_parameters( + tiny_encoder, monkeypatch, mode, n_options +): + model = make_model(tiny_encoder, mode) + texts = ["good", "bad", "other"][:n_options] + items = [item(*texts) for _ in range(5)] + # options need not all occur as selected labels + labels = ["option_0"] * len(items) + ids = None if mode == "fixed" else ["rater"] * len(items) + initial_encoder = { + name: value.clone() for name, value in model.encoder.state_dict().items() + } + initial_heads = {} + initial_biases = {} + initialize_classifier = model._initialize_classifier + get_slopes = RandomEffectsManager.get_slopes + get_intercepts = RandomEffectsManager.get_intercepts + + def capture_classifier(n): + initialize_classifier(n) + initial_heads["fixed"] = { + name: value.clone() + for name, value in model.classifier_head.state_dict().items() + } + + def capture_slopes(manager, participant_id, fixed_head, create_if_missing=True): + head = get_slopes(manager, participant_id, fixed_head, create_if_missing) + if create_if_missing and participant_id not in initial_heads: + initial_heads[participant_id] = { + name: value.clone() for name, value in head.state_dict().items() + } + return head + + def capture_intercepts( + manager, participant_id, n_classes, param_name, create_if_missing=True + ): + bias = get_intercepts( + manager, participant_id, n_classes, param_name, create_if_missing + ) + if create_if_missing and participant_id not in initial_biases: + initial_biases[participant_id] = bias.detach().clone() + return bias + + monkeypatch.setattr(model, "_initialize_classifier", capture_classifier) + monkeypatch.setattr(RandomEffectsManager, "get_slopes", capture_slopes) + monkeypatch.setattr(RandomEffectsManager, "get_intercepts", capture_intercepts) + metrics = model.train(items, labels, ids) + assert np.isfinite(metrics["train_loss"]) + assert model.num_classes == n_options + assert any( + not torch.equal(value, initial_encoder[name]) + for name, value in model.encoder.state_dict().items() + ) + head = ( + model.random_effects.slopes["rater"] + if mode == "random_slopes" + else model.classifier_head + ) + before = initial_heads["rater" if mode == "random_slopes" else "fixed"] + assert any( + not torch.equal(value, before[name]) + for name, value in head.state_dict().items() + ) + if mode == "random_intercepts": + assert not torch.equal( + model.random_effects.intercepts["mu"]["rater"], initial_biases["rater"] + ) + probabilities = model.predict_proba(items, ids) + assert probabilities.shape == (5, n_options) + np.testing.assert_allclose(probabilities.sum(1), 1.0, atol=1e-7) + + +def test_fixed_model_learns_content_preferences(tiny_encoder, monkeypatch): + model = make_model(tiny_encoder, num_epochs=20, learning_rate=0.02, batch_size=4) + items = [item("good", "bad"), item("bad", "good")] * 4 + labels = ["option_0", "option_1"] * 4 + initialize_classifier = model._initialize_classifier + + def no_head_dropout(n): + initialize_classifier(n) + model.classifier_head[2].p = 0.0 + + monkeypatch.setattr(model, "_initialize_classifier", no_head_dropout) + metrics = model.train(items, labels) + assert metrics["train_accuracy"] == 1.0 + proba = model.predict_proba(items) + assert np.mean(proba[np.arange(8), [0, 1] * 4]) > 0.9 + + +@pytest.mark.parametrize("mode", ["fixed", "random_intercepts", "random_slopes"]) +def test_checkpoint_round_trip_preserves_scoring_and_known_and_unknown_predictions( + tiny_encoder, tmp_path, mode +): + model = make_model(tiny_encoder, mode) + items = [item("good", "bad"), item("bad", "good")] + ids = None if mode == "fixed" else ["rater", "rater"] + model.train(items, ["option_0", "option_1"], ids) + checkpoint = tmp_path / "model" + model.save(str(checkpoint)) + restored = make_model( + tiny_encoder, scoring_mode="joint", encoder_mode="single_encoder" + ) + restored.load(str(checkpoint)) + assert restored.config.scoring_mode == "independent" + for participants in (ids, None if mode == "fixed" else ["unknown"] * 2): + np.testing.assert_allclose( + restored.predict_proba(items, participants), + model.predict_proba(items, participants), + atol=1e-7, + ) + assert AcceptabilityScorer(restored).score(items[0]) == pytest.approx( + AcceptabilityScorer(model).score(items[0]) + ) + + +def test_legacy_checkpoint_without_scoring_mode_loads_as_joint(tiny_encoder, tmp_path): + model = initialize( + make_model(tiny_encoder, scoring_mode="joint", encoder_mode="single_encoder") + ) + items = [item("good", "bad")] + model.save(str(tmp_path)) + config_path = tmp_path / "config.json" + config = json.loads(config_path.read_text()) + del config["scoring_mode"] + config_path.write_text(json.dumps(config)) + restored = make_model(tiny_encoder) + restored.load(str(tmp_path)) + assert restored.config.scoring_mode == "joint" + np.testing.assert_allclose( + restored.predict_proba(items), model.predict_proba(items), atol=1e-7 + ) + + +@pytest.mark.parametrize( + "bad_items", [[item("good")], [item("good", "bad"), item("good", "bad", "other")]] +) +def test_training_rejects_missing_or_inconsistent_options(tiny_encoder, bad_items): + model = make_model(tiny_encoder) + with pytest.raises(ValueError, match="at least two options|identical option names"): + model.train(bad_items, ["option_0"] * len(bad_items)) + + +def test_validation_and_prediction_reject_inconsistent_options(tiny_encoder): + model = make_model(tiny_encoder) + items = [item("good", "bad")] + wrong = [item("good", "bad", "other")] + with pytest.raises(ValueError, match="identical option names"): + model.train( + items, ["option_0"], validation_items=wrong, validation_labels=["option_0"] + ) + model.train(items, ["option_0"]) + with pytest.raises(ValueError, match="exactly the fitted option names"): + model.predict_proba(wrong) + + +def test_dual_joint_training_still_produces_option_logits(tiny_encoder): + model = make_model(tiny_encoder, scoring_mode="joint") + items = [item("good", "bad"), item("bad", "good")] + metrics = model.train(items, ["option_0", "option_1"]) + assert np.isfinite(metrics["train_loss"]) + assert model.classifier_head[0].in_features == 32 + assert model.classifier_head[-1].out_features == 2 + assert model.predict_proba(items).shape == (2, 2) + + +@pytest.mark.parametrize("mode", ["fixed", "random_intercepts", "random_slopes"]) +def test_early_stopping_restores_best_encoder_head_and_participant_effects( + tiny_encoder, monkeypatch, mode +): + model = make_model(tiny_encoder, mode, num_epochs=8, early_stopping_patience=2) + train = [item("good", "bad"), item("bad", "good")] + dev = [item("good", "other")] + ids = None if mode == "fixed" else ["rater"] * 2 + predict_proba = model._do_predict_proba + snapshots = [] + + def score_with_controlled_dev_loss(items, participant_ids): + actual = predict_proba(items, participant_ids) + if items is dev: + snapshots.append( + { + "encoder": { + name: value.clone() + for name, value in model.encoder.state_dict().items() + }, + "head": { + name: value.clone() + for name, value in model.classifier_head.state_dict().items() + }, + "participant": predict_proba(train, ["rater"] * 2), + } + ) + return np.array( + [ + [ + 0.9 if len(snapshots) == 1 else 0.4, + 0.1 if len(snapshots) == 1 else 0.6, + ] + ] + ) + return actual + + monkeypatch.setattr(model, "_do_predict_proba", score_with_controlled_dev_loss) + metrics = model.train( + train, + ["option_0", "option_1"], + ids, + validation_items=dev, + validation_labels=["option_0"], + ) + # three training epochs, followed by final validation metrics after restoration + assert len(snapshots) == 4 + assert "val_accuracy" in metrics + for component in ("encoder", "head"): + for name in snapshots[0][component]: + torch.testing.assert_close( + snapshots[-1][component][name], + snapshots[0][component][name], + rtol=0, + atol=0, + ) + np.testing.assert_array_equal( + snapshots[-1]["participant"], snapshots[0]["participant"] + ) + + +def test_common_score_offset_leaves_probabilities_unchanged(tiny_encoder): + model = initialize(make_model(tiny_encoder), 3) + items = [item("good", "bad", "other")] + before = model.predict_proba(items) + with torch.no_grad(): + model.classifier_head[-1].bias.add_(10.0) + np.testing.assert_allclose(model.predict_proba(items), before, atol=1e-6) + + +def test_empty_training_is_rejected_and_empty_prediction_keeps_option_dimension( + tiny_encoder, +): + model = make_model(tiny_encoder) + with pytest.raises(ValueError, match="nonempty training items"): + model.train([], []) + initialize(model) + assert model.predict_proba([]).shape == (0, 2) + assert model.predict([]) == [] + + +def test_unknown_training_response_is_rejected(tiny_encoder): + model = make_model(tiny_encoder) + with pytest.raises(ValueError, match="Invalid label"): + model.train([item("good", "bad")], ["option_2"]) + + +def test_option_order_is_defined_by_names_not_mapping_insertion_order(tiny_encoder): + model = initialize(make_model(tiny_encoder)) + original = item("good", "bad") + reordered = original.with_( + rendered_elements=dict(reversed(list(original.rendered_elements.items()))) + ) + np.testing.assert_array_equal( + model.predict_proba([original]), model.predict_proba([reordered]) + ) + + +@pytest.mark.parametrize("mode", ["random_intercepts", "random_slopes"]) +def test_participant_parameters_are_distinct_and_unknown_predictions_do_not_create_them( + tiny_encoder, mode +): + model = make_model(tiny_encoder, mode, num_epochs=3) + items = [item("good", "bad")] * 4 + model.train( + items, + ["option_0", "option_0", "option_1", "option_1"], + ["alice", "alice", "bob", "bob"], + ) + parameters = ( + model.random_effects.intercepts["mu"] + if mode == "random_intercepts" + else model.random_effects.slopes + ) + assert set(parameters) == {"alice", "bob"} + if mode == "random_intercepts": + assert parameters["alice"].data_ptr() != parameters["bob"].data_ptr() + assert not torch.equal(parameters["alice"], parameters["bob"]) + else: + for alice, bob in zip( + parameters["alice"].parameters(), + parameters["bob"].parameters(), + strict=True, + ): + assert alice.data_ptr() != bob.data_ptr() + predictions = model.predict_proba([items[0]], ["new-rater"]) + assert set(parameters) == {"alice", "bob"} + np.testing.assert_array_equal( + predictions, model.predict_proba([items[0]], ["new-rater"]) + ) + + +def test_original_single_encoder_joint_training_remains_available(tiny_encoder): + model = make_model( + tiny_encoder, scoring_mode="joint", encoder_mode="single_encoder", num_epochs=1 + ) + items = [item("good", "bad"), item("bad", "good")] + metrics = model.train(items, ["option_0", "option_1"]) + assert np.isfinite(metrics["train_loss"]) + assert model.classifier_head[0].in_features == 16 + assert model.classifier_head[-1].out_features == 2 + assert model.predict_proba(items).shape == (2, 2) diff --git a/tests/config/test_forced_choice_config.py b/tests/config/test_forced_choice_config.py new file mode 100644 index 0000000..d058b2e --- /dev/null +++ b/tests/config/test_forced_choice_config.py @@ -0,0 +1,72 @@ +"""Configuration contracts for forced-choice scoring architectures.""" + +import json + +import pytest +import yaml +from didactic.api import ValidationError + +from bead.config.active_learning import ForcedChoiceModelConfig +from gallery.eng.argument_structure.train_acceptability_model import build_model_config + + +def test_existing_configuration_defaults_to_joint_scoring(): + config = ForcedChoiceModelConfig() + assert config.encoder_mode == "single_encoder" + assert config.scoring_mode == "joint" + + +@pytest.mark.parametrize( + "encoder_mode,scoring_mode", + [ + ("single_encoder", "joint"), + ("dual_encoder", "joint"), + ("dual_encoder", "independent"), + ], +) +def test_valid_architectures_round_trip_through_json_and_yaml( + encoder_mode, scoring_mode +): + config = ForcedChoiceModelConfig( + encoder_mode=encoder_mode, scoring_mode=scoring_mode + ) + json_config = ForcedChoiceModelConfig(**json.loads(config.model_dump_json())) + yaml_config = ForcedChoiceModelConfig( + **yaml.safe_load(yaml.safe_dump(config.model_dump())) + ) + assert json_config == config + assert yaml_config == config + + +def test_independent_scoring_requires_separate_encoding(): + with pytest.raises( + ValidationError, match="independent scoring requires.*dual_encoder" + ): + ForcedChoiceModelConfig(scoring_mode="independent") + + +def test_config_updates_revalidate_architecture(): + config = ForcedChoiceModelConfig( + encoder_mode="dual_encoder", scoring_mode="independent" + ) + with pytest.raises(ValidationError, match="independent scoring requires"): + config.with_(encoder_mode="single_encoder") + + +def test_unknown_scoring_mode_is_rejected(): + with pytest.raises(ValidationError): + ForcedChoiceModelConfig(scoring_mode="bradley_terry") + + +def test_gallery_configuration_exposes_both_architecture_options(): + config = build_model_config( + {"encoder_mode": "dual_encoder", "scoring_mode": "independent"} + ) + assert config.encoder_mode == "dual_encoder" + assert config.scoring_mode == "independent" + assert config.mixed_effects.mode == "random_intercepts" + default = build_model_config({}) + assert default.encoder_mode == "single_encoder" + assert default.scoring_mode == "joint" + with pytest.raises(ValidationError, match="independent scoring requires"): + build_model_config({"scoring_mode": "independent"}) diff --git a/uv.lock b/uv.lock index b7302d5..77b2f1c 100644 --- a/uv.lock +++ b/uv.lock @@ -170,7 +170,7 @@ wheels = [ [[package]] name = "bead" -version = "0.12.0" +version = "0.13.0" source = { editable = "." } dependencies = [ { name = "accelerate" },