diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd7889..9bc9c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.12.0] - 2026-11-03 + +### Added + +- The task item factories in `bead.items` (`ordinal_scale`, `binary`, `categorical`, + `free_text`, `magnitude`, `span_labeling`, `cloze`) take keyword-only `text_element` + and `prompt_element` naming the rendered elements they write, plus `unit_element` + in `magnitude` and `instructions_element` in `cloze`. Every batch helper in those + modules threads the names through, and the defaults keep `"text"`, `"prompt"`, + `"unit"` and `"instructions"`. Span items key `tokenized_elements` and + `token_space_after` by `text_element`, and `add_spans_to_item` reads the prompt + from `prompt_element`. Two roles declared with the same name are refused with a + `ValueError`. +- `family_to_item_template`, `realization_to_item`, `protocol_to_item_templates` + and `realize_protocol_to_items` take `text_element` and `prompt_element`. + `realization_to_item` refuses a name its template does not declare, so the item's + rendered elements always agree with the template's elements under `validate_item`, + and the spans it derives reference the declared text element. + +### Changed + +- `create_cross_product_items`, `create_filtered_cross_product_items` and + `create_stratified_cross_product_items` require `cross_product_slot`; the `"verb"` + default is removed. The `_lemma` and `_form` rendered elements are + named after the slot the caller declares. + +### Fixed + +- `StratifiedStrategy` resolves each segment of `grouping_property` as a mapping key + as well as an attribute, so `"features."` groups lexical items by that feature + value; previously every item fell into the `"unknown"` group. + ## [0.11.0] - 2026-10-25 ### Added @@ -734,7 +766,8 @@ guards as type-checkers. - CI/CD: GitHub Actions for testing, docs, PyPI publishing - Read the Docs integration -[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.11.0...HEAD +[Unreleased]: https://github.com/FACTSlab/bead/compare/v0.12.0...HEAD +[0.12.0]: https://github.com/FACTSlab/bead/compare/v0.11.0...v0.12.0 [0.11.0]: https://github.com/FACTSlab/bead/compare/v0.10.0...v0.11.0 [0.10.0]: https://github.com/FACTSlab/bead/compare/v0.9.0...v0.10.0 [0.9.0]: https://github.com/FACTSlab/bead/compare/v0.8.0...v0.9.0 diff --git a/bead/items/binary.py b/bead/items/binary.py index 18e0374..989a261 100644 --- a/bead/items/binary.py +++ b/bead/items/binary.py @@ -30,6 +30,9 @@ def create_binary_item( binary_options: tuple[str, str] = ("yes", "no"), item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a binary judgment item for a single stimulus. @@ -46,16 +49,23 @@ def create_binary_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- Item - Binary item with text and prompt in rendered_elements. + Binary item with the stimulus and prompt stored under the declared + element names. Raises ------ ValueError - If text is empty or if binary_options doesn't have exactly 2 values. + If text is empty, if binary_options doesn't have exactly 2 values, + or if text_element and prompt_element name the same element. Examples -------- @@ -79,6 +89,13 @@ def create_binary_item( ... ) >>> item.item_metadata["binary_options"] ['true', 'false'] + + >>> # Declared element names + >>> item = create_binary_item( + ... "The sky is blue.", text_element="sentence", prompt_element="question" + ... ) + >>> item.rendered_elements["sentence"] + 'The sky is blue.' """ if not text or not text.strip(): raise ValueError("text cannot be empty") @@ -89,9 +106,12 @@ def create_binary_item( if item_template_id is None: item_template_id = uuid4() + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + rendered_elements: dict[str, str] = { - "text": text, - "prompt": prompt, + text_element: text, + prompt_element: prompt, } # Build item metadata @@ -115,6 +135,8 @@ def create_binary_items_from_texts( *, item_template_id: UUID | None = None, metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create binary items from a list of texts with the same prompt. @@ -130,6 +152,11 @@ def create_binary_items_from_texts( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str], dict[str, MetadataValue]] | None Function to generate metadata from each text. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -167,6 +194,8 @@ def create_binary_items_from_texts( binary_options=binary_options, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) binary_items.append(item) @@ -183,6 +212,8 @@ def create_binary_items_with_context( target_label: str = "Statement", item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create binary items with context + target structure. @@ -206,6 +237,11 @@ def create_binary_items_with_context( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (context, target). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -254,6 +290,8 @@ def create_binary_items_with_context( binary_options=binary_options, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) binary_items.append(item) @@ -269,6 +307,8 @@ def create_binary_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create binary items from grouped source items. @@ -291,6 +331,11 @@ def create_binary_items_from_groups( Whether to include group key in item metadata. item_template_id : UUID | None Template ID for all created items. If None, generates one per item. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -349,6 +394,8 @@ def create_binary_items_from_groups( binary_options=binary_options, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) binary_items.append(binary_item) @@ -362,6 +409,8 @@ def create_binary_items_cross_product( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create binary items from cross-product of texts and prompts. @@ -379,6 +428,11 @@ def create_binary_items_cross_product( Template ID for all created items. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -406,6 +460,8 @@ def create_binary_items_cross_product( binary_options=binary_options, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) binary_items.append(item) @@ -420,6 +476,8 @@ def create_filtered_binary_items( item_filter: Callable[[Item], bool] | None = None, extract_text: Callable[[Item], str] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create binary items with filtering. @@ -437,6 +495,11 @@ def create_filtered_binary_items( Text extraction function. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -476,6 +539,8 @@ def create_filtered_binary_items( binary_options=binary_options, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) binary_items.append(binary_item) diff --git a/bead/items/categorical.py b/bead/items/categorical.py index a3128f0..f8d7ccf 100644 --- a/bead/items/categorical.py +++ b/bead/items/categorical.py @@ -27,6 +27,9 @@ def create_categorical_item( prompt: str | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a categorical classification item. @@ -43,16 +46,23 @@ def create_categorical_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- Item - Categorical item with text and prompt in rendered_elements. + Categorical item with the stimulus and prompt stored under the + declared element names. Raises ------ ValueError - If text is empty or if fewer than 2 categories provided. + If text is empty, if fewer than 2 categories provided, or if + text_element and prompt_element name the same element. Examples -------- @@ -77,6 +87,15 @@ def create_categorical_item( ... ) >>> len(item.item_metadata["categories"]) 5 + + >>> # Declared element names + >>> item = create_categorical_item( + ... text="The cat sat.", + ... categories=["past", "present"], + ... text_element="sentence", + ... ) + >>> item.rendered_elements["sentence"] + 'The cat sat.' """ if not text or not text.strip(): raise ValueError("text cannot be empty") @@ -90,9 +109,12 @@ def create_categorical_item( if prompt is None: prompt = "Select a category:" + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + rendered_elements: dict[str, str] = { - "text": text, - "prompt": prompt, + text_element: text, + prompt_element: prompt, } # Build item metadata @@ -116,6 +138,9 @@ def create_nli_item( prompt: str | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a Natural Language Inference (NLI) item. @@ -136,6 +161,11 @@ def create_nli_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -190,6 +220,8 @@ def create_nli_item( prompt=prompt, item_template_id=item_template_id, metadata=nli_metadata, + text_element=text_element, + prompt_element=prompt_element, ) @@ -200,6 +232,8 @@ def create_categorical_items_from_texts( *, item_template_id: UUID | None = None, metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create categorical items from a list of texts with the same categories. @@ -215,6 +249,11 @@ def create_categorical_items_from_texts( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str], dict[str, MetadataValue]] | None Function to generate metadata from each text. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -248,6 +287,8 @@ def create_categorical_items_from_texts( prompt=prompt, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) categorical_items.append(item) @@ -263,6 +304,8 @@ def create_categorical_items_from_pairs( pair_label2: str = "Text 2", item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create categorical items from pairs of texts. @@ -284,6 +327,11 @@ def create_categorical_items_from_pairs( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text1, text2). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -327,6 +375,8 @@ def create_categorical_items_from_pairs( prompt=prompt, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) categorical_items.append(item) @@ -342,6 +392,8 @@ def create_categorical_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create categorical items from grouped source items. @@ -364,6 +416,11 @@ def create_categorical_items_from_groups( Whether to include group key in item metadata. item_template_id : UUID | None Template ID for all created items. If None, generates one per item. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -423,6 +480,8 @@ def create_categorical_items_from_groups( prompt=prompt, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) categorical_items.append(categorical_item) @@ -436,6 +495,8 @@ def create_categorical_items_cross_product( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create categorical items from cross-product of texts and prompts. @@ -453,6 +514,11 @@ def create_categorical_items_cross_product( Template ID for all created items. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -483,6 +549,8 @@ def create_categorical_items_cross_product( prompt=prompt, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) categorical_items.append(item) @@ -497,6 +565,8 @@ def create_filtered_categorical_items( item_filter: Callable[[Item], bool] | None = None, extract_text: Callable[[Item], str] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create categorical items with filtering. @@ -514,6 +584,11 @@ def create_filtered_categorical_items( Text extraction function. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -554,6 +629,8 @@ def create_filtered_categorical_items( prompt=prompt, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) categorical_items.append(categorical_item) diff --git a/bead/items/cloze.py b/bead/items/cloze.py index 77e86b8..b2abca8 100644 --- a/bead/items/cloze.py +++ b/bead/items/cloze.py @@ -43,6 +43,8 @@ def create_cloze_item( *, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + text_element: str = "text", + instructions_element: str = "instructions", ) -> Item: """Create a cloze item from a template with specific slots unfilled. @@ -60,6 +62,11 @@ def create_cloze_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the cloze text (default: "text"). + instructions_element : str + Name of the rendered element that holds the instructions, written only + when instructions are given (default: "instructions"). Returns ------- @@ -71,6 +78,8 @@ def create_cloze_item( ValueError If unfilled_slot_names not in template, if filled_slots not in template, if unfilled and filled overlap, if no unfilled slots, or if validation fails. + If instructions are given and text_element and instructions_element + are the same name. Examples -------- @@ -97,7 +106,22 @@ def create_cloze_item( 'verb' >>> item.unfilled_slots[0].position 2 + + >>> # Declared element name + >>> item = create_cloze_item( + ... template, + ... unfilled_slot_names=["verb"], + ... filled_slots={"det": "The", "noun": "cat"}, + ... text_element="sentence", + ... ) + >>> item.rendered_elements["sentence"] + 'The cat ___.' """ + if instructions and text_element == instructions_element: + raise ValueError( + "text_element and instructions_element must name different elements" + ) + if filled_slots is None: filled_slots = {} @@ -126,9 +150,9 @@ def create_cloze_item( ) # Build rendered_elements - rendered_elements: dict[str, str] = {"text": rendered_text} + rendered_elements: dict[str, str] = {text_element: rendered_text} if instructions: - rendered_elements["instructions"] = instructions + rendered_elements[instructions_element] = instructions # Build item_metadata # Convert filled_slots to MetadataValue format @@ -161,6 +185,8 @@ def create_cloze_items_from_template( *, item_template_id: UUID | None = None, metadata_fn: Callable[[list[str]], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + instructions_element: str = "instructions", ) -> list[Item]: """Create multiple cloze items from a template, varying unfilled slots. @@ -183,6 +209,11 @@ def create_cloze_items_from_template( Template ID for all items. metadata_fn : Callable[[list[str]], dict[str, MetadataValue]] | None Generate metadata from unfilled slot names. + text_element : str + Name of the rendered element that holds the cloze text (default: "text"). + instructions_element : str + Name of the rendered element that holds the instructions, written only + when instructions are given (default: "instructions"). Returns ------- @@ -267,6 +298,8 @@ def create_cloze_items_from_template( instructions=instructions, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + instructions_element=instructions_element, ) items.append(item) @@ -281,6 +314,8 @@ def create_simple_cloze_item( *, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + text_element: str = "text", + instructions_element: str = "instructions", ) -> Item: """Create a cloze item from plain text (no template). @@ -302,6 +337,11 @@ def create_simple_cloze_item( Template ID for the item. metadata : dict[str, MetadataValue] | None Additional metadata. + text_element : str + Name of the rendered element that holds the cloze text (default: "text"). + instructions_element : str + Name of the rendered element that holds the instructions, written only + when instructions are given (default: "instructions"). Returns ------- @@ -312,6 +352,8 @@ def create_simple_cloze_item( ------ ValueError If blank_positions out of range or if blank_labels length mismatch. + If instructions are given and text_element and instructions_element + are the same name. Examples -------- @@ -330,6 +372,11 @@ def create_simple_cloze_item( if not text or not text.strip(): raise ValueError("text cannot be empty") + if instructions and text_element == instructions_element: + raise ValueError( + "text_element and instructions_element must name different elements" + ) + if not blank_positions: raise ValueError( "blank_positions cannot be empty. " @@ -372,9 +419,9 @@ def create_simple_cloze_item( rendered_text = " ".join(blanked_tokens) # Build rendered_elements - rendered_elements: dict[str, str] = {"text": rendered_text} + rendered_elements: dict[str, str] = {text_element: rendered_text} if instructions: - rendered_elements["instructions"] = instructions + rendered_elements[instructions_element] = instructions # Build item_metadata item_metadata: dict[str, MetadataValue] = { @@ -403,6 +450,7 @@ def create_cloze_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", ) -> list[Item]: """Create cloze items from grouped source items. @@ -424,6 +472,8 @@ def create_cloze_items_from_groups( Whether to include group_key in metadata. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the cloze text (default: "text"). Returns ------- @@ -475,6 +525,7 @@ def create_cloze_items_from_groups( blank_positions=blank_positions, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, ) cloze_items.append(cloze_item) @@ -488,6 +539,7 @@ def create_filtered_cloze_items( template_filter: Callable[[Any], bool] | None = None, slot_filter: Callable[[str, Any], bool] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", ) -> list[Item]: """Create cloze items with multi-level filtering. @@ -505,6 +557,8 @@ def create_filtered_cloze_items( Filter which slots can be unfilled (receives slot_name and Slot object). item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the cloze text (default: "text"). Returns ------- @@ -548,6 +602,7 @@ def create_filtered_cloze_items( n_unfilled=n_slots_to_unfill, strategy="all_combinations", item_template_id=item_template_id, + text_element=text_element, ) # Further filter items if slot_filter was used diff --git a/bead/items/free_text.py b/bead/items/free_text.py index d92a710..2e98468 100644 --- a/bead/items/free_text.py +++ b/bead/items/free_text.py @@ -31,6 +31,9 @@ def create_free_text_item( multiline: bool = False, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a free text (open-ended) item. @@ -52,16 +55,24 @@ def create_free_text_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- Item - Free text item with text and prompt in rendered_elements. + Free text item with the stimulus and the prompt stored in + rendered_elements under the declared element names. Raises ------ ValueError - If text or prompt is empty, or if min_length > max_length. + If text or prompt is empty, if min_length > max_length, or if + text_element and prompt_element are the same. Examples -------- @@ -86,6 +97,16 @@ def create_free_text_item( ... ) >>> item.item_metadata["multiline"] True + + >>> # Declared element names + >>> item = create_free_text_item( + ... text="The dog chased the cat.", + ... prompt="Who chased whom?", + ... text_element="passage", + ... prompt_element="question", + ... ) + >>> item.rendered_elements["question"] + 'Who chased whom?' """ if not text or not text.strip(): raise ValueError("text cannot be empty") @@ -93,6 +114,9 @@ def create_free_text_item( if not prompt or not prompt.strip(): raise ValueError("prompt is required for free text items") + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + # Validate length constraints if min_length is not None and max_length is not None: if min_length > max_length: @@ -105,8 +129,8 @@ def create_free_text_item( item_template_id = uuid4() rendered_elements: dict[str, str] = { - "text": text, - "prompt": prompt, + text_element: text, + prompt_element: prompt, } # Build item metadata @@ -143,6 +167,8 @@ def create_free_text_items_from_texts( *, item_template_id: UUID | None = None, metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create free text items from a list of texts with the same prompt. @@ -164,6 +190,12 @@ def create_free_text_items_from_texts( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str], dict[str, MetadataValue]] | None Function to generate metadata from each text. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -201,6 +233,8 @@ def create_free_text_items_from_texts( multiline=multiline, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) free_text_items.append(item) @@ -217,6 +251,8 @@ def create_free_text_items_with_context( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create free text items with context + prompt pairs. @@ -241,6 +277,12 @@ def create_free_text_items_with_context( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (context, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -292,6 +334,8 @@ def create_free_text_items_with_context( multiline=multiline, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) free_text_items.append(item) @@ -310,6 +354,8 @@ def create_free_text_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create free text items from grouped source items. @@ -338,6 +384,12 @@ def create_free_text_items_from_groups( Whether to include group key in item metadata. item_template_id : UUID | None Template ID for all created items. If None, generates one per item. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -395,6 +447,8 @@ def create_free_text_items_from_groups( multiline=multiline, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) free_text_items.append(free_text_item) @@ -411,6 +465,8 @@ def create_free_text_items_cross_product( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create free text items from cross-product of texts and prompts. @@ -434,6 +490,12 @@ def create_free_text_items_cross_product( Template ID for all created items. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -466,6 +528,8 @@ def create_free_text_items_cross_product( multiline=multiline, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) free_text_items.append(item) @@ -483,6 +547,8 @@ def create_filtered_free_text_items( item_filter: Callable[[Item], bool] | None = None, extract_text: Callable[[Item], str] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create free text items with filtering. @@ -506,6 +572,12 @@ def create_filtered_free_text_items( Text extraction function. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -549,6 +621,8 @@ def create_filtered_free_text_items( multiline=multiline, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) free_text_items.append(free_text_item) @@ -560,6 +634,9 @@ def create_paraphrase_item( instruction: str = "Rewrite in your own words:", item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a paraphrase generation item. @@ -575,6 +652,12 @@ def create_paraphrase_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -598,6 +681,8 @@ def create_paraphrase_item( max_length=500, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) @@ -606,6 +691,9 @@ def create_wh_question_item( question_word: str = "Who", item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a WH-question answering item. @@ -621,6 +709,12 @@ def create_wh_question_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). Returns ------- @@ -645,6 +739,8 @@ def create_wh_question_item( max_length=100, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) diff --git a/bead/items/generation.py b/bead/items/generation.py index 9e55d31..15f1541 100644 --- a/bead/items/generation.py +++ b/bead/items/generation.py @@ -39,7 +39,7 @@ def create_cross_product_items( templates: list[Template], lexicons: dict[str, Lexicon], *, - cross_product_slot: str = "verb", + cross_product_slot: str, metadata_extractor: ( Callable[[Template, LexicalItem], dict[str, MetadataValue]] | None ) = None, @@ -61,8 +61,9 @@ def create_cross_product_items( lexicons : dict[str, Lexicon] Lexicons keyed by slot name. cross_product_slot : str - Slot name to vary across items (default: "verb"). - This slot's lexicon will be crossed with all templates. + Slot name to vary across items. This slot's lexicon is crossed with + all templates, and the ``_lemma`` and ``_form`` rendered + elements are named after it. metadata_extractor : Callable[[Template, LexicalItem], \ dict[str, MetadataValue]] | None Optional function to extract metadata from template and lexical item. @@ -79,7 +80,6 @@ def create_cross_product_items( Examples -------- Basic verb × template cross-product: - >>> from uuid import uuid4 >>> templates = [ ... Template( ... name="transitive", @@ -87,13 +87,21 @@ def create_cross_product_items( ... slots={} ... ) ... ] - >>> verb_lex = Lexicon(name="verbs") - >>> verb_lex.add(LexicalItem(lemma="walk")) - >>> verb_lex.add(LexicalItem(lemma="eat")) + >>> verb_lex = Lexicon( + ... name="verbs", + ... items=( + ... LexicalItem(lemma="walk", language_code="eng"), + ... LexicalItem(lemma="eat", language_code="eng"), + ... ), + ... ) >>> lexicons = {"verb": verb_lex} - >>> items = list(create_cross_product_items(templates, lexicons)) + >>> items = list( + ... create_cross_product_items(templates, lexicons, cross_product_slot="verb") + ... ) >>> len(items) 2 + >>> items[0].rendered_elements["verb_lemma"] + 'walk' With metadata extraction: >>> def extract_metadata(template, item): @@ -105,6 +113,7 @@ def create_cross_product_items( >>> items = list(create_cross_product_items( ... templates, ... lexicons, + ... cross_product_slot="verb", ... metadata_extractor=extract_metadata ... )) # doctest: +SKIP @@ -114,6 +123,7 @@ def create_cross_product_items( >>> items = list(create_cross_product_items( ... templates, ... lexicons, + ... cross_product_slot="verb", ... filter_fn=filter_transitive_only ... )) # doctest: +SKIP """ @@ -206,7 +216,7 @@ def create_filtered_cross_product_items( templates: list[Template], lexicons: dict[str, Lexicon], *, - cross_product_slot: str = "verb", + cross_product_slot: str, template_filter: Callable[[Template], bool] | None = None, item_filter: Callable[[LexicalItem], bool] | None = None, combination_filter: Callable[[Template, LexicalItem], bool] | None = None, @@ -255,6 +265,7 @@ def create_filtered_cross_product_items( >>> items = list(create_filtered_cross_product_items( ... templates, ... lexicons, + ... cross_product_slot="verb", ... template_filter=template_filter, ... item_filter=item_filter, ... combination_filter=combination_filter @@ -309,7 +320,7 @@ def create_stratified_cross_product_items( templates: list[Template], lexicons: dict[str, Lexicon], *, - cross_product_slot: str = "verb", + cross_product_slot: str, stratify_by: Callable[[LexicalItem], str], items_per_stratum: int, metadata_extractor: ( @@ -357,6 +368,7 @@ def create_stratified_cross_product_items( >>> items = list(create_stratified_cross_product_items( ... templates, ... lexicons, + ... cross_product_slot="verb", ... stratify_by=stratify_by_frequency, ... items_per_stratum=10 ... )) # doctest: +SKIP @@ -416,7 +428,9 @@ def items_to_jsonl( Examples -------- - >>> items = create_cross_product_items(templates, lexicons) # doctest: +SKIP + >>> items = create_cross_product_items( + ... templates, lexicons, cross_product_slot="verb" + ... ) # doctest: +SKIP >>> n = items_to_jsonl(items, "output.jsonl") # doctest: +SKIP >>> print(f"Wrote {n} items") # doctest: +SKIP """ diff --git a/bead/items/magnitude.py b/bead/items/magnitude.py index 1449399..c144bc3 100644 --- a/bead/items/magnitude.py +++ b/bead/items/magnitude.py @@ -30,6 +30,10 @@ def create_magnitude_item( step: int | float | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> Item: """Create a magnitude (numeric input) item. @@ -52,16 +56,27 @@ def create_magnitude_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- Item - Magnitude item with text and prompt in rendered_elements. + Magnitude item with the stimulus, the prompt, and any unit stored in + rendered_elements under the declared element names. Raises ------ ValueError - If text is empty or if both bounds are provided and min >= max. + If text is empty, if both bounds are provided and min >= max, or if + two declared element names that are both written are the same. Examples -------- @@ -89,10 +104,29 @@ def create_magnitude_item( ... ) >>> item.item_metadata["max_value"] 100 + + >>> # Declared element names + >>> item = create_magnitude_item( + ... text="How long was the pause?", + ... unit="ms", + ... text_element="sentence", + ... unit_element="measure", + ... ) + >>> sorted(item.rendered_elements) + ['measure', 'prompt', 'sentence'] """ if not text or not text.strip(): raise ValueError("text cannot be empty") + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + + if unit and unit_element in (text_element, prompt_element): + raise ValueError( + "unit_element must name an element other than text_element and " + "prompt_element" + ) + min_value, max_value = bounds # Validate bounds if both are provided @@ -109,12 +143,12 @@ def create_magnitude_item( prompt = "Enter a value:" rendered_elements: dict[str, str] = { - "text": text, - "prompt": prompt, + text_element: text, + prompt_element: prompt, } if unit: - rendered_elements["unit"] = unit + rendered_elements[unit_element] = unit # Build item metadata item_metadata: dict[str, MetadataValue] = { @@ -147,6 +181,9 @@ def create_magnitude_items_from_texts( *, item_template_id: UUID | None = None, metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> list[Item]: """Create magnitude items from a list of texts. @@ -166,6 +203,15 @@ def create_magnitude_items_from_texts( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str], dict[str, MetadataValue]] | None Function to generate metadata from each text. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -202,6 +248,9 @@ def create_magnitude_items_from_texts( step=step, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) magnitude_items.append(item) @@ -219,6 +268,9 @@ def create_magnitude_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> list[Item]: """Create magnitude items from grouped source items. @@ -245,6 +297,15 @@ def create_magnitude_items_from_groups( Whether to include group key in item metadata. item_template_id : UUID | None Template ID for all created items. If None, generates one per item. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -302,6 +363,9 @@ def create_magnitude_items_from_groups( step=step, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) magnitude_items.append(magnitude_item) @@ -317,6 +381,9 @@ def create_magnitude_items_cross_product( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> list[Item]: """Create magnitude items from cross-product of texts and prompts. @@ -338,6 +405,15 @@ def create_magnitude_items_cross_product( Template ID for all created items. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -369,6 +445,9 @@ def create_magnitude_items_cross_product( step=step, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) magnitude_items.append(item) @@ -385,6 +464,9 @@ def create_filtered_magnitude_items( item_filter: Callable[[Item], bool] | None = None, extract_text: Callable[[Item], str] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> list[Item]: """Create magnitude items with filtering. @@ -406,6 +488,15 @@ def create_filtered_magnitude_items( Text extraction function. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -449,6 +540,9 @@ def create_filtered_magnitude_items( step=step, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) magnitude_items.append(magnitude_item) @@ -459,6 +553,10 @@ def create_reading_time_item( text: str, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> Item: """Create a reading time measurement item. @@ -473,6 +571,15 @@ def create_reading_time_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -495,6 +602,9 @@ def create_reading_time_item( step=1, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) @@ -502,6 +612,10 @@ def create_confidence_item( text: str, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", + unit_element: str = "unit", ) -> Item: """Create a confidence rating item (0-100%). @@ -515,6 +629,15 @@ def create_confidence_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt + (default: "prompt"). + unit_element : str + Name of the rendered element that holds the unit, written only + when a unit is given (default: "unit"). Returns ------- @@ -537,6 +660,9 @@ def create_confidence_item( step=1, item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, + unit_element=unit_element, ) diff --git a/bead/items/ordinal_scale.py b/bead/items/ordinal_scale.py index a27e162..4d1a520 100644 --- a/bead/items/ordinal_scale.py +++ b/bead/items/ordinal_scale.py @@ -31,6 +31,9 @@ def create_ordinal_scale_item( scale_labels: dict[int, str] | tuple[ScalePointLabel, ...] | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create an ordinal scale rating item. @@ -51,17 +54,24 @@ def create_ordinal_scale_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- Item - Ordinal scale item with text and prompt in rendered_elements. + Ordinal scale item with the stimulus and prompt stored under the + declared element names. Raises ------ ValueError - If text is empty, if scale_bounds are invalid, or if scale_labels - contain values outside scale bounds. + If text is empty, if scale_bounds are invalid, if scale_labels + contain values outside scale bounds, or if text_element and + prompt_element name the same element. Examples -------- @@ -89,6 +99,13 @@ def create_ordinal_scale_item( ... ) >>> item.item_metadata["scale_labels"][1] 'Strongly Disagree' + + >>> # Declared element names + >>> item = create_ordinal_scale_item( + ... "The cat sat.", text_element="sentence", prompt_element="question" + ... ) + >>> sorted(item.rendered_elements) + ['question', 'sentence'] """ if not text or not text.strip(): raise ValueError("text cannot be empty") @@ -117,9 +134,12 @@ def create_ordinal_scale_item( if prompt is None: prompt = "Rate this item:" + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + rendered_elements: dict[str, str] = { - "text": text, - "prompt": prompt, + text_element: text, + prompt_element: prompt, } # Build item metadata @@ -149,6 +169,8 @@ def create_ordinal_scale_items_from_texts( *, item_template_id: UUID | None = None, metadata_fn: Callable[[str], dict[str, MetadataValue]] | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create ordinal scale items from a list of texts. @@ -166,6 +188,11 @@ def create_ordinal_scale_items_from_texts( Template ID for all created items. If None, generates one per item. metadata_fn : Callable[[str], dict[str, MetadataValue]] | None Function to generate metadata from each text. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -200,6 +227,8 @@ def create_ordinal_scale_items_from_texts( scale_labels=scale_labels, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) ordinal_items.append(item) @@ -216,6 +245,8 @@ def create_ordinal_scale_items_from_groups( extract_text: Callable[[Item], str] | None = None, include_group_metadata: bool = True, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create ordinal scale items from grouped source items. @@ -240,6 +271,11 @@ def create_ordinal_scale_items_from_groups( Whether to include group key in item metadata. item_template_id : UUID | None Template ID for all created items. If None, generates one per item. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -295,6 +331,8 @@ def create_ordinal_scale_items_from_groups( scale_labels=scale_labels, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) ordinal_items.append(ordinal_item) @@ -309,6 +347,8 @@ def create_ordinal_scale_items_cross_product( *, item_template_id: UUID | None = None, metadata_fn: (Callable[[str, str], dict[str, MetadataValue]] | None) = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create ordinal scale items from cross-product of texts and prompts. @@ -328,6 +368,11 @@ def create_ordinal_scale_items_cross_product( Template ID for all created items. metadata_fn : Callable[[str, str], dict[str, MetadataValue]] | None Function to generate metadata from (text, prompt). + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -358,6 +403,8 @@ def create_ordinal_scale_items_cross_product( scale_labels=scale_labels, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) ordinal_items.append(item) @@ -373,6 +420,8 @@ def create_filtered_ordinal_scale_items( item_filter: Callable[[Item], bool] | None = None, extract_text: Callable[[Item], str] | None = None, item_template_id: UUID | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Create ordinal scale items with filtering. @@ -392,6 +441,11 @@ def create_filtered_ordinal_scale_items( Text extraction function. item_template_id : UUID | None Template ID for created items. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -433,6 +487,8 @@ def create_filtered_ordinal_scale_items( scale_labels=scale_labels, item_template_id=item_template_id, metadata=item_metadata, + text_element=text_element, + prompt_element=prompt_element, ) ordinal_items.append(ordinal_item) @@ -444,6 +500,9 @@ def create_likert_5_item( prompt: str | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a 5-point Likert scale item. @@ -460,6 +519,11 @@ def create_likert_5_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -490,6 +554,8 @@ def create_likert_5_item( ), item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) @@ -498,6 +564,9 @@ def create_likert_7_item( prompt: str | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a 7-point Likert scale item. @@ -514,6 +583,11 @@ def create_likert_7_item( Template ID for the item. If None, generates new UUID. metadata : dict[str, MetadataValue] | None Additional metadata for item_metadata field. + text_element : str + Name of the rendered element that holds the stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -541,6 +615,8 @@ def create_likert_7_item( ), item_template_id=item_template_id, metadata=metadata, + text_element=text_element, + prompt_element=prompt_element, ) diff --git a/bead/items/span_labeling.py b/bead/items/span_labeling.py index a923cb7..f9c21ce 100644 --- a/bead/items/span_labeling.py +++ b/bead/items/span_labeling.py @@ -116,6 +116,9 @@ def create_span_item( span_spec: SpanSpec | None = None, item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create a standalone span labeling item. @@ -141,6 +144,11 @@ def create_span_item( Template ID. If None, generates a new UUID. metadata : dict[str, MetadataValue] | None Additional item metadata. + text_element : str + Name of the rendered element that holds the stimulus text, also used + for its tokens and spacing flags (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -150,11 +158,33 @@ def create_span_item( Raises ------ ValueError - If text is empty or span indices are out of bounds. + If text is empty, if span indices are out of bounds, or if + text_element and prompt_element are the same name. + + Examples + -------- + >>> from bead.items.spans import SpanLabel, SpanSegment + >>> span = Span( + ... span_id="span_0", + ... segments=(SpanSegment(element_name="sentence", indices=(0,)),), + ... label=SpanLabel(label="Person"), + ... ) + >>> item = create_span_item( + ... "Ann left.", + ... [span], + ... "Label the person.", + ... tokens=["Ann", "left."], + ... text_element="sentence", + ... ) + >>> item.tokenized_elements["sentence"] + ('Ann', 'left.') """ if not text or not text.strip(): raise ValueError("text cannot be empty") + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + if item_template_id is None: item_template_id = uuid4() @@ -173,15 +203,15 @@ def create_span_item( tokenized_elements: dict[str, tuple[str, ...]] token_space_after: dict[str, tuple[bool, ...]] if tokens is not None: - tokenized_elements = {"text": tuple(tokens)} - token_space_after = {"text": tuple(_infer_space_after(tokens, text))} + tokenized_elements = {text_element: tuple(tokens)} + token_space_after = {text_element: tuple(_infer_space_after(tokens, text))} else: if tokenizer_config is None: tokenizer_config = TokenizerConfig() tokenize = create_tokenizer(tokenizer_config) result = tokenize(text) - tokenized_elements = {"text": result.token_texts} - token_space_after = {"text": result.space_after_flags} + tokenized_elements = {text_element: result.token_texts} + token_space_after = {text_element: result.space_after_flags} _validate_span_indices(spans, tokenized_elements) @@ -191,7 +221,7 @@ def create_span_item( return Item( item_template_id=item_template_id, - rendered_elements={"text": text, "prompt": prompt}, + rendered_elements={text_element: text, prompt_element: prompt}, spans=tuple(spans), tokenized_elements=tokenized_elements, token_space_after=token_space_after, @@ -208,6 +238,9 @@ def create_interactive_span_item( label_source: LabelSourceType = "fixed", item_template_id: UUID | None = None, metadata: dict[str, MetadataValue] | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Create an item for interactive span selection by participants. @@ -229,15 +262,29 @@ def create_interactive_span_item( Template ID. If None, generates a new UUID. metadata : dict[str, MetadataValue] | None Additional item metadata. + text_element : str + Name of the rendered element that holds the stimulus text, also used + for its tokens and spacing flags (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- Item Interactive span labeling item (no pre-defined spans). + + Raises + ------ + ValueError + If text is empty or if text_element and prompt_element are the same + name. """ if not text or not text.strip(): raise ValueError("text cannot be empty") + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + if item_template_id is None: item_template_id = uuid4() @@ -253,15 +300,15 @@ def create_interactive_span_item( tokenized_elements: dict[str, tuple[str, ...]] token_space_after: dict[str, tuple[bool, ...]] if tokens is not None: - tokenized_elements = {"text": tuple(tokens)} - token_space_after = {"text": tuple(_infer_space_after(tokens, text))} + tokenized_elements = {text_element: tuple(tokens)} + token_space_after = {text_element: tuple(_infer_space_after(tokens, text))} else: if tokenizer_config is None: tokenizer_config = TokenizerConfig() tokenize = create_tokenizer(tokenizer_config) result = tokenize(text) - tokenized_elements = {"text": result.token_texts} - token_space_after = {"text": result.space_after_flags} + tokenized_elements = {text_element: result.token_texts} + token_space_after = {text_element: result.space_after_flags} item_metadata: dict[str, MetadataValue] = {"_span_spec": span_spec_data} if metadata: @@ -269,7 +316,7 @@ def create_interactive_span_item( return Item( item_template_id=item_template_id, - rendered_elements={"text": text, "prompt": prompt}, + rendered_elements={text_element: text, prompt_element: prompt}, spans=(), tokenized_elements=tokenized_elements, token_space_after=token_space_after, @@ -282,6 +329,8 @@ def add_spans_to_item( spans: list[Span], tokenizer_config: TokenizerConfig | None = None, span_spec: SpanSpec | None = None, + *, + prompt_element: str = "prompt", ) -> Item: """Add span annotations to any existing item. @@ -299,6 +348,9 @@ def add_spans_to_item( Tokenizer configuration (used only if item lacks tokenization). span_spec : SpanSpec | None Span specification. + prompt_element : str + Name of the rendered element whose ``[[label]]`` references are checked + against the span labels (default: "prompt"). Returns ------- @@ -318,7 +370,7 @@ def add_spans_to_item( _validate_span_indices(spans, item.tokenized_elements) # warn if prompt contains [[label]] references to nonexistent span labels - prompt_text = item.rendered_elements.get("prompt", "") + prompt_text = item.rendered_elements.get(prompt_element, "") if prompt_text: all_spans = list(item.spans) + spans span_labels = {s.label.label for s in all_spans if s.label is not None} @@ -353,6 +405,9 @@ def create_span_items_from_texts( tokenizer_config: TokenizerConfig | None = None, labels: list[str] | None = None, item_template_id: UUID | None = None, + *, + text_element: str = "text", + prompt_element: str = "prompt", ) -> list[Item]: """Batch create span items with automatic tokenization. @@ -370,6 +425,11 @@ def create_span_items_from_texts( Fixed label set. item_template_id : UUID | None Shared template ID. If None, generates one per item. + text_element : str + Name of the rendered element that holds each stimulus text + (default: "text"). + prompt_element : str + Name of the rendered element that holds the prompt (default: "prompt"). Returns ------- @@ -392,6 +452,8 @@ def create_span_items_from_texts( tokens=tokens, labels=labels, item_template_id=item_template_id, + text_element=text_element, + prompt_element=prompt_element, ) items.append(item) diff --git a/bead/protocol/items.py b/bead/protocol/items.py index 51def8f..9dd102b 100644 --- a/bead/protocol/items.py +++ b/bead/protocol/items.py @@ -96,6 +96,8 @@ def family_to_item_template( *, judgment_type: JudgmentType, presentation_spec: PresentationSpec | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> ItemTemplate: """Build the :class:`ItemTemplate` for a :class:`QuestionFamily`. @@ -113,7 +115,7 @@ def family_to_item_template( The ``prompt`` field of the template's :class:`TaskSpec` is the anchor's canonical prompt (with ``[[label]]`` references intact); individual realizations override the prompt at item-construction - time via the ``prompt`` rendered-element on the resulting + time via the ``prompt_element`` rendered element on the resulting :class:`Item`. Parameters @@ -127,14 +129,27 @@ def family_to_item_template( presentation_spec : PresentationSpec | None, optional Custom presentation spec. Defaults to a fresh :class:`PresentationSpec` with mode ``"static"``. + text_element : str, default="text" + Name of the element that holds the context sentence. + prompt_element : str, default="prompt" + Name of the element that holds the realized question. Returns ------- ItemTemplate Template with ``name`` set to the anchor name, ``task_type`` - derived from the scale, and ``elements`` covering ``"text"`` - (the sentence) and ``"prompt"`` (the realized question). + derived from the scale, and ``elements`` covering + ``text_element`` (the sentence) and ``prompt_element`` (the + realized question). + + Raises + ------ + ValueError + If ``text_element`` and ``prompt_element`` are the same name. """ + if text_element == prompt_element: + raise ValueError("text_element and prompt_element must name different elements") + encoding = encode_response_space(family.anchor.name, family.anchor.response_space) task_type = scale_type_to_task_type(encoding.scale_type) @@ -165,13 +180,13 @@ def family_to_item_template( elements = ( ItemElement( element_type="text", - element_name="text", + element_name=text_element, content="", order=0, ), ItemElement( element_type="text", - element_name="prompt", + element_name=prompt_element, content=family.anchor.canonical_prompt, order=1, ), @@ -189,6 +204,7 @@ def family_to_item_template( def _spans_from_context( realization: QuestionRealization, + text_element: str, ) -> tuple[Span, ...]: """Extract :class:`Span` objects from a realization's context. @@ -202,6 +218,8 @@ def _spans_from_context( ---------- realization : QuestionRealization The realized question carrying the context. + text_element : str + Name of the element whose tokens the spans index. Returns ------- @@ -228,7 +246,7 @@ def _spans_from_context( span_id=f"{anchor.name}-{label}", segments=( SpanSegment( - element_name="text", + element_name=text_element, indices=tuple(i - 1 for i in dep.span_positions), ), ), @@ -246,7 +264,7 @@ def _spans_from_context( span_id=f"{anchor.name}-{label}", segments=( SpanSegment( - element_name="text", + element_name=text_element, indices=tuple(i - 1 for i in context.target_span_positions), ), ), @@ -266,12 +284,14 @@ def realization_to_item( realization: QuestionRealization, *, item_template: ItemTemplate, + text_element: str = "text", + prompt_element: str = "prompt", ) -> Item: """Package a :class:`QuestionRealization` as an :class:`Item`. The resulting :class:`Item` references ``item_template`` by id, - rendering the realization's prompt as the ``"prompt"`` element and - the context's sentence as the ``"text"`` element. Tokenized + rendering the realization's prompt as the ``prompt_element`` element + and the context's sentence as the ``text_element`` element. Tokenized elements are populated from :attr:`ProtocolContext.tokens` when present; spans for the anchor's ``required_span_labels`` are derived via @@ -285,25 +305,45 @@ def realization_to_item( item_template : ItemTemplate The template returned by :func:`family_to_item_template` for the originating family. The bridge does not validate that the - template was produced from the same family — the caller is + template was produced from the same family; the caller is responsible for matching them. + text_element : str, default="text" + Name of the element that holds the context sentence. The + template must declare it. + prompt_element : str, default="prompt" + Name of the element that holds the realized question. The + template must declare it. Returns ------- Item Item bound to the template, with the realization's prompt and context materialized. + + Raises + ------ + ValueError + If ``item_template`` does not declare ``text_element`` and + ``prompt_element``. """ + declared = {element.element_name for element in item_template.elements} + undeclared = sorted({text_element, prompt_element} - declared) + if undeclared: + raise ValueError( + f"item template {item_template.name!r} does not declare the " + f"elements {undeclared}; it declares {sorted(declared)}" + ) + context = realization.context rendered_elements = { - "text": context.sentence, - "prompt": realization.prompt, + text_element: context.sentence, + prompt_element: realization.prompt, } tokenized_elements: dict[str, tuple[str, ...]] = {} if context.tokens: - tokenized_elements["text"] = context.tokens + tokenized_elements[text_element] = context.tokens - spans = _spans_from_context(realization) + spans = _spans_from_context(realization, text_element) return Item( item_template_id=item_template.id, @@ -318,6 +358,8 @@ def protocol_to_item_templates( *, judgment_type: JudgmentType, presentation_spec: PresentationSpec | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> dict[str, ItemTemplate]: """Build one :class:`ItemTemplate` per family in the protocol. @@ -329,6 +371,10 @@ def protocol_to_item_templates( Common judgment type to assign to every template. presentation_spec : PresentationSpec | None, optional Common presentation spec; defaults to a fresh one per call. + text_element : str, default="text" + Name of the element that holds the context sentence. + prompt_element : str, default="prompt" + Name of the element that holds the realized question. Returns ------- @@ -340,6 +386,8 @@ def protocol_to_item_templates( family, judgment_type=judgment_type, presentation_spec=presentation_spec, + text_element=text_element, + prompt_element=prompt_element, ) for family in protocol.families } @@ -353,6 +401,8 @@ def realize_protocol_to_items( item_templates: dict[str, ItemTemplate] | None = None, responses: dict[str, str] | None = None, presentation_spec: PresentationSpec | None = None, + text_element: str = "text", + prompt_element: str = "prompt", ) -> tuple[tuple[QuestionRealization, Item], ...]: """Realize a protocol against one context, packaging items. @@ -376,6 +426,12 @@ def realize_protocol_to_items( ``None``. presentation_spec : PresentationSpec | None, optional Common presentation spec when templates are built fresh. + text_element : str, default="text" + Name of the element that holds the context sentence, both in + freshly built templates and in the packaged items. + prompt_element : str, default="prompt" + Name of the element that holds the realized question, both in + freshly built templates and in the packaged items. Returns ------- @@ -387,9 +443,19 @@ def realize_protocol_to_items( protocol, judgment_type=judgment_type, presentation_spec=presentation_spec, + text_element=text_element, + prompt_element=prompt_element, ) realizations = protocol.realize_all(context, responses=responses) return tuple( - (r, realization_to_item(r, item_template=templates[r.anchor.name])) + ( + r, + realization_to_item( + r, + item_template=templates[r.anchor.name], + text_element=text_element, + prompt_element=prompt_element, + ), + ) for r in realizations ) diff --git a/bead/templates/strategies.py b/bead/templates/strategies.py index 119785b..e7fd43e 100644 --- a/bead/templates/strategies.py +++ b/bead/templates/strategies.py @@ -7,7 +7,7 @@ import re import time from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Literal, cast from uuid import UUID @@ -342,6 +342,10 @@ def generate_combinations( def _get_property_value(self, item: LexicalItem, property_path: str) -> str: """Get property value from item, handling nested properties. + Each dotted segment is looked up as a key when the current value is + a mapping (such as ``features``) and as an attribute otherwise, so + ``"features.transitivity"`` reads the ``transitivity`` feature. + Parameters ---------- item : LexicalItem @@ -352,12 +356,17 @@ def _get_property_value(self, item: LexicalItem, property_path: str) -> str: Returns ------- str - Property value as string. + Property value as string, or "unknown" when a segment of the + path does not resolve. """ parts = property_path.split(".") value = item for part in parts: - if hasattr(value, part): + if isinstance(value, Mapping): + if part not in value: + return "unknown" + value = value[part] + elif hasattr(value, part): value = getattr(value, part) else: return "unknown" diff --git a/docs/user-guide/api/items.md b/docs/user-guide/api/items.md index 418652f..280f724 100644 --- a/docs/user-guide/api/items.md +++ b/docs/user-guide/api/items.md @@ -88,6 +88,19 @@ item = create_ordinal_scale_item( ) ``` +**Element names**: each factory stores the stimulus and the prompt under the rendered elements `"text"` and `"prompt"` unless the caller declares other names with `text_element` and `prompt_element`. The batch helpers take the same keywords, and the names must match the `elements` of the `ItemTemplate` the items are validated against. + +```python +from bead.items.ordinal_scale import create_ordinal_scale_item + +item = create_ordinal_scale_item( + "The cat sleeps", + text_element="sentence", + prompt_element="question", +) +print(sorted(item.rendered_elements)) +``` + **Batch creation**: ```python diff --git a/docs/user-guide/api/templates.md b/docs/user-guide/api/templates.md index f336e36..f96b43e 100644 --- a/docs/user-guide/api/templates.md +++ b/docs/user-guide/api/templates.md @@ -119,6 +119,8 @@ combinations = strategy.generate_combinations(slot_items) print(f"Generated {len(list(combinations))} combinations") ``` +`grouping_property` is a dotted path. Each segment is read as a key when the value reached so far is a mapping and as an attribute otherwise, so `"features.frame"` groups items by the `frame` entry of their `features`. Items whose path does not resolve share the `"unknown"` group. + ### MLM-Based Strategies (generate_from_template) These strategies use masked language models and work directly with templates: diff --git a/docs/user-guide/protocols.md b/docs/user-guide/protocols.md index 156174c..2c6afc5 100644 --- a/docs/user-guide/protocols.md +++ b/docs/user-guide/protocols.md @@ -401,6 +401,29 @@ for realization, item in pairs: ... # downstream item processing ``` +The template's elements and the item's rendered elements are named +`"text"` (the context sentence) and `"prompt"` (the realized +question) unless the caller declares other names. Pass the same +`text_element` and `prompt_element` to `family_to_item_template` and +`realization_to_item` (or once to `realize_protocol_to_items`); +`realization_to_item` refuses a name its template does not declare, +so the item always renders exactly the template's elements: + +```python +template = family_to_item_template( + family_change, + judgment_type="acceptability", + text_element="sentence", + prompt_element="question", +) +item = realization_to_item( + realization, + item_template=template, + text_element="sentence", + prompt_element="question", +) +``` + `scale_type_to_task_type` is the canonical translation used here and in the active-learning registry. There is no other mapping: every protocol family produces exactly one `ItemTemplate`. diff --git a/pyproject.toml b/pyproject.toml index 08fd250..a20466b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bead" -version = "0.11.0" +version = "0.12.0" description = "Lexicon and Template Collection Construction Pipeline for Acceptability and Inference Judgment Data" authors = [{name = "Aaron Steven White", email = "aaron.white@rochester.edu"}] readme = "README.md" diff --git a/tests/items/test_binary.py b/tests/items/test_binary.py index acd1c43..ff57473 100644 --- a/tests/items/test_binary.py +++ b/tests/items/test_binary.py @@ -265,3 +265,54 @@ def test_no_filter_includes_all(self) -> None: binary_items = create_filtered_binary_items(items, prompt="Question?") assert len(binary_items) == 3 + + +class TestDeclaredElementNames: + """Test caller-declared element names across the binary factories.""" + + def test_defaults_are_text_and_prompt(self) -> None: + """Test the default element names.""" + item = create_binary_item("She walks.", prompt="Grammatical?") + + assert set(item.rendered_elements) == {"text", "prompt"} + + def test_single_item_uses_declared_names(self) -> None: + """Test the stimulus and prompt land under the declared names.""" + item = create_binary_item( + "She walks.", + prompt="Grammatical?", + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "She walks.", + "question": "Grammatical?", + } + + def test_colliding_names_raise(self) -> None: + """Test that one name for both roles is refused.""" + with pytest.raises(ValueError, match="must name different elements"): + create_binary_item( + "She walks.", text_element="sentence", prompt_element="sentence" + ) + + def test_batch_helpers_thread_declared_names(self) -> None: + """Test every batch helper passes the declared names through.""" + names = {"text_element": "sentence", "prompt_element": "question"} + source = [ + Item(item_template_id=uuid4(), rendered_elements={"text": "She walks."}) + ] + batches = [ + create_binary_items_from_texts(["She walks."], "Q?", **names), + create_binary_items_with_context(["C."], ["T."], "Q?", **names), + create_binary_items_from_groups( + source, group_by=lambda i: "g", prompt="Q?", **names + ), + create_binary_items_cross_product(["She walks."], ["Q?"], **names), + create_filtered_binary_items(source, "Q?", **names), + ] + + for batch in batches: + assert len(batch) == 1 + assert set(batch[0].rendered_elements) == {"sentence", "question"} diff --git a/tests/items/test_categorical.py b/tests/items/test_categorical.py index 5483be6..b200bcb 100644 --- a/tests/items/test_categorical.py +++ b/tests/items/test_categorical.py @@ -318,3 +318,60 @@ def test_no_filter_includes_all(self) -> None: ) assert len(categorical_items) == 3 + + +class TestDeclaredElementNames: + """Test caller-declared element names across the categorical factories.""" + + def test_defaults_are_text_and_prompt(self) -> None: + """Test the default element names.""" + item = create_categorical_item("The cat sat.", categories=["a", "b"]) + + assert set(item.rendered_elements) == {"text", "prompt"} + + def test_single_item_uses_declared_names(self) -> None: + """Test the stimulus and prompt land under the declared names.""" + item = create_categorical_item( + "The cat sat.", + categories=["a", "b"], + prompt="Which?", + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "The cat sat.", + "question": "Which?", + } + + def test_colliding_names_raise(self) -> None: + """Test that one name for both roles is refused.""" + with pytest.raises(ValueError, match="must name different elements"): + create_categorical_item( + "The cat sat.", + categories=["a", "b"], + text_element="sentence", + prompt_element="sentence", + ) + + def test_batch_helpers_thread_declared_names(self) -> None: + """Test every derived and batch helper passes the declared names through.""" + names = {"text_element": "sentence", "prompt_element": "question"} + categories = ["a", "b"] + source = [ + Item(item_template_id=uuid4(), rendered_elements={"text": "The cat sat."}) + ] + batches = [ + [create_nli_item("P.", "H.", **names)], + create_categorical_items_from_texts(["T."], categories, **names), + create_categorical_items_from_pairs([("A.", "B.")], categories, **names), + create_categorical_items_from_groups( + source, group_by=lambda i: "g", categories=categories, **names + ), + create_categorical_items_cross_product(["T."], ["Q?"], categories, **names), + create_filtered_categorical_items(source, categories, **names), + ] + + for batch in batches: + assert len(batch) == 1 + assert set(batch[0].rendered_elements) == {"sentence", "question"} diff --git a/tests/items/test_cloze.py b/tests/items/test_cloze.py index 8045dcd..55b6010 100644 --- a/tests/items/test_cloze.py +++ b/tests/items/test_cloze.py @@ -552,3 +552,132 @@ def test_slot_filter(self) -> None: # Only verb slot has constraints, so 1 item assert len(cloze_items) == 1 assert cloze_items[0].unfilled_slots[0].slot_name == "verb" + + +class TestDeclaredElementNames: + """Test that cloze factories write the element names the caller declares.""" + + def _template(self) -> Template: + return Template( + name="simple", + template_string="{det} {noun} {verb}.", + slots={ + "det": Slot(name="det"), + "noun": Slot(name="noun"), + "verb": Slot(name="verb"), + }, + ) + + def test_create_cloze_item_declared_names(self) -> None: + """Declared names key the text and instructions elements.""" + item = create_cloze_item( + self._template(), + unfilled_slot_names=["verb"], + filled_slots={"det": "The", "noun": "cat"}, + instructions="Fill in the verb.", + text_element="sentence", + instructions_element="directions", + ) + + assert item.rendered_elements == { + "sentence": "The cat ___.", + "directions": "Fill in the verb.", + } + + def test_create_cloze_item_default_names(self) -> None: + """Without declarations the elements are named text and instructions.""" + item = create_cloze_item( + self._template(), + unfilled_slot_names=["verb"], + instructions="Fill in the verb.", + ) + + assert set(item.rendered_elements) == {"text", "instructions"} + + def test_create_cloze_item_colliding_names_raise(self) -> None: + """Colliding names raise when instructions are written.""" + with pytest.raises(ValueError, match="must name different elements"): + create_cloze_item( + self._template(), + unfilled_slot_names=["verb"], + instructions="Fill in the verb.", + text_element="body", + instructions_element="body", + ) + + def test_create_cloze_item_colliding_names_without_instructions(self) -> None: + """Colliding names are accepted when no instructions are written.""" + item = create_cloze_item( + self._template(), + unfilled_slot_names=["verb"], + text_element="body", + instructions_element="body", + ) + + assert set(item.rendered_elements) == {"body"} + + def test_create_simple_cloze_item_declared_names(self) -> None: + """Simple cloze items write the declared element names.""" + item = create_simple_cloze_item( + text="The quick brown fox", + blank_positions=[1], + instructions="Fill in the blank.", + text_element="sentence", + instructions_element="directions", + ) + + assert item.rendered_elements == { + "sentence": "The ___ brown fox", + "directions": "Fill in the blank.", + } + + def test_create_simple_cloze_item_colliding_names_raise(self) -> None: + """Simple cloze items refuse colliding element names.""" + with pytest.raises(ValueError, match="must name different elements"): + create_simple_cloze_item( + text="The quick brown fox", + blank_positions=[1], + instructions="Fill in the blank.", + text_element="body", + instructions_element="body", + ) + + def test_from_template_threads_names(self) -> None: + """Template expansion threads the declared names to every item.""" + items = create_cloze_items_from_template( + self._template(), + n_unfilled=1, + instructions="Fill it in.", + text_element="sentence", + instructions_element="directions", + ) + + assert len(items) == 3 + assert all( + set(item.rendered_elements) == {"sentence", "directions"} for item in items + ) + + def test_from_groups_threads_names(self) -> None: + """Group-based creation threads the declared names to every item.""" + source = [ + Item(item_template_id=uuid4(), rendered_elements={"text": "She walks."}), + ] + + items = create_cloze_items_from_groups( + source, + group_by=lambda i: "all", + text_element="sentence", + ) + + assert items[0].rendered_elements == {"sentence": "___ walks."} + + def test_filtered_threads_names(self) -> None: + """Filtered creation threads the declared names to every item.""" + items = create_filtered_cloze_items( + [self._template()], + n_slots_to_unfill=1, + text_element="sentence", + ) + + assert len(items) == 3 + assert all(set(item.rendered_elements) == {"sentence"} for item in items) diff --git a/tests/items/test_free_text.py b/tests/items/test_free_text.py index 7bc72d5..db707c9 100644 --- a/tests/items/test_free_text.py +++ b/tests/items/test_free_text.py @@ -383,3 +383,121 @@ def test_with_metadata(self) -> None: assert item.item_metadata["task"] == "qa" assert item.item_metadata["multiline"] is False + + +class TestDeclaredElementNames: + """Test that free text factories write under declared element names.""" + + def test_defaults_are_text_and_prompt(self) -> None: + """Test the default element names.""" + item = create_free_text_item("The cat sat.", prompt="Who sat?") + + assert set(item.rendered_elements) == {"text", "prompt"} + + def test_single_item_uses_declared_names(self) -> None: + """Test create_free_text_item writes under the declared names.""" + item = create_free_text_item( + "The cat sat.", + prompt="Who sat?", + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "The cat sat.", + "question": "Who sat?", + } + + def test_colliding_names_raise(self) -> None: + """Test that one name for both elements is refused.""" + with pytest.raises(ValueError, match="text_element and prompt_element"): + create_free_text_item( + "The cat sat.", + prompt="Who sat?", + text_element="stimulus", + prompt_element="stimulus", + ) + + def test_from_texts_threads_names(self) -> None: + """Test create_free_text_items_from_texts threads the names.""" + items = create_free_text_items_from_texts( + ["A.", "B."], + prompt="Paraphrase:", + text_element="sentence", + prompt_element="question", + ) + + assert all(set(i.rendered_elements) == {"sentence", "question"} for i in items) + + def test_with_context_threads_names(self) -> None: + """Test create_free_text_items_with_context threads the names.""" + items = create_free_text_items_with_context( + ["The cat sat."], + ["Who sat?"], + text_element="passage", + prompt_element="question", + ) + + assert items[0].rendered_elements == { + "passage": "The cat sat.", + "question": "Who sat?", + } + + def test_from_groups_threads_names(self) -> None: + """Test create_free_text_items_from_groups threads the names.""" + source = [ + Item( + item_template_id=uuid4(), + rendered_elements={"text": "A."}, + item_metadata={"group": "g"}, + ) + ] + items = create_free_text_items_from_groups( + source, + group_by=lambda i: i.item_metadata["group"], + prompt="Paraphrase:", + text_element="sentence", + prompt_element="question", + ) + + assert set(items[0].rendered_elements) == {"sentence", "question"} + + def test_cross_product_threads_names(self) -> None: + """Test create_free_text_items_cross_product threads the names.""" + items = create_free_text_items_cross_product( + ["A."], + ["P1", "P2"], + text_element="sentence", + prompt_element="question", + ) + + assert all(set(i.rendered_elements) == {"sentence", "question"} for i in items) + + def test_filtered_threads_names(self) -> None: + """Test create_filtered_free_text_items threads the names.""" + source = [Item(item_template_id=uuid4(), rendered_elements={"text": "A."})] + items = create_filtered_free_text_items( + source, + prompt="Paraphrase:", + text_element="sentence", + prompt_element="question", + ) + + assert set(items[0].rendered_elements) == {"sentence", "question"} + + def test_paraphrase_threads_names(self) -> None: + """Test create_paraphrase_item threads the names.""" + item = create_paraphrase_item( + "A.", text_element="sentence", prompt_element="instruction" + ) + + assert item.rendered_elements["instruction"] == "Rewrite in your own words:" + assert item.rendered_elements["sentence"] == "A." + + def test_wh_question_threads_names(self) -> None: + """Test create_wh_question_item threads the names.""" + item = create_wh_question_item( + "A.", text_element="passage", prompt_element="question" + ) + + assert set(item.rendered_elements) == {"passage", "question"} diff --git a/tests/items/test_generation.py b/tests/items/test_generation.py new file mode 100644 index 0000000..8b152a0 --- /dev/null +++ b/tests/items/test_generation.py @@ -0,0 +1,147 @@ +"""Tests for cross-product item generation utilities.""" + +from __future__ import annotations + +import pytest + +from bead.items.generation import ( + create_cross_product_items, + create_filtered_cross_product_items, + create_stratified_cross_product_items, +) +from bead.resources.lexical_item import LexicalItem +from bead.resources.lexicon import Lexicon +from bead.resources.template import Template + + +@pytest.fixture +def templates() -> list[Template]: + """Provide two templates without declared slots.""" + return [ + Template(name="intransitive", template_string="{subject} {head}.", slots={}), + Template( + name="transitive", template_string="{subject} {head} {object}.", slots={} + ), + ] + + +@pytest.fixture +def lexicons() -> dict[str, Lexicon]: + """Provide a lexicon keyed by a slot name other than "verb".""" + head = Lexicon( + name="heads", + items=( + LexicalItem( + lemma="walk", + language_code="eng", + form="walks", + features={"freq": "high"}, + ), + LexicalItem(lemma="eat", language_code="eng", features={"freq": "high"}), + LexicalItem(lemma="devour", language_code="eng", features={"freq": "low"}), + ), + ) + return {"head": head} + + +class TestCreateCrossProductItems: + """Test create_cross_product_items() function.""" + + def test_slot_must_be_named( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test that the cross-product slot has no default.""" + with pytest.raises(TypeError, match="cross_product_slot"): + create_cross_product_items(templates, lexicons) + + def test_keys_derive_from_declared_slot( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test that rendered element keys are derived from the declared slot.""" + items = list( + create_cross_product_items(templates, lexicons, cross_product_slot="head") + ) + + assert len(items) == 6 + first = items[0] + assert first.rendered_elements["head_lemma"] == "walk" + assert first.rendered_elements["head_form"] == "walks" + assert "verb_lemma" not in first.rendered_elements + assert items[1].rendered_elements["head_form"] == "eat" + + def test_missing_slot_lexicon_raises( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test that naming a slot without a lexicon raises ValueError.""" + with pytest.raises(ValueError, match="Lexicon for slot 'verb' not found"): + list( + create_cross_product_items( + templates, lexicons, cross_product_slot="verb" + ) + ) + + +class TestCreateFilteredCrossProductItems: + """Test create_filtered_cross_product_items() function.""" + + def test_slot_must_be_named( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test that the cross-product slot has no default.""" + with pytest.raises(TypeError, match="cross_product_slot"): + create_filtered_cross_product_items(templates, lexicons) + + def test_keys_derive_from_declared_slot( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test filtering with keys derived from the declared slot.""" + items = list( + create_filtered_cross_product_items( + templates, + lexicons, + cross_product_slot="head", + template_filter=lambda t: t.name == "transitive", + item_filter=lambda i: i.lemma != "eat", + ) + ) + + assert [i.rendered_elements["head_lemma"] for i in items] == [ + "walk", + "devour", + ] + + +class TestCreateStratifiedCrossProductItems: + """Test create_stratified_cross_product_items() function.""" + + def test_slot_must_be_named( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test that the cross-product slot has no default.""" + with pytest.raises(TypeError, match="cross_product_slot"): + create_stratified_cross_product_items( + templates, + lexicons, + stratify_by=lambda i: str(i.features["freq"]), + items_per_stratum=1, + ) + + def test_keys_derive_from_declared_slot( + self, templates: list[Template], lexicons: dict[str, Lexicon] + ) -> None: + """Test stratified sampling with keys derived from the declared slot.""" + items = list( + create_stratified_cross_product_items( + templates, + lexicons, + cross_product_slot="head", + stratify_by=lambda i: str(i.features["freq"]), + items_per_stratum=1, + ) + ) + + assert sorted({i.rendered_elements["head_lemma"] for i in items}) == [ + "devour", + "walk", + ] + assert len(items) == 4 diff --git a/tests/items/test_magnitude.py b/tests/items/test_magnitude.py index 0a1779e..4ff43ea 100644 --- a/tests/items/test_magnitude.py +++ b/tests/items/test_magnitude.py @@ -338,3 +338,135 @@ def test_with_metadata(self) -> None: assert item.item_metadata["task"] == "nli" assert item.item_metadata["unit"] == "%" + + +class TestDeclaredElementNames: + """Test that magnitude factories write under declared element names.""" + + def test_defaults_are_text_prompt_and_unit(self) -> None: + """Test the default element names.""" + item = create_magnitude_item("How long?", unit="ms") + + assert set(item.rendered_elements) == {"text", "prompt", "unit"} + + def test_single_item_uses_declared_names(self) -> None: + """Test create_magnitude_item writes under the declared names.""" + item = create_magnitude_item( + "How long?", + unit="ms", + prompt="Enter a time:", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert item.rendered_elements == { + "sentence": "How long?", + "question": "Enter a time:", + "measure": "ms", + } + + def test_colliding_text_and_prompt_raise(self) -> None: + """Test that one name for text and prompt is refused.""" + with pytest.raises(ValueError, match="text_element and prompt_element"): + create_magnitude_item( + "How long?", text_element="stimulus", prompt_element="stimulus" + ) + + def test_colliding_unit_raises(self) -> None: + """Test that the unit element may not reuse another element's name.""" + with pytest.raises(ValueError, match="unit_element"): + create_magnitude_item("How long?", unit="ms", unit_element="prompt") + + def test_unit_name_unchecked_without_unit(self) -> None: + """Test that the unit element name is not checked when no unit is written.""" + item = create_magnitude_item("How long?", unit_element="prompt") + + assert set(item.rendered_elements) == {"text", "prompt"} + + def test_from_texts_threads_names(self) -> None: + """Test create_magnitude_items_from_texts threads the names.""" + items = create_magnitude_items_from_texts( + ["A.", "B."], + unit="ms", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert all( + set(i.rendered_elements) == {"sentence", "question", "measure"} + for i in items + ) + + def test_from_groups_threads_names(self) -> None: + """Test create_magnitude_items_from_groups threads the names.""" + source = [ + Item( + item_template_id=uuid4(), + rendered_elements={"text": "A."}, + item_metadata={"group": "g"}, + ) + ] + items = create_magnitude_items_from_groups( + source, + group_by=lambda i: i.item_metadata["group"], + unit="ms", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert set(items[0].rendered_elements) == {"sentence", "question", "measure"} + + def test_cross_product_threads_names(self) -> None: + """Test create_magnitude_items_cross_product threads the names.""" + items = create_magnitude_items_cross_product( + ["A."], + ["P1", "P2"], + unit="ms", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert all( + set(i.rendered_elements) == {"sentence", "question", "measure"} + for i in items + ) + + def test_filtered_threads_names(self) -> None: + """Test create_filtered_magnitude_items threads the names.""" + source = [Item(item_template_id=uuid4(), rendered_elements={"text": "A."})] + items = create_filtered_magnitude_items( + source, + unit="ms", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert set(items[0].rendered_elements) == {"sentence", "question", "measure"} + + def test_reading_time_threads_names(self) -> None: + """Test create_reading_time_item threads the names.""" + item = create_reading_time_item( + "A.", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert item.rendered_elements["measure"] == "ms" + assert set(item.rendered_elements) == {"sentence", "question", "measure"} + + def test_confidence_threads_names(self) -> None: + """Test create_confidence_item threads the names.""" + item = create_confidence_item( + "A.", + text_element="sentence", + prompt_element="question", + unit_element="measure", + ) + + assert item.rendered_elements["measure"] == "%" diff --git a/tests/items/test_ordinal_scale.py b/tests/items/test_ordinal_scale.py index 255dd60..e2fe1db 100644 --- a/tests/items/test_ordinal_scale.py +++ b/tests/items/test_ordinal_scale.py @@ -365,3 +365,55 @@ def test_default_prompt(self) -> None: item = create_likert_7_item("Statement") assert item.rendered_elements["prompt"] == "Rate your agreement:" + + +class TestDeclaredElementNames: + """Test caller-declared element names across the ordinal scale factories.""" + + def test_defaults_are_text_and_prompt(self) -> None: + """Test the default element names.""" + item = create_ordinal_scale_item("The cat sat.") + + assert set(item.rendered_elements) == {"text", "prompt"} + + def test_single_item_uses_declared_names(self) -> None: + """Test the stimulus and prompt land under the declared names.""" + item = create_ordinal_scale_item( + "The cat sat.", + prompt="How natural?", + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "The cat sat.", + "question": "How natural?", + } + + def test_colliding_names_raise(self) -> None: + """Test that one name for both roles is refused.""" + with pytest.raises(ValueError, match="must name different elements"): + create_ordinal_scale_item( + "The cat sat.", text_element="sentence", prompt_element="sentence" + ) + + def test_batch_helpers_thread_declared_names(self) -> None: + """Test every derived and batch helper passes the declared names through.""" + names = {"text_element": "sentence", "prompt_element": "question"} + source = [ + Item(item_template_id=uuid4(), rendered_elements={"text": "The cat sat."}) + ] + batches = [ + [create_likert_5_item("Statement", **names)], + [create_likert_7_item("Statement", **names)], + create_ordinal_scale_items_from_texts(["The cat sat."], **names), + create_ordinal_scale_items_from_groups( + source, group_by=lambda i: "g", **names + ), + create_ordinal_scale_items_cross_product(["T."], ["Q?"], **names), + create_filtered_ordinal_scale_items(source, **names), + ] + + for batch in batches: + assert len(batch) == 1 + assert set(batch[0].rendered_elements) == {"sentence", "question"} diff --git a/tests/items/test_span_labeling.py b/tests/items/test_span_labeling.py index 07e33e4..b7f8637 100644 --- a/tests/items/test_span_labeling.py +++ b/tests/items/test_span_labeling.py @@ -341,3 +341,135 @@ def extractor(text: str, tokens: list[str]) -> list[Span]: assert all(len(item.spans) == 1 for item in items) assert items[0].rendered_elements["text"] == "Hello world." assert items[1].rendered_elements["text"] == "Goodbye world." + + +class TestDeclaredElementNames: + """Test that span factories write the element names the caller declares.""" + + def _sentence_span(self) -> Span: + return Span( + span_id="span_0", + segments=[SpanSegment(element_name="sentence", indices=[0])], + label=SpanLabel(label="First"), + ) + + def test_create_span_item_declared_names(self) -> None: + """Declared names key the rendered, tokenized, and spacing elements.""" + item = create_span_item( + text="Hello world.", + spans=[self._sentence_span()], + prompt="Label it.", + tokens=["Hello", "world."], + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "Hello world.", + "question": "Label it.", + } + assert item.tokenized_elements == {"sentence": ("Hello", "world.")} + assert set(item.token_space_after) == {"sentence"} + + def test_create_span_item_tokenizer_uses_declared_name(self) -> None: + """Tokenizer output is keyed by the declared text element.""" + item = create_span_item( + text="Hello world.", + spans=[self._sentence_span()], + prompt="Label it.", + tokenizer_config=TokenizerConfig(backend="whitespace"), + text_element="sentence", + ) + + assert set(item.tokenized_elements) == {"sentence"} + assert set(item.token_space_after) == {"sentence"} + assert item.rendered_elements["prompt"] == "Label it." + + def test_create_span_item_default_names(self) -> None: + """Without declarations the elements are named text and prompt.""" + item = create_span_item( + text="Hello world.", + spans=[], + prompt="Label it.", + tokens=["Hello", "world."], + ) + + assert set(item.rendered_elements) == {"text", "prompt"} + assert set(item.tokenized_elements) == {"text"} + + def test_create_span_item_colliding_names_raise(self) -> None: + """Declaring the same name for text and prompt raises ValueError.""" + with pytest.raises(ValueError, match="must name different elements"): + create_span_item( + text="Hello world.", + spans=[], + prompt="Label it.", + tokens=["Hello", "world."], + text_element="body", + prompt_element="body", + ) + + def test_create_interactive_span_item_declared_names(self) -> None: + """Interactive span items write the declared element names.""" + item = create_interactive_span_item( + text="Hello world.", + prompt="Select spans.", + tokens=["Hello", "world."], + text_element="sentence", + prompt_element="question", + ) + + assert item.rendered_elements == { + "sentence": "Hello world.", + "question": "Select spans.", + } + assert set(item.tokenized_elements) == {"sentence"} + assert set(item.token_space_after) == {"sentence"} + + def test_create_interactive_span_item_colliding_names_raise(self) -> None: + """Interactive span items refuse colliding element names.""" + with pytest.raises(ValueError, match="must name different elements"): + create_interactive_span_item( + text="Hello world.", + prompt="Select spans.", + tokens=["Hello", "world."], + text_element="body", + prompt_element="body", + ) + + def test_add_spans_to_item_reads_declared_prompt(self) -> None: + """The label-reference check reads the declared prompt element.""" + item = Item( + item_template_id=uuid4(), + rendered_elements={ + "sentence": "Hello world.", + "question": "Is [[Missing]] there?", + }, + tokenized_elements={ + "sentence": ("Hello", "world."), + "question": ("Is", "[[Missing]]", "there?"), + }, + ) + + with pytest.warns(UserWarning, match=r"\[\[Missing\]\]"): + add_spans_to_item(item, [self._sentence_span()], prompt_element="question") + + def test_create_span_items_from_texts_threads_names(self) -> None: + """Batch creation threads the declared names to every item.""" + + def extractor(text: str, tokens: list[str]) -> list[Span]: + return [self._sentence_span()] + + items = create_span_items_from_texts( + texts=["Hello world.", "Goodbye world."], + span_extractor=extractor, + prompt="Label first word.", + tokenizer_config=TokenizerConfig(backend="whitespace"), + text_element="sentence", + prompt_element="question", + ) + + assert all( + set(item.rendered_elements) == {"sentence", "question"} for item in items + ) + assert all(set(item.tokenized_elements) == {"sentence"} for item in items) diff --git a/tests/protocol/test_items_bridge.py b/tests/protocol/test_items_bridge.py index d89a4b6..53bb115 100644 --- a/tests/protocol/test_items_bridge.py +++ b/tests/protocol/test_items_bridge.py @@ -6,6 +6,7 @@ from bead.active_learning.models import ForcedChoiceModel, model_class_for_encoding from bead.items.item_template import ItemTemplate, PresentationSpec +from bead.items.validation import validate_item from bead.protocol import ( AnnotationProtocol, ContextItem, @@ -222,6 +223,106 @@ def test_dependent_span_picked_when_label_matches_lemma(self) -> None: assert labels_to_indices["participant"] == (0, 1) +class TestDeclaredElementNames: + """Element names are declared by the caller and agree across the bridge.""" + + def test_template_declares_elements(self) -> None: + family = QuestionFamily(anchor=_build_binary_anchor()) + template = family_to_item_template( + family, + judgment_type="acceptability", + text_element="sentence", + prompt_element="question", + ) + names = {e.element_name for e in template.elements} + assert names == {"sentence", "question"} + + def test_item_uses_declared_elements(self) -> None: + family = QuestionFamily(anchor=_build_binary_anchor()) + template = family_to_item_template( + family, + judgment_type="acceptability", + text_element="sentence", + prompt_element="question", + ) + ctx = ProtocolContext( + sentence="Mary built a sandcastle.", + tokens=("Mary", "built", "a", "sandcastle", "."), + target_position=2, + target_span_text="built a sandcastle", + target_span_positions=(2, 3, 4), + ) + realization = family.realize(ctx) + item = realization_to_item( + realization, + item_template=template, + text_element="sentence", + prompt_element="question", + ) + assert item.rendered_elements == { + "sentence": "Mary built a sandcastle.", + "question": realization.prompt, + } + assert item.tokenized_elements == {"sentence": ctx.tokens} + assert item.spans[0].segments[0].element_name == "sentence" + assert validate_item(item, template) == [] + + def test_item_refuses_undeclared_elements(self) -> None: + family = QuestionFamily(anchor=_build_binary_anchor()) + template = family_to_item_template(family, judgment_type="acceptability") + realization = family.realize(ProtocolContext(sentence="Plain text.")) + with pytest.raises(ValueError, match="sentence"): + realization_to_item( + realization, item_template=template, text_element="sentence" + ) + + def test_colliding_names_refused(self) -> None: + family = QuestionFamily(anchor=_build_binary_anchor()) + with pytest.raises(ValueError, match="must name different elements"): + family_to_item_template( + family, + judgment_type="acceptability", + text_element="text", + prompt_element="text", + ) + + def test_protocol_threads_declared_elements(self) -> None: + proto = AnnotationProtocol( + families=[QuestionFamily(anchor=_build_binary_anchor())] + ) + ctx = ProtocolContext( + sentence="Mary built a sandcastle.", + target_span_text="built a sandcastle", + target_span_positions=(2, 3, 4), + ) + templates = protocol_to_item_templates( + proto, + judgment_type="acceptability", + text_element="sentence", + prompt_element="question", + ) + pairs = realize_protocol_to_items( + proto, + ctx, + judgment_type="acceptability", + item_templates=templates, + text_element="sentence", + prompt_element="question", + ) + realization, item = pairs[0] + assert item.rendered_elements["question"] == realization.prompt + assert validate_item(item, templates["completion"]) == [] + + fresh = realize_protocol_to_items( + proto, + ctx, + judgment_type="acceptability", + text_element="sentence", + prompt_element="question", + ) + assert set(fresh[0][1].rendered_elements) == {"sentence", "question"} + + class TestProtocolToItemTemplates: """Tests for :func:`protocol_to_item_templates` and :func:`realize_protocol_to_items`.""" diff --git a/tests/templates/test_strategies.py b/tests/templates/test_strategies.py index 089b177..f724fa3 100644 --- a/tests/templates/test_strategies.py +++ b/tests/templates/test_strategies.py @@ -266,3 +266,63 @@ def test_stratified_strategy_multiple_slots() -> None: assert len(combinations) == 10 assert all("a" in c and "b" in c for c in combinations) + + +def test_stratified_strategy_groups_by_feature_key() -> None: + """Test stratified strategy resolves a dotted path through feature keys.""" + strategy = StratifiedStrategy( + n_samples=200, grouping_property="features.frame", seed=42 + ) + common = [ + LexicalItem(lemma=f"c{i}", language_code="en", features={"frame": "NP"}) + for i in range(9) + ] + rare = LexicalItem(lemma="r", language_code="en", features={"frame": "S"}) + + combinations = strategy.generate_combinations({"slot": [*common, rare]}) + + # Two strata are sampled evenly, so the single S-frame item is drawn about + # half the time; a single unresolved stratum would draw it about 1 in 10. + rare_draws = sum(1 for c in combinations if c["slot"].lemma == "r") + assert 80 <= rare_draws <= 120 + + +def test_stratified_strategy_groups_by_nested_feature_key() -> None: + """Test stratified strategy resolves nested feature dictionaries.""" + strategy = StratifiedStrategy( + n_samples=200, grouping_property="features.agreement.number", seed=42 + ) + singular = [ + LexicalItem( + lemma=f"s{i}", + language_code="en", + features={"agreement": {"number": "sg"}}, + ) + for i in range(9) + ] + plural = LexicalItem( + lemma="p", language_code="en", features={"agreement": {"number": "pl"}} + ) + + combinations = strategy.generate_combinations({"slot": [*singular, plural]}) + + plural_draws = sum(1 for c in combinations if c["slot"].lemma == "p") + assert 80 <= plural_draws <= 120 + + +def test_stratified_strategy_missing_feature_key() -> None: + """Test that items without the feature key share one stratum.""" + strategy = StratifiedStrategy( + n_samples=200, grouping_property="features.frame", seed=42 + ) + with_frame = LexicalItem(lemma="f", language_code="en", features={"frame": "S"}) + without_frame = [ + LexicalItem(lemma=f"n{i}", language_code="en", features={}) for i in range(9) + ] + + combinations = strategy.generate_combinations( + {"slot": [with_frame, *without_frame]} + ) + + framed_draws = sum(1 for c in combinations if c["slot"].lemma == "f") + assert 80 <= framed_draws <= 120 diff --git a/uv.lock b/uv.lock index 263b268..b7302d5 100644 --- a/uv.lock +++ b/uv.lock @@ -170,7 +170,7 @@ wheels = [ [[package]] name = "bead" -version = "0.11.0" +version = "0.12.0" source = { editable = "." } dependencies = [ { name = "accelerate" },