diff --git a/.gitignore b/.gitignore index 2618793..97b2ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # python test files .coverage -.pytype +.mypy_cache # build time temp files VERSION* diff --git a/.pytype.cfg b/.pytype.cfg deleted file mode 100644 index ee9e67c..0000000 --- a/.pytype.cfg +++ /dev/null @@ -1,10 +0,0 @@ -# NOTE: All relative paths are relative to the location of this file. - -[pytype] - -# Space-separated list of files or directories to exclude. -exclude = - **/*_test.py - **/test_*.py - clams/develop/templates/**/*.py - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e177ca..7c35d39 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Unlike the old `setup.py`-based workflow, an editable install (`pip install -e .`) is now required before running tests or building docs. The package uses `importlib.metadata` for version resolution at runtime, which only works when the package is registered in the -environment. You can no longer run `pytest` or `pytype` directly +environment. You can no longer run `pytest` or `mypy` directly against the source tree without installing first. If you want to avoid pulling in all dependencies, `pip install -e . --no-deps` is sufficient to register the package metadata. diff --git a/build-tools/clean.py b/build-tools/clean.py index 784dfda..db4e5c8 100644 --- a/build-tools/clean.py +++ b/build-tools/clean.py @@ -13,7 +13,7 @@ # Directories to remove CLEAN_DIRS = [ "build", "dist", "*.egg-info", "clams_python-*", - ".pytest_cache", ".pytype", ".hypothesis", + ".pytest_cache", ".mypy_cache", ".hypothesis", "tests/.hypothesis", "htmlcov", "docs-test", ] diff --git a/build-tools/test.py b/build-tools/test.py index 1a93227..ea64a8c 100644 --- a/build-tools/test.py +++ b/build-tools/test.py @@ -3,7 +3,7 @@ This script is equivalent to ``make test`` in the Makefile-based repos: pip install -e ".[test]" - pytype --config .pytype.cfg clams + python -m mypy python -m pytest --cov=clams --cov-report=xml """ import argparse @@ -50,10 +50,10 @@ def main(): cwd=project_root, ) - # Run pytype static analysis - print("\n--- Running pytype ---") + # Run mypy static analysis (configured in pyproject.toml) + print("\n--- Running mypy ---") run_command( - ["pytype", "--config", ".pytype.cfg", "clams"], + [sys.executable, "-m", "mypy"], cwd=project_root, ) diff --git a/clams/__init__.py b/clams/__init__.py index 5ed8716..7968261 100644 --- a/clams/__init__.py +++ b/clams/__init__.py @@ -11,7 +11,7 @@ from clams.restify import Restifier from clams.ver import __version__ -__all__ = [AppMetadata, Restifier] + app_all +__all__ = ['AppMetadata', 'Restifier'] + app_all version_template = "{} (based on MMIF spec: {})" diff --git a/clams/app/__init__.py b/clams/app/__init__.py index de43bcb..0f4bac3 100644 --- a/clams/app/__init__.py +++ b/clams/app/__init__.py @@ -1,3 +1,4 @@ +import inspect import json import logging import os @@ -6,19 +7,22 @@ import warnings from abc import ABC, abstractmethod from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum from datetime import datetime from urllib import parse as urlparser -__all__ = ['ClamsApp', 'ClamsPromptableApp', 'ClamsHFPromptableApp'] +__all__ = ['ClamsApp', 'ClamsPromptableApp', 'ClamsHFPromptableApp', + 'PromptTask', 'ImagesPerPromptMode'] -from typing import Union, Any, Optional, Dict, List, Tuple, cast +from typing import Union, Any, Optional, Dict, List, Mapping, Tuple, cast from mmif import Mmif, Document, DocumentTypes, View, AnnotationTypes from mmif.utils.video_document_helper import ( SamplingMode, SAMPLING_MODE_DESCRIPTIONS, SAMPLING_MODE_DEFAULT, _sampling_mode, ) -from mmif.utils.workflow_helper import generate_param_hash # pytype: disable=import-error +from mmif.utils.workflow_helper import generate_param_hash from clams.appmetadata import AppMetadata, real_valued_primitives, python_type, map_param_kv_delimiter from clams.envelop import unwrap_if_envelope @@ -53,6 +57,47 @@ class EmptyOutputWarning(UserWarning): """Raised when an app's ``_annotate()`` produces no non-empty views.""" +class ImagesPerPromptMode(Enum): + """ + How :py:meth:`ClamsPromptableApp.chunk_tasks` sizes the chunks of a + task that has more images than the cap. Both modes produce the same + number of chunks; they differ in how images are spread over them. + """ + #: Fill each chunk to the cap; the last chunk holds the remainder. + MAX = 'max' + #: Spread images evenly, so chunk sizes differ by at most one. + BALANCED = 'balanced' + + +@dataclass(frozen=True) +class PromptTask: + """ + One unit of promptable work: the images bundled into a single prompt, + the annotations they were drawn from, and the annotation the response + is anchored to. + + ``images`` and ``origins`` are parallel: ``origins[i]`` is the ``id`` + of the annotation (typically a ``TimePoint``) that ``images[i]`` was + extracted from. ``source`` is the ``id`` of the coarse anchor + (typically the parent ``TimeFrame``). The fields map directly onto + :py:meth:`ClamsPromptableApp.response_to_grounded_textdocument`. + ``source`` also serves as the join key for app-side data about the + anchor, such as the TimeFrame label. + + :raises ValueError: if ``images`` and ``origins`` differ in length. + """ + images: List[Any] + origins: List[str] + source: str + + def __post_init__(self): + if len(self.images) != len(self.origins): + raise ValueError( + f"PromptTask images and origins must be parallel; got " + f"{len(self.images)} images and {len(self.origins)} " + f"origins for source {self.source!r}.") + + class ClamsApp(ABC): """ An abstract class to define API's for ClamsApps. A CLAMS app should inherit @@ -62,7 +107,7 @@ class ClamsApp(ABC): # A set of "universal runtime parameters that can be used for both GET and POST anytime". # The behavioral changes based on these parameters must be implemented on the SDK level. # This needs to stay as a list of dicts for compatibility with the metadata.py file (templated). - universal_parameters = [ + universal_parameters: List[Dict[str, Any]] = [ { 'name': 'pretty', 'type': 'boolean', 'choices': None, 'default': False, 'multivalued': False, 'description': 'The JSON body of the HTTP response will be re-formatted with 2-space indentation', @@ -137,10 +182,10 @@ def _load_appmetadata(self) -> AppMetadata: For metadata specification, see `https://clams.ai/clams-python/appmetadata.jsonschema <../appmetadata.jsonschema>`_. """ - cwd = pathlib.Path(sys.modules[self.__module__].__file__).parent + cwd = pathlib.Path(inspect.getfile(type(self))).parent if (cwd / 'metadata.py').exists(): - import metadata as metadatapy # pytype: disable=import-error + import metadata as metadatapy metadata = metadatapy.appmetadata() else: metadata = self._appmetadata() @@ -180,7 +225,7 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) -> mmif, runtime_params = unwrap_if_envelope(mmif, runtime_params) mmif = Mmif(mmif) existing_view_ids = {view.id for view in mmif.views} - issued_warnings = [] + issued_warnings: List[Union[Warning, warnings.WarningMessage]] = [] for key in runtime_params: if key not in self.annotate_param_spec: issued_warnings.append(UserWarning(f'An undefined parameter "{key}" (value: "{runtime_params[key]}") is passed')) @@ -214,7 +259,7 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) -> td = run_id - t runningTime = refined.get('runningTime', False) hwFetch = refined.get('hwFetch', False) - runtime_recs = {} + runtime_recs: Dict[str, Any] = {} if hwFetch: import multiprocessing import platform, shutil, subprocess @@ -237,7 +282,7 @@ def annotate(self, mmif: Union[str, dict, Mmif], **runtime_params: List[str]) -> for annotated_view in annotated.views: if annotated_view.id not in existing_view_ids and annotated_view.metadata.app == str(self.metadata.identifier): annotated_view.metadata.timestamp = run_id - profiling_data = {} + profiling_data: Dict[str, Any] = {} if runningTime: profiling_data['runningTime'] = str(td) if len(runtime_recs) > 0: @@ -280,7 +325,7 @@ def _refine_params(self, **runtime_params: List[str]): if self._RAW_PARAMS_KEY in runtime_params: # meaning the dict is already refined, just return it return runtime_params - refined = {} + refined: Dict[str, Any] = {} casted = self.annotate_param_caster.cast(runtime_params) for parameter in self.metadata.parameters: @@ -491,7 +536,7 @@ def _get_available_vram() -> int: # Fallback to torch (only sees current process memory) try: - import torch # pytype: disable=import-error + import torch if not torch.cuda.is_available(): return 0 @@ -595,7 +640,7 @@ def wrapper(*args, **kwargs): available_before = {} try: - import torch # pytype: disable=import-error + import torch torch_available = True cuda_available = torch.cuda.is_available() device_count = torch.cuda.device_count() @@ -656,11 +701,9 @@ def wrapper(*args, **kwargs): } # Record peak memory for future requests (if GPU app) - gpu_app = ( - hasattr(app_instance, 'metadata') and - getattr(app_instance.metadata, 'est_gpu_mem_min', 0) > 0 - ) - if gpu_app and total_peak > 0: + app_metadata = getattr(app_instance, 'metadata', None) + gpu_app = getattr(app_metadata, 'est_gpu_mem_min', 0) > 0 + if app_instance is not None and gpu_app and total_peak > 0: app_instance._record_vram_usage(kwargs, total_peak) return result, cuda_profiler @@ -729,6 +772,11 @@ class ClamsPromptableApp(ClamsApp): parameter set to ``AppMetadata`` * :py:meth:`build_conversation` : assembles a chat-template-compatible message list from a prompt plus optional images/audios + * :py:meth:`chunk_tasks` : caps images per prompt by splitting + :class:`PromptTask`\\ s (``maxImagesPerPrompt``) + * :py:meth:`generate_batched` : runs :py:meth:`generate` in batches + of at most ``parallelPrompts`` prompts, reporting through + :py:meth:`report_progress` * :py:meth:`response_to_grounded_textdocument` : persists a generated response into a view as ``TextDocument`` + ``Alignment`` (+ optional ``origins`` / ``origination``) @@ -737,7 +785,7 @@ class ClamsPromptableApp(ClamsApp): #: SDK-managed runtime parameters injected into every promptable app. #: These names are reserved; apps cannot redeclare them with #: customized specs. - promptable_parameters = [ + promptable_parameters: List[Dict[str, Any]] = [ { 'name': 'prompt', 'type': 'string', 'multivalued': True, 'description': @@ -827,6 +875,38 @@ class ClamsPromptableApp(ClamsApp): 'Keep at ``1`` on memory-tight setups; raise only when ' 'per-prompt content is small and bounded.', }, + { + 'name': 'maxImagesPerPrompt', 'type': 'integer', 'default': 0, + 'description': + 'Maximum number of images the app bundles into a single ' + 'prompt (one generation, one output TextDocument). ``0`` ' + 'means no limit, so all images for a TimeFrame go into one ' + 'prompt and yield one TextDocument. With a positive N, the ' + 'app splits a TimeFrame whose frames exceed N into ' + 'consecutive groups of at most N; each group produces its ' + 'own TextDocument aligned to the same TimeFrame and grounded ' + 'to its own source TimePoints via ``origins``. This bounds ' + 'GPU memory for long dynamic scenes. The value is ' + 'operator-set, so the number of output TextDocuments is ' + 'deterministic and does not vary by GPU. Composes with ' + '``parallelPrompts``, which batches independent prompts ' + 'into one forward pass.', + }, + { + 'name': 'imagesPerPromptMode', 'type': 'string', + 'choices': [m.value for m in ImagesPerPromptMode], + 'default': ImagesPerPromptMode.MAX.value, + 'description': + 'How the images of a TimeFrame that exceeds ' + '``maxImagesPerPrompt`` are spread over its prompts. Has ' + 'no effect when ``maxImagesPerPrompt`` is ``0``. Both ' + 'choices produce the same number of prompts. "max" fills ' + 'each prompt to the cap and puts the remainder in the ' + 'last one (33 images at a cap of 32 give 32 and 1). ' + '"balanced" spreads the images evenly, so prompt sizes ' + 'differ by at most one (33 images at a cap of 32 give 17 ' + 'and 16).', + }, ] @staticmethod @@ -978,7 +1058,7 @@ def build_conversation( if isinstance(prompt, str): prompts = [prompt] else: - prompts = list(prompt) + prompts = cast(List[str], list(prompt)) if len(prompts) == 1: return self._build_single_turn( @@ -1072,6 +1152,148 @@ def _build_user_only(self, prompts, system_prompt, images, audios): base.append({'role': 'assistant', 'content': None}) return convs + def generate_batched( + self, + prompt: List[str], + system_prompt: str = '', + images: Optional[List[List[Any]]] = None, + audios: Optional[List[List[Any]]] = None, + prompt_mode: str = 'turn-taking', + parallel_prompts: int = 1, + **generation_params, + ) -> List[str]: + """ + Run N prompts through :py:meth:`generate` in batches of at most + ``parallel_prompts`` prompts per call, and return N outputs in + input order. Apps typically pass the ``parallelPrompts`` runtime + parameter as ``parallel_prompts``. + + The arguments other than ``parallel_prompts`` have the same + meaning as in :py:meth:`generate` and are forwarded to it. A + text-only call (``images`` and ``audios`` both ``None``) is one + prompt and runs as a single :py:meth:`generate` call. + + Progress is reported through :py:meth:`report_progress`: once + before the first batch and once after each batch. + + :param parallel_prompts: maximum number of prompts per + :py:meth:`generate` call. + :returns: one output per prompt, in input order. + :rtype: List[str] + :raises ValueError: if ``parallel_prompts`` is less than 1, or if + ``images`` and ``audios`` are both given with different outer + lengths. + """ + if parallel_prompts < 1: + raise ValueError( + f"parallel_prompts must be at least 1; " + f"got {parallel_prompts}.") + if images is not None and audios is not None \ + and len(images) != len(audios): + raise ValueError( + f"images and audios must have the same outer length " + f"when both are given; got " + f"{len(images)} vs {len(audios)}.") + groups = images if images is not None else audios + if groups is None: + self.report_progress(0, 1) + outputs = self.generate( + prompt, system_prompt=system_prompt, + prompt_mode=prompt_mode, **generation_params) + self.report_progress(1, 1) + return outputs + total = len(groups) + outputs = [] + self.report_progress(0, total) + for start in range(0, total, parallel_prompts): + end = start + parallel_prompts + outputs.extend(self.generate( + prompt, system_prompt=system_prompt, + images=images[start:end] if images is not None else None, + audios=audios[start:end] if audios is not None else None, + prompt_mode=prompt_mode, **generation_params)) + self.report_progress(min(end, total), total) + return outputs + + def report_progress(self, done: int, total: int) -> None: + """ + Receive generation progress from :py:meth:`generate_batched`, + which calls this once with ``done=0`` before the first batch and + once after each batch. The default implementation writes one + INFO log line. Override to report differently (another log + level or wording, a progress bar) or to stay silent. + + The app instance serves all requests, so an override that keeps + state on ``self`` between calls is shared by concurrent + requests. + + :param done: number of prompts generated so far. + :param total: number of prompts in this + :py:meth:`generate_batched` call. + """ + self.logger.info(f"Generated {done}/{total} prompts") + + @staticmethod + def chunk_tasks( + tasks: List[PromptTask], + max_images: int, + mode: Union[str, ImagesPerPromptMode] = ImagesPerPromptMode.MAX, + ) -> List[PromptTask]: + """ + Split each task with more than ``max_images`` images into + consecutive tasks of at most ``max_images`` images, chunking + ``origins`` in step and keeping ``source``. Tasks at or under the + cap pass through unchanged, and task order is preserved. Apps + typically pass the ``maxImagesPerPrompt`` and + ``imagesPerPromptMode`` runtime parameters as ``max_images`` and + ``mode``. + + Each resulting task yields one response and one ``TextDocument``, + so an over-cap TimeFrame produces several ``TextDocument``\\ s + aligned to the same ``source``, each grounded to its own + ``origins``. + + :param tasks: the tasks to split. + :param max_images: maximum number of images per task. ``0`` means + no cap. + :param mode: how the images of an over-cap task are spread over + its chunks; see :class:`ImagesPerPromptMode`. + :returns: the capped tasks, in order. + :rtype: List[PromptTask] + :raises ValueError: if ``max_images`` is negative or ``mode`` is + not a valid :class:`ImagesPerPromptMode`. + """ + mode = ImagesPerPromptMode(mode) + if max_images < 0: + raise ValueError( + f"max_images must be 0 (no cap) or positive; " + f"got {max_images}.") + if max_images == 0: + return list(tasks) + capped: List[PromptTask] = [] + for task in tasks: + n_images = len(task.images) + if n_images <= max_images: + capped.append(task) + continue + n_chunks = -(-n_images // max_images) + if mode is ImagesPerPromptMode.BALANCED: + size, n_larger = divmod(n_images, n_chunks) + sizes = ([size + 1] * n_larger + + [size] * (n_chunks - n_larger)) + else: + sizes = [max_images] * (n_chunks - 1) + sizes.append(n_images - sum(sizes)) + start = 0 + for size in sizes: + end = start + size + capped.append(PromptTask( + images=task.images[start:end], + origins=task.origins[start:end], + source=task.source)) + start = end + return capped + def response_to_grounded_textdocument( self, view: View, @@ -1330,7 +1552,8 @@ def inject_promptable_parameters(metadata: AppMetadata) -> None: "non-empty dict (HF model id -> commit hash). Set " "it on the ``AppMetadata`` constructor call before " "invoking this helper.") - choices = list(analyzer_versions.keys()) + choices: List[real_valued_primitives] = list( + analyzer_versions.keys()) default = choices[0] if len(choices) == 1 else None metadata.add_parameter( name='model', @@ -1438,7 +1661,7 @@ def load_model( model_id, _, revision = model_id_or_with_rev.rpartition('@') else: model_id = model_id_or_with_rev - revision = self.metadata.analyzer_versions[model_id] + revision = (self.metadata.analyzer_versions or {})[model_id] cache_key = (model_id, revision) cached = self._model_cache.get(cache_key) if cached is not None: @@ -1590,7 +1813,7 @@ def build_gen_kwargs( ignores them, so subclasses can pass through the full ``**parameters`` dict from ``_annotate`` without filtering. """ - gen_kwargs = {'max_new_tokens': max_new_tokens} + gen_kwargs: Dict[str, Any] = {'max_new_tokens': max_new_tokens} if temperature > 0: gen_kwargs.update({ 'do_sample': True, @@ -1628,7 +1851,7 @@ def build_template_kwargs(self, **generation_params) -> dict: class ParameterCaster(object): - def __init__(self, param_spec: Dict[str, Tuple[str, bool]]): + def __init__(self, param_spec: Mapping[str, Tuple[str, bool]]): """ A helper class to convert parameters passed by HTTP query strings to proper python data types. @@ -1657,7 +1880,7 @@ def cast(self, args: Dict[str, List[str]]) \ With the third case, developers can further process the nested values into a more complex data types or structures, but that is not in the scope of this Caster class. """ - casted = {} + casted: Dict[str, Any] = {} for k, vs in args.items(): assert isinstance(vs, list), f"Expected a list of values for key {k}, but got {vs} of type {type(vs)}" assert all(isinstance(v, str) for v in vs), f"Expected a list of strings for key {k}, but got {vs} of types {[type(v) for v in vs]}" @@ -1665,29 +1888,24 @@ def cast(self, args: Dict[str, List[str]]) \ valuetype, multivalued = self.param_spec[k] for v in vs: if multivalued or k not in casted: # effectively only keeps the first value for non-multi params + value: Any = v if valuetype == bool: - v = self.bool_param(v) + value = self.bool_param(v) elif valuetype == float: - v = self.float_param(v) + value = self.float_param(v) elif valuetype == int: - v = self.int_param(v) + value = self.int_param(v) elif valuetype == str: - v = self.str_param(v) + value = self.str_param(v) elif valuetype == dict: - v = self.kv_param(v) + value = self.kv_param(v) if multivalued: if valuetype == dict: - casted.setdefault(k, {}).update(v) + casted.setdefault(k, {}).update(value) else: - # pytype will complain about the next line, but it is actually correct - # casted.setdefault(k, []).append(v) - # so doing it in a more explicit way - if k in casted and isinstance(casted[k], list): - casted[k].append(v) - else: - casted[k] = [v] + casted.setdefault(k, []).append(value) else: - casted[k] = v + casted[k] = value # when an empty value is passed (usually as a default value) # just add an empty list or dict as a placeholder # explicit check for `len == 0` is required to prevent @@ -1704,7 +1922,7 @@ def cast(self, args: Dict[str, List[str]]) \ casted[k] = vs[0] else: casted[k] = vs - return casted # pytype: disable=bad-return-type + return casted @staticmethod def bool_param(value) -> bool: diff --git a/clams/appmetadata/__init__.py b/clams/appmetadata/__init__.py index b6fe185..588a867 100644 --- a/clams/appmetadata/__init__.py +++ b/clams/appmetadata/__init__.py @@ -1,10 +1,11 @@ +import inspect import json import os import shutil import subprocess import sys from pathlib import Path -from typing import Union, Dict, List, Optional, Literal, Any +from typing import Union, Dict, List, Optional, Literal, Any, get_args import mmif import pydantic @@ -22,7 +23,7 @@ # dict to map app parameter data types to Python data types python_type = {"boolean": bool, "number": float, "integer": int, "string": str, "map": dict} -param_value_types_values = param_value_types.__args__ # pytype: disable=attribute-error +param_value_types_values = get_args(param_value_types) app_directory_baseurl = "http://apps.clams.ai" @@ -42,7 +43,9 @@ def get_clams_pyver(): def generate_app_version(cwd=None): gitcmd = shutil.which('git') - gitdir = (Path(sys.modules["__main__"].__file__).parent.resolve() if cwd is None else Path(cwd)) / '.git' + if cwd is None: + cwd = Path(inspect.getfile(sys.modules["__main__"])).parent.resolve() + gitdir = Path(cwd) / '.git' if gitcmd is not None and gitdir.exists(): try: proc = subprocess.run([gitcmd, '--git-dir', str(gitdir), 'describe', '--tags', '--always'], @@ -60,6 +63,12 @@ def get_mmif_specver(): return mmif.__specver__ +# TODO (krim @ 10/05/26): in the 2.0.0 cycle, make the pydantic-specific +# helpers in this module private (underscore-prefixed), as they are not +# part of the developer-facing API: the schema adjusters (`pop_titles`, +# `jsonschema_versioning`) and the validators (`validate_properties`, +# `at_type_must_be_str`, `assign_versions`, `validate_gpu_memory`, +# `append_version`). def pop_titles(js): for prop in js.get('properties', {}).values(): prop.pop('title', None) @@ -70,6 +79,11 @@ def jsonschema_versioning(js): js['$comment'] = f"clams-python SDK {get_clams_pyver()} was used to generate this schema" +def _adjust_appmetadata_schema(js): + pop_titles(js) + jsonschema_versioning(js) + + class _BaseModel(pydantic.BaseModel): model_config = { @@ -381,7 +395,7 @@ class AppMetadata(pydantic.BaseModel): 'title': 'CLAMS AppMetadata', 'extra': 'forbid', 'validate_by_name': True, - 'json_schema_extra': lambda schema, model: [adjust(schema) for adjust in [pop_titles, jsonschema_versioning]], + 'json_schema_extra': _adjust_appmetadata_schema, } @pydantic.model_validator(mode='after') diff --git a/clams/backends/hf.py b/clams/backends/hf.py index b2dcfab..1dc0a06 100644 --- a/clams/backends/hf.py +++ b/clams/backends/hf.py @@ -112,14 +112,14 @@ class with ``from_pretrained()``. Defaults to installed. Install the ``[hf]`` extra to fix. """ try: - import torch # pytype: disable=import-error + import torch except ImportError as e: raise ImportError( "clams.backends.hf requires the `torch` package. " "Install with: pip install clams-python[hf]" ) from e try: - import transformers # pytype: disable=import-error + import transformers except ImportError as e: raise ImportError( "clams.backends.hf requires the `transformers` package. " @@ -212,14 +212,14 @@ def load_hf_pipeline( installed. Install the ``[hf]`` extra to fix. """ try: - import torch # pytype: disable=import-error + import torch except ImportError as e: raise ImportError( "clams.backends.hf requires the `torch` package. " "Install with: pip install clams-python[hf]" ) from e try: - from transformers import pipeline # pytype: disable=import-error + from transformers import pipeline except ImportError as e: raise ImportError( "clams.backends.hf requires the `transformers` package. " diff --git a/clams/develop/__init__.py b/clams/develop/__init__.py index 5cbd84f..cb682e4 100644 --- a/clams/develop/__init__.py +++ b/clams/develop/__init__.py @@ -37,7 +37,7 @@ def __init__(self, name: str, outdir: str, recipes: List[str]): if recipes: self.recipes = recipes else: - self.recipes = available_recipes.keys() + self.recipes = list(available_recipes.keys()) def tokenize_rawname(self): word_pat = re.compile('[A-Z][a-z]+|[0-9A-Z]+(?=[A-Z][a-z])|[0-9A-Z]{2,}|[a-z0-9]{2,}|[a-zA-Z0-9]') diff --git a/clams/develop/templates/app/Containerfile.template b/clams/develop/templates/app/Containerfile.template index 36ffdaf..c832af9 100644 --- a/clams/develop/templates/app/Containerfile.template +++ b/clams/develop/templates/app/Containerfile.template @@ -14,20 +14,6 @@ ARG CLAMS_APP_VERSION ENV CLAMS_APP_VERSION ${CLAMS_APP_VERSION} ################################################################################ -################################################################################ -# This is duplicate from the base image Containerfile -# but makes sure the cache directories are consistent across all CLAMS apps - -# https://github.com/openai/whisper/blob/ba3f3cd54b0e5b8ce1ab3de13e32122d0d5f98ab/whisper/__init__.py#L130 -ENV XDG_CACHE_HOME='/cache' -# https://huggingface.co/docs/huggingface_hub/main/en/package_reference/environment_variables#hfhome -ENV HF_HOME="/cache/huggingface" -# https://pytorch.org/docs/stable/hub.html#where-are-my-downloaded-models-saved -ENV TORCH_HOME="/cache/torch" - -RUN mkdir /cache ; rm -rf /root/.cache ; ln -s /cache /root/.cache -################################################################################ - ################################################################################ # clams-python base images are based on debian distro # install more system packages as needed using the apt manager diff --git a/clams/develop/templates/utl-tf/timeframe.py.template b/clams/develop/templates/utl-tf/timeframe.py.template index d2c8d7b..5dc9585 100644 --- a/clams/develop/templates/utl-tf/timeframe.py.template +++ b/clams/develop/templates/utl-tf/timeframe.py.template @@ -16,13 +16,14 @@ processing video by TimeFrames tends to write: 3. when ``vdh`` returns a fallback timestamp (milliseconds, no existing TP behind it), mint a fresh ``TimePoint`` annotation in the app's new view so downstream code has a stable anchor id - 4. assemble per-TF task tuples that downstream batching / - inference / annotation code can consume uniformly + 4. assemble one ``clams.app.PromptTask`` per TF that downstream + batching / inference / annotation code can consume uniformly The helpers are backend-agnostic: tasks can feed a HuggingFace VLM, a remote LLM API, a classical CV pipeline, or any other per-frame -processor. They have no dependency on ``clams.app.ClamsPromptableApp`` -or any other promptable / inference machinery. +processor. Their only SDK dependency is the ``PromptTask`` data type; +they do not use ``clams.app.ClamsPromptableApp`` or any other +inference machinery. These functions are scaffolded into each app so individual apps can edit them freely while the pattern stabilizes across the ecosystem. @@ -30,12 +31,13 @@ Once the shape converges across several apps, the helpers are good candidates for promotion into a shared package -- either ``mmif.utils`` (for the pure-MMIF iteration / TP minting pieces, which have no clams-app dependency) or ``clams.`` (for the -task-tuple composition that does presuppose the "writing into a new -view" CLAMS-app idiom). If/when that happens, apps would import the +task composition that does presuppose the "writing into a new view" +CLAMS-app idiom). If/when that happens, apps would import the shared version and delete this local copy. """ -from typing import Any, Iterator, List, Optional, Tuple, Union +from typing import Iterator, List, Union +from clams.app import PromptTask from mmif import Annotation, Document, Mmif, View, AnnotationTypes from mmif.utils import video_document_helper as vdh @@ -117,21 +119,37 @@ def collect_timeframes_of_interest( parent_view: View, video_doc: Document, tflabels_of_interest: List[str], -) -> List[Tuple[List[Any], List[str], str, Optional[str]]]: +) -> List[PromptTask]: """ Convenience composition of :func:`iter_timeframes`, :func:`vdh.extract_images_by_mode_with_sources`, and - :func:`to_timepoints`. Returns one - ``(images, tp_ids, tf_id, tf_label)`` task per matching TimeFrame - that produced at least one sampled frame. + :func:`to_timepoints`. Returns one ``PromptTask`` per matching + TimeFrame that produced at least one sampled frame. - Each task's ``images`` and ``tp_ids`` are parallel lists -- one + Each task's ``images`` and ``origins`` are parallel lists -- one entry per frame sampled from that TF (length 1 for ``tfSamplingMode=single``, N for ``representatives`` / ``all``). - Each entry of ``tp_ids`` is either the id of an existing + Each entry of ``origins`` is either the id of an existing TimePoint or the id of a freshly-minted one (see - :func:`to_timepoints`). ``tf_label`` is the source TimeFrame's - ``label`` property value, or ``None`` if unset. + :func:`to_timepoints`). ``source`` is the TimeFrame id; look up + other TF properties through it, e.g. + ``mmif[task.source].get_property('label')``. + + In a ``ClamsPromptableApp``, the tasks feed the SDK helpers that + honor the ``maxImagesPerPrompt``, ``imagesPerPromptMode``, and + ``parallelPrompts`` runtime parameters:: + + tasks = self.chunk_tasks( + collect_timeframes_of_interest(...), + parameters['maxImagesPerPrompt'], + parameters['imagesPerPromptMode']) + responses = self.generate_batched( + parameters['prompt'], images=[t.images for t in tasks], + parallel_prompts=parameters['parallelPrompts'], ...) + for task, response in zip(tasks, responses): + self.response_to_grounded_textdocument( + view, task.source, response, + origins=task.origins, origination='derived') :param mmif: the input MMIF. :param parent_view: the view this app is writing into. @@ -139,16 +157,17 @@ def collect_timeframes_of_interest( extracted from. :param tflabels_of_interest: optional label filter; empty list = no filter. - :return: per-TF task tuples, ready to feed a batched inference - loop or any other per-frame processor. + :return: per-TF tasks, ready to feed a batched inference loop or + any other per-frame processor. """ - tasks: List[Tuple[List[Any], List[str], str, Optional[str]]] = [] + tasks: List[PromptTask] = [] for tf in iter_timeframes(mmif, tflabels_of_interest): images, sources = vdh.extract_images_by_mode_with_sources( mmif, tf, as_PIL=True) if not images: continue - tp_ids = to_timepoints(parent_view, video_doc, sources) - tf_label = tf.get_property('label') - tasks.append((list(images), tp_ids, tf.id, tf_label)) + tasks.append(PromptTask( + images=list(images), + origins=to_timepoints(parent_view, video_doc, sources), + source=tf.id)) return tasks diff --git a/clams/envelop/__init__.py b/clams/envelop/__init__.py index c925bb8..c9ca535 100644 --- a/clams/envelop/__init__.py +++ b/clams/envelop/__init__.py @@ -107,7 +107,7 @@ def unwrap_if_envelope(data, runtime_params): body = json.loads(raw) except (json.JSONDecodeError, ValueError): return data, runtime_params - if is_envelope(body): + if isinstance(body, dict) and is_envelope(body): inner_mmif, envelope_params = unwrap_envelope(body) return inner_mmif, {**envelope_params, **runtime_params} return data, runtime_params diff --git a/documentation/app-baseclasses.rst b/documentation/app-baseclasses.rst index 8a33f5e..895d1a9 100644 --- a/documentation/app-baseclasses.rst +++ b/documentation/app-baseclasses.rst @@ -213,6 +213,24 @@ from :class:`~clams.app.ClamsApp`. These names are reserved; see setups; see the parameter's own description in :py:attr:`~clams.app.ClamsPromptableApp.promptable_parameters` for an OOM-risk example. + * - ``maxImagesPerPrompt`` + - integer + - ``0`` + - no + - Maximum number of images the app bundles into a single prompt (one + generation, one output ``TextDocument``). ``0`` means no limit. With a + positive value, the app splits a TimeFrame with more frames into + consecutive capped groups, each producing its own ``TextDocument`` + aligned to that TimeFrame and grounded via ``origins``. Bounds GPU + memory for long dynamic scenes; composes with ``parallelPrompts``. + * - ``imagesPerPromptMode`` + - string + - ``max`` + - no + - How the images of a TimeFrame over ``maxImagesPerPrompt`` are spread + over its prompts; no effect when the cap is ``0``. ``max`` fills each + prompt to the cap and leaves the remainder last. ``balanced`` spreads + the images evenly. Both give the same number of prompts. .. _promptable-customizing-defaults: @@ -291,6 +309,31 @@ This separation lets HF-backed apps inherit the default ``generate()`` without restating backend mechanics, and lets non-HF apps swap in a new ``generate()`` without rewriting their MMIF I/O. +.. _promptable-grouping: + +From TimeFrames to forward passes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Three parameters group the visual input at three different steps. The +documentation and the helper names use one word for each step. + +.. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - Step + - Parameter + - Word + * - TimeFrame to frames + - ``tfSamplingMode`` + - sample + * - One TimeFrame's frames to prompts + - ``maxImagesPerPrompt``, ``imagesPerPromptMode`` + - chunk + * - Prompts to forward passes + - ``parallelPrompts`` + - batch + .. _promptable-multiturn: Multi-turn handling (``promptMode``) @@ -394,6 +437,28 @@ Helpers may override to access model-specific state (e.g. ``self.processor``) when formatting messages. +:meth:`~clams.app.ClamsPromptableApp.chunk_tasks` + A static method that applies ``maxImagesPerPrompt``: splits each + :class:`~clams.app.PromptTask` (images, their ``origins``, and the + anchoring ``source``) with more images than the cap into consecutive + capped tasks. Each task yields one ``TextDocument``, so with a cap + set, one TimeFrame can yield several ``TextDocument`` annotations + aligned to it, each grounded to its own ``origins``. Consumers of + promptable-app output must not assume one ``TextDocument`` per + TimeFrame. ``imagesPerPromptMode`` + (:class:`~clams.app.ImagesPerPromptMode`) selects how the images are + spread over the chunks. + +:meth:`~clams.app.ClamsPromptableApp.generate_batched` + Applies ``parallelPrompts``: calls ``generate()`` on batches of at + most that many prompts and returns one output per prompt, in order. + +:meth:`~clams.app.ClamsPromptableApp.report_progress` + Called by ``generate_batched()`` before the first batch and after + each batch with the number of prompts done and the total. The + default writes one INFO log line; override it to report differently + or to stay silent. + :meth:`~clams.app.ClamsPromptableApp.response_to_grounded_textdocument` Writes a ``TextDocument`` plus an ``Alignment`` (``source -> TD``) into a view. ``source`` is the coarse cross-modal anchor; the diff --git a/documentation/target-versions.csv b/documentation/target-versions.csv index ae3e625..9fa6ba8 100644 --- a/documentation/target-versions.csv +++ b/documentation/target-versions.csv @@ -1,4 +1,5 @@ "``clams-python`` version","``mmif-python`` version","Target MMIF Specification" +`1.8.0 `__,`1.5.4 `__,`1.2.0 `__ `1.7.4 `__,`1.5.3 `__,`1.2.0 `__ `1.7.3 `__,`1.5.1 `__,`1.2.0 `__ `1.7.2 `__,`1.5.1 `__,`1.2.0 `__ diff --git a/pyproject.toml b/pyproject.toml index 2abe89a..28989b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3 :: Only", ] dependencies = [ - "mmif-python==1.5.3", + "mmif-python==1.5.4", "Flask>=2", "Flask-RESTful>=0.3.9", "gunicorn>=20", @@ -36,9 +36,9 @@ homepage = "https://clams.ai" source = "https://github.com/clamsproject/clams-python" [project.optional-dependencies] -dev = ["pytype", "pytest", "pytest-cov", "pillow", "setuptools"] +dev = ["mypy", "pytest", "pytest-cov", "pillow", "setuptools"] docs = ["sphinx>=7.0,<8.0", "furo", "m2r2", "sphinx-jsonschema"] -test = ["pytype", "pytest", "pytest-cov", "pillow"] +test = ["mypy", "pytest", "pytest-cov", "pillow"] # Required for apps using the HuggingFace transformers backend # (clams.backends.hf). Heavy deps; opt-in only. hf = ["torch", "transformers", "pillow", "tqdm"] @@ -53,3 +53,18 @@ clams = ["develop/templates/**/*", "develop/templates/**/.*"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--cov=clams --cov-report=xml" + +[tool.mypy] +files = ["clams"] +exclude = ["clams/develop/templates/"] +check_untyped_defs = true + +[[tool.mypy.overrides]] +# dependencies that ship neither inline types nor a py.typed marker +module = ["mmif.*", "flask_restful.*", "gunicorn.*", "jsonschema.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +# optional backends (the `hf` extra) and the `metadata.py` of a running app +module = ["torch.*", "transformers.*", "metadata"] +ignore_missing_imports = true diff --git a/tests/test_develop.py b/tests/test_develop.py new file mode 100644 index 0000000..95a8c02 --- /dev/null +++ b/tests/test_develop.py @@ -0,0 +1,122 @@ +""" +Tests for the code-generation recipes in :mod:`clams.develop`. +""" +import contextlib +import importlib.util +import io +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from mmif import AnnotationTypes, Document, DocumentTypes, Mmif + +from clams.app import PromptTask +from clams.develop import CookieCutter + + +class TestUtlTfRecipe(unittest.TestCase): + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + with contextlib.redirect_stdout(io.StringIO()): + CookieCutter( + name='tfapp', outdir=tmp.name, recipes=['utl-tf']).bake() + path = Path(tmp.name) / 'tfapp' / 'utils' / 'timeframe.py' + mod_name = f'_baked_timeframe_{id(self)}' + spec = importlib.util.spec_from_file_location(mod_name, path) + self.baked = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = self.baked + self.addCleanup(sys.modules.pop, mod_name, None) + spec.loader.exec_module(self.baked) + + self.mmif = Mmif(validate=False) + self.vdoc = Document() + self.vdoc.at_type = DocumentTypes.VideoDocument + self.vdoc.id = 'v1' + self.vdoc.location = 'file:///video.mp4' + self.mmif.add_document(self.vdoc) + up_view = self.mmif.new_view() + up_view.metadata.app = 'http://upstream/1' + self.tp = up_view.new_annotation( + AnnotationTypes.TimePoint, document=self.vdoc.id, + timePoint=500) + self.tf_a = up_view.new_annotation( + AnnotationTypes.TimeFrame, document=self.vdoc.id, + label='scene') + self.tf_b = up_view.new_annotation( + AnnotationTypes.TimeFrame, document=self.vdoc.id, + label='slate') + self.out_view = self.mmif.new_view() + self.out_view.metadata.app = 'http://downstream/1' + + def patch_extractor(self, results): + """ + Patch the extractor used by the baked module. + + :param results: map of TimeFrame id to ``(images, sources)`` + """ + def fake(mmif, tf, **kwargs): + return results[tf.id] + + return mock.patch.object( + self.baked.vdh, 'extract_images_by_mode_with_sources', fake) + + def collect(self, labels=()): + return self.baked.collect_timeframes_of_interest( + self.mmif, self.out_view, self.vdoc, list(labels)) + + def test_baked_module_exposes_helpers(self): + for name in ('iter_timeframes', 'to_timepoints', + 'collect_timeframes_of_interest'): + self.assertTrue(callable(getattr(self.baked, name))) + + def test_tasks_carry_images_sources_and_origins(self): + results = { + self.tf_a.id: (['i0', 'i1', 'i2'], [self.tp.id, 1500, 2500]), + self.tf_b.id: (['j0'], [self.tp.id]), + } + with self.patch_extractor(results): + tasks = self.collect() + self.assertEqual(len(tasks), 2) + for task in tasks: + self.assertIsInstance(task, PromptTask) + self.assertEqual(len(task.images), len(task.origins)) + task_a, task_b = tasks + self.assertEqual(task_a.source, self.tf_a.id) + self.assertEqual(task_a.images, ['i0', 'i1', 'i2']) + self.assertEqual(task_a.origins[0], self.tp.id) + out_ids = [a.id for a in self.out_view.get_annotations()] + for origin, expected_ms in zip(task_a.origins[1:], (1500, 2500)): + self.assertNotEqual(origin, self.tp.id) + self.assertIn(origin, out_ids) + minted = self.mmif[origin] + self.assertEqual(minted.at_type, AnnotationTypes.TimePoint) + self.assertEqual(minted.get_property('timePoint'), expected_ms) + self.assertEqual( + minted.get_property('document'), self.vdoc.id) + self.assertEqual(task_b.source, self.tf_b.id) + self.assertEqual(task_b.images, ['j0']) + self.assertEqual(task_b.origins, [self.tp.id]) + # only the two int sources mint new TimePoints + self.assertEqual(len(out_ids), 2) + + def test_label_filter_keeps_only_matching_timeframes(self): + results = { + self.tf_a.id: (['i0'], [self.tp.id]), + self.tf_b.id: (['j0'], [self.tp.id]), + } + with self.patch_extractor(results): + tasks = self.collect(['slate']) + self.assertEqual([t.source for t in tasks], [self.tf_b.id]) + + def test_timeframe_without_images_yields_no_task(self): + results = { + self.tf_a.id: ([], []), + self.tf_b.id: (['j0'], [self.tp.id]), + } + with self.patch_extractor(results): + tasks = self.collect() + self.assertEqual([t.source for t in tasks], [self.tf_b.id]) diff --git a/tests/test_promptable.py b/tests/test_promptable.py index f394fa7..2cd2ab6 100644 --- a/tests/test_promptable.py +++ b/tests/test_promptable.py @@ -13,6 +13,7 @@ from mmif import AnnotationTypes, Document, DocumentTypes, Mmif from clams import AppMetadata, ClamsPromptableApp +from clams.app import PromptTask # --------------------------------------------------------------------------- @@ -127,6 +128,266 @@ def test_prompt_mode_choices(self): self.assertEqual(pm.default, 'turn-taking') +# --------------------------------------------------------------------------- +# PromptTask and chunk_tasks (maxImagesPerPrompt) +# --------------------------------------------------------------------------- + +def make_task(n_images, source='tf1'): + """ + Build a PromptTask whose ``origins[i]`` names the frame in + ``images[i]``, so alignment can be checked after chunking. + """ + return PromptTask( + images=[f'{source}-img{i}' for i in range(n_images)], + origins=[f'{source}-tp{i}' for i in range(n_images)], + source=source) + + +class TestPromptTask(unittest.TestCase): + + def test_mismatched_images_and_origins_raise(self): + with self.assertRaises(ValueError): + PromptTask(images=['a', 'b'], origins=['tp1'], source='tf1') + + +class TestChunkTasks(unittest.TestCase): + + def test_no_cap_keeps_one_task_per_timeframe(self): + tasks = [make_task(90, 'tf1'), make_task(3, 'tf2')] + self.assertEqual(ClamsPromptableApp.chunk_tasks(tasks, 0), tasks) + + def test_task_at_cap_is_not_split(self): + task = make_task(32) + self.assertEqual( + ClamsPromptableApp.chunk_tasks([task], 32), [task]) + + def test_over_cap_splits_in_order(self): + chunks = ClamsPromptableApp.chunk_tasks([make_task(90)], 32) + self.assertEqual([len(c.images) for c in chunks], [32, 32, 26]) + self.assertEqual( + [img for c in chunks for img in c.images], + make_task(90).images) + self.assertEqual({c.source for c in chunks}, {'tf1'}) + + def test_exact_multiple_splits_evenly(self): + chunks = ClamsPromptableApp.chunk_tasks([make_task(64)], 32) + self.assertEqual([len(c.images) for c in chunks], [32, 32]) + + def test_origins_stay_aligned_with_images(self): + chunks = ClamsPromptableApp.chunk_tasks([make_task(7)], 3) + for c in chunks: + self.assertEqual( + [o.replace('-tp', '-img') for o in c.origins], c.images) + + def test_timeframe_order_is_preserved(self): + tasks = [make_task(5, 'tf1'), make_task(1, 'tf2'), + make_task(3, 'tf3')] + chunks = ClamsPromptableApp.chunk_tasks(tasks, 2) + self.assertEqual( + [c.source for c in chunks], + ['tf1', 'tf1', 'tf1', 'tf2', 'tf3', 'tf3']) + + def test_negative_cap_raises(self): + with self.assertRaises(ValueError): + ClamsPromptableApp.chunk_tasks([make_task(5)], -1) + + def test_balanced_mode_evens_out_chunk_sizes(self): + for n_images, expected in [(33, [17, 16]), (90, [30, 30, 30]), + (65, [22, 22, 21]), (64, [32, 32])]: + with self.subTest(n_images=n_images): + chunks = ClamsPromptableApp.chunk_tasks( + [make_task(n_images)], 32, 'balanced') + self.assertEqual( + [len(c.images) for c in chunks], expected) + self.assertEqual( + [img for c in chunks for img in c.images], + make_task(n_images).images) + + def test_unknown_mode_raises(self): + with self.assertRaises(ValueError): + ClamsPromptableApp.chunk_tasks([make_task(5)], 2, 'even') + + +# --------------------------------------------------------------------------- +# generate_batched (parallelPrompts) +# --------------------------------------------------------------------------- + +class TestGenerateBatched(unittest.TestCase): + + def setUp(self): + self.app = make_test_app(make_metadata(call_helper=True)) + self.calls = [] + self.call_kwargs = [] + + def stub_generate(prompt, images=None, audios=None, **kwargs): + self.calls.append(images) + self.call_kwargs.append( + dict(prompt=prompt, images=images, audios=audios, **kwargs)) + if images is None and audios is None: + return ['text-only'] + # echo the first item of each group so output order is visible + groups = images if images is not None else audios + return [group[0] for group in groups] + + self.app.generate = stub_generate + self.progress = [] + self.app.report_progress = ( + lambda done, total: self.progress.append((done, total))) + + def test_slices_by_parallel_prompts_and_keeps_order(self): + images = [[f'img{i}'] for i in range(5)] + outputs = self.app.generate_batched( + ['describe'], images=images, parallel_prompts=2) + self.assertEqual([len(c) for c in self.calls], [2, 2, 1]) + self.assertEqual(outputs, [f'img{i}' for i in range(5)]) + + def test_default_runs_one_prompt_per_call(self): + images = [['a'], ['b'], ['c']] + outputs = self.app.generate_batched(['describe'], images=images) + self.assertEqual([len(c) for c in self.calls], [1, 1, 1]) + self.assertEqual(outputs, ['a', 'b', 'c']) + + def test_arguments_are_forwarded_to_generate(self): + outputs = self.app.generate_batched( + ['q1', 'a1', 'q2'], system_prompt='be brief', + images=[[f'img{i}'] for i in range(3)], + prompt_mode='user-only', parallel_prompts=2, + max_new_tokens=64, use_reasoning=True) + self.assertEqual(len(self.call_kwargs), 2) + for kw in self.call_kwargs: + self.assertEqual(kw['prompt'], ['q1', 'a1', 'q2']) + self.assertEqual(kw['system_prompt'], 'be brief') + self.assertEqual(kw['prompt_mode'], 'user-only') + self.assertEqual(kw['max_new_tokens'], 64) + self.assertIs(kw['use_reasoning'], True) + self.assertNotIn('parallel_prompts', kw) + self.assertEqual(outputs, ['img0', 'img1', 'img2']) + + def test_audio_only_input_is_batched(self): + audios = [[f'aud{i}'] for i in range(5)] + outputs = self.app.generate_batched( + ['transcribe'], audios=audios, parallel_prompts=2) + self.assertEqual( + [kw['audios'] for kw in self.call_kwargs], + [audios[0:2], audios[2:4], audios[4:5]]) + self.assertEqual([kw['images'] for kw in self.call_kwargs], + [None, None, None]) + self.assertEqual(outputs, [f'aud{i}' for i in range(5)]) + + def test_images_and_audios_are_sliced_in_step(self): + images = [[f'img{i}'] for i in range(5)] + audios = [[f'aud{i}'] for i in range(5)] + self.app.generate_batched( + ['describe'], images=images, audios=audios, + parallel_prompts=2) + self.assertEqual( + [kw['images'] for kw in self.call_kwargs], + [images[0:2], images[2:4], images[4:5]]) + self.assertEqual( + [kw['audios'] for kw in self.call_kwargs], + [audios[0:2], audios[2:4], audios[4:5]]) + + def test_text_only_runs_single_call(self): + outputs = self.app.generate_batched( + ['describe'], parallel_prompts=4) + self.assertEqual(self.calls, [None]) + self.assertEqual(outputs, ['text-only']) + + def test_progress_is_reported_before_and_after_each_batch(self): + self.app.generate_batched( + ['describe'], images=[[f'img{i}'] for i in range(5)], + parallel_prompts=2) + self.assertEqual(self.progress, [(0, 5), (2, 5), (4, 5), (5, 5)]) + + def test_text_only_progress_counts_one_prompt(self): + self.app.generate_batched(['describe']) + self.assertEqual(self.progress, [(0, 1), (1, 1)]) + + def test_default_progress_report_logs_at_info(self): + del self.app.report_progress + with self.assertLogs(self.app.logger, level='INFO') as logs: + self.app.generate_batched( + ['describe'], images=[['a'], ['b']], parallel_prompts=2) + self.assertEqual( + [r.getMessage() for r in logs.records], + ['Generated 0/2 prompts', 'Generated 2/2 prompts']) + + def test_parallel_prompts_below_one_raises(self): + with self.assertRaises(ValueError): + self.app.generate_batched( + ['describe'], images=[['a']], parallel_prompts=0) + + def test_mismatched_images_and_audios_raise(self): + with self.assertRaises(ValueError): + self.app.generate_batched( + ['describe'], images=[['a'], ['b']], audios=[['x']], + parallel_prompts=1) + + +# --------------------------------------------------------------------------- +# Chunked grounding (chunk_tasks + generate_batched + response writing) +# --------------------------------------------------------------------------- + +class TestChunkedGrounding(unittest.TestCase): + + def setUp(self): + self.app = make_test_app(make_metadata(call_helper=True)) + self.app.report_progress = lambda done, total: None + self.mmif = Mmif(validate=False) + vdoc = Document() + vdoc.at_type = DocumentTypes.VideoDocument + vdoc.id = 'v1' + vdoc.location = 'file:///video.mp4' + self.mmif.add_document(vdoc) + src_view = self.mmif.new_view() + src_view.metadata.app = 'http://upstream/1' + self.tf = src_view.new_annotation( + AnnotationTypes.TimeFrame, document=vdoc.id, label='scene') + self.tp_ids = [ + src_view.new_annotation( + AnnotationTypes.TimePoint, document=vdoc.id, + timePoint=1000 * (i + 1)).id + for i in range(5)] + self.view = self.mmif.new_view() + self.app.sign_view(self.view, {}) + self.view.new_contain(DocumentTypes.TextDocument) + self.view.new_contain(AnnotationTypes.Alignment) + + def test_chunks_yield_one_grounded_textdocument_each(self): + task = PromptTask( + images=[f'img{i}' for i in range(5)], + origins=list(self.tp_ids), source=self.tf.id) + tasks = ClamsPromptableApp.chunk_tasks([task], 2) + self.app.generate = lambda prompt, images=None, **kw: [ + '+'.join(group) for group in images] + responses = self.app.generate_batched( + ['describe'], images=[t.images for t in tasks], + parallel_prompts=2) + results = [ + self.app.response_to_grounded_textdocument( + self.view, t.source, r, origins=t.origins, + origination='derived') + for t, r in zip(tasks, responses)] + + self.assertEqual(len(results), 3) + tds = [td for td, _ in results] + aligns = [al for _, al in results] + self.assertEqual( + [al.get_property('source') for al in aligns], + [self.tf.id] * 3) + self.assertEqual( + len({al.get_property('target') for al in aligns}), 3) + self.assertEqual( + [al.get_property('target') for al in aligns], + [td.id for td in tds]) + self.assertEqual( + [td.get_property('origins') for td in tds], + [self.tp_ids[0:2], self.tp_ids[2:4], self.tp_ids[4:5]]) + self.assertEqual( + [td.text_value for td in tds], + ['img0+img1', 'img2+img3', 'img4']) + + # --------------------------------------------------------------------------- # Required-prompt validation # ---------------------------------------------------------------------------