diff --git a/.github/workflow-actions/cleanup/action.yml b/.github/workflow-actions/cleanup/action.yml new file mode 100644 index 0000000..3199800 --- /dev/null +++ b/.github/workflow-actions/cleanup/action.yml @@ -0,0 +1,9 @@ +name: 'Cleanup Environment' +description: 'Clean up build artifacts and temporary files' +runs: + using: "composite" + steps: + - name: Clean up + shell: bash + run: | + yes | make clean || true diff --git a/.github/workflow-actions/setup/action.yml b/.github/workflow-actions/setup/action.yml new file mode 100644 index 0000000..6af7167 --- /dev/null +++ b/.github/workflow-actions/setup/action.yml @@ -0,0 +1,26 @@ +name: 'Setup Environment' +description: 'Setup Python 3.13.7 and print system information' +runs: + using: "composite" + steps: + - name: Set up Python 3.13.7 + uses: actions/setup-python@v5 + with: + python-version: '3.13.7' + + - name: Print system information + shell: bash + run: | + echo "::group::System information" + echo "::notice title=Operating system::Linux." + echo "::notice title=Kernel::$(uname -a)." + echo "::notice title=Python::$(python --version 2>&1)." + echo "::endgroup::" + + - name: Setup virtual environment + shell: bash + run: | + echo "::group::ARES environment setup" + pipx install uv + make setup-venv + echo "::endgroup::" diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml new file mode 100644 index 0000000..045a15b --- /dev/null +++ b/.github/workflows/check_format.yml @@ -0,0 +1,29 @@ +name: Check Format Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + check-format: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run ruff for format checking + run: | + make format-check + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/.github/workflows/test_coverage.yml b/.github/workflows/test_coverage.yml new file mode 100644 index 0000000..c7cb4e9 --- /dev/null +++ b/.github/workflows/test_coverage.yml @@ -0,0 +1,29 @@ +name: Test Coverage Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test-coverage: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run coverage tests + run: | + make test-coverage + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/.github/workflows/test_requirements.yml b/.github/workflows/test_requirements.yml new file mode 100644 index 0000000..e7c180a --- /dev/null +++ b/.github/workflows/test_requirements.yml @@ -0,0 +1,29 @@ +name: Test Requirements Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test-requirements: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run requirements tests + run: | + make test-requirements + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/.gitignore b/.gitignore index 3af7067..75d898a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ docs/_build/* *.pyc *.pyo .python-version +version.py # coverage .coverage diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index e69de29..1516a69 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,37 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. diff --git a/Makefile b/Makefile index 83b79c3..6ef6cef 100644 --- a/Makefile +++ b/Makefile @@ -2,32 +2,23 @@ # commands: # - make setup-venv # - make examples -# - make test-examples # - make test-requirements # - make test-coverage # - make format # - make format-check -# - make build-executable -# - make docs -# - make clean -# - make clean-light -# - make release-checklist -# - make release-changelog # - make thirdpartycheck -# - make release-upload -# - make release +# - make clean -VENV_DIR := .venv -VENV_RECREATE := false -VENV_RELEASE := false +# Release metadata is maintained in pyproject.toml and checked against the tag. +RELEASE_TAG := $(shell git describe --tags --exact-match --match 'v[0-9]*' 2>/dev/null) +VERSION := $(shell sed -n 's/^version = "\([^"]*\)"/\1/p' pyproject.toml) +RELEASE_ARCHIVE := dist/ares-$(VERSION)-release.zip # Platform detection for virtual environment binary path ifeq ($(OS),Windows_NT) - VENV_BIN := $(VENV_DIR)/Scripts PLATFORM := windows PATHSEP := ; else - VENV_BIN := $(VENV_DIR)/bin PLATFORM := unix PATHSEP := : endif @@ -35,118 +26,41 @@ endif # Main setup-venv target - uses POSIX shell commands (works with Git Bash on Windows) .PHONY: setup-venv setup-venv: - @if [ -d "$(VENV_DIR)" ]; then \ - echo "Virtual environment '$(VENV_DIR)' already exists. For recreation add 'VENV_RECREATE=true' to cli."; \ - if [ "$(VENV_RECREATE)" = "true" ] || [ "$(VENV_RECREATE)" = "TRUE" ]; then \ - echo "Removing existing virtual environment '$(VENV_DIR)'..."; \ - rm -rf "$(VENV_DIR)"; \ - else \ - exit 0; \ - fi; \ - fi; \ - echo "Creating virtual environment '$(VENV_DIR)'..."; \ - python -m venv "$(VENV_DIR)" || { echo "Error: Failed to create virtual environment."; exit 1; }; \ - echo "Installing project dependencies in virtual environment '$(VENV_DIR)'..."; \ - \ - # Check if .git exists to decide on versioning strategy for editable install \ - if [ ! -d ".git" ]; then \ - echo "NOTE: .git directory not found. Setting pretend version for installation."; \ - if [ "$(VENV_RELEASE)" = "true" ] || [ "$(VENV_RELEASE)" = "TRUE" ]; then \ - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[release]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ - else \ - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[dev]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ - fi; \ - else \ - echo "NOTE: .git directory found. Using setuptools_scm for versioning."; \ - if [ "$(VENV_RELEASE)" = "true" ] || [ "$(VENV_RELEASE)" = "TRUE" ]; then \ - "$(VENV_BIN)/pip" install -e ".[release]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ - else \ - "$(VENV_BIN)/pip" install -e ".[dev]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ - fi; \ - fi - -.PHONY: examples -examples: setup-venv - @echo "TODO: Implementation" - -.PHONY: test-examples -test-examples: setup-venv - @echo "TODO: Implementation" - + echo "Syncing project dependencies:" + uv sync --all-extras + .PHONY: test-requirements test-requirements: setup-venv - @echo "TODO: Implementation" + uv run pytest .PHONY: test-coverage test-coverage: setup-venv - @echo "TODO: Implementation" + NUMBA_DISABLE_JIT=1 uv run pytest --cov --cov-report=html --cov-report=term-missing .PHONY: format format: setup-venv - @"$(VENV_BIN)/ruff" format . + uv run ruff format . .PHONY: format-check format-check: setup-venv - @"$(VENV_BIN)/ruff" format --check . - -.PHONY: build-executable -build-executable: setup-venv - @echo "TODO: Implementation" - -.PHONY: release-checklist -release-checklist: - @echo "TODO: Implementation" - -.PHONY: release-changelog -release-changelog: - @echo "TODO: Implementation" + uv run ruff format --check . .PHONY: thirdpartycheck thirdpartycheck: setup-venv @echo "" @echo "Running third-party dependency analysis..." - @"$(VENV_BIN)/python" scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility + uv run scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility @echo "Third-party dependency check complete." -.PHONY: release-upload -release-upload: - @echo "TODO: Implementation" - -.PHONY: release -release: release-checklist release-changelog thirdpartycheck build-executable release-upload - @echo "TODO: Implementation" - -.PHONY: docs -docs: setup-venv - @echo "TODO: Implementation" - .PHONY: clean clean: - @printf "WARNING: This will permanently delete all generated files, caches, and logs. Continue? [y/n] "; \ - read -r REPLY; \ - if [ "$$REPLY" = "y" ] || [ "$$REPLY" = "Y" ]; then \ - echo "Cleaning project in mode full..."; \ - $(MAKE) clean-light; \ - rm -rf logs; \ - rm -rf build; \ - rm -rf dist; \ - rm -rf examples/output; \ - rm -rf docs/_build; \ - echo "Project cleaned successfully in mode full."; \ - else \ - echo "Clean cancelled."; \ - fi - -.PHONY: clean-light -clean-light: - echo "Cleaning project in mode light..."; \ + echo "Cleaning project..."; \ find . -type f -name "*.pyc" -delete; \ find . -type d -name "__pycache__" -exec rm -rf {} +; \ find . -type d -name "log" -exec rm -rf {} +; \ find . -type d -name ".pytest_cache" -exec rm -rf {} +; \ find . -type d -name ".ruff_cache" -exec rm -rf {} +; \ - find . -type f -name "*.spec" -delete; \ rm -f .coverage .coverage.*; \ rm -rf htmlcov; \ - $(MAKE) -C examples/sim_unit clean; \ + rm -rf logs; \ echo "Project cleaned successfully in mode light."; \ diff --git a/NOTICE b/NOTICE index e69de29..d03c84c 100644 --- a/NOTICE +++ b/NOTICE @@ -0,0 +1,84 @@ +MATI (*.mat file interface) +Copyright 2025 olympus-tools contributors. + +This file is automatically generated. +For complete license texts, please visit the respective project URLs. + +============================================================================== + +This product contains dependencies with the following licenses: + +Apache Software License License: +------------------------------------------------------------------------------ + * mati 0.1.dev7+g08c181fbb.d19800101 + URL: UNKNOWN + + +Apache-2.0 License: +------------------------------------------------------------------------------ + * coverage 7.13.5 + URL: https://github.com/coveragepy/coveragepy + + +Apache-2.0 OR BSD-2-Clause License: +------------------------------------------------------------------------------ + * packaging 26.0 + URL: https://github.com/pypa/packaging + + +BSD License License: +------------------------------------------------------------------------------ + * Pygments 2.19.2 + URL: https://pygments.org + + * scipy 1.17.1 + URL: https://scipy.org/ + + +BSD-3-Clause License: +------------------------------------------------------------------------------ + * h5py 3.16.0 + URL: https://www.h5py.org/ + + +BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0 License: +------------------------------------------------------------------------------ + * numpy 2.4.3 + URL: https://numpy.org + + +MIT License: +------------------------------------------------------------------------------ + * iniconfig 2.3.0 + URL: https://github.com/pytest-dev/iniconfig + + * pipdeptree 2.34.0 + URL: https://github.com/tox-dev/pipdeptree + + * ruff 0.15.8 + URL: https://docs.astral.sh/ruff + + * typeguard 4.5.1 + URL: UNKNOWN + + +MIT License License: +------------------------------------------------------------------------------ + * debugpy 1.8.20 + URL: https://aka.ms/debugpy + + * pluggy 1.6.0 + URL: UNKNOWN + + * pytest 8.4.1 + URL: https://docs.pytest.org/en/latest/ + + * pytest-cov 6.0.0 + URL: https://github.com/pytest-dev/pytest-cov + + +PSF-2.0 License: +------------------------------------------------------------------------------ + * typing_extensions 4.15.0 + URL: https://github.com/python/typing_extensions + diff --git a/README.md b/README.md index a8169d1..f8deb23 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,132 @@ -# template -THIS IS THE TEMPLATE DESCRIPTION +# MATI + +This package provides a standalone interface for reading and writing MATLAB MAT-files. It supports reading version v7.3 (HDF5) via `h5py` and older versions (v7.0 and lower) via `scipy.io.loadmat`. The `get` method intelligently determines the MAT-file version and uses the appropriate backend for reading. +The default `write` function specifically targets v7.3 files. +It is designed for easy usage with minimal dependencies. + +## Usage + +To use this interface, you can import the `MatInterface` class and call its static methods. + +```python +from pathlib import Path +from mat_interface.mat_interface import MatInterface +import numpy as np + +# Example: Writing a MAT file +signals_to_write = [ + { + "label": "signal1", + "timestamps": np.array([0.0, 1.0, 2.0]), + "value": np.array([10, 20, 30]), + }, + { + "label": "signal2", + "timestamps": np.array([0.0, 1.0, 2.0]), + "value": np.array([[1, 2], [3, 4], [5, 6]]), + }, +] +output_file = Path("output.mat") +MatInterface.write(output_path=output_file, signals=signals_to_write) + +# Example: Reading a MAT file +signals_read = MatInterface.get(file_path=output_file) +print(signals_read) + +# Example: Reading specific signals from a struct +signals_from_struct = MatInterface.get(file_path=output_file, struct_list=["my_struct"]) + +# Example: Reading signals with a label filter +signal_1_data = MatInterface.get(file_path=output_file, label_filter=["signal1"]) +``` + +## Supported Data Layouts + +The `MatInterface` supports two primary layouts when reading MAT files. + +### 1. Timeseries Struct Layout (Default for `write`) +Each signal is encapsulated in its own group containing its specific time vector and data. + +```text +/ (root or struct_group) +├── signal_1/ +│ ├── Time (dataset: 1-D timestamps) +│ ├── Data (dataset: N-D values) +│ └── Events (group: empty, necessary metadata for MATLAB) +└── signal_2/ + ├── Time (dataset: 1-D timestamps) + ├── Data (dataset: N-D values) + └── Events (group: empty, necessary metadata for MATLAB) +``` + +### 2. Flat Signal Arrays +Signals are stored as datasets at the same level, sharing a common timestamp vector. + +```text +/ (root or struct_group) +├── signal_1 (dataset: N-D values) +├── signal_2 (dataset: N-D values) +└── timestamps (dataset: 1-D shared timestamps, can also be named 'time') +``` + +### 3. Nested Struct Layout (via `struct_list`) +The interface can target signals nested within MATLAB structs (HDF5 groups). Within a single targeted struct, all signals must follow the **same** layout; combining layouts within one struct is not supported. + +#### Layout: Struct with Flat Signals +All datasets within the struct share a common timestamp vector. + +```text +/ (root) +└── my_struct_A/ <-- Targeted via struct_list=['my_struct_A'] + ├── signal_1 (dataset: N-D values) + ├── signal_2 (dataset: N-D values) + └── timestamps (dataset: 1-D shared timestamps) +``` + +#### Layout: Struct with Timeseries Groups +Each signal within the struct is its own group containing its specific time vector. + +```text +/ (root) +└── my_struct_B/ <-- Targeted via struct_list=['my_struct_B'] + ├── signal_1/ + │ ├── Time + │ └── Data + └── signal_2/ + ├── Time + └── Data +``` + +## API Documentation + +```python +MatInterface.write(output_path: Path, signals: list[dict[str, str | np.typing.NDArray[np.generic]]]) -> None +``` + +* **Description:** Writes a list of signals to a MAT 7.3 file (HDF5). Each signal is stored as a top-level HDF5 group containing 'Time', 'Data', and 'Events' members ('Events' can be ignored, just for MATLAB compatibility). +Additional the group carries `MATLAB_class = "timeseries"` and `MATLAB_fields` attributes for MATLAB compatibility. +* **Arguments:** + * `output_path` (Path): The path to the output MAT file. + * `signals` (list[dict]): A list of dictionaries, where each dictionary represents a signal and must contain: + * `'label'` (str): The name of the signal. + * `'timestamps'` (np.ndarray): A 1-D NumPy array of timestamps (float). + * `'value'` (np.ndarray): A 1-D, 2-D, or 3-D NumPy array representing the signal's values. The dtype will be mapped to a corresponding MATLAB numerical type. +* **Note:** This method adds MATLAB-specific headers and attributes for compatibility. MATLAB's `load()` will recognize the structure, but true MATLAB `timeseries` objects require MATLAB's MCOS serialization, which cannot be produced from Python. + +```python +MatInterface.get(file_path: Path, label_filter: list[str] | None = None, struct_list: list[str] | None = None) -> list[dict[str, str | np.typing.NDArray[np.generic]]] +``` + +* **Description:** Reads signals from a MAT 7.3 or lower file. It intelligently determines the MAT-file version and uses either `h5py` for v7.3 (HDF5) files or `scipy.io.loadmat` for older versions. It supports 2 different data layouts: + * **Timeseries struct layout:** Each signal is a group containing 'Time' and 'Data' sub-datasets. + * **Flat signal arrays:** Signals are top-level datasets, with timestamps assumed to be in a separate dataset named 'timestamps' or 'time'. + It can optionally filter signals by `label_filter` or by reading from specific `struct_list` groups within the MAT file. +* **Arguments:** + * `file_path` (Path): The path to the MAT file to read. + * `label_filter` (list[str] | None, optional): A list of signal names (labels) to filter and read. If `None`, all readable signals are returned. + * `struct_list` (list[str] | None, optional): A list of struct group names within the MAT file from which to extract signals. If `None`, signals are read from the root level of the MAT file. +* **Returns:** A list of dictionaries, where each dictionary represents a signal and contains: + * `'label'` (str): The name of the signal. + * `'timestamps'` (np.ndarray | None): A 1-D NumPy array of timestamps, or `None` if not found. + * `'value'` (np.ndarray): A NumPy array containing the signal's values. +* **Note:** This method does not support MATLAB MCOS `timeseries` objects due to their opaque serialization. It also skips internal HDF5 groups like `#refs#` and `#subsystem#`. diff --git a/architecture.md b/architecture.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/examples/mat_v7_flat_signals.mat b/examples/mat_v7_flat_signals.mat new file mode 100644 index 0000000..6db7c43 Binary files /dev/null and b/examples/mat_v7_flat_signals.mat differ diff --git a/examples/mat_v7_flat_timeseries.mat b/examples/mat_v7_flat_timeseries.mat new file mode 100644 index 0000000..92de7db Binary files /dev/null and b/examples/mat_v7_flat_timeseries.mat differ diff --git a/TEMPLATE/README.md b/mati/__init__.py similarity index 100% rename from TEMPLATE/README.md rename to mati/__init__.py diff --git a/mati/dummy.py b/mati/dummy.py deleted file mode 100644 index e69de29..0000000 diff --git a/mati/mati.py b/mati/mati.py new file mode 100644 index 0000000..f21c79f --- /dev/null +++ b/mati/mati.py @@ -0,0 +1,553 @@ +r""" +________________________________________________________________________ +| | +| $$\ $$\ $$$$$$\ $$$$$$$$\ $$$$$$\ | +| $$$\ $$$ |$$ __$$\\__$$ __|\_$$ _| | +| $$$$\ $$$$ |$$ / $$ | $$ | $$ | | +| $$\$$\$$ $$ |$$$$$$$$ | $$ | $$ | | +| $$ \$$$ $$ |$$ __$$ | $$ | $$ | | +| $$ |\$ /$$ |$$ | $$ | $$ | $$ | | +| $$ | \_/ $$ |$$ | $$ | $$ | $$$$$$\ | +| \__| \__|\__| \__| \__| \______| | +| | +| MATI (*.mat file interface) (c) | +|______________________________________________________________________| + +Copyright 2025 olympus-tools contributors. Dependencies and licenses +are listed in the NOTICE file: + + https://github.com/olympus-tools/MATI/blob/master/NOTICE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License: + + https://github.com/olympus-tools/MATI/blob/master/LICENSE +""" + +import datetime +import struct +import warnings +from pathlib import Path +from typing import Any, Iterable, Literal, TypedDict + +import h5py +import numpy as np +import scipy.io as sio + +DEFAULT_TIMESTAMP = "timestamps" # default name of "timestamps" for flat signals + + +class MatSignal(TypedDict): + label: str + timestamps: np.ndarray + value: np.ndarray + + +class MatInterface: + """A standalone interface for reading and writing MATLAB MAT-files supporting + - v7.3 files (HDF5 based) + - v7.0/6.0 files (via scipy) + + The MatInterface is based on python standard interfaces to guarantee easy usage and minimum dependencies. + + additional information see: + https://de.mathworks.com/help/matlab/import_export/mat-file-versions.html?s_tid=srchtitle_site_search_1_mat+files + https://www.hdfgroup.org/solutions/hdf5/ + https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.loadmat.html + """ + + @staticmethod + def write( + output_path: Path, + signals: list[MatSignal], + format: Literal["v7.3", "v7"] = "v7.3", + **kwargs, + ) -> None: + """Write signals to a MAT 7.3 (HDF5) or lower. + + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + if format == "v7.3": + MatInterface._write73(output_path=output_path, signals=signals, **kwargs) + elif format == "v7": + MatInterface._write7(output_path=output_path, signals=signals, **kwargs) + + @staticmethod + def get( + file_path: Path, + label_filter: list[str] | None = None, + struct_list: list[str] | None = None, + **kwargs, + ) -> list[MatSignal]: + """Read signals from a MAT 7.3 (HDF5) or lower. + + Supports two modes (defined via struct_list): + - flat : signals exist at the highest layer + - struct : signals are divided into structs each with providing its own timestamp + + where each mode can read two types of signals: + - **timeseries struct layout**: each signal is a struct containing "Time","Data" + - **flat signal arrays**: each signal is a array whereas one array represents the shared timestamp + + Note: MATLAB MCOS ``timeseries`` *objects* (``MATLAB_object_decode==3``) + store their data in an opaque ``#refs#`` pool and cannot be decoded + without a full MATLAB OOP deserialiser. + Therefore not the 'real' MATLAB timeseries supported. + + Assumptions: + Considering the ''flat'' signal structure the assumption that the shared timevector is called + - timestamps + is made. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + matfile_version = sio.matlab.matfile_version(file_path) + + if matfile_version[0] == 2: + signal_data = MatInterface._get73( + file_path=file_path, + label_filter=label_filter, + struct_list=struct_list, + **kwargs, + ) + elif matfile_version[0] == 1: + signal_data = MatInterface._get7( + file_path=file_path, + label_filter=label_filter, + struct_list=struct_list, + **kwargs, + ) + else: + raise ValueError(f"Unsupported/Unknown matfile-version {matfile_version}.") + return signal_data + + # MAT 7.0 functions (scipy wrapper) + @staticmethod + def _write7(output_path: Path, signals: list[MatSignal]) -> None: + """Write signals to a MAT 7.0 file via scipy. + + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + # defaults for saving with scipy.sio + kw = dict( + oned_as="column", + long_field_names=True, + do_compression=True, + ) + + mat_data_dict = {} + for signal in signals: + mat_data_dict[signal["label"]] = { + "Time": signal["timestamps"], + "Data": signal["value"], + "Events": np.array([]), # Empty array for MATLAB compatibility + } + sio.savemat(output_path, mat_data_dict, **kw) + + @staticmethod + def _get7( + file_path: Path, + label_filter: list[str] | None = None, + struct_list: list[str] | None = None, + **kwargs, + ) -> list[MatSignal]: + """Read signals from a MAT 7.0 or lower file via scipy.io package. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + **kwargs : additional arguments passed to scipy "loadmat" + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + _OBSOLETE_MATFIELDS = [ + "__header__", + "__version__", + "__globals__", + ] # define obsolete mat fields thar are always skipped during reading and define default timestamp + _TIMESTAMP = DEFAULT_TIMESTAMP + mat_data: list[MatSignal] = [] + + # helper functions for scipy (transform to standard format) + def _get_timeseries( + tmp_data: dict[str, Any], label_filter: list[str] + ) -> list[MatSignal]: + """Read timeseries struct from mat file.""" + nonlocal mat_data + signals_to_add: Iterable[MatSignal] = [ + { + "label": signal_name, + "timestamps": signal_dict.get("Time"), + "value": signal_dict.get("Data"), + } + for signal_name, signal_dict in tmp_data.items() + if signal_name not in _OBSOLETE_MATFIELDS # skip mat-fields + and (not label_filter or signal_name in label_filter) # label-filter + and type(signal_dict) is dict # check 'timeseries'-struct + and "Time" in signal_dict + and "Data" in signal_dict + ] + mat_data.extend(signals_to_add) + return mat_data + + def _get_signals( + tmp_data: dict[str, Any], + timestamps: np.ndarray, + label_filter: list[str], + ) -> list[MatSignal]: + """Read flat signals from mat file.""" + nonlocal mat_data + signals_to_add: Iterable[MatSignal] = [ + { + "label": signal_name, + "timestamps": timestamps, + "value": signal_value, + } + for signal_name, signal_value in tmp_data.items() + if signal_name not in _OBSOLETE_MATFIELDS # skip mat-fields + and signal_name != _TIMESTAMP # skip timestamp itself + and (not label_filter or signal_name in label_filter) # label-filter + and isinstance(signal_value, np.ndarray) # Ensure it's a numpy array + ] + mat_data.extend(signals_to_add) + return mat_data + + # define standard flags to guarantee readability + kwargs = {**kwargs} + kwargs.setdefault("simplify_cells", True) + tmp_data = sio.loadmat(file_path, **kwargs) + + # differentiate between struct-mode, plain-mode, timeseries + if struct_list: + tmp_data.update([tmp_data.get(struct) for struct in struct_list]) + + # check for timessignal -> timeseries or plain-mode + timestamps = tmp_data.get(_TIMESTAMP, None) + if timestamps is None: + mat_data = _get_timeseries(tmp_data, label_filter if label_filter else []) + else: + mat_data = _get_signals( + tmp_data, timestamps, label_filter if label_filter else [] + ) + return mat_data + + # MAT 7.3 functions + @staticmethod + def _write73( + output_path: Path, + signals: list[MatSignal], + ) -> None: + """Write signals to a MAT 7.3 file (HDF5). + Each signal is stored as a top-level HDF5 group containing three members: + - ``Time`` – 1-D dataset, timestamps (double) + - ``Data`` – 1-D or N-D dataset, signal values + - ``Events`` – empty group (reserved for MATLAB compatibility) + + The group carries ``MATLAB_class = "timeseries"`` and ``MATLAB_fields`` + attributes so that MATLAB's ``load()`` recognises the layout. + Note: + MATLAB's ``load()`` will return each signal as a **struct** with fields + ``Time``, ``Data``, and ``Events``. + The struct represents the base of a MATLAB ``timeseries``. + A true ``timeseries`` object requires MATLAB's MCOS object serialisation, + which cannot be produced from Python. To convert after loading in MATLAB:: + 'ts = timeseries(sig.Data, sig.Time, 'Name', 'my_signal');' + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + with h5py.File(output_path, "w", userblock_size=512) as h5file: # type: ignore[reportGeneralTypeIssues] + # HDF5 attribute encoding for MATLAB compatibility (hdf5 tweaked to match *.mat expectations) + _ds_class_tid = h5py.h5t.py_create(np.dtype("S8")) + _ds_class_tid.set_strpad(h5py.h5t.STR_NULLTERM) + _ds_class_sid = h5py.h5s.create(h5py.h5s.SCALAR) + _vlen_s1_dt = h5py.vlen_dtype(np.dtype("S1")) # pyright: ignore[reportAttributeAccessIssue] + + # helper functions for writing + def _write_ds_matlab_class(dataset_id, class_bytes: bytes) -> None: + """Add matlab-class attributes to represent datatype.""" + padded_bytes = class_bytes.ljust(8, b"\x00")[:8] + attribute = h5py.h5a.create( # type: ignore + dataset_id, b"MATLAB_class", _ds_class_tid, _ds_class_sid + ) + attribute.write(np.frombuffer(padded_bytes, dtype="S8")) + + def _mat_fields(*names: str) -> np.ndarray: + """Helper function for '_write_mat_fields'. Write names as ascii array.""" + array = np.empty(len(names), dtype=object) + for i, name in enumerate(names): + array[i] = np.array([c.encode("ascii") for c in name], dtype="S1") + return array + + def _write_mat_fields(group, *names: str) -> None: + """Initialize timeseries-struct with given names.""" + group.attrs.create( + "MATLAB_fields", + data=_mat_fields(*names), + dtype=_vlen_s1_dt, + ) + + # write HDF5 content + for signal in signals: + label: str = signal["label"] + timestamps: np.typing.NDArray[np.generic] = signal["timestamps"] + value: np.typing.NDArray[np.generic] = signal["value"] + dtype = value.dtype + + # initialize matlab group (add important metadata) + group = h5file.create_group(label) + group.attrs["MATLAB_class"] = np.bytes_(b"timeseries") + _write_mat_fields(group, "Time", "Data", "Events") + event_group = group.create_group("Events") + event_group.attrs["MATLAB_class"] = np.bytes_(b"struct") + _write_mat_fields(event_group) + + # write timestamp + timestamps_single = timestamps.astype(np.float32) + ti = group.create_dataset("Time", data=timestamps_single.T) + _write_ds_matlab_class(ti.id, b"single") + + # bool/logical: MATLAB requires uint8 storage + if np.issubdtype(dtype, np.bool_): + value = value.astype(np.uint8) + # write signal data + if value.ndim <= 2: + ds = group.create_dataset("Data", data=value.T) + elif value.ndim == 3: + ds = group.create_dataset("Data", data=value.transpose(2, 1, 0)) + else: + raise ValueError( + "MatInterface does not support more than 3 dimensions." + ) + _write_ds_matlab_class(ds.id, MatInterface._get_matlab_class(dtype)) + + # bool/logical: MATLAB requires MATLAB_int_decode=1 + if np.issubdtype(dtype, np.bool_): + ds.attrs["MATLAB_int_decode"] = np.uint8(1) + + # add MATLAB specific 128-byte header + date_str = datetime.datetime.now().strftime("%a %b %d %H:%M:%S %Y") + # byte: 0-115 | text | human readable + header = ( + f"MATLAB 7.3 MAT-file, Platform: Python/ARES, Created on: {date_str}".ljust( + 116 + ) + ) + subsystem_offset = ( + b"\x00" * 8 + ) # byte: 116- 123 | subsystem specific data | all 0 | + version = 0x0200 # byte: 124 - 125 | version | 0x0200 or 0x0100 | + endian = b"IM" # byte: 126 - 127 | Endian Indicator | MI BitEndian, IM LittleEndian (Intel/AMD = IM) + signature = ( + header.encode("ascii") + + subsystem_offset + + struct.pack(" list[MatSignal]: + """Read signals from a MAT 7.3 file. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + _HDF5_INTERNAL = {"#refs#", "#subsystem#"} + _NUMERIC_KINDS = set("bifcu") + _TIMESERIES_KEYS = { + "Data", + "Time", + } # Events can be ignored, only important for writing + _TIMESTAMP_NAMES = {DEFAULT_TIMESTAMP} + + # helper functions for reading + def _is_timeseries_group(obj) -> bool: + """Droup = plain-struct Time/Data groups.""" + if not hasattr(obj, "keys"): + return False + if obj.attrs.get("MATLAB_object_decode") is not None: + # MCOS object – cannot decode without MATLAB OOP deserialiser + return False + return _TIMESERIES_KEYS.issubset(set(obj.keys())) + + def _is_numeric_dataset(obj) -> bool: + """Dataset = numeric""" + return ( + isinstance(obj, h5py.Dataset) # type: ignore + and obj.attrs.get("MATLAB_object_decode") is None + and obj.dtype.kind in _NUMERIC_KINDS + ) + + def _read_from_group(group) -> list[dict]: + """Decode all signals from an HDF5 group (root or struct sub-group). + MCOS objects, char arrays, non-timeseries structs ares skipped silently. + """ + result = [] + + candidate_keys = [k for k in group.keys() if k not in _HDF5_INTERNAL] + if label_filter is not None: + candidate_keys = [k for k in candidate_keys if k in label_filter] + + # resolve the shared flat timestamp vector + _ts: np.ndarray | None = None + _ts_resolved = False + + def _get_flat_timestamps() -> np.ndarray | None: + nonlocal _ts, _ts_resolved + if _ts_resolved: + return _ts + _ts_resolved = True + for k in group.keys(): + if k.lower() in _TIMESTAMP_NAMES and isinstance( + group[k], + h5py.Dataset, # type: ignore + ): + _ts = MatInterface._mat_load_num(group[k]) + break + return _ts + + for key in candidate_keys: + element = group[key] + + if _is_timeseries_group(element): + # Layout A / C – group with Time + Data sub-datasets + result.append( + { + "label": key, + "timestamps": MatInterface._mat_load_num(element["Time"]), + "value": MatInterface._mat_load_num(element["Data"]), + } + ) + elif _is_numeric_dataset(element): + # Layout B / D – flat numeric dataset; skip timestamp key itself + if key.lower() in _TIMESTAMP_NAMES: + continue + result.append( + { + "label": key, + "timestamps": _get_flat_timestamps(), + "value": MatInterface._mat_load_num(element), + } + ) + + return result + + # read HDF5 (mat-file) + with h5py.File(file_path, mode="r") as matfile: # type: ignore + signals = [] + if struct_list: + for struct in struct_list: + if struct not in matfile: + warnings.warn( + f"Struct group '{struct}' not found in {file_path}. ", + RuntimeWarning, + ) + continue + signals.extend(_read_from_group(matfile[struct])) + else: + signals.extend(_read_from_group(matfile)) + return signals + + # static, general helper functions + @staticmethod + def _get_matlab_class(dtype: np.dtype) -> bytes: + """Maps a NumPy dtype to the corresponding MATLAB class string.""" + if np.issubdtype(dtype, np.float64): + return b"double" + elif np.issubdtype(dtype, np.float32): + return b"single" + elif np.issubdtype(dtype, np.int8): + return b"int8" + elif np.issubdtype(dtype, np.uint8): + return b"uint8" + elif np.issubdtype(dtype, np.int16): + return b"int16" + elif np.issubdtype(dtype, np.uint16): + return b"uint16" + elif np.issubdtype(dtype, np.int32): + return b"int32" + elif np.issubdtype(dtype, np.uint32): + return b"uint32" + elif np.issubdtype(dtype, np.int64): + return b"int64" + elif np.issubdtype(dtype, np.uint64): + return b"uint64" + elif np.issubdtype(dtype, np.bool_): + return b"logical" + else: + return b"double" + + @staticmethod + def _mat_load_num( + dataset: h5py.Dataset, # type: ignore + ) -> np.ndarray: + """Load numerical data from h5py dataset and adjust dimensions.""" + samples = np.squeeze(dataset[()], axis=None) + if len(samples.shape) > 1: + samples = samples.T + return samples + + # INFO: the following functions are currently not used, useful to read more complex data + @staticmethod + def _mat_load_string(file: h5py.File, dataset: h5py.Dataset) -> np.ndarray: # type: ignore + """Load string data from h5py dataset.""" + n_el = dataset.shape[1] + el_string = [] + for ii in range(n_el): + el_string.append(bytes(np.array(file[dataset[0, ii]]))[::2].decode()) + return np.asarray(el_string) + + @staticmethod + def _mat_load_numstring(dataset: h5py.Dataset) -> np.ndarray | str: # type: ignore + """Load numeric string data from h5py dataset.""" + try: + samples = bytes(dataset[()])[::2].decode() + except Exception: + samples = np.squeeze(dataset[()], axis=None) + if len(samples.shape) > 1: + samples = samples.T + return samples diff --git a/pyproject.toml b/pyproject.toml index a08aff1..e979826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + [project] name = "mati" description = "*.mat file interface" @@ -33,9 +37,10 @@ classifiers = [ # Environment "Environment :: Console", ] - dependencies = [ - "typeguard==4.4.4", + "h5py>=3.16.0", + "scipy>=1.17.1", + "typeguard>=4.4.4", ] dynamic = ["version"] @@ -45,40 +50,28 @@ test = [ "pytest-cov==6.0.0", ] dev = [ - "ruff==0.12.11", - "gitpython==3.1.45", - "pip-licenses==5.0.0", - "pipdeptree==2.23.4", + "ruff>=0.15.8", + "pip-licenses>=5.5.1", + "pipdeptree>=2.34.0", "mati[test]", ] -docs = [] -release = [ - "mati[dev]", - "mati[docs]", -] -[project.urls] -"Homepage" = "https://github.com/olympus-tools/MATI" -"Documentation" = "https://github.com/olympus-tools/MATI#readme" -"Bug Tracker" = "https://github.com/olympus-tools/MATI/issues" -"Repository" = "https://github.com/olympus-tools/MATI" -"Changelog" = "https://github.com/olympus-tools/MATI/releases" -"Source" = "https://github.com/olympus-tools/MATI" -"Security Policy" = "https://github.com/olympus-tools/MATI/security/policy" -"Contributing" = "https://github.com/olympus-tools/MATI/blob/master/CONTRIBUTING.md" +debug = [ + "debugpy>=1.8.20", +] -[tool.setuptools.packages.find] -include = ["mati*"] +[tool.hatch.build.targets.wheel] +packages = ["mati"] -[tool.setuptools.package-data] -mati = ["py.typed"] +[tool.hatch.version] +source = "vcs" -[tool.setuptools_scm] -write_to = "mati/core/version.py" +[tool.hatch.build.hooks.vcs] +version-file = "mati/version.py" [tool.pytest.ini_options] +pythonpath = ["."] testpaths = ["test"] -python_files = ["test_*.py"] addopts = [ "--ignore=test/examples", ] @@ -87,12 +80,9 @@ addopts = [ # The maximum line length. line-length = 88 # The target Python version. -target-version = "py312" +target-version = "py313" # Excluded files and directories. exclude = [ - "ares/core/version.py", - "packages", - "submodules", ".venv", ] diff --git a/test/dummy.py b/test/dummy.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/test_mati.py b/test/test_mati.py new file mode 100644 index 0000000..4f90eae --- /dev/null +++ b/test/test_mati.py @@ -0,0 +1,242 @@ +r""" +________________________________________________________________________ +| | +| $$\ $$\ $$$$$$\ $$$$$$$$\ $$$$$$\ | +| $$$\ $$$ |$$ __$$\\__$$ __|\_$$ _| | +| $$$$\ $$$$ |$$ / $$ | $$ | $$ | | +| $$\$$\$$ $$ |$$$$$$$$ | $$ | $$ | | +| $$ \$$$ $$ |$$ __$$ | $$ | $$ | | +| $$ |\$ /$$ |$$ | $$ | $$ | $$ | | +| $$ | \_/ $$ |$$ | $$ | $$ | $$$$$$\ | +| \__| \__|\__| \__| \__| \______| | +| | +| MATI (*.mat file interface) (c) | +|______________________________________________________________________| + +Copyright 2025 olympus-tools contributors. Dependencies and licenses +are listed in the NOTICE file: + + https://github.com/olympus-tools/MATI/blob/master/NOTICE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License: + + https://github.com/olympus-tools/MATI/blob/master/LICENSE +""" + +import struct +from pathlib import Path + +import numpy as np +import pytest + +from mati.mati import MatInterface + + +def test_get_matlab_class(): + """Test mapping of numpy dtypes to MATLAB classes.""" + assert MatInterface._get_matlab_class(np.dtype(np.float64)) == b"double" + assert MatInterface._get_matlab_class(np.dtype(np.float32)) == b"single" + assert MatInterface._get_matlab_class(np.dtype(np.int32)) == b"int32" + assert MatInterface._get_matlab_class(np.dtype(np.uint8)) == b"uint8" + assert MatInterface._get_matlab_class(np.dtype(np.bool_)) == b"logical" + assert MatInterface._get_matlab_class(np.dtype(np.complex128)) == b"double" + + +def test_write_header(tmp_path): + """Verify the 128-byte MATLAB 7.3 header is correctly written.""" + test_file = tmp_path / "test_header.mat" + signals = [ + { + "label": "sig", + "timestamps": np.array([0.0, 1.0]), + "value": np.array([1.0, 2.0]), + } + ] + MatInterface._write73(test_file, signals) + + with open(test_file, "rb") as f: + header = f.read(128) + + assert len(header) == 128 + assert header.startswith(b"MATLAB 7.3 MAT-file") + assert header[126:128] == b"IM" # Little-endian indicator + version = struct.unpack("3 dimensions.""" + test_file = tmp_path / "test_4d.mat" + signals = [ + { + "label": "sig_4d", + "timestamps": np.array([0]), + "value": np.zeros((1, 1, 1, 1)), + } + ] + with pytest.raises(ValueError, match="does not support more than 3 dimensions"): + MatInterface._write73(test_file, signals) + + +def test_round_trip_v7(tmp_path): + """Test writing and reading back 1D signals in v7 format.""" + test_file = tmp_path / "test_v7.mat" + signals = [ + { + "label": "sig_float", + "timestamps": np.linspace(0, 5, 10), + "value": np.cos(np.linspace(0, 5, 10)), + }, + { + "label": "sig_bool", + "timestamps": np.linspace(0, 1, 2), + "value": np.array([True, False], dtype=bool), + }, + ] + + MatInterface.write(test_file, signals, format="v7") + read_signals = MatInterface.get(test_file) + assert len(read_signals) == 2 + + # Check sig_float + sig_float = next(s for s in read_signals if s["label"] == "sig_float") + np.testing.assert_array_almost_equal( + sig_float["timestamps"], signals[0]["timestamps"] + ) + np.testing.assert_array_almost_equal(sig_float["value"], signals[0]["value"]) + + # Check sig_bool + sig_bool = next(s for s in read_signals if s["label"] == "sig_bool") + np.testing.assert_array_almost_equal( + sig_bool["timestamps"], signals[1]["timestamps"] + ) + np.testing.assert_array_equal(sig_bool["value"], signals[1]["value"]) + + +def test_read_v7_flat_signals_example(): + """Test reading flat signals from an existing v7 MAT file.""" + file_path = Path("examples/mat_v7_flat_signals.mat") + # Use label_filter to get specific signals we know exist + signals = MatInterface.get(file_path, label_filter=["sig_a_f32", "sig_b_f32"]) + + assert len(signals) == 2 + + sig_a = next(s for s in signals if s["label"] == "sig_a_f32") + sig_b = next(s for s in signals if s["label"] == "sig_b_f32") + + assert sig_a["value"].shape == (11,) + assert sig_a["value"].dtype == np.float32 + assert sig_a["timestamps"].shape == (11,) + + assert sig_b["value"].shape == (11, 3) + assert sig_b["value"].dtype == np.float32 + assert sig_b["timestamps"].shape == (11,) + + +def test_read_v7_timeseries_example(): + """Test reading timeseries structs from an existing v7 MAT file.""" + file_path = Path("examples/mat_v7_flat_timeseries.mat") + # Use label_filter to get specific signals we know exist + signals = MatInterface.get(file_path, label_filter=["ts_sig_a_f32", "ts_sig_b_f32"]) + + assert len(signals) == 2 + + sig_a = next(s for s in signals if s["label"] == "ts_sig_a_f32") + sig_b = next(s for s in signals if s["label"] == "ts_sig_b_f32") + + assert sig_a["label"] == "ts_sig_a_f32" + assert sig_a["timestamps"].shape == (11,) + assert sig_a["value"].shape == (11,) + assert sig_a["value"].dtype == np.float32 + + assert sig_b["label"] == "ts_sig_b_f32" + assert sig_b["timestamps"].shape == (11,) + assert sig_b["value"].shape == (11, 3) + assert sig_b["value"].dtype == np.float32