From b8d4f42cf6311670efd6e5fa8675f57f55221e72 Mon Sep 17 00:00:00 2001 From: jonas Date: Sat, 28 Mar 2026 09:28:05 +0100 Subject: [PATCH 1/7] jonas: moving from ARES --- CODE_OF_CONDUCT.md | 37 ++ Makefile | 64 +-- NOTICE | 72 ++++ README.md | 130 +++++- architecture.md | 0 docs/README.md | 0 examples/README.md | 0 examples/mat_v7_flat_signals.mat | Bin 0 -> 2320 bytes examples/mat_v7_flat_timeseries.mat | Bin 0 -> 4016 bytes TEMPLATE/README.md => mati/__init__.py | 0 mati/dummy.py | 0 mati/mati.py | 553 +++++++++++++++++++++++++ pyproject.toml | 56 +-- test/dummy.py | 0 test/test_mati.py | 242 +++++++++++ 15 files changed, 1053 insertions(+), 101 deletions(-) delete mode 100644 architecture.md delete mode 100644 docs/README.md delete mode 100644 examples/README.md create mode 100644 examples/mat_v7_flat_signals.mat create mode 100644 examples/mat_v7_flat_timeseries.mat rename TEMPLATE/README.md => mati/__init__.py (100%) delete mode 100644 mati/dummy.py create mode 100644 mati/mati.py delete mode 100644 test/dummy.py create mode 100644 test/test_mati.py diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index e69de29..1516a69 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,37 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. diff --git a/Makefile b/Makefile index 83b79c3..9060c72 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,12 @@ # commands: # - make setup-venv # - make examples -# - make test-examples # - make test-requirements # - make test-coverage # - make format # - make format-check -# - make build-executable -# - make docs -# - make clean -# - make clean-light -# - make release-checklist -# - make release-changelog # - make thirdpartycheck -# - make release-upload -# - make release +# - make clean VENV_DIR := .venv VENV_RECREATE := false @@ -69,17 +61,13 @@ setup-venv: examples: setup-venv @echo "TODO: Implementation" -.PHONY: test-examples -test-examples: setup-venv - @echo "TODO: Implementation" - .PHONY: test-requirements test-requirements: setup-venv - @echo "TODO: Implementation" + @"$(VENV_BIN)/pytest" .PHONY: test-coverage test-coverage: setup-venv - @echo "TODO: Implementation" + @"$(VENV_BIN)/pytest" --cov --cov-report=html --cov-report=term-missing .PHONY: format format: setup-venv @@ -89,18 +77,6 @@ format: setup-venv format-check: setup-venv @"$(VENV_BIN)/ruff" format --check . -.PHONY: build-executable -build-executable: setup-venv - @echo "TODO: Implementation" - -.PHONY: release-checklist -release-checklist: - @echo "TODO: Implementation" - -.PHONY: release-changelog -release-changelog: - @echo "TODO: Implementation" - .PHONY: thirdpartycheck thirdpartycheck: setup-venv @echo "" @@ -108,45 +84,15 @@ thirdpartycheck: setup-venv @"$(VENV_BIN)/python" scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility @echo "Third-party dependency check complete." -.PHONY: release-upload -release-upload: - @echo "TODO: Implementation" - -.PHONY: release -release: release-checklist release-changelog thirdpartycheck build-executable release-upload - @echo "TODO: Implementation" - -.PHONY: docs -docs: setup-venv - @echo "TODO: Implementation" - .PHONY: clean clean: - @printf "WARNING: This will permanently delete all generated files, caches, and logs. Continue? [y/n] "; \ - read -r REPLY; \ - if [ "$$REPLY" = "y" ] || [ "$$REPLY" = "Y" ]; then \ - echo "Cleaning project in mode full..."; \ - $(MAKE) clean-light; \ - rm -rf logs; \ - rm -rf build; \ - rm -rf dist; \ - rm -rf examples/output; \ - rm -rf docs/_build; \ - echo "Project cleaned successfully in mode full."; \ - else \ - echo "Clean cancelled."; \ - fi - -.PHONY: clean-light -clean-light: - echo "Cleaning project in mode light..."; \ + echo "Cleaning project..."; \ find . -type f -name "*.pyc" -delete; \ find . -type d -name "__pycache__" -exec rm -rf {} +; \ find . -type d -name "log" -exec rm -rf {} +; \ find . -type d -name ".pytest_cache" -exec rm -rf {} +; \ find . -type d -name ".ruff_cache" -exec rm -rf {} +; \ - find . -type f -name "*.spec" -delete; \ rm -f .coverage .coverage.*; \ rm -rf htmlcov; \ - $(MAKE) -C examples/sim_unit clean; \ + rm -rf logs; \ echo "Project cleaned successfully in mode light."; \ diff --git a/NOTICE b/NOTICE index e69de29..8069743 100644 --- a/NOTICE +++ b/NOTICE @@ -0,0 +1,72 @@ +MATI (*.mat file interface) +Copyright 2025 olympus-tools contributors. + +This file is automatically generated. +For complete license texts, please visit the respective project URLs. + +============================================================================== + +This product contains dependencies with the following licenses: + +Apache Software License License: +------------------------------------------------------------------------------ + * meson 1.10.1 + URL: https://mesonbuild.com + + +Apache-2.0 OR BSD-2-Clause License: +------------------------------------------------------------------------------ + * packaging 26.0 + URL: https://github.com/pypa/packaging + + +BSD License License: +------------------------------------------------------------------------------ + * Pygments 2.19.2 + URL: https://pygments.org + + * scipy 1.17.1 + URL: https://scipy.org/ + + +BSD-3-Clause License: +------------------------------------------------------------------------------ + * h5py 3.16.0 + URL: https://www.h5py.org/ + + +BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0 License: +------------------------------------------------------------------------------ + * numpy 2.4.3 + URL: https://numpy.org + + +MIT License: +------------------------------------------------------------------------------ + * iniconfig 2.3.0 + URL: https://github.com/pytest-dev/iniconfig + + * pipdeptree 2.34.0 + URL: https://github.com/tox-dev/pipdeptree + + * pytest 9.0.2 + URL: https://docs.pytest.org/en/latest/ + + * ruff 0.15.8 + URL: https://docs.astral.sh/ruff + + * typeguard 4.5.1 + URL: UNKNOWN + + +MIT License License: +------------------------------------------------------------------------------ + * pluggy 1.6.0 + URL: UNKNOWN + + +PSF-2.0 License: +------------------------------------------------------------------------------ + * typing_extensions 4.15.0 + URL: https://github.com/python/typing_extensions + diff --git a/README.md b/README.md index a8169d1..d522762 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,128 @@ -# template -THIS IS THE TEMPLATE DESCRIPTION +# MATI + +This package provides a standalone interface for reading and writing MATLAB MAT-files. It supports reading version v7.3 (HDF5) via `h5py` and older versions (v7.0 and lower) via `scipy.io.loadmat`. The `get` method intelligently determines the MAT-file version and uses the appropriate backend for reading. +The default `write` function specifically targets v7.3 files. +It is designed for easy usage with minimal dependencies. + +## Usage + +To use this interface, you can import the `MatInterface` class and call its static methods. + +```python +from pathlib import Path +from mat_interface.mat_interface import MatInterface +import numpy as np + +# Example: Writing a MAT file +signals_to_write = [ + { + 'label': 'signal1', + 'timestamps': np.array([0.0, 1.0, 2.0]), + 'value': np.array([10, 20, 30]) + }, + { + 'label': 'signal2', + 'timestamps': np.array([0.0, 1.0, 2.0]), + 'value': np.array([[1, 2], [3, 4], [5, 6]]) + } +] +output_file = Path('output.mat') +MatInterface.write(output_path=output_file, signals=signals_to_write) + +# Example: Reading a MAT file +signals_read = MatInterface.get(file_path=output_file) +print(signals_read) + +# Example: Reading specific signals from a struct +signals_from_struct = MatInterface.get(file_path=output_file, struct_name=['my_struct']) + +# Example: Reading signals with a label filter +signal_1_data = MatInterface.get(file_path=output_file, label_filter=['signal1']) +``` + +## Supported Data Layouts + +The `MatInterface` supports two primary layouts when reading MAT files. + +### 1. Timeseries Struct Layout (Default for `write`) +Each signal is encapsulated in its own group containing its specific time vector and data. + +```text +/ (root or struct_group) +├── signal_1/ +│ ├── Time (dataset: 1-D timestamps) +│ ├── Data (dataset: N-D values) +│ └── Events (group: empty, necessary metadata for MATLAB) +└── signal_2/ + ├── Time (dataset: 1-D timestamps) + ├── Data (dataset: N-D values) + └── Events (group: empty, necessary metadata for MATLAB) +``` + +### 2. Flat Signal Arrays +Signals are stored as datasets at the same level, sharing a common timestamp vector. + +```text +/ (root or struct_group) +├── signal_1 (dataset: N-D values) +├── signal_2 (dataset: N-D values) +└── timestamps (dataset: 1-D shared timestamps, can also be named 'time') +``` + +### 3. Nested Struct Layout (via `struct_name`) +The interface can target signals nested within MATLAB structs (HDF5 groups). Within a single targeted struct, all signals must follow the **same** layout; combining layouts within one struct is not supported. + +#### Layout: Struct with Flat Signals +All datasets within the struct share a common timestamp vector. + +```text +/ (root) +└── my_struct_A/ <-- Targeted via struct_name=['my_struct_A'] + ├── signal_1 (dataset: N-D values) + ├── signal_2 (dataset: N-D values) + └── timestamps (dataset: 1-D shared timestamps) +``` + +#### Layout: Struct with Timeseries Groups +Each signal within the struct is its own group containing its specific time vector. + +```text +/ (root) +└── my_struct_B/ <-- Targeted via struct_name=['my_struct_B'] + ├── signal_1/ + │ ├── Time + │ └── Data + └── signal_2/ + ├── Time + └── Data +``` + +## API Documentation + +### `MatInterface.write(output_path: Path, signals: list[dict[str, str | np.typing.NDArray[np.generic]]]) -> None` + +* **Description:** Writes a list of signals to a MAT 7.3 file (HDF5). Each signal is stored as a top-level HDF5 group containing 'Time', 'Data', and 'Events' members ('Events' can be ignored, just for MATLAB compatibility). +Additional the group carries `MATLAB_class = "timeseries"` and `MATLAB_fields` attributes for MATLAB compatibility. +* **Parameters:** + * `output_path` (Path): The path to the output MAT file. + * `signals` (list[dict]): A list of dictionaries, where each dictionary represents a signal and must contain: + * `'label'` (str): The name of the signal. + * `'timestamps'` (np.ndarray): A 1-D NumPy array of timestamps (float). + * `'value'` (np.ndarray): A 1-D, 2-D, or 3-D NumPy array representing the signal's values. The dtype will be mapped to a corresponding MATLAB numerical type. +* **Note:** This method adds MATLAB-specific headers and attributes for compatibility. MATLAB's `load()` will recognize the structure, but true MATLAB `timeseries` objects require MATLAB's MCOS serialization, which cannot be produced from Python. + +### `MatInterface.get(file_path: Path, label_filter: list[str] | None = None, struct_name: list[str] | None = None) -> list[dict[str, str | np.typing.NDArray[np.generic]]]` + +* **Description:** Reads signals from a MAT 7.3 or lower file. It intelligently determines the MAT-file version and uses either `h5py` for v7.3 (HDF5) files or `scipy.io.loadmat` for older versions. It supports 2 different data layouts: + * **Timeseries struct layout:** Each signal is a group containing 'Time' and 'Data' sub-datasets. + * **Flat signal arrays:** Signals are top-level datasets, with timestamps assumed to be in a separate dataset named 'timestamps' or 'time'. + It can optionally filter signals by `label_filter` or by reading from specific `struct_name` groups within the MAT file. +* **Parameters:** + * `file_path` (Path): The path to the MAT file to read. + * `label_filter` (list[str] | None, optional): A list of signal names (labels) to filter and read. If `None`, all readable signals are returned. + * `struct_name` (list[str] | None, optional): A list of struct group names within the MAT file from which to extract signals. If `None`, signals are read from the root level of the MAT file. +* **Returns:** A list of dictionaries, where each dictionary represents a signal and contains: + * `'label'` (str): The name of the signal. + * `'timestamps'` (np.ndarray | None): A 1-D NumPy array of timestamps, or `None` if not found. + * `'value'` (np.ndarray): A NumPy array containing the signal's values. +* **Note:** This method does not support MATLAB MCOS `timeseries` objects due to their opaque serialization. It also skips internal HDF5 groups like `#refs#` and `#subsystem#`. diff --git a/architecture.md b/architecture.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/examples/mat_v7_flat_signals.mat b/examples/mat_v7_flat_signals.mat new file mode 100644 index 0000000000000000000000000000000000000000..6db7c43cc783577ea0fe7792996453cabe4fb45a GIT binary patch literal 2320 zcmb_dT}V_>5I*bL?pkU@A%+AOw4N;8AAj1So7#h`yS0U(^rLOG%7Ui3D`f}@2|o4E zQ)QAwq&*dZB0+U4C@YExp8_8u`_YIr3KUYd`OZD(wycZmLm$k}oH^gloik_dTwhs@ zx2#+#+EJ+Z%4)VZhMI%4t8aV!3cTF?GsSH4ow9t=c+ro z?TWp~U0CFHmMV@yN3pU>FbR{#mq~UTiG{>QVkH6AkYK%z#3Uq*bU4)15NK$0I>-jV z&*v6n*_~PHnR8(-upRb{t)w;bae?iyU%Ul6o_mEvJ{KBD^4wyVPGiWWPS{ZOM>4UuWt0 zRnkJ%T|Z6Y{II9@W2h88ie6>aot@Qew6AXz3q78nw2YTV_W-PX-saXOoyI`taZP}5 zl8WyS&LRKGaXXCuNF^@%W7_Pp3|a@RF3Yrez+`83Hei~j=85l*4B`Sw;v4;e^P)fY zmak_XcQ!ltYzr6I@Rd|qSb@g4z=jWq?+@LZ^tncVAWz;Ouljc(Uf!LX_0YXft0J*l z`kCNOe!i5A{jPW4_zg)d{gQ|LV*3Mg07z7qfBn%>D(|tI!)39!y7~VQ?*)nMk3{ni z?=r4l&cBm!^RJiofYr~x9`#|)0rh;z^*tny$fld<(V(%0AN%AZVGJ`BvE?kq3|;Qa z7y_5qhmw`&fd*phli)nY{o#zZm_GmCbCoB<=G5E2FJC)5HKY1x3QA_5PpF4}%zp==kuj=;McTTw@JC_T@W^*^k(A=A&|neDKG= z---GI*AKxrFA{1Ah9iNNwlFOS@!#G4tH6C#*DqC<6POU|_u^hYsNC(GQX4tGi2WJ$ L&1>MhagToik9FW8 literal 0 HcmV?d00001 diff --git a/examples/mat_v7_flat_timeseries.mat b/examples/mat_v7_flat_timeseries.mat new file mode 100644 index 0000000000000000000000000000000000000000..92de7db88043d30b5490cbb8c3859b8ae85ba771 GIT binary patch literal 4016 zcmds3Ur3Wt6u-;qwA6@7j9xy7dRnz_o6|=0`R_r|)rg4_)m*j7KpW{KjUI#oed#$W zNqX>4z4ag^WM2^U2aEzC3VN`N+9-5UV9>+5zx(Gy&A~T&ng`C#x#yhk?A~+Fz4zy< z2{hN#DisGDimxWHKN9T>+ZBIjC?1J*cX<_mU2Ee}kK3-)b%#Uo@Hr)R-mA1y=?ir$ zs;a0JUPp!3Ri!u`PLE=eh$Ks+uaxS33n2wm7E<0MeEDKbT|{y*)#E+Eo@hrf6pXl> z2+b!}$~m4RqinAC^jAvh@^{}5%F9}*<&m}YRO(I zb!uY_-Kkz1ML7Pak1kC9kNLWK-1t5h>-sPVR8+ z8@qG(6d&s%PA4raUT+t69(VuIy8Al1G5Q*36_n|m!7lr{=yxI6cBPatbkLG*DZP04 z2V>~WNPkdQhsnOi**5NrXcbG5i+x>9XD-h%uTJK=;x5CwNmGA+6PX+woWvlzx~;fp zJ7XRAKG*A3Brd>A{RlU!n+1dwb3(sWne4X}b?a3<$dHS5YoPO%{l;p-*1_Lrj8Pt0 zA}iI)C@1pV1tYc*o4X*9H*8UfN`@_oJl=N`+1Kw(F2C;&X+329)_UJ*S&aI@f0Z)G z{QEwjP47OYU8=l&gz{sm>x2yGkQl=bKeaL30nC8h5-R{B{A(*2!yM4>>)Njzez%tI z*IMt^^8Ld7A@|GdyH#Dgl^kj-Cc7S3$=8NI>F47=(@&;TOkxZ>{D=eLEkthl24=Fs zj{SO({rNhZ&G~vq|3mgOBuL()JE^|>J^+56K!6~Qo9_xdJSFh+Qv*-)ur~&-?qK-& zt%0vOe*VDvn2h-}N zV1Hu%3}$myv7hDiF_QB!iQDOZn#b)MeLvT~zLYlc>-w#W^GVI0JXpE-Y*stYr9RH> sI@@}Z(#OPD`kO_(h%@?YROD+Be!IvgznA&$iF~cX-yq`U_aaXF3;lFUoB#j- literal 0 HcmV?d00001 diff --git a/TEMPLATE/README.md b/mati/__init__.py similarity index 100% rename from TEMPLATE/README.md rename to mati/__init__.py diff --git a/mati/dummy.py b/mati/dummy.py deleted file mode 100644 index e69de29..0000000 diff --git a/mati/mati.py b/mati/mati.py new file mode 100644 index 0000000..f21c79f --- /dev/null +++ b/mati/mati.py @@ -0,0 +1,553 @@ +r""" +________________________________________________________________________ +| | +| $$\ $$\ $$$$$$\ $$$$$$$$\ $$$$$$\ | +| $$$\ $$$ |$$ __$$\\__$$ __|\_$$ _| | +| $$$$\ $$$$ |$$ / $$ | $$ | $$ | | +| $$\$$\$$ $$ |$$$$$$$$ | $$ | $$ | | +| $$ \$$$ $$ |$$ __$$ | $$ | $$ | | +| $$ |\$ /$$ |$$ | $$ | $$ | $$ | | +| $$ | \_/ $$ |$$ | $$ | $$ | $$$$$$\ | +| \__| \__|\__| \__| \__| \______| | +| | +| MATI (*.mat file interface) (c) | +|______________________________________________________________________| + +Copyright 2025 olympus-tools contributors. Dependencies and licenses +are listed in the NOTICE file: + + https://github.com/olympus-tools/MATI/blob/master/NOTICE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License: + + https://github.com/olympus-tools/MATI/blob/master/LICENSE +""" + +import datetime +import struct +import warnings +from pathlib import Path +from typing import Any, Iterable, Literal, TypedDict + +import h5py +import numpy as np +import scipy.io as sio + +DEFAULT_TIMESTAMP = "timestamps" # default name of "timestamps" for flat signals + + +class MatSignal(TypedDict): + label: str + timestamps: np.ndarray + value: np.ndarray + + +class MatInterface: + """A standalone interface for reading and writing MATLAB MAT-files supporting + - v7.3 files (HDF5 based) + - v7.0/6.0 files (via scipy) + + The MatInterface is based on python standard interfaces to guarantee easy usage and minimum dependencies. + + additional information see: + https://de.mathworks.com/help/matlab/import_export/mat-file-versions.html?s_tid=srchtitle_site_search_1_mat+files + https://www.hdfgroup.org/solutions/hdf5/ + https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.loadmat.html + """ + + @staticmethod + def write( + output_path: Path, + signals: list[MatSignal], + format: Literal["v7.3", "v7"] = "v7.3", + **kwargs, + ) -> None: + """Write signals to a MAT 7.3 (HDF5) or lower. + + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + if format == "v7.3": + MatInterface._write73(output_path=output_path, signals=signals, **kwargs) + elif format == "v7": + MatInterface._write7(output_path=output_path, signals=signals, **kwargs) + + @staticmethod + def get( + file_path: Path, + label_filter: list[str] | None = None, + struct_list: list[str] | None = None, + **kwargs, + ) -> list[MatSignal]: + """Read signals from a MAT 7.3 (HDF5) or lower. + + Supports two modes (defined via struct_list): + - flat : signals exist at the highest layer + - struct : signals are divided into structs each with providing its own timestamp + + where each mode can read two types of signals: + - **timeseries struct layout**: each signal is a struct containing "Time","Data" + - **flat signal arrays**: each signal is a array whereas one array represents the shared timestamp + + Note: MATLAB MCOS ``timeseries`` *objects* (``MATLAB_object_decode==3``) + store their data in an opaque ``#refs#`` pool and cannot be decoded + without a full MATLAB OOP deserialiser. + Therefore not the 'real' MATLAB timeseries supported. + + Assumptions: + Considering the ''flat'' signal structure the assumption that the shared timevector is called + - timestamps + is made. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + matfile_version = sio.matlab.matfile_version(file_path) + + if matfile_version[0] == 2: + signal_data = MatInterface._get73( + file_path=file_path, + label_filter=label_filter, + struct_list=struct_list, + **kwargs, + ) + elif matfile_version[0] == 1: + signal_data = MatInterface._get7( + file_path=file_path, + label_filter=label_filter, + struct_list=struct_list, + **kwargs, + ) + else: + raise ValueError(f"Unsupported/Unknown matfile-version {matfile_version}.") + return signal_data + + # MAT 7.0 functions (scipy wrapper) + @staticmethod + def _write7(output_path: Path, signals: list[MatSignal]) -> None: + """Write signals to a MAT 7.0 file via scipy. + + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + # defaults for saving with scipy.sio + kw = dict( + oned_as="column", + long_field_names=True, + do_compression=True, + ) + + mat_data_dict = {} + for signal in signals: + mat_data_dict[signal["label"]] = { + "Time": signal["timestamps"], + "Data": signal["value"], + "Events": np.array([]), # Empty array for MATLAB compatibility + } + sio.savemat(output_path, mat_data_dict, **kw) + + @staticmethod + def _get7( + file_path: Path, + label_filter: list[str] | None = None, + struct_list: list[str] | None = None, + **kwargs, + ) -> list[MatSignal]: + """Read signals from a MAT 7.0 or lower file via scipy.io package. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + **kwargs : additional arguments passed to scipy "loadmat" + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + _OBSOLETE_MATFIELDS = [ + "__header__", + "__version__", + "__globals__", + ] # define obsolete mat fields thar are always skipped during reading and define default timestamp + _TIMESTAMP = DEFAULT_TIMESTAMP + mat_data: list[MatSignal] = [] + + # helper functions for scipy (transform to standard format) + def _get_timeseries( + tmp_data: dict[str, Any], label_filter: list[str] + ) -> list[MatSignal]: + """Read timeseries struct from mat file.""" + nonlocal mat_data + signals_to_add: Iterable[MatSignal] = [ + { + "label": signal_name, + "timestamps": signal_dict.get("Time"), + "value": signal_dict.get("Data"), + } + for signal_name, signal_dict in tmp_data.items() + if signal_name not in _OBSOLETE_MATFIELDS # skip mat-fields + and (not label_filter or signal_name in label_filter) # label-filter + and type(signal_dict) is dict # check 'timeseries'-struct + and "Time" in signal_dict + and "Data" in signal_dict + ] + mat_data.extend(signals_to_add) + return mat_data + + def _get_signals( + tmp_data: dict[str, Any], + timestamps: np.ndarray, + label_filter: list[str], + ) -> list[MatSignal]: + """Read flat signals from mat file.""" + nonlocal mat_data + signals_to_add: Iterable[MatSignal] = [ + { + "label": signal_name, + "timestamps": timestamps, + "value": signal_value, + } + for signal_name, signal_value in tmp_data.items() + if signal_name not in _OBSOLETE_MATFIELDS # skip mat-fields + and signal_name != _TIMESTAMP # skip timestamp itself + and (not label_filter or signal_name in label_filter) # label-filter + and isinstance(signal_value, np.ndarray) # Ensure it's a numpy array + ] + mat_data.extend(signals_to_add) + return mat_data + + # define standard flags to guarantee readability + kwargs = {**kwargs} + kwargs.setdefault("simplify_cells", True) + tmp_data = sio.loadmat(file_path, **kwargs) + + # differentiate between struct-mode, plain-mode, timeseries + if struct_list: + tmp_data.update([tmp_data.get(struct) for struct in struct_list]) + + # check for timessignal -> timeseries or plain-mode + timestamps = tmp_data.get(_TIMESTAMP, None) + if timestamps is None: + mat_data = _get_timeseries(tmp_data, label_filter if label_filter else []) + else: + mat_data = _get_signals( + tmp_data, timestamps, label_filter if label_filter else [] + ) + return mat_data + + # MAT 7.3 functions + @staticmethod + def _write73( + output_path: Path, + signals: list[MatSignal], + ) -> None: + """Write signals to a MAT 7.3 file (HDF5). + Each signal is stored as a top-level HDF5 group containing three members: + - ``Time`` – 1-D dataset, timestamps (double) + - ``Data`` – 1-D or N-D dataset, signal values + - ``Events`` – empty group (reserved for MATLAB compatibility) + + The group carries ``MATLAB_class = "timeseries"`` and ``MATLAB_fields`` + attributes so that MATLAB's ``load()`` recognises the layout. + Note: + MATLAB's ``load()`` will return each signal as a **struct** with fields + ``Time``, ``Data``, and ``Events``. + The struct represents the base of a MATLAB ``timeseries``. + A true ``timeseries`` object requires MATLAB's MCOS object serialisation, + which cannot be produced from Python. To convert after loading in MATLAB:: + 'ts = timeseries(sig.Data, sig.Time, 'Name', 'my_signal');' + Args: + output_path (Path): Path to the output file. + signals (list[dict]): List of dicts, each with: + - ``'label'`` (str): signal name + - ``'timestamps'`` (np.ndarray): 1-D time vector + - ``'value'`` (np.ndarray): 1-D / 2-D / 3-D value array + """ + with h5py.File(output_path, "w", userblock_size=512) as h5file: # type: ignore[reportGeneralTypeIssues] + # HDF5 attribute encoding for MATLAB compatibility (hdf5 tweaked to match *.mat expectations) + _ds_class_tid = h5py.h5t.py_create(np.dtype("S8")) + _ds_class_tid.set_strpad(h5py.h5t.STR_NULLTERM) + _ds_class_sid = h5py.h5s.create(h5py.h5s.SCALAR) + _vlen_s1_dt = h5py.vlen_dtype(np.dtype("S1")) # pyright: ignore[reportAttributeAccessIssue] + + # helper functions for writing + def _write_ds_matlab_class(dataset_id, class_bytes: bytes) -> None: + """Add matlab-class attributes to represent datatype.""" + padded_bytes = class_bytes.ljust(8, b"\x00")[:8] + attribute = h5py.h5a.create( # type: ignore + dataset_id, b"MATLAB_class", _ds_class_tid, _ds_class_sid + ) + attribute.write(np.frombuffer(padded_bytes, dtype="S8")) + + def _mat_fields(*names: str) -> np.ndarray: + """Helper function for '_write_mat_fields'. Write names as ascii array.""" + array = np.empty(len(names), dtype=object) + for i, name in enumerate(names): + array[i] = np.array([c.encode("ascii") for c in name], dtype="S1") + return array + + def _write_mat_fields(group, *names: str) -> None: + """Initialize timeseries-struct with given names.""" + group.attrs.create( + "MATLAB_fields", + data=_mat_fields(*names), + dtype=_vlen_s1_dt, + ) + + # write HDF5 content + for signal in signals: + label: str = signal["label"] + timestamps: np.typing.NDArray[np.generic] = signal["timestamps"] + value: np.typing.NDArray[np.generic] = signal["value"] + dtype = value.dtype + + # initialize matlab group (add important metadata) + group = h5file.create_group(label) + group.attrs["MATLAB_class"] = np.bytes_(b"timeseries") + _write_mat_fields(group, "Time", "Data", "Events") + event_group = group.create_group("Events") + event_group.attrs["MATLAB_class"] = np.bytes_(b"struct") + _write_mat_fields(event_group) + + # write timestamp + timestamps_single = timestamps.astype(np.float32) + ti = group.create_dataset("Time", data=timestamps_single.T) + _write_ds_matlab_class(ti.id, b"single") + + # bool/logical: MATLAB requires uint8 storage + if np.issubdtype(dtype, np.bool_): + value = value.astype(np.uint8) + # write signal data + if value.ndim <= 2: + ds = group.create_dataset("Data", data=value.T) + elif value.ndim == 3: + ds = group.create_dataset("Data", data=value.transpose(2, 1, 0)) + else: + raise ValueError( + "MatInterface does not support more than 3 dimensions." + ) + _write_ds_matlab_class(ds.id, MatInterface._get_matlab_class(dtype)) + + # bool/logical: MATLAB requires MATLAB_int_decode=1 + if np.issubdtype(dtype, np.bool_): + ds.attrs["MATLAB_int_decode"] = np.uint8(1) + + # add MATLAB specific 128-byte header + date_str = datetime.datetime.now().strftime("%a %b %d %H:%M:%S %Y") + # byte: 0-115 | text | human readable + header = ( + f"MATLAB 7.3 MAT-file, Platform: Python/ARES, Created on: {date_str}".ljust( + 116 + ) + ) + subsystem_offset = ( + b"\x00" * 8 + ) # byte: 116- 123 | subsystem specific data | all 0 | + version = 0x0200 # byte: 124 - 125 | version | 0x0200 or 0x0100 | + endian = b"IM" # byte: 126 - 127 | Endian Indicator | MI BitEndian, IM LittleEndian (Intel/AMD = IM) + signature = ( + header.encode("ascii") + + subsystem_offset + + struct.pack(" list[MatSignal]: + """Read signals from a MAT 7.3 file. + + Args: + file_path (Path): Path to the MAT file. + label_filter (list[str] | None): List of signal names to read. + struct_list (list[str] | None): List of structs which contain the signals to extract. + + Returns: + list[dict]: List of ``{'label', 'timestamps', 'value'}`` dicts. + """ + _HDF5_INTERNAL = {"#refs#", "#subsystem#"} + _NUMERIC_KINDS = set("bifcu") + _TIMESERIES_KEYS = { + "Data", + "Time", + } # Events can be ignored, only important for writing + _TIMESTAMP_NAMES = {DEFAULT_TIMESTAMP} + + # helper functions for reading + def _is_timeseries_group(obj) -> bool: + """Droup = plain-struct Time/Data groups.""" + if not hasattr(obj, "keys"): + return False + if obj.attrs.get("MATLAB_object_decode") is not None: + # MCOS object – cannot decode without MATLAB OOP deserialiser + return False + return _TIMESERIES_KEYS.issubset(set(obj.keys())) + + def _is_numeric_dataset(obj) -> bool: + """Dataset = numeric""" + return ( + isinstance(obj, h5py.Dataset) # type: ignore + and obj.attrs.get("MATLAB_object_decode") is None + and obj.dtype.kind in _NUMERIC_KINDS + ) + + def _read_from_group(group) -> list[dict]: + """Decode all signals from an HDF5 group (root or struct sub-group). + MCOS objects, char arrays, non-timeseries structs ares skipped silently. + """ + result = [] + + candidate_keys = [k for k in group.keys() if k not in _HDF5_INTERNAL] + if label_filter is not None: + candidate_keys = [k for k in candidate_keys if k in label_filter] + + # resolve the shared flat timestamp vector + _ts: np.ndarray | None = None + _ts_resolved = False + + def _get_flat_timestamps() -> np.ndarray | None: + nonlocal _ts, _ts_resolved + if _ts_resolved: + return _ts + _ts_resolved = True + for k in group.keys(): + if k.lower() in _TIMESTAMP_NAMES and isinstance( + group[k], + h5py.Dataset, # type: ignore + ): + _ts = MatInterface._mat_load_num(group[k]) + break + return _ts + + for key in candidate_keys: + element = group[key] + + if _is_timeseries_group(element): + # Layout A / C – group with Time + Data sub-datasets + result.append( + { + "label": key, + "timestamps": MatInterface._mat_load_num(element["Time"]), + "value": MatInterface._mat_load_num(element["Data"]), + } + ) + elif _is_numeric_dataset(element): + # Layout B / D – flat numeric dataset; skip timestamp key itself + if key.lower() in _TIMESTAMP_NAMES: + continue + result.append( + { + "label": key, + "timestamps": _get_flat_timestamps(), + "value": MatInterface._mat_load_num(element), + } + ) + + return result + + # read HDF5 (mat-file) + with h5py.File(file_path, mode="r") as matfile: # type: ignore + signals = [] + if struct_list: + for struct in struct_list: + if struct not in matfile: + warnings.warn( + f"Struct group '{struct}' not found in {file_path}. ", + RuntimeWarning, + ) + continue + signals.extend(_read_from_group(matfile[struct])) + else: + signals.extend(_read_from_group(matfile)) + return signals + + # static, general helper functions + @staticmethod + def _get_matlab_class(dtype: np.dtype) -> bytes: + """Maps a NumPy dtype to the corresponding MATLAB class string.""" + if np.issubdtype(dtype, np.float64): + return b"double" + elif np.issubdtype(dtype, np.float32): + return b"single" + elif np.issubdtype(dtype, np.int8): + return b"int8" + elif np.issubdtype(dtype, np.uint8): + return b"uint8" + elif np.issubdtype(dtype, np.int16): + return b"int16" + elif np.issubdtype(dtype, np.uint16): + return b"uint16" + elif np.issubdtype(dtype, np.int32): + return b"int32" + elif np.issubdtype(dtype, np.uint32): + return b"uint32" + elif np.issubdtype(dtype, np.int64): + return b"int64" + elif np.issubdtype(dtype, np.uint64): + return b"uint64" + elif np.issubdtype(dtype, np.bool_): + return b"logical" + else: + return b"double" + + @staticmethod + def _mat_load_num( + dataset: h5py.Dataset, # type: ignore + ) -> np.ndarray: + """Load numerical data from h5py dataset and adjust dimensions.""" + samples = np.squeeze(dataset[()], axis=None) + if len(samples.shape) > 1: + samples = samples.T + return samples + + # INFO: the following functions are currently not used, useful to read more complex data + @staticmethod + def _mat_load_string(file: h5py.File, dataset: h5py.Dataset) -> np.ndarray: # type: ignore + """Load string data from h5py dataset.""" + n_el = dataset.shape[1] + el_string = [] + for ii in range(n_el): + el_string.append(bytes(np.array(file[dataset[0, ii]]))[::2].decode()) + return np.asarray(el_string) + + @staticmethod + def _mat_load_numstring(dataset: h5py.Dataset) -> np.ndarray | str: # type: ignore + """Load numeric string data from h5py dataset.""" + try: + samples = bytes(dataset[()])[::2].decode() + except Exception: + samples = np.squeeze(dataset[()], axis=None) + if len(samples.shape) > 1: + samples = samples.T + return samples diff --git a/pyproject.toml b/pyproject.toml index a08aff1..41878dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [project] name = "mati" +version = "0.1.0" description = "*.mat file interface" readme = "README.md" authors = [ @@ -35,50 +36,27 @@ classifiers = [ ] dependencies = [ - "typeguard==4.4.4", + "h5py>=3.16.0", + "pip-licenses>=5.5.1", + "pipdeptree>=2.34.0", + "scipy>=1.17.1", + "typeguard>=4.5.1", ] -dynamic = ["version"] -[project.optional-dependencies] -test = [ - "pytest==8.4.1", - "pytest-cov==6.0.0", -] +[dependency-groups] dev = [ - "ruff==0.12.11", - "gitpython==3.1.45", - "pip-licenses==5.0.0", - "pipdeptree==2.23.4", - "mati[test]", -] -docs = [] -release = [ - "mati[dev]", - "mati[docs]", + "pytest>=9.0.2", + "ruff>=0.15.8", + "pytest-cov>=7.1.0", + "debugpy>=1.8.20", ] -[project.urls] -"Homepage" = "https://github.com/olympus-tools/MATI" -"Documentation" = "https://github.com/olympus-tools/MATI#readme" -"Bug Tracker" = "https://github.com/olympus-tools/MATI/issues" -"Repository" = "https://github.com/olympus-tools/MATI" -"Changelog" = "https://github.com/olympus-tools/MATI/releases" -"Source" = "https://github.com/olympus-tools/MATI" -"Security Policy" = "https://github.com/olympus-tools/MATI/security/policy" -"Contributing" = "https://github.com/olympus-tools/MATI/blob/master/CONTRIBUTING.md" - -[tool.setuptools.packages.find] -include = ["mati*"] - -[tool.setuptools.package-data] -mati = ["py.typed"] - -[tool.setuptools_scm] -write_to = "mati/core/version.py" +[tool.hatch.build.targets.wheel] +packages = ["mati"] [tool.pytest.ini_options] +pythonpath = ["."] testpaths = ["test"] -python_files = ["test_*.py"] addopts = [ "--ignore=test/examples", ] @@ -87,12 +65,10 @@ addopts = [ # The maximum line length. line-length = 88 # The target Python version. -target-version = "py312" +target-version = "py313" # Excluded files and directories. exclude = [ - "ares/core/version.py", - "packages", - "submodules", + "version.py", ".venv", ] diff --git a/test/dummy.py b/test/dummy.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/test_mati.py b/test/test_mati.py new file mode 100644 index 0000000..4f90eae --- /dev/null +++ b/test/test_mati.py @@ -0,0 +1,242 @@ +r""" +________________________________________________________________________ +| | +| $$\ $$\ $$$$$$\ $$$$$$$$\ $$$$$$\ | +| $$$\ $$$ |$$ __$$\\__$$ __|\_$$ _| | +| $$$$\ $$$$ |$$ / $$ | $$ | $$ | | +| $$\$$\$$ $$ |$$$$$$$$ | $$ | $$ | | +| $$ \$$$ $$ |$$ __$$ | $$ | $$ | | +| $$ |\$ /$$ |$$ | $$ | $$ | $$ | | +| $$ | \_/ $$ |$$ | $$ | $$ | $$$$$$\ | +| \__| \__|\__| \__| \__| \______| | +| | +| MATI (*.mat file interface) (c) | +|______________________________________________________________________| + +Copyright 2025 olympus-tools contributors. Dependencies and licenses +are listed in the NOTICE file: + + https://github.com/olympus-tools/MATI/blob/master/NOTICE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License: + + https://github.com/olympus-tools/MATI/blob/master/LICENSE +""" + +import struct +from pathlib import Path + +import numpy as np +import pytest + +from mati.mati import MatInterface + + +def test_get_matlab_class(): + """Test mapping of numpy dtypes to MATLAB classes.""" + assert MatInterface._get_matlab_class(np.dtype(np.float64)) == b"double" + assert MatInterface._get_matlab_class(np.dtype(np.float32)) == b"single" + assert MatInterface._get_matlab_class(np.dtype(np.int32)) == b"int32" + assert MatInterface._get_matlab_class(np.dtype(np.uint8)) == b"uint8" + assert MatInterface._get_matlab_class(np.dtype(np.bool_)) == b"logical" + assert MatInterface._get_matlab_class(np.dtype(np.complex128)) == b"double" + + +def test_write_header(tmp_path): + """Verify the 128-byte MATLAB 7.3 header is correctly written.""" + test_file = tmp_path / "test_header.mat" + signals = [ + { + "label": "sig", + "timestamps": np.array([0.0, 1.0]), + "value": np.array([1.0, 2.0]), + } + ] + MatInterface._write73(test_file, signals) + + with open(test_file, "rb") as f: + header = f.read(128) + + assert len(header) == 128 + assert header.startswith(b"MATLAB 7.3 MAT-file") + assert header[126:128] == b"IM" # Little-endian indicator + version = struct.unpack("3 dimensions.""" + test_file = tmp_path / "test_4d.mat" + signals = [ + { + "label": "sig_4d", + "timestamps": np.array([0]), + "value": np.zeros((1, 1, 1, 1)), + } + ] + with pytest.raises(ValueError, match="does not support more than 3 dimensions"): + MatInterface._write73(test_file, signals) + + +def test_round_trip_v7(tmp_path): + """Test writing and reading back 1D signals in v7 format.""" + test_file = tmp_path / "test_v7.mat" + signals = [ + { + "label": "sig_float", + "timestamps": np.linspace(0, 5, 10), + "value": np.cos(np.linspace(0, 5, 10)), + }, + { + "label": "sig_bool", + "timestamps": np.linspace(0, 1, 2), + "value": np.array([True, False], dtype=bool), + }, + ] + + MatInterface.write(test_file, signals, format="v7") + read_signals = MatInterface.get(test_file) + assert len(read_signals) == 2 + + # Check sig_float + sig_float = next(s for s in read_signals if s["label"] == "sig_float") + np.testing.assert_array_almost_equal( + sig_float["timestamps"], signals[0]["timestamps"] + ) + np.testing.assert_array_almost_equal(sig_float["value"], signals[0]["value"]) + + # Check sig_bool + sig_bool = next(s for s in read_signals if s["label"] == "sig_bool") + np.testing.assert_array_almost_equal( + sig_bool["timestamps"], signals[1]["timestamps"] + ) + np.testing.assert_array_equal(sig_bool["value"], signals[1]["value"]) + + +def test_read_v7_flat_signals_example(): + """Test reading flat signals from an existing v7 MAT file.""" + file_path = Path("examples/mat_v7_flat_signals.mat") + # Use label_filter to get specific signals we know exist + signals = MatInterface.get(file_path, label_filter=["sig_a_f32", "sig_b_f32"]) + + assert len(signals) == 2 + + sig_a = next(s for s in signals if s["label"] == "sig_a_f32") + sig_b = next(s for s in signals if s["label"] == "sig_b_f32") + + assert sig_a["value"].shape == (11,) + assert sig_a["value"].dtype == np.float32 + assert sig_a["timestamps"].shape == (11,) + + assert sig_b["value"].shape == (11, 3) + assert sig_b["value"].dtype == np.float32 + assert sig_b["timestamps"].shape == (11,) + + +def test_read_v7_timeseries_example(): + """Test reading timeseries structs from an existing v7 MAT file.""" + file_path = Path("examples/mat_v7_flat_timeseries.mat") + # Use label_filter to get specific signals we know exist + signals = MatInterface.get(file_path, label_filter=["ts_sig_a_f32", "ts_sig_b_f32"]) + + assert len(signals) == 2 + + sig_a = next(s for s in signals if s["label"] == "ts_sig_a_f32") + sig_b = next(s for s in signals if s["label"] == "ts_sig_b_f32") + + assert sig_a["label"] == "ts_sig_a_f32" + assert sig_a["timestamps"].shape == (11,) + assert sig_a["value"].shape == (11,) + assert sig_a["value"].dtype == np.float32 + + assert sig_b["label"] == "ts_sig_b_f32" + assert sig_b["timestamps"].shape == (11,) + assert sig_b["value"].shape == (11, 3) + assert sig_b["value"].dtype == np.float32 From f11d7f26461e8f92fa0984f219f000f2628e8bd0 Mon Sep 17 00:00:00 2001 From: jonas Date: Sat, 28 Mar 2026 15:45:44 +0100 Subject: [PATCH 2/7] jonas: add workflows --- .github/workflow-actions/cleanup/action.yml | 9 +++++++ .github/workflow-actions/setup/action.yml | 21 +++++++++++++++ .github/workflows/check_format.yml | 29 +++++++++++++++++++++ .github/workflows/test_coverage.yml | 29 +++++++++++++++++++++ .github/workflows/test_requirements.yml | 29 +++++++++++++++++++++ Makefile | 16 ++---------- pyproject.toml | 12 +++++---- 7 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 .github/workflow-actions/cleanup/action.yml create mode 100644 .github/workflow-actions/setup/action.yml create mode 100644 .github/workflows/check_format.yml create mode 100644 .github/workflows/test_coverage.yml create mode 100644 .github/workflows/test_requirements.yml diff --git a/.github/workflow-actions/cleanup/action.yml b/.github/workflow-actions/cleanup/action.yml new file mode 100644 index 0000000..3199800 --- /dev/null +++ b/.github/workflow-actions/cleanup/action.yml @@ -0,0 +1,9 @@ +name: 'Cleanup Environment' +description: 'Clean up build artifacts and temporary files' +runs: + using: "composite" + steps: + - name: Clean up + shell: bash + run: | + yes | make clean || true diff --git a/.github/workflow-actions/setup/action.yml b/.github/workflow-actions/setup/action.yml new file mode 100644 index 0000000..6312767 --- /dev/null +++ b/.github/workflow-actions/setup/action.yml @@ -0,0 +1,21 @@ +name: 'Setup Environment' +description: 'Setup Python 3.13.7 and print system information' +runs: + using: "composite" + steps: + - name: Set up Python 3.13.7 + uses: actions/setup-python@v5 + with: + python-version: '3.13.7' + + - name: Print system information + shell: bash + run: | + echo "Running on Linux" + uname -a + python --version + + - name: Setup virtual environment + shell: bash + run: | + make setup-venv VENV_RELEASE=true VENV_RECREATE=true diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml new file mode 100644 index 0000000..045a15b --- /dev/null +++ b/.github/workflows/check_format.yml @@ -0,0 +1,29 @@ +name: Check Format Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + check-format: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run ruff for format checking + run: | + make format-check + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/.github/workflows/test_coverage.yml b/.github/workflows/test_coverage.yml new file mode 100644 index 0000000..c7cb4e9 --- /dev/null +++ b/.github/workflows/test_coverage.yml @@ -0,0 +1,29 @@ +name: Test Coverage Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test-coverage: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run coverage tests + run: | + make test-coverage + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/.github/workflows/test_requirements.yml b/.github/workflows/test_requirements.yml new file mode 100644 index 0000000..e7c180a --- /dev/null +++ b/.github/workflows/test_requirements.yml @@ -0,0 +1,29 @@ +name: Test Requirements Pipeline + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test-requirements: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Environment + uses: ./.github/workflow-actions/setup + + - name: Run requirements tests + run: | + make test-requirements + + - name: Cleanup Environment + uses: ./.github/workflow-actions/cleanup diff --git a/Makefile b/Makefile index 9060c72..a71e230 100644 --- a/Makefile +++ b/Makefile @@ -43,24 +43,12 @@ setup-venv: # Check if .git exists to decide on versioning strategy for editable install \ if [ ! -d ".git" ]; then \ echo "NOTE: .git directory not found. Setting pretend version for installation."; \ - if [ "$(VENV_RELEASE)" = "true" ] || [ "$(VENV_RELEASE)" = "TRUE" ]; then \ - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[release]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ - else \ - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[dev]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ - fi; \ + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[dev,test]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ else \ echo "NOTE: .git directory found. Using setuptools_scm for versioning."; \ - if [ "$(VENV_RELEASE)" = "true" ] || [ "$(VENV_RELEASE)" = "TRUE" ]; then \ - "$(VENV_BIN)/pip" install -e ".[release]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ - else \ - "$(VENV_BIN)/pip" install -e ".[dev]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ - fi; \ + "$(VENV_BIN)/pip" install -e ".[dev,test]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ fi -.PHONY: examples -examples: setup-venv - @echo "TODO: Implementation" - .PHONY: test-requirements test-requirements: setup-venv @"$(VENV_BIN)/pytest" diff --git a/pyproject.toml b/pyproject.toml index 41878dc..ec8fc91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,18 +37,20 @@ classifiers = [ dependencies = [ "h5py>=3.16.0", - "pip-licenses>=5.5.1", - "pipdeptree>=2.34.0", "scipy>=1.17.1", "typeguard>=4.5.1", ] -[dependency-groups] +[project.optional-dependencies] +test = [ + "pytest==8.4.1", + "pytest-cov==6.0.0", +] dev = [ - "pytest>=9.0.2", "ruff>=0.15.8", - "pytest-cov>=7.1.0", "debugpy>=1.8.20", + "pip-licenses>=5.5.1", + "pipdeptree>=2.34.0", ] [tool.hatch.build.targets.wheel] From 1626918cca3f35b477d0fe615d998140578bb128 Mon Sep 17 00:00:00 2001 From: jonas Date: Sun, 29 Mar 2026 10:12:24 +0200 Subject: [PATCH 3/7] jonas: update version to match ARES --- .gitignore | 1 + pyproject.toml | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3af7067..75d898a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ docs/_build/* *.pyc *.pyo .python-version +version.py # coverage .coverage diff --git a/pyproject.toml b/pyproject.toml index ec8fc91..feaa8f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,9 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + [project] name = "mati" -version = "0.1.0" description = "*.mat file interface" readme = "README.md" authors = [ @@ -34,12 +37,12 @@ classifiers = [ # Environment "Environment :: Console", ] - dependencies = [ "h5py>=3.16.0", "scipy>=1.17.1", - "typeguard>=4.5.1", + "typeguard>=4.4.4", ] +dynamic = ["version"] [project.optional-dependencies] test = [ @@ -51,11 +54,18 @@ dev = [ "debugpy>=1.8.20", "pip-licenses>=5.5.1", "pipdeptree>=2.34.0", + "mati[test]", ] [tool.hatch.build.targets.wheel] packages = ["mati"] +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "mati/version.py" + [tool.pytest.ini_options] pythonpath = ["."] testpaths = ["test"] From 2f36b35986a5565bce70658a849538aa3834c700 Mon Sep 17 00:00:00 2001 From: jonas Date: Sun, 29 Mar 2026 10:41:21 +0200 Subject: [PATCH 4/7] jonas: update thirdparty --- NOTICE | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/NOTICE b/NOTICE index 8069743..9b34680 100644 --- a/NOTICE +++ b/NOTICE @@ -10,10 +10,25 @@ This product contains dependencies with the following licenses: Apache Software License License: ------------------------------------------------------------------------------ + * mati 0.1.0 + URL: UNKNOWN + + * mati 0.1.0 + URL: UNKNOWN + + * mati 0.1.dev4+g65bcbb1a1.d19800101 + URL: UNKNOWN + * meson 1.10.1 URL: https://mesonbuild.com +Apache-2.0 License: +------------------------------------------------------------------------------ + * coverage 7.13.5 + URL: https://github.com/coveragepy/coveragepy + + Apache-2.0 OR BSD-2-Clause License: ------------------------------------------------------------------------------ * packaging 26.0 @@ -49,9 +64,6 @@ MIT License: * pipdeptree 2.34.0 URL: https://github.com/tox-dev/pipdeptree - * pytest 9.0.2 - URL: https://docs.pytest.org/en/latest/ - * ruff 0.15.8 URL: https://docs.astral.sh/ruff @@ -61,9 +73,18 @@ MIT License: MIT License License: ------------------------------------------------------------------------------ + * debugpy 1.8.20 + URL: https://aka.ms/debugpy + * pluggy 1.6.0 URL: UNKNOWN + * pytest 8.4.1 + URL: https://docs.pytest.org/en/latest/ + + * pytest-cov 6.0.0 + URL: https://github.com/pytest-dev/pytest-cov + PSF-2.0 License: ------------------------------------------------------------------------------ From 19423238ab94e148adf88639f8d948e512a52b64 Mon Sep 17 00:00:00 2001 From: jonas Date: Sun, 29 Mar 2026 12:43:57 +0200 Subject: [PATCH 5/7] jonas: update of readme considering "graphical" stuff --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d522762..9d32fee 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ signals_read = MatInterface.get(file_path=output_file) print(signals_read) # Example: Reading specific signals from a struct -signals_from_struct = MatInterface.get(file_path=output_file, struct_name=['my_struct']) +signals_from_struct = MatInterface.get(file_path=output_file, struct_list=['my_struct']) # Example: Reading signals with a label filter signal_1_data = MatInterface.get(file_path=output_file, label_filter=['signal1']) @@ -69,7 +69,7 @@ Signals are stored as datasets at the same level, sharing a common timestamp vec └── timestamps (dataset: 1-D shared timestamps, can also be named 'time') ``` -### 3. Nested Struct Layout (via `struct_name`) +### 3. Nested Struct Layout (via `struct_list`) The interface can target signals nested within MATLAB structs (HDF5 groups). Within a single targeted struct, all signals must follow the **same** layout; combining layouts within one struct is not supported. #### Layout: Struct with Flat Signals @@ -77,7 +77,7 @@ All datasets within the struct share a common timestamp vector. ```text / (root) -└── my_struct_A/ <-- Targeted via struct_name=['my_struct_A'] +└── my_struct_A/ <-- Targeted via struct_list=['my_struct_A'] ├── signal_1 (dataset: N-D values) ├── signal_2 (dataset: N-D values) └── timestamps (dataset: 1-D shared timestamps) @@ -88,7 +88,7 @@ Each signal within the struct is its own group containing its specific time vect ```text / (root) -└── my_struct_B/ <-- Targeted via struct_name=['my_struct_B'] +└── my_struct_B/ <-- Targeted via struct_list=['my_struct_B'] ├── signal_1/ │ ├── Time │ └── Data @@ -99,11 +99,13 @@ Each signal within the struct is its own group containing its specific time vect ## API Documentation -### `MatInterface.write(output_path: Path, signals: list[dict[str, str | np.typing.NDArray[np.generic]]]) -> None` +```python +MatInterface.write(output_path: Path, signals: list[dict[str, str | np.typing.NDArray[np.generic]]]) -> None +``` * **Description:** Writes a list of signals to a MAT 7.3 file (HDF5). Each signal is stored as a top-level HDF5 group containing 'Time', 'Data', and 'Events' members ('Events' can be ignored, just for MATLAB compatibility). Additional the group carries `MATLAB_class = "timeseries"` and `MATLAB_fields` attributes for MATLAB compatibility. -* **Parameters:** +* **Arguments:** * `output_path` (Path): The path to the output MAT file. * `signals` (list[dict]): A list of dictionaries, where each dictionary represents a signal and must contain: * `'label'` (str): The name of the signal. @@ -111,16 +113,18 @@ Additional the group carries `MATLAB_class = "timeseries"` and `MATLAB_fields` a * `'value'` (np.ndarray): A 1-D, 2-D, or 3-D NumPy array representing the signal's values. The dtype will be mapped to a corresponding MATLAB numerical type. * **Note:** This method adds MATLAB-specific headers and attributes for compatibility. MATLAB's `load()` will recognize the structure, but true MATLAB `timeseries` objects require MATLAB's MCOS serialization, which cannot be produced from Python. -### `MatInterface.get(file_path: Path, label_filter: list[str] | None = None, struct_name: list[str] | None = None) -> list[dict[str, str | np.typing.NDArray[np.generic]]]` +```python +MatInterface.get(file_path: Path, label_filter: list[str] | None = None, struct_list: list[str] | None = None) -> list[dict[str, str | np.typing.NDArray[np.generic]]] +``` * **Description:** Reads signals from a MAT 7.3 or lower file. It intelligently determines the MAT-file version and uses either `h5py` for v7.3 (HDF5) files or `scipy.io.loadmat` for older versions. It supports 2 different data layouts: * **Timeseries struct layout:** Each signal is a group containing 'Time' and 'Data' sub-datasets. * **Flat signal arrays:** Signals are top-level datasets, with timestamps assumed to be in a separate dataset named 'timestamps' or 'time'. - It can optionally filter signals by `label_filter` or by reading from specific `struct_name` groups within the MAT file. -* **Parameters:** + It can optionally filter signals by `label_filter` or by reading from specific `struct_list` groups within the MAT file. +* **Arguments:** * `file_path` (Path): The path to the MAT file to read. * `label_filter` (list[str] | None, optional): A list of signal names (labels) to filter and read. If `None`, all readable signals are returned. - * `struct_name` (list[str] | None, optional): A list of struct group names within the MAT file from which to extract signals. If `None`, signals are read from the root level of the MAT file. + * `struct_list` (list[str] | None, optional): A list of struct group names within the MAT file from which to extract signals. If `None`, signals are read from the root level of the MAT file. * **Returns:** A list of dictionaries, where each dictionary represents a signal and contains: * `'label'` (str): The name of the signal. * `'timestamps'` (np.ndarray | None): A 1-D NumPy array of timestamps, or `None` if not found. From 08c181fbba70fb18ae5424e80b31b6b10fe9a5ec Mon Sep 17 00:00:00 2001 From: jonas Date: Wed, 1 Jul 2026 21:26:02 +0200 Subject: [PATCH 6/7] jonas: update makefile to use uv like main repository --- Makefile | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index a71e230..f2d67c0 100644 --- a/Makefile +++ b/Makefile @@ -27,49 +27,30 @@ endif # Main setup-venv target - uses POSIX shell commands (works with Git Bash on Windows) .PHONY: setup-venv setup-venv: - @if [ -d "$(VENV_DIR)" ]; then \ - echo "Virtual environment '$(VENV_DIR)' already exists. For recreation add 'VENV_RECREATE=true' to cli."; \ - if [ "$(VENV_RECREATE)" = "true" ] || [ "$(VENV_RECREATE)" = "TRUE" ]; then \ - echo "Removing existing virtual environment '$(VENV_DIR)'..."; \ - rm -rf "$(VENV_DIR)"; \ - else \ - exit 0; \ - fi; \ - fi; \ - echo "Creating virtual environment '$(VENV_DIR)'..."; \ - python -m venv "$(VENV_DIR)" || { echo "Error: Failed to create virtual environment."; exit 1; }; \ - echo "Installing project dependencies in virtual environment '$(VENV_DIR)'..."; \ - \ - # Check if .git exists to decide on versioning strategy for editable install \ - if [ ! -d ".git" ]; then \ - echo "NOTE: .git directory not found. Setting pretend version for installation."; \ - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MATI=0.0.1 "$(VENV_BIN)/pip" install -e ".[dev,test]" || { echo "Error: Failed to install dependencies (no Git)."; exit 1; }; \ - else \ - echo "NOTE: .git directory found. Using setuptools_scm for versioning."; \ - "$(VENV_BIN)/pip" install -e ".[dev,test]" || { echo "Error: Failed to install dependencies (with Git)."; exit 1; }; \ - fi - + echo "Syncing project dependencies:" + uv sync --all-extras + .PHONY: test-requirements test-requirements: setup-venv - @"$(VENV_BIN)/pytest" + uv run pytest .PHONY: test-coverage test-coverage: setup-venv - @"$(VENV_BIN)/pytest" --cov --cov-report=html --cov-report=term-missing + uv run pytest --cov --cov-report=html --cov-report=term-missing .PHONY: format format: setup-venv - @"$(VENV_BIN)/ruff" format . + uv run ruff format . .PHONY: format-check format-check: setup-venv - @"$(VENV_BIN)/ruff" format --check . + uv run ruff format --check . .PHONY: thirdpartycheck thirdpartycheck: setup-venv @echo "" @echo "Running third-party dependency analysis..." - @"$(VENV_BIN)/python" scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility + uv run python scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility @echo "Third-party dependency check complete." .PHONY: clean From 159d26e75e84a21b7c094fd3eaf1844a486826ab Mon Sep 17 00:00:00 2001 From: jonas Date: Sun, 11 Oct 2026 12:56:34 +0200 Subject: [PATCH 7/7] jonas: update "tools" in mati according to ares --- .github/workflow-actions/setup/action.yml | 13 +++++++++---- Makefile | 13 ++++++------- NOTICE | 11 +---------- README.md | 20 ++++++++++---------- pyproject.toml | 6 ++++-- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.github/workflow-actions/setup/action.yml b/.github/workflow-actions/setup/action.yml index 6312767..6af7167 100644 --- a/.github/workflow-actions/setup/action.yml +++ b/.github/workflow-actions/setup/action.yml @@ -11,11 +11,16 @@ runs: - name: Print system information shell: bash run: | - echo "Running on Linux" - uname -a - python --version + echo "::group::System information" + echo "::notice title=Operating system::Linux." + echo "::notice title=Kernel::$(uname -a)." + echo "::notice title=Python::$(python --version 2>&1)." + echo "::endgroup::" - name: Setup virtual environment shell: bash run: | - make setup-venv VENV_RELEASE=true VENV_RECREATE=true + echo "::group::ARES environment setup" + pipx install uv + make setup-venv + echo "::endgroup::" diff --git a/Makefile b/Makefile index f2d67c0..6ef6cef 100644 --- a/Makefile +++ b/Makefile @@ -9,17 +9,16 @@ # - make thirdpartycheck # - make clean -VENV_DIR := .venv -VENV_RECREATE := false -VENV_RELEASE := false +# Release metadata is maintained in pyproject.toml and checked against the tag. +RELEASE_TAG := $(shell git describe --tags --exact-match --match 'v[0-9]*' 2>/dev/null) +VERSION := $(shell sed -n 's/^version = "\([^"]*\)"/\1/p' pyproject.toml) +RELEASE_ARCHIVE := dist/ares-$(VERSION)-release.zip # Platform detection for virtual environment binary path ifeq ($(OS),Windows_NT) - VENV_BIN := $(VENV_DIR)/Scripts PLATFORM := windows PATHSEP := ; else - VENV_BIN := $(VENV_DIR)/bin PLATFORM := unix PATHSEP := : endif @@ -36,7 +35,7 @@ test-requirements: setup-venv .PHONY: test-coverage test-coverage: setup-venv - uv run pytest --cov --cov-report=html --cov-report=term-missing + NUMBA_DISABLE_JIT=1 uv run pytest --cov --cov-report=html --cov-report=term-missing .PHONY: format format: setup-venv @@ -50,7 +49,7 @@ format-check: setup-venv thirdpartycheck: setup-venv @echo "" @echo "Running third-party dependency analysis..." - uv run python scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility + uv run scripts/analyze_dependencies.py --format json --generate-notice --check-compatibility @echo "Third-party dependency check complete." .PHONY: clean diff --git a/NOTICE b/NOTICE index 9b34680..d03c84c 100644 --- a/NOTICE +++ b/NOTICE @@ -10,18 +10,9 @@ This product contains dependencies with the following licenses: Apache Software License License: ------------------------------------------------------------------------------ - * mati 0.1.0 + * mati 0.1.dev7+g08c181fbb.d19800101 URL: UNKNOWN - * mati 0.1.0 - URL: UNKNOWN - - * mati 0.1.dev4+g65bcbb1a1.d19800101 - URL: UNKNOWN - - * meson 1.10.1 - URL: https://mesonbuild.com - Apache-2.0 License: ------------------------------------------------------------------------------ diff --git a/README.md b/README.md index 9d32fee..f8deb23 100644 --- a/README.md +++ b/README.md @@ -16,17 +16,17 @@ import numpy as np # Example: Writing a MAT file signals_to_write = [ { - 'label': 'signal1', - 'timestamps': np.array([0.0, 1.0, 2.0]), - 'value': np.array([10, 20, 30]) + "label": "signal1", + "timestamps": np.array([0.0, 1.0, 2.0]), + "value": np.array([10, 20, 30]), }, { - 'label': 'signal2', - 'timestamps': np.array([0.0, 1.0, 2.0]), - 'value': np.array([[1, 2], [3, 4], [5, 6]]) - } + "label": "signal2", + "timestamps": np.array([0.0, 1.0, 2.0]), + "value": np.array([[1, 2], [3, 4], [5, 6]]), + }, ] -output_file = Path('output.mat') +output_file = Path("output.mat") MatInterface.write(output_path=output_file, signals=signals_to_write) # Example: Reading a MAT file @@ -34,10 +34,10 @@ signals_read = MatInterface.get(file_path=output_file) print(signals_read) # Example: Reading specific signals from a struct -signals_from_struct = MatInterface.get(file_path=output_file, struct_list=['my_struct']) +signals_from_struct = MatInterface.get(file_path=output_file, struct_list=["my_struct"]) # Example: Reading signals with a label filter -signal_1_data = MatInterface.get(file_path=output_file, label_filter=['signal1']) +signal_1_data = MatInterface.get(file_path=output_file, label_filter=["signal1"]) ``` ## Supported Data Layouts diff --git a/pyproject.toml b/pyproject.toml index feaa8f3..e979826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,12 +51,15 @@ test = [ ] dev = [ "ruff>=0.15.8", - "debugpy>=1.8.20", "pip-licenses>=5.5.1", "pipdeptree>=2.34.0", "mati[test]", ] +debug = [ + "debugpy>=1.8.20", +] + [tool.hatch.build.targets.wheel] packages = ["mati"] @@ -80,7 +83,6 @@ line-length = 88 target-version = "py313" # Excluded files and directories. exclude = [ - "version.py", ".venv", ]