diff --git a/src/isp_workbook_parser/__init__.py b/src/isp_workbook_parser/__init__.py index 92591b1..ca16998 100644 --- a/src/isp_workbook_parser/__init__.py +++ b/src/isp_workbook_parser/__init__.py @@ -7,11 +7,10 @@ import pandas as pd -from .config_model import TableConfig as TableConfig -from .config_model import load_yaml as load_yaml +from .config_model import TableConfig, load_yaml from .parser import Parser -from .read_table import read_table as read_table +from .read_table import read_table __all__ = ["Parser", "TableConfig", "load_yaml", "read_table"] -pd.set_option("future.no_silent_downcasting", True) +pd.set_option("future.no_silent_downcasting", True) # noqa: FBT003 diff --git a/src/isp_workbook_parser/parser.py b/src/isp_workbook_parser/parser.py index 9abbdcb..892b739 100644 --- a/src/isp_workbook_parser/parser.py +++ b/src/isp_workbook_parser/parser.py @@ -114,7 +114,7 @@ def _determine_config_path( def _check_version_is_supported(self, config_path: Path) -> None: """Check the default config directory contains a subdirectory that matches the workbook version number.""" - versions = [dir.name for dir in config_path.iterdir()] + versions = [directory.name for directory in config_path.iterdir()] if self.workbook_version not in versions: msg = f"The workbook version {self.workbook_version} is not supported." raise ValueError(msg) @@ -509,7 +509,7 @@ def _postprocess_percentage_columns_between_0_and_100( if isinstance(sr, int) and cell.row == sr: skipped_rows += 1 continue - if isinstance(cell.value, (int, float)) and "%" in cell.number_format: + if isinstance(cell.value, int | float) and "%" in cell.number_format: percentage_cells.append( ( cell.row - min_row - skipped_rows, @@ -553,7 +553,7 @@ def get_table_names(self) -> list[str]: return self.table_names_by_sheet def get_table_from_config( - self, table_config: TableConfig, config_checks: bool = True + self, table_config: TableConfig, *, config_checks: bool = True ) -> pd.DataFrame: """Retrieve a table from the assumptions workbook using the config provided and returns as pd.DataFrame. @@ -603,7 +603,7 @@ def get_table_from_config( self._check_table(data, table_config) return data - def get_table(self, table_name: str, config_checks: bool = True) -> pd.DataFrame: + def get_table(self, table_name: str, *, config_checks: bool = True) -> pd.DataFrame: """Retrieve a table from the assumptions workbook and returns as `pd.DataFrame`. Examples: @@ -643,6 +643,7 @@ def save_tables( self, directory: str | Path, tables: list[str] | str = "all", + *, config_checks: bool = True, ) -> None: """Save tables from the provided workbook to the specified directory as CSV files. @@ -672,7 +673,7 @@ def save_tables( msg = "The path provided is not a directory." raise ValueError(msg) - if not (isinstance(tables, (str, list))): + if not (isinstance(tables, str | list)): msg = "The parameter tables must be provided as str or list[str]." raise TypeError(msg) diff --git a/src/isp_workbook_parser/read_table.py b/src/isp_workbook_parser/read_table.py index 6815e36..730b06a 100644 --- a/src/isp_workbook_parser/read_table.py +++ b/src/isp_workbook_parser/read_table.py @@ -132,7 +132,10 @@ def read_table(workbook_file: pd.ExcelFile, table: TableConfig) -> pd.DataFrame: series[series != ""] = "_" + series[series != ""] merged_headers = ffilled_initial_header.str.cat(filled_headers) df_cleaned = _build_cleaned_dataframe( - df_initial, header_rows_in_table, merged_headers, table.forward_fill_values + df_initial, + header_rows_in_table, + merged_headers, + forward_fill_values=table.forward_fill_values, ) if table.skip_rows: df_cleaned = _skip_rows_in_dataframe( @@ -202,6 +205,7 @@ def _build_cleaned_dataframe( df_initial: pd.DataFrame, header_rows_in_table: int, new_headers: pd.Series, + *, forward_fill_values: bool, ) -> pd.DataFrame: """ diff --git a/src/isp_workbook_parser/sanitisers.py b/src/isp_workbook_parser/sanitisers.py index 0d34606..a48c5f5 100644 --- a/src/isp_workbook_parser/sanitisers.py +++ b/src/isp_workbook_parser/sanitisers.py @@ -64,7 +64,7 @@ def _values_casting_and_sanitisation(df: pd.DataFrame) -> pd.DataFrame: will return `pd.NA` """ df = _replace_dataframe_hyphens_with_na(df) - for object_col in df.dtypes[df.dtypes == "object"].keys(): + for object_col in df.dtypes[df.dtypes == "object"].keys(): # noqa: SIM118 try: df.loc[:, object_col] = pd.to_numeric(df[object_col]) except (ValueError, TypeError): diff --git a/tests/conftest.py b/tests/conftest.py index 49028b4..441bd19 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ def workbook_v6() -> Parser: @pytest.fixture(scope="module") -def sample_series(): +def sample_series() -> pd.Series: return pd.Series( [ "First line\nSecond line", diff --git a/tests/test_packaged_table_configs.py b/tests/test_packaged_table_configs.py index c782daa..ab77072 100644 --- a/tests/test_packaged_table_configs.py +++ b/tests/test_packaged_table_configs.py @@ -15,8 +15,8 @@ @pytest.mark.parametrize("workbook_version_folder", list(workbook_path.iterdir())) -def test_packaged_table_configs_for_each_version(workbook_version_folder: Path): - xl_file = [file for file in workbook_version_folder.glob("[!.]*.xls*")] +def test_packaged_table_configs_for_each_version(workbook_version_folder: Path) -> None: + xl_file = list(workbook_version_folder.glob("[!.]*.xls*")) assert len(xl_file) == 1, ( f"There should only be one Excel workbook in each version sub-directory, got {xl_file}" ) @@ -34,9 +34,7 @@ def test_packaged_table_configs_for_each_version(workbook_version_folder: Path): for index, value in enumerate(sheet_header_end_row_combos): if sheet_header_end_row_combos.count(value) > 1: duplicate_configs.append(table_names[index]) - if len(duplicate_configs) > 0: - print(duplicate_configs) - assert len(duplicate_configs) == 0 + assert len(duplicate_configs) == 0, duplicate_configs save_dir = Path(f"example_output/{workbook.workbook_version}") save_dir.mkdir(parents=True, exist_ok=True) @@ -46,14 +44,14 @@ def test_packaged_table_configs_for_each_version(workbook_version_folder: Path): table = workbook.get_table(table_name) save_path = save_dir / Path(f"{table_name}.csv") table.to_csv(save_path, index=False) - except Exception as e: + except (FileNotFoundError, PermissionError, OSError) as e: error_tables[table_name] = e if error_tables: error_str = "" - for key in error_tables: - error_str += key + ":" + str(error_tables[key]) + "\n" + for key, value in error_tables.items(): + error_str += key + ":" + str(value) + "\n" raise TableLoadError(error_str) class TableLoadError(Exception): - """Exception to throw if table loading fails""" + """Exception to throw if table loading fails.""" diff --git a/tests/test_read_table_functionality.py b/tests/test_read_table_functionality.py index 580d103..f333064 100644 --- a/tests/test_read_table_functionality.py +++ b/tests/test_read_table_functionality.py @@ -6,9 +6,10 @@ # (at your option) any later version. from isp_workbook_parser.config_model import TableConfig +from isp_workbook_parser.parser import Parser -def test_skip_single_row_in_single_header_row_table(workbook_v6): +def test_skip_single_row_in_single_header_row_table(workbook_v6: Parser) -> None: table_config = TableConfig( name="build_cost_current_policies", sheet_name="Build costs", @@ -22,7 +23,7 @@ def test_skip_single_row_in_single_header_row_table(workbook_v6): assert df[df.Technology.str.contains("Hydrogen")].empty -def test_skip_multiple_rows_in_single_header_row_table(workbook_v6): +def test_skip_multiple_rows_in_single_header_row_table(workbook_v6: Parser) -> None: table_config = TableConfig( name="existing_generator_maintenance_rates", sheet_name="Maintenance", @@ -37,7 +38,7 @@ def test_skip_multiple_rows_in_single_header_row_table(workbook_v6): assert df[df["Generator type"].str.contains("Coal")].empty -def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6): +def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6: Parser) -> None: table_config = TableConfig( name="wind_high_capacity_factors", sheet_name="Capacity Factors ", @@ -52,7 +53,7 @@ def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6): assert df[df["Wind High_REZ ID"].str.contains("V")].empty -def test_no_forward_fill_in_rows(workbook_v6): +def test_no_forward_fill_in_rows(workbook_v6: Parser) -> None: table_config = TableConfig( name="outages_new_entrants", sheet_name="Generator Reliability Settings", diff --git a/tests/test_sanitisers.py b/tests/test_sanitisers.py index 1f449ca..53319b4 100644 --- a/tests/test_sanitisers.py +++ b/tests/test_sanitisers.py @@ -8,7 +8,7 @@ from pathlib import Path import pandas as pd -import pytest +from pandas import Series from isp_workbook_parser.sanitisers import ( _extract_numeric_value_millions, @@ -25,7 +25,7 @@ ) -def test_sanitisation_on_flow_path_transfer_capability(): +def test_sanitisation_on_flow_path_transfer_capability() -> None: unsanitised = pd.read_csv(Path("tests", "test_data", "unsanitised.csv")) expected = pd.read_csv(Path("tests", "test_data", "sanitised.csv")) # handle carriage return on Windows @@ -38,7 +38,7 @@ def test_sanitisation_on_flow_path_transfer_capability(): pd.testing.assert_frame_equal(test_sanitised, expected, check_dtype=False) -def test_replace_series_newlines_with_whitespace(sample_series): +def test_replace_series_newlines_with_whitespace(sample_series: Series) -> None: result = _replace_series_newlines_with_whitespace(sample_series) expected = pd.Series( [ @@ -63,7 +63,7 @@ def test_replace_series_newlines_with_whitespace(sample_series): pd.testing.assert_series_equal(result, expected) -def test_extract_numeric_value_millions(sample_series): +def test_extract_numeric_value_millions(sample_series: Series) -> None: result = _extract_numeric_value_millions(sample_series) expected = pd.Series( [ @@ -88,7 +88,7 @@ def test_extract_numeric_value_millions(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_double_whitespaces(sample_series): +def test_remove_series_double_whitespaces(sample_series: Series) -> None: result = _remove_series_double_whitespaces(sample_series) expected = pd.Series( [ @@ -113,7 +113,7 @@ def test_remove_series_double_whitespaces(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_trailing_asterisks(sample_series): +def test_remove_series_trailing_asterisks(sample_series: Series) -> None: result = _remove_series_trailing_asterisks(sample_series) expected = pd.Series( [ @@ -138,7 +138,7 @@ def test_remove_series_trailing_asterisks(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_trailing_footnotes(sample_series): +def test_remove_series_trailing_footnotes(sample_series: Series) -> None: result = _remove_series_trailing_footnotes(sample_series) expected = pd.Series( [ @@ -163,7 +163,7 @@ def test_remove_series_trailing_footnotes(sample_series): pd.testing.assert_series_equal(result, expected) -def test_strip_series_whitespaces(sample_series): +def test_strip_series_whitespaces(sample_series: Series) -> None: result = _strip_series_whitespaces(sample_series) expected = pd.Series( [ @@ -188,7 +188,7 @@ def test_strip_series_whitespaces(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_thousands_commas(sample_series): +def test_remove_series_thousands_commas(sample_series: Series) -> None: result = _remove_series_thousands_commas(sample_series) expected = pd.Series( [ @@ -213,7 +213,7 @@ def test_remove_series_thousands_commas(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_notes_after_values(sample_series): +def test_remove_series_notes_after_values(sample_series: Series) -> None: result = _remove_series_notes_after_values(sample_series) expected = pd.Series( [ @@ -238,19 +238,25 @@ def test_remove_series_notes_after_values(sample_series): pd.testing.assert_series_equal(result, expected) -def test_remove_series_notes_after_values_with_special_characters(): +def test_remove_series_notes_after_values_with_special_characters() -> None: unsanitised = pd.Series( [ - "4758 (Marinus Link Pty Ltd and TasNetworks have advised that $534 million, " - "in $2023, of this amount relates to approved early works and other incurred " - "costs that should be excluded from the cost estimate for the 2026 ISP in " - "accordance with the AER's CBA Guidelines. AEMO has removed this from the " - "estimate of $5035 million in $2023, and has then adjusted to $2025.)", - "7035 (Transgrid has advised $565 million of this amount relates to approved " - "early works and other incurred costs that should be excluded from the total " - "cost estimate of $7600 million for the 2026 ISP.)", - "2431 (This figure reflects the estimate from Option 2 with a portion costed " - "at Class 5b removed.)", + ( + "4758 (Marinus Link Pty Ltd and TasNetworks have advised that $534 million, " + "in $2023, of this amount relates to approved early works and other incurred " + "costs that should be excluded from the cost estimate for the 2026 ISP in " + "accordance with the AER's CBA Guidelines. AEMO has removed this from the " + "estimate of $5035 million in $2023, and has then adjusted to $2025.)" + ), + ( + "7035 (Transgrid has advised $565 million of this amount relates to approved " + "early works and other incurred costs that should be excluded from the total " + "cost estimate of $7600 million for the 2026 ISP.)" + ), + ( + "2431 (This figure reflects the estimate from Option 2 with a portion costed " + "at Class 5b removed.)" + ), "1749.5 (only part of this figure is included)", ] ) @@ -259,7 +265,7 @@ def test_remove_series_notes_after_values_with_special_characters(): pd.testing.assert_series_equal(result, expected) -def test_remove_series_bracketed_footnotes(): +def test_remove_series_bracketed_footnotes() -> None: unsanitised = pd.Series( [ "750[footnote14]", @@ -308,8 +314,10 @@ def test_remove_series_bracketed_footnotes(): ("0.16 (apply from 5,400 MW)", False, "0.16"), # 6.0 Build limits, decimal value ( # 6.0 Network Capability. The digit after the note is a footnote reference, not # a second value, so the separator between them may not contain letters. - "400 (with VNI SIPS) - Note 8 (Snowy 2.0 generation or pump load <= 660 " - "- Note 11)", + ( + "400 (with VNI SIPS) - Note 8 (Snowy 2.0 generation or pump load <= 660 " + "- Note 11)" + ), False, "400", ), @@ -341,10 +349,12 @@ def test_remove_series_bracketed_footnotes(): ] -def test_multiple_values_with_notes_detection_and_sanitisation(): +def test_multiple_values_with_notes_detection_and_sanitisation() -> None: """Cells holding two values are detected and kept whole; cells holding one value and a note are still cut down to that value.""" - cells, fires, sanitised = (list(field) for field in zip(*MULTIPLE_VALUE_CASES)) + cells, fires, sanitised = ( + list(field) for field in zip(*MULTIPLE_VALUE_CASES, strict=True) + ) unsanitised = pd.Series(cells) pd.testing.assert_series_equal( _where_multiple_values_with_notes(unsanitised), pd.Series(fires) @@ -354,7 +364,7 @@ def test_multiple_values_with_notes_detection_and_sanitisation(): ) -def test_where_multiple_values_with_notes_on_mixed_and_index_input(): +def test_where_multiple_values_with_notes_on_mixed_and_index_input() -> None: """Columns reaching the sanitisers hold a mix of strings, numbers and nulls, and the sanitisers are also applied to a `pandas.Index` of column names.""" series = pd.Series(["250 (generation) 325 (pump)", "250 (generation)", 42.0, None]) @@ -366,7 +376,7 @@ def test_where_multiple_values_with_notes_on_mixed_and_index_input(): assert list(_where_multiple_values_with_notes(index)) == [True, False] -def test_values_casting_and_sanitisation_leaves_multi_value_column_as_text(): +def test_values_casting_and_sanitisation_leaves_multi_value_column_as_text() -> None: """A column containing a multi-value cell cannot be cast to a numeric type, which is the signal to consumers that the cell holds more than one value.""" df = pd.DataFrame({"capacity": ["250 (generation) 325 (pump)", "500 (generation)"]}) diff --git a/tests/test_validation.py b/tests/test_validation.py index 5289fd3..af5c4d8 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -13,10 +13,10 @@ from pydantic import ValidationError from isp_workbook_parser.config_model import CheckName, TableConfig -from isp_workbook_parser.parser import TableConfigError +from isp_workbook_parser.parser import Parser, TableConfigError -def test_end_row_not_on_sheet_throws_error(workbook_v6): +def test_end_row_not_on_sheet_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Aggregated energy storages", @@ -31,7 +31,7 @@ def test_end_row_not_on_sheet_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_first_header_row_not_on_sheet_throws_error(workbook_v6): +def test_first_header_row_not_on_sheet_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Aggregated energy storages", @@ -44,7 +44,7 @@ def test_first_header_row_not_on_sheet_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_first_column_not_on_sheet_throws_error(workbook_v6): +def test_first_column_not_on_sheet_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Aggregated energy storages", @@ -59,7 +59,7 @@ def test_first_column_not_on_sheet_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_last_column_not_on_sheet_throws_error(workbook_v6): +def test_last_column_not_on_sheet_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Aggregated energy storages", @@ -74,7 +74,7 @@ def test_last_column_not_on_sheet_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_last_column_empty_throws_error(workbook_v6): +def test_last_column_empty_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Generation limits", @@ -87,7 +87,7 @@ def test_last_column_empty_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_end_row_runs_into_another_table_throws_error(workbook_v6): +def test_end_row_runs_into_another_table_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Aggregated energy storages", @@ -103,7 +103,7 @@ def test_end_row_runs_into_another_table_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_end_row_runs_into_notes_throws_error(workbook_v6): +def test_end_row_runs_into_notes_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -118,7 +118,7 @@ def test_end_row_runs_into_notes_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_first_header_row_too_late_throws_error(workbook_v6): +def test_first_header_row_too_late_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Generator Reliability Settings", @@ -131,7 +131,7 @@ def test_first_header_row_too_late_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_end_row_too_soon_throws_error(workbook_v6): +def test_end_row_too_soon_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -146,7 +146,7 @@ def test_end_row_too_soon_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_end_column_too_soon_throws_error(workbook_v6): +def test_end_column_too_soon_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -161,7 +161,7 @@ def test_end_column_too_soon_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_start_column_too_far_throws_error(workbook_v6): +def test_start_column_too_far_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -176,7 +176,7 @@ def test_start_column_too_far_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_duplicate_column_names_throws_error(workbook_v6): +def test_duplicate_column_names_throws_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -189,7 +189,7 @@ def test_duplicate_column_names_throws_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_good_config_throws_no_error(workbook_v6): +def test_good_config_throws_no_error(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -200,14 +200,14 @@ def test_good_config_throws_no_error(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_incorrect_table_name_throws_error(workbook_v6): +def test_incorrect_table_name_throws_error(workbook_v6: Parser) -> None: error_message = "The table_name (affine_heat_rates_new_entrant) provided is not in the config for this workbook version. Did you mean 'affine_heat_rates_new_entrants'?" error_message = re.escape(error_message) with pytest.raises(ValueError, match=error_message): workbook_v6.get_table("affine_heat_rates_new_entrant") -def test_skip_checks_silences_named_check(workbook_v6): +def test_skip_checks_silences_named_check(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -219,7 +219,7 @@ def test_skip_checks_silences_named_check(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_skip_checks_does_not_silence_other_checks(workbook_v6): +def test_skip_checks_does_not_silence_other_checks(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", @@ -235,7 +235,7 @@ def test_skip_checks_does_not_silence_other_checks(workbook_v6): workbook_v6.get_table_from_config(table_config) -def test_skip_checks_invalid_check_name_throws_error(): +def test_skip_checks_invalid_check_name_throws_error() -> None: with pytest.raises(ValidationError): TableConfig( name="DUMMY", @@ -247,7 +247,7 @@ def test_skip_checks_invalid_check_name_throws_error(): ) -def test_skippable_check_names_match_config_literal(workbook_v6): +def test_skippable_check_names_match_config_literal(workbook_v6: Parser) -> None: table_config = TableConfig( name="DUMMY", sheet_name="Network Capability", diff --git a/tests/test_workbook_sheets_exist_check.py b/tests/test_workbook_sheets_exist_check.py index c29da09..0d29dae 100644 --- a/tests/test_workbook_sheets_exist_check.py +++ b/tests/test_workbook_sheets_exist_check.py @@ -11,7 +11,7 @@ from isp_workbook_parser.parser import TableConfigError -def test_end_row_not_on_sheet_throws_error(): +def test_end_row_not_on_sheet_throws_error() -> None: with pytest.raises(TableConfigError): Parser( "tests/test_data/2024-isp-inputs-and-assumptions-workbook-missing-sheets.xlsx"