More ruff fixes

src/isp_workbook_parser/__init__.py

@@ -7,11 +7,10 @@
 
 import pandas as pd
 
-from .config_model import TableConfig as TableConfig
-from .config_model import load_yaml as load_yaml
+from .config_model import TableConfig, load_yaml
 from .parser import Parser
-from .read_table import read_table as read_table
+from .read_table import read_table
 
 __all__ = ["Parser", "TableConfig", "load_yaml", "read_table"]
 
-pd.set_option("future.no_silent_downcasting", True)
+pd.set_option("future.no_silent_downcasting", True)  # noqa: FBT003

src/isp_workbook_parser/parser.py

@@ -114,7 +114,7 @@ def _determine_config_path(
 
     def _check_version_is_supported(self, config_path: Path) -> None:
         """Check the default config directory contains a subdirectory that matches the workbook version number."""
-        versions = [dir.name for dir in config_path.iterdir()]
+        versions = [directory.name for directory in config_path.iterdir()]
         if self.workbook_version not in versions:
             msg = f"The workbook version {self.workbook_version} is not supported."
             raise ValueError(msg)
@@ -509,7 +509,7 @@ def _postprocess_percentage_columns_between_0_and_100(
                     if isinstance(sr, int) and cell.row == sr:
                         skipped_rows += 1
                         continue
-                if isinstance(cell.value, (int, float)) and "%" in cell.number_format:
+                if isinstance(cell.value, int | float) and "%" in cell.number_format:
                     percentage_cells.append(
                         (
                             cell.row - min_row - skipped_rows,
@@ -553,7 +553,7 @@ def get_table_names(self) -> list[str]:
         return self.table_names_by_sheet
 
     def get_table_from_config(
-        self, table_config: TableConfig, config_checks: bool = True
+        self, table_config: TableConfig, *, config_checks: bool = True
     ) -> pd.DataFrame:
         """Retrieve a table from the assumptions workbook using the config provided and returns as pd.DataFrame.
 
@@ -603,7 +603,7 @@ def get_table_from_config(
             self._check_table(data, table_config)
         return data
 
-    def get_table(self, table_name: str, config_checks: bool = True) -> pd.DataFrame:
+    def get_table(self, table_name: str, *, config_checks: bool = True) -> pd.DataFrame:
         """Retrieve a table from the assumptions workbook and returns as `pd.DataFrame`.
 
         Examples:
@@ -643,6 +643,7 @@ def save_tables(
         self,
         directory: str | Path,
         tables: list[str] | str = "all",
+        *,
         config_checks: bool = True,
     ) -> None:
         """Save tables from the provided workbook to the specified directory as CSV files.
@@ -672,7 +673,7 @@ def save_tables(
             msg = "The path provided is not a directory."
             raise ValueError(msg)
 
-        if not (isinstance(tables, (str, list))):
+        if not (isinstance(tables, str | list)):
             msg = "The parameter tables must be provided as str or list[str]."
             raise TypeError(msg)
 

src/isp_workbook_parser/read_table.py

@@ -132,7 +132,10 @@ def read_table(workbook_file: pd.ExcelFile, table: TableConfig) -> pd.DataFrame:
         series[series != ""] = "_" + series[series != ""]
     merged_headers = ffilled_initial_header.str.cat(filled_headers)
     df_cleaned = _build_cleaned_dataframe(
-        df_initial, header_rows_in_table, merged_headers, table.forward_fill_values
+        df_initial,
+        header_rows_in_table,
+        merged_headers,
+        forward_fill_values=table.forward_fill_values,
     )
     if table.skip_rows:
         df_cleaned = _skip_rows_in_dataframe(
@@ -202,6 +205,7 @@ def _build_cleaned_dataframe(
     df_initial: pd.DataFrame,
     header_rows_in_table: int,
     new_headers: pd.Series,
+    *,
     forward_fill_values: bool,
 ) -> pd.DataFrame:
     """

src/isp_workbook_parser/sanitisers.py

@@ -64,7 +64,7 @@ def _values_casting_and_sanitisation(df: pd.DataFrame) -> pd.DataFrame:
     will return `pd.NA`
     """
     df = _replace_dataframe_hyphens_with_na(df)
-    for object_col in df.dtypes[df.dtypes == "object"].keys():
+    for object_col in df.dtypes[df.dtypes == "object"].keys():  # noqa: SIM118
         try:
             df.loc[:, object_col] = pd.to_numeric(df[object_col])
         except (ValueError, TypeError):

tests/conftest.py

@@ -17,7 +17,7 @@ def workbook_v6() -> Parser:
 
 
 @pytest.fixture(scope="module")
-def sample_series():
+def sample_series() -> pd.Series:
     return pd.Series(
         [
             "First line\nSecond line",

tests/test_packaged_table_configs.py

@@ -15,8 +15,8 @@
 
 
 @pytest.mark.parametrize("workbook_version_folder", list(workbook_path.iterdir()))
-def test_packaged_table_configs_for_each_version(workbook_version_folder: Path):
-    xl_file = [file for file in workbook_version_folder.glob("[!.]*.xls*")]
+def test_packaged_table_configs_for_each_version(workbook_version_folder: Path) -> None:
+    xl_file = list(workbook_version_folder.glob("[!.]*.xls*"))
     assert len(xl_file) == 1, (
         f"There should only be one Excel workbook in each version sub-directory, got {xl_file}"
     )
@@ -34,9 +34,7 @@ def test_packaged_table_configs_for_each_version(workbook_version_folder: Path):
     for index, value in enumerate(sheet_header_end_row_combos):
         if sheet_header_end_row_combos.count(value) > 1:
             duplicate_configs.append(table_names[index])
-    if len(duplicate_configs) > 0:
-        print(duplicate_configs)
-    assert len(duplicate_configs) == 0
+    assert len(duplicate_configs) == 0, duplicate_configs
 
     save_dir = Path(f"example_output/{workbook.workbook_version}")
     save_dir.mkdir(parents=True, exist_ok=True)
@@ -46,14 +44,14 @@ def test_packaged_table_configs_for_each_version(workbook_version_folder: Path):
             table = workbook.get_table(table_name)
             save_path = save_dir / Path(f"{table_name}.csv")
             table.to_csv(save_path, index=False)
-        except Exception as e:
+        except (FileNotFoundError, PermissionError, OSError) as e:
             error_tables[table_name] = e
     if error_tables:
         error_str = ""
-        for key in error_tables:
-            error_str += key + ":" + str(error_tables[key]) + "\n"
+        for key, value in error_tables.items():
+            error_str += key + ":" + str(value) + "\n"
         raise TableLoadError(error_str)
 
 
 class TableLoadError(Exception):
-    """Exception to throw if table loading fails"""
+    """Exception to throw if table loading fails."""

tests/test_read_table_functionality.py

@@ -6,9 +6,10 @@
 # (at your option) any later version.
 
 from isp_workbook_parser.config_model import TableConfig
+from isp_workbook_parser.parser import Parser
 
 
-def test_skip_single_row_in_single_header_row_table(workbook_v6):
+def test_skip_single_row_in_single_header_row_table(workbook_v6: Parser) -> None:
     table_config = TableConfig(
         name="build_cost_current_policies",
         sheet_name="Build costs",
@@ -22,7 +23,7 @@ def test_skip_single_row_in_single_header_row_table(workbook_v6):
     assert df[df.Technology.str.contains("Hydrogen")].empty
 
 
-def test_skip_multiple_rows_in_single_header_row_table(workbook_v6):
+def test_skip_multiple_rows_in_single_header_row_table(workbook_v6: Parser) -> None:
     table_config = TableConfig(
         name="existing_generator_maintenance_rates",
         sheet_name="Maintenance",
@@ -37,7 +38,7 @@ def test_skip_multiple_rows_in_single_header_row_table(workbook_v6):
     assert df[df["Generator type"].str.contains("Coal")].empty
 
 
-def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6):
+def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6: Parser) -> None:
     table_config = TableConfig(
         name="wind_high_capacity_factors",
         sheet_name="Capacity Factors ",
@@ -52,7 +53,7 @@ def test_skip_multiple_rows_in_multiple_header_row_table(workbook_v6):
     assert df[df["Wind High_REZ ID"].str.contains("V")].empty
 
 
-def test_no_forward_fill_in_rows(workbook_v6):
+def test_no_forward_fill_in_rows(workbook_v6: Parser) -> None:
     table_config = TableConfig(
         name="outages_new_entrants",
         sheet_name="Generator Reliability Settings",

tests/test_sanitisers.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 
 import pandas as pd
-import pytest
+from pandas import Series
 
 from isp_workbook_parser.sanitisers import (
     _extract_numeric_value_millions,
@@ -25,7 +25,7 @@
 )
 
 
-def test_sanitisation_on_flow_path_transfer_capability():
+def test_sanitisation_on_flow_path_transfer_capability() -> None:
     unsanitised = pd.read_csv(Path("tests", "test_data", "unsanitised.csv"))
     expected = pd.read_csv(Path("tests", "test_data", "sanitised.csv"))
     # handle carriage return on Windows
@@ -38,7 +38,7 @@ def test_sanitisation_on_flow_path_transfer_capability():
     pd.testing.assert_frame_equal(test_sanitised, expected, check_dtype=False)
 
 
-def test_replace_series_newlines_with_whitespace(sample_series):
+def test_replace_series_newlines_with_whitespace(sample_series: Series) -> None:
     result = _replace_series_newlines_with_whitespace(sample_series)
     expected = pd.Series(
         [
@@ -63,7 +63,7 @@ def test_replace_series_newlines_with_whitespace(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_extract_numeric_value_millions(sample_series):
+def test_extract_numeric_value_millions(sample_series: Series) -> None:
     result = _extract_numeric_value_millions(sample_series)
     expected = pd.Series(
         [
@@ -88,7 +88,7 @@ def test_extract_numeric_value_millions(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_double_whitespaces(sample_series):
+def test_remove_series_double_whitespaces(sample_series: Series) -> None:
     result = _remove_series_double_whitespaces(sample_series)
     expected = pd.Series(
         [
@@ -113,7 +113,7 @@ def test_remove_series_double_whitespaces(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_trailing_asterisks(sample_series):
+def test_remove_series_trailing_asterisks(sample_series: Series) -> None:
     result = _remove_series_trailing_asterisks(sample_series)
     expected = pd.Series(
         [
@@ -138,7 +138,7 @@ def test_remove_series_trailing_asterisks(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_trailing_footnotes(sample_series):
+def test_remove_series_trailing_footnotes(sample_series: Series) -> None:
     result = _remove_series_trailing_footnotes(sample_series)
     expected = pd.Series(
         [
@@ -163,7 +163,7 @@ def test_remove_series_trailing_footnotes(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_strip_series_whitespaces(sample_series):
+def test_strip_series_whitespaces(sample_series: Series) -> None:
     result = _strip_series_whitespaces(sample_series)
     expected = pd.Series(
         [
@@ -188,7 +188,7 @@ def test_strip_series_whitespaces(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_thousands_commas(sample_series):
+def test_remove_series_thousands_commas(sample_series: Series) -> None:
     result = _remove_series_thousands_commas(sample_series)
     expected = pd.Series(
         [
@@ -213,7 +213,7 @@ def test_remove_series_thousands_commas(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_notes_after_values(sample_series):
+def test_remove_series_notes_after_values(sample_series: Series) -> None:
     result = _remove_series_notes_after_values(sample_series)
     expected = pd.Series(
         [
@@ -238,19 +238,25 @@ def test_remove_series_notes_after_values(sample_series):
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_notes_after_values_with_special_characters():
+def test_remove_series_notes_after_values_with_special_characters() -> None:
     unsanitised = pd.Series(
         [
-            "4758 (Marinus Link Pty Ltd and TasNetworks have advised that $534 million, "
-            "in $2023, of this amount relates to approved early works and other incurred "
-            "costs that should be excluded from the cost estimate for the 2026 ISP in "
-            "accordance with the AER's CBA Guidelines. AEMO has removed this from the "
-            "estimate of $5035 million in $2023, and has then adjusted to $2025.)",
-            "7035 (Transgrid has advised $565 million of this amount relates to approved "
-            "early works and other incurred costs that should be excluded from the total "
-            "cost estimate of $7600 million for the 2026 ISP.)",
-            "2431 (This figure reflects the estimate from Option 2 with a portion costed "
-            "at Class 5b removed.)",
+            (
+                "4758 (Marinus Link Pty Ltd and TasNetworks have advised that $534 million, "
+                "in $2023, of this amount relates to approved early works and other incurred "
+                "costs that should be excluded from the cost estimate for the 2026 ISP in "
+                "accordance with the AER's CBA Guidelines. AEMO has removed this from the "
+                "estimate of $5035 million in $2023, and has then adjusted to $2025.)"
+            ),
+            (
+                "7035 (Transgrid has advised $565 million of this amount relates to approved "
+                "early works and other incurred costs that should be excluded from the total "
+                "cost estimate of $7600 million for the 2026 ISP.)"
+            ),
+            (
+                "2431 (This figure reflects the estimate from Option 2 with a portion costed "
+                "at Class 5b removed.)"
+            ),
             "1749.5 (only part of this figure is included)",
         ]
     )
@@ -259,7 +265,7 @@ def test_remove_series_notes_after_values_with_special_characters():
     pd.testing.assert_series_equal(result, expected)
 
 
-def test_remove_series_bracketed_footnotes():
+def test_remove_series_bracketed_footnotes() -> None:
     unsanitised = pd.Series(
         [
             "750[footnote14]",
@@ -308,8 +314,10 @@ def test_remove_series_bracketed_footnotes():
     ("0.16 (apply from 5,400 MW)", False, "0.16"),  # 6.0 Build limits, decimal value
     (  # 6.0 Network Capability. The digit after the note is a footnote reference, not
         # a second value, so the separator between them may not contain letters.
-        "400 (with VNI SIPS) - Note 8 (Snowy 2.0 generation or pump load <= 660 "
-        "- Note 11)",
+        (
+            "400 (with VNI SIPS) - Note 8 (Snowy 2.0 generation or pump load <= 660 "
+            "- Note 11)"
+        ),
         False,
         "400",
     ),
@@ -341,10 +349,12 @@ def test_remove_series_bracketed_footnotes():
 ]
 
 
-def test_multiple_values_with_notes_detection_and_sanitisation():
+def test_multiple_values_with_notes_detection_and_sanitisation() -> None:
     """Cells holding two values are detected and kept whole; cells holding one value
     and a note are still cut down to that value."""
-    cells, fires, sanitised = (list(field) for field in zip(*MULTIPLE_VALUE_CASES))
+    cells, fires, sanitised = (
+        list(field) for field in zip(*MULTIPLE_VALUE_CASES, strict=True)
+    )
     unsanitised = pd.Series(cells)
     pd.testing.assert_series_equal(
         _where_multiple_values_with_notes(unsanitised), pd.Series(fires)
@@ -354,7 +364,7 @@ def test_multiple_values_with_notes_detection_and_sanitisation():
     )
 
 
-def test_where_multiple_values_with_notes_on_mixed_and_index_input():
+def test_where_multiple_values_with_notes_on_mixed_and_index_input() -> None:
     """Columns reaching the sanitisers hold a mix of strings, numbers and nulls, and
     the sanitisers are also applied to a `pandas.Index` of column names."""
     series = pd.Series(["250 (generation) 325 (pump)", "250 (generation)", 42.0, None])
@@ -366,7 +376,7 @@ def test_where_multiple_values_with_notes_on_mixed_and_index_input():
     assert list(_where_multiple_values_with_notes(index)) == [True, False]
 
 
-def test_values_casting_and_sanitisation_leaves_multi_value_column_as_text():
+def test_values_casting_and_sanitisation_leaves_multi_value_column_as_text() -> None:
     """A column containing a multi-value cell cannot be cast to a numeric type, which
     is the signal to consumers that the cell holds more than one value."""
     df = pd.DataFrame({"capacity": ["250 (generation) 325 (pump)", "500 (generation)"]})