From d03aacdabc8ba7c4d071defec3514ef2100b716d Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Wed, 2 Sep 2026 21:12:35 +1000 Subject: [PATCH 01/18] Use ruff v0.16.5 (the latest). --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7912a80..2b209ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: exclude: ^(example_input_data/.*|example_parsed_data/.*)$ - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.15.20 + rev: v0.16.5 hooks: # Run the linter. - id: ruff From 0cdaa0d4fff6d1998984ceb83d325f0e9ff09a67 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:14:00 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 256 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 133 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index d5785d7..dfab17b 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,9 @@ The zipped data is also archived in publicly accessible object storage ([data.op ```python from isp_trace_parser.remote import fetch_trace_data -fetch_trace_data("full", dataset_src="isp_2024", save_directory="data/archive", data_format="archive") +fetch_trace_data( + "full", dataset_src="isp_2024", save_directory="data/archive", data_format="archive" +) ``` This will download all the archived zip files into the provided directory with the following structure: @@ -84,7 +86,12 @@ Trace data that has been processed into the hive-partitioned format is also avai from isp_trace_parser.remote import fetch_trace_data # Download example dataset (2018 reference year only) -fetch_trace_data("example", dataset_src="isp_2024", save_directory="data/trace_data", data_format="processed") +fetch_trace_data( + "example", + dataset_src="isp_2024", + save_directory="data/trace_data", + data_format="processed", +) ``` This will download the processed parquet files with the following structure: @@ -147,42 +154,42 @@ from isp_trace_parser import ( parse_wind_traces, WindMetadataFilter, parse_demand_traces, - DemandMetadataFilter + DemandMetadataFilter, ) # Note: to not filter on a component of the metadata it can be excluded from the filter definition. filters = SolarMetadataFilter(file_type=["project"]) parse_solar_traces( - input_directory='', - parsed_directory='', - filters = filters, + input_directory="", + parsed_directory="", + filters=filters, ) filters = WindMetadataFilter(file_type=["project"]) parse_wind_traces( - input_directory='', - parsed_directory='', - filters = filters, + input_directory="", + parsed_directory="", + filters=filters, ) filters = SolarMetadataFilter(file_type=["zone"]) parse_solar_traces( - input_directory='', - parsed_directory='', - filters = filters, + input_directory="", + parsed_directory="", + filters=filters, ) filters = WindMetadataFilter(file_type=["zone"]) parse_wind_traces( - input_directory='', - parsed_directory='', - filters = filters, + input_directory="", + parsed_directory="", + filters=filters, ) parse_demand_traces( - input_directory='', - parsed_directory='', + input_directory="", + parsed_directory="", ) ``` @@ -196,14 +203,18 @@ The following code illustrates how the parsed parquet files can be consolidated from isp_trace_parser import optimise_parquet # For optimising `zone` and `project`, suggest partitioning on reference year -optimise_parquet.partition_traces_by_columns(input_directory="", - output_directory="", - partition_cols=["reference_year"]) +optimise_parquet.partition_traces_by_columns( + input_directory="", + output_directory="", + partition_cols=["reference_year"], +) # For optimising `demand`, suggest partitioning on scenario and reference year -optimise_parquet.partition_traces_by_columns(input_directory="", - output_directory="", - partition_cols=["scenario", "reference_year"]) +optimise_parquet.partition_traces_by_columns( + input_directory="", + output_directory="", + partition_cols=["scenario", "reference_year"], +) ``` @@ -228,11 +239,9 @@ from isp_trace_parser import get_data # Define location of parsed data. -parsed_workbook_data = Path( - "/path/to/parsed/workbook/data" -) +parsed_workbook_data = Path("/path/to/parsed/workbook/data") -parsed_solar_data = Path('path/to/parsed/solar/traces') +parsed_solar_data = Path("path/to/parsed/solar/traces") # Wind and solar generator names are stored across four IASR workbook tables @@ -259,30 +268,29 @@ generator_tables = [ existing_generators, committed_generators, anticipated_generators, - additional_generators + additional_generators, ] for table in generator_tables: - table.rename( - columns={table.columns.values[0]: "Generator"}, - inplace=True - ) + table.rename(columns={table.columns.values[0]: "Generator"}, inplace=True) generator_data = pd.concat(generator_tables) # The names of solar and wind projects/generators can be retrieved by filtering -solar_generators = generator_data[generator_data['Technology type'] == 'Large scale Solar PV'] +solar_generators = generator_data[ + generator_data["Technology type"] == "Large scale Solar PV" +] -solar_generator_names = list(solar_generators['Generator']) +solar_generator_names = list(solar_generators["Generator"]) print(solar_generator_names) # ['Avonlie Solar Farm', 'Beryl Solar Farm', 'Bomen Solar Farm', 'Broken Hill Solar Farm' . . . -wind_generators = generator_data[generator_data['Technology type'] == 'Wind'] +wind_generators = generator_data[generator_data["Technology type"] == "Wind"] -wind_generator_names = list(wind_generators['Generator']) +wind_generator_names = list(wind_generators["Generator"]) print(wind_generator_names) # ['Bango 973 Wind Farm', 'Bango 999 Wind Farm', 'Boco Rock Wind Farm', 'Bodangora Wind Farm' . . . @@ -295,9 +303,8 @@ solar_traces = get_data.get_project_single_reference_year( end_year=2030, reference_year=2011, project=solar_generator_names, - directory="parsed_project_data" - ) - + directory="parsed_project_data", +) ``` @@ -316,32 +323,32 @@ from isp_trace_parser import get_data # Define location of parsed data. -parsed_workbook_data = Path( - "/path/to/parsed/workbook/data" -) +parsed_workbook_data = Path("/path/to/parsed/workbook/data") -parsed_wind_data = Path('path/to/parsed/wind/traces') +parsed_wind_data = Path("path/to/parsed/wind/traces") # ISP REZ IDs and wind resource types can be retrieved from the parsed workbook data -build_limits = pd.read_csv( - parsed_workbook_data / Path("initial_build_limits.csv") -) +build_limits = pd.read_csv(parsed_workbook_data / Path("initial_build_limits.csv")) # If a unit has a non-nan offshore floating build limit then it will have the wind # resource qualities WFL and WFX (wind offshore floating and wind offshore fixed). -offshore_rezs = build_limits[~build_limits["Wind generation total limits (MW)_Offshore -floating"].isna()] +offshore_rezs = build_limits[ + ~build_limits["Wind generation total limits (MW)_Offshore -floating"].isna() +] -print(list(offshore_rezs['REZ ID'])) +print(list(offshore_rezs["REZ ID"])) # ['N10', 'N11', 'V7', 'V8', 'S10', 'T4'] # If a unit has a nonzero high build limit then it will be an on shore REZ and have the wind # resource qualities WH and WM (wind high and wind medium). -onshore_rezs = build_limits[build_limits["Wind generation total limits (MW)_High"] > 0.1] +onshore_rezs = build_limits[ + build_limits["Wind generation total limits (MW)_High"] > 0.1 +] -print(list(onshore_rezs['REZ ID'])) +print(list(onshore_rezs["REZ ID"])) # ['Q1', 'Q2', 'Q3', 'Q4', 'Q5', 'Q6', . . . # These sets of onshore and offshore REZ IDs can the be used to retrieve a dataframes containing all relevant traces, which can be filtered by REZ name using the 'zone' column" @@ -350,20 +357,19 @@ wind_offshore_rez_traces = get_data.get_zone_single_reference_year( start_year=2025, end_year=2026, reference_year=2011, - zone=list(offshore_rezs['REZ ID']), + zone=list(offshore_rezs["REZ ID"]), resource_type="WFL", - directory="parsed_zone_data" + directory="parsed_zone_data", ) wind_onshore_rez_traces = get_data.get_zone_single_reference_year( start_year=2025, end_year=2026, reference_year=2011, - zone=list(onshore_rezs['REZ ID']), + zone=list(onshore_rezs["REZ ID"]), resource_type="WH", - directory="parsed_zone_data" + directory="parsed_zone_data", ) - ``` @@ -381,24 +387,22 @@ from isp_trace_parser import get_data # Define location of parsed data. -parsed_workbook_data = Path( - "/path/to/parsed/workbook/data" -) +parsed_workbook_data = Path("/path/to/parsed/workbook/data") -parsed_solar_data = Path('path/to/parsed/wind/traces') +parsed_solar_data = Path("path/to/parsed/wind/traces") # ISP REZ IDs and types can be retrieved from the parsed workbook data -build_limits = pd.read_csv( - parsed_workbook_data / Path("initial_build_limits.csv") -) +build_limits = pd.read_csv(parsed_workbook_data / Path("initial_build_limits.csv")) # If a unit has a nonzero high build limit then it will be an onshore REZ and have the # solar traces for SAT (single axis tracking) and CST (concentrating solar thermal). -onshore_solar_rezs = build_limits[build_limits["Solar PV plus Solar thermal Limits (MW)_Solar"] > 0.1] +onshore_solar_rezs = build_limits[ + build_limits["Solar PV plus Solar thermal Limits (MW)_Solar"] > 0.1 +] -print(list(onshore_solar_rezs['REZ ID'])) +print(list(onshore_solar_rezs["REZ ID"])) # ['Q1', 'Q2', 'Q3', 'Q4', 'Q5', 'Q6', . . . # The set of REZ IDs can be used to retrieves a dataframe containing all REZ traces, which can be filtered by REZ name using the 'zone' column" @@ -407,11 +411,10 @@ single_axis_tracking_traces = get_data.get_zone_single_reference_year( start_year=2025, end_year=2026, reference_year=2011, - zone=onshore_solar_rezs['REZ ID'], + zone=onshore_solar_rezs["REZ ID"], resource_type="SAT", - directory="parsed_zone_data" + directory="parsed_zone_data", ) - ``` @@ -430,17 +433,13 @@ from isp_trace_parser import get_data # Define location of parsed data. -parsed_workbook_data = Path( - "/path/to/parsed/workbook/data" -) +parsed_workbook_data = Path("/path/to/parsed/workbook/data") -parsed_demand_data = Path('path/to/parsed/demand/traces') +parsed_demand_data = Path("path/to/parsed/demand/traces") # ISP Subregion ID can be retrieved from renewable energy zones table -rez_definitions = pd.read_csv( - parsed_workbook_data / Path("renewable_energy_zones.csv") -) +rez_definitions = pd.read_csv(parsed_workbook_data / Path("renewable_energy_zones.csv")) subregions = list(set(rez_definitions["ISP Sub-region"])) print(subregions) @@ -456,8 +455,8 @@ demand_trace = get_demand_single_reference_year( subregion=subregions, demand_type="OPSO_MODELLING", poe="POE50", - directory="parsed_data/demand" - ) + directory="parsed_data/demand", +) ``` @@ -473,18 +472,22 @@ Once trace data has been parsed it can also queried using legacy API functionali ```python from isp_trace_parser import get_data -solar_project_trace_single_reference_year = get_data.solar_project_single_reference_year( - start_year=2022, - end_year=2024, - reference_year=2011, - project='Adelaide Desalination Plant Solar Farm', - directory='example_project_data/' +solar_project_trace_single_reference_year = ( + get_data.solar_project_single_reference_year( + start_year=2022, + end_year=2024, + reference_year=2011, + project="Adelaide Desalination Plant Solar Farm", + directory="example_project_data/", + ) ) -solar_project_trace_many_reference_years = get_data.solar_project_multiple_reference_years( - reference_years={2022: 2011, 2024: 2012}, - project='Adelaide Desalination Plant Solar Farm', - directory='example_project_data/' +solar_project_trace_many_reference_years = ( + get_data.solar_project_multiple_reference_years( + reference_years={2022: 2011, 2024: 2012}, + project="Adelaide Desalination Plant Solar Farm", + directory="example_project_data/", + ) ) ``` @@ -496,20 +499,21 @@ solar_project_trace_many_reference_years = get_data.solar_project_multiple_refer ```python from isp_trace_parser import get_data + solar_rez_trace_single_reference_years = get_data.solar_area_single_reference_year( start_year=2022, end_year=2024, reference_year=2011, - area='Q1', - technology='SAT', - directory='example_rez_data/' + area="Q1", + technology="SAT", + directory="example_rez_data/", ) solar_rez_trace_many_reference_years = get_data.solar_area_multiple_reference_years( reference_years={2022: 2011, 2024: 2012}, - area='Q1', - technology='SAT', - directory='example_rez_data/' + area="Q1", + technology="SAT", + directory="example_rez_data/", ) ``` @@ -521,18 +525,21 @@ solar_rez_trace_many_reference_years = get_data.solar_area_multiple_reference_ye ```python from isp_trace_parser import get_data + wind_project_trace_single_reference_years = get_data.wind_project_single_reference_year( start_year=2022, end_year=2024, reference_year=2011, - project='Bango 973 Wind Farm', - directory='parsed_project_data/' + project="Bango 973 Wind Farm", + directory="parsed_project_data/", ) -wind_project_trace_many_reference_years = get_data.wind_project_multiple_reference_years( - reference_years={2022: 2011, 2024: 2012}, - project='Bango 973 Wind Farm', - directory='parsed_project_data/' +wind_project_trace_many_reference_years = ( + get_data.wind_project_multiple_reference_years( + reference_years={2022: 2011, 2024: 2012}, + project="Bango 973 Wind Farm", + directory="parsed_project_data/", + ) ) ``` @@ -544,20 +551,21 @@ wind_project_trace_many_reference_years = get_data.wind_project_multiple_referen ```python from isp_trace_parser import get_data + wind_rez_trace_single_reference_years = get_data.wind_area_single_reference_year( start_year=2022, end_year=2024, reference_year=2011, - area='Q1', - resource_quality='WH', - directory='parsed_rez_data/' + area="Q1", + resource_quality="WH", + directory="parsed_rez_data/", ) wind_rez_trace_many_reference_years = get_data.wind_area_multiple_reference_years( reference_years={2022: 2011, 2024: 2012}, - area='Q1', - resource_quality='WH', - directory='parsed_rez_data/' + area="Q1", + resource_quality="WH", + directory="parsed_rez_data/", ) ``` @@ -569,26 +577,26 @@ wind_rez_trace_many_reference_years = get_data.wind_area_multiple_reference_year ```python from isp_trace_parser import get_data + demand_subregion_trace_single_reference_years = get_data.demand_single_reference_year( start_year=2024, end_year=2024, reference_year=2011, - subregion='CNSW', - scenario='Green Energy Exports', - poe='POE10', - demand_type='OPSO_MODELLING', - directory='parsed_demand_data/' + subregion="CNSW", + scenario="Green Energy Exports", + poe="POE10", + demand_type="OPSO_MODELLING", + directory="parsed_demand_data/", ) demand_subregion_trace_many_reference_years = get_data.demand_multiple_reference_years( reference_years={2024: 2011}, - subregion='CNSW', - scenario='Green Energy Exports', - poe='POE10', - demand_type='OPSO_MODELLING', - directory='parsed_demand_data/' + subregion="CNSW", + scenario="Green Energy Exports", + poe="POE10", + demand_type="OPSO_MODELLING", + directory="parsed_demand_data/", ) - ``` @@ -626,14 +634,16 @@ import polars as pl import pandas as pd from isp_trace_parser import trace_formatter -aemo_format_data = pd.DataFrame({ - 'Year': [2024, 2024], - 'Month': [6, 6], - 'Day': [1, 2], - '01': [11.2, 15.3], - '02': [30.7, 20.4], - '48': [17.1, 18.9] -}) +aemo_format_data = pd.DataFrame( + { + "Year": [2024, 2024], + "Month": [6, 6], + "Day": [1, 2], + "01": [11.2, 15.3], + "02": [30.7, 20.4], + "48": [17.1, 18.9], + } +) aemo_format_data_as_polars = pl.from_pandas(aemo_format_data) From c8ae91a2eb42701b42cdc5f07943939bcedceddc Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 14:59:24 +1000 Subject: [PATCH 03/18] Use {..} set comprehension syntax. --- src/isp_trace_parser/solar_traces.py | 2 +- src/isp_trace_parser/trace_restructure_helper_functions.py | 4 +--- src/isp_trace_parser/wind_traces.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/isp_trace_parser/solar_traces.py b/src/isp_trace_parser/solar_traces.py index 17fc881..13ebc5e 100644 --- a/src/isp_trace_parser/solar_traces.py +++ b/src/isp_trace_parser/solar_traces.py @@ -276,7 +276,7 @@ def get_unique_resource_types_in_metadata( A list of unique resource types. """ return list( - set(metadata["resource_type"] for metadata in metadata_for_trace_files.values()) + {metadata["resource_type"] for metadata in metadata_for_trace_files.values()} ) diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index 25c8229..f882398 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -99,9 +99,7 @@ def get_unique_reference_years_in_metadata( metadata_for_trace_files: dict[Path, dict[str, str]], ) -> list[str]: return list( - set( - metadata["reference_year"] for metadata in metadata_for_trace_files.values() - ) + {metadata["reference_year"] for metadata in metadata_for_trace_files.values()} ) diff --git a/src/isp_trace_parser/wind_traces.py b/src/isp_trace_parser/wind_traces.py index 775f6e5..d5c8257 100644 --- a/src/isp_trace_parser/wind_traces.py +++ b/src/isp_trace_parser/wind_traces.py @@ -323,7 +323,7 @@ def get_unique_resource_types_in_metadata( metadata_for_trace_files: dict[str:str], ) -> list: return list( - set(metadata["resource_type"] for metadata in metadata_for_trace_files.values()) + {metadata["resource_type"] for metadata in metadata_for_trace_files.values()} ) From 481b2993b07ce4d56301c1b91a91c8cd6118d30f Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:10:18 +1000 Subject: [PATCH 04/18] Fix RET class messages from Ruff. --- src/isp_trace_parser/get_data.py | 6 +++--- .../trace_restructure_helper_functions.py | 12 ++++-------- tests/create_end_to_end_test_data.py | 3 +-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/isp_trace_parser/get_data.py b/src/isp_trace_parser/get_data.py index 6d620c2..87bbd6e 100644 --- a/src/isp_trace_parser/get_data.py +++ b/src/isp_trace_parser/get_data.py @@ -37,10 +37,11 @@ def _year_range_to_dt_range( end_year, 7, 1 ) - elif year_type == "calendar": + if year_type == "calendar": return datetime.datetime(start_year, 1, 1), datetime.datetime( end_year + 1, 1, 1 ) + raise ValueError(year_type) def _query_parquet_single_reference_year( @@ -136,8 +137,7 @@ def _query_parquet_multiple_reference_years( start_year=year, end_year=year, reference_year=reference_year, **kwargs ) ) - data = pd.concat(data).reset_index(drop=True) - return data + return pd.concat(data).reset_index(drop=True) @validate_call diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index f882398..b0be7f7 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -10,14 +10,12 @@ def get_all_filepaths(directory: Path) -> list[Path]: if directory.is_dir(): return [path for path in Path(directory).rglob("*.csv") if path.is_file()] - else: - raise ValueError(f"{directory} not found.") + raise ValueError(f"{directory} not found.") def read_trace_csv(file: Path) -> pl.DataFrame: pl_types = [pl.Int64] * 3 + [pl.Float64] * 48 - data = pl.read_csv(file, schema_overrides=pl_types) - return data + return pl.read_csv(file, schema_overrides=pl_types) def read_and_format_traces(files: list[Path]) -> list[pl.DataFrame]: @@ -31,10 +29,9 @@ def read_and_format_traces(files: list[Path]) -> list[pl.DataFrame]: def calculate_average_trace(traces: list[pl.DataFrame]) -> pl.DataFrame: combined_traces = pl.concat(traces) - average_trace = combined_traces.group_by("datetime").agg( + return combined_traces.group_by("datetime").agg( [pl.col("value").mean().alias("value")] ) - return average_trace def _frame_with_metadata(trace: pl.DataFrame, file_metadata: dict) -> pl.DataFrame: @@ -87,12 +84,11 @@ def get_metadata_that_matches_trace_names( ) -> dict[Path, dict[str, str]]: if isinstance(trace_names, str): trace_names = [trace_names] - matching_meta_data = { + return { f: metadata.copy() for f, metadata in all_input_file_metadata.items() if metadata["name"] in trace_names } - return matching_meta_data def get_unique_reference_years_in_metadata( diff --git a/tests/create_end_to_end_test_data.py b/tests/create_end_to_end_test_data.py index e9f2086..b107cc4 100644 --- a/tests/create_end_to_end_test_data.py +++ b/tests/create_end_to_end_test_data.py @@ -24,8 +24,7 @@ def generate_random_data(start_year, end_year): half_hour_columns = [f"{i:02d}" for i in range(1, 49)] # Combine the date components with the random data - df = pd.concat([df, pd.DataFrame(random_data, columns=half_hour_columns)], axis=1) - return df + return pd.concat([df, pd.DataFrame(random_data, columns=half_hour_columns)], axis=1) data = generate_random_data(start_year=config.start, end_year=config.end) From 0c26802ff81749c93e6e5d9a5efc2a268e3e94cb Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:21:36 +1000 Subject: [PATCH 05/18] Fix UP class warnings from Ruff. --- src/isp_trace_parser/demand_traces.py | 19 ++++++++++--------- src/isp_trace_parser/get_data.py | 2 +- src/isp_trace_parser/optimise_parquet.py | 2 +- src/isp_trace_parser/solar_traces.py | 8 ++++---- src/isp_trace_parser/wind_traces.py | 8 ++++---- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/isp_trace_parser/demand_traces.py b/src/isp_trace_parser/demand_traces.py index ab17556..c59c4e1 100644 --- a/src/isp_trace_parser/demand_traces.py +++ b/src/isp_trace_parser/demand_traces.py @@ -42,18 +42,19 @@ class DemandMetadataFilter(BaseModel): scenario: list of scenarios, only including "Step Change", "Progressive Change", and "Green Energy Exports" poe: list of POE levels, only including "POE10" and "POE50" demand_type, list of demand types, only including "OPSO_MODELLING", "OPSO_MODELLING_PVLITE", and "PV_TOT" - reference_year: list of ints specifying reference_years + reference_year: list of ints specifying reference_subregion """ - subregion: Optional[list[str]] = None - scenario: Optional[ + years: list[str] | None = None + scenario: ( list[Literal["Step Change", "Progressive Change", "Green Energy Exports"]] - ] = None - poe: Optional[list[Literal["POE50", "POE10"]]] = None - demand_type: Optional[ - list[Literal["OPSO_MODELLING", "OPSO_MODELLING_PVLITE", "PV_TOT"]] - ] = None - reference_year: Optional[list[int]] = None + | None + ) = None + poe: list[Literal["POE50", "POE10"]] | None = None + demand_type: ( + list[Literal["OPSO_MODELLING", "OPSO_MODELLING_PVLITE", "PV_TOT"]] | None + ) = None + reference_year: list[int] | None = None @validate_call diff --git a/src/isp_trace_parser/get_data.py b/src/isp_trace_parser/get_data.py index 87bbd6e..e848c89 100644 --- a/src/isp_trace_parser/get_data.py +++ b/src/isp_trace_parser/get_data.py @@ -1,6 +1,6 @@ import datetime from pathlib import Path -from typing import List, Literal +from typing import Literal import pandas as pd import polars as pl diff --git a/src/isp_trace_parser/optimise_parquet.py b/src/isp_trace_parser/optimise_parquet.py index da00e07..327196c 100644 --- a/src/isp_trace_parser/optimise_parquet.py +++ b/src/isp_trace_parser/optimise_parquet.py @@ -23,7 +23,7 @@ def partition_traces_by_columns( input_directory: str | Path, output_directory: str | Path, partition_cols: list[str], - sort_by: Optional[list[str]] = ["datetime"], + sort_by: list[str] | None = ["datetime"], ) -> None: """Partition parquet traces by specified columns with optional sorting. diff --git a/src/isp_trace_parser/solar_traces.py b/src/isp_trace_parser/solar_traces.py index 13ebc5e..9bef13c 100644 --- a/src/isp_trace_parser/solar_traces.py +++ b/src/isp_trace_parser/solar_traces.py @@ -49,10 +49,10 @@ class SolarMetadataFilter(BaseModel): reference_year: list of ints specifying reference_years """ - name: Optional[list[str]] = None - file_type: Optional[list[Literal["zone", "project"]]] = None - resource_type: Optional[list[Literal["SAT", "FFP", "CST"]]] = None - reference_year: Optional[list[int]] = None + name: list[str] | None = None + file_type: list[Literal["zone", "project"]] | None = None + resource_type: list[Literal["SAT", "FFP", "CST"]] | None = None + reference_year: list[int] | None = None @validate_call diff --git a/src/isp_trace_parser/wind_traces.py b/src/isp_trace_parser/wind_traces.py index d5c8257..81cdb12 100644 --- a/src/isp_trace_parser/wind_traces.py +++ b/src/isp_trace_parser/wind_traces.py @@ -49,10 +49,10 @@ class WindMetadataFilter(BaseModel): reference_year: list of ints specifying reference_years """ - name: Optional[list[str]] = None - file_type: Optional[list[Literal["zone", "project"]]] = None - resource_type: Optional[list[Literal["WH", "WM", "WL", "WX", "wind"]]] = None - reference_year: Optional[list[int]] = None + name: list[str] | None = None + file_type: list[Literal["zone", "project"]] | None = None + resource_type: list[Literal["WH", "WM", "WL", "WX", "wind"]] | None = None + reference_year: list[int] | None = None @validate_call From a79d095662f78f866fc61c9f11d9181bceaaeb7c Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:28:06 +1000 Subject: [PATCH 06/18] Use list not List throughout for type annotations. --- src/isp_trace_parser/get_data.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/isp_trace_parser/get_data.py b/src/isp_trace_parser/get_data.py index e848c89..e2a4e5e 100644 --- a/src/isp_trace_parser/get_data.py +++ b/src/isp_trace_parser/get_data.py @@ -145,7 +145,7 @@ def get_project_single_reference_year( start_year: int, end_year: int, reference_year: int, - project: str | List, + project: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, @@ -237,8 +237,8 @@ def get_zone_single_reference_year( start_year: int, end_year: int, reference_year: int, - zone: str | List, - resource_type: str | List, + zone: str | list, + resource_type: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, @@ -333,10 +333,10 @@ def get_demand_single_reference_year( start_year: int, end_year: int, reference_year: int, - scenario: str | List, - subregion: str | List, - demand_type: str | List, - poe: str | List, + scenario: str | list, + subregion: str | list, + demand_type: str | list, + poe: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, @@ -441,7 +441,7 @@ def get_demand_single_reference_year( @validate_call def get_project_multiple_reference_years( reference_year_mapping: dict[int, int], - project: str | List, + project: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, @@ -530,8 +530,8 @@ def get_project_multiple_reference_years( @validate_call def get_zone_multiple_reference_years( reference_year_mapping: dict[int, int], - zone: str | List, - resource_type: str | List, + zone: str | list, + resource_type: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, @@ -623,10 +623,10 @@ def get_zone_multiple_reference_years( @validate_call def get_demand_multiple_reference_years( reference_year_mapping: dict[int, int], - scenario: str | List, - subregion: str | List, - demand_type: str | List, - poe: str | List, + scenario: str | list, + subregion: str | list, + demand_type: str | list, + poe: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, From 839361296119971da4a6f41b4cbe4e50a7a2b77e Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:30:24 +1000 Subject: [PATCH 07/18] Fix Ruff "F" class messages (e.g. unused imports). --- src/isp_trace_parser/demand_traces.py | 2 +- src/isp_trace_parser/optimise_parquet.py | 1 - src/isp_trace_parser/solar_traces.py | 2 +- src/isp_trace_parser/trace_restructure_helper_functions.py | 1 - src/isp_trace_parser/wind_traces.py | 2 +- tests/conftest.py | 2 +- tests/test_download.py | 2 -- tests/test_trace_parsers.py | 2 +- 8 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/isp_trace_parser/demand_traces.py b/src/isp_trace_parser/demand_traces.py index c59c4e1..f39561b 100644 --- a/src/isp_trace_parser/demand_traces.py +++ b/src/isp_trace_parser/demand_traces.py @@ -1,7 +1,7 @@ import functools import os from pathlib import Path -from typing import Literal, Optional +from typing import Literal import polars as pl from joblib import Parallel, delayed diff --git a/src/isp_trace_parser/optimise_parquet.py b/src/isp_trace_parser/optimise_parquet.py index 327196c..71ef51d 100644 --- a/src/isp_trace_parser/optimise_parquet.py +++ b/src/isp_trace_parser/optimise_parquet.py @@ -1,6 +1,5 @@ from itertools import product from pathlib import Path -from typing import Optional import duckdb from pydantic import validate_call diff --git a/src/isp_trace_parser/solar_traces.py b/src/isp_trace_parser/solar_traces.py index 9bef13c..353412a 100644 --- a/src/isp_trace_parser/solar_traces.py +++ b/src/isp_trace_parser/solar_traces.py @@ -1,7 +1,7 @@ import functools import os from pathlib import Path -from typing import Literal, Optional +from typing import Literal from joblib import Parallel, delayed from pydantic import BaseModel, validate_call diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index b0be7f7..03ae988 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -1,4 +1,3 @@ -from datetime import timedelta from pathlib import Path import polars as pl diff --git a/src/isp_trace_parser/wind_traces.py b/src/isp_trace_parser/wind_traces.py index 81cdb12..e7f11d0 100644 --- a/src/isp_trace_parser/wind_traces.py +++ b/src/isp_trace_parser/wind_traces.py @@ -1,7 +1,7 @@ import functools import os from pathlib import Path -from typing import Literal, Optional +from typing import Literal from joblib import Parallel, delayed from pydantic import BaseModel, validate_call diff --git a/tests/conftest.py b/tests/conftest.py index 09d5a86..dc61c97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ def parsed_trace_trace_directory(request): optimise_parquet.partition_traces_by_columns( input_directory=tmp_parsed_directory / "demand", - output_directory=tmp_parsed_directory / f"demand_optimised", + output_directory=tmp_parsed_directory / "demand_optimised", partition_cols=["scenario", "reference_year"], ) yield tmp_parsed_directory diff --git a/tests/test_download.py b/tests/test_download.py index 5911c63..bf8df91 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -117,8 +117,6 @@ def test_wrong_type(): def test_empty_manifest(monkeypatch): """Test that empty manifest raises ValueError.""" - from importlib.resources import files - with TemporaryDirectory() as tmp_path: tmp_path = Path(tmp_path) diff --git a/tests/test_trace_parsers.py b/tests/test_trace_parsers.py index ac72a08..21e07aa 100644 --- a/tests/test_trace_parsers.py +++ b/tests/test_trace_parsers.py @@ -5,7 +5,7 @@ import pytest from polars.testing import assert_frame_equal -from isp_trace_parser import demand_traces, solar_traces, wind_traces +from isp_trace_parser import demand_traces TEST_DATA = Path(__file__).parent / "test_data" From faeef5f5fdb33ffbcbe5f3d56ba411cd3f6d4a4f Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:31:43 +1000 Subject: [PATCH 08/18] Remove commented out code (fixeds Ruff ERA warnings). --- src/isp_trace_parser/optimise_parquet.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/isp_trace_parser/optimise_parquet.py b/src/isp_trace_parser/optimise_parquet.py index 71ef51d..0804708 100644 --- a/src/isp_trace_parser/optimise_parquet.py +++ b/src/isp_trace_parser/optimise_parquet.py @@ -69,8 +69,6 @@ def partition_traces_by_columns( partitions = [tuple(val[0] for val in vals) for vals in product(*distinct_values)] for partition_values in partitions: - # print(*partition_values) - conditions = [] for col, val in zip(partition_cols, partition_values): if isinstance(val, str): From f1c61f644b68237f4fb461f0db15624f336c8b48 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:48:34 +1000 Subject: [PATCH 09/18] Fix some TRY class messages. --- src/isp_trace_parser/input_validation.py | 2 +- src/isp_trace_parser/remote/download.py | 5 +++-- src/isp_trace_parser/trace_restructure_helper_functions.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/isp_trace_parser/input_validation.py b/src/isp_trace_parser/input_validation.py index cfa9164..6326112 100644 --- a/src/isp_trace_parser/input_validation.py +++ b/src/isp_trace_parser/input_validation.py @@ -4,7 +4,7 @@ def input_directory(path: Path | str) -> Path: path = is_valid_path(path) if not path.is_dir(): - raise ValueError(f"Directory {path} does not exist") + raise FileNotFoundError(path) return path diff --git a/src/isp_trace_parser/remote/download.py b/src/isp_trace_parser/remote/download.py index 462887c..4d5e549 100644 --- a/src/isp_trace_parser/remote/download.py +++ b/src/isp_trace_parser/remote/download.py @@ -54,7 +54,7 @@ def _download_from_manifest( manifest_path = files("isp_trace_parser.remote.manifests") / f"{manifest_name}.txt" if not manifest_path.exists(): - raise FileNotFoundError(f"Manifest file not found: {manifest_path}") + raise FileNotFoundError(manifest_path) # Read URLs from manifest with open(manifest_path) as f: @@ -81,12 +81,13 @@ def _download_with_retry( for attempt in range(max_retries): try: _download_file(url, save_directory, strip_levels, unquote_path) - return except requests.exceptions.RequestException: if attempt < max_retries - 1: time.sleep(2**attempt) else: raise + else: + return def _download_file( diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index 03ae988..b9011f6 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -9,7 +9,7 @@ def get_all_filepaths(directory: Path) -> list[Path]: if directory.is_dir(): return [path for path in Path(directory).rglob("*.csv") if path.is_file()] - raise ValueError(f"{directory} not found.") + raise FileNotFoundError(directory) def read_trace_csv(file: Path) -> pl.DataFrame: From 5c832b4f6aacd6f77da66831f89d4c4a64a354c5 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:52:34 +1000 Subject: [PATCH 10/18] Fix B905 messages -- call zip() with strict=True so that we strictly ensure that both lists are of equal length and error if not. --- .../construct_reference_year_mapping.py | 2 +- src/isp_trace_parser/optimise_parquet.py | 2 +- src/isp_trace_parser/solar_traces.py | 6 +++--- src/isp_trace_parser/wind_traces.py | 20 +++++++++++++------ 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/isp_trace_parser/construct_reference_year_mapping.py b/src/isp_trace_parser/construct_reference_year_mapping.py index 6e384b2..932dd22 100644 --- a/src/isp_trace_parser/construct_reference_year_mapping.py +++ b/src/isp_trace_parser/construct_reference_year_mapping.py @@ -35,4 +35,4 @@ def construct_reference_year_mapping( reference_years = ( reference_years * full_reference_year_cycles ) + reference_years[:partial_cycle_length] - return dict(zip(years, reference_years)) + return dict(zip(years, reference_years, strict=True)) diff --git a/src/isp_trace_parser/optimise_parquet.py b/src/isp_trace_parser/optimise_parquet.py index 0804708..cb32520 100644 --- a/src/isp_trace_parser/optimise_parquet.py +++ b/src/isp_trace_parser/optimise_parquet.py @@ -70,7 +70,7 @@ def partition_traces_by_columns( for partition_values in partitions: conditions = [] - for col, val in zip(partition_cols, partition_values): + for col, val in zip(partition_cols, partition_values, strict=True): if isinstance(val, str): conditions.append(f"{col}='{val}'") else: diff --git a/src/isp_trace_parser/solar_traces.py b/src/isp_trace_parser/solar_traces.py index 353412a..15238ab 100644 --- a/src/isp_trace_parser/solar_traces.py +++ b/src/isp_trace_parser/solar_traces.py @@ -157,7 +157,7 @@ def parse_solar_traces( } project_and_zone_output_names, project_and_zone_input_names = zip( - *name_mappings.items() + *name_mappings.items(), strict=True ) partial_func = functools.partial( @@ -172,12 +172,12 @@ def parse_solar_traces( Parallel(n_jobs=max_workers)( delayed(partial_func)(save_name, old_trace_name) for save_name, old_trace_name in zip( - project_and_zone_output_names, project_and_zone_input_names + project_and_zone_output_names, project_and_zone_input_names, strict=True ) ) else: for save_name, old_trace_name in zip( - project_and_zone_output_names, project_and_zone_input_names + project_and_zone_output_names, project_and_zone_input_names, strict=True ): partial_func(save_name, old_trace_name) diff --git a/src/isp_trace_parser/wind_traces.py b/src/isp_trace_parser/wind_traces.py index e7f11d0..2fa1305 100644 --- a/src/isp_trace_parser/wind_traces.py +++ b/src/isp_trace_parser/wind_traces.py @@ -156,12 +156,14 @@ def parse_wind_traces( zone_name_mappings = filter_mapping_by_names_in_input_files( zone_name_mappings, project_and_zone_input_names ) - zone_output_names, zone_input_names = zip(*zone_name_mappings.items()) + zone_output_names, zone_input_names = zip(*zone_name_mappings.items(), strict=True) project_name_mappings = filter_mapping_by_names_in_input_files( project_name_mappings, project_and_zone_input_names ) - project_output_names, project_input_names = zip(*project_name_mappings.items()) + project_output_names, project_input_names = zip( + *project_name_mappings.items(), strict=True + ) zone_partial_func = functools.partial( restructure_wind_zone_files, @@ -182,21 +184,27 @@ def parse_wind_traces( Parallel(n_jobs=max_workers)( delayed(zone_partial_func)(save_name, old_trace_name) - for save_name, old_trace_name in zip(zone_output_names, zone_input_names) + for save_name, old_trace_name in zip( + zone_output_names, zone_input_names, strict=True + ) ) Parallel(n_jobs=max_workers)( delayed(project_partial_func)(save_name, old_trace_name) for save_name, old_trace_name in zip( - project_output_names, project_input_names + project_output_names, project_input_names, strict=True ) ) else: - for save_name, old_trace_name in zip(zone_output_names, zone_input_names): + for save_name, old_trace_name in zip( + zone_output_names, zone_input_names, strict=True + ): zone_partial_func(save_name, old_trace_name) - for save_name, old_trace_name in zip(project_output_names, project_input_names): + for save_name, old_trace_name in zip( + project_output_names, project_input_names, strict=True + ): project_partial_func(save_name, old_trace_name) From c3e1decd75d0b413db51accf1677c8bd8d604573 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 15:57:40 +1000 Subject: [PATCH 11/18] Fix RUF022 (sort __all__ list members). --- src/isp_trace_parser/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/isp_trace_parser/__init__.py b/src/isp_trace_parser/__init__.py index 6467868..2f27dd3 100644 --- a/src/isp_trace_parser/__init__.py +++ b/src/isp_trace_parser/__init__.py @@ -8,13 +8,13 @@ from isp_trace_parser.wind_traces import WindMetadataFilter, parse_wind_traces __all__ = [ - "trace_formatter", + "DemandMetadataFilter", + "SolarMetadataFilter", + "WindMetadataFilter", + "construct_reference_year_mapping", "get_data", - "parse_wind_traces", "parse_demand_traces", "parse_solar_traces", - "construct_reference_year_mapping", - "WindMetadataFilter", - "SolarMetadataFilter", - "DemandMetadataFilter", + "parse_wind_traces", + "trace_formatter", ] From cad03bc86aee169ca48d8ae3821d6f9e59d0b9a8 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 16:27:27 +1000 Subject: [PATCH 12/18] Fix Ruff ANN201 warnings. --- .../construct_reference_year_mapping.py | 2 +- src/isp_trace_parser/demand_traces.py | 2 +- src/isp_trace_parser/get_data.py | 14 +++---- src/isp_trace_parser/input_validation.py | 2 +- src/isp_trace_parser/solar_traces.py | 2 +- src/isp_trace_parser/trace_formatter.py | 2 +- .../trace_restructure_helper_functions.py | 2 +- src/isp_trace_parser/wind_traces.py | 2 +- tests/conftest.py | 2 +- tests/create_end_to_end_test_data.py | 10 ++--- tests/test_demand_trace_metadata.py | 4 +- tests/test_download.py | 20 ++++----- tests/test_get_data.py | 42 +++++++++++-------- tests/test_input_validation.py | 32 +++++++------- tests/test_optimise_parquet.py | 2 +- tests/test_resource_trace_metadata.py | 4 +- tests/test_trace_formatter.py | 2 +- tests/test_trace_parsers.py | 8 ++-- tests/test_writing_save_names.py | 4 +- 19 files changed, 84 insertions(+), 74 deletions(-) diff --git a/src/isp_trace_parser/construct_reference_year_mapping.py b/src/isp_trace_parser/construct_reference_year_mapping.py index 932dd22..19cf0d9 100644 --- a/src/isp_trace_parser/construct_reference_year_mapping.py +++ b/src/isp_trace_parser/construct_reference_year_mapping.py @@ -6,7 +6,7 @@ @validate_call def construct_reference_year_mapping( start_year: int, end_year: int, reference_years: list[int] -): +) -> dict: """Constructs a dictionary mapping a sequence of modeling years to a cycle of reference years. Examples: diff --git a/src/isp_trace_parser/demand_traces.py b/src/isp_trace_parser/demand_traces.py index f39561b..045a478 100644 --- a/src/isp_trace_parser/demand_traces.py +++ b/src/isp_trace_parser/demand_traces.py @@ -63,7 +63,7 @@ def parse_demand_traces( parsed_directory: str | Path, use_concurrency: bool = True, filters: DemandMetadataFilter | None = None, -): +) -> None: """Takes a directory with AEMO demand trace data and reformats the data, saving it to a new directory. AEMO demand trace data comes in CSVs with columns specifying the year, day, and month, and data columns diff --git a/src/isp_trace_parser/get_data.py b/src/isp_trace_parser/get_data.py index e2a4e5e..081fd38 100644 --- a/src/isp_trace_parser/get_data.py +++ b/src/isp_trace_parser/get_data.py @@ -9,7 +9,7 @@ def _year_range_to_dt_range( start_year: int, end_year: int, year_type: Literal["fy", "calendar"] = "fy" -): +) -> datetime.datetime: """ Convert year range to datetime boundaries for efficient time filtering. @@ -149,7 +149,7 @@ def get_project_single_reference_year( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query project trace data for a single reference year. @@ -242,7 +242,7 @@ def get_zone_single_reference_year( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query zone trace data for a single reference year. @@ -340,7 +340,7 @@ def get_demand_single_reference_year( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query demand trace data for a single reference year. @@ -445,7 +445,7 @@ def get_project_multiple_reference_years( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query project trace data across multiple reference years. @@ -535,7 +535,7 @@ def get_zone_multiple_reference_years( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query zone trace data across multiple reference years. @@ -630,7 +630,7 @@ def get_demand_multiple_reference_years( directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", select_columns: list[str] = None, -): +) -> pd.DataFrame: """ Query demand trace data across multiple reference years. diff --git a/src/isp_trace_parser/input_validation.py b/src/isp_trace_parser/input_validation.py index 6326112..2a74de0 100644 --- a/src/isp_trace_parser/input_validation.py +++ b/src/isp_trace_parser/input_validation.py @@ -19,6 +19,6 @@ def is_valid_path(path: str | Path) -> Path: raise ValueError(f"Invalid parsed directory path: {path}") -def start_year_before_end_year(start_year, end_year): +def start_year_before_end_year(start_year, end_year) -> None: if end_year < start_year: raise ValueError(f"Start year {end_year} < end year {start_year}") diff --git a/src/isp_trace_parser/solar_traces.py b/src/isp_trace_parser/solar_traces.py index 15238ab..d93525b 100644 --- a/src/isp_trace_parser/solar_traces.py +++ b/src/isp_trace_parser/solar_traces.py @@ -61,7 +61,7 @@ def parse_solar_traces( parsed_directory: str | Path, use_concurrency: bool = True, filters: SolarMetadataFilter | None = None, -): +) -> None: """Takes a directory with AEMO solar trace data and reformats the data, saving it to a new directory. AEMO solar trace data comes in CSVs with columns specifying the year, day, and month, and data columns diff --git a/src/isp_trace_parser/trace_formatter.py b/src/isp_trace_parser/trace_formatter.py index b6262e4..8d43cd9 100644 --- a/src/isp_trace_parser/trace_formatter.py +++ b/src/isp_trace_parser/trace_formatter.py @@ -65,7 +65,7 @@ def trace_formatter(trace_data: pl.DataFrame) -> pl.DataFrame: value_name="value", ) - def get_hour(time_label): + def get_hour(time_label) -> int: return timedelta(hours=int(time_label) // 2) def get_minute(time_label): diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index b9011f6..e5f531b 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -158,5 +158,5 @@ def filter_mapping_by_names_in_input_files( return filtered_mapping -def get_just_filepaths(metadata_for_files): +def get_just_filepaths(metadata_for_files) -> list: return [file for file, metadata in metadata_for_files.items()] diff --git a/src/isp_trace_parser/wind_traces.py b/src/isp_trace_parser/wind_traces.py index 2fa1305..a30d653 100644 --- a/src/isp_trace_parser/wind_traces.py +++ b/src/isp_trace_parser/wind_traces.py @@ -61,7 +61,7 @@ def parse_wind_traces( parsed_directory: str | Path, use_concurrency: bool = True, filters: WindMetadataFilter | None = None, -): +) -> None: """Takes a directory with AEMO wind trace data and reformats the data, saving it to a new directory. AEMO wind trace data comes in CSVs with columns specifying the year, day, and month, and data columns diff --git a/tests/conftest.py b/tests/conftest.py index dc61c97..c95861b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ @pytest.fixture(params=[True, False], ids=["concurrent", "sequential"], scope="module") -def parsed_trace_trace_directory(request): +def parsed_trace_trace_directory(request) -> Path: """Fixture that performs parsing of wind and solar trace directory once, providing the output directory to multiple test cases that validate different files. diff --git a/tests/create_end_to_end_test_data.py b/tests/create_end_to_end_test_data.py index b107cc4..85e1fa9 100644 --- a/tests/create_end_to_end_test_data.py +++ b/tests/create_end_to_end_test_data.py @@ -6,7 +6,7 @@ import pandas as pd -def generate_random_data(start_year, end_year): +def generate_random_data(start_year, end_year) -> pd.DataFrame: # Generate date range from July 1st of the start year to July 1st of the end year (excluding end) date_range = pd.date_range( start=f"{start_year}-01-01", end=f"{end_year}-01-01", freq="D", inclusive="left" @@ -30,7 +30,7 @@ def generate_random_data(start_year, end_year): data = generate_random_data(start_year=config.start, end_year=config.end) -def simple_flatten(nested_list): +def simple_flatten(nested_list) -> list: flattened = [] for item in nested_list: if isinstance(item, list): @@ -40,7 +40,7 @@ def simple_flatten(nested_list): return flattened -def create_solar_csvs(directory): +def create_solar_csvs(directory) -> None: combos = itertools.product(config.reference_years, config.solar_projects) for y, project in combos: data.to_csv(directory / Path(f"{project}_FFP_RefYear{y}.csv"), index=False) @@ -52,7 +52,7 @@ def create_solar_csvs(directory): ) -def create_wind_csvs(directory): +def create_wind_csvs(directory) -> None: combos = itertools.product( config.reference_years, simple_flatten(config.wind_projects.values()) ) @@ -70,7 +70,7 @@ def create_wind_csvs(directory): ) -def create_demand_csvs(directory): +def create_demand_csvs(directory) -> None: combos = itertools.product( config.reference_years, config.sub_regions, diff --git a/tests/test_demand_trace_metadata.py b/tests/test_demand_trace_metadata.py index 93a3499..44ffd47 100644 --- a/tests/test_demand_trace_metadata.py +++ b/tests/test_demand_trace_metadata.py @@ -5,7 +5,7 @@ from isp_trace_parser import demand_trace_metadata -def test_build(): +def test_build() -> None: """Two examples spanning different scenario / poe / demand_type / subregion values. Every combination resolves through the same single dict lookup, so two are enough for testing. @@ -40,6 +40,6 @@ def test_build(): "VIC_RefYear_2011_MYSTERY_POE10_OPSO_MODELLING.csv", # lookup miss ], ) -def test_build_rejects_unexpected_filename(filename): +def test_build_rejects_unexpected_filename(filename) -> None: with pytest.raises(ValueError, match="Unexpected trace filename"): demand_trace_metadata.build([Path(filename)], version="2024") diff --git a/tests/test_download.py b/tests/test_download.py index bf8df91..1285745 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -9,7 +9,7 @@ TEST_EXPECTED_CONTENT = b"ISP Trace Parser Test File\n" -def test_download_test_file(): +def test_download_test_file() -> None: """Test download with actual server file.""" with TemporaryDirectory() as tmp_path: @@ -21,7 +21,7 @@ def test_download_test_file(): assert downloaded.read_bytes() == TEST_EXPECTED_CONTENT -def test_download_with_retry(): +def test_download_with_retry() -> None: """Test retry logic with real server.""" with TemporaryDirectory() as tmp_path: @@ -31,7 +31,7 @@ def test_download_with_retry(): assert (tmp_path / "test" / "test" / "test_file.txt").exists() -def test_fetch_trace_data_with_test_manifest(monkeypatch): +def test_fetch_trace_data_with_test_manifest(monkeypatch) -> None: """Test downloading from a small, test manifest. The testing manifest, while still named "full_isp_2024" here, is just a test manifest with containing a single url ("https://data.openisp.au/test/test/test_file.txt") @@ -58,7 +58,7 @@ def mock_files(package): assert downloaded.read_bytes() == TEST_EXPECTED_CONTENT -def test_manifest_not_found(): +def test_manifest_not_found() -> None: """Test downloading from a small, test manifest.""" with pytest.raises(FileNotFoundError): @@ -68,7 +68,7 @@ def test_manifest_not_found(): @pytest.mark.parametrize("unquote", [True, False]) -def test_fetch_trace_data(unquote: bool, monkeypatch): +def test_fetch_trace_data(unquote: bool, monkeypatch) -> None: """Test downloading via fetch_trace_data with test fixtures. This, while still download a dataset name "full", is just a pointing to a test manifest manifest with containing a single url ("https://data.openisp.au/test/test/test_file.txt") @@ -95,13 +95,13 @@ def mock_files(package): assert downloaded.read_bytes() == TEST_EXPECTED_CONTENT -def test_wrong_source(): +def test_wrong_source() -> None: # no ISP 2025 data with pytest.raises(ValueError, match="Only isp_2024 is currently supported"): download.fetch_trace_data("example", "isp_2025", "/", "archive") -def test_wrong_format(): +def test_wrong_format() -> None: # only archive or processed data (not other) with pytest.raises( ValueError, match="data_format must be 'processed' or 'archive'" @@ -109,13 +109,13 @@ def test_wrong_format(): download.fetch_trace_data("example", "isp_2024", "/", "other") -def test_wrong_type(): +def test_wrong_type() -> None: # only full or example type with pytest.raises(ValueError): download.fetch_trace_data("other", "isp_2024", "/", "archive") -def test_empty_manifest(monkeypatch): +def test_empty_manifest(monkeypatch) -> None: """Test that empty manifest raises ValueError.""" with TemporaryDirectory() as tmp_path: tmp_path = Path(tmp_path) @@ -130,7 +130,7 @@ def mock_files(package): download._download_from_manifest("empty_manifest", tmp_path, strip_levels=0) -def test_strip_levels_too_high(): +def test_strip_levels_too_high() -> None: """Test that strip_levels >= path parts raises ValueError.""" with TemporaryDirectory() as tmp_path: tmp_path = Path(tmp_path) diff --git a/tests/test_get_data.py b/tests/test_get_data.py index 609226a..fa5438a 100644 --- a/tests/test_get_data.py +++ b/tests/test_get_data.py @@ -25,7 +25,7 @@ TEST_DATA = Path(__file__).parent / "test_data" -def test_year_range_to_dt_range_fy(): +def test_year_range_to_dt_range_fy() -> None: """Test financial year conversion.""" start_dt, end_dt = _year_range_to_dt_range(2022, 2024, year_type="fy") @@ -33,7 +33,7 @@ def test_year_range_to_dt_range_fy(): assert end_dt == datetime.datetime(2024, 7, 1, 0, 0) -def test_year_range_to_dt_range_calendar(): +def test_year_range_to_dt_range_calendar() -> None: """Test calendar year conversion.""" start_dt, end_dt = _year_range_to_dt_range(2022, 2024, year_type="calendar") @@ -42,7 +42,9 @@ def test_year_range_to_dt_range_calendar(): @pytest.mark.parametrize("year_type", ["fy", "calendar"]) -def test_get_zone_single_reference_year(parsed_trace_trace_directory: Path, year_type): +def test_get_zone_single_reference_year( + parsed_trace_trace_directory: Path, year_type +) -> None: test_df_lazy = pl.scan_parquet(TEST_DATA / "output" / "RefYear2022_N2_CST.parquet") start_dt, end_dt = _year_range_to_dt_range(2023, 2024, year_type=year_type) @@ -69,7 +71,7 @@ def test_get_zone_single_reference_year(parsed_trace_trace_directory: Path, year pd.testing.assert_frame_equal(test_df, df) -def test_get_zone_multiple_reference_year(parsed_trace_trace_directory: Path): +def test_get_zone_multiple_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet(TEST_DATA / "output" / "RefYear2022_N1_WM.parquet") test_df = ( @@ -93,7 +95,7 @@ def test_get_zone_multiple_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_get_project_single_reference_year(parsed_trace_trace_directory: Path): +def test_get_project_single_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Bodangora_Wind_Farm.parquet" ) @@ -120,7 +122,9 @@ def test_get_project_single_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_get_project_multiple_reference_year(parsed_trace_trace_directory: Path): +def test_get_project_multiple_reference_year( + parsed_trace_trace_directory: Path, +) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Broken_Hill_Solar_Farm_FFP.parquet" ) @@ -145,7 +149,7 @@ def test_get_project_multiple_reference_year(parsed_trace_trace_directory: Path) pd.testing.assert_frame_equal(test_df, df) -def test_get_demand_single_reference_year(parsed_trace_trace_directory: Path): +def test_get_demand_single_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" @@ -178,7 +182,7 @@ def test_get_demand_single_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_get_demand_multiple_reference_year(parsed_trace_trace_directory: Path): +def test_get_demand_multiple_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" @@ -208,7 +212,7 @@ def test_get_demand_multiple_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_explicit_select_columns(parsed_trace_trace_directory): +def test_explicit_select_columns(parsed_trace_trace_directory) -> None: df = get_zone_single_reference_year( start_year=2023, end_year=2024, @@ -221,7 +225,7 @@ def test_explicit_select_columns(parsed_trace_trace_directory): assert list(df.columns) == ["datetime", "value", "zone"] -def test_multi_value_filter(parsed_trace_trace_directory): +def test_multi_value_filter(parsed_trace_trace_directory) -> None: df = get_zone_single_reference_year( start_year=2023, end_year=2024, @@ -234,7 +238,7 @@ def test_multi_value_filter(parsed_trace_trace_directory): assert "zone" in df.columns -def test_wind_project_single_reference_year(parsed_trace_trace_directory): +def test_wind_project_single_reference_year(parsed_trace_trace_directory) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Bodangora_Wind_Farm.parquet" ) @@ -260,7 +264,7 @@ def test_wind_project_single_reference_year(parsed_trace_trace_directory): pd.testing.assert_frame_equal(test_df, df) -def test_solar_project_single_reference_year(parsed_trace_trace_directory): +def test_solar_project_single_reference_year(parsed_trace_trace_directory) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Broken_Hill_Solar_Farm_FFP.parquet" ) @@ -286,7 +290,9 @@ def test_solar_project_single_reference_year(parsed_trace_trace_directory): pd.testing.assert_frame_equal(test_df, df) -def test_solar_project_multiple_reference_years(parsed_trace_trace_directory: Path): +def test_solar_project_multiple_reference_years( + parsed_trace_trace_directory: Path, +) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Broken_Hill_Solar_Farm_FFP.parquet" ) @@ -311,7 +317,9 @@ def test_solar_project_multiple_reference_years(parsed_trace_trace_directory: Pa pd.testing.assert_frame_equal(test_df, df) -def test_wind_project_multiple_reference_years(parsed_trace_trace_directory: Path): +def test_wind_project_multiple_reference_years( + parsed_trace_trace_directory: Path, +) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Bodangora_Wind_Farm.parquet" ) @@ -336,7 +344,7 @@ def test_wind_project_multiple_reference_years(parsed_trace_trace_directory: Pat pd.testing.assert_frame_equal(test_df, df) -def test_solar_area_single_reference_year(parsed_trace_trace_directory: Path): +def test_solar_area_single_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet(TEST_DATA / "output" / "RefYear2022_N2_CST.parquet") start_dt, end_dt = _year_range_to_dt_range(2023, 2024, year_type="fy") @@ -362,7 +370,7 @@ def test_solar_area_single_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_demand_single_reference_year(parsed_trace_trace_directory: Path): +def test_demand_single_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" @@ -395,7 +403,7 @@ def test_demand_single_reference_year(parsed_trace_trace_directory: Path): pd.testing.assert_frame_equal(test_df, df) -def test_demand_multiple_reference_years(parsed_trace_trace_directory: Path): +def test_demand_multiple_reference_years(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py index b380d46..56c571a 100644 --- a/tests/test_input_validation.py +++ b/tests/test_input_validation.py @@ -31,7 +31,7 @@ }, ], ) -def test_solar_metadata_filter_valid(valid_input): +def test_solar_metadata_filter_valid(valid_input) -> None: assert SolarMetadataFilter(**valid_input) @@ -44,7 +44,7 @@ def test_solar_metadata_filter_valid(valid_input): ({"name": 123}, "Input should be a valid list"), ], ) -def test_solar_metadata_filter_invalid(invalid_input, expected_error): +def test_solar_metadata_filter_invalid(invalid_input, expected_error) -> None: with pytest.raises(ValidationError, match=expected_error): SolarMetadataFilter(**invalid_input) @@ -64,7 +64,7 @@ def test_solar_metadata_filter_invalid(invalid_input, expected_error): }, ], ) -def test_wind_metadata_filter_valid(valid_input): +def test_wind_metadata_filter_valid(valid_input) -> None: assert WindMetadataFilter(**valid_input) @@ -80,7 +80,7 @@ def test_wind_metadata_filter_valid(valid_input): ({"name": 123}, "Input should be a valid list"), ], ) -def test_wind_metadata_filter_invalid(invalid_input, expected_error): +def test_wind_metadata_filter_invalid(invalid_input, expected_error) -> None: with pytest.raises(ValidationError, match=expected_error): WindMetadataFilter(**invalid_input) @@ -102,7 +102,7 @@ def test_wind_metadata_filter_invalid(invalid_input, expected_error): }, ], ) -def test_demand_metadata_filter_valid(valid_input): +def test_demand_metadata_filter_valid(valid_input) -> None: assert DemandMetadataFilter(**valid_input) @@ -122,7 +122,7 @@ def test_demand_metadata_filter_valid(valid_input): ({"subregion": 123}, "Input should be a valid list"), ], ) -def test_demand_metadata_filter_invalid(invalid_input, expected_error): +def test_demand_metadata_filter_invalid(invalid_input, expected_error) -> None: with pytest.raises(ValidationError, match=expected_error): DemandMetadataFilter(**invalid_input) @@ -145,7 +145,7 @@ def test_demand_metadata_filter_invalid(invalid_input, expected_error): }, ], ) -def test_parse_traces_validation(invalid_input): +def test_parse_traces_validation(invalid_input) -> None: with pytest.raises(ValidationError): parse_solar_traces(**invalid_input) with pytest.raises(ValidationError): @@ -163,12 +163,12 @@ def test_parse_traces_validation(invalid_input): {"start_year": 2030, "end_year": 2035, "reference_years": [2011, "x", 2018]}, ], ) -def test_construct_reference_year_mapping_validation_invalid(invalid_input): +def test_construct_reference_year_mapping_validation_invalid(invalid_input) -> None: with pytest.raises(ValidationError): construct_reference_year_mapping(**invalid_input) -def test_construct_reference_year_mapping_validation_valid(): +def test_construct_reference_year_mapping_validation_valid() -> None: result = construct_reference_year_mapping( start_year=2030, end_year=2035, reference_years=[2011, 2013, 2018] ) @@ -178,7 +178,7 @@ def test_construct_reference_year_mapping_validation_valid(): # Tests for custom input validation functions -def test_input_directory(tmp_path): +def test_input_directory(tmp_path) -> None: valid_dir = tmp_path / "valid_dir" valid_dir.mkdir() assert input_validation.input_directory(valid_dir) == valid_dir @@ -194,7 +194,7 @@ def test_input_directory(tmp_path): Path("/valid/path"), ], ) -def test_parsed_directory_valid(valid_path): +def test_parsed_directory_valid(valid_path) -> None: result = input_validation.parsed_directory(valid_path) assert isinstance(result, Path) @@ -207,7 +207,7 @@ def test_parsed_directory_valid(valid_path): [], ], ) -def test_parsed_directory_invalid(invalid_path): +def test_parsed_directory_invalid(invalid_path) -> None: with pytest.raises(ValueError, match="Invalid parsed directory path"): input_validation.parsed_directory(invalid_path) @@ -219,7 +219,7 @@ def test_parsed_directory_invalid(invalid_path): Path("/valid/path"), ], ) -def test_is_valid_path_valid(valid_path): +def test_is_valid_path_valid(valid_path) -> None: result = input_validation.is_valid_path(valid_path) assert isinstance(result, Path) @@ -232,7 +232,7 @@ def test_is_valid_path_valid(valid_path): [], ], ) -def test_is_valid_path_invalid(invalid_path): +def test_is_valid_path_invalid(invalid_path) -> None: with pytest.raises(ValueError, match="Invalid parsed directory path"): input_validation.is_valid_path(invalid_path) @@ -245,7 +245,7 @@ def test_is_valid_path_invalid(invalid_path): (-10, 0), ], ) -def test_start_year_before_end_year_valid(start, end): +def test_start_year_before_end_year_valid(start, end) -> None: assert input_validation.start_year_before_end_year(start, end) is None @@ -257,6 +257,6 @@ def test_start_year_before_end_year_valid(start, end): (2020, 2019), ], ) -def test_start_year_before_end_year_invalid(start, end): +def test_start_year_before_end_year_invalid(start, end) -> None: with pytest.raises(ValueError, match="Start year .* < end year"): input_validation.start_year_before_end_year(start, end) diff --git a/tests/test_optimise_parquet.py b/tests/test_optimise_parquet.py index f7b0efe..16f64ed 100644 --- a/tests/test_optimise_parquet.py +++ b/tests/test_optimise_parquet.py @@ -13,7 +13,7 @@ "expected_data, file_type", [("zone_data_0.parquet", "zone"), ("project_data_0.parquet", "project")], ) -def test_optimisation(parsed_trace_trace_directory, expected_data, file_type): +def test_optimisation(parsed_trace_trace_directory, expected_data, file_type) -> None: """Test wind trace parsing produces expected parquet outputs (both for a sample wind project and wind zone)""" test_output_parquet = TEST_DATA / "output" / expected_data diff --git a/tests/test_resource_trace_metadata.py b/tests/test_resource_trace_metadata.py index a209d04..5a724b8 100644 --- a/tests/test_resource_trace_metadata.py +++ b/tests/test_resource_trace_metadata.py @@ -5,7 +5,7 @@ from isp_trace_parser import resource_trace_metadata -def test_build(): +def test_build() -> None: """One test covers function logic compared with regex approach Solar zones / wind zones / extra reference years add no new code-path @@ -35,6 +35,6 @@ def test_build(): "Mystery_Plant_RefYear2011.csv", # stem not in mapping ], ) -def test_build_rejects_unexpected_filename(filename): +def test_build_rejects_unexpected_filename(filename) -> None: with pytest.raises(ValueError, match="Unexpected trace filename"): resource_trace_metadata.build([Path(filename)], version="2024") diff --git a/tests/test_trace_formatter.py b/tests/test_trace_formatter.py index 7e7e5fa..391a1e5 100644 --- a/tests/test_trace_formatter.py +++ b/tests/test_trace_formatter.py @@ -4,7 +4,7 @@ from isp_trace_parser import trace_formatter, trace_restructure_helper_functions -def test_trace_formatter(): +def test_trace_formatter() -> None: # Test trace formatting works by using formatting function works by performing formatting and then # reversing the formatting changes and checking the result matches the original data. filepath = ( diff --git a/tests/test_trace_parsers.py b/tests/test_trace_parsers.py index 21e07aa..c0b255f 100644 --- a/tests/test_trace_parsers.py +++ b/tests/test_trace_parsers.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize("use_concurrency", [True, False]) -def test_demand_trace_parsing(use_concurrency: bool): +def test_demand_trace_parsing(use_concurrency: bool) -> None: """Test demand trace parsing produces expected parquet output.""" test_demand_csv_directory = TEST_DATA / "demand" expected_filename = "CNSW_RefYear_2011_HYDROGEN_EXPORT_POE10_OPSO_MODELLING.parquet" @@ -43,7 +43,9 @@ def test_demand_trace_parsing(use_concurrency: bool): ("RefYear2022_N1_WM.parquet", "zone"), ], ) -def test_wind_trace_parsing(parsed_trace_trace_directory, expected_filename, file_type): +def test_wind_trace_parsing( + parsed_trace_trace_directory, expected_filename, file_type +) -> None: """Test wind trace parsing produces expected parquet outputs (both for a sample wind project and wind zone)""" test_output_parquet = TEST_DATA / "output" / expected_filename @@ -64,7 +66,7 @@ def test_wind_trace_parsing(parsed_trace_trace_directory, expected_filename, fil ) def test_solar_trace_parsing( parsed_trace_trace_directory, expected_filename, file_type -): +) -> None: """Test solar trace parsing produces expected parquet output (both for a sample solar project and solar zone)""" test_output_parquet = TEST_DATA / "output" / expected_filename diff --git a/tests/test_writing_save_names.py b/tests/test_writing_save_names.py index 3fc47a1..d06051c 100644 --- a/tests/test_writing_save_names.py +++ b/tests/test_writing_save_names.py @@ -1,7 +1,7 @@ import isp_trace_parser -def test_write_solar_save_names(): +def test_write_solar_save_names() -> None: meta_data = { "name": "a", "reference_year": "1", @@ -25,7 +25,7 @@ def test_write_solar_save_names(): assert str(save_filepath) == "RefYear1_a_x.parquet" -def test_write_wind_save_names(): +def test_write_wind_save_names() -> None: meta_data = { "name": "a", "reference_year": "1", From e00e8ea770fc98c1ca6e89361d047a183f3e19a9 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 16:29:43 +1000 Subject: [PATCH 13/18] Fix remaining ANN2* messages. --- src/isp_trace_parser/trace_formatter.py | 4 ++-- tests/test_download.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/isp_trace_parser/trace_formatter.py b/src/isp_trace_parser/trace_formatter.py index 8d43cd9..53c3c31 100644 --- a/src/isp_trace_parser/trace_formatter.py +++ b/src/isp_trace_parser/trace_formatter.py @@ -65,10 +65,10 @@ def trace_formatter(trace_data: pl.DataFrame) -> pl.DataFrame: value_name="value", ) - def get_hour(time_label) -> int: + def get_hour(time_label) -> timedelta: return timedelta(hours=int(time_label) // 2) - def get_minute(time_label): + def get_minute(time_label) -> timedelta: return timedelta(minutes=int(time_label) % 2 * 30) trace_data = trace_data.with_columns( diff --git a/tests/test_download.py b/tests/test_download.py index 1285745..f805014 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -41,7 +41,7 @@ def test_fetch_trace_data_with_test_manifest(monkeypatch) -> None: tmp_path = Path(tmp_path) # Point to test fixtures instead of production manifests - def mock_files(package): + def mock_files(package) -> Path: return Path(__file__).parent / "fixtures" / "manifests" monkeypatch.setattr("isp_trace_parser.remote.download.files", mock_files) @@ -78,7 +78,7 @@ def test_fetch_trace_data(unquote: bool, monkeypatch) -> None: tmp_path = Path(tmp_path) # Point to test manifests instead of production manifests - def mock_files(package): + def mock_files(package) -> Path: return Path(__file__).parent / "fixtures" / "manifests" monkeypatch.setattr("isp_trace_parser.remote.download.files", mock_files) @@ -121,7 +121,7 @@ def test_empty_manifest(monkeypatch) -> None: tmp_path = Path(tmp_path) # Point to test manifest instead of production manifests - def mock_files(package): + def mock_files(package) -> Path: return Path(__file__).parent / "fixtures" / "manifests" monkeypatch.setattr("isp_trace_parser.remote.download.files", mock_files) From 36109bfcbb783b3b22d5d4d537c0e44a30bbc591 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 17:41:48 +1000 Subject: [PATCH 14/18] Fix most all ANN001 messages. --- src/isp_trace_parser/input_validation.py | 2 +- src/isp_trace_parser/trace_formatter.py | 4 +- .../trace_restructure_helper_functions.py | 2 +- tests/create_end_to_end_test_data.py | 10 ++--- tests/test_demand_trace_metadata.py | 2 +- tests/test_download.py | 6 +-- tests/test_get_data.py | 12 +++--- tests/test_input_validation.py | 38 +++++++++++-------- tests/test_optimise_parquet.py | 4 +- tests/test_resource_trace_metadata.py | 2 +- tests/test_trace_parsers.py | 4 +- 11 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/isp_trace_parser/input_validation.py b/src/isp_trace_parser/input_validation.py index 2a74de0..f9b38f3 100644 --- a/src/isp_trace_parser/input_validation.py +++ b/src/isp_trace_parser/input_validation.py @@ -19,6 +19,6 @@ def is_valid_path(path: str | Path) -> Path: raise ValueError(f"Invalid parsed directory path: {path}") -def start_year_before_end_year(start_year, end_year) -> None: +def start_year_before_end_year(start_year: int, end_year: int) -> None: if end_year < start_year: raise ValueError(f"Start year {end_year} < end year {start_year}") diff --git a/src/isp_trace_parser/trace_formatter.py b/src/isp_trace_parser/trace_formatter.py index 53c3c31..d693c4c 100644 --- a/src/isp_trace_parser/trace_formatter.py +++ b/src/isp_trace_parser/trace_formatter.py @@ -65,10 +65,10 @@ def trace_formatter(trace_data: pl.DataFrame) -> pl.DataFrame: value_name="value", ) - def get_hour(time_label) -> timedelta: + def get_hour(time_label: str) -> timedelta: return timedelta(hours=int(time_label) // 2) - def get_minute(time_label) -> timedelta: + def get_minute(time_label: str) -> timedelta: return timedelta(minutes=int(time_label) % 2 * 30) trace_data = trace_data.with_columns( diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index e5f531b..540cd1e 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -158,5 +158,5 @@ def filter_mapping_by_names_in_input_files( return filtered_mapping -def get_just_filepaths(metadata_for_files) -> list: +def get_just_filepaths(metadata_for_files: dict) -> list: return [file for file, metadata in metadata_for_files.items()] diff --git a/tests/create_end_to_end_test_data.py b/tests/create_end_to_end_test_data.py index 85e1fa9..d0d5127 100644 --- a/tests/create_end_to_end_test_data.py +++ b/tests/create_end_to_end_test_data.py @@ -6,7 +6,7 @@ import pandas as pd -def generate_random_data(start_year, end_year) -> pd.DataFrame: +def generate_random_data(start_year: int, end_year: int) -> pd.DataFrame: # Generate date range from July 1st of the start year to July 1st of the end year (excluding end) date_range = pd.date_range( start=f"{start_year}-01-01", end=f"{end_year}-01-01", freq="D", inclusive="left" @@ -30,7 +30,7 @@ def generate_random_data(start_year, end_year) -> pd.DataFrame: data = generate_random_data(start_year=config.start, end_year=config.end) -def simple_flatten(nested_list) -> list: +def simple_flatten(nested_list: list) -> list: flattened = [] for item in nested_list: if isinstance(item, list): @@ -40,7 +40,7 @@ def simple_flatten(nested_list) -> list: return flattened -def create_solar_csvs(directory) -> None: +def create_solar_csvs(directory: Path) -> None: combos = itertools.product(config.reference_years, config.solar_projects) for y, project in combos: data.to_csv(directory / Path(f"{project}_FFP_RefYear{y}.csv"), index=False) @@ -52,7 +52,7 @@ def create_solar_csvs(directory) -> None: ) -def create_wind_csvs(directory) -> None: +def create_wind_csvs(directory: Path) -> None: combos = itertools.product( config.reference_years, simple_flatten(config.wind_projects.values()) ) @@ -70,7 +70,7 @@ def create_wind_csvs(directory) -> None: ) -def create_demand_csvs(directory) -> None: +def create_demand_csvs(directory: Path) -> None: combos = itertools.product( config.reference_years, config.sub_regions, diff --git a/tests/test_demand_trace_metadata.py b/tests/test_demand_trace_metadata.py index 44ffd47..1581a7d 100644 --- a/tests/test_demand_trace_metadata.py +++ b/tests/test_demand_trace_metadata.py @@ -40,6 +40,6 @@ def test_build() -> None: "VIC_RefYear_2011_MYSTERY_POE10_OPSO_MODELLING.csv", # lookup miss ], ) -def test_build_rejects_unexpected_filename(filename) -> None: +def test_build_rejects_unexpected_filename(filename: str) -> None: with pytest.raises(ValueError, match="Unexpected trace filename"): demand_trace_metadata.build([Path(filename)], version="2024") diff --git a/tests/test_download.py b/tests/test_download.py index f805014..391713e 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -31,7 +31,7 @@ def test_download_with_retry() -> None: assert (tmp_path / "test" / "test" / "test_file.txt").exists() -def test_fetch_trace_data_with_test_manifest(monkeypatch) -> None: +def test_fetch_trace_data_with_test_manifest(monkeypatch: pytest.MonkeyPatch) -> None: """Test downloading from a small, test manifest. The testing manifest, while still named "full_isp_2024" here, is just a test manifest with containing a single url ("https://data.openisp.au/test/test/test_file.txt") @@ -68,7 +68,7 @@ def test_manifest_not_found() -> None: @pytest.mark.parametrize("unquote", [True, False]) -def test_fetch_trace_data(unquote: bool, monkeypatch) -> None: +def test_fetch_trace_data(unquote: bool, monkeypatch: pytest.MonkeyPatch) -> None: """Test downloading via fetch_trace_data with test fixtures. This, while still download a dataset name "full", is just a pointing to a test manifest manifest with containing a single url ("https://data.openisp.au/test/test/test_file.txt") @@ -115,7 +115,7 @@ def test_wrong_type() -> None: download.fetch_trace_data("other", "isp_2024", "/", "archive") -def test_empty_manifest(monkeypatch) -> None: +def test_empty_manifest(monkeypatch: pytest.MonkeyPatch) -> None: """Test that empty manifest raises ValueError.""" with TemporaryDirectory() as tmp_path: tmp_path = Path(tmp_path) diff --git a/tests/test_get_data.py b/tests/test_get_data.py index fa5438a..ffe76e7 100644 --- a/tests/test_get_data.py +++ b/tests/test_get_data.py @@ -43,7 +43,7 @@ def test_year_range_to_dt_range_calendar() -> None: @pytest.mark.parametrize("year_type", ["fy", "calendar"]) def test_get_zone_single_reference_year( - parsed_trace_trace_directory: Path, year_type + parsed_trace_trace_directory: Path, year_type: str ) -> None: test_df_lazy = pl.scan_parquet(TEST_DATA / "output" / "RefYear2022_N2_CST.parquet") @@ -212,7 +212,7 @@ def test_get_demand_multiple_reference_year(parsed_trace_trace_directory: Path) pd.testing.assert_frame_equal(test_df, df) -def test_explicit_select_columns(parsed_trace_trace_directory) -> None: +def test_explicit_select_columns(parsed_trace_trace_directory: Path) -> None: df = get_zone_single_reference_year( start_year=2023, end_year=2024, @@ -225,7 +225,7 @@ def test_explicit_select_columns(parsed_trace_trace_directory) -> None: assert list(df.columns) == ["datetime", "value", "zone"] -def test_multi_value_filter(parsed_trace_trace_directory) -> None: +def test_multi_value_filter(parsed_trace_trace_directory: Path) -> None: df = get_zone_single_reference_year( start_year=2023, end_year=2024, @@ -238,7 +238,7 @@ def test_multi_value_filter(parsed_trace_trace_directory) -> None: assert "zone" in df.columns -def test_wind_project_single_reference_year(parsed_trace_trace_directory) -> None: +def test_wind_project_single_reference_year(parsed_trace_trace_directory: Path) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Bodangora_Wind_Farm.parquet" ) @@ -264,7 +264,9 @@ def test_wind_project_single_reference_year(parsed_trace_trace_directory) -> Non pd.testing.assert_frame_equal(test_df, df) -def test_solar_project_single_reference_year(parsed_trace_trace_directory) -> None: +def test_solar_project_single_reference_year( + parsed_trace_trace_directory: Path, +) -> None: test_df_lazy = pl.scan_parquet( TEST_DATA / "output" / "RefYear2022_Broken_Hill_Solar_Farm_FFP.parquet" ) diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py index 56c571a..892b375 100644 --- a/tests/test_input_validation.py +++ b/tests/test_input_validation.py @@ -31,7 +31,7 @@ }, ], ) -def test_solar_metadata_filter_valid(valid_input) -> None: +def test_solar_metadata_filter_valid(valid_input: dict[str, list]) -> None: assert SolarMetadataFilter(**valid_input) @@ -44,7 +44,9 @@ def test_solar_metadata_filter_valid(valid_input) -> None: ({"name": 123}, "Input should be a valid list"), ], ) -def test_solar_metadata_filter_invalid(invalid_input, expected_error) -> None: +def test_solar_metadata_filter_invalid( + invalid_input: dict[str, list], expected_error: str +) -> None: with pytest.raises(ValidationError, match=expected_error): SolarMetadataFilter(**invalid_input) @@ -64,7 +66,7 @@ def test_solar_metadata_filter_invalid(invalid_input, expected_error) -> None: }, ], ) -def test_wind_metadata_filter_valid(valid_input) -> None: +def test_wind_metadata_filter_valid(valid_input: dict[str, list]) -> None: assert WindMetadataFilter(**valid_input) @@ -80,7 +82,9 @@ def test_wind_metadata_filter_valid(valid_input) -> None: ({"name": 123}, "Input should be a valid list"), ], ) -def test_wind_metadata_filter_invalid(invalid_input, expected_error) -> None: +def test_wind_metadata_filter_invalid( + invalid_input: dict[str, list], expected_error: str +) -> None: with pytest.raises(ValidationError, match=expected_error): WindMetadataFilter(**invalid_input) @@ -102,7 +106,7 @@ def test_wind_metadata_filter_invalid(invalid_input, expected_error) -> None: }, ], ) -def test_demand_metadata_filter_valid(valid_input) -> None: +def test_demand_metadata_filter_valid(valid_input: dict[str, list]) -> None: assert DemandMetadataFilter(**valid_input) @@ -122,7 +126,9 @@ def test_demand_metadata_filter_valid(valid_input) -> None: ({"subregion": 123}, "Input should be a valid list"), ], ) -def test_demand_metadata_filter_invalid(invalid_input, expected_error) -> None: +def test_demand_metadata_filter_invalid( + invalid_input: dict[str, str | int | list], expected_error: str +) -> None: with pytest.raises(ValidationError, match=expected_error): DemandMetadataFilter(**invalid_input) @@ -145,7 +151,7 @@ def test_demand_metadata_filter_invalid(invalid_input, expected_error) -> None: }, ], ) -def test_parse_traces_validation(invalid_input) -> None: +def test_parse_traces_validation(invalid_input: dict[str, str | int | list]) -> None: with pytest.raises(ValidationError): parse_solar_traces(**invalid_input) with pytest.raises(ValidationError): @@ -163,7 +169,9 @@ def test_parse_traces_validation(invalid_input) -> None: {"start_year": 2030, "end_year": 2035, "reference_years": [2011, "x", 2018]}, ], ) -def test_construct_reference_year_mapping_validation_invalid(invalid_input) -> None: +def test_construct_reference_year_mapping_validation_invalid( + invalid_input: dict, +) -> None: with pytest.raises(ValidationError): construct_reference_year_mapping(**invalid_input) @@ -178,7 +186,7 @@ def test_construct_reference_year_mapping_validation_valid() -> None: # Tests for custom input validation functions -def test_input_directory(tmp_path) -> None: +def test_input_directory(tmp_path: Path) -> None: valid_dir = tmp_path / "valid_dir" valid_dir.mkdir() assert input_validation.input_directory(valid_dir) == valid_dir @@ -194,7 +202,7 @@ def test_input_directory(tmp_path) -> None: Path("/valid/path"), ], ) -def test_parsed_directory_valid(valid_path) -> None: +def test_parsed_directory_valid(valid_path: Path | str) -> None: result = input_validation.parsed_directory(valid_path) assert isinstance(result, Path) @@ -207,7 +215,7 @@ def test_parsed_directory_valid(valid_path) -> None: [], ], ) -def test_parsed_directory_invalid(invalid_path) -> None: +def test_parsed_directory_invalid(invalid_path: Path | str) -> None: with pytest.raises(ValueError, match="Invalid parsed directory path"): input_validation.parsed_directory(invalid_path) @@ -219,7 +227,7 @@ def test_parsed_directory_invalid(invalid_path) -> None: Path("/valid/path"), ], ) -def test_is_valid_path_valid(valid_path) -> None: +def test_is_valid_path_valid(valid_path: Path | str) -> None: result = input_validation.is_valid_path(valid_path) assert isinstance(result, Path) @@ -232,7 +240,7 @@ def test_is_valid_path_valid(valid_path) -> None: [], ], ) -def test_is_valid_path_invalid(invalid_path) -> None: +def test_is_valid_path_invalid(invalid_path: Path | str) -> None: with pytest.raises(ValueError, match="Invalid parsed directory path"): input_validation.is_valid_path(invalid_path) @@ -245,7 +253,7 @@ def test_is_valid_path_invalid(invalid_path) -> None: (-10, 0), ], ) -def test_start_year_before_end_year_valid(start, end) -> None: +def test_start_year_before_end_year_valid(start: int, end: int) -> None: assert input_validation.start_year_before_end_year(start, end) is None @@ -257,6 +265,6 @@ def test_start_year_before_end_year_valid(start, end) -> None: (2020, 2019), ], ) -def test_start_year_before_end_year_invalid(start, end) -> None: +def test_start_year_before_end_year_invalid(start: int, end: int) -> None: with pytest.raises(ValueError, match="Start year .* < end year"): input_validation.start_year_before_end_year(start, end) diff --git a/tests/test_optimise_parquet.py b/tests/test_optimise_parquet.py index 16f64ed..9a6dc73 100644 --- a/tests/test_optimise_parquet.py +++ b/tests/test_optimise_parquet.py @@ -13,7 +13,9 @@ "expected_data, file_type", [("zone_data_0.parquet", "zone"), ("project_data_0.parquet", "project")], ) -def test_optimisation(parsed_trace_trace_directory, expected_data, file_type) -> None: +def test_optimisation( + parsed_trace_trace_directory: Path, expected_data: str, file_type: str +) -> None: """Test wind trace parsing produces expected parquet outputs (both for a sample wind project and wind zone)""" test_output_parquet = TEST_DATA / "output" / expected_data diff --git a/tests/test_resource_trace_metadata.py b/tests/test_resource_trace_metadata.py index 5a724b8..f14d6b3 100644 --- a/tests/test_resource_trace_metadata.py +++ b/tests/test_resource_trace_metadata.py @@ -35,6 +35,6 @@ def test_build() -> None: "Mystery_Plant_RefYear2011.csv", # stem not in mapping ], ) -def test_build_rejects_unexpected_filename(filename) -> None: +def test_build_rejects_unexpected_filename(filename: str) -> None: with pytest.raises(ValueError, match="Unexpected trace filename"): resource_trace_metadata.build([Path(filename)], version="2024") diff --git a/tests/test_trace_parsers.py b/tests/test_trace_parsers.py index c0b255f..cd45801 100644 --- a/tests/test_trace_parsers.py +++ b/tests/test_trace_parsers.py @@ -44,7 +44,7 @@ def test_demand_trace_parsing(use_concurrency: bool) -> None: ], ) def test_wind_trace_parsing( - parsed_trace_trace_directory, expected_filename, file_type + parsed_trace_trace_directory: Path, expected_filename: str, file_type: str ) -> None: """Test wind trace parsing produces expected parquet outputs (both for a sample wind project and wind zone)""" test_output_parquet = TEST_DATA / "output" / expected_filename @@ -65,7 +65,7 @@ def test_wind_trace_parsing( ], ) def test_solar_trace_parsing( - parsed_trace_trace_directory, expected_filename, file_type + parsed_trace_trace_directory: Path, expected_filename: str, file_type: str ) -> None: """Test solar trace parsing produces expected parquet output (both for a sample solar project and solar zone)""" test_output_parquet = TEST_DATA / "output" / expected_filename From 55fcfc85eaea04436f30768d17519cd0b8ec4b1c Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 18:31:02 +1000 Subject: [PATCH 15/18] Fix test to reflect change of exception type. --- tests/test_input_validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py index 892b375..ca3fcbb 100644 --- a/tests/test_input_validation.py +++ b/tests/test_input_validation.py @@ -190,8 +190,7 @@ def test_input_directory(tmp_path: Path) -> None: valid_dir = tmp_path / "valid_dir" valid_dir.mkdir() assert input_validation.input_directory(valid_dir) == valid_dir - - with pytest.raises(ValueError, match="Directory .* does not exist"): + with pytest.raises(FileNotFoundError): input_validation.input_directory(tmp_path / "non_existent_dir") From e59dfe00a04153e8f987a39aef9a7a1a8af5d7ad Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Tue, 8 Sep 2026 20:18:58 +1000 Subject: [PATCH 16/18] Fix typo introduced by Emacs 'transport-words. Whoops! --- src/isp_trace_parser/demand_traces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/isp_trace_parser/demand_traces.py b/src/isp_trace_parser/demand_traces.py index 045a478..7b0f3de 100644 --- a/src/isp_trace_parser/demand_traces.py +++ b/src/isp_trace_parser/demand_traces.py @@ -42,10 +42,10 @@ class DemandMetadataFilter(BaseModel): scenario: list of scenarios, only including "Step Change", "Progressive Change", and "Green Energy Exports" poe: list of POE levels, only including "POE10" and "POE50" demand_type, list of demand types, only including "OPSO_MODELLING", "OPSO_MODELLING_PVLITE", and "PV_TOT" - reference_year: list of ints specifying reference_subregion + reference_year: list of ints specifying reference_years """ - years: list[str] | None = None + subregion: list[str] | None = None scenario: ( list[Literal["Step Change", "Progressive Change", "Green Energy Exports"]] | None From c0a8709a006c6b13cfc81b6603a8b2e847311dff Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Wed, 9 Sep 2026 13:40:02 +1000 Subject: [PATCH 17/18] Fix more Ruff warnings. A few subtle things to point out here: -- We silence S608 about forming SQL queries using user inputs; -- We now use Python logging instead of calling print(); -- Avoid implicit Optional typing, ie. prefer: varname: list[str] | None = None over: varname: list[str] = None --- create_trace_mapping.py | 10 +++-- src/isp_trace_parser/demand_trace_metadata.py | 3 +- src/isp_trace_parser/get_data.py | 16 ++++---- src/isp_trace_parser/input_validation.py | 8 ++-- src/isp_trace_parser/optimise_parquet.py | 12 ++++-- src/isp_trace_parser/remote/download.py | 39 +++++++++++-------- .../resource_trace_metadata.py | 3 +- .../trace_restructure_helper_functions.py | 20 ++++------ 8 files changed, 62 insertions(+), 49 deletions(-) diff --git a/create_trace_mapping.py b/create_trace_mapping.py index 2b8ef57..1dc449a 100644 --- a/create_trace_mapping.py +++ b/create_trace_mapping.py @@ -1,3 +1,5 @@ +from pathlib import Path + import yaml from generator_to_trace_draft_mapper import ( draft_solar_generator_to_trace_mapping, @@ -19,14 +21,14 @@ solar_generator_mapping = draft_solar_generator_to_trace_mapping( solar_gens, solar_traces ) -with open("draft_solar_generator_mapping.yaml", "w") as file: +with Path("draft_solar_generator_mapping.yaml").open("w") as file: yaml.dump(solar_generator_mapping, file, default_flow_style=False) solar_traces = "/media/nick/Samsung_T5/isp_2024_data/trace_data/solar/solar_2023" rezs = gets_rezs(workbook) solar_rez_mapping = draft_solar_rez_mapping(rezs, solar_traces) -with open("solar_area_mapping.yaml", "w") as file: +with Path("solar_area_mapping.yaml").open("w") as file: yaml.dump(solar_rez_mapping, file, default_flow_style=False) duids_and_station_names = static_table( @@ -47,12 +49,12 @@ wind_generator_mapping = draft_wind_generator_to_trace_mapping( wind_gens, wind_duids_and_station_names, wind_traces ) -with open("draft_wind_generator_mapping.yaml", "w") as file: +with Path("draft_wind_generator_mapping.yaml").open("w") as file: yaml.dump(wind_generator_mapping, file, default_flow_style=False, sort_keys=False) wind_traces = "D:/isp_2024_data/trace_data/wind/wind_2023" rezs = gets_rezs(workbook) wind_rez_mapping = draft_wind_rez_mapping(rezs, wind_traces) -with open("draft_wind_rez_mapping.yaml", "w") as file: +with Path("draft_wind_rez_mapping.yaml").open("w") as file: yaml.dump(wind_rez_mapping, file, default_flow_style=False) diff --git a/src/isp_trace_parser/demand_trace_metadata.py b/src/isp_trace_parser/demand_trace_metadata.py index 7a1eff9..2b77896 100644 --- a/src/isp_trace_parser/demand_trace_metadata.py +++ b/src/isp_trace_parser/demand_trace_metadata.py @@ -29,7 +29,8 @@ def build( refyear, _, dimensions_suffix = after.partition("_") key = (location_prefix, dimensions_suffix) if not refyear.isdigit() or key not in lookup: - raise ValueError(f"Unexpected trace filename: {path.name}") + msg = f"Unexpected trace filename: {path.name}" + raise ValueError(msg) file_metadata[path] = {**lookup[key], "reference_year": int(refyear)} return file_metadata diff --git a/src/isp_trace_parser/get_data.py b/src/isp_trace_parser/get_data.py index 078b220..1cf4f91 100644 --- a/src/isp_trace_parser/get_data.py +++ b/src/isp_trace_parser/get_data.py @@ -56,8 +56,8 @@ def _query_parquet_single_reference_year( end_year: int, reference_year: int, directory: str | Path, - filters: dict[str, any] = None, - select_columns: list[str] = None, + filters: dict[str, any] | None = None, + select_columns: list[str] | None = None, year_type: Literal["fy", "calendar"] = "fy", ) -> pd.DataFrame: """ @@ -155,7 +155,7 @@ def get_project_single_reference_year( project: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query project trace data for a single reference year. @@ -248,7 +248,7 @@ def get_zone_single_reference_year( resource_type: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query zone trace data for a single reference year. @@ -346,7 +346,7 @@ def get_demand_single_reference_year( poe: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query demand trace data for a single reference year. @@ -451,7 +451,7 @@ def get_project_multiple_reference_years( project: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query project trace data across multiple reference years. @@ -541,7 +541,7 @@ def get_zone_multiple_reference_years( resource_type: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query zone trace data across multiple reference years. @@ -636,7 +636,7 @@ def get_demand_multiple_reference_years( poe: str | list, directory: str | Path, year_type: Literal["fy", "calendar"] = "fy", - select_columns: list[str] = None, + select_columns: list[str] | None = None, ) -> pd.DataFrame: """ Query demand trace data across multiple reference years. diff --git a/src/isp_trace_parser/input_validation.py b/src/isp_trace_parser/input_validation.py index aeee6b9..a56cc44 100644 --- a/src/isp_trace_parser/input_validation.py +++ b/src/isp_trace_parser/input_validation.py @@ -22,10 +22,12 @@ def parsed_directory(path: str | Path) -> Path: def is_valid_path(path: str | Path) -> Path: try: return Path(path) - except (TypeError, ValueError): - raise ValueError(f"Invalid parsed directory path: {path}") + except (TypeError, ValueError) as exc: + msg = f"Invalid parsed directory path: {path}" + raise ValueError(msg) from exc def start_year_before_end_year(start_year: int, end_year: int) -> None: if end_year < start_year: - raise ValueError(f"Start year {end_year} < end year {start_year}") + msg = f"Start year {end_year} < end year {start_year}" + raise ValueError(msg) diff --git a/src/isp_trace_parser/optimise_parquet.py b/src/isp_trace_parser/optimise_parquet.py index 283a3bf..a549377 100644 --- a/src/isp_trace_parser/optimise_parquet.py +++ b/src/isp_trace_parser/optimise_parquet.py @@ -29,7 +29,7 @@ def partition_traces_by_columns( input_directory: str | Path, output_directory: str | Path, partition_cols: list[str], - sort_by: list[str] | None = ["datetime"], + sort_by: list[str] | None = None, ) -> None: """Partition parquet traces by specified columns with optional sorting. @@ -59,6 +59,12 @@ def partition_traces_by_columns( ... partition_cols=["scenario", "reference_year"] ... ) # doctest: +SKIP """ + + if sort_by is None: + # Avoid use of mutable data structure for argument defaults + # (see Ruff rule B006). + sort_by = ["datetime"] + output_path = Path(output_directory) output_path.mkdir(parents=True, exist_ok=True) @@ -70,7 +76,7 @@ def partition_traces_by_columns( values = con.execute(f""" SELECT DISTINCT {col} FROM read_parquet('{input_directory}') - """).fetchall() + """).fetchall() # noqa: S608 distinct_values.append(values) partitions = [tuple(val[0] for val in vals) for vals in product(*distinct_values)] @@ -84,7 +90,7 @@ def partition_traces_by_columns( conditions.append(f"{col}={val}") where_clause = " AND ".join(conditions) - query = f"SELECT * FROM read_parquet('{input_directory}') WHERE {where_clause}" + query = f"SELECT * FROM read_parquet('{input_directory}') WHERE {where_clause}" # noqa: S608 if sort_by: query += f" ORDER BY {', '.join(sort_by)}" diff --git a/src/isp_trace_parser/remote/download.py b/src/isp_trace_parser/remote/download.py index b9106a7..401b753 100644 --- a/src/isp_trace_parser/remote/download.py +++ b/src/isp_trace_parser/remote/download.py @@ -7,6 +7,7 @@ """Download data files from manifests.""" +import logging import time from importlib.resources import files from pathlib import Path @@ -16,6 +17,9 @@ import requests from tqdm import tqdm +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + def _download_from_manifest( manifest_name: str, @@ -64,11 +68,12 @@ def _download_from_manifest( raise FileNotFoundError(manifest_path) # Read URLs from manifest - with open(manifest_path) as f: + with Path(manifest_path).open("r") as f: urls = [line.strip() for line in f if line.strip()] if not urls: - raise ValueError(f"No URLs found in manifest: {manifest_path}") + msg = f"No URLs found in manifest: {manifest_path}" + raise ValueError(msg) save_directory = Path(save_directory) @@ -88,7 +93,7 @@ def _download_with_retry( for attempt in range(max_retries): try: _download_file(url, save_directory, strip_levels, unquote_path) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException: # noqa: PERF203 if attempt < max_retries - 1: time.sleep(2**attempt) else: @@ -132,10 +137,9 @@ def _download_file( # Strip specified number of directory levels path_parts = url_path.split("/") if strip_levels >= len(path_parts): - raise ValueError( - f"Cannot strip {strip_levels} levels from path with only " - f"{len(path_parts)} parts: {url_path}" - ) + msg = f"Cannot strip {strip_levels} levels from path with only " + f"{len(path_parts)} parts: {url_path}" + raise ValueError(msg) stripped_path = "/".join(path_parts[strip_levels:]) destination = save_directory / stripped_path @@ -152,7 +156,7 @@ def _download_file( # Write file with progress bar with ( - open(destination, "wb") as f, + Path(destination).open("wb") as f, tqdm( total=total_size, unit="B", @@ -219,23 +223,24 @@ def fetch_trace_data( # Validate inputs if dataset_type not in ["full", "example"]: - raise ValueError( - f"dataset_type must be 'full' or 'example', got: {dataset_type}" - ) + msg = f"dataset_type must be 'full' or 'example', got: {dataset_type}" + raise ValueError(msg) if dataset_src != "isp_2024": - raise ValueError(f"Only isp_2024 is currently supported, got: {dataset_src}") + msg = f"Only isp_2024 is currently supported, got: {dataset_src}" + raise ValueError(msg) if data_format not in ["processed", "archive"]: - raise ValueError( - f"data_format must be 'processed' or 'archive', got: {data_format}" - ) + msg = f"data_format must be 'processed' or 'archive', got: {data_format}" + raise ValueError(msg) # Construct manifest name and download manifest_name = f"{data_format}/{dataset_type}_{dataset_src}" - print(f"Downloading {dataset_type} {data_format} trace data for {dataset_src}") + logger.info( + "Downloading %s %s trace data for %s", dataset_type, data_format, dataset_src + ) _download_from_manifest( manifest_name, save_directory, strip_levels=2, unquote_path=unquote_path ) - print(f"Trace data saved to: {save_directory}") + logger.info("Trace data saved to: %s", save_directory) diff --git a/src/isp_trace_parser/resource_trace_metadata.py b/src/isp_trace_parser/resource_trace_metadata.py index f5ffa95..8cedc7a 100644 --- a/src/isp_trace_parser/resource_trace_metadata.py +++ b/src/isp_trace_parser/resource_trace_metadata.py @@ -40,7 +40,8 @@ def build( for path in files: stem, sep, ref = path.stem.rpartition("_RefYear") if not sep or not ref.isdigit() or stem not in resource_mapping: - raise ValueError(f"Unexpected trace filename: {path.name}") + msg = f"Unexpected trace filename: {path.name}" + raise ValueError(msg) entry = resource_mapping[stem] file_metadata[path] = { "name": entry["location"], diff --git a/src/isp_trace_parser/trace_restructure_helper_functions.py b/src/isp_trace_parser/trace_restructure_helper_functions.py index bddeda4..0a74dbb 100644 --- a/src/isp_trace_parser/trace_restructure_helper_functions.py +++ b/src/isp_trace_parser/trace_restructure_helper_functions.py @@ -5,7 +5,6 @@ # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. -from datetime import timedelta from pathlib import Path import polars as pl @@ -76,11 +75,7 @@ def process_and_save_files( ) -> None: traces = read_and_format_traces(files) - if len(traces) > 1: - trace = calculate_average_trace(traces) - else: - trace = traces[0] - + trace = calculate_average_trace(traces) if len(traces) > 1 else traces[0] trace = _frame_with_metadata(trace, file_metadata) save_trace(trace, file_metadata, output_directory, write_output_filepath) @@ -136,9 +131,12 @@ def check_filter_by_metadata( return True for field, allowed_values in filters.model_dump(exclude_unset=True).items(): - if field in metadata and allowed_values is not None: - if metadata[field] not in allowed_values: - return False + if ( + field in metadata + and allowed_values is not None + and metadata[field] not in allowed_values + ): + return False return True @@ -146,9 +144,7 @@ def check_filter_by_metadata( def get_unique_project_and_zone_names_in_input_files( metadata_for_trace_files: dict[Path, dict[str, str]], ) -> list[str]: - names = [] - for filepath, meta_data in metadata_for_trace_files.items(): - names.append(meta_data["name"]) + names = [meta_data["name"] for meta_data in metadata_for_trace_files.values()] return list(set(names)) From 04c853e9e13463bfd7394021b868b07b4ddc9967 Mon Sep 17 00:00:00 2001 From: Ben Elliston Date: Fri, 11 Sep 2026 17:19:24 +1000 Subject: [PATCH 18/18] Back out logging changes. Fix multi-line construction of 'msg'. --- src/isp_trace_parser/remote/download.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/isp_trace_parser/remote/download.py b/src/isp_trace_parser/remote/download.py index 401b753..d16bfbd 100644 --- a/src/isp_trace_parser/remote/download.py +++ b/src/isp_trace_parser/remote/download.py @@ -7,7 +7,6 @@ """Download data files from manifests.""" -import logging import time from importlib.resources import files from pathlib import Path @@ -17,9 +16,6 @@ import requests from tqdm import tqdm -logging.basicConfig(level=logging.INFO, format="%(message)s") -logger = logging.getLogger(__name__) - def _download_from_manifest( manifest_name: str, @@ -137,8 +133,10 @@ def _download_file( # Strip specified number of directory levels path_parts = url_path.split("/") if strip_levels >= len(path_parts): - msg = f"Cannot strip {strip_levels} levels from path with only " - f"{len(path_parts)} parts: {url_path}" + msg = ( + f"Cannot strip {strip_levels} levels from path with only " + f"{len(path_parts)} parts: {url_path}" + ) raise ValueError(msg) stripped_path = "/".join(path_parts[strip_levels:]) @@ -237,10 +235,8 @@ def fetch_trace_data( # Construct manifest name and download manifest_name = f"{data_format}/{dataset_type}_{dataset_src}" - logger.info( - "Downloading %s %s trace data for %s", dataset_type, data_format, dataset_src - ) + print(f"Downloading {dataset_type} {data_format} trace data for {dataset_src}") _download_from_manifest( manifest_name, save_directory, strip_levels=2, unquote_path=unquote_path ) - logger.info("Trace data saved to: %s", save_directory) + print(f"Trace data saved to: {save_directory}")