diff --git a/.gitignore b/.gitignore index 94a5e52..5dec1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,6 @@ cython_debug/ .idea/ scratch.py .~lock*.xlsx# + +# macOS +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d5030c1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ +# Changelog + +Notable changes to `isp-workbook-parser`, for people using the package +(new workbook versions, renamed/removed/added tables, and new or changed public config/API) + +## [2.9.0] - 2026-09-15 + +### Added + +- Support for IASR workbook version 7.8 (final 2026 ISP workbook). + - Includes new "Hybrid site limits" sheet and 7 new tables. + - More details of workbook changes here: + [workbook_content_changes_7.8.md](https://github.com/Open-ISP/project-planning/blob/main/ISP2026/workbook_content_changes_7.8.md). +- `skip_checks` table config option: Allows you to opt a specific table out from specific validation checks, (rather than disabling all checks for it). + +### Changed / Fixed + +- **Table renames** (impacts version 7.8 of the workbook, as well as 7.3/7.5 retrospectivelly). + - `*_reduced_energy_efficiency` → `*_lower_energy_efficiency` + - `flow_path_augmentation_cost_slower_growth_CNSW-NNSW` → + `flow_path_augmentation_costs_slower_growth_CNSW-NNSW` + - Two clashing table names resolved (one of tables was previously being shadowed/ignored): + - `water_for_hydrogen` → + `desalination_electricity_demand_for_h2` + - `embedded_storage_consultant_scenario_mapping` + → `aggregated_storage_consultant_scenario_mapping` +- Fixed a validation issue that resulted in some tables dropping rows when a table's second column happened to be blank + - (affected `marginal_loss_factors_existing_generators` in 7.8). +- Fixed sanitiser handling of `[footnote]`-style markers and notes containing `$` amounts (which were previously partly sanitising the values). +- Cells holding two distinct values (e.g. `"250 (generation) 325 (pump)"`) are intentionally returned as text (instead of choosing, for example, the first value). +- Dropped Python 3.9 support; added Python 3.14. diff --git a/README.md b/README.md index b9d1ce0..ceca1b7 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install isp-workbook-parser 1. Load a workbook using `Parser` (see examples below). - While we do not include workbooks with the package distribution, you can find the versions for which table configurations are written within `workbooks/`. -2. Table configuration files for data tables are located in `src/config/` +2. Table configuration files for data tables are located in `src/isp_table_configs/` - These specify the name, location, columns and data range of tables to be extracted from a particular workbook version. Optionally, rows to skip and not read in (e.g. where AEMO has formatted a row with a strike through to indicate that the data is no longer being used) and columns with merged rows can also be specified and handled. - These are included with the package distributions. 3. `Parser` loads the MS Excel workbook and, by default, will check if the version of the workbook is supported by seeing if configuration files are included in the package for that version. @@ -40,7 +40,7 @@ pip install isp-workbook-parser > [!NOTE] > This package makes some opinionated decisions when processing tables. For example, > multiple header row tables are reduced to a single header, data in merged cells is inferred from surrounding cells, -> and notes and footnotes are dropped (amonst other ways in which the data is sanitised). +> and notes and footnotes are dropped (amongst other ways in which the data is sanitised). > For more detail, refer to the docstring and code in [`read_table.py`](https://github.com/Open-ISP/isp-workbook-parser/blob/main/src/isp_workbook_parser/read_table.py) > and [`sanitisers.py`](https://github.com/Open-ISP/isp-workbook-parser/blob/main/src/isp_workbook_parser/sanitisers.py). diff --git a/example_output/7.5/flow_path_augmentation_cost_slower_growth_CNSW-NNSW.csv b/example_output/7.5/flow_path_augmentation_costs_slower_growth_CNSW-NNSW.csv similarity index 100% rename from example_output/7.5/flow_path_augmentation_cost_slower_growth_CNSW-NNSW.csv rename to example_output/7.5/flow_path_augmentation_costs_slower_growth_CNSW-NNSW.csv diff --git a/example_output/7.8/marginal_loss_factors_existing_generators.csv b/example_output/7.8/marginal_loss_factors_existing_generators.csv index d77bc9a..b3771ae 100644 --- a/example_output/7.8/marginal_loss_factors_existing_generators.csv +++ b/example_output/7.8/marginal_loss_factors_existing_generators.csv @@ -641,3 +641,87 @@ Kiamal BESS,,Kiamal BESS,Additional policy-supported project,0.8785,Kiamal Solar Pine Lodge BESS,,Pine Lodge BESS,Committed,0.9463,Numurkah Solar Farm Dubbo GT,,Dubbo Firming Power Station,Anticipated,1.0, Axedale Solar Farm,,Axedale Solar Farm,Anticipated,0.9463,VSHS +Bendemeer Energy Hub Solar Farm,,Bendemeer Energy Hub Solar Farm,Additional policy-supported project,0.8803,NURR +Blind Creek Solar Farm,,Blind Creek Solar Farm,Committed,0.9628,NCLW +Bundey Solar Farm,,Bundey Solar Farm,Additional policy-supported project,0.9697,SROB +Corop Solar Farm,,Corop Solar Farm,Additional policy-supported project,0.9463,VSHS +Dunedoo Solar Farm,,Dunedoo Solar Farm,Anticipated,0.9429,NSTB +Fosterville Solar Farm,,Fosterville Solar Farm,Anticipated,1.0315,VSHT +Fraser Coast Hybrid Solar Farm,,Fraser Coast Hybrid Solar Farm,Anticipated,0.9787,QWLS +GUSF1,,Gunsynd Solar Farm,Committed,0.98,QMLN +Guthries Gap Solar Power Station,,Guthries Gap Solar Power Station,Anticipated,0.9328,QMRR +Kerang Solar Plant,,Kerang Solar Plant,Existing,0.9305, +Lower Wonga Solar Farm,,Lower Wonga Solar Farm,Anticipated,0.9787,QWLS +Marulan Solar Farm,,Marulan Solar Farm,Anticipated,0.962,NCW3 +Merino Solar Farm,,Merino Solar Farm,Additional policy-supported project,0.9649,NCWF +Middlebrook Solar Farm,,Middlebrook Solar Farm,Anticipated,0.8803,NURR +Moama Solar Farm,,Moama Solar Farm,Anticipated,0.8898,NFNS +Mulwala Solar Farm,,Mulwala Solar Farm,Committed,0.8898,NFNS +Nowingi Solar Power Station,,Nowingi Solar Power Station,Additional policy-supported project,0.8895,VRCS +Smoky Creek Solar Power Station,,Smoky Creek Solar Power Station,Anticipated,0.9328,QMRR +Wandoan South Solar Farm - Stage 2,,Wandoan South Solar Farm - Stage 2,Committed,0.9975,QWSR +Winton North Solar Farm,,Winton North Solar Farm,Anticipated,0.9402,VGNS +Bell Bay Wind Farm,,Bell Bay Wind Farm,Additional policy-supported project,1.0,TTV1 +Carmodys Hill Wind Farm,,Carmodys Hill Wind Farm,Committed,0.9735,SDAP +Goyder North Wind Farm 2,,Goyder North Wind Farm 2,Anticipated,0.9697,SROB +Hexham Wind Farm,,Hexham Wind Farm,Additional policy-supported project,0.9829,VTRT +Willogoleche Wind Farm 2,,Willogoleche Wind Farm 2,Additional policy-supported project,0.9601, +Armidale East BESS,,Armidale East BESS,Additional policy-supported project,0.8557,NMTZ +Axedale Solar Farm BESS,,Axedale Solar Farm BESS,Anticipated,1.0315,VSHT +Bannaby BESS,,Bannaby BESS,Additional policy-supported project,0.962,NCW3 +Baranduda BESS,,Baranduda BESS,Anticipated,0.9298,NWLA +Bellambi Heights BESS,,Bellambi Heights BESS,Committed,0.9284,NBES +Bendemeer Energy Hub BESS,,Bendemeer Energy Hub BESS,Additional policy-supported project,0.8803,NURR +Blackstone BESS,,Blackstone BESS,Anticipated,0.9994,QSWE +Blackstone BESS 2,,Blackstone BESS 2,Anticipated,0.9994,QSWE +Blind Creek Solar Farm BESS,,Blind Creek Solar Farm BESS,Committed,0.9628,NCLW +Bowmans Creek BESS,,Bowmans Creek BESS,Additional policy-supported project,0.9637,NBAY +Bulabul BESS 1,,Bulabul BESS 1,Committed,0.923,NWLS +Bulabul BESS 2,,Bulabul BESS 2,Additional policy-supported project,0.923,NWLS +Bundey Solar Farm BESS 1,,Bundey Solar Farm BESS 1,Additional policy-supported project,0.9697,SROB +Bundey Solar Farm BESS 2,,Bundey Solar Farm BESS 2,Additional policy-supported project,0.9697,SROB +Bundey Solar Farm BESS 3,,Bundey Solar Farm BESS 3,Additional policy-supported project,0.9697,SROB +Capricorn BESS,,Capricorn BESS,Additional policy-supported project,0.9358,QBCB +Carmodys Hill BESS,,Carmodys Hill BESS,Anticipated,0.9735,SDAP +Corop Solar Farm BESS,,Corop Solar Farm BESS,Additional policy-supported project,0.9463,VSHS +Culcairn BESS,,Culcairn BESS,Anticipated,0.9278,NCRN +Deer Park BESS,,Deer Park BESS,Additional policy-supported project,0.9982,VAT2 +Ebor BESS,,Ebor BESS,Additional policy-supported project,0.8803,NURR +Elaine BESS,,Elaine BESS,Committed,0.9569,VELT +Emeroo BESS,,Emeroo BESS,Anticipated,0.9725,SBEM +Fosterville Solar Farm BESS,,Fosterville Solar Farm BESS,Anticipated,1.0315,VSHT +Fraser Coast Hybrid BESS,,Fraser Coast Hybrid BESS,Anticipated,0.9787,QWLS +Golden Plains BESS,,Golden Plains BESS,Anticipated,0.9834,VGPT +Goulburn River Solar Farm BESS,,Goulburn River Solar Farm BESS,Committed,0.957,NWOW +Goyder BESS,,Goyder BESS,Anticipated,0.9697,SROB +Great Western Battery,,Great Western Battery,Anticipated,0.9727,NMTP +Guthries Gap Solar Power Station BESS,,Guthries Gap Solar Power Station BESS,Anticipated,0.9461,QGLH +Joel Joel BESS,,Joel Joel BESS,Anticipated,0.8907,VBGT +Kingswood BESS,,Kingswood BESS,Additional policy-supported project,0.8803,NURR +Koolunga BESS,,Koolunga BESS,Additional policy-supported project,0.9735,SDAP +Lindley BESS,,Lindley BESS,Anticipated,1.0146,SMW1 +Little River BESS,,Little River BESS,Anticipated,0.9628,NCLW +Lower Wonga BESS,,Lower Wonga BESS,Anticipated,0.9787,QWLS +Marulan BESS,,Marulan BESS,Anticipated,0.962,NCW3 +Merino Solar Farm BESS,,Merino Solar Farm BESS,Additional policy-supported project,0.9649,NCWF +Middlebrook Solar Farm BESS,,Middlebrook Solar Farm BESS,Anticipated,0.8803,NURR +Mt Piper BESS,,Mt Piper BESS,Anticipated,0.9727,NMTP +Noblevale BESS,,Noblevale BESS,Anticipated,0.9994,QSWE +Nowingi Solar Power Station BESS,,Nowingi Solar Power Station BESS,Additional policy-supported project,0.8895,VRCS +Pelican Point BESS,,Pelican Point BESS,Committed,0.9983,SPPT +Punchs Creek Solar Farm BESS,,Punchs Creek Solar Farm BESS,Additional policy-supported project,0.98,QMLN +Reeves Plains Power Station BESS,,Reeves Plains Power Station BESS,Anticipated,1.0146,STET +Smoky Creek Solar Power Station BESS,,Smoky Creek Solar Power Station BESS,Anticipated,0.9358,QBCB +Summerfield BESS 1,,Summerfield BESS 1,Committed,1.0252,SMA3 +Summerfield BESS 2,,Summerfield BESS 2,Committed,1.0252,SMA3 +Supernode BESS 3,,Supernode BESS 3,Anticipated,0.9913,QWIV +Swallow Tail BESS,,Swallow Tail BESS,Additional policy-supported project,0.962,NCW3 +Tailem Bend BESS - Stage 3,,Tailem Bend BESS - Stage 3,Committed,0.9952,STBS +Teebar Creek BESS,,Teebar Creek BESS,Anticipated,0.9787,QWLS +Tomago BESS,,Tomago BESS,Anticipated,0.9901,NERB +Wagga North BESS,,Wagga North BESS,Anticipated,0.9183,NWGG +WDBESS2,,Western Downs Battery,Committed,0.9767,QWDR +Williamsdale BESS,,Williamsdale BESS,Committed,1.0389,NQBC +Yabulu BESS,,Yabulu BESS,Anticipated,0.9508,QTYP +Yanco BESS,,Yanco BESS,Anticipated,0.9396,NRBB +Mugga Lane Renewable Hybrid,,Mugga Lane Renewable Hybrid,Committed,1.0, diff --git a/pyproject.toml b/pyproject.toml index da0ca89..41d9d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "isp-workbook-parser" -version = "2.8.0" +version = "2.9.0" description = "A Python package for reading data from the Inputs, Assumptions and Scenarios Report (IASR) Microsoft Excel workbook published by the Australian Energy Market Operator for use in their Integrated System Plan modelling." authors = [ { name = "nick-gorman", email = "n.gorman305@gmail.com" }, diff --git a/uv.lock b/uv.lock index 56f6f22..72fcf5d 100644 --- a/uv.lock +++ b/uv.lock @@ -306,7 +306,7 @@ wheels = [ [[package]] name = "isp-workbook-parser" -version = "2.8.0" +version = "2.9.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },