Apply the new-format custom constraints in pypsa_build

CLAUDE.md

@@ -45,6 +45,11 @@ def template_network_transmission_paths(iasr_tables, scenario):
 - **Prefer explicit data over clever detection.** If the set of special cases is small and
   stable, declare them as data rather than building logic to infer them from surrounding
   context.
+- **Say "investment period", never bare "period".** On its own, "period" reads as a
+  snapshot or time window. In prose (docstrings, comments, schema descriptions, I/O
+  Example notes) write "investment period", or name the `investment_period` column
+  ("a row with a blank investment_period", not "a blank-period row"). Bare `period` is
+  fine only where it is a literal name, such as PyPSA's snapshot index level.
 
 ### Control flow
 

src/ispypsa/templater/existing_planned.py

@@ -2,10 +2,11 @@
 summary into generator and storage tables, and builds the generator identity and
 property columns.
 
-Both target tables — see schemas/generators_existing_planned.yaml and
-schemas/storage_existing_planned.yaml — are built from the single IASR
-existing_committed_anticipated_additional_generator_summary table, which already lists
-one row per real generating/storage unit (DUID-level). TODO: finish templating storage.
+Both target tables — see schemas/ispypsa_tables/generators_existing_planned.yaml
+and schemas/ispypsa_tables/storage_existing_planned.yaml — are built from the
+single IASR existing_committed_anticipated_additional_generator_summary table,
+which already lists one row per real generating/storage unit (DUID-level). TODO:
+finish templating storage.
 
     existing_committed_anticipated_additional_generator_summary:
         IASR ID / DLT names  Power Station  Technology Type      REZ ID  Sub-region  Fuel type  Fuel cost mapping

src/ispypsa/templater/new_entrants.py

@@ -3,7 +3,8 @@
 Both tables are built from the IASR ``new_entrants_summary`` table (for identity
 columns) plus per-technology property tables. This module splits the summary into
 its two subsets and shapes each into the columns of its target schema (see
-schemas/generators_new_entrant.yaml and schemas/storage_new_entrant.yaml).
+schemas/ispypsa_tables/generators_new_entrant.yaml and
+schemas/ispypsa_tables/storage_new_entrant.yaml).
 
 There are two independent public orchestrators, one per output table. Each one:
     1. Filters the summary to its technology group (generators or storage)

src/ispypsa/translator/timeslices.py

@@ -164,20 +164,90 @@ def _log_referenced_timeslices_without_snapshots(
     custom_constraints_rhs: pd.DataFrame,
 ) -> None:
     """Logs the timeslices referenced by a limit or constraint but mapped to
-    no snapshots — those limits and constraints will never apply.
+    no snapshots where they would apply — those limits and constraints will
+    never apply.
 
     This is expected when snapshot aggregation (e.g. representative weeks)
     selects no snapshots inside a timeslice's windows, and for calendar
     timeslices that never activate (tas_peak_demand in the Draft 2026 ISP
     calendar), but the user should know the affected inputs will not bind.
+
+    Transmission limits apply in every investment period, so they are checked
+    against the model as a whole. Custom constraint rows each apply in one
+    investment period, so they are checked period by period: a short
+    timeslice like qld_peak_demand can be caught by the representative weeks
+    in 2025 but missed in 2030.
     """
-    referenced = set(link_timeslice_limits["timeslice"]) | set(
-        custom_constraints_rhs["timeslice"].dropna()
+    _log_link_timeslices_without_snapshots(timeslice_snapshots, link_timeslice_limits)
+    _log_constraint_timeslices_without_snapshots_in_period(
+        timeslice_snapshots, custom_constraints_rhs
     )
+
+
+def _log_link_timeslices_without_snapshots(
+    timeslice_snapshots: pd.DataFrame, link_timeslice_limits: pd.DataFrame
+) -> None:
+    """Logs the named timeslices in link_timeslice_limits with no snapshots in
+    any investment period. Fallback rows (blank timeslice) aren't checked.
+
+    I/O Example:
+        timeslice_snapshots:
+            timeslice        investment_periods  snapshots
+            nsw_peak_demand  2026                2026-01-13 12:00:00
+
+        link_timeslice_limits:
+            name            attribute  timeslice        value
+            CQ-NQ_existing  p_max_pu   nsw_peak_demand  0.8
+            CQ-NQ_existing  p_max_pu   tas_peak_demand  0.9   # never mapped: logged
+            CQ-NQ_existing  p_max_pu   ,                1.0   # fallback: not checked
+
+        logs: [...] will never apply): ['tas_peak_demand']
+    """
+    referenced = set(link_timeslice_limits["timeslice"].dropna())
     without_snapshots = referenced - set(timeslice_snapshots["timeslice"])
     if without_snapshots:
         logger.warning(
-            f"Timeslices referenced by transmission limits or custom constraints "
-            f"but with no snapshots in the model (these limits and constraints "
-            f"will never apply): {sorted(without_snapshots)}"
+            f"Timeslices referenced by transmission limits but with no snapshots "
+            f"in the model (these limits will never apply): "
+            f"{sorted(without_snapshots)}"
+        )
+
+
+def _log_constraint_timeslices_without_snapshots_in_period(
+    timeslice_snapshots: pd.DataFrame, custom_constraints_rhs: pd.DataFrame
+) -> None:
+    """Logs each (timeslice, investment_period) pair named in
+    custom_constraints_rhs with no snapshots in timeslice_snapshots. Fallback
+    rows (blank timeslice) aren't checked; a named timeslice always has an
+    investment_period (custom_constraints_rhs schema).
+
+    I/O Example:
+        timeslice_snapshots:
+            timeslice        investment_periods  snapshots
+            qld_peak_demand  2025                2025-01-31 12:00:00
+
+        custom_constraints_rhs:
+            constraint_name  investment_period  timeslice
+            SWQLD1           2025               qld_peak_demand
+            SWQLD1           2030               qld_peak_demand   # not in 2030: logged
+            SWQLD1           2030               ,                 # fallback: not checked
+
+        logs: [...] will never apply): [('qld_peak_demand', 2030)]
+    """
+    named = custom_constraints_rhs.dropna(subset=["timeslice"])
+    referenced = set(
+        zip(named["timeslice"], named["investment_period"].astype(int).tolist())
+    )
+    mapped = set(
+        zip(
+            timeslice_snapshots["timeslice"],
+            timeslice_snapshots["investment_periods"].tolist(),
+        )
+    )
+    without_snapshots = referenced - mapped
+    if without_snapshots:
+        logger.warning(
+            f"Timeslices referenced by custom constraints but with no snapshots in "
+            f"the constraint's investment period (these constraints will never "
+            f"apply): {sorted(without_snapshots)}"
         )

src/ispypsa/validation/schemas/pypsa_friendly_tables/custom_constraints_lhs.yaml

@@ -0,0 +1,91 @@
+table: custom_constraints_lhs
+required: false
+unique:
+  - [constraint_name, investment_period, variable_name, component, attribute]
+custom_validation:
+  - name: supported_component_attribute_pairs
+    description: >
+      Each row's (component, attribute) must be one pypsa_build maps onto the
+      model: (Link, p), (Link, p_nom), (Generator, p), (Generator, p_nom),
+      (Storage, p) or (Load, p_set). pypsa_build raises on any other pair,
+      listing every unsupported pair found.
+  - name: terms_belong_to_an_rhs_row
+    description: >
+      Every (constraint_name, investment_period) here must match a
+      custom_constraints_rhs row with the same pair, blank matching blank. So a
+      term with a blank investment_period needs its constraint to have an RHS
+      row with a blank investment_period, and a term with an investment_period
+      an RHS row in that investment period. pypsa_build picks each
+      constraint's terms from its RHS rows, so a term with no matching row is
+      silently ignored. The translator drops the investment periods in which a
+      constraint has only one side before this table is written.
+  - name: variables_name_components_in_the_input_tables
+    description: >
+      variable_name must name a component in the pypsa-friendly table
+      pypsa_build builds that component type from: a Link in links.name; a
+      Generator in generators.name or custom_constraints_generators.name (the
+      relaxation generators); a Storage unit in batteries.name; and a Load as
+      "load_<bus>", where <bus> is in buses.name and has a demand trace
+      (demand_traces/<bus>.parquet), since pypsa_build only attaches loads to
+      buses with one. A p_nom term must also name a row with
+      p_nom_extendable true, as only extendable components have a p_nom
+      variable. The translator raises on terms whose component isn't in the
+      model before this table is written.
+description: >
+  Left-hand-side terms of each custom constraint: one row per (constraint,
+  investment period, model component) coefficient.
+
+  Produced by ispypsa.translator.constraints, which expands each templated
+  term from its input ID into one term per matching PyPSA component (e.g. a
+  path's existing and expansion links) in each investment period the component
+  is in service. Consumed by
+  ispypsa.pypsa_build.custom_constraints._add_custom_constraints_with_temporal_scope.
+
+  If absent:
+  No custom constraints are added to the model.
+columns:
+  constraint_name:
+    type: string
+    required: true
+    allowed_values_from:
+      - custom_constraints_rhs: constraint_name
+    description: Constraint the term belongs to.
+  investment_period:
+    type: int
+    required: false
+    description: >
+      Investment period the term applies in. A term joins the constraints of
+      the custom_constraints_rhs rows with the same constraint_name and
+      investment_period.
+
+      If absent (or empty):
+      The term applies regardless of investment period, joining its
+      constraint's RHS row with a blank investment_period. Only the expansion
+      limits take this form (see terms_belong_to_an_rhs_row).
+  variable_name:
+    type: string
+    required: true
+    description: >
+      Name of the PyPSA component the term's variable belongs to: a link, a
+      generator, a storage unit, or a load ("load_<bus>").
+  component:
+    type: string
+    required: true
+    allowed_values: [Link, Generator, Storage, Load]
+    description: >
+      Type of the component variable_name names. Storage terms apply to
+      StorageUnit components.
+  attribute:
+    type: string
+    required: true
+    allowed_values: [p, p_nom, p_set]
+    description: >
+      The component's variable the term applies to: p is dispatch (a Link's
+      flow, a Generator's output, or a Storage unit's net dispatch, discharging
+      minus charging), p_nom is installed capacity, and p_set (Load only) is
+      demand. p_set is data rather than a variable: coefficient x demand moves
+      to the constraint's right-hand side at each snapshot.
+  coefficient:
+    type: float
+    required: true
+    description: Coefficient the variable is multiplied by in the constraint.

src/ispypsa/validation/schemas/pypsa_friendly_tables/custom_constraints_rhs.yaml

@@ -0,0 +1,76 @@
+table: custom_constraints_rhs
+required: false
+unique:
+  - [constraint_name, investment_period, timeslice]
+custom_validation:
+  - name: blank_investment_period_rows_have_a_blank_timeslice
+    description: >
+      A row with a blank investment_period must also have a blank timeslice.
+      A blank investment_period is reserved for constraints that apply
+      regardless of time: in every investment period and at every snapshot,
+      so the timeslice must be blank too. Only the expansion limits take this
+      form. A row with a blank investment_period and a named timeslice would
+      match no snapshots, so pypsa_build would skip it and its limit would
+      silently never apply.
+  - name: constraint_investment_periods_have_lhs_terms
+    description: >
+      Every (constraint_name, investment_period) here must have at least one
+      custom_constraints_lhs term on a model variable (anything but a Load
+      p_set term) for the same pair, blank matching blank. pypsa_build raises
+      on a constraint with no such term, since its LHS would be empty.
+description: >
+  Right-hand side of each custom constraint: one row per linopy constraint
+  pypsa_build adds, scoped in time by its investment_period and timeslice.
+
+  Produced by ispypsa.translator.constraints, which resolves the templated
+  custom-constraint tables onto the investment periods and appends the
+  endogenous expansion-limit constraints. Consumed by
+  ispypsa.pypsa_build.custom_constraints._add_custom_constraints_with_temporal_scope.
+
+  If absent:
+  No custom constraints are added to the model.
+columns:
+  constraint_name:
+    type: string
+    required: true
+    description: >
+      Constraint the row belongs to: a templated constraint_id, or
+      "<expansion_id>_expansion_limit" for an expansion-limit constraint.
+  investment_period:
+    type: int
+    required: false
+    description: >
+      Investment period the row applies in. Its LHS terms are the
+      custom_constraints_lhs rows with the same constraint_name and
+      investment_period.
+
+      If absent (or empty):
+      The row applies regardless of investment period, and its LHS terms are
+      the ones with a blank investment_period. Only the expansion-limit
+      constraints take this form.
+  timeslice:
+    type: string
+    required: false
+    description: >
+      Timeslice the row applies in: the constraint holds at that timeslice's
+      snapshots (from timeslice_snapshots) in its investment_period. A
+      timeslice with no snapshots in timeslice_snapshots (e.g. one that never
+      activates, or one missed by snapshot aggregation) is valid; the
+      constraint just doesn't apply.
+
+      If absent (or empty):
+      The row is the constraint's fallback: it applies at the snapshots in its
+      investment_period that none of the same constraint's named timeslices
+      cover. A constraint with only named-timeslice rows doesn't bind outside
+      them.
+  rhs:
+    type: float
+    required: true
+    description: >
+      Limit value the constraint's LHS is compared against, before any load
+      terms' per-snapshot demand is moved across.
+  constraint_type:
+    type: string
+    required: true
+    allowed_values: ["<=", ">=", "=="]
+    description: Comparison between the constraint's LHS and rhs.

src/ispypsa/validation/schemas/pypsa_friendly_tables/timeslice_snapshots.yaml

@@ -0,0 +1,40 @@
+table: timeslice_snapshots
+required: false
+unique:
+  - [timeslice, investment_periods, snapshots]
+custom_validation:
+  - name: snapshots_are_in_the_snapshots_table
+    description: >
+      Every (investment_periods, snapshots) pair must be a row of the
+      pypsa-friendly snapshots table, which pypsa_build sets the network's
+      snapshots from. The translator builds this table from those same
+      snapshots, so the check guards the contract rather than a known gap;
+      pypsa_build selects model variables at these snapshots and would fail on
+      one the network doesn't have.
+description: >
+  Which snapshots each timeslice is active at: one row per (timeslice,
+  snapshot) pair. A snapshot appears once per region's timeslice active at it,
+  since each region's timeslice windows tile the year.
+
+  Produced by ispypsa.translator.timeslices, which re-sequences the templated
+  timeslice window patterns onto the model's snapshots. Consumed by
+  ispypsa.pypsa_build.links (per-timeslice link limits) and
+  ispypsa.pypsa_build.custom_constraints (named-timeslice custom constraints).
+
+  If absent (or empty):
+  No snapshot belongs to a named timeslice, so only the fallback rows (blank
+  timeslice) of link_timeslice_limits and custom_constraints_rhs apply.
+columns:
+  timeslice:
+    type: string
+    required: true
+    description: Timeslice active at the snapshot, e.g. qld_peak_demand.
+  investment_periods:
+    type: int
+    required: true
+    description: Investment period of the snapshot.
+  snapshots:
+    type: date
+    required: true
+    format: "%Y-%m-%d %H:%M:%S"
+    description: The snapshot's timestamp.