From 9e64cc83f1b4e0274218a95d0fb34db9ef4750ae Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 4 Sep 2026 14:30:00 +0200 Subject: [PATCH 1/3] Refresh the Ember carbon intensity data Regenerated with scripts/fetch_ember_co2_intensity.sh from the current Ember release: 78 of the 157 regions move, most by a few gCO2/kWh. The larger shift is us-east-1, from 384.4 to 327.17, which now matches the Virginia figure used for the Azure region in the same state. --- .../resources/ember/ember_co2_intensity.csv | 156 +++++++++--------- .../ember/AverageCarbonIntensityTest.java | 6 +- 2 files changed, 81 insertions(+), 81 deletions(-) diff --git a/src/main/resources/ember/ember_co2_intensity.csv b/src/main/resources/ember/ember_co2_intensity.csv index b898465c..c08d0039 100644 --- a/src/main/resources/ember/ember_co2_intensity.csv +++ b/src/main/resources/ember/ember_co2_intensity.csv @@ -1,32 +1,32 @@ # https://ember-energy.org/creative-commons/ # Creative Commons Attribution Licence (CC-BY-4.0) #provider,region,gCO2_per_kWh -aws,ap-southeast-4,525.18 -aws,ap-southeast-2,525.18 +aws,ap-southeast-4,524.59 +aws,ap-southeast-2,524.59 aws,me-south-1,902.24 aws,sa-east-1,109.95 -aws,ca-west-1,190.72 -aws,ca-central-1,190.72 -aws,cn-north-1,525.34 -aws,ap-east-1,525.34 -aws,cn-northwest-1,525.34 -aws,eu-west-3,41.44 +aws,ca-west-1,190.73 +aws,ca-central-1,190.73 +aws,cn-north-1,526.19 +aws,ap-east-1,526.19 +aws,cn-northwest-1,526.19 +aws,eu-west-3,41.45 aws,eu-central-1,329.65 aws,ap-south-2,679.88 aws,ap-south-1,658.97 -aws,ap-southeast-3,680.25 -aws,eu-west-1,256.54 +aws,ap-southeast-3,681.11 +aws,eu-west-1,255.9 aws,il-central-1,492.69 -aws,eu-south-1,284.78 -aws,ap-northeast-3,477.26 -aws,ap-northeast-1,477.26 +aws,eu-south-1,284.63 +aws,ap-northeast-3,477.37 +aws,ap-northeast-1,477.37 aws,ap-southeast-1,497.09 -aws,af-south-1,699.29 -aws,ap-northeast-2,417.06 -aws,eu-south-2,153.6 -aws,eu-north-1,35.26 +aws,af-south-1,698.96 +aws,ap-northeast-2,417.14 +aws,eu-south-2,153.56 +aws,eu-north-1,35.35 aws,eu-central-2,39.22 -aws,ap-southeast-7,545.74 +aws,ap-southeast-7,546.32 aws,me-central-1,467.51 aws,eu-west-2,217.41 aws,us-gov-east-1,479.05 @@ -34,42 +34,42 @@ aws,us-east-2,479.05 aws,us-gov-west-1,184.02 aws,us-west-1,177.11 aws,us-west-2,184.02 -aws,us-east-1,384.4 +aws,us-east-1,327.17 aws,ap-southeast-6,92.76 -aws,ap-southeast-5,601.97 +aws,ap-southeast-5,603.21 aws,mx-central-1,474.02 -aws,ap-east-2,633.21 -gcp,australia-southeast2,525.18 -gcp,australia-southeast1,525.18 -gcp,europe-west1,149.82 +aws,ap-east-2,634.07 +gcp,australia-southeast2,524.59 +gcp,australia-southeast1,524.59 +gcp,europe-west1,109.3 gcp,southamerica-east1,109.95 -gcp,northamerica-northeast1,190.72 -gcp,northamerica-northeast2,190.72 -gcp,southamerica-west1,289.49 -gcp,asia-east2,525.34 +gcp,northamerica-northeast1,190.73 +gcp,northamerica-northeast2,190.73 +gcp,southamerica-west1,290.51 +gcp,asia-east2,526.19 gcp,europe-north1,57.47 -gcp,europe-west9,41.44 +gcp,europe-west9,41.45 gcp,europe-west10,329.65 gcp,europe-west3,329.65 gcp,asia-south1,658.97 gcp,asia-south2,421.04 -gcp,asia-southeast2,680.25 +gcp,asia-southeast2,681.11 gcp,me-west1,492.69 -gcp,europe-west8,284.78 -gcp,europe-west12,284.78 -gcp,asia-northeast2,477.26 -gcp,asia-northeast1,477.26 +gcp,europe-west8,284.63 +gcp,europe-west12,284.63 +gcp,asia-northeast2,477.37 +gcp,asia-northeast1,477.37 gcp,europe-west4,253.56 -gcp,europe-central2,588.6 +gcp,europe-central2,590.8 gcp,me-central1,581.52 gcp,me-central2,691.95 gcp,asia-southeast1,497.09 -gcp,africa-south1,699.29 -gcp,asia-northeast3,417.06 -gcp,europe-southwest1,153.6 -gcp,europe-north2,35.26 +gcp,africa-south1,698.96 +gcp,asia-northeast3,417.14 +gcp,europe-southwest1,153.56 +gcp,europe-north2,35.35 gcp,europe-west6,39.22 -gcp,asia-east1,633.21 +gcp,asia-east1,634.07 gcp,europe-west2,217.41 gcp,us-east1,259.37 gcp,us-east5,479.05 @@ -81,62 +81,62 @@ gcp,us-west3,542.89 gcp,us-west1,184.02 gcp,us-east4,327.17 gcp,northamerica-south1,474.02 -gcp,asia-southeast3,545.74 -azure,australiacentral,525.18 -azure,australiacentral2,525.18 -azure,australiasoutheast,525.18 -azure,australiaeast,525.18 -azure,austriaeast,116.91 -azure,belgiumcentral,149.82 +gcp,asia-southeast3,546.32 +azure,australiacentral,524.59 +azure,australiacentral2,524.59 +azure,australiasoutheast,524.59 +azure,australiaeast,524.59 +azure,austriaeast,117.22 +azure,belgiumcentral,109.3 azure,brazilsoutheast,109.95 azure,brazilsouth,109.95 -azure,canadaeast,190.72 -azure,canadacentral,190.72 -azure,chilecentral,289.49 -azure,chinanorth,525.34 -azure,chinanorth2,525.34 -azure,chinanorth3,525.34 -azure,eastasia,525.34 -azure,chinaeast3,525.34 -azure,chinaeast,525.34 -azure,chinaeast2,525.34 -azure,denmarkeast,114.41 +azure,canadaeast,190.73 +azure,canadacentral,190.73 +azure,chilecentral,290.51 +azure,chinanorth,526.19 +azure,chinanorth2,526.19 +azure,chinanorth3,526.19 +azure,eastasia,526.19 +azure,chinaeast3,526.19 +azure,chinaeast,526.19 +azure,chinaeast2,526.19 +azure,denmarkeast,99.62 azure,finlandcentral,57.47 -azure,francesouth,41.44 -azure,francecentral,41.44 +azure,francesouth,41.45 +azure,francecentral,41.45 azure,germanynorth,329.65 azure,germanywestcentral,329.65 -azure,greececentral,315.09 +azure,greececentral,324.41 azure,southindia,493.19 azure,indiasouthcentral,679.88 azure,westindia,658.97 azure,centralindia,658.97 -azure,indonesiacentral,680.25 -azure,northeurope,256.54 +azure,indonesiacentral,681.11 +azure,northeurope,255.9 azure,israelcentral,492.69 -azure,italynorth,284.78 -azure,japanwest,477.26 -azure,japaneast,477.26 -azure,malaysiawest,601.97 +azure,italynorth,284.63 +azure,japanwest,477.37 +azure,japaneast,477.37 +azure,malaysiawest,603.21 azure,mexicocentral,474.02 azure,westeurope,253.56 azure,newzealandnorth,92.76 -azure,norwayeast,28.11 -azure,norwaywest,28.11 -azure,polandcentral,588.6 +azure,norwayeast,28.1 +azure,norwaywest,28.1 +azure,polandcentral,590.8 azure,qatarcentral,581.52 azure,saudiarabiasouthcentral,691.95 azure,southeastasia,497.09 -azure,southafricawest,699.29 -azure,southafricanorth,699.29 -azure,koreasouth,417.06 -azure,koreacentral,417.06 -azure,spaincentral,153.6 -azure,swedencentral,35.26 -azure,swedensouth,35.26 +azure,southafricawest,698.96 +azure,southafricanorth,698.96 +azure,koreasouth,417.14 +azure,koreacentral,417.14 +azure,spaincentral,153.56 +azure,swedencentral,35.35 +azure,swedensouth,35.35 azure,switzerlandwest,39.22 azure,switzerlandnorth,39.22 -azure,taiwannorth,633.21 +azure,taiwannorth,634.07 azure,uaecentral,467.51 azure,uaenorth,467.51 azure,ukwest,217.41 diff --git a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java index 85f2804f..3ba629aa 100644 --- a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java +++ b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java @@ -29,12 +29,12 @@ void setUp() { @Test void awsKnownRegionReturnsIntensity() { - assertEquals(384.4, module.getIntensity(Provider.AWS, "us-east-1")); + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1")); } @Test void awsAnotherRegionReturnsIntensity() { - assertEquals(256.54, module.getIntensity(Provider.AWS, "eu-west-1")); + assertEquals(255.9, module.getIntensity(Provider.AWS, "eu-west-1")); } @Test @@ -69,7 +69,7 @@ void enrichSetsCarbonIntensity() { module.enrich(row, enriched); - assertEquals(384.4, enriched.get(CARBON_INTENSITY)); + assertEquals(327.17, enriched.get(CARBON_INTENSITY)); } @Test From f8e87f850d853e0726ebf56447bdd80998eb4635 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Thu, 10 Sep 2026 13:42:32 +0200 Subject: [PATCH 2/3] Key Ember's carbon intensity by the year of the usage The csv carries every year from 2022 on, one row per region and year, and AverageCarbonIntensity picks the figure of the year the usage was billed in, as PWUE does: the latest earlier year when Ember has not published that one, the latest year when the row has no date. The usage date lookup moves out of PWUE into UsageDate, and timestamps are read in UTC instead of the JVM zone. --- docs/src/modules.md | 15 +- scripts/README.md | 10 +- scripts/fetch_ember_co2_intensity.sh | 92 +-- .../com/digitalpebble/spruce/RowColumn.java | 2 +- .../com/digitalpebble/spruce/UsageDate.java | 39 + .../digitalpebble/spruce/modules/PWUE.java | 26 +- .../ember/AbstractEmberCarbonIntensity.java | 40 +- .../modules/ember/AverageCarbonIntensity.java | 6 +- .../resources/ember/ember_co2_intensity.csv | 770 ++++++++++++++---- .../digitalpebble/spruce/RowColumnTest.java | 14 + .../digitalpebble/spruce/UsageDateTest.java | 53 ++ .../ember/AverageCarbonIntensityTest.java | 105 +-- 12 files changed, 867 insertions(+), 305 deletions(-) create mode 100644 src/main/java/com/digitalpebble/spruce/UsageDate.java create mode 100644 src/test/java/com/digitalpebble/spruce/UsageDateTest.java diff --git a/docs/src/modules.md b/docs/src/modules.md index 139b1087..711e0bea 100644 --- a/docs/src/modules.md +++ b/docs/src/modules.md @@ -342,6 +342,19 @@ in countries with sub-national data (currently the US and India), the carbon int taken from the Ember value for the state hosting the data centre; otherwise the country-level Ember value is used. +The figures are keyed by region and year, like [PWUE](#pwue): a line item gets the figure of +the year it was incurred in, read from its usage date. When Ember has not published that year +yet (the figures of a year come out in the course of the following one, some series lag +further), the latest year published for the region is used; rows without a usable date get the +latest year as well, and usage before 2022, the first year the file carries, gets the 2022 +figure. Ember revises published figures and its most recent year can include estimates, so +refreshing the file can move past years too. + +The figures follow +[Ember's methodology](https://files.ember-energy.org/public-downloads/ember_electricity_data_methodology.pdf): +full lifecycle emissions, including upstream methane, supply chain and manufacturing, with all +gases converted to CO2 equivalent over 100 years. + The data is loaded from `ember/ember_co2_intensity.csv`, which is generated from [`cloud_regions.json`](#cloud-region-metadata) — see the scripts under [`scripts/`](https://github.com/DigitalPebble/spruce/tree/main/scripts) and the dedicated @@ -351,7 +364,7 @@ refresh it. | | | |---|---| | **Class** | `com.digitalpebble.spruce.modules.ember.AverageCarbonIntensity` | -| **Reads** | `region` | +| **Reads** | `region`, the usage date of the line item | | **Writes** | `carbon_intensity` | ## Stage 4 — Impacts diff --git a/scripts/README.md b/scripts/README.md index 23f9bfd5..a9235308 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -21,7 +21,7 @@ scripts/fix_cloud_regions.sh src/main/resources/cloud_regions.json scripts/fetch_ember_co2_intensity.sh src/main/resources/cloud_regions.json ``` -End result: `src/main/resources/ember/ember_co2_intensity.csv`, columns `provider,region,gCO2_per_kWh`. +End result: `src/main/resources/ember/ember_co2_intensity.csv`, columns `provider,region,year,gCO2_per_kWh`, one row per region and year from 2022 on. ## Scripts @@ -71,7 +71,7 @@ Usage: `./fix_cloud_regions.sh [cloud_regions.json]` (no default — pass the pa ### `fetch_ember_co2_intensity.sh` -Downloads three Ember CSVs and emits one CSV row per keyed cloud region: +Downloads three Ember CSVs and emits one CSV row per keyed cloud region and year: - `yearly_full_release_long_format.csv` — per-country power-sector intensity. - `us_yearly_full_release_long_format.csv` — per-US-state intensity. @@ -80,15 +80,15 @@ Downloads three Ember CSVs and emits one CSV row per keyed cloud region: Filtering and reduction (all datasets): - `Unit == "gCO2/kWh"`. -- Keep only the row with the highest `Year` per ISO3 code / state code. +- Keep every year from `FROM_YEAR` (2022) on, per ISO3 code / state code. - Country rows are further restricted to countries that appear in `cloud_regions.json` (one alias: Ember's "United States of America" ↔ cloud_regions' "United States"). Joining to cloud regions: -- For every keyed region under `aws`/`gcp`/`azure.cloud_regions`, emit - `(provider, region_code, gCO2_per_kWh)`. +- For every keyed region under `aws`/`gcp`/`azure.cloud_regions`, emit one + `(provider, region_code, year, gCO2_per_kWh)` row per year. - For regions whose country has a sub-national source (US, India), reverse-geocode the region's `latitude`/`longitude` via OpenStreetMap Nominatim (`zoom=5`, read `address["ISO3166-2-lvl4"]`) to get an ISO diff --git a/scripts/fetch_ember_co2_intensity.sh b/scripts/fetch_ember_co2_intensity.sh index 4cd97d66..f44a0455 100755 --- a/scripts/fetch_ember_co2_intensity.sh +++ b/scripts/fetch_ember_co2_intensity.sh @@ -8,13 +8,13 @@ # - Unit == "gCO2/kWh" # Country CSV: # - ISO 3 code non-empty, Area in cloud_regions.json country set. -# - One value per ISO3: the row with the highest Year. +# - One value per ISO3 and year, from FROM_YEAR on. # Sub-national CSVs (US, India): # - State code non-empty. -# - One value per state code: the row with the highest Year. +# - One value per state code and year, from FROM_YEAR on. # Join: -# - For every keyed region under aws/gcp/azure.cloud_regions, emit -# (provider, region_code, gCO2_per_kWh). +# - For every keyed region under aws/gcp/azure.cloud_regions, emit one +# (provider, region_code, year, gCO2_per_kWh) row per year. # - For regions in a country with a sub-national source, reverse-geocode # the region's lat/lon via OpenStreetMap Nominatim to get the ISO 3166-2 # subdivision code and use the sub-national value. If the subdivision @@ -25,7 +25,7 @@ # - _unresolved entries are skipped (no region code). # - Regions with no country/subdivision match are skipped. # -# Output columns: provider,region,gCO2_per_kWh +# Output columns: provider,region,year,gCO2_per_kWh # # Usage: ./fetch_ember_co2_intensity.sh [cloud_regions.json] [output.csv] # @@ -39,6 +39,9 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" URL="https://files.ember-energy.org/public-downloads/yearly_full_release_long_format.csv" CLOUD_REGIONS="${1:-cloud_regions.json}" OUTPUT="${2:-$PROJECT_ROOT/src/main/resources/ember/ember_co2_intensity.csv}" +# First year of figures to ship. Usage before it gets the figures of this year (see +# AverageCarbonIntensity); the PUE / WUE file starts the same year. +FROM_YEAR=2022 GEO_CACHE="${EMBER_GEOCACHE:-$SCRIPT_DIR/.geocode_cache}" NOMINATIM_UA="ember-cloud-region-script/1.0" @@ -76,7 +79,8 @@ tmp_countries="$(mktemp)" tmp_lookup="$(mktemp)" tmp_sub_lookup="$(mktemp)" tmp_sub_csv="$(mktemp)" -trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv"' EXIT +tmp_alias="$(mktemp)" +trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv" "$tmp_alias"' EXIT echo "Downloading $URL..." >&2 curl --fail -sSL "$URL" -o "$tmp_csv" @@ -93,9 +97,9 @@ jq -r ' | .[] ' "$CLOUD_REGIONS" > "$tmp_countries" -# Build country -> latest gCO2/kWh lookup (TSV: countryvalue). -# Canonicalise the one known name mismatch so the join works. -awk -F',' -v countries_file="$tmp_countries" ' +# Build country -> gCO2/kWh by year lookup (TSV: countryyearvalue), +# from FROM_YEAR on. Canonicalise the one known name mismatch so the join works. +awk -F',' -v countries_file="$tmp_countries" -v from="$FROM_YEAR" ' BEGIN { while ((getline line < countries_file) > 0) { if (line != "") ok[line] = 1 @@ -104,26 +108,15 @@ awk -F',' -v countries_file="$tmp_countries" ' if (ok["United States"]) ok["United States of America"] = 1 } NR == 1 { next } - $15 == "gCO2/kWh" && $2 != "" && ($1 in ok) { - iso = $2 - year = $3 + 0 - if (year > best_year[iso]) { - best_year[iso] = year - best_country[iso] = $1 - best_value[iso] = $16 - } - } - END { - for (iso in best_value) { - name = best_country[iso] - if (name == "United States of America") name = "United States" - print name "\t" best_value[iso] - } + $15 == "gCO2/kWh" && $2 != "" && ($1 in ok) && $3 + 0 >= from { + name = $1 + if (name == "United States of America") name = "United States" + print name "\t" $3 "\t" $16 } ' "$tmp_csv" > "$tmp_lookup" -# Build combined sub-national lookup (TSV: ISO_3166-2_codevalue) by -# pulling each configured source and prefixing bare state codes. +# Build combined sub-national lookup (TSV: ISO_3166-2_codeyearvalue) +# by pulling each configured source and prefixing bare state codes. # Ember sub-national CSVs share a schema: col 4=State code, 6=Year, 10=Unit, # 11=Value. : > "$tmp_sub_lookup" @@ -131,18 +124,10 @@ for entry in "${SUBNATIONAL[@]}"; do IFS='|' read -r sn_country sn_url sn_prefix <<< "$entry" echo "Downloading $sn_url..." >&2 curl --fail -sSL "$sn_url" -o "$tmp_sub_csv" - awk -F',' -v prefix="$sn_prefix" ' + awk -F',' -v prefix="$sn_prefix" -v from="$FROM_YEAR" ' NR == 1 { next } - $10 == "gCO2/kWh" && $4 != "" { - code = $4 - year = $6 + 0 - if (year > best_year[code]) { - best_year[code] = year - best_value[code] = $11 - } - } - END { - for (code in best_value) print prefix code "\t" best_value[code] + $10 == "gCO2/kWh" && $4 != "" && $6 + 0 >= from { + print prefix $4 "\t" $6 "\t" $11 } ' "$tmp_sub_csv" >> "$tmp_sub_lookup" done @@ -151,11 +136,10 @@ done # even when the two systems use different codes for the same region. for alias in "${SUBNATIONAL_ALIASES[@]}"; do IFS='|' read -r nom_code ember_code <<< "$alias" - val=$(awk -F'\t' -v k="$ember_code" '$1==k {print $2; exit}' "$tmp_sub_lookup") - if [[ -n "$val" ]]; then - printf '%s\t%s\n' "$nom_code" "$val" >> "$tmp_sub_lookup" - fi + awk -F'\t' -v k="$ember_code" -v n="$nom_code" -v OFS='\t' '$1 == k {print n, $2, $3}' \ + "$tmp_sub_lookup" >> "$tmp_alias" done +cat "$tmp_alias" >> "$tmp_sub_lookup" # Check whether a country has a sub-national source configured. has_subnational() { @@ -190,15 +174,9 @@ geo_to_subdivision() { printf '%s' "$code" } -lookup_value() { - awk -F'\t' -v key="$1" -v file="$2" ' - BEGIN { - while ((getline line < file) > 0) { - split(line, a, "\t") - if (a[1] == key) { print a[2]; exit } - } - } - ' +# Print "yearvalue" lines for a key, in year order. +lookup_years() { + awk -F'\t' -v key="$1" '$1 == key { print $2 "\t" $3 }' "$2" | sort -n } mkdir -p "$(dirname "$OUTPUT")" @@ -207,20 +185,20 @@ mkdir -p "$(dirname "$OUTPUT")" { echo "# https://ember-energy.org/creative-commons/" echo "# Creative Commons Attribution Licence (CC-BY-4.0)" - echo "#provider,region,gCO2_per_kWh" + echo "#provider,region,year,gCO2_per_kWh" while IFS=$'\t' read -r provider region country lat lon; do - value="" + years="" if [[ -n "$lat" && -n "$lon" ]] && has_subnational "$country"; then code=$(geo_to_subdivision "$lat" "$lon") if [[ -n "$code" ]]; then - value=$(lookup_value "$code" "$tmp_sub_lookup") + years=$(lookup_years "$code" "$tmp_sub_lookup") fi fi - if [[ -z "$value" ]]; then - value=$(lookup_value "$country" "$tmp_lookup") + if [[ -z "$years" ]]; then + years=$(lookup_years "$country" "$tmp_lookup") fi - if [[ -n "$value" ]]; then - echo "$provider,$region,$value" + if [[ -n "$years" ]]; then + awk -F'\t' -v p="$provider" -v r="$region" '{ print p "," r "," $1 "," $2 }' <<< "$years" fi done < <(jq -r ' ["aws","gcp","azure"][] as $p diff --git a/src/main/java/com/digitalpebble/spruce/RowColumn.java b/src/main/java/com/digitalpebble/spruce/RowColumn.java index 7d6b41ca..e4fb332c 100644 --- a/src/main/java/com/digitalpebble/spruce/RowColumn.java +++ b/src/main/java/com/digitalpebble/spruce/RowColumn.java @@ -99,7 +99,7 @@ public Integer getYear(Row r) { return null; } if (value instanceof java.sql.Timestamp timestamp) { - return timestamp.toLocalDateTime().getYear(); + return timestamp.toInstant().atZone(ZoneOffset.UTC).getYear(); } if (value instanceof java.time.Instant instant) { return instant.atZone(ZoneOffset.UTC).getYear(); diff --git a/src/main/java/com/digitalpebble/spruce/UsageDate.java b/src/main/java/com/digitalpebble/spruce/UsageDate.java new file mode 100644 index 00000000..b11e11f6 --- /dev/null +++ b/src/main/java/com/digitalpebble/spruce/UsageDate.java @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalpebble.spruce; + +import org.apache.spark.sql.Row; + +/** + * Reads the date a line item was incurred on, whichever column the report carries it in. + * Shared by the modules that key their factors by year (PUE / WUE, carbon intensity) so they + * agree on which column wins. + */ +public final class UsageDate { + + /** + * Columns a usage date can be read from, in the order they are probed. All are optional: a + * report only carries the ones its provider and format define, and the FOCUS columns exist + * but are still null in native reports, where the FOCUS bridge module fills them in later. + */ + private static final RowColumn[] DATE_COLUMNS = { + FOCUSColumn.CHARGE_PERIOD_START, + CURColumn.LINE_ITEM_USAGE_START_DATE, + AzureColumn.DATE, + CURColumn.BILLING_PERIOD + }; + + private UsageDate() { + } + + /** Returns the year the line item was incurred in, or null if the row has no usable date. */ + public static Integer year(Row row) { + for (RowColumn column : DATE_COLUMNS) { + Integer year = column.getYear(row); + if (year != null) { + return year; + } + } + return null; + } +} diff --git a/src/main/java/com/digitalpebble/spruce/modules/PWUE.java b/src/main/java/com/digitalpebble/spruce/modules/PWUE.java index 9725997c..6d9d066d 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/PWUE.java +++ b/src/main/java/com/digitalpebble/spruce/modules/PWUE.java @@ -2,14 +2,11 @@ package com.digitalpebble.spruce.modules; -import com.digitalpebble.spruce.AzureColumn; -import com.digitalpebble.spruce.CURColumn; import com.digitalpebble.spruce.Column; import com.digitalpebble.spruce.EnrichmentModule; -import com.digitalpebble.spruce.FOCUSColumn; import com.digitalpebble.spruce.Provider; -import com.digitalpebble.spruce.RowColumn; import com.digitalpebble.spruce.SpruceColumn; +import com.digitalpebble.spruce.UsageDate; import com.digitalpebble.spruce.Utils; import org.apache.spark.sql.Row; @@ -62,18 +59,6 @@ public class PWUE implements EnrichmentModule { /** Year used for rows with no usable usage date: yields the most recent figures. */ private static final int LATEST = Integer.MAX_VALUE; - /** - * Columns a usage date can be read from, in the order they are probed. All are optional: a - * report only carries the ones its provider and format define, and the FOCUS columns exist - * but are still null in native reports, where the FOCUS bridge module fills them in later. - */ - private static final RowColumn[] DATE_COLUMNS = { - FOCUSColumn.CHARGE_PERIOD_START, - CURColumn.LINE_ITEM_USAGE_START_DATE, - AzureColumn.DATE, - CURColumn.BILLING_PERIOD - }; - /** PUE and WUE by year for one CSV key — a region id, or a regex over region ids. */ private static class Factors implements Serializable { /** null when the key is an exact region id rather than a pattern. */ @@ -215,13 +200,8 @@ public void enrich(Row row, Map enrichedValues) { /** Returns the year the line item was incurred in, or {@link #LATEST} if it has no date. */ private static int usageYear(Row row) { - for (RowColumn column : DATE_COLUMNS) { - Integer year = column.getYear(row); - if (year != null) { - return year; - } - } - return LATEST; + Integer year = UsageDate.year(row); + return year != null ? year : LATEST; } private Double lookup(String region, int year, boolean water) { diff --git a/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java b/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java index d85a1127..f3c62a58 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java +++ b/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java @@ -13,6 +13,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import static com.digitalpebble.spruce.SpruceColumn.CARBON_INTENSITY; @@ -22,8 +24,8 @@ public abstract class AbstractEmberCarbonIntensity implements EnrichmentModule { private static final String DEFAULT_RESOURCE_LOCATION = "ember/ember_co2_intensity.csv"; - // keyed by "provider:region" e.g. "aws:us-east-1" - private final Map intensities = new HashMap<>(); + // keyed by "provider:region" e.g. "aws:us-east-1", then by year + private final Map> intensities = new HashMap<>(); /** Set via {@link #init(Map, Provider)} — left null on purpose so any call path that * bypasses provider-aware init fails loudly rather than silently using AWS. */ @@ -49,10 +51,11 @@ public void init(Map params) { return; } String[] parts = line.split(","); - if (parts.length == 3) { + if (parts.length == 4) { String key = parts[0].trim() + ":" + parts[1].trim(); - double value = Double.parseDouble(parts[2].trim()); - intensities.put(key, value); + int year = Integer.parseInt(parts[2].trim()); + double value = Double.parseDouble(parts[3].trim()); + intensities.computeIfAbsent(key, k -> new TreeMap<>()).put(year, value); } else { throw new RuntimeException("Invalid ember intensity line: " + line); } @@ -68,15 +71,28 @@ public Column[] columnsAdded() { } /** - * Returns the carbon intensity in gCO2/kWh for the given provider and region, - * or null if not found. + * Returns the carbon intensity in gCO2/kWh for the given provider and region in that year, + * see {@link #forYear} for the years the file does not cover; null if the region is not + * found. */ - protected Double getIntensity(Provider provider, String region) { - String key = provider.csvKey + ":" + region; - Double value = intensities.get(key); - if (value == null) { + protected Double getIntensity(Provider provider, String region, Integer year) { + NavigableMap byYear = intensities.get(provider.csvKey + ":" + region); + if (byYear == null) { log.info("No Ember carbon intensity for {} region {}", provider, region); + return null; } - return value; + return forYear(byYear, year); + } + + /** + * Returns the figure published for that year, else the closest earlier year, else the first + * year published; the latest year when the year is null, i.e. the row has no usable date. + */ + static Double forYear(NavigableMap byYear, Integer year) { + Map.Entry entry = year == null ? byYear.lastEntry() : byYear.floorEntry(year); + if (entry == null) { + entry = byYear.firstEntry(); + } + return entry == null ? null : entry.getValue(); } } diff --git a/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java b/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java index 72c2c948..56d26de0 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java +++ b/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java @@ -3,6 +3,7 @@ package com.digitalpebble.spruce.modules.ember; import com.digitalpebble.spruce.Column; +import com.digitalpebble.spruce.UsageDate; import org.apache.spark.sql.Row; import java.util.Map; @@ -10,7 +11,8 @@ import static com.digitalpebble.spruce.SpruceColumn.*; /** - * Populate the CARBON_INTENSITY field using Ember's cloud region carbon intensity data. + * Populate the CARBON_INTENSITY field with Ember's figure for the cloud region and the year the + * usage was incurred in. */ public class AverageCarbonIntensity extends AbstractEmberCarbonIntensity { @@ -30,7 +32,7 @@ public void enrich(Row row, Map enrichedValues) { return; } - Double coeff = getIntensity(getProvider(), locationCode); + Double coeff = getIntensity(getProvider(), locationCode, UsageDate.year(row)); if (coeff == null) { return; } diff --git a/src/main/resources/ember/ember_co2_intensity.csv b/src/main/resources/ember/ember_co2_intensity.csv index c08d0039..8ebdf7a6 100644 --- a/src/main/resources/ember/ember_co2_intensity.csv +++ b/src/main/resources/ember/ember_co2_intensity.csv @@ -1,160 +1,614 @@ # https://ember-energy.org/creative-commons/ # Creative Commons Attribution Licence (CC-BY-4.0) -#provider,region,gCO2_per_kWh -aws,ap-southeast-4,524.59 -aws,ap-southeast-2,524.59 -aws,me-south-1,902.24 -aws,sa-east-1,109.95 -aws,ca-west-1,190.73 -aws,ca-central-1,190.73 -aws,cn-north-1,526.19 -aws,ap-east-1,526.19 -aws,cn-northwest-1,526.19 -aws,eu-west-3,41.45 -aws,eu-central-1,329.65 -aws,ap-south-2,679.88 -aws,ap-south-1,658.97 -aws,ap-southeast-3,681.11 -aws,eu-west-1,255.9 -aws,il-central-1,492.69 -aws,eu-south-1,284.63 -aws,ap-northeast-3,477.37 -aws,ap-northeast-1,477.37 -aws,ap-southeast-1,497.09 -aws,af-south-1,698.96 -aws,ap-northeast-2,417.14 -aws,eu-south-2,153.56 -aws,eu-north-1,35.35 -aws,eu-central-2,39.22 -aws,ap-southeast-7,546.32 -aws,me-central-1,467.51 -aws,eu-west-2,217.41 -aws,us-gov-east-1,479.05 -aws,us-east-2,479.05 -aws,us-gov-west-1,184.02 -aws,us-west-1,177.11 -aws,us-west-2,184.02 -aws,us-east-1,327.17 -aws,ap-southeast-6,92.76 -aws,ap-southeast-5,603.21 -aws,mx-central-1,474.02 -aws,ap-east-2,634.07 -gcp,australia-southeast2,524.59 -gcp,australia-southeast1,524.59 -gcp,europe-west1,109.3 -gcp,southamerica-east1,109.95 -gcp,northamerica-northeast1,190.73 -gcp,northamerica-northeast2,190.73 -gcp,southamerica-west1,290.51 -gcp,asia-east2,526.19 -gcp,europe-north1,57.47 -gcp,europe-west9,41.45 -gcp,europe-west10,329.65 -gcp,europe-west3,329.65 -gcp,asia-south1,658.97 -gcp,asia-south2,421.04 -gcp,asia-southeast2,681.11 -gcp,me-west1,492.69 -gcp,europe-west8,284.63 -gcp,europe-west12,284.63 -gcp,asia-northeast2,477.37 -gcp,asia-northeast1,477.37 -gcp,europe-west4,253.56 -gcp,europe-central2,590.8 -gcp,me-central1,581.52 -gcp,me-central2,691.95 -gcp,asia-southeast1,497.09 -gcp,africa-south1,698.96 -gcp,asia-northeast3,417.14 -gcp,europe-southwest1,153.56 -gcp,europe-north2,35.35 -gcp,europe-west6,39.22 -gcp,asia-east1,634.07 -gcp,europe-west2,217.41 -gcp,us-east1,259.37 -gcp,us-east5,479.05 -gcp,us-south1,342.52 -gcp,us-west4,299.02 -gcp,us-west2,177.11 -gcp,us-central1,400.49 -gcp,us-west3,542.89 -gcp,us-west1,184.02 -gcp,us-east4,327.17 -gcp,northamerica-south1,474.02 -gcp,asia-southeast3,546.32 -azure,australiacentral,524.59 -azure,australiacentral2,524.59 -azure,australiasoutheast,524.59 -azure,australiaeast,524.59 -azure,austriaeast,117.22 -azure,belgiumcentral,109.3 -azure,brazilsoutheast,109.95 -azure,brazilsouth,109.95 -azure,canadaeast,190.73 -azure,canadacentral,190.73 -azure,chilecentral,290.51 -azure,chinanorth,526.19 -azure,chinanorth2,526.19 -azure,chinanorth3,526.19 -azure,eastasia,526.19 -azure,chinaeast3,526.19 -azure,chinaeast,526.19 -azure,chinaeast2,526.19 -azure,denmarkeast,99.62 -azure,finlandcentral,57.47 -azure,francesouth,41.45 -azure,francecentral,41.45 -azure,germanynorth,329.65 -azure,germanywestcentral,329.65 -azure,greececentral,324.41 -azure,southindia,493.19 -azure,indiasouthcentral,679.88 -azure,westindia,658.97 -azure,centralindia,658.97 -azure,indonesiacentral,681.11 -azure,northeurope,255.9 -azure,israelcentral,492.69 -azure,italynorth,284.63 -azure,japanwest,477.37 -azure,japaneast,477.37 -azure,malaysiawest,603.21 -azure,mexicocentral,474.02 -azure,westeurope,253.56 -azure,newzealandnorth,92.76 -azure,norwayeast,28.1 -azure,norwaywest,28.1 -azure,polandcentral,590.8 -azure,qatarcentral,581.52 -azure,saudiarabiasouthcentral,691.95 -azure,southeastasia,497.09 -azure,southafricawest,698.96 -azure,southafricanorth,698.96 -azure,koreasouth,417.14 -azure,koreacentral,417.14 -azure,spaincentral,153.56 -azure,swedencentral,35.35 -azure,swedensouth,35.35 -azure,switzerlandwest,39.22 -azure,switzerlandnorth,39.22 -azure,taiwannorth,634.07 -azure,uaecentral,467.51 -azure,uaenorth,467.51 -azure,ukwest,217.41 -azure,uksouth,217.41 -azure,eastus3,317.96 -azure,usgovtexas,342.52 -azure,westcentralus,539.33 -azure,northcentralus,222.41 -azure,centralus,275.36 -azure,usdodwest,275.36 -azure,westus2,124.36 -azure,usgovarizona,282.98 -azure,westus3,282.98 -azure,eastus2,327.17 -azure,usdodeast,327.17 -azure,eastus,327.17 -azure,usseceast,327.17 -azure,southcentralus,342.52 -azure,westus,177.11 -azure,ussecwest,177.11 -azure,usgovvirginia,384.4 +#provider,region,year,gCO2_per_kWh +aws,ap-southeast-4,2022,578.01 +aws,ap-southeast-4,2023,556.85 +aws,ap-southeast-4,2024,554.01 +aws,ap-southeast-4,2025,524.59 +aws,ap-southeast-2,2022,578.01 +aws,ap-southeast-2,2023,556.85 +aws,ap-southeast-2,2024,554.01 +aws,ap-southeast-2,2025,524.59 +aws,me-south-1,2022,902.75 +aws,me-south-1,2023,902.59 +aws,me-south-1,2024,902.24 +aws,sa-east-1,2022,103.39 +aws,sa-east-1,2023,96.26 +aws,sa-east-1,2024,106.06 +aws,sa-east-1,2025,109.95 +aws,ca-west-1,2022,160.69 +aws,ca-west-1,2023,174.43 +aws,ca-west-1,2024,185.41 +aws,ca-west-1,2025,190.73 +aws,ca-central-1,2022,160.69 +aws,ca-central-1,2023,174.43 +aws,ca-central-1,2024,185.41 +aws,ca-central-1,2025,190.73 +aws,cn-north-1,2022,586.86 +aws,cn-north-1,2023,583.25 +aws,cn-north-1,2024,556.27 +aws,cn-north-1,2025,526.19 +aws,ap-east-1,2022,586.86 +aws,ap-east-1,2023,583.25 +aws,ap-east-1,2024,556.27 +aws,ap-east-1,2025,526.19 +aws,cn-northwest-1,2022,586.86 +aws,cn-northwest-1,2023,583.25 +aws,cn-northwest-1,2024,556.27 +aws,cn-northwest-1,2025,526.19 +aws,eu-west-3,2022,78.55 +aws,eu-west-3,2023,53.26 +aws,eu-west-3,2024,40.49 +aws,eu-west-3,2025,41.45 +aws,eu-central-1,2022,419.74 +aws,eu-central-1,2023,363.59 +aws,eu-central-1,2024,337.12 +aws,eu-central-1,2025,329.65 +aws,ap-south-2,2022,655.89 +aws,ap-south-2,2023,710.49 +aws,ap-south-2,2024,679.88 +aws,ap-south-1,2022,658.3 +aws,ap-south-1,2023,663.23 +aws,ap-south-1,2024,658.97 +aws,ap-southeast-3,2022,676.05 +aws,ap-southeast-3,2023,681.03 +aws,ap-southeast-3,2024,681.11 +aws,eu-west-1,2022,331.16 +aws,eu-west-1,2023,282.5 +aws,eu-west-1,2024,270.91 +aws,eu-west-1,2025,255.9 +aws,il-central-1,2022,583.32 +aws,il-central-1,2023,558.83 +aws,il-central-1,2024,540.55 +aws,il-central-1,2025,492.69 +aws,eu-south-1,2022,377.84 +aws,eu-south-1,2023,323.3 +aws,eu-south-1,2024,281.43 +aws,eu-south-1,2025,284.63 +aws,ap-northeast-3,2022,519.7 +aws,ap-northeast-3,2023,492.6 +aws,ap-northeast-3,2024,483.58 +aws,ap-northeast-3,2025,477.37 +aws,ap-northeast-1,2022,519.7 +aws,ap-northeast-1,2023,492.6 +aws,ap-northeast-1,2024,483.58 +aws,ap-northeast-1,2025,477.37 +aws,ap-southeast-1,2022,504.22 +aws,ap-southeast-1,2023,500.87 +aws,ap-southeast-1,2024,498.74 +aws,ap-southeast-1,2025,497.09 +aws,af-south-1,2022,729.02 +aws,af-south-1,2023,714.4 +aws,af-south-1,2024,717.72 +aws,af-south-1,2025,698.96 +aws,ap-northeast-2,2022,438.11 +aws,ap-northeast-2,2023,427.34 +aws,ap-northeast-2,2024,415.62 +aws,ap-northeast-2,2025,417.14 +aws,eu-south-2,2022,217.13 +aws,eu-south-2,2023,169.68 +aws,eu-south-2,2024,146.22 +aws,eu-south-2,2025,153.56 +aws,eu-north-1,2022,41.03 +aws,eu-north-1,2023,38.42 +aws,eu-north-1,2024,34.91 +aws,eu-north-1,2025,35.35 +aws,eu-central-2,2022,37.27 +aws,eu-central-2,2023,34.82 +aws,eu-central-2,2024,34.58 +aws,eu-central-2,2025,39.22 +aws,ap-southeast-7,2022,561.12 +aws,ap-southeast-7,2023,549.93 +aws,ap-southeast-7,2024,555.38 +aws,ap-southeast-7,2025,546.32 +aws,me-central-1,2022,558.49 +aws,me-central-1,2023,483.79 +aws,me-central-1,2024,467.51 +aws,eu-west-2,2022,254.98 +aws,eu-west-2,2023,235.6 +aws,eu-west-2,2024,216.5 +aws,eu-west-2,2025,217.41 +aws,us-gov-east-1,2022,519.12 +aws,us-gov-east-1,2023,492.61 +aws,us-gov-east-1,2024,475.34 +aws,us-gov-east-1,2025,479.05 +aws,us-east-2,2022,519.12 +aws,us-east-2,2023,492.61 +aws,us-east-2,2024,475.34 +aws,us-east-2,2025,479.05 +aws,us-gov-west-1,2022,170.58 +aws,us-gov-west-1,2023,201.49 +aws,us-gov-west-1,2024,204.66 +aws,us-gov-west-1,2025,184.02 +aws,us-west-1,2022,237.28 +aws,us-west-1,2023,216.73 +aws,us-west-1,2024,199.57 +aws,us-west-1,2025,177.11 +aws,us-west-2,2022,170.58 +aws,us-west-2,2023,201.49 +aws,us-west-2,2024,204.66 +aws,us-west-2,2025,184.02 +aws,us-east-1,2022,321.4 +aws,us-east-1,2023,302.97 +aws,us-east-1,2024,322.4 +aws,us-east-1,2025,327.17 +aws,ap-southeast-6,2022,103.12 +aws,ap-southeast-6,2023,87.2 +aws,ap-southeast-6,2024,112.02 +aws,ap-southeast-6,2025,92.76 +aws,ap-southeast-5,2022,605.24 +aws,ap-southeast-5,2023,609.11 +aws,ap-southeast-5,2024,601.71 +aws,ap-southeast-5,2025,603.21 +aws,mx-central-1,2022,464.38 +aws,mx-central-1,2023,489.05 +aws,mx-central-1,2024,483.14 +aws,mx-central-1,2025,474.02 +aws,ap-east-2,2022,639.96 +aws,ap-east-2,2023,645.32 +aws,ap-east-2,2024,636.23 +aws,ap-east-2,2025,634.07 +gcp,australia-southeast2,2022,578.01 +gcp,australia-southeast2,2023,556.85 +gcp,australia-southeast2,2024,554.01 +gcp,australia-southeast2,2025,524.59 +gcp,australia-southeast1,2022,578.01 +gcp,australia-southeast1,2023,556.85 +gcp,australia-southeast1,2024,554.01 +gcp,australia-southeast1,2025,524.59 +gcp,europe-west1,2022,139.4 +gcp,europe-west1,2023,134.67 +gcp,europe-west1,2024,127.28 +gcp,europe-west1,2025,109.3 +gcp,southamerica-east1,2022,103.39 +gcp,southamerica-east1,2023,96.26 +gcp,southamerica-east1,2024,106.06 +gcp,southamerica-east1,2025,109.95 +gcp,northamerica-northeast1,2022,160.69 +gcp,northamerica-northeast1,2023,174.43 +gcp,northamerica-northeast1,2024,185.41 +gcp,northamerica-northeast1,2025,190.73 +gcp,northamerica-northeast2,2022,160.69 +gcp,northamerica-northeast2,2023,174.43 +gcp,northamerica-northeast2,2024,185.41 +gcp,northamerica-northeast2,2025,190.73 +gcp,southamerica-west1,2022,373.9 +gcp,southamerica-west1,2023,304.83 +gcp,southamerica-west1,2024,260.89 +gcp,southamerica-west1,2025,290.51 +gcp,asia-east2,2022,586.86 +gcp,asia-east2,2023,583.25 +gcp,asia-east2,2024,556.27 +gcp,asia-east2,2025,526.19 +gcp,europe-north1,2022,129.97 +gcp,europe-north1,2023,81.33 +gcp,europe-north1,2024,66.75 +gcp,europe-north1,2025,57.47 +gcp,europe-west9,2022,78.55 +gcp,europe-west9,2023,53.26 +gcp,europe-west9,2024,40.49 +gcp,europe-west9,2025,41.45 +gcp,europe-west10,2022,419.74 +gcp,europe-west10,2023,363.59 +gcp,europe-west10,2024,337.12 +gcp,europe-west10,2025,329.65 +gcp,europe-west3,2022,419.74 +gcp,europe-west3,2023,363.59 +gcp,europe-west3,2024,337.12 +gcp,europe-west3,2025,329.65 +gcp,asia-south1,2022,658.3 +gcp,asia-south1,2023,663.23 +gcp,asia-south1,2024,658.97 +gcp,asia-south2,2022,442.37 +gcp,asia-south2,2023,417.71 +gcp,asia-south2,2024,421.04 +gcp,asia-southeast2,2022,676.05 +gcp,asia-southeast2,2023,681.03 +gcp,asia-southeast2,2024,681.11 +gcp,me-west1,2022,583.32 +gcp,me-west1,2023,558.83 +gcp,me-west1,2024,540.55 +gcp,me-west1,2025,492.69 +gcp,europe-west8,2022,377.84 +gcp,europe-west8,2023,323.3 +gcp,europe-west8,2024,281.43 +gcp,europe-west8,2025,284.63 +gcp,europe-west12,2022,377.84 +gcp,europe-west12,2023,323.3 +gcp,europe-west12,2024,281.43 +gcp,europe-west12,2025,284.63 +gcp,asia-northeast2,2022,519.7 +gcp,asia-northeast2,2023,492.6 +gcp,asia-northeast2,2024,483.58 +gcp,asia-northeast2,2025,477.37 +gcp,asia-northeast1,2022,519.7 +gcp,asia-northeast1,2023,492.6 +gcp,asia-northeast1,2024,483.58 +gcp,asia-northeast1,2025,477.37 +gcp,europe-west4,2022,325.61 +gcp,europe-west4,2023,268.17 +gcp,europe-west4,2024,250.72 +gcp,europe-west4,2025,253.56 +gcp,europe-central2,2022,732.53 +gcp,europe-central2,2023,650.86 +gcp,europe-central2,2024,608.36 +gcp,europe-central2,2025,590.8 +gcp,me-central1,2022,596.9 +gcp,me-central1,2023,584.25 +gcp,me-central1,2024,581.69 +gcp,me-central1,2025,581.52 +gcp,me-central2,2022,700.05 +gcp,me-central2,2023,697.31 +gcp,me-central2,2024,691.95 +gcp,asia-southeast1,2022,504.22 +gcp,asia-southeast1,2023,500.87 +gcp,asia-southeast1,2024,498.74 +gcp,asia-southeast1,2025,497.09 +gcp,africa-south1,2022,729.02 +gcp,africa-south1,2023,714.4 +gcp,africa-south1,2024,717.72 +gcp,africa-south1,2025,698.96 +gcp,asia-northeast3,2022,438.11 +gcp,asia-northeast3,2023,427.34 +gcp,asia-northeast3,2024,415.62 +gcp,asia-northeast3,2025,417.14 +gcp,europe-southwest1,2022,217.13 +gcp,europe-southwest1,2023,169.68 +gcp,europe-southwest1,2024,146.22 +gcp,europe-southwest1,2025,153.56 +gcp,europe-north2,2022,41.03 +gcp,europe-north2,2023,38.42 +gcp,europe-north2,2024,34.91 +gcp,europe-north2,2025,35.35 +gcp,europe-west6,2022,37.27 +gcp,europe-west6,2023,34.82 +gcp,europe-west6,2024,34.58 +gcp,europe-west6,2025,39.22 +gcp,asia-east1,2022,639.96 +gcp,asia-east1,2023,645.32 +gcp,asia-east1,2024,636.23 +gcp,asia-east1,2025,634.07 +gcp,europe-west2,2022,254.98 +gcp,europe-west2,2023,235.6 +gcp,europe-west2,2024,216.5 +gcp,europe-west2,2025,217.41 +gcp,us-east1,2022,251.47 +gcp,us-east1,2023,250.4 +gcp,us-east1,2024,260.88 +gcp,us-east1,2025,259.37 +gcp,us-east5,2022,519.12 +gcp,us-east5,2023,492.61 +gcp,us-east5,2024,475.34 +gcp,us-east5,2025,479.05 +gcp,us-south1,2022,380.22 +gcp,us-south1,2023,367.37 +gcp,us-south1,2024,356.02 +gcp,us-south1,2025,342.52 +gcp,us-west4,2022,344.07 +gcp,us-west4,2023,330.46 +gcp,us-west4,2024,314.76 +gcp,us-west4,2025,299.02 +gcp,us-west2,2022,237.28 +gcp,us-west2,2023,216.73 +gcp,us-west2,2024,199.57 +gcp,us-west2,2025,177.11 +gcp,us-central1,2022,423.47 +gcp,us-central1,2023,396.73 +gcp,us-central1,2024,385.61 +gcp,us-central1,2025,400.49 +gcp,us-west3,2022,602.72 +gcp,us-west3,2023,551.68 +gcp,us-west3,2024,533.93 +gcp,us-west3,2025,542.89 +gcp,us-west1,2022,170.58 +gcp,us-west1,2023,201.49 +gcp,us-west1,2024,204.66 +gcp,us-west1,2025,184.02 +gcp,us-east4,2022,321.4 +gcp,us-east4,2023,302.97 +gcp,us-east4,2024,322.4 +gcp,us-east4,2025,327.17 +gcp,northamerica-south1,2022,464.38 +gcp,northamerica-south1,2023,489.05 +gcp,northamerica-south1,2024,483.14 +gcp,northamerica-south1,2025,474.02 +gcp,asia-southeast3,2022,561.12 +gcp,asia-southeast3,2023,549.93 +gcp,asia-southeast3,2024,555.38 +gcp,asia-southeast3,2025,546.32 +azure,australiacentral,2022,578.01 +azure,australiacentral,2023,556.85 +azure,australiacentral,2024,554.01 +azure,australiacentral,2025,524.59 +azure,australiacentral2,2022,578.01 +azure,australiacentral2,2023,556.85 +azure,australiacentral2,2024,554.01 +azure,australiacentral2,2025,524.59 +azure,australiasoutheast,2022,578.01 +azure,australiasoutheast,2023,556.85 +azure,australiasoutheast,2024,554.01 +azure,australiasoutheast,2025,524.59 +azure,australiaeast,2022,578.01 +azure,australiaeast,2023,556.85 +azure,australiaeast,2024,554.01 +azure,australiaeast,2025,524.59 +azure,austriaeast,2022,140.84 +azure,austriaeast,2023,111.72 +azure,austriaeast,2024,103.48 +azure,austriaeast,2025,117.22 +azure,belgiumcentral,2022,139.4 +azure,belgiumcentral,2023,134.67 +azure,belgiumcentral,2024,127.28 +azure,belgiumcentral,2025,109.3 +azure,brazilsoutheast,2022,103.39 +azure,brazilsoutheast,2023,96.26 +azure,brazilsoutheast,2024,106.06 +azure,brazilsoutheast,2025,109.95 +azure,brazilsouth,2022,103.39 +azure,brazilsouth,2023,96.26 +azure,brazilsouth,2024,106.06 +azure,brazilsouth,2025,109.95 +azure,canadaeast,2022,160.69 +azure,canadaeast,2023,174.43 +azure,canadaeast,2024,185.41 +azure,canadaeast,2025,190.73 +azure,canadacentral,2022,160.69 +azure,canadacentral,2023,174.43 +azure,canadacentral,2024,185.41 +azure,canadacentral,2025,190.73 +azure,chilecentral,2022,373.9 +azure,chilecentral,2023,304.83 +azure,chilecentral,2024,260.89 +azure,chilecentral,2025,290.51 +azure,chinanorth,2022,586.86 +azure,chinanorth,2023,583.25 +azure,chinanorth,2024,556.27 +azure,chinanorth,2025,526.19 +azure,chinanorth2,2022,586.86 +azure,chinanorth2,2023,583.25 +azure,chinanorth2,2024,556.27 +azure,chinanorth2,2025,526.19 +azure,chinanorth3,2022,586.86 +azure,chinanorth3,2023,583.25 +azure,chinanorth3,2024,556.27 +azure,chinanorth3,2025,526.19 +azure,eastasia,2022,586.86 +azure,eastasia,2023,583.25 +azure,eastasia,2024,556.27 +azure,eastasia,2025,526.19 +azure,chinaeast3,2022,586.86 +azure,chinaeast3,2023,583.25 +azure,chinaeast3,2024,556.27 +azure,chinaeast3,2025,526.19 +azure,chinaeast,2022,586.86 +azure,chinaeast,2023,583.25 +azure,chinaeast,2024,556.27 +azure,chinaeast,2025,526.19 +azure,chinaeast2,2022,586.86 +azure,chinaeast2,2023,583.25 +azure,chinaeast2,2024,556.27 +azure,chinaeast2,2025,526.19 +azure,denmarkeast,2022,203.64 +azure,denmarkeast,2023,152.26 +azure,denmarkeast,2024,131.92 +azure,denmarkeast,2025,99.62 +azure,finlandcentral,2022,129.97 +azure,finlandcentral,2023,81.33 +azure,finlandcentral,2024,66.75 +azure,finlandcentral,2025,57.47 +azure,francesouth,2022,78.55 +azure,francesouth,2023,53.26 +azure,francesouth,2024,40.49 +azure,francesouth,2025,41.45 +azure,francecentral,2022,78.55 +azure,francecentral,2023,53.26 +azure,francecentral,2024,40.49 +azure,francecentral,2025,41.45 +azure,germanynorth,2022,419.74 +azure,germanynorth,2023,363.59 +azure,germanynorth,2024,337.12 +azure,germanynorth,2025,329.65 +azure,germanywestcentral,2022,419.74 +azure,germanywestcentral,2023,363.59 +azure,germanywestcentral,2024,337.12 +azure,germanywestcentral,2025,329.65 +azure,greececentral,2022,376.81 +azure,greececentral,2023,336.6 +azure,greececentral,2024,321.83 +azure,greececentral,2025,324.41 +azure,southindia,2022,476.65 +azure,southindia,2023,489.65 +azure,southindia,2024,493.19 +azure,indiasouthcentral,2022,655.89 +azure,indiasouthcentral,2023,710.49 +azure,indiasouthcentral,2024,679.88 +azure,westindia,2022,658.3 +azure,westindia,2023,663.23 +azure,westindia,2024,658.97 +azure,centralindia,2022,658.3 +azure,centralindia,2023,663.23 +azure,centralindia,2024,658.97 +azure,indonesiacentral,2022,676.05 +azure,indonesiacentral,2023,681.03 +azure,indonesiacentral,2024,681.11 +azure,northeurope,2022,331.16 +azure,northeurope,2023,282.5 +azure,northeurope,2024,270.91 +azure,northeurope,2025,255.9 +azure,israelcentral,2022,583.32 +azure,israelcentral,2023,558.83 +azure,israelcentral,2024,540.55 +azure,israelcentral,2025,492.69 +azure,italynorth,2022,377.84 +azure,italynorth,2023,323.3 +azure,italynorth,2024,281.43 +azure,italynorth,2025,284.63 +azure,japanwest,2022,519.7 +azure,japanwest,2023,492.6 +azure,japanwest,2024,483.58 +azure,japanwest,2025,477.37 +azure,japaneast,2022,519.7 +azure,japaneast,2023,492.6 +azure,japaneast,2024,483.58 +azure,japaneast,2025,477.37 +azure,malaysiawest,2022,605.24 +azure,malaysiawest,2023,609.11 +azure,malaysiawest,2024,601.71 +azure,malaysiawest,2025,603.21 +azure,mexicocentral,2022,464.38 +azure,mexicocentral,2023,489.05 +azure,mexicocentral,2024,483.14 +azure,mexicocentral,2025,474.02 +azure,westeurope,2022,325.61 +azure,westeurope,2023,268.17 +azure,westeurope,2024,250.72 +azure,westeurope,2025,253.56 +azure,newzealandnorth,2022,103.12 +azure,newzealandnorth,2023,87.2 +azure,newzealandnorth,2024,112.02 +azure,newzealandnorth,2025,92.76 +azure,norwayeast,2022,29.64 +azure,norwayeast,2023,30.46 +azure,norwayeast,2024,29.66 +azure,norwayeast,2025,28.1 +azure,norwaywest,2022,29.64 +azure,norwaywest,2023,30.46 +azure,norwaywest,2024,29.66 +azure,norwaywest,2025,28.1 +azure,polandcentral,2022,732.53 +azure,polandcentral,2023,650.86 +azure,polandcentral,2024,608.36 +azure,polandcentral,2025,590.8 +azure,qatarcentral,2022,596.9 +azure,qatarcentral,2023,584.25 +azure,qatarcentral,2024,581.69 +azure,qatarcentral,2025,581.52 +azure,saudiarabiasouthcentral,2022,700.05 +azure,saudiarabiasouthcentral,2023,697.31 +azure,saudiarabiasouthcentral,2024,691.95 +azure,southeastasia,2022,504.22 +azure,southeastasia,2023,500.87 +azure,southeastasia,2024,498.74 +azure,southeastasia,2025,497.09 +azure,southafricawest,2022,729.02 +azure,southafricawest,2023,714.4 +azure,southafricawest,2024,717.72 +azure,southafricawest,2025,698.96 +azure,southafricanorth,2022,729.02 +azure,southafricanorth,2023,714.4 +azure,southafricanorth,2024,717.72 +azure,southafricanorth,2025,698.96 +azure,koreasouth,2022,438.11 +azure,koreasouth,2023,427.34 +azure,koreasouth,2024,415.62 +azure,koreasouth,2025,417.14 +azure,koreacentral,2022,438.11 +azure,koreacentral,2023,427.34 +azure,koreacentral,2024,415.62 +azure,koreacentral,2025,417.14 +azure,spaincentral,2022,217.13 +azure,spaincentral,2023,169.68 +azure,spaincentral,2024,146.22 +azure,spaincentral,2025,153.56 +azure,swedencentral,2022,41.03 +azure,swedencentral,2023,38.42 +azure,swedencentral,2024,34.91 +azure,swedencentral,2025,35.35 +azure,swedensouth,2022,41.03 +azure,swedensouth,2023,38.42 +azure,swedensouth,2024,34.91 +azure,swedensouth,2025,35.35 +azure,switzerlandwest,2022,37.27 +azure,switzerlandwest,2023,34.82 +azure,switzerlandwest,2024,34.58 +azure,switzerlandwest,2025,39.22 +azure,switzerlandnorth,2022,37.27 +azure,switzerlandnorth,2023,34.82 +azure,switzerlandnorth,2024,34.58 +azure,switzerlandnorth,2025,39.22 +azure,taiwannorth,2022,639.96 +azure,taiwannorth,2023,645.32 +azure,taiwannorth,2024,636.23 +azure,taiwannorth,2025,634.07 +azure,uaecentral,2022,558.49 +azure,uaecentral,2023,483.79 +azure,uaecentral,2024,467.51 +azure,uaenorth,2022,558.49 +azure,uaenorth,2023,483.79 +azure,uaenorth,2024,467.51 +azure,ukwest,2022,254.98 +azure,ukwest,2023,235.6 +azure,ukwest,2024,216.5 +azure,ukwest,2025,217.41 +azure,uksouth,2022,254.98 +azure,uksouth,2023,235.6 +azure,uksouth,2024,216.5 +azure,uksouth,2025,217.41 +azure,eastus3,2022,359.2 +azure,eastus3,2023,349.92 +azure,eastus3,2024,320.85 +azure,eastus3,2025,317.96 +azure,usgovtexas,2022,380.22 +azure,usgovtexas,2023,367.37 +azure,usgovtexas,2024,356.02 +azure,usgovtexas,2025,342.52 +azure,westcentralus,2022,620.1 +azure,westcentralus,2023,620.19 +azure,westcentralus,2024,567.45 +azure,westcentralus,2025,539.33 +azure,northcentralus,2022,241.3 +azure,northcentralus,2023,213.11 +azure,northcentralus,2024,208.45 +azure,northcentralus,2025,222.41 +azure,centralus,2022,261.2 +azure,centralus,2023,271.5 +azure,centralus,2024,243.43 +azure,centralus,2025,275.36 +azure,usdodwest,2022,261.2 +azure,usdodwest,2023,271.5 +azure,usdodwest,2024,243.43 +azure,usdodwest,2025,275.36 +azure,westus2,2022,108.64 +azure,westus2,2023,146.37 +azure,westus2,2024,130.24 +azure,westus2,2025,124.36 +azure,usgovarizona,2022,314.9 +azure,usgovarizona,2023,316.59 +azure,usgovarizona,2024,298.63 +azure,usgovarizona,2025,282.98 +azure,westus3,2022,314.9 +azure,westus3,2023,316.59 +azure,westus3,2024,298.63 +azure,westus3,2025,282.98 +azure,eastus2,2022,321.4 +azure,eastus2,2023,302.97 +azure,eastus2,2024,322.4 +azure,eastus2,2025,327.17 +azure,usdodeast,2022,321.4 +azure,usdodeast,2023,302.97 +azure,usdodeast,2024,322.4 +azure,usdodeast,2025,327.17 +azure,eastus,2022,321.4 +azure,eastus,2023,302.97 +azure,eastus,2024,322.4 +azure,eastus,2025,327.17 +azure,usseceast,2022,321.4 +azure,usseceast,2023,302.97 +azure,usseceast,2024,322.4 +azure,usseceast,2025,327.17 +azure,southcentralus,2022,380.22 +azure,southcentralus,2023,367.37 +azure,southcentralus,2024,356.02 +azure,southcentralus,2025,342.52 +azure,westus,2022,237.28 +azure,westus,2023,216.73 +azure,westus,2024,199.57 +azure,westus,2025,177.11 +azure,ussecwest,2022,237.28 +azure,ussecwest,2023,216.73 +azure,ussecwest,2024,199.57 +azure,ussecwest,2025,177.11 +azure,usgovvirginia,2022,410.39 +azure,usgovvirginia,2023,392.9 +azure,usgovvirginia,2024,383.81 +azure,usgovvirginia,2025,384.4 diff --git a/src/test/java/com/digitalpebble/spruce/RowColumnTest.java b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java index e0c276a1..e4395f0d 100644 --- a/src/test/java/com/digitalpebble/spruce/RowColumnTest.java +++ b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java @@ -42,6 +42,20 @@ void readsTheYearFromTheRepresentationsTheReportsUse() { assertEquals(2025, COLUMN.getYear(row(java.time.Instant.parse("2025-06-15T12:00:00Z")))); } + @Test + void bucketsTimestampsByUtcYearWhateverTheJvmTimeZone() { + // CUR timestamps are UTC; a JVM west of Greenwich must not push the first hours of a + // year into the previous one + java.util.TimeZone jvmZone = java.util.TimeZone.getDefault(); + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Los_Angeles")); + try { + java.sql.Timestamp newYear = java.sql.Timestamp.from(java.time.Instant.parse("2026-01-01T00:00:00Z")); + assertEquals(2026, COLUMN.getYear(row(newYear))); + } finally { + java.util.TimeZone.setDefault(jvmZone); + } + } + @Test void returnsNullWhenThereIsNoYearToRead() { assertNull(COLUMN.getYear(row(null))); diff --git a/src/test/java/com/digitalpebble/spruce/UsageDateTest.java b/src/test/java/com/digitalpebble/spruce/UsageDateTest.java new file mode 100644 index 00000000..5ae9d691 --- /dev/null +++ b/src/test/java/com/digitalpebble/spruce/UsageDateTest.java @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalpebble.spruce; + +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class UsageDateTest { + + private static final StructType SCHEMA = new StructType(new StructField[]{ + StructField.apply(FOCUSColumn.CHARGE_PERIOD_START.getLabel(), DataTypes.StringType, true, null), + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null), + StructField.apply(CURColumn.BILLING_PERIOD.getLabel(), DataTypes.StringType, true, null)}); + + private static Row row(String focusStart, String curStart, String billingPeriod) { + return new GenericRowWithSchema(new Object[]{focusStart, curStart, billingPeriod}, SCHEMA); + } + + @Test + void readsTheYearFromTheFirstDateColumnThatIsFilled() { + assertEquals(2025, UsageDate.year(row("2025-03-01T00:00:00Z", "2024-01-01T00:00:00Z", "2023-01"))); + assertEquals(2024, UsageDate.year(row(null, "2024-01-01T00:00:00Z", "2023-01"))); + assertEquals(2023, UsageDate.year(row(null, null, "2023-01"))); + } + + @Test + void returnsNullWhenNoDateColumnIsFilled() { + assertNull(UsageDate.year(row(null, null, null))); + } + + @Test + void readsTheAzureDateColumn() { + StructType azure = new StructType(new StructField[]{ + StructField.apply(AzureColumn.DATE.getLabel(), DataTypes.DateType, true, null)}); + Row row = new GenericRowWithSchema(new Object[]{java.sql.Date.valueOf("2024-06-01")}, azure); + assertEquals(2024, UsageDate.year(row)); + } + + @Test + void ignoresColumnsTheReportDoesNotCarry() { + StructType curOnly = new StructType(new StructField[]{ + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null)}); + Row row = new GenericRowWithSchema(new Object[]{"2024-11-05"}, curOnly); + assertEquals(2024, UsageDate.year(row)); + } +} diff --git a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java index 3ba629aa..5e727e04 100644 --- a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java +++ b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java @@ -2,23 +2,30 @@ package com.digitalpebble.spruce.modules.ember; +import com.digitalpebble.spruce.CURColumn; import com.digitalpebble.spruce.Column; import com.digitalpebble.spruce.Provider; -import com.digitalpebble.spruce.Utils; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import static com.digitalpebble.spruce.SpruceColumn.*; import static org.junit.jupiter.api.Assertions.*; class AverageCarbonIntensityTest { + private static final StructType SCHEMA = new StructType(new StructField[]{ + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null)}); + private AverageCarbonIntensity module; @BeforeEach @@ -27,88 +34,94 @@ void setUp() { module.init(Map.of(), Provider.AWS); } + private static Row row(String usageStart) { + return new GenericRowWithSchema(new Object[]{usageStart}, SCHEMA); + } + + private Map enrich(Row row, String region) { + Map enriched = new HashMap<>(); + enriched.put(ENERGY_USED, 1.0); + enriched.put(REGION, region); + module.enrich(row, enriched); + return enriched; + } + + @Test + void returnsTheFigureOfTheYear() { + // Virginia moved from 322.4 in 2024 to 327.17 in 2025 + assertEquals(322.4, module.getIntensity(Provider.AWS, "us-east-1", 2024)); + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", 2025)); + } + + @Test + void usesTheLatestYearWhenTheYearIsNotPublishedYet() { + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", 2030)); + } + + @Test + void usesTheFirstYearBeforeTheFileStarts() { + // the file starts in 2022 + assertEquals(321.4, module.getIntensity(Provider.AWS, "us-east-1", 2019)); + } + @Test - void awsKnownRegionReturnsIntensity() { - assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1")); + void usesTheLatestYearWithoutAYear() { + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", null)); } @Test - void awsAnotherRegionReturnsIntensity() { - assertEquals(255.9, module.getIntensity(Provider.AWS, "eu-west-1")); + void usesTheClosestEarlierYearAcrossAGap() { + NavigableMap byYear = new TreeMap<>(Map.of(2022, 100.0, 2024, 200.0)); + assertEquals(100.0, AbstractEmberCarbonIntensity.forYear(byYear, 2023)); } @Test void gcpKnownRegionReturnsIntensity() { - assertEquals(259.37, module.getIntensity(Provider.GOOGLE, "us-east1")); + assertEquals(259.37, module.getIntensity(Provider.GOOGLE, "us-east1", 2025)); } @Test void azureKnownRegionReturnsIntensity() { - assertEquals(327.17, module.getIntensity(Provider.AZURE, "eastus")); + assertEquals(327.17, module.getIntensity(Provider.AZURE, "eastus", 2025)); } @Test void unknownRegionReturnsNull() { - assertNull(module.getIntensity(Provider.AWS, "us-fake-99")); + assertNull(module.getIntensity(Provider.AWS, "us-fake-99", 2025)); } @Test void wrongProviderReturnsNull() { // us-east-1 is valid for AWS but not for GCP - assertNull(module.getIntensity(Provider.GOOGLE, "us-east-1")); + assertNull(module.getIntensity(Provider.GOOGLE, "us-east-1", 2025)); } @Test - void enrichSetsCarbonIntensity() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - enriched.put(REGION, "us-east-1"); - - module.enrich(row, enriched); - - assertEquals(327.17, enriched.get(CARBON_INTENSITY)); + void enrichSetsTheCarbonIntensityOfTheUsageYear() { + assertEquals(322.4, enrich(row("2024-06-15T00:00:00Z"), "us-east-1").get(CARBON_INTENSITY)); + assertEquals(327.17, enrich(row("2025-06-15T00:00:00Z"), "us-east-1").get(CARBON_INTENSITY)); } @Test - void enrichSkipsWhenNoEnergy() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(REGION, "us-east-1"); - - module.enrich(row, enriched); - - assertFalse(enriched.containsKey(CARBON_INTENSITY)); + void enrichUsesTheLatestYearWithoutADate() { + assertEquals(327.17, enrich(row(null), "us-east-1").get(CARBON_INTENSITY)); } @Test void enrichSkipsWhenNoRegion() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - - module.enrich(row, enriched); - - assertFalse(enriched.containsKey(CARBON_INTENSITY)); + assertFalse(enrich(row("2025-06-15T00:00:00Z"), null).containsKey(CARBON_INTENSITY)); } @Test void enrichSkipsUnknownRegion() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); + assertFalse(enrich(row("2025-06-15T00:00:00Z"), "us-nowhere-99").containsKey(CARBON_INTENSITY)); + } + @Test + void enrichSkipsWhenNoEnergy() { Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - enriched.put(REGION, "us-nowhere-99"); - - module.enrich(row, enriched); - + enriched.put(REGION, "us-east-1"); + module.enrich(row("2025-06-15T00:00:00Z"), enriched); assertFalse(enriched.containsKey(CARBON_INTENSITY)); } } From de35cb308f196e1f77549aea8af3493e82c6ae15 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Thu, 10 Sep 2026 13:42:32 +0200 Subject: [PATCH 3/3] Fix the generation of the carbon intensity csv usgovvirginia gets coordinates in Virginia; cloud_regions.json placed it in Washington DC. The script reads Ember's state files by counting fields from the end of the line, so "Washington, D.C." no longer shifts the columns and drops the state. When a region in a country with state figures would get the national figure, the script lists the reasons and leaves the csv unchanged. Failed Nominatim requests are not cached and cached empty codes are ignored. --- scripts/README.md | 19 ++++--- scripts/fetch_ember_co2_intensity.sh | 54 ++++++++++++++----- scripts/fix_cloud_regions.sh | 8 +++ src/main/resources/cloud_regions.json | 4 +- .../resources/ember/ember_co2_intensity.csv | 8 +-- 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index a9235308..a6d66398 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -94,13 +94,16 @@ Joining to cloud regions: Nominatim (`zoom=5`, read `address["ISO3166-2-lvl4"]`) to get an ISO 3166-2 subdivision code (e.g. `US-VA`, `IN-MH`) and use the state-level Ember value. -- If the subdivision can't be resolved (e.g. the coordinates point at DC, - which isn't a state) or has no Ember entry, fall back to the - country-level value. -- Nominatim results are cached in `geocode_cache.tsv` (`lat`, `lon`, - ISO 3166-2 code). Repeat runs are free; the 1 req/sec rate limit only - matters on the first run or when new regions in supported countries - appear. Delete the cache to force a refresh. +- If a region in one of those countries has no coordinates, or Nominatim + places its coordinates in no subdivision, or Ember has no figure for that + subdivision, the script stops without writing the csv and lists the + regions with the reason: each of them would otherwise get the national + figure. +- Resolved subdivisions are cached in `scripts/.geocode_cache` (`lat`, + `lon`, ISO 3166-2 code); failed requests are not, so they are retried on + the next run. Repeat runs are free; the 1 req/sec rate limit only matters + on the first run or when new regions in supported countries appear. + Delete the cache to force a refresh. - `_unresolved` entries are skipped (no region code to emit). Sub-national configuration lives in two arrays at the top of the script: @@ -121,7 +124,7 @@ Usage: `./fetch_ember_co2_intensity.sh [cloud_regions.json] [output.csv]`. Environment variables: - `EMBER_GEOCACHE` — override the geocode cache path (default - `./geocode_cache.tsv`). + `scripts/.geocode_cache`). ## Cloud region water data diff --git a/scripts/fetch_ember_co2_intensity.sh b/scripts/fetch_ember_co2_intensity.sh index f44a0455..68dbc451 100755 --- a/scripts/fetch_ember_co2_intensity.sh +++ b/scripts/fetch_ember_co2_intensity.sh @@ -80,7 +80,8 @@ tmp_lookup="$(mktemp)" tmp_sub_lookup="$(mktemp)" tmp_sub_csv="$(mktemp)" tmp_alias="$(mktemp)" -trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv" "$tmp_alias"' EXIT +tmp_out="$(mktemp)" +trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv" "$tmp_alias" "$tmp_out"' EXIT echo "Downloading $URL..." >&2 curl --fail -sSL "$URL" -o "$tmp_csv" @@ -117,8 +118,11 @@ awk -F',' -v countries_file="$tmp_countries" -v from="$FROM_YEAR" ' # Build combined sub-national lookup (TSV: ISO_3166-2_codeyearvalue) # by pulling each configured source and prefixing bare state codes. -# Ember sub-national CSVs share a schema: col 4=State code, 6=Year, 10=Unit, -# 11=Value. +# Ember sub-national CSVs share a schema: State code, Year, Unit and Value are +# the 4th, 6th, 10th and 11th of 13 columns. A state name with a comma +# ("Washington, D.C.") is quoted and shifts the columns after it, so the fields +# are counted from the end of the line: the last three columns are Value, +# YoY absolute change and YoY % change. : > "$tmp_sub_lookup" for entry in "${SUBNATIONAL[@]}"; do IFS='|' read -r sn_country sn_url sn_prefix <<< "$entry" @@ -126,8 +130,8 @@ for entry in "${SUBNATIONAL[@]}"; do curl --fail -sSL "$sn_url" -o "$tmp_sub_csv" awk -F',' -v prefix="$sn_prefix" -v from="$FROM_YEAR" ' NR == 1 { next } - $10 == "gCO2/kWh" && $4 != "" && $6 + 0 >= from { - print prefix $4 "\t" $6 "\t" $11 + $(NF-3) == "gCO2/kWh" && $(NF-9) != "" && $(NF-7) + 0 >= from { + print prefix $(NF-9) "\t" $(NF-7) "\t" $(NF-2) } ' "$tmp_sub_csv" >> "$tmp_sub_lookup" done @@ -152,13 +156,15 @@ has_subnational() { } # Reverse-geocode (lat, lon) -> ISO 3166-2 subdivision code via Nominatim, -# with caching. Returns empty if no subdivision can be resolved. +# with caching. Prints the code, or nothing when Nominatim places the point in +# no subdivision; returns 1 when the request fails. Only resolved codes are +# cached, so a failed request is retried on the next run. touch "$GEO_CACHE" geo_to_subdivision() { local lat="$1" lon="$2" local hit hit=$(awk -F'\t' -v lat="$lat" -v lon="$lon" ' - $1 == lat && $2 == lon { print $3; found=1; exit } + $1 == lat && $2 == lon && $3 != "" { print $3; found=1; exit } END { if (!found) exit 1 } ' "$GEO_CACHE") && { printf '%s' "$hit"; return; } @@ -166,11 +172,13 @@ geo_to_subdivision() { local resp code resp=$(curl --fail -sSL -A "$NOMINATIM_UA" \ "https://nominatim.openstreetmap.org/reverse?format=jsonv2&zoom=5&lat=${lat}&lon=${lon}" \ - 2>/dev/null || echo '{}') + 2>/dev/null) || return 1 code=$(jq -r '.address["ISO3166-2-lvl4"] // ""' <<< "$resp") # Validate shape: "XX-..." where XX is a 2-letter country code. if [[ ! "$code" =~ ^[A-Z]{2}-[A-Z0-9]+$ ]]; then code=""; fi - printf '%s\t%s\t%s\n' "$lat" "$lon" "$code" >> "$GEO_CACHE" + if [[ -n "$code" ]]; then + printf '%s\t%s\t%s\n' "$lat" "$lon" "$code" >> "$GEO_CACHE" + fi printf '%s' "$code" } @@ -181,6 +189,10 @@ lookup_years() { mkdir -p "$(dirname "$OUTPUT")" +# Regions of a country with state figures that would get the national figure, +# each with the reason. The csv is written only when there are none. +problems=() + # Emit a row per keyed cloud region. { echo "# https://ember-energy.org/creative-commons/" @@ -188,10 +200,18 @@ mkdir -p "$(dirname "$OUTPUT")" echo "#provider,region,year,gCO2_per_kWh" while IFS=$'\t' read -r provider region country lat lon; do years="" - if [[ -n "$lat" && -n "$lon" ]] && has_subnational "$country"; then - code=$(geo_to_subdivision "$lat" "$lon") - if [[ -n "$code" ]]; then + if has_subnational "$country"; then + if [[ -z "$lat" || -z "$lon" ]]; then + problems+=("$provider $region: no coordinates in $CLOUD_REGIONS") + elif ! code=$(geo_to_subdivision "$lat" "$lon"); then + problems+=("$provider $region: Nominatim request failed for $lat,$lon") + elif [[ -z "$code" ]]; then + problems+=("$provider $region: no subdivision at $lat,$lon") + else years=$(lookup_years "$code" "$tmp_sub_lookup") + if [[ -z "$years" ]]; then + problems+=("$provider $region: no Ember figure for $code") + fi fi fi if [[ -z "$years" ]]; then @@ -206,7 +226,15 @@ mkdir -p "$(dirname "$OUTPUT")" | to_entries[] | [$p, .key, .value.country, .value.latitude, .value.longitude] | @tsv ' "$CLOUD_REGIONS") -} > "$OUTPUT" +} > "$tmp_out" + +if (( ${#problems[@]} )); then + echo "error: these regions would get the national figure instead of their state's:" >&2 + printf ' %s\n' "${problems[@]}" >&2 + echo "$OUTPUT left unchanged" >&2 + exit 1 +fi +mv "$tmp_out" "$OUTPUT" rows=$(($(wc -l < "$OUTPUT") - 1)) echo "Wrote $OUTPUT ($rows rows)" >&2 diff --git a/scripts/fix_cloud_regions.sh b/scripts/fix_cloud_regions.sh index 6c10bae3..f4156dbd 100755 --- a/scripts/fix_cloud_regions.sh +++ b/scripts/fix_cloud_regions.sh @@ -9,6 +9,8 @@ # East US 2 at Richmond; East US is actually Ashburn/Sterling). # - Fixes AWS us-east-1 coordinates (upstream points to Washington DC; # the region is actually in Ashburn/Loudoun County, VA). +# - Fixes Azure US Gov Virginia coordinates (upstream points to Washington +# DC; the region is in Virginia). # - Fixes AWS cn-northwest-1 (upstream labels it "Ningxiang" in Hunan; # the region is in Ningxia, near Yinchuan). # - Fixes AWS ca-central-1 coordinates (upstream points ~50 km west of @@ -99,6 +101,12 @@ jq ' | .azure.cloud_regions.eastus.longitude = "-77.487442" | .azure.cloud_regions.eastus.metro_area = "Ashburn" | .azure.cloud_regions.eastus.name = "East US (Ashburn)" + + # US Gov Virginia is in Virginia, not Washington DC. Microsoft names only the + # state, so the coordinates are those of Richmond: any point in the state + # selects the state figure. + | .azure.cloud_regions.usgovvirginia.latitude = "37.540700" + | .azure.cloud_regions.usgovvirginia.longitude = "-77.433654" ' "$FILE" > "$tmp" mv "$tmp" "$FILE" diff --git a/src/main/resources/cloud_regions.json b/src/main/resources/cloud_regions.json index abf697c9..f6502670 100644 --- a/src/main/resources/cloud_regions.json +++ b/src/main/resources/cloud_regions.json @@ -1516,8 +1516,8 @@ "zones": "1" }, "usgovvirginia": { - "latitude": "38.890370", - "longitude": "-77.031959", + "latitude": "37.540700", + "longitude": "-77.433654", "metro_area": "Washington", "country": "United States", "name": "US Gov Virginia ", diff --git a/src/main/resources/ember/ember_co2_intensity.csv b/src/main/resources/ember/ember_co2_intensity.csv index 8ebdf7a6..74fcbb5f 100644 --- a/src/main/resources/ember/ember_co2_intensity.csv +++ b/src/main/resources/ember/ember_co2_intensity.csv @@ -608,7 +608,7 @@ azure,ussecwest,2022,237.28 azure,ussecwest,2023,216.73 azure,ussecwest,2024,199.57 azure,ussecwest,2025,177.11 -azure,usgovvirginia,2022,410.39 -azure,usgovvirginia,2023,392.9 -azure,usgovvirginia,2024,383.81 -azure,usgovvirginia,2025,384.4 +azure,usgovvirginia,2022,321.4 +azure,usgovvirginia,2023,302.97 +azure,usgovvirginia,2024,322.4 +azure,usgovvirginia,2025,327.17