diff --git a/docs/src/modules.md b/docs/src/modules.md index 139b1087..711e0bea 100644 --- a/docs/src/modules.md +++ b/docs/src/modules.md @@ -342,6 +342,19 @@ in countries with sub-national data (currently the US and India), the carbon int taken from the Ember value for the state hosting the data centre; otherwise the country-level Ember value is used. +The figures are keyed by region and year, like [PWUE](#pwue): a line item gets the figure of +the year it was incurred in, read from its usage date. When Ember has not published that year +yet (the figures of a year come out in the course of the following one, some series lag +further), the latest year published for the region is used; rows without a usable date get the +latest year as well, and usage before 2022, the first year the file carries, gets the 2022 +figure. Ember revises published figures and its most recent year can include estimates, so +refreshing the file can move past years too. + +The figures follow +[Ember's methodology](https://files.ember-energy.org/public-downloads/ember_electricity_data_methodology.pdf): +full lifecycle emissions, including upstream methane, supply chain and manufacturing, with all +gases converted to CO2 equivalent over 100 years. + The data is loaded from `ember/ember_co2_intensity.csv`, which is generated from [`cloud_regions.json`](#cloud-region-metadata) — see the scripts under [`scripts/`](https://github.com/DigitalPebble/spruce/tree/main/scripts) and the dedicated @@ -351,7 +364,7 @@ refresh it. | | | |---|---| | **Class** | `com.digitalpebble.spruce.modules.ember.AverageCarbonIntensity` | -| **Reads** | `region` | +| **Reads** | `region`, the usage date of the line item | | **Writes** | `carbon_intensity` | ## Stage 4 — Impacts diff --git a/scripts/README.md b/scripts/README.md index 23f9bfd5..a6d66398 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -21,7 +21,7 @@ scripts/fix_cloud_regions.sh src/main/resources/cloud_regions.json scripts/fetch_ember_co2_intensity.sh src/main/resources/cloud_regions.json ``` -End result: `src/main/resources/ember/ember_co2_intensity.csv`, columns `provider,region,gCO2_per_kWh`. +End result: `src/main/resources/ember/ember_co2_intensity.csv`, columns `provider,region,year,gCO2_per_kWh`, one row per region and year from 2022 on. ## Scripts @@ -71,7 +71,7 @@ Usage: `./fix_cloud_regions.sh [cloud_regions.json]` (no default — pass the pa ### `fetch_ember_co2_intensity.sh` -Downloads three Ember CSVs and emits one CSV row per keyed cloud region: +Downloads three Ember CSVs and emits one CSV row per keyed cloud region and year: - `yearly_full_release_long_format.csv` — per-country power-sector intensity. - `us_yearly_full_release_long_format.csv` — per-US-state intensity. @@ -80,27 +80,30 @@ Downloads three Ember CSVs and emits one CSV row per keyed cloud region: Filtering and reduction (all datasets): - `Unit == "gCO2/kWh"`. -- Keep only the row with the highest `Year` per ISO3 code / state code. +- Keep every year from `FROM_YEAR` (2022) on, per ISO3 code / state code. - Country rows are further restricted to countries that appear in `cloud_regions.json` (one alias: Ember's "United States of America" ↔ cloud_regions' "United States"). Joining to cloud regions: -- For every keyed region under `aws`/`gcp`/`azure.cloud_regions`, emit - `(provider, region_code, gCO2_per_kWh)`. +- For every keyed region under `aws`/`gcp`/`azure.cloud_regions`, emit one + `(provider, region_code, year, gCO2_per_kWh)` row per year. - For regions whose country has a sub-national source (US, India), reverse-geocode the region's `latitude`/`longitude` via OpenStreetMap Nominatim (`zoom=5`, read `address["ISO3166-2-lvl4"]`) to get an ISO 3166-2 subdivision code (e.g. `US-VA`, `IN-MH`) and use the state-level Ember value. -- If the subdivision can't be resolved (e.g. the coordinates point at DC, - which isn't a state) or has no Ember entry, fall back to the - country-level value. -- Nominatim results are cached in `geocode_cache.tsv` (`lat`, `lon`, - ISO 3166-2 code). Repeat runs are free; the 1 req/sec rate limit only - matters on the first run or when new regions in supported countries - appear. Delete the cache to force a refresh. +- If a region in one of those countries has no coordinates, or Nominatim + places its coordinates in no subdivision, or Ember has no figure for that + subdivision, the script stops without writing the csv and lists the + regions with the reason: each of them would otherwise get the national + figure. +- Resolved subdivisions are cached in `scripts/.geocode_cache` (`lat`, + `lon`, ISO 3166-2 code); failed requests are not, so they are retried on + the next run. Repeat runs are free; the 1 req/sec rate limit only matters + on the first run or when new regions in supported countries appear. + Delete the cache to force a refresh. - `_unresolved` entries are skipped (no region code to emit). Sub-national configuration lives in two arrays at the top of the script: @@ -121,7 +124,7 @@ Usage: `./fetch_ember_co2_intensity.sh [cloud_regions.json] [output.csv]`. Environment variables: - `EMBER_GEOCACHE` — override the geocode cache path (default - `./geocode_cache.tsv`). + `scripts/.geocode_cache`). ## Cloud region water data diff --git a/scripts/fetch_ember_co2_intensity.sh b/scripts/fetch_ember_co2_intensity.sh index 4cd97d66..68dbc451 100755 --- a/scripts/fetch_ember_co2_intensity.sh +++ b/scripts/fetch_ember_co2_intensity.sh @@ -8,13 +8,13 @@ # - Unit == "gCO2/kWh" # Country CSV: # - ISO 3 code non-empty, Area in cloud_regions.json country set. -# - One value per ISO3: the row with the highest Year. +# - One value per ISO3 and year, from FROM_YEAR on. # Sub-national CSVs (US, India): # - State code non-empty. -# - One value per state code: the row with the highest Year. +# - One value per state code and year, from FROM_YEAR on. # Join: -# - For every keyed region under aws/gcp/azure.cloud_regions, emit -# (provider, region_code, gCO2_per_kWh). +# - For every keyed region under aws/gcp/azure.cloud_regions, emit one +# (provider, region_code, year, gCO2_per_kWh) row per year. # - For regions in a country with a sub-national source, reverse-geocode # the region's lat/lon via OpenStreetMap Nominatim to get the ISO 3166-2 # subdivision code and use the sub-national value. If the subdivision @@ -25,7 +25,7 @@ # - _unresolved entries are skipped (no region code). # - Regions with no country/subdivision match are skipped. # -# Output columns: provider,region,gCO2_per_kWh +# Output columns: provider,region,year,gCO2_per_kWh # # Usage: ./fetch_ember_co2_intensity.sh [cloud_regions.json] [output.csv] # @@ -39,6 +39,9 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" URL="https://files.ember-energy.org/public-downloads/yearly_full_release_long_format.csv" CLOUD_REGIONS="${1:-cloud_regions.json}" OUTPUT="${2:-$PROJECT_ROOT/src/main/resources/ember/ember_co2_intensity.csv}" +# First year of figures to ship. Usage before it gets the figures of this year (see +# AverageCarbonIntensity); the PUE / WUE file starts the same year. +FROM_YEAR=2022 GEO_CACHE="${EMBER_GEOCACHE:-$SCRIPT_DIR/.geocode_cache}" NOMINATIM_UA="ember-cloud-region-script/1.0" @@ -76,7 +79,9 @@ tmp_countries="$(mktemp)" tmp_lookup="$(mktemp)" tmp_sub_lookup="$(mktemp)" tmp_sub_csv="$(mktemp)" -trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv"' EXIT +tmp_alias="$(mktemp)" +tmp_out="$(mktemp)" +trap 'rm -f "$tmp_csv" "$tmp_countries" "$tmp_lookup" "$tmp_sub_lookup" "$tmp_sub_csv" "$tmp_alias" "$tmp_out"' EXIT echo "Downloading $URL..." >&2 curl --fail -sSL "$URL" -o "$tmp_csv" @@ -93,9 +98,9 @@ jq -r ' | .[] ' "$CLOUD_REGIONS" > "$tmp_countries" -# Build country -> latest gCO2/kWh lookup (TSV: countryvalue). -# Canonicalise the one known name mismatch so the join works. -awk -F',' -v countries_file="$tmp_countries" ' +# Build country -> gCO2/kWh by year lookup (TSV: countryyearvalue), +# from FROM_YEAR on. Canonicalise the one known name mismatch so the join works. +awk -F',' -v countries_file="$tmp_countries" -v from="$FROM_YEAR" ' BEGIN { while ((getline line < countries_file) > 0) { if (line != "") ok[line] = 1 @@ -104,45 +109,29 @@ awk -F',' -v countries_file="$tmp_countries" ' if (ok["United States"]) ok["United States of America"] = 1 } NR == 1 { next } - $15 == "gCO2/kWh" && $2 != "" && ($1 in ok) { - iso = $2 - year = $3 + 0 - if (year > best_year[iso]) { - best_year[iso] = year - best_country[iso] = $1 - best_value[iso] = $16 - } - } - END { - for (iso in best_value) { - name = best_country[iso] - if (name == "United States of America") name = "United States" - print name "\t" best_value[iso] - } + $15 == "gCO2/kWh" && $2 != "" && ($1 in ok) && $3 + 0 >= from { + name = $1 + if (name == "United States of America") name = "United States" + print name "\t" $3 "\t" $16 } ' "$tmp_csv" > "$tmp_lookup" -# Build combined sub-national lookup (TSV: ISO_3166-2_codevalue) by -# pulling each configured source and prefixing bare state codes. -# Ember sub-national CSVs share a schema: col 4=State code, 6=Year, 10=Unit, -# 11=Value. +# Build combined sub-national lookup (TSV: ISO_3166-2_codeyearvalue) +# by pulling each configured source and prefixing bare state codes. +# Ember sub-national CSVs share a schema: State code, Year, Unit and Value are +# the 4th, 6th, 10th and 11th of 13 columns. A state name with a comma +# ("Washington, D.C.") is quoted and shifts the columns after it, so the fields +# are counted from the end of the line: the last three columns are Value, +# YoY absolute change and YoY % change. : > "$tmp_sub_lookup" for entry in "${SUBNATIONAL[@]}"; do IFS='|' read -r sn_country sn_url sn_prefix <<< "$entry" echo "Downloading $sn_url..." >&2 curl --fail -sSL "$sn_url" -o "$tmp_sub_csv" - awk -F',' -v prefix="$sn_prefix" ' + awk -F',' -v prefix="$sn_prefix" -v from="$FROM_YEAR" ' NR == 1 { next } - $10 == "gCO2/kWh" && $4 != "" { - code = $4 - year = $6 + 0 - if (year > best_year[code]) { - best_year[code] = year - best_value[code] = $11 - } - } - END { - for (code in best_value) print prefix code "\t" best_value[code] + $(NF-3) == "gCO2/kWh" && $(NF-9) != "" && $(NF-7) + 0 >= from { + print prefix $(NF-9) "\t" $(NF-7) "\t" $(NF-2) } ' "$tmp_sub_csv" >> "$tmp_sub_lookup" done @@ -151,11 +140,10 @@ done # even when the two systems use different codes for the same region. for alias in "${SUBNATIONAL_ALIASES[@]}"; do IFS='|' read -r nom_code ember_code <<< "$alias" - val=$(awk -F'\t' -v k="$ember_code" '$1==k {print $2; exit}' "$tmp_sub_lookup") - if [[ -n "$val" ]]; then - printf '%s\t%s\n' "$nom_code" "$val" >> "$tmp_sub_lookup" - fi + awk -F'\t' -v k="$ember_code" -v n="$nom_code" -v OFS='\t' '$1 == k {print n, $2, $3}' \ + "$tmp_sub_lookup" >> "$tmp_alias" done +cat "$tmp_alias" >> "$tmp_sub_lookup" # Check whether a country has a sub-national source configured. has_subnational() { @@ -168,13 +156,15 @@ has_subnational() { } # Reverse-geocode (lat, lon) -> ISO 3166-2 subdivision code via Nominatim, -# with caching. Returns empty if no subdivision can be resolved. +# with caching. Prints the code, or nothing when Nominatim places the point in +# no subdivision; returns 1 when the request fails. Only resolved codes are +# cached, so a failed request is retried on the next run. touch "$GEO_CACHE" geo_to_subdivision() { local lat="$1" lon="$2" local hit hit=$(awk -F'\t' -v lat="$lat" -v lon="$lon" ' - $1 == lat && $2 == lon { print $3; found=1; exit } + $1 == lat && $2 == lon && $3 != "" { print $3; found=1; exit } END { if (!found) exit 1 } ' "$GEO_CACHE") && { printf '%s' "$hit"; return; } @@ -182,45 +172,53 @@ geo_to_subdivision() { local resp code resp=$(curl --fail -sSL -A "$NOMINATIM_UA" \ "https://nominatim.openstreetmap.org/reverse?format=jsonv2&zoom=5&lat=${lat}&lon=${lon}" \ - 2>/dev/null || echo '{}') + 2>/dev/null) || return 1 code=$(jq -r '.address["ISO3166-2-lvl4"] // ""' <<< "$resp") # Validate shape: "XX-..." where XX is a 2-letter country code. if [[ ! "$code" =~ ^[A-Z]{2}-[A-Z0-9]+$ ]]; then code=""; fi - printf '%s\t%s\t%s\n' "$lat" "$lon" "$code" >> "$GEO_CACHE" + if [[ -n "$code" ]]; then + printf '%s\t%s\t%s\n' "$lat" "$lon" "$code" >> "$GEO_CACHE" + fi printf '%s' "$code" } -lookup_value() { - awk -F'\t' -v key="$1" -v file="$2" ' - BEGIN { - while ((getline line < file) > 0) { - split(line, a, "\t") - if (a[1] == key) { print a[2]; exit } - } - } - ' +# Print "yearvalue" lines for a key, in year order. +lookup_years() { + awk -F'\t' -v key="$1" '$1 == key { print $2 "\t" $3 }' "$2" | sort -n } mkdir -p "$(dirname "$OUTPUT")" +# Regions of a country with state figures that would get the national figure, +# each with the reason. The csv is written only when there are none. +problems=() + # Emit a row per keyed cloud region. { echo "# https://ember-energy.org/creative-commons/" echo "# Creative Commons Attribution Licence (CC-BY-4.0)" - echo "#provider,region,gCO2_per_kWh" + echo "#provider,region,year,gCO2_per_kWh" while IFS=$'\t' read -r provider region country lat lon; do - value="" - if [[ -n "$lat" && -n "$lon" ]] && has_subnational "$country"; then - code=$(geo_to_subdivision "$lat" "$lon") - if [[ -n "$code" ]]; then - value=$(lookup_value "$code" "$tmp_sub_lookup") + years="" + if has_subnational "$country"; then + if [[ -z "$lat" || -z "$lon" ]]; then + problems+=("$provider $region: no coordinates in $CLOUD_REGIONS") + elif ! code=$(geo_to_subdivision "$lat" "$lon"); then + problems+=("$provider $region: Nominatim request failed for $lat,$lon") + elif [[ -z "$code" ]]; then + problems+=("$provider $region: no subdivision at $lat,$lon") + else + years=$(lookup_years "$code" "$tmp_sub_lookup") + if [[ -z "$years" ]]; then + problems+=("$provider $region: no Ember figure for $code") + fi fi fi - if [[ -z "$value" ]]; then - value=$(lookup_value "$country" "$tmp_lookup") + if [[ -z "$years" ]]; then + years=$(lookup_years "$country" "$tmp_lookup") fi - if [[ -n "$value" ]]; then - echo "$provider,$region,$value" + if [[ -n "$years" ]]; then + awk -F'\t' -v p="$provider" -v r="$region" '{ print p "," r "," $1 "," $2 }' <<< "$years" fi done < <(jq -r ' ["aws","gcp","azure"][] as $p @@ -228,7 +226,15 @@ mkdir -p "$(dirname "$OUTPUT")" | to_entries[] | [$p, .key, .value.country, .value.latitude, .value.longitude] | @tsv ' "$CLOUD_REGIONS") -} > "$OUTPUT" +} > "$tmp_out" + +if (( ${#problems[@]} )); then + echo "error: these regions would get the national figure instead of their state's:" >&2 + printf ' %s\n' "${problems[@]}" >&2 + echo "$OUTPUT left unchanged" >&2 + exit 1 +fi +mv "$tmp_out" "$OUTPUT" rows=$(($(wc -l < "$OUTPUT") - 1)) echo "Wrote $OUTPUT ($rows rows)" >&2 diff --git a/scripts/fix_cloud_regions.sh b/scripts/fix_cloud_regions.sh index 6c10bae3..f4156dbd 100755 --- a/scripts/fix_cloud_regions.sh +++ b/scripts/fix_cloud_regions.sh @@ -9,6 +9,8 @@ # East US 2 at Richmond; East US is actually Ashburn/Sterling). # - Fixes AWS us-east-1 coordinates (upstream points to Washington DC; # the region is actually in Ashburn/Loudoun County, VA). +# - Fixes Azure US Gov Virginia coordinates (upstream points to Washington +# DC; the region is in Virginia). # - Fixes AWS cn-northwest-1 (upstream labels it "Ningxiang" in Hunan; # the region is in Ningxia, near Yinchuan). # - Fixes AWS ca-central-1 coordinates (upstream points ~50 km west of @@ -99,6 +101,12 @@ jq ' | .azure.cloud_regions.eastus.longitude = "-77.487442" | .azure.cloud_regions.eastus.metro_area = "Ashburn" | .azure.cloud_regions.eastus.name = "East US (Ashburn)" + + # US Gov Virginia is in Virginia, not Washington DC. Microsoft names only the + # state, so the coordinates are those of Richmond: any point in the state + # selects the state figure. + | .azure.cloud_regions.usgovvirginia.latitude = "37.540700" + | .azure.cloud_regions.usgovvirginia.longitude = "-77.433654" ' "$FILE" > "$tmp" mv "$tmp" "$FILE" diff --git a/src/main/java/com/digitalpebble/spruce/RowColumn.java b/src/main/java/com/digitalpebble/spruce/RowColumn.java index 7d6b41ca..e4fb332c 100644 --- a/src/main/java/com/digitalpebble/spruce/RowColumn.java +++ b/src/main/java/com/digitalpebble/spruce/RowColumn.java @@ -99,7 +99,7 @@ public Integer getYear(Row r) { return null; } if (value instanceof java.sql.Timestamp timestamp) { - return timestamp.toLocalDateTime().getYear(); + return timestamp.toInstant().atZone(ZoneOffset.UTC).getYear(); } if (value instanceof java.time.Instant instant) { return instant.atZone(ZoneOffset.UTC).getYear(); diff --git a/src/main/java/com/digitalpebble/spruce/UsageDate.java b/src/main/java/com/digitalpebble/spruce/UsageDate.java new file mode 100644 index 00000000..b11e11f6 --- /dev/null +++ b/src/main/java/com/digitalpebble/spruce/UsageDate.java @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalpebble.spruce; + +import org.apache.spark.sql.Row; + +/** + * Reads the date a line item was incurred on, whichever column the report carries it in. + * Shared by the modules that key their factors by year (PUE / WUE, carbon intensity) so they + * agree on which column wins. + */ +public final class UsageDate { + + /** + * Columns a usage date can be read from, in the order they are probed. All are optional: a + * report only carries the ones its provider and format define, and the FOCUS columns exist + * but are still null in native reports, where the FOCUS bridge module fills them in later. + */ + private static final RowColumn[] DATE_COLUMNS = { + FOCUSColumn.CHARGE_PERIOD_START, + CURColumn.LINE_ITEM_USAGE_START_DATE, + AzureColumn.DATE, + CURColumn.BILLING_PERIOD + }; + + private UsageDate() { + } + + /** Returns the year the line item was incurred in, or null if the row has no usable date. */ + public static Integer year(Row row) { + for (RowColumn column : DATE_COLUMNS) { + Integer year = column.getYear(row); + if (year != null) { + return year; + } + } + return null; + } +} diff --git a/src/main/java/com/digitalpebble/spruce/modules/PWUE.java b/src/main/java/com/digitalpebble/spruce/modules/PWUE.java index 9725997c..6d9d066d 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/PWUE.java +++ b/src/main/java/com/digitalpebble/spruce/modules/PWUE.java @@ -2,14 +2,11 @@ package com.digitalpebble.spruce.modules; -import com.digitalpebble.spruce.AzureColumn; -import com.digitalpebble.spruce.CURColumn; import com.digitalpebble.spruce.Column; import com.digitalpebble.spruce.EnrichmentModule; -import com.digitalpebble.spruce.FOCUSColumn; import com.digitalpebble.spruce.Provider; -import com.digitalpebble.spruce.RowColumn; import com.digitalpebble.spruce.SpruceColumn; +import com.digitalpebble.spruce.UsageDate; import com.digitalpebble.spruce.Utils; import org.apache.spark.sql.Row; @@ -62,18 +59,6 @@ public class PWUE implements EnrichmentModule { /** Year used for rows with no usable usage date: yields the most recent figures. */ private static final int LATEST = Integer.MAX_VALUE; - /** - * Columns a usage date can be read from, in the order they are probed. All are optional: a - * report only carries the ones its provider and format define, and the FOCUS columns exist - * but are still null in native reports, where the FOCUS bridge module fills them in later. - */ - private static final RowColumn[] DATE_COLUMNS = { - FOCUSColumn.CHARGE_PERIOD_START, - CURColumn.LINE_ITEM_USAGE_START_DATE, - AzureColumn.DATE, - CURColumn.BILLING_PERIOD - }; - /** PUE and WUE by year for one CSV key — a region id, or a regex over region ids. */ private static class Factors implements Serializable { /** null when the key is an exact region id rather than a pattern. */ @@ -215,13 +200,8 @@ public void enrich(Row row, Map enrichedValues) { /** Returns the year the line item was incurred in, or {@link #LATEST} if it has no date. */ private static int usageYear(Row row) { - for (RowColumn column : DATE_COLUMNS) { - Integer year = column.getYear(row); - if (year != null) { - return year; - } - } - return LATEST; + Integer year = UsageDate.year(row); + return year != null ? year : LATEST; } private Double lookup(String region, int year, boolean water) { diff --git a/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java b/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java index d85a1127..f3c62a58 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java +++ b/src/main/java/com/digitalpebble/spruce/modules/ember/AbstractEmberCarbonIntensity.java @@ -13,6 +13,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import static com.digitalpebble.spruce.SpruceColumn.CARBON_INTENSITY; @@ -22,8 +24,8 @@ public abstract class AbstractEmberCarbonIntensity implements EnrichmentModule { private static final String DEFAULT_RESOURCE_LOCATION = "ember/ember_co2_intensity.csv"; - // keyed by "provider:region" e.g. "aws:us-east-1" - private final Map intensities = new HashMap<>(); + // keyed by "provider:region" e.g. "aws:us-east-1", then by year + private final Map> intensities = new HashMap<>(); /** Set via {@link #init(Map, Provider)} — left null on purpose so any call path that * bypasses provider-aware init fails loudly rather than silently using AWS. */ @@ -49,10 +51,11 @@ public void init(Map params) { return; } String[] parts = line.split(","); - if (parts.length == 3) { + if (parts.length == 4) { String key = parts[0].trim() + ":" + parts[1].trim(); - double value = Double.parseDouble(parts[2].trim()); - intensities.put(key, value); + int year = Integer.parseInt(parts[2].trim()); + double value = Double.parseDouble(parts[3].trim()); + intensities.computeIfAbsent(key, k -> new TreeMap<>()).put(year, value); } else { throw new RuntimeException("Invalid ember intensity line: " + line); } @@ -68,15 +71,28 @@ public Column[] columnsAdded() { } /** - * Returns the carbon intensity in gCO2/kWh for the given provider and region, - * or null if not found. + * Returns the carbon intensity in gCO2/kWh for the given provider and region in that year, + * see {@link #forYear} for the years the file does not cover; null if the region is not + * found. */ - protected Double getIntensity(Provider provider, String region) { - String key = provider.csvKey + ":" + region; - Double value = intensities.get(key); - if (value == null) { + protected Double getIntensity(Provider provider, String region, Integer year) { + NavigableMap byYear = intensities.get(provider.csvKey + ":" + region); + if (byYear == null) { log.info("No Ember carbon intensity for {} region {}", provider, region); + return null; } - return value; + return forYear(byYear, year); + } + + /** + * Returns the figure published for that year, else the closest earlier year, else the first + * year published; the latest year when the year is null, i.e. the row has no usable date. + */ + static Double forYear(NavigableMap byYear, Integer year) { + Map.Entry entry = year == null ? byYear.lastEntry() : byYear.floorEntry(year); + if (entry == null) { + entry = byYear.firstEntry(); + } + return entry == null ? null : entry.getValue(); } } diff --git a/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java b/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java index 72c2c948..56d26de0 100644 --- a/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java +++ b/src/main/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensity.java @@ -3,6 +3,7 @@ package com.digitalpebble.spruce.modules.ember; import com.digitalpebble.spruce.Column; +import com.digitalpebble.spruce.UsageDate; import org.apache.spark.sql.Row; import java.util.Map; @@ -10,7 +11,8 @@ import static com.digitalpebble.spruce.SpruceColumn.*; /** - * Populate the CARBON_INTENSITY field using Ember's cloud region carbon intensity data. + * Populate the CARBON_INTENSITY field with Ember's figure for the cloud region and the year the + * usage was incurred in. */ public class AverageCarbonIntensity extends AbstractEmberCarbonIntensity { @@ -30,7 +32,7 @@ public void enrich(Row row, Map enrichedValues) { return; } - Double coeff = getIntensity(getProvider(), locationCode); + Double coeff = getIntensity(getProvider(), locationCode, UsageDate.year(row)); if (coeff == null) { return; } diff --git a/src/main/resources/cloud_regions.json b/src/main/resources/cloud_regions.json index abf697c9..f6502670 100644 --- a/src/main/resources/cloud_regions.json +++ b/src/main/resources/cloud_regions.json @@ -1516,8 +1516,8 @@ "zones": "1" }, "usgovvirginia": { - "latitude": "38.890370", - "longitude": "-77.031959", + "latitude": "37.540700", + "longitude": "-77.433654", "metro_area": "Washington", "country": "United States", "name": "US Gov Virginia ", diff --git a/src/main/resources/ember/ember_co2_intensity.csv b/src/main/resources/ember/ember_co2_intensity.csv index b898465c..74fcbb5f 100644 --- a/src/main/resources/ember/ember_co2_intensity.csv +++ b/src/main/resources/ember/ember_co2_intensity.csv @@ -1,160 +1,614 @@ # https://ember-energy.org/creative-commons/ # Creative Commons Attribution Licence (CC-BY-4.0) -#provider,region,gCO2_per_kWh -aws,ap-southeast-4,525.18 -aws,ap-southeast-2,525.18 -aws,me-south-1,902.24 -aws,sa-east-1,109.95 -aws,ca-west-1,190.72 -aws,ca-central-1,190.72 -aws,cn-north-1,525.34 -aws,ap-east-1,525.34 -aws,cn-northwest-1,525.34 -aws,eu-west-3,41.44 -aws,eu-central-1,329.65 -aws,ap-south-2,679.88 -aws,ap-south-1,658.97 -aws,ap-southeast-3,680.25 -aws,eu-west-1,256.54 -aws,il-central-1,492.69 -aws,eu-south-1,284.78 -aws,ap-northeast-3,477.26 -aws,ap-northeast-1,477.26 -aws,ap-southeast-1,497.09 -aws,af-south-1,699.29 -aws,ap-northeast-2,417.06 -aws,eu-south-2,153.6 -aws,eu-north-1,35.26 -aws,eu-central-2,39.22 -aws,ap-southeast-7,545.74 -aws,me-central-1,467.51 -aws,eu-west-2,217.41 -aws,us-gov-east-1,479.05 -aws,us-east-2,479.05 -aws,us-gov-west-1,184.02 -aws,us-west-1,177.11 -aws,us-west-2,184.02 -aws,us-east-1,384.4 -aws,ap-southeast-6,92.76 -aws,ap-southeast-5,601.97 -aws,mx-central-1,474.02 -aws,ap-east-2,633.21 -gcp,australia-southeast2,525.18 -gcp,australia-southeast1,525.18 -gcp,europe-west1,149.82 -gcp,southamerica-east1,109.95 -gcp,northamerica-northeast1,190.72 -gcp,northamerica-northeast2,190.72 -gcp,southamerica-west1,289.49 -gcp,asia-east2,525.34 -gcp,europe-north1,57.47 -gcp,europe-west9,41.44 -gcp,europe-west10,329.65 -gcp,europe-west3,329.65 -gcp,asia-south1,658.97 -gcp,asia-south2,421.04 -gcp,asia-southeast2,680.25 -gcp,me-west1,492.69 -gcp,europe-west8,284.78 -gcp,europe-west12,284.78 -gcp,asia-northeast2,477.26 -gcp,asia-northeast1,477.26 -gcp,europe-west4,253.56 -gcp,europe-central2,588.6 -gcp,me-central1,581.52 -gcp,me-central2,691.95 -gcp,asia-southeast1,497.09 -gcp,africa-south1,699.29 -gcp,asia-northeast3,417.06 -gcp,europe-southwest1,153.6 -gcp,europe-north2,35.26 -gcp,europe-west6,39.22 -gcp,asia-east1,633.21 -gcp,europe-west2,217.41 -gcp,us-east1,259.37 -gcp,us-east5,479.05 -gcp,us-south1,342.52 -gcp,us-west4,299.02 -gcp,us-west2,177.11 -gcp,us-central1,400.49 -gcp,us-west3,542.89 -gcp,us-west1,184.02 -gcp,us-east4,327.17 -gcp,northamerica-south1,474.02 -gcp,asia-southeast3,545.74 -azure,australiacentral,525.18 -azure,australiacentral2,525.18 -azure,australiasoutheast,525.18 -azure,australiaeast,525.18 -azure,austriaeast,116.91 -azure,belgiumcentral,149.82 -azure,brazilsoutheast,109.95 -azure,brazilsouth,109.95 -azure,canadaeast,190.72 -azure,canadacentral,190.72 -azure,chilecentral,289.49 -azure,chinanorth,525.34 -azure,chinanorth2,525.34 -azure,chinanorth3,525.34 -azure,eastasia,525.34 -azure,chinaeast3,525.34 -azure,chinaeast,525.34 -azure,chinaeast2,525.34 -azure,denmarkeast,114.41 -azure,finlandcentral,57.47 -azure,francesouth,41.44 -azure,francecentral,41.44 -azure,germanynorth,329.65 -azure,germanywestcentral,329.65 -azure,greececentral,315.09 -azure,southindia,493.19 -azure,indiasouthcentral,679.88 -azure,westindia,658.97 -azure,centralindia,658.97 -azure,indonesiacentral,680.25 -azure,northeurope,256.54 -azure,israelcentral,492.69 -azure,italynorth,284.78 -azure,japanwest,477.26 -azure,japaneast,477.26 -azure,malaysiawest,601.97 -azure,mexicocentral,474.02 -azure,westeurope,253.56 -azure,newzealandnorth,92.76 -azure,norwayeast,28.11 -azure,norwaywest,28.11 -azure,polandcentral,588.6 -azure,qatarcentral,581.52 -azure,saudiarabiasouthcentral,691.95 -azure,southeastasia,497.09 -azure,southafricawest,699.29 -azure,southafricanorth,699.29 -azure,koreasouth,417.06 -azure,koreacentral,417.06 -azure,spaincentral,153.6 -azure,swedencentral,35.26 -azure,swedensouth,35.26 -azure,switzerlandwest,39.22 -azure,switzerlandnorth,39.22 -azure,taiwannorth,633.21 -azure,uaecentral,467.51 -azure,uaenorth,467.51 -azure,ukwest,217.41 -azure,uksouth,217.41 -azure,eastus3,317.96 -azure,usgovtexas,342.52 -azure,westcentralus,539.33 -azure,northcentralus,222.41 -azure,centralus,275.36 -azure,usdodwest,275.36 -azure,westus2,124.36 -azure,usgovarizona,282.98 -azure,westus3,282.98 -azure,eastus2,327.17 -azure,usdodeast,327.17 -azure,eastus,327.17 -azure,usseceast,327.17 -azure,southcentralus,342.52 -azure,westus,177.11 -azure,ussecwest,177.11 -azure,usgovvirginia,384.4 +#provider,region,year,gCO2_per_kWh +aws,ap-southeast-4,2022,578.01 +aws,ap-southeast-4,2023,556.85 +aws,ap-southeast-4,2024,554.01 +aws,ap-southeast-4,2025,524.59 +aws,ap-southeast-2,2022,578.01 +aws,ap-southeast-2,2023,556.85 +aws,ap-southeast-2,2024,554.01 +aws,ap-southeast-2,2025,524.59 +aws,me-south-1,2022,902.75 +aws,me-south-1,2023,902.59 +aws,me-south-1,2024,902.24 +aws,sa-east-1,2022,103.39 +aws,sa-east-1,2023,96.26 +aws,sa-east-1,2024,106.06 +aws,sa-east-1,2025,109.95 +aws,ca-west-1,2022,160.69 +aws,ca-west-1,2023,174.43 +aws,ca-west-1,2024,185.41 +aws,ca-west-1,2025,190.73 +aws,ca-central-1,2022,160.69 +aws,ca-central-1,2023,174.43 +aws,ca-central-1,2024,185.41 +aws,ca-central-1,2025,190.73 +aws,cn-north-1,2022,586.86 +aws,cn-north-1,2023,583.25 +aws,cn-north-1,2024,556.27 +aws,cn-north-1,2025,526.19 +aws,ap-east-1,2022,586.86 +aws,ap-east-1,2023,583.25 +aws,ap-east-1,2024,556.27 +aws,ap-east-1,2025,526.19 +aws,cn-northwest-1,2022,586.86 +aws,cn-northwest-1,2023,583.25 +aws,cn-northwest-1,2024,556.27 +aws,cn-northwest-1,2025,526.19 +aws,eu-west-3,2022,78.55 +aws,eu-west-3,2023,53.26 +aws,eu-west-3,2024,40.49 +aws,eu-west-3,2025,41.45 +aws,eu-central-1,2022,419.74 +aws,eu-central-1,2023,363.59 +aws,eu-central-1,2024,337.12 +aws,eu-central-1,2025,329.65 +aws,ap-south-2,2022,655.89 +aws,ap-south-2,2023,710.49 +aws,ap-south-2,2024,679.88 +aws,ap-south-1,2022,658.3 +aws,ap-south-1,2023,663.23 +aws,ap-south-1,2024,658.97 +aws,ap-southeast-3,2022,676.05 +aws,ap-southeast-3,2023,681.03 +aws,ap-southeast-3,2024,681.11 +aws,eu-west-1,2022,331.16 +aws,eu-west-1,2023,282.5 +aws,eu-west-1,2024,270.91 +aws,eu-west-1,2025,255.9 +aws,il-central-1,2022,583.32 +aws,il-central-1,2023,558.83 +aws,il-central-1,2024,540.55 +aws,il-central-1,2025,492.69 +aws,eu-south-1,2022,377.84 +aws,eu-south-1,2023,323.3 +aws,eu-south-1,2024,281.43 +aws,eu-south-1,2025,284.63 +aws,ap-northeast-3,2022,519.7 +aws,ap-northeast-3,2023,492.6 +aws,ap-northeast-3,2024,483.58 +aws,ap-northeast-3,2025,477.37 +aws,ap-northeast-1,2022,519.7 +aws,ap-northeast-1,2023,492.6 +aws,ap-northeast-1,2024,483.58 +aws,ap-northeast-1,2025,477.37 +aws,ap-southeast-1,2022,504.22 +aws,ap-southeast-1,2023,500.87 +aws,ap-southeast-1,2024,498.74 +aws,ap-southeast-1,2025,497.09 +aws,af-south-1,2022,729.02 +aws,af-south-1,2023,714.4 +aws,af-south-1,2024,717.72 +aws,af-south-1,2025,698.96 +aws,ap-northeast-2,2022,438.11 +aws,ap-northeast-2,2023,427.34 +aws,ap-northeast-2,2024,415.62 +aws,ap-northeast-2,2025,417.14 +aws,eu-south-2,2022,217.13 +aws,eu-south-2,2023,169.68 +aws,eu-south-2,2024,146.22 +aws,eu-south-2,2025,153.56 +aws,eu-north-1,2022,41.03 +aws,eu-north-1,2023,38.42 +aws,eu-north-1,2024,34.91 +aws,eu-north-1,2025,35.35 +aws,eu-central-2,2022,37.27 +aws,eu-central-2,2023,34.82 +aws,eu-central-2,2024,34.58 +aws,eu-central-2,2025,39.22 +aws,ap-southeast-7,2022,561.12 +aws,ap-southeast-7,2023,549.93 +aws,ap-southeast-7,2024,555.38 +aws,ap-southeast-7,2025,546.32 +aws,me-central-1,2022,558.49 +aws,me-central-1,2023,483.79 +aws,me-central-1,2024,467.51 +aws,eu-west-2,2022,254.98 +aws,eu-west-2,2023,235.6 +aws,eu-west-2,2024,216.5 +aws,eu-west-2,2025,217.41 +aws,us-gov-east-1,2022,519.12 +aws,us-gov-east-1,2023,492.61 +aws,us-gov-east-1,2024,475.34 +aws,us-gov-east-1,2025,479.05 +aws,us-east-2,2022,519.12 +aws,us-east-2,2023,492.61 +aws,us-east-2,2024,475.34 +aws,us-east-2,2025,479.05 +aws,us-gov-west-1,2022,170.58 +aws,us-gov-west-1,2023,201.49 +aws,us-gov-west-1,2024,204.66 +aws,us-gov-west-1,2025,184.02 +aws,us-west-1,2022,237.28 +aws,us-west-1,2023,216.73 +aws,us-west-1,2024,199.57 +aws,us-west-1,2025,177.11 +aws,us-west-2,2022,170.58 +aws,us-west-2,2023,201.49 +aws,us-west-2,2024,204.66 +aws,us-west-2,2025,184.02 +aws,us-east-1,2022,321.4 +aws,us-east-1,2023,302.97 +aws,us-east-1,2024,322.4 +aws,us-east-1,2025,327.17 +aws,ap-southeast-6,2022,103.12 +aws,ap-southeast-6,2023,87.2 +aws,ap-southeast-6,2024,112.02 +aws,ap-southeast-6,2025,92.76 +aws,ap-southeast-5,2022,605.24 +aws,ap-southeast-5,2023,609.11 +aws,ap-southeast-5,2024,601.71 +aws,ap-southeast-5,2025,603.21 +aws,mx-central-1,2022,464.38 +aws,mx-central-1,2023,489.05 +aws,mx-central-1,2024,483.14 +aws,mx-central-1,2025,474.02 +aws,ap-east-2,2022,639.96 +aws,ap-east-2,2023,645.32 +aws,ap-east-2,2024,636.23 +aws,ap-east-2,2025,634.07 +gcp,australia-southeast2,2022,578.01 +gcp,australia-southeast2,2023,556.85 +gcp,australia-southeast2,2024,554.01 +gcp,australia-southeast2,2025,524.59 +gcp,australia-southeast1,2022,578.01 +gcp,australia-southeast1,2023,556.85 +gcp,australia-southeast1,2024,554.01 +gcp,australia-southeast1,2025,524.59 +gcp,europe-west1,2022,139.4 +gcp,europe-west1,2023,134.67 +gcp,europe-west1,2024,127.28 +gcp,europe-west1,2025,109.3 +gcp,southamerica-east1,2022,103.39 +gcp,southamerica-east1,2023,96.26 +gcp,southamerica-east1,2024,106.06 +gcp,southamerica-east1,2025,109.95 +gcp,northamerica-northeast1,2022,160.69 +gcp,northamerica-northeast1,2023,174.43 +gcp,northamerica-northeast1,2024,185.41 +gcp,northamerica-northeast1,2025,190.73 +gcp,northamerica-northeast2,2022,160.69 +gcp,northamerica-northeast2,2023,174.43 +gcp,northamerica-northeast2,2024,185.41 +gcp,northamerica-northeast2,2025,190.73 +gcp,southamerica-west1,2022,373.9 +gcp,southamerica-west1,2023,304.83 +gcp,southamerica-west1,2024,260.89 +gcp,southamerica-west1,2025,290.51 +gcp,asia-east2,2022,586.86 +gcp,asia-east2,2023,583.25 +gcp,asia-east2,2024,556.27 +gcp,asia-east2,2025,526.19 +gcp,europe-north1,2022,129.97 +gcp,europe-north1,2023,81.33 +gcp,europe-north1,2024,66.75 +gcp,europe-north1,2025,57.47 +gcp,europe-west9,2022,78.55 +gcp,europe-west9,2023,53.26 +gcp,europe-west9,2024,40.49 +gcp,europe-west9,2025,41.45 +gcp,europe-west10,2022,419.74 +gcp,europe-west10,2023,363.59 +gcp,europe-west10,2024,337.12 +gcp,europe-west10,2025,329.65 +gcp,europe-west3,2022,419.74 +gcp,europe-west3,2023,363.59 +gcp,europe-west3,2024,337.12 +gcp,europe-west3,2025,329.65 +gcp,asia-south1,2022,658.3 +gcp,asia-south1,2023,663.23 +gcp,asia-south1,2024,658.97 +gcp,asia-south2,2022,442.37 +gcp,asia-south2,2023,417.71 +gcp,asia-south2,2024,421.04 +gcp,asia-southeast2,2022,676.05 +gcp,asia-southeast2,2023,681.03 +gcp,asia-southeast2,2024,681.11 +gcp,me-west1,2022,583.32 +gcp,me-west1,2023,558.83 +gcp,me-west1,2024,540.55 +gcp,me-west1,2025,492.69 +gcp,europe-west8,2022,377.84 +gcp,europe-west8,2023,323.3 +gcp,europe-west8,2024,281.43 +gcp,europe-west8,2025,284.63 +gcp,europe-west12,2022,377.84 +gcp,europe-west12,2023,323.3 +gcp,europe-west12,2024,281.43 +gcp,europe-west12,2025,284.63 +gcp,asia-northeast2,2022,519.7 +gcp,asia-northeast2,2023,492.6 +gcp,asia-northeast2,2024,483.58 +gcp,asia-northeast2,2025,477.37 +gcp,asia-northeast1,2022,519.7 +gcp,asia-northeast1,2023,492.6 +gcp,asia-northeast1,2024,483.58 +gcp,asia-northeast1,2025,477.37 +gcp,europe-west4,2022,325.61 +gcp,europe-west4,2023,268.17 +gcp,europe-west4,2024,250.72 +gcp,europe-west4,2025,253.56 +gcp,europe-central2,2022,732.53 +gcp,europe-central2,2023,650.86 +gcp,europe-central2,2024,608.36 +gcp,europe-central2,2025,590.8 +gcp,me-central1,2022,596.9 +gcp,me-central1,2023,584.25 +gcp,me-central1,2024,581.69 +gcp,me-central1,2025,581.52 +gcp,me-central2,2022,700.05 +gcp,me-central2,2023,697.31 +gcp,me-central2,2024,691.95 +gcp,asia-southeast1,2022,504.22 +gcp,asia-southeast1,2023,500.87 +gcp,asia-southeast1,2024,498.74 +gcp,asia-southeast1,2025,497.09 +gcp,africa-south1,2022,729.02 +gcp,africa-south1,2023,714.4 +gcp,africa-south1,2024,717.72 +gcp,africa-south1,2025,698.96 +gcp,asia-northeast3,2022,438.11 +gcp,asia-northeast3,2023,427.34 +gcp,asia-northeast3,2024,415.62 +gcp,asia-northeast3,2025,417.14 +gcp,europe-southwest1,2022,217.13 +gcp,europe-southwest1,2023,169.68 +gcp,europe-southwest1,2024,146.22 +gcp,europe-southwest1,2025,153.56 +gcp,europe-north2,2022,41.03 +gcp,europe-north2,2023,38.42 +gcp,europe-north2,2024,34.91 +gcp,europe-north2,2025,35.35 +gcp,europe-west6,2022,37.27 +gcp,europe-west6,2023,34.82 +gcp,europe-west6,2024,34.58 +gcp,europe-west6,2025,39.22 +gcp,asia-east1,2022,639.96 +gcp,asia-east1,2023,645.32 +gcp,asia-east1,2024,636.23 +gcp,asia-east1,2025,634.07 +gcp,europe-west2,2022,254.98 +gcp,europe-west2,2023,235.6 +gcp,europe-west2,2024,216.5 +gcp,europe-west2,2025,217.41 +gcp,us-east1,2022,251.47 +gcp,us-east1,2023,250.4 +gcp,us-east1,2024,260.88 +gcp,us-east1,2025,259.37 +gcp,us-east5,2022,519.12 +gcp,us-east5,2023,492.61 +gcp,us-east5,2024,475.34 +gcp,us-east5,2025,479.05 +gcp,us-south1,2022,380.22 +gcp,us-south1,2023,367.37 +gcp,us-south1,2024,356.02 +gcp,us-south1,2025,342.52 +gcp,us-west4,2022,344.07 +gcp,us-west4,2023,330.46 +gcp,us-west4,2024,314.76 +gcp,us-west4,2025,299.02 +gcp,us-west2,2022,237.28 +gcp,us-west2,2023,216.73 +gcp,us-west2,2024,199.57 +gcp,us-west2,2025,177.11 +gcp,us-central1,2022,423.47 +gcp,us-central1,2023,396.73 +gcp,us-central1,2024,385.61 +gcp,us-central1,2025,400.49 +gcp,us-west3,2022,602.72 +gcp,us-west3,2023,551.68 +gcp,us-west3,2024,533.93 +gcp,us-west3,2025,542.89 +gcp,us-west1,2022,170.58 +gcp,us-west1,2023,201.49 +gcp,us-west1,2024,204.66 +gcp,us-west1,2025,184.02 +gcp,us-east4,2022,321.4 +gcp,us-east4,2023,302.97 +gcp,us-east4,2024,322.4 +gcp,us-east4,2025,327.17 +gcp,northamerica-south1,2022,464.38 +gcp,northamerica-south1,2023,489.05 +gcp,northamerica-south1,2024,483.14 +gcp,northamerica-south1,2025,474.02 +gcp,asia-southeast3,2022,561.12 +gcp,asia-southeast3,2023,549.93 +gcp,asia-southeast3,2024,555.38 +gcp,asia-southeast3,2025,546.32 +azure,australiacentral,2022,578.01 +azure,australiacentral,2023,556.85 +azure,australiacentral,2024,554.01 +azure,australiacentral,2025,524.59 +azure,australiacentral2,2022,578.01 +azure,australiacentral2,2023,556.85 +azure,australiacentral2,2024,554.01 +azure,australiacentral2,2025,524.59 +azure,australiasoutheast,2022,578.01 +azure,australiasoutheast,2023,556.85 +azure,australiasoutheast,2024,554.01 +azure,australiasoutheast,2025,524.59 +azure,australiaeast,2022,578.01 +azure,australiaeast,2023,556.85 +azure,australiaeast,2024,554.01 +azure,australiaeast,2025,524.59 +azure,austriaeast,2022,140.84 +azure,austriaeast,2023,111.72 +azure,austriaeast,2024,103.48 +azure,austriaeast,2025,117.22 +azure,belgiumcentral,2022,139.4 +azure,belgiumcentral,2023,134.67 +azure,belgiumcentral,2024,127.28 +azure,belgiumcentral,2025,109.3 +azure,brazilsoutheast,2022,103.39 +azure,brazilsoutheast,2023,96.26 +azure,brazilsoutheast,2024,106.06 +azure,brazilsoutheast,2025,109.95 +azure,brazilsouth,2022,103.39 +azure,brazilsouth,2023,96.26 +azure,brazilsouth,2024,106.06 +azure,brazilsouth,2025,109.95 +azure,canadaeast,2022,160.69 +azure,canadaeast,2023,174.43 +azure,canadaeast,2024,185.41 +azure,canadaeast,2025,190.73 +azure,canadacentral,2022,160.69 +azure,canadacentral,2023,174.43 +azure,canadacentral,2024,185.41 +azure,canadacentral,2025,190.73 +azure,chilecentral,2022,373.9 +azure,chilecentral,2023,304.83 +azure,chilecentral,2024,260.89 +azure,chilecentral,2025,290.51 +azure,chinanorth,2022,586.86 +azure,chinanorth,2023,583.25 +azure,chinanorth,2024,556.27 +azure,chinanorth,2025,526.19 +azure,chinanorth2,2022,586.86 +azure,chinanorth2,2023,583.25 +azure,chinanorth2,2024,556.27 +azure,chinanorth2,2025,526.19 +azure,chinanorth3,2022,586.86 +azure,chinanorth3,2023,583.25 +azure,chinanorth3,2024,556.27 +azure,chinanorth3,2025,526.19 +azure,eastasia,2022,586.86 +azure,eastasia,2023,583.25 +azure,eastasia,2024,556.27 +azure,eastasia,2025,526.19 +azure,chinaeast3,2022,586.86 +azure,chinaeast3,2023,583.25 +azure,chinaeast3,2024,556.27 +azure,chinaeast3,2025,526.19 +azure,chinaeast,2022,586.86 +azure,chinaeast,2023,583.25 +azure,chinaeast,2024,556.27 +azure,chinaeast,2025,526.19 +azure,chinaeast2,2022,586.86 +azure,chinaeast2,2023,583.25 +azure,chinaeast2,2024,556.27 +azure,chinaeast2,2025,526.19 +azure,denmarkeast,2022,203.64 +azure,denmarkeast,2023,152.26 +azure,denmarkeast,2024,131.92 +azure,denmarkeast,2025,99.62 +azure,finlandcentral,2022,129.97 +azure,finlandcentral,2023,81.33 +azure,finlandcentral,2024,66.75 +azure,finlandcentral,2025,57.47 +azure,francesouth,2022,78.55 +azure,francesouth,2023,53.26 +azure,francesouth,2024,40.49 +azure,francesouth,2025,41.45 +azure,francecentral,2022,78.55 +azure,francecentral,2023,53.26 +azure,francecentral,2024,40.49 +azure,francecentral,2025,41.45 +azure,germanynorth,2022,419.74 +azure,germanynorth,2023,363.59 +azure,germanynorth,2024,337.12 +azure,germanynorth,2025,329.65 +azure,germanywestcentral,2022,419.74 +azure,germanywestcentral,2023,363.59 +azure,germanywestcentral,2024,337.12 +azure,germanywestcentral,2025,329.65 +azure,greececentral,2022,376.81 +azure,greececentral,2023,336.6 +azure,greececentral,2024,321.83 +azure,greececentral,2025,324.41 +azure,southindia,2022,476.65 +azure,southindia,2023,489.65 +azure,southindia,2024,493.19 +azure,indiasouthcentral,2022,655.89 +azure,indiasouthcentral,2023,710.49 +azure,indiasouthcentral,2024,679.88 +azure,westindia,2022,658.3 +azure,westindia,2023,663.23 +azure,westindia,2024,658.97 +azure,centralindia,2022,658.3 +azure,centralindia,2023,663.23 +azure,centralindia,2024,658.97 +azure,indonesiacentral,2022,676.05 +azure,indonesiacentral,2023,681.03 +azure,indonesiacentral,2024,681.11 +azure,northeurope,2022,331.16 +azure,northeurope,2023,282.5 +azure,northeurope,2024,270.91 +azure,northeurope,2025,255.9 +azure,israelcentral,2022,583.32 +azure,israelcentral,2023,558.83 +azure,israelcentral,2024,540.55 +azure,israelcentral,2025,492.69 +azure,italynorth,2022,377.84 +azure,italynorth,2023,323.3 +azure,italynorth,2024,281.43 +azure,italynorth,2025,284.63 +azure,japanwest,2022,519.7 +azure,japanwest,2023,492.6 +azure,japanwest,2024,483.58 +azure,japanwest,2025,477.37 +azure,japaneast,2022,519.7 +azure,japaneast,2023,492.6 +azure,japaneast,2024,483.58 +azure,japaneast,2025,477.37 +azure,malaysiawest,2022,605.24 +azure,malaysiawest,2023,609.11 +azure,malaysiawest,2024,601.71 +azure,malaysiawest,2025,603.21 +azure,mexicocentral,2022,464.38 +azure,mexicocentral,2023,489.05 +azure,mexicocentral,2024,483.14 +azure,mexicocentral,2025,474.02 +azure,westeurope,2022,325.61 +azure,westeurope,2023,268.17 +azure,westeurope,2024,250.72 +azure,westeurope,2025,253.56 +azure,newzealandnorth,2022,103.12 +azure,newzealandnorth,2023,87.2 +azure,newzealandnorth,2024,112.02 +azure,newzealandnorth,2025,92.76 +azure,norwayeast,2022,29.64 +azure,norwayeast,2023,30.46 +azure,norwayeast,2024,29.66 +azure,norwayeast,2025,28.1 +azure,norwaywest,2022,29.64 +azure,norwaywest,2023,30.46 +azure,norwaywest,2024,29.66 +azure,norwaywest,2025,28.1 +azure,polandcentral,2022,732.53 +azure,polandcentral,2023,650.86 +azure,polandcentral,2024,608.36 +azure,polandcentral,2025,590.8 +azure,qatarcentral,2022,596.9 +azure,qatarcentral,2023,584.25 +azure,qatarcentral,2024,581.69 +azure,qatarcentral,2025,581.52 +azure,saudiarabiasouthcentral,2022,700.05 +azure,saudiarabiasouthcentral,2023,697.31 +azure,saudiarabiasouthcentral,2024,691.95 +azure,southeastasia,2022,504.22 +azure,southeastasia,2023,500.87 +azure,southeastasia,2024,498.74 +azure,southeastasia,2025,497.09 +azure,southafricawest,2022,729.02 +azure,southafricawest,2023,714.4 +azure,southafricawest,2024,717.72 +azure,southafricawest,2025,698.96 +azure,southafricanorth,2022,729.02 +azure,southafricanorth,2023,714.4 +azure,southafricanorth,2024,717.72 +azure,southafricanorth,2025,698.96 +azure,koreasouth,2022,438.11 +azure,koreasouth,2023,427.34 +azure,koreasouth,2024,415.62 +azure,koreasouth,2025,417.14 +azure,koreacentral,2022,438.11 +azure,koreacentral,2023,427.34 +azure,koreacentral,2024,415.62 +azure,koreacentral,2025,417.14 +azure,spaincentral,2022,217.13 +azure,spaincentral,2023,169.68 +azure,spaincentral,2024,146.22 +azure,spaincentral,2025,153.56 +azure,swedencentral,2022,41.03 +azure,swedencentral,2023,38.42 +azure,swedencentral,2024,34.91 +azure,swedencentral,2025,35.35 +azure,swedensouth,2022,41.03 +azure,swedensouth,2023,38.42 +azure,swedensouth,2024,34.91 +azure,swedensouth,2025,35.35 +azure,switzerlandwest,2022,37.27 +azure,switzerlandwest,2023,34.82 +azure,switzerlandwest,2024,34.58 +azure,switzerlandwest,2025,39.22 +azure,switzerlandnorth,2022,37.27 +azure,switzerlandnorth,2023,34.82 +azure,switzerlandnorth,2024,34.58 +azure,switzerlandnorth,2025,39.22 +azure,taiwannorth,2022,639.96 +azure,taiwannorth,2023,645.32 +azure,taiwannorth,2024,636.23 +azure,taiwannorth,2025,634.07 +azure,uaecentral,2022,558.49 +azure,uaecentral,2023,483.79 +azure,uaecentral,2024,467.51 +azure,uaenorth,2022,558.49 +azure,uaenorth,2023,483.79 +azure,uaenorth,2024,467.51 +azure,ukwest,2022,254.98 +azure,ukwest,2023,235.6 +azure,ukwest,2024,216.5 +azure,ukwest,2025,217.41 +azure,uksouth,2022,254.98 +azure,uksouth,2023,235.6 +azure,uksouth,2024,216.5 +azure,uksouth,2025,217.41 +azure,eastus3,2022,359.2 +azure,eastus3,2023,349.92 +azure,eastus3,2024,320.85 +azure,eastus3,2025,317.96 +azure,usgovtexas,2022,380.22 +azure,usgovtexas,2023,367.37 +azure,usgovtexas,2024,356.02 +azure,usgovtexas,2025,342.52 +azure,westcentralus,2022,620.1 +azure,westcentralus,2023,620.19 +azure,westcentralus,2024,567.45 +azure,westcentralus,2025,539.33 +azure,northcentralus,2022,241.3 +azure,northcentralus,2023,213.11 +azure,northcentralus,2024,208.45 +azure,northcentralus,2025,222.41 +azure,centralus,2022,261.2 +azure,centralus,2023,271.5 +azure,centralus,2024,243.43 +azure,centralus,2025,275.36 +azure,usdodwest,2022,261.2 +azure,usdodwest,2023,271.5 +azure,usdodwest,2024,243.43 +azure,usdodwest,2025,275.36 +azure,westus2,2022,108.64 +azure,westus2,2023,146.37 +azure,westus2,2024,130.24 +azure,westus2,2025,124.36 +azure,usgovarizona,2022,314.9 +azure,usgovarizona,2023,316.59 +azure,usgovarizona,2024,298.63 +azure,usgovarizona,2025,282.98 +azure,westus3,2022,314.9 +azure,westus3,2023,316.59 +azure,westus3,2024,298.63 +azure,westus3,2025,282.98 +azure,eastus2,2022,321.4 +azure,eastus2,2023,302.97 +azure,eastus2,2024,322.4 +azure,eastus2,2025,327.17 +azure,usdodeast,2022,321.4 +azure,usdodeast,2023,302.97 +azure,usdodeast,2024,322.4 +azure,usdodeast,2025,327.17 +azure,eastus,2022,321.4 +azure,eastus,2023,302.97 +azure,eastus,2024,322.4 +azure,eastus,2025,327.17 +azure,usseceast,2022,321.4 +azure,usseceast,2023,302.97 +azure,usseceast,2024,322.4 +azure,usseceast,2025,327.17 +azure,southcentralus,2022,380.22 +azure,southcentralus,2023,367.37 +azure,southcentralus,2024,356.02 +azure,southcentralus,2025,342.52 +azure,westus,2022,237.28 +azure,westus,2023,216.73 +azure,westus,2024,199.57 +azure,westus,2025,177.11 +azure,ussecwest,2022,237.28 +azure,ussecwest,2023,216.73 +azure,ussecwest,2024,199.57 +azure,ussecwest,2025,177.11 +azure,usgovvirginia,2022,321.4 +azure,usgovvirginia,2023,302.97 +azure,usgovvirginia,2024,322.4 +azure,usgovvirginia,2025,327.17 diff --git a/src/test/java/com/digitalpebble/spruce/RowColumnTest.java b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java index e0c276a1..e4395f0d 100644 --- a/src/test/java/com/digitalpebble/spruce/RowColumnTest.java +++ b/src/test/java/com/digitalpebble/spruce/RowColumnTest.java @@ -42,6 +42,20 @@ void readsTheYearFromTheRepresentationsTheReportsUse() { assertEquals(2025, COLUMN.getYear(row(java.time.Instant.parse("2025-06-15T12:00:00Z")))); } + @Test + void bucketsTimestampsByUtcYearWhateverTheJvmTimeZone() { + // CUR timestamps are UTC; a JVM west of Greenwich must not push the first hours of a + // year into the previous one + java.util.TimeZone jvmZone = java.util.TimeZone.getDefault(); + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Los_Angeles")); + try { + java.sql.Timestamp newYear = java.sql.Timestamp.from(java.time.Instant.parse("2026-01-01T00:00:00Z")); + assertEquals(2026, COLUMN.getYear(row(newYear))); + } finally { + java.util.TimeZone.setDefault(jvmZone); + } + } + @Test void returnsNullWhenThereIsNoYearToRead() { assertNull(COLUMN.getYear(row(null))); diff --git a/src/test/java/com/digitalpebble/spruce/UsageDateTest.java b/src/test/java/com/digitalpebble/spruce/UsageDateTest.java new file mode 100644 index 00000000..5ae9d691 --- /dev/null +++ b/src/test/java/com/digitalpebble/spruce/UsageDateTest.java @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalpebble.spruce; + +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class UsageDateTest { + + private static final StructType SCHEMA = new StructType(new StructField[]{ + StructField.apply(FOCUSColumn.CHARGE_PERIOD_START.getLabel(), DataTypes.StringType, true, null), + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null), + StructField.apply(CURColumn.BILLING_PERIOD.getLabel(), DataTypes.StringType, true, null)}); + + private static Row row(String focusStart, String curStart, String billingPeriod) { + return new GenericRowWithSchema(new Object[]{focusStart, curStart, billingPeriod}, SCHEMA); + } + + @Test + void readsTheYearFromTheFirstDateColumnThatIsFilled() { + assertEquals(2025, UsageDate.year(row("2025-03-01T00:00:00Z", "2024-01-01T00:00:00Z", "2023-01"))); + assertEquals(2024, UsageDate.year(row(null, "2024-01-01T00:00:00Z", "2023-01"))); + assertEquals(2023, UsageDate.year(row(null, null, "2023-01"))); + } + + @Test + void returnsNullWhenNoDateColumnIsFilled() { + assertNull(UsageDate.year(row(null, null, null))); + } + + @Test + void readsTheAzureDateColumn() { + StructType azure = new StructType(new StructField[]{ + StructField.apply(AzureColumn.DATE.getLabel(), DataTypes.DateType, true, null)}); + Row row = new GenericRowWithSchema(new Object[]{java.sql.Date.valueOf("2024-06-01")}, azure); + assertEquals(2024, UsageDate.year(row)); + } + + @Test + void ignoresColumnsTheReportDoesNotCarry() { + StructType curOnly = new StructType(new StructField[]{ + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null)}); + Row row = new GenericRowWithSchema(new Object[]{"2024-11-05"}, curOnly); + assertEquals(2024, UsageDate.year(row)); + } +} diff --git a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java index 85f2804f..5e727e04 100644 --- a/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java +++ b/src/test/java/com/digitalpebble/spruce/modules/ember/AverageCarbonIntensityTest.java @@ -2,23 +2,30 @@ package com.digitalpebble.spruce.modules.ember; +import com.digitalpebble.spruce.CURColumn; import com.digitalpebble.spruce.Column; import com.digitalpebble.spruce.Provider; -import com.digitalpebble.spruce.Utils; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import static com.digitalpebble.spruce.SpruceColumn.*; import static org.junit.jupiter.api.Assertions.*; class AverageCarbonIntensityTest { + private static final StructType SCHEMA = new StructType(new StructField[]{ + StructField.apply(CURColumn.LINE_ITEM_USAGE_START_DATE.getLabel(), DataTypes.StringType, true, null)}); + private AverageCarbonIntensity module; @BeforeEach @@ -27,88 +34,94 @@ void setUp() { module.init(Map.of(), Provider.AWS); } + private static Row row(String usageStart) { + return new GenericRowWithSchema(new Object[]{usageStart}, SCHEMA); + } + + private Map enrich(Row row, String region) { + Map enriched = new HashMap<>(); + enriched.put(ENERGY_USED, 1.0); + enriched.put(REGION, region); + module.enrich(row, enriched); + return enriched; + } + + @Test + void returnsTheFigureOfTheYear() { + // Virginia moved from 322.4 in 2024 to 327.17 in 2025 + assertEquals(322.4, module.getIntensity(Provider.AWS, "us-east-1", 2024)); + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", 2025)); + } + + @Test + void usesTheLatestYearWhenTheYearIsNotPublishedYet() { + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", 2030)); + } + + @Test + void usesTheFirstYearBeforeTheFileStarts() { + // the file starts in 2022 + assertEquals(321.4, module.getIntensity(Provider.AWS, "us-east-1", 2019)); + } + @Test - void awsKnownRegionReturnsIntensity() { - assertEquals(384.4, module.getIntensity(Provider.AWS, "us-east-1")); + void usesTheLatestYearWithoutAYear() { + assertEquals(327.17, module.getIntensity(Provider.AWS, "us-east-1", null)); } @Test - void awsAnotherRegionReturnsIntensity() { - assertEquals(256.54, module.getIntensity(Provider.AWS, "eu-west-1")); + void usesTheClosestEarlierYearAcrossAGap() { + NavigableMap byYear = new TreeMap<>(Map.of(2022, 100.0, 2024, 200.0)); + assertEquals(100.0, AbstractEmberCarbonIntensity.forYear(byYear, 2023)); } @Test void gcpKnownRegionReturnsIntensity() { - assertEquals(259.37, module.getIntensity(Provider.GOOGLE, "us-east1")); + assertEquals(259.37, module.getIntensity(Provider.GOOGLE, "us-east1", 2025)); } @Test void azureKnownRegionReturnsIntensity() { - assertEquals(327.17, module.getIntensity(Provider.AZURE, "eastus")); + assertEquals(327.17, module.getIntensity(Provider.AZURE, "eastus", 2025)); } @Test void unknownRegionReturnsNull() { - assertNull(module.getIntensity(Provider.AWS, "us-fake-99")); + assertNull(module.getIntensity(Provider.AWS, "us-fake-99", 2025)); } @Test void wrongProviderReturnsNull() { // us-east-1 is valid for AWS but not for GCP - assertNull(module.getIntensity(Provider.GOOGLE, "us-east-1")); + assertNull(module.getIntensity(Provider.GOOGLE, "us-east-1", 2025)); } @Test - void enrichSetsCarbonIntensity() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - enriched.put(REGION, "us-east-1"); - - module.enrich(row, enriched); - - assertEquals(384.4, enriched.get(CARBON_INTENSITY)); + void enrichSetsTheCarbonIntensityOfTheUsageYear() { + assertEquals(322.4, enrich(row("2024-06-15T00:00:00Z"), "us-east-1").get(CARBON_INTENSITY)); + assertEquals(327.17, enrich(row("2025-06-15T00:00:00Z"), "us-east-1").get(CARBON_INTENSITY)); } @Test - void enrichSkipsWhenNoEnergy() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(REGION, "us-east-1"); - - module.enrich(row, enriched); - - assertFalse(enriched.containsKey(CARBON_INTENSITY)); + void enrichUsesTheLatestYearWithoutADate() { + assertEquals(327.17, enrich(row(null), "us-east-1").get(CARBON_INTENSITY)); } @Test void enrichSkipsWhenNoRegion() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); - - Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - - module.enrich(row, enriched); - - assertFalse(enriched.containsKey(CARBON_INTENSITY)); + assertFalse(enrich(row("2025-06-15T00:00:00Z"), null).containsKey(CARBON_INTENSITY)); } @Test void enrichSkipsUnknownRegion() { - StructType schema = Utils.getSchema(module); - Row row = new GenericRowWithSchema(new Object[schema.fields().length], schema); + assertFalse(enrich(row("2025-06-15T00:00:00Z"), "us-nowhere-99").containsKey(CARBON_INTENSITY)); + } + @Test + void enrichSkipsWhenNoEnergy() { Map enriched = new HashMap<>(); - enriched.put(ENERGY_USED, 1.0); - enriched.put(REGION, "us-nowhere-99"); - - module.enrich(row, enriched); - + enriched.put(REGION, "us-east-1"); + module.enrich(row("2025-06-15T00:00:00Z"), enriched); assertFalse(enriched.containsKey(CARBON_INTENSITY)); } }