Transform vceregen renewable generation profiles

docs/conf.py

@@ -162,6 +162,7 @@ def data_sources_metadata_to_rst(app):
         "epacems",
         "phmsagas",
         "gridpathratoolkit",
+        "vcerare",
     ]
     package = PUDL_PACKAGE
     extra_etl_groups = {"eia860": ["entity_eia"], "ferc1": ["glue"]}
@@ -213,6 +214,7 @@ def cleanup_rsts(app, exception):
     (DOCS_DIR / "data_sources/epacems.rst").unlink()
     (DOCS_DIR / "data_sources/phmsagas.rst").unlink()
     (DOCS_DIR / "data_sources/gridpathratoolkit.rst").unlink()
+    (DOCS_DIR / "data_sources/vcerare.rst").unlink()
 
 
 def cleanup_csv_dir(app, exception):

docs/data_sources/index.rst

@@ -18,6 +18,7 @@ The following data sources serve as the foundation for our data pipeline.
    ferc714
    phmsagas
    gridpathratoolkit
+   vcerare
    other_data
 
 .. toctree::

docs/release_notes.rst

@@ -6,6 +6,21 @@ PUDL Release Notes
 v2024.X.x (2024-XX-XX)
 ---------------------------------------------------------------------------------------
 
+New Data
+^^^^^^^^
+
+Vibrant Clean Energy Resource Adequacy Renewable Energy (RARE) Power Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Integrate the VCE hourly capacity factor data for solar PV, onshore wind, and
+  offshore wind from 2019 through 2023. The data in this table were produced by
+  Vibrant Clean Energy, and are licensed to the public under the Creative Commons
+  Attribution 4.0 International license (CC-BY-4.0). This data complements the
+  WECC-wide GridPath RA Toolkit data currently incorporated into PUDL, providing
+  capacity factor data nation-wide with a different set of modeling assumptions and
+  a different granularity for the aggregation of outputs.
+  See :doc:`data_sources/gridpathratoolkit` and :doc:`data_sources/vcerare` for
+  more information.  See :issue:`#3872`.
+
 New Data Coverage
 ^^^^^^^^^^^^^^^^^
 

docs/templates/vcerare_child.rst.jinja

@@ -0,0 +1,70 @@
+{% extends "data_source_parent.rst.jinja" %}
+{% block background %}
+The data in the Resource Adequacy Renewable Energy (RARE) Power Dataset was produced by
+Vibrant Clean Energy based on outputs from the NOA HRRR model and are licensed
+to the public under the Creative Commons Attribution 4.0 International license
+(CC-BY-4.0).
+
+See the `Zenodo archive README <https://doi.org/10.5281/zenodo.13937523>`__ for more
+detailed information.
+
+The cleaned PUDL table is hundreds of millions of rows, so don't try and open it with Excel!
+{% endblock %}
+
+{% block download_docs %}
+{% for filename in download_paths %}
+* :download:`{{ filename.stem.replace("-", " ").replace("_", " ").title() }} ({{ filename.suffix.replace('.', '').upper() }}) <{{ filename }}>`
+{% endfor %}
+* `NOAA HRRR Model Overview <https://rapidrefresh.noaa.gov/hrrr/>`__
+{% endblock %}
+
+
+{% block availability %}
+Hourly, county-level data from 2019 - 2023 is integrated into PUDL. There is a
+second release of data for the years 2014 - 2018 expected in Q1 of 2025, which will be
+integrated into PUDL pending funding availability.
+{% endblock %}
+
+{% block respondents %}
+This data does not come from a government agency, and is not the result of compulsory
+data reporting.
+{% endblock %}
+
+{% block original_data %}
+The contents of the original CSVs are formatted so that Excel can display the
+data without crashing. There's one file per year per generation type, and each
+file contains an index column for time (simply 1, 2, 3...8760 to
+represent the hours in a year) and columns for each county populated with capacity
+factor values as a percentage from 0-100.
+{% endblock %}
+
+{% block notable_irregularities %}
+Non-county regions
+------------------
+
+The original data include capacity factors for some non-county areas including the Great
+Lakes and 2 small cities (Bedford City, VA and Clifton Forge City, VA). It associated
+"county" FIPS IDs with those areas, meaning that there was not a 1:1 relationship
+between the FIPS IDs and the named areas, and the geographic region implied by the
+FIPS IDs did not correspond to the named area. We've dropped the cities -- one of which
+contained no data -- and set the FIPS codes for the Great Lakes to NA. Note that lakes
+bordering multiple states will appear more than once in the data. VCE used a nearest
+neighbor technique to assign the state waters to the counties (this pertains to coastal
+areas as well).
+
+Capacity factors > 1
+--------------------
+There are a couple of capacity factor values for the solar pv data that exceed
+the maximum value of 1 for capacity factor (or 100 for the raw data--PUDL converts the
+data from a percentage to a fraction to match other reported capacity factor data). This
+is due to power production performance being correlated with panel temperatures. During
+cold sunny periods, some solar capacity factor values are greater than 1 (but less that
+1.1).
+
+8760-hour years
+---------------
+This data is primarily used for modeling purposes and conforms to the 8760 hour/year
+standard regardless of leap years. This means that 2020 is missing data for December
+31st.
+
+{% endblock %}

src/pudl/etl/__init__.py

@@ -82,6 +82,7 @@
     "core_ferc714": [pudl.transform.ferc714],
     "core_gridpathratoolkit": [pudl.transform.gridpathratoolkit],
     "core_nrelatb": [pudl.transform.nrelatb],
+    "core_vcerare": [pudl.transform.vcerare],
 }
 
 out_module_groups = {

src/pudl/metadata/classes.py

@@ -1284,6 +1284,7 @@ class Resource(PudlMeta):
             "ppe",
             "pudl",
             "nrelatb",
+            "vcerare",
         ]
         | None
     ) = None
@@ -1311,6 +1312,7 @@ class Resource(PudlMeta):
             "static_pudl",
             "service_territories",
             "nrelatb",
+            "vcerare",
         ]
         | None
     ) = None

src/pudl/metadata/fields.py

@@ -466,6 +466,56 @@
         "type": "number",
         "description": "Fraction of potential generation that was actually reported for a plant part.",
     },
+    "capacity_factor_offshore_wind": {
+        "type": "number",
+        "description": (
+            "Estimated capacity factor (0-1) calculated for offshore wind "
+            "assuming a 140m hub height and 120m rotor diameter."
+            "Based on outputs from the NOAA HRRR operational numerical "
+            "weather prediction model. Capacity factors are normalized "
+            "to unity for maximal power output. "
+            "Vertical slices of the atmosphere are considered across the "
+            "defined rotor swept area. Bringing together wind speed, density, "
+            "temperature and icing information, a power capacity is estimated "
+            "using a representative power coefficient (Cp) curve to determine "
+            "the power from a given wind speed, atmospheric density and "
+            "temperature. There is no wake modeling included in the dataset."
+        ),
+    },
+    "capacity_factor_onshore_wind": {
+        "type": "number",
+        "description": (
+            "Estimated capacity factor (0-1) calculated for onshore wind "
+            "assuming a 100m hub height and 120m rotor diameter."
+            "Based on outputs from the NOAA HRRR operational numerical "
+            "weather prediction model. Capacity factors are normalized "
+            "to unity for maximal power output. "
+            "Vertical slices of the atmosphere are considered across the "
+            "defined rotor swept area. Bringing together wind speed, density, "
+            "temperature and icing information, a power capacity is estimated "
+            "using a representative power coefficient (Cp) curve to determine "
+            "the power from a given wind speed, atmospheric density and "
+            "temperature. There is no wake modeling included in the dataset."
+        ),
+    },
+    "capacity_factor_solar_pv": {
+        "type": "number",
+        "description": (
+            "Estimated capacity factor (0-1) calculated for solar PV "
+            "assuming a fixed axis panel tilted at latitude and DC power "
+            "outputs. Due to power production performance being correlated "
+            "with panel temperatures, during cold sunny periods, some solar "
+            "capacity factor values are greater than 1 (but less that 1.1)."
+            "All values are based on outputs from the NOAA HRRR operational "
+            "numerical weather prediction model. Capacity factors are "
+            "normalized to unity for maximal power output. "
+            "Pertinent surface weather variables are pulled such as "
+            "incoming short wave radiation, direct normal irradiance "
+            "(calculated in the HRRR 2016 forward), surface temperature "
+            "and other parameters. These are used in a non-linear I-V curve "
+            "translation to power capacity factors."
+        ),
+    },
     "capacity_mw": {
         "type": "number",
         "description": "Total installed (nameplate) capacity, in megawatts.",
@@ -836,6 +886,13 @@
         "type": "string",
         "description": "County name as specified in Census DP1 Data.",
     },
+    "county_or_lake_name": {
+        "type": "string",
+        "description": (
+            "County or lake name. Lake names may also appear several times--once for "
+            "each state it touches. FIPS ID values for lakes have been nulled."
+        ),
+    },
     "country_code": {
         "type": "string",
         "description": "Three letter ISO-3166 country code (e.g. USA or CAN).",
@@ -1970,6 +2027,10 @@
         "description": "The energy contained in fuel burned, measured in million BTU.",
         "unit": "MMBtu",
     },
+    "hour_of_year": {
+        "type": "integer",
+        "description": "Integer between 1 and 8670 representing the hour in a given year.",
+    },
     "unit_heat_rate_mmbtu_per_mwh": {
         "type": "number",
         "description": "Fuel content per unit of electricity generated. Coming from MCOE calculation.",

src/pudl/metadata/resources/vcerare.py

@@ -0,0 +1,64 @@
+"""Table definitions for data coming from the Vibrant Clean Energy Renewable Generation Profiles."""
+
+from typing import Any
+
+RESOURCE_METADATA: dict[str, dict[str, Any]] = {
+    "out_vcerare__hourly_available_capacity_factor": {
+        "description": (
+            "The data in this table were produced by Vibrant Clean Energy, and are "
+            "licensed to the public under the Creative Commons Attribution 4.0 International "
+            "license (CC-BY-4.0). The table consists of estimated county-averaged hourly "
+            "capacity factors for wind and solar generating facilities across the contiguous "
+            "United States (US) to be used as a tool and input for resource adequacy modeling "
+            "and planning. The hourly capacity factors are normalized to unity for maximal power "
+            "output. To convert to units of power, the user must multiply by the installed capacity "
+            "within the county.\n\n"
+            "The technologies provided are:\n"
+            "1) Onshore wind assuming a 100m hub height and 120m rotor diameter;\n"
+            "2) Offshore wind assuming a 140m hub height and 120m rotor diameter;\n"
+            "3) Utility solar assuming a fixed axis panel tilted at latitude.\n\n"
+            "The foundation of the capacity factors provided here is the NOAA HRRR "
+            "operational numerical weather prediction model. The HRRR covers the entire "
+            "contiguous US at a horizontal resolution of 3 km. Forecasts are intialized each "
+            "hour of the year. Forecast hour two (2) is used as the input data for the power "
+            "algorithms. This forecast hour is chosen to trade-off the impact of the measurement "
+            "and data assimilation procedure of the HRRR with the physics of the model to derive "
+            "the most complete picture of the atmosphere at the forecast time horizon. "
+            "Hourly capacity factors are spatially averaged across each county over the contiguous "
+            "USA. There are a handful of counties that are too small to pick up representation on "
+            "the HRRR operational forecast grid. As such, these counties will have no wind or solar "
+            "power production curves.\n\n"
+            "For wind capacity factors: vertical slices of the atmosphere are considered across "
+            "the defined rotor swept area. Bringing together wind speed, density, temperature and "
+            "icing information, a power capacity is estimated using a representative power coefficient "
+            "(Cp) curve to determine the power from a given wind speed, atmospheric density and "
+            "temperature. There is no wake modeling included in the dataset.\n\n"
+            "For solar capacity factors: pertinent surface weather variables are pulled such as "
+            "incoming short wave radiation, direct normal irradiance (calculated in the HRRR 2016 "
+            "forward), surface temperature and other parameters. These are used in a non-linear "
+            "I-V curve translation to power capacity factors. Due to power production "
+            "performance being correlated with panel temperatures, during cold sunny periods, "
+            "some solar capacity factor values are greater than 1 (but less that 1.1)."
+        ),
+        "schema": {
+            "fields": [
+                "datetime_utc",
+                "hour_of_year",
+                "report_year",
+                "county_id_fips",
+                "county_or_lake_name",
+                "state",
+                "latitude",
+                "longitude",
+                "capacity_factor_solar_pv",
+                "capacity_factor_onshore_wind",
+                "capacity_factor_offshore_wind",
+            ],
+            "primary_key": ["datetime_utc", "state", "county_or_lake_name"],
+        },
+        "sources": ["vcerare"],
+        "field_namespace": "vcerare",
+        "etl_group": "vcerare",
+        "create_database_schema": False,
+    },
+}

src/pudl/transform/__init__.py

@@ -77,4 +77,5 @@
     gridpathratoolkit,
     nrelatb,
     params,
+    vcerare,
 )