diff --git a/docs/docs/Methodology/Converting Gross to Net Generation.md b/docs/docs/Methodology/Converting Gross to Net Generation.md index 9f9cdac..8a63628 100644 --- a/docs/docs/Methodology/Converting Gross to Net Generation.md +++ b/docs/docs/Methodology/Converting Gross to Net Generation.md @@ -55,11 +55,15 @@ The conversion factors are applied in the following hierarchical order: 1. Subplant-specific, annual gross-to-net ratio 2. Plant-specific, annual gross-to-net ratio 3. Fleet-specific, annual gross-to-net ratio -4. Prime mover-specific, default gross-to-net ratios from EIA -5. Apply an assumed gross to net ratio of 0.97 +4. Subplant-specific, annual gross-to-net shift factor +5. Plant-specific, annual gross-to-net shift factor +6. Prime mover-specific, default gross-to-net ratios from EIA +7. Apply an assumed gross to net ratio of 0.97 If no plant-specific factors are available (methods 1 and 2), the pipeline uses a fleet-specific ratio that represents the average gross-to-net ratio for all subplant (nationally) that consume the same fuel category (natural gas, coal, etc). Otherwise, we use default gross to net ratios published in the EIA Electric Power Monthly Technical Notes (Appendix C). As a final backstop, we use an assumed gross to net ratio of 0.97 +Shift factors are used instead of ratios in cases where the reported gross generation data in CEMS is zero, but EIA reports non-zero net generation data. In these cases, multiplying by a ratio would result in 0 calculated net generation. Applying a shift factor (which is added to the gross generation) effectively applies a flat generation profile to the data, but ensures that the annual total net mwh will match the data in EIA. + The following table shows what percent of gross generation reported in CEMS was converted to net generation using each method. @@ -119,10 +123,10 @@ The following table shows what percent of gross generation reported in CEMS was Before calculating net generation, the conversion factors are filtered to remove any factors that would lead to anomalous net generation values being calculated. -Based on analysis of historical data, the interquartile range of gross to net generation values is between 0.75 and 1.0, with an upper bound of approximately 1.25. +Based on analysis of historical data, about 90% of the gross to net generation values fall between approximately 0.5 and 1.25. The first filter removes factors that would lead to net generation values that significantly exceed the subplant’s nameplate capacity. In this case, the factor is filtered out if the 98th percentile of calculated hourly net generation values in a month exceed 125% of the subplant’s nameplate capacity. We use the 98th percentile instead of the maximum value to allow for a small handful of hours (approximately 14 hours in a 30-day month) to exceed the value. The use of the 125% exceedance threshold allows for the fact that a plant’s nameplate capacity can vary throughout the year and sometimes be exceeded, and the use of the 98th percentile prevents a small number of anomalous hours from causing the factor to be filtered out. -The second filter removes any ratios that are less than 0.75 (including negative ratios), since multiplying a negative ratio by gross generation would invert the shape of the hourly gross generation data. +The second filter removes any ratios that are less than 0.5 (including negative ratios), since multiplying a negative ratio by gross generation would invert the shape of the hourly gross generation data. After these filters have been applied for individual factors for each subplant-month, the pipeline removes all factors of a certain type for an entire plant if there are any missing factors for any subplant-month. For example, if a subplant ratio for a single month at a single subplant is filtered out, all subplant ratios for all other months and subplants at the same plant will also be removed. \ No newline at end of file diff --git a/notebooks/work_in_progress/sandbox.ipynb b/notebooks/work_in_progress/sandbox.ipynb index d3df7e8..c6b493d 100644 --- a/notebooks/work_in_progress/sandbox.ipynb +++ b/notebooks/work_in_progress/sandbox.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -37,9 +37,998 @@ "logger = get_logger(\"test\")\n", "\n", "\n", - "year = 2022\n", + "year = 2018\n", "path_prefix = f\"{year}/\"" ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "cems = pd.read_csv(\n", + " outputs_folder(\"2018/cems_cleaned_2018.csv.zip\"),\n", + " compression=\"zip\",\n", + " dtype=get_dtypes(),\n", + ")\n", + "eia923_allocated = pd.read_csv(\n", + " outputs_folder(\"2018/eia923_allocated_2018.csv.zip\"),\n", + " compression=\"zip\",\n", + " dtype=get_dtypes(),\n", + ")\n", + "primary_fuel_table = pd.read_csv(\n", + " outputs_folder(\"2018/primary_fuel_table_2018.csv.zip\"),\n", + " compression=\"zip\",\n", + " dtype=get_dtypes(),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "cems[\"report_date\"] = cems[\"report_date\"].astype(\"datetime64[s]\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "eia923_allocated[\"report_date\"] = eia923_allocated[\"report_date\"].astype(\n", + " \"datetime64[s]\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-06-29 12:55:04 [INFO] oge.oge.validation:480 Checking that all data has an associated `subplant_id`... \n", + "2024-06-29 12:55:04 [INFO] oge.oge.validation:493 OK\n", + "2024-06-29 12:56:22 [INFO] oge.oge.validation:480 Checking that all data has an associated `subplant_id`... \n", + "2024-06-29 12:56:22 [INFO] oge.oge.validation:493 OK\n", + "2024-06-29 12:56:28 [WARNING] oge.oge.validation:520 There are 1235 subplants at 517 plants for which there is zero gross generation associated with positive net generation.\n", + "2024-06-29 12:56:31 [WARNING] oge.oge.validation:523 \n", + " plant_id_eia subplant_id report_date gross_generation_mwh net_generation_mwh data_source ba_code\n", + "0 7 1 2018-01-01 0.0 14990.0 both SOCO\n", + "1 7 1 2018-02-01 0.0 13488.0 both SOCO\n", + "2 7 1 2018-03-01 0.0 5462.0 both SOCO\n", + "3 7 1 2018-04-01 0.0 16896.0 both SOCO\n", + "4 7 1 2018-05-01 0.0 14882.0 both SOCO\n", + "5 7 1 2018-06-01 0.0 14478.0 both SOCO\n", + "6 7 1 2018-07-01 0.0 15459.0 both SOCO\n", + "7 7 1 2018-08-01 0.0 14953.0 both SOCO\n", + "8 7 1 2018-09-01 0.0 14631.0 both SOCO\n", + "9 7 1 2018-10-01 0.0 15031.0 both SOCO\n", + "...\n", + " plant_id_eia subplant_id report_date gross_generation_mwh net_generation_mwh data_source ba_code\n", + "5021 60340 1 2018-03-01 0.0 17426.0 both SOCO\n", + "5022 60357 1 2018-04-01 0.0 17378.2 both PJM\n", + "5023 60357 2 2018-07-01 0.0 180774.7 both PJM\n", + "5024 60357 2 2018-08-01 0.0 224580.2 both PJM\n", + "5025 60357 3 2018-10-01 0.0 19787.7 both PJM\n", + "5026 60368 1 2018-04-01 0.0 33847.0 both PJM\n", + "5027 60376 1 2018-07-01 0.0 2486.0 both PJM\n", + "5028 60459 2 2018-10-01 0.0 1840.5 both ERCO\n", + "5029 60460 1 2018-10-01 0.0 1795.0 both ERCO\n", + "5030 60903 2 2018-11-01 0.0 2262.0 both ISNE\n", + "2024-06-29 12:56:31 [WARNING] oge.oge.validation:569 There are 52 subplants at 40 plants for which there is zero net generation associated with positive gross generation.\n", + "2024-06-29 12:56:33 [WARNING] oge.oge.validation:572 \n", + " plant_id_eia subplant_id report_date gross_generation_mwh net_generation_mwh data_source ba_code\n", + "0 127 1 2018-05-01 13.0 0.0 both SWPP\n", + "1 302 4 2018-10-01 21.0 0.0 both CISO\n", + "2 315 5 2018-03-01 9.0 0.0 both CISO\n", + "3 315 6 2018-03-01 369.0 0.0 both CISO\n", + "4 963 4 2018-10-01 2325.0 0.0 both MISO\n", + "5 1001 7 2018-10-01 4.0 0.0 both MISO\n", + "6 1167 2 2018-06-01 31.0 0.0 both MISO\n", + "7 1353 1 2018-12-01 12.0 0.0 both PJM\n", + "8 1417 3 2018-02-01 1.0 0.0 both SWPP\n", + "9 1417 3 2018-12-01 95.0 0.0 both SWPP\n", + "...\n", + " plant_id_eia subplant_id report_date gross_generation_mwh net_generation_mwh data_source ba_code\n", + "70 54844 2 2018-12-01 517.0 0.0 both PJM\n", + "71 55062 1 2018-10-01 171.0 0.0 both ERCO\n", + "72 55132 1 2018-03-01 254.0 0.0 both ERCO\n", + "73 55221 1 2018-02-01 1.0 0.0 both MISO\n", + "74 55225 2 2018-10-01 33.0 0.0 both SWPP\n", + "75 55282 1 2018-01-01 3.0 0.0 both DEAA\n", + "76 55364 1 2018-11-01 59.0 0.0 both MISO\n", + "77 55841 1 2018-01-01 4.0 0.0 both NEVP\n", + "78 56298 1 2018-05-01 5074.0 0.0 both BANC\n", + "79 60903 1 2018-11-01 2502.0 0.0 both ISNE\n", + "2024-06-29 12:56:33 [WARNING] oge.oge.validation:624 The following plants have annual net generation that is >125% of annual gross generation:\n", + "2024-06-29 12:56:35 [WARNING] oge.oge.validation:627 \n", + " plant_id_eia gross_generation_mwh net_generation_mwh annual_plant_ratio ba_code\n", + "192 60468 1349.0 23311.0 17.280208 ERCO\n", + "188 58284 16528.0 242057.8 14.645317 PNM\n", + "116 10349 37504.0 226368.0 6.035836 CISO\n", + "140 54802 72720.0 400556.0 5.508196 SOCO\n", + "117 10567 36109.0 162121.9 4.489792 ISNE\n", + "14 607 10539.0 40495.4 3.842433 PSEI\n", + "167 55470 1150808.0 3900260.0 3.389149 ERCO\n", + "13 597 68.0 224.6 3.302941 PJM\n", + "131 54096 114921.0 303247.0 2.638743 SOCO\n", + "63 2712 3011259.0 5927875.8 1.968571 CPLE\n", + "129 52168 42146.0 80497.3 1.909963 NYIS\n", + "190 60302 1201796.0 2189603.0 1.821942 PJM\n", + "113 10156 3256.0 5886.8 1.807985 CISO\n", + "68 3168 7922.0 13364.2 1.686973 PJM\n", + "111 8059 122031.0 196644.0 1.611427 SWPP\n", + "60 2503 42.0 67.6 1.609524 NYIS\n", + "17 689 761400.0 1195461.0 1.570083 TAL\n", + "145 55068 150162.0 234887.0 1.564224 ISNE\n", + "180 56565 1713394.0 2677131.0 1.562472 SWPP\n", + "151 55149 3347638.0 5211758.0 1.556846 ISNE\n", + "152 55154 1505234.0 2320257.0 1.541459 ERCO\n", + "92 7302 8693052.0 13368185.0 1.537801 FPC\n", + "79 4937 2011320.0 3086140.0 1.534385 ERCO\n", + "158 55231 2790774.0 4273036.0 1.531129 PJM\n", + "147 55079 732798.0 1118788.0 1.526735 ISNE\n", + "93 7314 423227.0 645840.0 1.525990 NYIS\n", + "97 7699 795077.0 1212922.0 1.525540 FPC\n", + "54 2398 2605917.0 3949317.0 1.515519 PJM\n", + "133 54324 551999.0 835958.0 1.514419 ISNE\n", + "166 55457 2359809.0 3529681.0 1.495749 SWPP\n", + "110 7985 427955.0 638948.0 1.493026 MISO\n", + "153 55170 1459514.0 2177483.0 1.491923 ISNE\n", + "159 55239 2863316.0 4268420.0 1.490726 PJM\n", + "105 7873 4465039.0 6654175.4 1.490284 TEC\n", + "178 55976 3564682.0 5311726.0 1.490098 PJM\n", + "157 55212 1446007.0 2152772.0 1.488770 ISNE\n", + "62 2539 3457482.0 5142115.0 1.487243 NYIS\n", + "149 55089 3561178.0 5289832.0 1.485416 MISO\n", + "127 51030 457334.0 679062.0 1.484827 ISNE\n", + "187 57978 947293.0 1406540.0 1.484799 CISO\n", + "148 55088 3348057.0 4969984.7 1.484438 MISO\n", + "174 55706 3430552.0 5070830.0 1.478138 TVA\n", + "103 7846 2909820.0 4292338.0 1.475121 JEA\n", + "163 55337 3534798.0 5209815.0 1.473865 PJM\n", + "164 55406 1229748.0 1808046.0 1.470257 SOCO\n", + "150 55096 99357.0 145667.2 1.466099 MISO\n", + "108 7946 1423393.0 2084949.0 1.464774 SOCO\n", + "3 533 2704642.0 3957128.9 1.463088 AEC\n", + "136 54592 1156.0 1690.0 1.461938 NYIS\n", + "175 55710 2668528.0 3893141.0 1.458910 PJM\n", + "162 55295 1849168.0 2695876.0 1.457886 CISO\n", + "135 54537 454384.0 660732.9 1.454129 PSEI\n", + "94 7380 2505963.0 3637056.9 1.451361 SEC\n", + "104 7870 138485.0 200543.0 1.448121 PSEI\n", + "138 54640 540322.0 780087.1 1.443745 PJM\n", + "100 7783 2501.0 3591.8 1.436146 PJM\n", + "69 3236 1304788.0 1866081.0 1.430179 ISNE\n", + "156 55211 1568625.0 2231535.0 1.422606 ISNE\n", + "134 54476 191987.0 272946.0 1.421690 PSEI\n", + "101 7826 3316477.0 4693043.5 1.415069 DUK\n", + "31 1393 3057068.0 4283357.8 1.401133 MISO\n", + "21 1007 1075456.0 1506408.0 1.400716 MISO\n", + "2 404 1277166.0 1786840.0 1.399066 LDWP\n", + "90 7153 1966967.0 2751210.0 1.398707 PJM\n", + "184 57564 125938.0 176139.0 1.398617 CISO\n", + "70 3295 1524011.0 2126871.7 1.395575 SCEG\n", + "141 54805 180567.0 250898.0 1.389501 ISNE\n", + "119 10726 365540.0 498909.0 1.364855 ISNE\n", + "132 54131 17676.0 24116.0 1.364336 NYIS\n", + "130 54041 322878.0 438704.1 1.358730 NYIS\n", + "143 55043 387596.0 523894.0 1.351650 DUK\n", + "120 50002 88618.0 117955.0 1.331050 ISNE\n", + "176 55821 1277743.0 1690975.0 1.323408 FPC\n", + "125 50744 19146.0 25240.0 1.318291 NYIS\n", + "122 50497 2562.0 3339.8 1.303591 PJM\n", + "115 10307 98697.0 128416.0 1.301114 ISNE\n", + "102 7834 3927020.0 5099120.3 1.298471 SC\n", + "123 50555 21795.0 28278.0 1.297454 PJM\n", + "118 10633 1108945.0 1434087.0 1.293199 PJM\n", + "137 54593 27099.0 34949.0 1.289679 NYIS\n", + "25 1206 7956.0 10246.0 1.287833 MISO\n", + "15 675 63587.0 81871.0 1.287543 FMPP\n", + "139 54785 791148.0 998083.1 1.261563 PJM\n", + "74 3469 453278.0 570606.4 1.258844 ERCO\n", + "81 6081 87763.0 110102.0 1.254538 ISNE\n", + "185 57703 149706.0 187201.0 1.250458 WACM\n", + "144 55064 1972915.0 1478679.0 0.749489 SWPP\n", + "18 715 13126.0 9808.2 0.747234 SOCO\n", + "91 7155 2489.0 1844.8 0.741181 MISO\n", + "98 7721 1790530.0 1325015.0 0.740013 SOCO\n", + "75 3492 12316.0 9111.6 0.739818 ERCO\n", + "80 6063 7490.0 5538.0 0.739386 MISO\n", + "171 55513 16628.0 12241.1 0.736174 CISO\n", + "66 3113 8683.0 6380.7 0.734850 PJM\n", + "20 995 250738.0 183970.0 0.733714 MISO\n", + "155 55206 2792738.0 2045715.0 0.732512 ERCO\n", + "182 56806 853160.0 623340.0 0.730625 ERCO\n", + "170 55510 15605.0 11380.8 0.729305 CISO\n", + "35 1592 5938.0 4329.6 0.729134 ISNE\n", + "67 3116 1111.0 806.2 0.725653 PJM\n", + "179 56397 8501.0 6127.0 0.720739 PJM\n", + "173 55625 18003.0 12968.0 0.720324 CISO\n", + "24 1131 4016.0 2888.0 0.719124 MISO\n", + "82 6124 81897.0 58476.4 0.714024 SOCO\n", + "121 50074 76.0 54.0 0.710526 PJM\n", + "128 52088 1348810.0 955109.0 0.708112 ERCO\n", + "124 50625 5960507.0 4210849.8 0.706458 MISO\n", + "109 7967 7144.0 5042.0 0.705767 PNM\n", + "169 55508 20173.0 14215.0 0.704655 CISO\n", + "168 55499 16217.0 11339.9 0.699260 CISO\n", + "86 6584 1139.0 794.2 0.697278 FPL\n", + "55 2401 1796.0 1242.9 0.692038 PJM\n", + "154 55187 7824369.0 5347392.0 0.683428 ERCO\n", + "41 1913 20843.0 14126.4 0.677753 MISO\n", + "107 7914 48992.0 33058.4 0.674771 NYIS\n", + "36 1595 1711191.0 1143916.0 0.668491 ISNE\n", + "106 7909 73151.0 48862.2 0.667964 NYIS\n", + "146 55075 1954556.0 1305210.0 0.667778 MISO\n", + "1 399 56056.0 37140.0 0.662552 LDWP\n", + "28 1248 9519.0 6202.1 0.651550 SWPP\n", + "8 563 1579.0 1004.8 0.636352 ISNE\n", + "96 7698 934575.0 571146.0 0.611129 SOCO\n", + "29 1295 45077.0 27497.2 0.610005 SWPP\n", + "49 2290 2198.0 1337.2 0.608371 SWPP\n", + "40 1730 10491.0 6251.8 0.595920 MISO\n", + "160 55243 66239.0 39453.9 0.595629 NYIS\n", + "95 7697 1374058.0 809934.0 0.589447 SOCO\n", + "189 58378 474167.0 276596.0 0.583330 ERCO\n", + "0 118 81300.0 46413.3 0.570889 AZPS\n", + "71 3318 1508.0 860.2 0.570424 SC\n", + "84 6258 7545.0 4153.8 0.550537 SOCO\n", + "191 60464 265224.0 145905.0 0.550120 PJM\n", + "33 1564 8262.0 4415.9 0.534483 PJM\n", + "99 7759 242333.0 127147.6 0.524681 MISO\n", + "83 6156 61869.0 31998.3 0.517194 ISNE\n", + "59 2499 77843.0 39870.4 0.512190 NYIS\n", + "11 591 1632.0 835.8 0.512132 PJM\n", + "142 54907 222836.0 111973.8 0.502494 ISNE\n", + "126 50949 21747.0 10709.2 0.492445 TEC\n", + "181 56668 105069.0 50618.0 0.481760 PJM\n", + "72 3320 1103.0 529.6 0.480145 SC\n", + "57 2494 18800.0 9022.4 0.479915 NYIS\n", + "112 8906 2319483.0 1092925.9 0.471194 NYIS\n", + "46 2176 2016.0 931.7 0.462153 MISO\n", + "32 1404 6878.0 3131.0 0.455220 MISO\n", + "165 55419 9855971.0 4441565.0 0.450647 MISO\n", + "52 2390 3869.0 1724.8 0.445800 PJM\n", + "87 6650 732.0 311.9 0.426093 MISO\n", + "44 2082 1087.0 459.0 0.422263 MISO\n", + "177 55833 815.0 340.0 0.417178 TEC\n", + "161 55247 267243.0 108652.2 0.406567 PJM\n", + "5 542 1338.0 539.4 0.403139 ISNE\n", + "88 6651 717.0 279.0 0.389121 MISO\n", + "43 2081 4106.0 1583.8 0.385728 SWPP\n", + "89 6652 727.0 279.0 0.383769 MISO\n", + "73 3406 852414.0 321910.4 0.377646 TVA\n", + "9 565 170.0 60.4 0.355294 ISNE\n", + "114 10176 1793.0 635.1 0.354211 ISNE\n", + "183 57068 9221.0 3162.0 0.342913 ISNE\n", + "26 1233 3494.0 1193.0 0.341442 SWPP\n", + "47 2241 4442.0 1308.8 0.294642 SWPP\n", + "39 1719 884.0 250.8 0.283710 MISO\n", + "6 557 182.0 49.4 0.271429 ISNE\n", + "38 1643 211.0 53.0 0.251185 ISNE\n", + "51 2369 315.0 72.3 0.229524 ISNE\n", + "37 1631 235.0 50.0 0.212766 ISNE\n", + "7 561 314.0 65.0 0.207006 ISNE\n", + "172 55600 3616.0 687.7 0.190183 NYIS\n", + "4 540 357.0 62.7 0.175630 ISNE\n", + "50 2362 337.0 56.4 0.167359 ISNE\n", + "34 1586 558.0 90.0 0.161290 ISNE\n", + "10 581 250.0 39.7 0.158800 ISNE\n", + "42 2001 520.0 66.7 0.128269 MISO\n", + "22 1048 2472.0 303.0 0.122573 MISO\n", + "61 2504 154.0 17.2 0.111688 NYIS\n", + "12 592 26.0 2.8 0.107692 PJM\n", + "53 2393 12190.0 1062.8 0.087186 PJM\n", + "186 57865 647458.0 45384.6 0.070097 SWPP\n", + "85 6463 468.0 31.2 0.066667 MISO\n", + "16 683 23895.0 1308.0 0.054739 FMPP\n", + "58 2496 588.0 25.5 0.043367 NYIS\n", + "48 2250 311.0 -93.9 -0.301929 SWPP\n", + "77 4005 591.0 -215.0 -0.363790 MISO\n", + "78 4195 1660.0 -692.0 -0.416867 ERCO\n", + "27 1235 1418.0 -661.0 -0.466150 SWPP\n", + "23 1081 2828.0 -1973.0 -0.697666 MISO\n", + "45 2098 15018.0 -11969.9 -0.797037 SWPP\n", + "56 2410 52.0 -65.0 -1.250000 PJM\n", + "76 3991 50.0 -150.0 -3.000000 MISO\n", + "30 1383 3500.0 -14868.0 -4.248000 MISO\n", + "19 874 13406.0 -57596.0 -4.296285 PJM\n", + "65 2861 42.0 -329.0 -7.833333 PJM\n", + "64 2848 53.0 -461.2 -8.701887 PJM\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
plant_id_eiasubplant_idreport_datehours_in_month_subplantgross_generation_mwhminimum_gross_generation_mwhmaximum_gross_generation_mwhnet_generation_mwhdata_sourcecapacity_mwmonthly_subplant_ratiomonthly_subplant_shift_mwannual_subplant_ratioannual_subplant_shift_mwmonthly_plant_ratiohours_in_month_plantannual_plant_ratioannual_plant_shift_mwannual_fleet_ratiosubplant_regression_ratiosubplant_regression_shift_mwsubplant_regression_rsq_adjplant_regression_ratioplant_regression_shift_mwplant_regression_rsq_adjdefault_gtn_ratio
0212018-01-01NaNNaNNaNNaN-35.5eia_only53.9NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
1212018-02-01NaNNaNNaNNaN-35.2eia_only53.9NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
2212018-03-01NaNNaNNaNNaN-35.8eia_only53.9NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
3212018-04-01NaNNaNNaNNaN-37.8eia_only53.9NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
4212018-05-01NaNNaNNaNNaN-42.3eia_only53.9NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
.................................................................................
2403696277812018-12-01NaNNaNNaNNaN445.4eia_only4.0NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
2403706278412018-12-01NaNNaNNaNNaN13.0eia_only1.6NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
2403716278512018-12-01NaNNaNNaNNaN9.0eia_only1.1NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
2403726280512018-12-01NaNNaNNaNNaN99.0eia_only2.0NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
2403736281012018-12-01NaNNaNNaNNaN3525.0eia_only1.6NaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN0.99
\n", + "

240374 rows × 26 columns

\n", + "
" + ], + "text/plain": [ + " plant_id_eia subplant_id report_date hours_in_month_subplant gross_generation_mwh minimum_gross_generation_mwh maximum_gross_generation_mwh net_generation_mwh data_source capacity_mw monthly_subplant_ratio monthly_subplant_shift_mw annual_subplant_ratio annual_subplant_shift_mw monthly_plant_ratio hours_in_month_plant annual_plant_ratio annual_plant_shift_mw annual_fleet_ratio subplant_regression_ratio subplant_regression_shift_mw subplant_regression_rsq_adj plant_regression_ratio plant_regression_shift_mw plant_regression_rsq_adj default_gtn_ratio\n", + "0 2 1 2018-01-01 NaN NaN NaN NaN -35.5 eia_only 53.9 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "1 2 1 2018-02-01 NaN NaN NaN NaN -35.2 eia_only 53.9 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "2 2 1 2018-03-01 NaN NaN NaN NaN -35.8 eia_only 53.9 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "3 2 1 2018-04-01 NaN NaN NaN NaN -37.8 eia_only 53.9 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "4 2 1 2018-05-01 NaN NaN NaN NaN -42.3 eia_only 53.9 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...\n", + "240369 62778 1 2018-12-01 NaN NaN NaN NaN 445.4 eia_only 4.0 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "240370 62784 1 2018-12-01 NaN NaN NaN NaN 13.0 eia_only 1.6 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "240371 62785 1 2018-12-01 NaN NaN NaN NaN 9.0 eia_only 1.1 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "240372 62805 1 2018-12-01 NaN NaN NaN NaN 99.0 eia_only 2.0 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "240373 62810 1 2018-12-01 NaN NaN NaN NaN 3525.0 eia_only 1.6 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.99\n", + "\n", + "[240374 rows x 26 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gtn_conversions = gross_to_net_generation.calculate_gross_to_net_conversion_factors(\n", + " cems, eia923_allocated, primary_fuel_table, year\n", + ")\n", + "gtn_conversions" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "factors_to_use = gross_to_net_generation.filter_gtn_conversion_factors(gtn_conversions)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
plant_id_eiasubplant_idreport_datedata_sourcegross_generation_mwhnet_generation_mwhminimum_gross_generation_mwhmaximum_gross_generation_mwhcapacity_mwannual_subplant_ratioannual_plant_ratioannual_fleet_ratioannual_subplant_shift_mwannual_plant_shift_mwdefault_gtn_ratio
0212018-01-01eia_onlyNaN-35.5NaNNaN53.9NaNNaNNaNNaNNaN0.99
1212018-02-01eia_onlyNaN-35.2NaNNaN53.9NaNNaNNaNNaNNaN0.99
2212018-03-01eia_onlyNaN-35.8NaNNaN53.9NaNNaNNaNNaNNaN0.99
3212018-04-01eia_onlyNaN-37.8NaNNaN53.9NaNNaNNaNNaNNaN0.99
4212018-05-01eia_onlyNaN-42.3NaNNaN53.9NaNNaNNaNNaNNaN0.99
................................................
2403696277812018-12-01eia_onlyNaN445.4NaNNaN4.0NaNNaNNaNNaNNaN0.99
2403706278412018-12-01eia_onlyNaN13.0NaNNaN1.6NaNNaNNaNNaNNaN0.99
2403716278512018-12-01eia_onlyNaN9.0NaNNaN1.1NaNNaNNaNNaNNaN0.99
2403726280512018-12-01eia_onlyNaN99.0NaNNaN2.0NaNNaNNaNNaNNaN0.99
2403736281012018-12-01eia_onlyNaN3525.0NaNNaN1.6NaNNaNNaNNaNNaN0.99
\n", + "

240374 rows × 15 columns

\n", + "
" + ], + "text/plain": [ + " plant_id_eia subplant_id report_date data_source gross_generation_mwh net_generation_mwh minimum_gross_generation_mwh maximum_gross_generation_mwh capacity_mw annual_subplant_ratio annual_plant_ratio annual_fleet_ratio annual_subplant_shift_mw annual_plant_shift_mw default_gtn_ratio\n", + "0 2 1 2018-01-01 eia_only NaN -35.5 NaN NaN 53.9 NaN NaN NaN NaN NaN 0.99\n", + "1 2 1 2018-02-01 eia_only NaN -35.2 NaN NaN 53.9 NaN NaN NaN NaN NaN 0.99\n", + "2 2 1 2018-03-01 eia_only NaN -35.8 NaN NaN 53.9 NaN NaN NaN NaN NaN 0.99\n", + "3 2 1 2018-04-01 eia_only NaN -37.8 NaN NaN 53.9 NaN NaN NaN NaN NaN 0.99\n", + "4 2 1 2018-05-01 eia_only NaN -42.3 NaN NaN 53.9 NaN NaN NaN NaN NaN 0.99\n", + "... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...\n", + "240369 62778 1 2018-12-01 eia_only NaN 445.4 NaN NaN 4.0 NaN NaN NaN NaN NaN 0.99\n", + "240370 62784 1 2018-12-01 eia_only NaN 13.0 NaN NaN 1.6 NaN NaN NaN NaN NaN 0.99\n", + "240371 62785 1 2018-12-01 eia_only NaN 9.0 NaN NaN 1.1 NaN NaN NaN NaN NaN 0.99\n", + "240372 62805 1 2018-12-01 eia_only NaN 99.0 NaN NaN 2.0 NaN NaN NaN NaN NaN 0.99\n", + "240373 62810 1 2018-12-01 eia_only NaN 3525.0 NaN NaN 1.6 NaN NaN NaN NaN NaN 0.99\n", + "\n", + "[240374 rows x 15 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "factors_to_use" + ] } ], "metadata": { diff --git a/src/oge/column_checks.py b/src/oge/column_checks.py index 5a34f6e..8579ad7 100644 --- a/src/oge/column_checks.py +++ b/src/oge/column_checks.py @@ -325,6 +325,9 @@ "default_gtn_ratio", "monthly_subplant_ratio", "monthly_plant_ratio", + "annual_plant_shift_mw", + "monthly_subplant_shift_mw", + "annual_subplant_shift_mw", "subplant_regression_ratio", "subplant_regression_shift_mw", "subplant_regression_rsq_adj", @@ -461,7 +464,7 @@ def get_dtypes() -> dict: "nox_mass_lb_for_electricity": "float64", "nox_mass_lb_for_electricity_adjusted": "float64", "nox_mass_measurement_code": "category", - "operating_time_hours": "float16", + "operating_time_hours": "float32", "ozone_season_nox_emission_rate_lb_per_mmbtu": "float64", "particulate_emission_rate_lb_per_mmbtu": "float64", "particulate_removal_efficiency_annual": "float64", diff --git a/src/oge/gross_to_net_generation.py b/src/oge/gross_to_net_generation.py index fee21cc..05021bc 100644 --- a/src/oge/gross_to_net_generation.py +++ b/src/oge/gross_to_net_generation.py @@ -28,8 +28,10 @@ def convert_gross_to_net_generation( 1. Annual GTN ratio at the subplant level 2. Annual GTN ratio at the plant level 3. Annual GTN ratio at the national fleet (fuel category-prime mover) level - 4. Default GTN ratio published by EIA based on prime mover - 5. Assumed GTN ratio of 0.97 + 4. Annual GTN shift factor at the subplant level (used where gross = 0) + 5. Annual GTN shift factor at the plant level (used where gross = 0) + 6. Default GTN ratio published by EIA based on prime mover + 7. Assumed GTN ratio of 0.97 Args: cems (pd.DataFrame): hourly CEMS data aggregated to the subplant level @@ -58,6 +60,8 @@ def convert_gross_to_net_generation( "data_source", "annual_subplant_ratio", "annual_plant_ratio", + "annual_subplant_shift_mw", + "annual_plant_shift_mw", "annual_fleet_ratio", "default_gtn_ratio", ] @@ -82,14 +86,28 @@ def convert_gross_to_net_generation( cems["gross_generation_mwh"] * cems["annual_fleet_ratio"] ) - cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = "4_default_eia_ratio" + cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = ( + "4_annual_subplant_shift_factor" + ) + cems["net_generation_mwh"] = cems["net_generation_mwh"].fillna( + cems["gross_generation_mwh"] + cems["annual_subplant_shift_mw"] + ) + + cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = ( + "5_annual_plant_shift_factor" + ) + cems["net_generation_mwh"] = cems["net_generation_mwh"].fillna( + cems["gross_generation_mwh"] + cems["annual_plant_shift_mw"] + ) + + cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = "6_default_eia_ratio" cems["net_generation_mwh"] = cems["net_generation_mwh"].fillna( cems["gross_generation_mwh"] * cems["default_gtn_ratio"] ) # warn if there are any missing default gtn ratios for plants that would use them. missing_defaults = cems.loc[ - (cems["gtn_method"] == "4_default_eia_ratio") + (cems["gtn_method"] == "6_default_eia_ratio") & (cems["default_gtn_ratio"].isna()) ] if len(missing_defaults) > 0: @@ -109,7 +127,7 @@ def convert_gross_to_net_generation( .to_string() ) # if there is a missing default gtn ratio, fill with 0.97 - cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = "5_assumed_gtn_ratio" + cems.loc[cems["net_generation_mwh"].isna(), "gtn_method"] = "7_assumed_gtn_ratio" cems["default_gtn_ratio"] = cems["default_gtn_ratio"].fillna(0.97) cems["net_generation_mwh"] = cems["net_generation_mwh"].fillna( cems["gross_generation_mwh"] * cems["default_gtn_ratio"] @@ -122,6 +140,8 @@ def convert_gross_to_net_generation( "annual_subplant_ratio", "annual_plant_ratio", "annual_fleet_ratio", + "annual_subplant_shift_mw", + "annual_plant_shift_mw", "default_gtn_ratio", ] ) @@ -301,7 +321,7 @@ def calculate_gross_to_net_conversion_factors( ).replace([np.inf, -np.inf], np.nan) annual_fleet_ratio = annual_fleet_ratio[ (annual_fleet_ratio["annual_fleet_ratio"] <= 1.25) - & (annual_fleet_ratio["annual_fleet_ratio"] >= 0.75) + & (annual_fleet_ratio["annual_fleet_ratio"] >= 0.5) ] # sum data by fuel category @@ -325,10 +345,18 @@ def calculate_gross_to_net_conversion_factors( combined_gen_data["net_generation_mwh"] / combined_gen_data["gross_generation_mwh"] ).replace([np.inf, -np.inf], np.nan) + combined_gen_data["monthly_subplant_shift_mw"] = ( + combined_gen_data["net_generation_mwh"] + - combined_gen_data["gross_generation_mwh"] + ) / (combined_gen_data["hours_in_month"]) annual_subplant_ratio["annual_subplant_ratio"] = ( annual_subplant_ratio["net_generation_mwh"] / annual_subplant_ratio["gross_generation_mwh"] ).replace([np.inf, -np.inf], np.nan) + annual_subplant_ratio["annual_subplant_shift_mw"] = ( + annual_subplant_ratio["net_generation_mwh"] + - annual_subplant_ratio["gross_generation_mwh"] + ) / (annual_subplant_ratio["hours_in_month"]) monthly_plant_ratio["monthly_plant_ratio"] = ( monthly_plant_ratio["net_generation_mwh"] / monthly_plant_ratio["gross_generation_mwh"] @@ -337,6 +365,10 @@ def calculate_gross_to_net_conversion_factors( annual_plant_ratio["net_generation_mwh"] / annual_plant_ratio["gross_generation_mwh"] ).replace([np.inf, -np.inf], np.nan) + annual_plant_ratio["annual_plant_shift_mw"] = ( + annual_plant_ratio["net_generation_mwh"] + - annual_plant_ratio["gross_generation_mwh"] + ) / (annual_plant_ratio["hours_in_month"]) # flag anomalous plant ratios validation.identify_anomalous_annual_plant_gtn_ratios(annual_plant_ratio, year) @@ -518,7 +550,8 @@ def calculate_subplant_nameplate_capacity(year): def filter_gtn_conversion_factors(gtn_conversions: pd.DataFrame) -> pd.DataFrame: """Filters the calculated GTN ratios to remove anomalous or incomplete factors. - First, we remove any ratios that are less than 0.75 or greater than 1.25. + First, we remove any ratios that are less than 0.5 or would cause the generation to + exceed 125% of nameplate capacity. We also want to ensure that at each plant, we either use all annual_subplant_ratio or all annual_plant_ratio so that the annual plant total net generation matches. We remove any annual_subplant_ratios if they are not available for all subplants at a @@ -545,6 +578,8 @@ def filter_gtn_conversion_factors(gtn_conversions: pd.DataFrame) -> pd.DataFrame "annual_subplant_ratio", "annual_plant_ratio", "annual_fleet_ratio", + "annual_subplant_shift_mw", + "annual_plant_shift_mw", "default_gtn_ratio", ] ] @@ -554,11 +589,9 @@ def filter_gtn_conversion_factors(gtn_conversions: pd.DataFrame) -> pd.DataFrame "annual_plant_ratio", "annual_fleet_ratio", ]: - # remove any factors that would scale net generation to less than 75% of gross - # generation. In general, the IQR of GTN ratios is between 0.75 and 1, with an - # upper bound around 1.25. Remove any ratios that are negative to avoid flipping + # Remove any ratios that are negative to avoid flipping # the shape of the profile - factors_to_use.loc[factors_to_use[scaling_factor] < 0.75, scaling_factor] = ( + factors_to_use.loc[factors_to_use[scaling_factor] < 0.5, scaling_factor] = ( np.NaN ) # remove any factors that would cause the generation in any hour to exceed 125% @@ -572,12 +605,40 @@ def filter_gtn_conversion_factors(gtn_conversions: pd.DataFrame) -> pd.DataFrame scaling_factor, ] = np.NaN + for shift_factor in ["annual_subplant_shift_mw", "annual_plant_shift_mw"]: + # remove any shift factors that would lead net generation in any hour to be less than -50 MW + factors_to_use.loc[ + factors_to_use["minimum_gross_generation_mwh"] + + factors_to_use[shift_factor] + <= -50, + shift_factor, + ] = np.NaN + # remove any shift factors that would lead net generation in any hour to be greater than 125% of nameplate capacity + factors_to_use.loc[ + factors_to_use["maximum_gross_generation_mwh"] + + factors_to_use[shift_factor] + > (factors_to_use["capacity_mw"] * 1.25), + shift_factor, + ] = np.NaN + + # for subplants where gross generation is zero, drop the ratio data so that we use + # shift factors + factors_to_use.loc[ + factors_to_use.groupby(["plant_id_eia", "subplant_id"])[ + "gross_generation_mwh" + ].transform("sum") + == 0, + ["annual_subplant_ratio", "annual_plant_ratio", "annual_fleet_ratio"], + ] = np.NaN + # All subplant-months at each plant should use the same method # if any annual_subplant_ratio are missing at a plant, revert to using # annual_plant_ratio for the entire plant method_hierarchy = [ "annual_subplant_ratio", "annual_plant_ratio", + "annual_subplant_shift_mw", + "annual_plant_shift_mw", ] for method in method_hierarchy: diff --git a/src/oge/validation.py b/src/oge/validation.py index 1a4b07e..c6dac54 100644 --- a/src/oge/validation.py +++ b/src/oge/validation.py @@ -617,12 +617,12 @@ def identify_anomalous_annual_plant_gtn_ratios(annual_plant_ratio, year): anomalous_gtn = annual_plant_ratio[ (annual_plant_ratio["annual_plant_ratio"] > 1.25) - | (annual_plant_ratio["annual_plant_ratio"] < 0.75) + | (annual_plant_ratio["annual_plant_ratio"] < 0.5) ] if len(anomalous_gtn) > 0: logger.warning( - "The following plants have annual net generation that is >125% of annual gross generation:" + "The following plants have annual net generation that is >125% or <50% of annual gross generation:" ) logger.warning( "\n" @@ -1715,7 +1715,7 @@ def summarize_cems_measurement_quality(cems): def identify_cems_gtn_method(cems): method_summary = cems.groupby("gtn_method", dropna=False)[ - "gross_generation_mwh" + "net_generation_mwh" ].sum() method_summary = method_summary / method_summary.sum(axis=0) method_summary = method_summary.reset_index()