Merge pull request #3489 from cal-itp/3483-ntd-validation-2024

Use dynamic years instead of fixed years on models used for NTD Validation
cal-itp · Oct 7, 2024 · b6a153e · b6a153e
2 parents 58cd9fa + 81a31d9
commit b6a153e
Show file tree

Hide file tree

Showing 14 changed files with 443 additions and 410 deletions.
diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql
@@ -1,6 +1,7 @@
 ---------
 ---- For assessing A10-032: Check that sum of total facilities for each agency, across all modes, is a whole number.
 ---------
+{% set this_year = run_started_at.year %}
 
 WITH fac_by_mode as (
 SELECT
@@ -17,7 +18,7 @@ SELECT
     as total_facilities,
     MAX(api_report_last_modified_date) as max_api_report_last_modified_date
 FROM {{ ref('stg_ntd_a10') }}
-WHERE api_report_period = 2023
+WHERE api_report_period = {{this_year}}
 GROUP BY organization, api_report_period, service_mode, total_facilities
 ),
 

diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql
@@ -1,6 +1,8 @@
 -- need fare rev and upt for each year.
+{% set this_year = run_started_at.year %}
+{% set last_year = this_year - 1 %}
 
-WITH fare_rev_2023 AS (
+WITH fare_rev_this_year AS (
   SELECT
     organization,
     api_report_period AS fiscal_year,
@@ -9,10 +11,11 @@ WITH fare_rev_2023 AS (
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
      FROM {{ ref('stg_ntd_rr20_rural') }}
      WHERE type = "Fare Revenues"
-     AND api_report_period = 2023
+     AND api_report_period = {{this_year}}
      GROUP BY organization, fiscal_year, mode, Fare_Revenues
 ),
-upt_2023 AS (
+
+upt_this_year AS (
   SELECT
     organization,
     api_report_period AS fiscal_year,
@@ -21,45 +24,62 @@ upt_2023 AS (
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
     FROM {{ ref('stg_ntd_rr20_rural') }}
      WHERE type = "Service Data"
-     AND api_report_period = 2023
+     AND api_report_period = {{this_year}}
     GROUP BY organization, fiscal_year, mode, Annual_UPT
 ),
-all_2023 AS (
-  select fare_rev_2023.organization,
-    fare_rev_2023.fiscal_year,
-    fare_rev_2023.mode,
-    fare_rev_2023.Fare_Revenues,
-    upt_2023.Annual_UPT
-  FROM fare_rev_2023
-  FULL OUTER JOIN upt_2023
-    ON fare_rev_2023.organization = upt_2023.organization
-    AND fare_rev_2023.mode = upt_2023.mode
+
+all_this_year AS (
+  select fare_rev_this_year.organization,
+    fare_rev_this_year.fiscal_year,
+    fare_rev_this_year.mode,
+    fare_rev_this_year.Fare_Revenues,
+    upt_this_year.Annual_UPT
+  FROM fare_rev_this_year
+  FULL OUTER JOIN upt_this_year
+    ON fare_rev_this_year.organization = upt_this_year.organization
+    AND fare_rev_this_year.mode = upt_this_year.mode
 ),
-fare_rev_2022 AS (
-  SELECT Organization_Legal_Name AS organization,
-  Fiscal_Year AS fiscal_year,
-  SUM(Fare_Revenues) AS Fare_Revenues
-  FROM {{ ref('stg_ntd_2022_rr20_financial') }}
-  GROUP BY organization, fiscal_year
+
+fare_rev_last_year AS (
+  SELECT
+    organization,
+    api_report_period AS fiscal_year,
+    item AS mode,
+    operations_expended + capital_expended AS Fare_Revenues,
+    MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
+     FROM {{ ref('stg_ntd_rr20_rural') }}
+     WHERE type = "Fare Revenues"
+     AND api_report_period = {{last_year}}
+     GROUP BY organization, fiscal_year, mode, Fare_Revenues
 ),
-upt_2022 AS (
-  select
-    Organization_Legal_Name AS organization,
-    Fiscal_Year AS  fiscal_year,
-    Mode AS mode,
-    Annual_UPT
-FROM {{ ref('stg_ntd_2022_rr20_service') }}
+
+upt_last_year AS (
+  SELECT
+    organization,
+    api_report_period AS fiscal_year,
+    item AS mode,
+    annual_unlinked_pass_trips AS Annual_UPT,
+    MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
+    FROM {{ ref('stg_ntd_rr20_rural') }}
+     WHERE type = "Service Data"
+     AND api_report_period = {{last_year}}
+    GROUP BY organization, fiscal_year, mode, Annual_UPT
 ),
-all_2022 AS (
-  select fare_rev_2022.organization, fare_rev_2022.fiscal_year,
-    upt_2022.Mode, fare_rev_2022.Fare_Revenues, upt_2022.Annual_UPT
-  FROM fare_rev_2022
-  FULL OUTER JOIN upt_2022
-    ON fare_rev_2022.organization = upt_2022.organization
+
+all_last_year AS (
+  select fare_rev_last_year.organization,
+    fare_rev_last_year.fiscal_year,
+    fare_rev_last_year.mode,
+    fare_rev_last_year.Fare_Revenues,
+    upt_last_year.Annual_UPT
+  FROM fare_rev_last_year
+  FULL OUTER JOIN upt_last_year
+    ON fare_rev_last_year.organization = upt_last_year.organization
+    AND fare_rev_last_year.mode = upt_last_year.mode
 )
 
-SELECT * FROM all_2023
+SELECT * FROM all_this_year
 
 UNION ALL
 
-SELECT * FROM all_2022
+SELECT * FROM all_last_year
diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql
@@ -1,11 +1,11 @@
 -------
 -- NTD validation errors about these 1 specific funding sources.
 --- ID #s RR20F-070, RR20F-065, RR20F-068, RR20F-066, RR20F-013. Sums the capital expenses across all funding sources
---- In 2022 the data is a different format than 2023 **and onwards**.
---- Only needed for the 2023 error checking (to compare to "last year"). In 2024 you don't need 2022 data.
 -------
+{% set this_year = run_started_at.year %}
+{% set last_year = this_year - 1 %}
 
-WITH longform_2023 AS (
+WITH longform_this_year AS (
     SELECT
     organization,
     api_report_period AS fiscal_year,
@@ -14,40 +14,52 @@ WITH longform_2023 AS (
       REPLACE(
         REPLACE(item, 'FTA Formula Grants for Rural Areas (§5311)', 'FTA_Formula_Grants_for_Rural_Areas_5311'),
         'Other Directly Generated Funds', 'Other_Directly_Generated_Funds'),
-    'Local Funds', 'Local_Funds') as item,
-    MAX(api_report_last_modified_date) as max_api_report_last_modified_date
+      'Local Funds', 'Local_Funds') AS item,
+    MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
      FROM {{ ref('stg_ntd_rr20_rural') }}
-     WHERE (item LIKE "%Directly Generated Funds%"
-      OR item LIKE "%Formula Grants for Rural Areas%"
-      OR item LIKE "Local Funds")
-      AND api_report_period = 2023
+    WHERE item LIKE ANY("%Directly Generated Funds%","%Formula Grants for Rural Areas%","Local Funds")
+      AND api_report_period = {{this_year}}
     GROUP BY organization, fiscal_year, total_expended, item
 ),
-wide_2023 AS (
+
+wide_this_year AS (
     SELECT * FROM
-    (SELECT * FROM longform_2023)
+    (SELECT * FROM longform_this_year)
     PIVOT(AVG(total_expended) FOR item IN ('FTA_Formula_Grants_for_Rural_Areas_5311', 'Other_Directly_Generated_Funds', 'Local_Funds'))
     ORDER BY organization
 ),
-data_2022 AS (
-    SELECT Organization_Legal_Name as organization,
-        Fiscal_Year as fiscal_year,
-        SUM(Other_Directly_Generated_Funds) as Other_Directly_Generated_Funds_2022,
-        SUM(FTA_Formula_Grants_for_Rural_Areas_5311) as FTA_Formula_Grants_for_Rural_Areas_5311_2022,
-        Null as Local_Funds_2022
-    FROM {{ ref('stg_ntd_2022_rr20_financial') }}
-    GROUP BY 1,2 -- noqa: L054
+
+longform_last_year AS (
+    SELECT
+    organization,
+    api_report_period AS fiscal_year,
+    operations_expended + capital_expended AS total_expended,
+    REPLACE(
+      REPLACE(
+        REPLACE(item, 'FTA Formula Grants for Rural Areas (§5311)', 'FTA_Formula_Grants_for_Rural_Areas_5311'),
+        'Other Directly Generated Funds', 'Other_Directly_Generated_Funds'),
+      'Local Funds', 'Local_Funds') AS item,
+    MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
+     FROM {{ ref('stg_ntd_rr20_rural') }}
+    WHERE item LIKE ANY("%Directly Generated Funds%","%Formula Grants for Rural Areas%","Local Funds")
+      AND api_report_period = {{last_year}}
+    GROUP BY organization, fiscal_year, total_expended, item
+),
+
+wide_last_year AS (
+    SELECT * FROM
+    (SELECT * FROM longform_last_year)
+    PIVOT(AVG(total_expended) FOR item IN ('FTA_Formula_Grants_for_Rural_Areas_5311', 'Other_Directly_Generated_Funds', 'Local_Funds'))
     ORDER BY organization
 )
 
-select wide_2023.organization,
-    wide_2023.FTA_Formula_Grants_for_Rural_Areas_5311 as FTA_Formula_Grants_for_Rural_Areas_5311_2023,
-    wide_2023.Other_Directly_Generated_Funds as Other_Directly_Generated_Funds_2023,
-    wide_2023.Local_Funds as Local_Funds_2023,
-    data_2022.FTA_Formula_Grants_for_Rural_Areas_5311_2022,
-    data_2022.Other_Directly_Generated_Funds_2022,
-    data_2022.Local_Funds_2022
-from wide_2023
-FULL OUTER JOIN data_2022
-    ON wide_2023.organization = data_2022.organization
-ORDER BY organization
+SELECT wide_this_year.organization,
+       wide_this_year.FTA_Formula_Grants_for_Rural_Areas_5311 AS FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,
+       wide_this_year.Other_Directly_Generated_Funds AS Other_Directly_Generated_Funds_This_Year,
+       wide_this_year.Local_Funds AS Local_Funds_This_Year,
+       wide_last_year.FTA_Formula_Grants_for_Rural_Areas_5311 AS FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year,
+       wide_last_year.Other_Directly_Generated_Funds AS Other_Directly_Generated_Funds_Last_Year
+  FROM wide_this_year
+  FULL OUTER JOIN wide_last_year
+    ON wide_this_year.organization = wide_last_year.organization
+ ORDER BY organization
diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql
@@ -3,62 +3,66 @@
 --- into one table for downstream validation checks. "Prior year" data not needed
 --- NTD error ID #s RR20F-001OA, RR20F-001C, RR20F-182
 ------
+{% set this_year = run_started_at.year %}
 
-WITH total_operations_exp_2023 AS(
+WITH total_operations_exp AS(
     SELECT organization,
     api_report_period AS fiscal_year,
     SUM(operations_expended) AS Total_Annual_Op_Expenses_by_Mode,
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
     FROM {{ ref('stg_ntd_rr20_rural') }}
      WHERE css_class = "expense"
-     AND api_report_period = 2023
+     AND api_report_period = {{this_year}}
      GROUP BY organization, api_report_period
 ),
-total_capital_exp_bymode_2023 AS (
+
+total_capital_exp_bymode AS (
     SELECT organization,
     api_report_period AS fiscal_year,
     SUM(capital_expended) AS Total_Annual_Cap_Expenses_byMode,
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
     FROM {{ ref('stg_ntd_rr20_rural') }}
      WHERE css_class = "expense"
-     AND api_report_period = 2023
+     AND api_report_period = {{this_year}}
     GROUP BY organization, api_report_period
 ),
-total_operations_rev_2023 AS (
+
+total_operations_rev AS (
     SELECT organization,
     api_report_period AS fiscal_year,
     SUM(operations_expended) AS Total_Annual_Op_Revenues_Expended,
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
     FROM {{ ref('stg_ntd_rr20_rural') }}
      WHERE css_class = "revenue"
-     AND api_report_period = 2023
+     AND api_report_period = {{this_year}}
     GROUP BY organization, api_report_period
 ),
-total_cap_exp_byfunds_2023 AS (
+
+total_cap_exp_byfunds AS (
     SELECT organization,
     api_report_period AS fiscal_year,
     SUM(capital_expended) AS Total_Annual_Cap_Expenses_byFunds,
     MAX(api_report_last_modified_date) AS max_api_report_last_modified_date
     FROM {{ ref('stg_ntd_rr20_rural') }}
     WHERE css_class = "revenue"
-    AND api_report_period = 2023
+    AND api_report_period = {{this_year}}
     GROUP BY organization, api_report_period
 )
 
 SELECT
-    total_operations_exp_2023.organization,
-    total_operations_exp_2023.fiscal_year,
-    total_operations_exp_2023.Total_Annual_Op_Expenses_by_Mode,
-    total_capital_exp_bymode_2023.Total_Annual_Cap_Expenses_byMode,
-    total_operations_rev_2023.Total_Annual_Op_Revenues_Expended,
-    total_cap_exp_byfunds_2023.Total_Annual_Cap_Expenses_byFunds
-FROM total_operations_exp_2023
-FULL OUTER JOIN total_capital_exp_bymode_2023
-    ON total_operations_exp_2023.organization = total_capital_exp_bymode_2023.organization
-    AND total_operations_exp_2023.fiscal_year = total_capital_exp_bymode_2023.fiscal_year
-FULL OUTER JOIN total_operations_rev_2023
-    ON total_operations_exp_2023.organization = total_operations_rev_2023.organization
-    AND total_operations_exp_2023.fiscal_year = total_operations_rev_2023.fiscal_year
-FULL OUTER JOIN total_cap_exp_byfunds_2023
-    ON total_operations_exp_2023.organization = total_cap_exp_byfunds_2023.organization
-    AND total_operations_exp_2023.fiscal_year = total_cap_exp_byfunds_2023.fiscal_year
+    total_operations_exp.organization,
+    total_operations_exp.fiscal_year,
+    total_operations_exp.Total_Annual_Op_Expenses_by_Mode,
+    total_capital_exp_bymode.Total_Annual_Cap_Expenses_byMode,
+    total_operations_rev.Total_Annual_Op_Revenues_Expended,
+    total_cap_exp_byfunds.Total_Annual_Cap_Expenses_byFunds
+FROM total_operations_exp
+FULL OUTER JOIN total_capital_exp_bymode
+    ON total_operations_exp.organization = total_capital_exp_bymode.organization
+    AND total_operations_exp.fiscal_year = total_capital_exp_bymode.fiscal_year
+FULL OUTER JOIN total_operations_rev
+    ON total_operations_exp.organization = total_operations_rev.organization
+    AND total_operations_exp.fiscal_year = total_operations_rev.fiscal_year
+FULL OUTER JOIN total_cap_exp_byfunds
+    ON total_operations_exp.organization = total_cap_exp_byfunds.organization
+    AND total_operations_exp.fiscal_year = total_cap_exp_byfunds.fiscal_year