From 81a31d902a125e49a0bf0662a884909ebf1c8821 Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Tue, 1 Oct 2024 12:31:02 -0700 Subject: [PATCH] Replace fixed years on NTD validation tables to use dynamic years based on the timestamp that the DAG run started. [#3483] --- .../int_ntd_a10_facilitiesdata.sql | 3 +- .../int_ntd_rr20_financial_fare_revenues.sql | 90 +++++--- .../int_ntd_rr20_financial_specific_funds.sql | 72 +++--- .../int_ntd_rr20_financial_total_exp.sql | 52 +++-- .../int_ntd_rr20_service_1alldata.sql | 156 ++++++------- .../int_ntd_rr20_service_3ratios_wide.sql | 79 ++++--- .../ntd_validation/int_ntd_validation.yml | 22 +- .../ntd_validation/_mart_ntd_validation.yml | 3 +- .../fct_ntd_rr20_funds_checks.sql | 148 +++++++------ .../fct_ntd_rr20_service_checks.sql | 209 +++++++++--------- .../stg_ntd_2022_rr20_exp_by_mode.sql | 6 - .../stg_ntd_2022_rr20_financial.sql | 6 - .../stg_ntd_2022_rr20_service.sql | 6 - .../stg_ntd_rr20_urban_tribal.sql | 1 - 14 files changed, 443 insertions(+), 410 deletions(-) delete mode 100644 warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_exp_by_mode.sql delete mode 100644 warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_financial.sql delete mode 100644 warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_service.sql diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql index 9e78b66f2b..460113aaba 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_a10_facilitiesdata.sql @@ -1,6 +1,7 @@ --------- ---- For assessing A10-032: Check that sum of total facilities for each agency, across all modes, is a whole number. --------- +{% set this_year = run_started_at.year %} WITH fac_by_mode as ( SELECT @@ -17,7 +18,7 @@ SELECT as total_facilities, MAX(api_report_last_modified_date) as max_api_report_last_modified_date FROM {{ ref('stg_ntd_a10') }} -WHERE api_report_period = 2023 +WHERE api_report_period = {{this_year}} GROUP BY organization, api_report_period, service_mode, total_facilities ), diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql index 3e5b899027..c3997924f0 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_fare_revenues.sql @@ -1,6 +1,8 @@ -- need fare rev and upt for each year. +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} -WITH fare_rev_2023 AS ( +WITH fare_rev_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -9,10 +11,11 @@ WITH fare_rev_2023 AS ( MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE type = "Fare Revenues" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year, mode, Fare_Revenues ), -upt_2023 AS ( + +upt_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -21,45 +24,62 @@ upt_2023 AS ( MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE type = "Service Data" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year, mode, Annual_UPT ), -all_2023 AS ( - select fare_rev_2023.organization, - fare_rev_2023.fiscal_year, - fare_rev_2023.mode, - fare_rev_2023.Fare_Revenues, - upt_2023.Annual_UPT - FROM fare_rev_2023 - FULL OUTER JOIN upt_2023 - ON fare_rev_2023.organization = upt_2023.organization - AND fare_rev_2023.mode = upt_2023.mode + +all_this_year AS ( + select fare_rev_this_year.organization, + fare_rev_this_year.fiscal_year, + fare_rev_this_year.mode, + fare_rev_this_year.Fare_Revenues, + upt_this_year.Annual_UPT + FROM fare_rev_this_year + FULL OUTER JOIN upt_this_year + ON fare_rev_this_year.organization = upt_this_year.organization + AND fare_rev_this_year.mode = upt_this_year.mode ), -fare_rev_2022 AS ( - SELECT Organization_Legal_Name AS organization, - Fiscal_Year AS fiscal_year, - SUM(Fare_Revenues) AS Fare_Revenues - FROM {{ ref('stg_ntd_2022_rr20_financial') }} - GROUP BY organization, fiscal_year + +fare_rev_last_year AS ( + SELECT + organization, + api_report_period AS fiscal_year, + item AS mode, + operations_expended + capital_expended AS Fare_Revenues, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + FROM {{ ref('stg_ntd_rr20_rural') }} + WHERE type = "Fare Revenues" + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year, mode, Fare_Revenues ), -upt_2022 AS ( - select - Organization_Legal_Name AS organization, - Fiscal_Year AS fiscal_year, - Mode AS mode, - Annual_UPT -FROM {{ ref('stg_ntd_2022_rr20_service') }} + +upt_last_year AS ( + SELECT + organization, + api_report_period AS fiscal_year, + item AS mode, + annual_unlinked_pass_trips AS Annual_UPT, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + FROM {{ ref('stg_ntd_rr20_rural') }} + WHERE type = "Service Data" + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year, mode, Annual_UPT ), -all_2022 AS ( - select fare_rev_2022.organization, fare_rev_2022.fiscal_year, - upt_2022.Mode, fare_rev_2022.Fare_Revenues, upt_2022.Annual_UPT - FROM fare_rev_2022 - FULL OUTER JOIN upt_2022 - ON fare_rev_2022.organization = upt_2022.organization + +all_last_year AS ( + select fare_rev_last_year.organization, + fare_rev_last_year.fiscal_year, + fare_rev_last_year.mode, + fare_rev_last_year.Fare_Revenues, + upt_last_year.Annual_UPT + FROM fare_rev_last_year + FULL OUTER JOIN upt_last_year + ON fare_rev_last_year.organization = upt_last_year.organization + AND fare_rev_last_year.mode = upt_last_year.mode ) -SELECT * FROM all_2023 +SELECT * FROM all_this_year UNION ALL -SELECT * FROM all_2022 +SELECT * FROM all_last_year diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql index 2a3de1de70..27e1fbe6c9 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_specific_funds.sql @@ -1,11 +1,11 @@ ------- -- NTD validation errors about these 1 specific funding sources. --- ID #s RR20F-070, RR20F-065, RR20F-068, RR20F-066, RR20F-013. Sums the capital expenses across all funding sources ---- In 2022 the data is a different format than 2023 **and onwards**. ---- Only needed for the 2023 error checking (to compare to "last year"). In 2024 you don't need 2022 data. ------- +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} -WITH longform_2023 AS ( +WITH longform_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -14,40 +14,52 @@ WITH longform_2023 AS ( REPLACE( REPLACE(item, 'FTA Formula Grants for Rural Areas (§5311)', 'FTA_Formula_Grants_for_Rural_Areas_5311'), 'Other Directly Generated Funds', 'Other_Directly_Generated_Funds'), - 'Local Funds', 'Local_Funds') as item, - MAX(api_report_last_modified_date) as max_api_report_last_modified_date + 'Local Funds', 'Local_Funds') AS item, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} - WHERE (item LIKE "%Directly Generated Funds%" - OR item LIKE "%Formula Grants for Rural Areas%" - OR item LIKE "Local Funds") - AND api_report_period = 2023 + WHERE item LIKE ANY("%Directly Generated Funds%","%Formula Grants for Rural Areas%","Local Funds") + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year, total_expended, item ), -wide_2023 AS ( + +wide_this_year AS ( SELECT * FROM - (SELECT * FROM longform_2023) + (SELECT * FROM longform_this_year) PIVOT(AVG(total_expended) FOR item IN ('FTA_Formula_Grants_for_Rural_Areas_5311', 'Other_Directly_Generated_Funds', 'Local_Funds')) ORDER BY organization ), -data_2022 AS ( - SELECT Organization_Legal_Name as organization, - Fiscal_Year as fiscal_year, - SUM(Other_Directly_Generated_Funds) as Other_Directly_Generated_Funds_2022, - SUM(FTA_Formula_Grants_for_Rural_Areas_5311) as FTA_Formula_Grants_for_Rural_Areas_5311_2022, - Null as Local_Funds_2022 - FROM {{ ref('stg_ntd_2022_rr20_financial') }} - GROUP BY 1,2 -- noqa: L054 + +longform_last_year AS ( + SELECT + organization, + api_report_period AS fiscal_year, + operations_expended + capital_expended AS total_expended, + REPLACE( + REPLACE( + REPLACE(item, 'FTA Formula Grants for Rural Areas (§5311)', 'FTA_Formula_Grants_for_Rural_Areas_5311'), + 'Other Directly Generated Funds', 'Other_Directly_Generated_Funds'), + 'Local Funds', 'Local_Funds') AS item, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + FROM {{ ref('stg_ntd_rr20_rural') }} + WHERE item LIKE ANY("%Directly Generated Funds%","%Formula Grants for Rural Areas%","Local Funds") + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year, total_expended, item +), + +wide_last_year AS ( + SELECT * FROM + (SELECT * FROM longform_last_year) + PIVOT(AVG(total_expended) FOR item IN ('FTA_Formula_Grants_for_Rural_Areas_5311', 'Other_Directly_Generated_Funds', 'Local_Funds')) ORDER BY organization ) -select wide_2023.organization, - wide_2023.FTA_Formula_Grants_for_Rural_Areas_5311 as FTA_Formula_Grants_for_Rural_Areas_5311_2023, - wide_2023.Other_Directly_Generated_Funds as Other_Directly_Generated_Funds_2023, - wide_2023.Local_Funds as Local_Funds_2023, - data_2022.FTA_Formula_Grants_for_Rural_Areas_5311_2022, - data_2022.Other_Directly_Generated_Funds_2022, - data_2022.Local_Funds_2022 -from wide_2023 -FULL OUTER JOIN data_2022 - ON wide_2023.organization = data_2022.organization -ORDER BY organization +SELECT wide_this_year.organization, + wide_this_year.FTA_Formula_Grants_for_Rural_Areas_5311 AS FTA_Formula_Grants_for_Rural_Areas_5311_This_Year, + wide_this_year.Other_Directly_Generated_Funds AS Other_Directly_Generated_Funds_This_Year, + wide_this_year.Local_Funds AS Local_Funds_This_Year, + wide_last_year.FTA_Formula_Grants_for_Rural_Areas_5311 AS FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year, + wide_last_year.Other_Directly_Generated_Funds AS Other_Directly_Generated_Funds_Last_Year + FROM wide_this_year + FULL OUTER JOIN wide_last_year + ON wide_this_year.organization = wide_last_year.organization + ORDER BY organization diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql index 98da2364d1..929077bcf7 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_financial_total_exp.sql @@ -3,62 +3,66 @@ --- into one table for downstream validation checks. "Prior year" data not needed --- NTD error ID #s RR20F-001OA, RR20F-001C, RR20F-182 ------ +{% set this_year = run_started_at.year %} -WITH total_operations_exp_2023 AS( +WITH total_operations_exp AS( SELECT organization, api_report_period AS fiscal_year, SUM(operations_expended) AS Total_Annual_Op_Expenses_by_Mode, MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE css_class = "expense" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, api_report_period ), -total_capital_exp_bymode_2023 AS ( + +total_capital_exp_bymode AS ( SELECT organization, api_report_period AS fiscal_year, SUM(capital_expended) AS Total_Annual_Cap_Expenses_byMode, MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE css_class = "expense" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, api_report_period ), -total_operations_rev_2023 AS ( + +total_operations_rev AS ( SELECT organization, api_report_period AS fiscal_year, SUM(operations_expended) AS Total_Annual_Op_Revenues_Expended, MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE css_class = "revenue" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, api_report_period ), -total_cap_exp_byfunds_2023 AS ( + +total_cap_exp_byfunds AS ( SELECT organization, api_report_period AS fiscal_year, SUM(capital_expended) AS Total_Annual_Cap_Expenses_byFunds, MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE css_class = "revenue" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, api_report_period ) SELECT - total_operations_exp_2023.organization, - total_operations_exp_2023.fiscal_year, - total_operations_exp_2023.Total_Annual_Op_Expenses_by_Mode, - total_capital_exp_bymode_2023.Total_Annual_Cap_Expenses_byMode, - total_operations_rev_2023.Total_Annual_Op_Revenues_Expended, - total_cap_exp_byfunds_2023.Total_Annual_Cap_Expenses_byFunds -FROM total_operations_exp_2023 -FULL OUTER JOIN total_capital_exp_bymode_2023 - ON total_operations_exp_2023.organization = total_capital_exp_bymode_2023.organization - AND total_operations_exp_2023.fiscal_year = total_capital_exp_bymode_2023.fiscal_year -FULL OUTER JOIN total_operations_rev_2023 - ON total_operations_exp_2023.organization = total_operations_rev_2023.organization - AND total_operations_exp_2023.fiscal_year = total_operations_rev_2023.fiscal_year -FULL OUTER JOIN total_cap_exp_byfunds_2023 - ON total_operations_exp_2023.organization = total_cap_exp_byfunds_2023.organization - AND total_operations_exp_2023.fiscal_year = total_cap_exp_byfunds_2023.fiscal_year + total_operations_exp.organization, + total_operations_exp.fiscal_year, + total_operations_exp.Total_Annual_Op_Expenses_by_Mode, + total_capital_exp_bymode.Total_Annual_Cap_Expenses_byMode, + total_operations_rev.Total_Annual_Op_Revenues_Expended, + total_cap_exp_byfunds.Total_Annual_Cap_Expenses_byFunds +FROM total_operations_exp +FULL OUTER JOIN total_capital_exp_bymode + ON total_operations_exp.organization = total_capital_exp_bymode.organization + AND total_operations_exp.fiscal_year = total_capital_exp_bymode.fiscal_year +FULL OUTER JOIN total_operations_rev + ON total_operations_exp.organization = total_operations_rev.organization + AND total_operations_exp.fiscal_year = total_operations_rev.fiscal_year +FULL OUTER JOIN total_cap_exp_byfunds + ON total_operations_exp.organization = total_cap_exp_byfunds.organization + AND total_operations_exp.fiscal_year = total_cap_exp_byfunds.fiscal_year diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_1alldata.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_1alldata.sql index 9708e95ed7..954313e28f 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_1alldata.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_1alldata.sql @@ -1,14 +1,10 @@ ------ --- Compiles data for RR-20 Service checks from all years into one table for future computation ------ +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} ---- The 2022 data was *not* from the API and so formatted differently ---- We are *assuming* that data in 2024 and onwards will be the same format as 2023 ---- If you get errors in 2024, check which columns may differ and read errors carefully. - ----TO DO: insert parameter for loop, for each year, do what 2023 is doing, ---- and at the end, add another union statement -WITH service_2023 AS ( +WITH service_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -21,11 +17,11 @@ WITH service_2023 AS ( MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE type = "Service Data" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year, mode, Annual_VRM, Annual_VRH, Annual_UPT, Sponsored_UPT, VOMX ), -expenses_2023 AS ( +expenses_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -34,11 +30,11 @@ expenses_2023 AS ( MAX(api_report_last_modified_date) AS max_api_report_last_modified_date FROM {{ ref('stg_ntd_rr20_rural') }} WHERE type = "Expenses by Mode" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year, mode, Total_Annual_Expenses_By_Mode ), -fare_rev_2023 AS ( +fare_rev_this_year AS ( SELECT organization, api_report_period AS fiscal_year, @@ -46,88 +42,98 @@ fare_rev_2023 AS ( MAX(api_report_last_modified_date) AS max_api_report_last_modified_date from {{ ref('stg_ntd_rr20_rural') }} WHERE type = "Fare Revenues" - AND api_report_period = 2023 + AND api_report_period = {{this_year}} GROUP BY organization, fiscal_year ), -all_2023 as ( +all_this_year as ( SELECT DISTINCT - service_2023.organization, - service_2023.fiscal_year, - service_2023.mode, - expenses_2023.Total_Annual_Expenses_By_Mode, - service_2023.Annual_VRM, - service_2023.Annual_VRH, - service_2023.Annual_UPT, - service_2023.Sponsored_UPT, - service_2023.VOMX, - fare_rev_2023.Fare_Revenues - FROM service_2023 - FULL OUTER JOIN expenses_2023 - ON service_2023.organization = expenses_2023.organization - AND service_2023.fiscal_year = expenses_2023.fiscal_year - AND service_2023.mode = expenses_2023.mode - FULL OUTER JOIN fare_rev_2023 - ON service_2023.organization = fare_rev_2023.organization - AND service_2023.fiscal_year = fare_rev_2023.fiscal_year + service_this_year.organization, + service_this_year.fiscal_year, + service_this_year.mode, + expenses_this_year.Total_Annual_Expenses_By_Mode, + service_this_year.Annual_VRM, + service_this_year.Annual_VRH, + service_this_year.Annual_UPT, + service_this_year.Sponsored_UPT, + service_this_year.VOMX, + fare_rev_this_year.Fare_Revenues + FROM service_this_year + FULL OUTER JOIN expenses_this_year + ON service_this_year.organization = expenses_this_year.organization + AND service_this_year.fiscal_year = expenses_this_year.fiscal_year + AND service_this_year.mode = expenses_this_year.mode + FULL OUTER JOIN fare_rev_this_year + ON service_this_year.organization = fare_rev_this_year.organization + AND service_this_year.fiscal_year = fare_rev_this_year.fiscal_year ), -service2022 AS ( +service_last_year AS ( SELECT - Organization_Legal_Name AS organization, - Fiscal_Year AS fiscal_year, - Mode AS mode, - Annual_VRM, - Annual_VRH, - Annual_UPT, - Sponsored_UPT, - VOMX - FROM {{ ref('stg_ntd_2022_rr20_service') }} + organization, + api_report_period AS fiscal_year, + item AS mode, + annual_vehicle_rev_miles AS Annual_VRM, + annual_vehicle_rev_hours AS Annual_VRH, + annual_unlinked_pass_trips AS Annual_UPT, + sponsored_service_upt AS Sponsored_UPT, + annual_vehicle_max_service AS VOMX, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + FROM {{ ref('stg_ntd_rr20_rural') }} + WHERE type = "Service Data" + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year, mode, Annual_VRM, Annual_VRH, Annual_UPT, Sponsored_UPT, VOMX ), -expenses2022 AS ( +expenses_last_year AS ( SELECT - Organization_Legal_Name AS organization, - Fiscal_Year AS fiscal_year, - Mode AS mode, - Total_Annual_Expenses_By_Mode - FROM {{ ref('stg_ntd_2022_rr20_exp_by_mode') }} - WHERE Operating_Capital = "Operating" + organization, + api_report_period AS fiscal_year, + item AS mode, + operations_expended AS Total_Annual_Expenses_By_Mode, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + FROM {{ ref('stg_ntd_rr20_rural') }} + WHERE type = "Expenses by Mode" + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year, mode, Total_Annual_Expenses_By_Mode ), -fare_rev_2022 AS ( +fare_rev_last_year AS ( SELECT - Organization_Legal_Name AS organization, - Fiscal_Year AS fiscal_year, - Fare_Revenues - FROM {{ ref('stg_ntd_2022_rr20_financial') }} - WHERE Operating_Capital = "Operating" + organization, + api_report_period AS fiscal_year, + SUM(operations_expended) AS Fare_Revenues, + MAX(api_report_last_modified_date) AS max_api_report_last_modified_date + from {{ ref('stg_ntd_rr20_rural') }} + WHERE type = "Fare Revenues" + AND api_report_period = {{last_year}} + GROUP BY organization, fiscal_year ), -all_2022 AS ( +all_last_year as ( SELECT DISTINCT - service2022.organization, - service2022.fiscal_year, - service2022.mode, - expenses2022.Total_Annual_Expenses_By_Mode, - service2022.Annual_VRM, - service2022.Annual_VRH, - service2022.Annual_UPT, - service2022.Sponsored_UPT, - service2022.VOMX, - fare_rev_2022.Fare_Revenues -FROM service2022 -FULL OUTER JOIN expenses2022 - ON service2022.organization = expenses2022.organization - AND service2022.fiscal_year = expenses2022.fiscal_year - AND service2022.mode = expenses2022.mode -INNER JOIN fare_rev_2022 - ON service2022.organization = fare_rev_2022.organization - AND service2022.fiscal_year = fare_rev_2022.fiscal_year + service_last_year.organization, + service_last_year.fiscal_year, + service_last_year.mode, + expenses_last_year.Total_Annual_Expenses_By_Mode, + service_last_year.Annual_VRM, + service_last_year.Annual_VRH, + service_last_year.Annual_UPT, + service_last_year.Sponsored_UPT, + service_last_year.VOMX, + fare_rev_last_year.Fare_Revenues + FROM service_last_year + FULL OUTER JOIN expenses_last_year + ON service_last_year.organization = expenses_last_year.organization + AND service_last_year.fiscal_year = expenses_last_year.fiscal_year + AND service_last_year.mode = expenses_last_year.mode + FULL OUTER JOIN fare_rev_last_year + ON service_last_year.organization = fare_rev_last_year.organization + AND service_last_year.fiscal_year = fare_rev_last_year.fiscal_year ) -SELECT * FROM all_2022 +SELECT * FROM all_last_year UNION ALL -SELECT * FROM all_2023 +SELECT * FROM all_this_year diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_3ratios_wide.sql b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_3ratios_wide.sql index a37d390129..a782b678e9 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_3ratios_wide.sql +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_rr20_service_3ratios_wide.sql @@ -1,7 +1,9 @@ +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} + with longform as ( select organization, - -- CAST(fiscal_year as STRING) as fiscal_year, fiscal_year, mode, VOMX, @@ -19,90 +21,86 @@ with longform as ( cph as ( select * from (select organization, fiscal_year, mode,cost_per_hr from longform) - -- PIVOT(AVG(cost_per_hr) FOR fiscal_year IN (', check_period, ')) - PIVOT(AVG(cost_per_hr) FOR fiscal_year IN (2022,2023,2024)) as cost_per_hr + PIVOT(AVG(cost_per_hr) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as cost_per_hr ORDER BY organization ), mpv as ( select * from (select organization, fiscal_year, mode, miles_per_veh from longform) - PIVOT(AVG(miles_per_veh) FOR fiscal_year IN (2022,2023,2024)) as miles_per_veh + PIVOT(AVG(miles_per_veh) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as miles_per_veh ORDER BY organization ), + frpt as ( select * from (select organization, fiscal_year, mode, fare_rev_per_trip from longform) - PIVOT(AVG(fare_rev_per_trip) FOR fiscal_year IN (2022,2023,2024)) as fare_rev_per_trip + PIVOT(AVG(fare_rev_per_trip) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as fare_rev_per_trip ORDER BY organization ), + rev_speed as ( select * from (select organization, fiscal_year, mode, rev_speed from longform) - PIVOT(AVG(rev_speed) FOR fiscal_year IN (2022,2023,2024)) as rev_speed + PIVOT(AVG(rev_speed) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as rev_speed ORDER BY organization ), + tph as ( select * from (select organization, fiscal_year, mode, trips_per_hr from longform) - PIVOT(AVG(trips_per_hr) FOR fiscal_year IN (2022,2023,2024)) as trips_per_hr + PIVOT(AVG(trips_per_hr) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as trips_per_hr ORDER BY organization ), + voms as ( select * from (select organization, fiscal_year, mode, VOMX from longform) - PIVOT(AVG(VOMX) FOR fiscal_year IN (2022,2023,2024)) as VOMX + PIVOT(AVG(VOMX) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as VOMX ORDER BY organization ), + vrm as ( select * from (select organization, fiscal_year, mode, Annual_VRM from longform) - PIVOT(AVG(Annual_VRM) FOR fiscal_year IN (2022,2023,2024)) as Annual_VRM + PIVOT(AVG(Annual_VRM) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as Annual_VRM ORDER BY organization ), + vrh as ( select * from (select organization, fiscal_year, mode, Annual_VRH from longform) - PIVOT(AVG(Annual_VRH) FOR fiscal_year IN (2022,2023,2024)) as Annual_VRH + PIVOT(AVG(Annual_VRH) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as Annual_VRH ORDER BY organization ), + upt as ( select * from (select organization, fiscal_year, mode, Annual_UPT from longform) - PIVOT(AVG(Annual_UPT) FOR fiscal_year IN (2022,2023,2024)) as Annual_UPT + PIVOT(AVG(Annual_UPT) FOR fiscal_year IN ({{last_year}} AS _last_year, {{this_year}} AS _this_year)) as Annual_UPT ORDER BY organization ) --- select * from mpv select distinct cph.organization, cph.mode, - cph._2022 as cph_2022, - cph._2023 as cph_2023, - cph._2024 as cph_2024, - mpv._2022 as mpv_2022, - mpv._2023 as mpv_2023, - mpv._2024 as mpv_2024, - frpt._2022 as frpt_2022, - frpt._2023 as frpt_2023, - frpt._2024 as frpt_2024, - rev_speed._2022 as rev_speed_2022, - rev_speed._2023 as rev_speed_2023, - rev_speed._2024 as rev_speed_2024, - tph._2022 as tph_2022, - tph._2023 as tph_2023, - tph._2024 as tph_2024, - voms._2022 as voms_2022, - voms._2023 as voms_2023, - voms._2024 as voms_2024, - vrm._2022 as vrm_2022, - vrm._2023 as vrm_2023, - vrm._2024 as vrm_2024, - vrh._2022 as vrh_2022, - vrh._2023 as vrh_2023, - vrh._2024 as vrh_2024, - upt._2022 as upt_2022, - upt._2023 as upt_2023, - upt._2024 as upt_2024 + cph._last_year as cph_last_year, + cph._this_year as cph_this_year, + mpv._last_year as mpv_last_year, + mpv._this_year as mpv_this_year, + frpt._last_year as frpt_last_year, + frpt._this_year as frpt_this_year, + rev_speed._last_year as rev_speed_last_year, + rev_speed._this_year as rev_speed_this_year, + tph._last_year as tph_last_year, + tph._this_year as tph_this_year, + voms._last_year as voms_last_year, + voms._this_year as voms_this_year, + vrm._last_year as vrm_last_year, + vrm._this_year as vrm_this_year, + vrh._last_year as vrh_last_year, + vrh._this_year as vrh_this_year, + upt._last_year as upt_last_year, + upt._this_year as upt_this_year from cph FULL OUTER JOIN mpv on cph.organization = mpv.organization @@ -129,6 +127,3 @@ FULL OUTER JOIN upt on cph.organization = upt.organization AND cph.mode = upt.mode ORDER BY organization --- ); - --- EXECUTE IMMEDIATE query; diff --git a/warehouse/models/intermediate/ntd_validation/int_ntd_validation.yml b/warehouse/models/intermediate/ntd_validation/int_ntd_validation.yml index 6092f26b22..1d43fef494 100644 --- a/warehouse/models/intermediate/ntd_validation/int_ntd_validation.yml +++ b/warehouse/models/intermediate/ntd_validation/int_ntd_validation.yml @@ -1,9 +1,17 @@ version: 2 models: + - name: int_ntd_a10_facilitiesdata + description: | + For assessing A10-032: Check that sum of total facilities for each agency, across all modes, is a whole number. + NOTE: This model uses a dynamic variable for "this_year" which is based on the timestamp that this run started. + - name: int_ntd_a30_voms_vins_totals + description: | + Gets the number of VOMS in the RR-20 and the number of VINS in the A30 - name: int_ntd_rr20_financial_fare_revenues description: | Setting up the RR-20 data for comparing fare revenues to previous year + NOTE: This model uses dynamic variables for "this_year" and "last_year" which are based on the timestamp that this run started. # tests: # - dbt_utils.expression_is_true: # expression: 'status != {{ guidelines_to_be_assessed_status() }}' @@ -12,17 +20,18 @@ models: description: | Setting up the RR-20 data for comparing specific funding sources - the 5311 funds, and Other directly generated funds For NTD validation error ID #s RR20F-070, RR20F-065, RR20F-068, RR20F-066, RR20F-013 + NOTE: This model uses dynamic variables for "this_year" and "last_year" which are based on the timestamp that this run started. - name: int_ntd_rr20_financial_total_exp description: | Setting up the RR-20 data for comparing totals, for operating and capital expenses, reported in different areas of the RR-20 For NTD validation error ID #s RR20F-001OA, RR20F-001C, RR20F-182 + NOTE: This model uses a dynamic variable for "this_year" which is based on the timestamp that this run started. - name: int_ntd_rr20_service_1alldata description: | - 1st intermediate cleaning step for service data. Combines 2023 and 2022 data in preparation for doing NTD validation checks. - The 2022 data was *not* from the API and so formatted differently - We are *assuming* that data in 2024 and onwards will be the same format as 2023 - If you get errors in 2024, check which columns may differ and read errors carefully. - NOTE!!!! You must add in the 2024 data when it is available with another CTE. IF NOT THERE WILL BE NO 2024 DATA CHECKED. + 1st intermediate cleaning step for service data. Combines the current year and previous year data in preparation for doing NTD validation checks. + NOTE: This model uses dynamic variables for "this_year" and "last_year" which are based on the timestamp that this run started. + config: + materialized: table - name: int_ntd_rr20_service_2ratioslong description: | 2nd intermediate cleaning step for service data. Calculates all needed NTD metrics that are ratios of two other values. E.g., cost per hour, @@ -32,7 +41,6 @@ models: - name: int_ntd_rr20_service_3ratios_wide description: | 3rd intermediate cleaning step for service data. Pivots data from the 2nd cleaning step (above) from longform to wide form. - NOTE!!!! A column for 2024 has already been added, but in 2025 and beyond you must go in and add a column for the year of interest - e.g., 2025, (and up to you to delete any columns you don't want, e.g. 2022) + NOTE: This model uses dynamic variables for "this_year" and "last_year" which are based on the timestamp that this run started. config: materialized: table diff --git a/warehouse/models/mart/ntd_validation/_mart_ntd_validation.yml b/warehouse/models/mart/ntd_validation/_mart_ntd_validation.yml index d33b8a5946..f176098d7f 100644 --- a/warehouse/models/mart/ntd_validation/_mart_ntd_validation.yml +++ b/warehouse/models/mart/ntd_validation/_mart_ntd_validation.yml @@ -4,8 +4,8 @@ models: - name: fct_ntd_rr20_service_checks description: | Runs validation checks on the RR-20 service data. Source data is int_ntd_rr20_service_3ratios_wide. - NOTE: This model uses "this_year" and "last_year" as dynamic variables based on the date on which it is run. For NTD validation error ID #s rr20f_005, rr20f_146, rr20f_139, rr20f_008, rr20f_137, rr20f_154, rr20f_179, rr20f_171, rr20f_143 + NOTE: This model uses a dynamic variable for "this_year" which is based on the timestamp that this run started. - name: fct_ntd_a30_vomscheck description: | Runs various checks on VOMS data submitted to NTD, that are also in the file voms_inventory_check.py. @@ -19,6 +19,7 @@ models: description: | Runs various validation checks on specific RR-20 funding source data. For NTD validation error ID #s rr20f_070, rr20f_066, rr20f_065, rr20f_013, rr20f_068, rr20f_024 + NOTE: This model uses a dynamic variable for "this_year" which is based on the timestamp that this run started. - name: fct_ntd_a10_facilitiescheck description: | Runs various validation checks on specific A-10 facilities source data. diff --git a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_funds_checks.sql b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_funds_checks.sql index efa6294282..63a715b19b 100644 --- a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_funds_checks.sql +++ b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_funds_checks.sql @@ -1,125 +1,127 @@ --- We do identical CASE WHEN clauses in each CTE. The results determine 2 different column values but one can only specify 1 col/statement +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} -WITH rr20f_070 as ( - select +WITH rr20f_070 AS ( + SELECT organization, - "RR20F-070: 5311 Funds not reported" as name_of_check, - CASE WHEN ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023) = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NULL + "RR20F-070: 5311 Funds not reported" AS name_of_check, + CASE WHEN ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year) = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NULL THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2023 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS STRING)) as value_checked, - CASE WHEN ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023) = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NULL + END AS check_status, + CONCAT("{{this_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS STRING)) AS value_checked, + CASE WHEN ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year) = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NULL THEN "The §5311 program is not listed as a revenue source in your report, Please double check and provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ), -rr20f_066 as ( - select +rr20f_066 AS ( + SELECT organization, - "RR20F-066: change from zero" as name_of_check, - CASE WHEN ((FTA_Formula_Grants_for_Rural_Areas_5311_2023 = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NOT NULL)) + "RR20F-066: change from zero" AS name_of_check, + CASE WHEN ((FTA_Formula_Grants_for_Rural_Areas_5311_This_Year = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NOT NULL)) OR - ((FTA_Formula_Grants_for_Rural_Areas_5311_2023 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NULL)) + ((FTA_Formula_Grants_for_Rural_Areas_5311_This_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NULL)) THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2022 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2022,0) AS STRING), - "2023 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS STRING)) as value_checked, - CASE WHEN ((FTA_Formula_Grants_for_Rural_Areas_5311_2023 = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NOT NULL)) + END AS check_status, + CONCAT("{{last_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year,0) AS STRING), + ", {{this_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS STRING)) AS value_checked, + CASE WHEN ((FTA_Formula_Grants_for_Rural_Areas_5311_This_Year = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NOT NULL)) OR - ((FTA_Formula_Grants_for_Rural_Areas_5311_2023 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NULL)) + ((FTA_Formula_Grants_for_Rural_Areas_5311_This_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year = 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NULL)) THEN "FTA_Formula_Grants_for_Rural_Areas_5311 funding changed either from or to zero compared to last year. Please provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ), -rr20f_065 as ( - select +rr20f_065 AS ( + SELECT organization, - "RR20F-065: 5311 Funds same value" as name_of_check, - CASE WHEN (FTA_Formula_Grants_for_Rural_Areas_5311_2023 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2023 = FTA_Formula_Grants_for_Rural_Areas_5311_2022) + "RR20F-065: 5311 Funds same value" AS name_of_check, + CASE WHEN (FTA_Formula_Grants_for_Rural_Areas_5311_This_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_This_Year = FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year) THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2022 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2022,0) AS STRING), - "2023 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS STRING)) as value_checked, - CASE WHEN (FTA_Formula_Grants_for_Rural_Areas_5311_2023 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2022 != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_2022 IS NOT NULL) - AND (FTA_Formula_Grants_for_Rural_Areas_5311_2023 = FTA_Formula_Grants_for_Rural_Areas_5311_2022) - THEN "You have identical values for FTA_Formula_Grants_for_Rural_Areas_5311 funding in 2022 and 2023, which is unusual. Please provide a narrative justification." + END AS check_status, + CONCAT("{{last_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year,0) AS STRING), + ", {{this_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS STRING)) AS value_checked, + CASE WHEN (FTA_Formula_Grants_for_Rural_Areas_5311_This_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year != 0 OR FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year IS NOT NULL) + AND (FTA_Formula_Grants_for_Rural_Areas_5311_This_Year = FTA_Formula_Grants_for_Rural_Areas_5311_Last_Year) + THEN "You have identical values for FTA_Formula_Grants_for_Rural_Areas_5311 funding in {{last_year}} and {{this_year}}, which is unusual. Please provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ), -rr20f_013 as ( - select +rr20f_013 AS ( + SELECT organization, - "RR20F-013: Other Directly Generated Funds same value" as name_of_check, - CASE WHEN (Other_Directly_Generated_Funds_2023 != 0 OR Other_Directly_Generated_Funds_2023 IS NOT NULL) - AND (Other_Directly_Generated_Funds_2022 != 0 OR Other_Directly_Generated_Funds_2022 IS NOT NULL) - AND (Other_Directly_Generated_Funds_2023 = Other_Directly_Generated_Funds_2022) + "RR20F-013: Other Directly Generated Funds same value" AS name_of_check, + CASE WHEN (Other_Directly_Generated_Funds_This_Year != 0 OR Other_Directly_Generated_Funds_This_Year IS NOT NULL) + AND (Other_Directly_Generated_Funds_Last_Year != 0 OR Other_Directly_Generated_Funds_Last_Year IS NOT NULL) + AND (Other_Directly_Generated_Funds_This_Year = Other_Directly_Generated_Funds_Last_Year) THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2022 = ", CAST(ROUND(Other_Directly_Generated_Funds_2022,0) AS STRING), - "2023 = ", CAST(ROUND(Other_Directly_Generated_Funds_2023,0) AS STRING)) as value_checked, - CASE WHEN (Other_Directly_Generated_Funds_2023 != 0 OR Other_Directly_Generated_Funds_2023 IS NOT NULL) - AND (Other_Directly_Generated_Funds_2022 != 0 OR Other_Directly_Generated_Funds_2022 IS NOT NULL) - AND (Other_Directly_Generated_Funds_2023 = Other_Directly_Generated_Funds_2022) - THEN "You have identical values for Other_Directly_Generated_Funds funding in 2022 and 2023, which is unusual. Please provide a narrative justification." + END AS check_status, + CONCAT("last_year = ", CAST(ROUND(Other_Directly_Generated_Funds_Last_Year,0) AS STRING), + ", {{this_year}} = ", CAST(ROUND(Other_Directly_Generated_Funds_This_Year,0) AS STRING)) AS value_checked, + CASE WHEN (Other_Directly_Generated_Funds_This_Year != 0 OR Other_Directly_Generated_Funds_This_Year IS NOT NULL) + AND (Other_Directly_Generated_Funds_Last_Year != 0 OR Other_Directly_Generated_Funds_Last_Year IS NOT NULL) + AND (Other_Directly_Generated_Funds_This_Year = Other_Directly_Generated_Funds_Last_Year) + THEN "You have identical values for Other_Directly_Generated_Funds funding in {{last_year}} and {{this_year}}, which is unusual. Please provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ), -rr20f_068 as ( - select +rr20f_068 AS ( + SELECT organization, - "RR20F-068: 5311 Funds rounded to thousand" as name_of_check, - CASE WHEN MOD(CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS INT),1000) = 0 AND FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL + "RR20F-068: 5311 Funds rounded to thousand" AS name_of_check, + CASE WHEN MOD(CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS INT),1000) = 0 AND FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2023 = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS STRING)) as value_checked, - CASE WHEN MOD(CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_2023,0) AS INT),1000) = 0 AND FTA_Formula_Grants_for_Rural_Areas_5311_2023 IS NOT NULL + END AS check_status, + CONCAT("{{this_year}} = ", CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS STRING)) AS value_checked, + CASE WHEN MOD(CAST(ROUND(FTA_Formula_Grants_for_Rural_Areas_5311_This_Year,0) AS INT),1000) = 0 AND FTA_Formula_Grants_for_Rural_Areas_5311_This_Year IS NOT NULL THEN "FTA_Formula_Grants_for_Rural_Areas_5311 are rounded to the nearest thousand, but should be reported as exact values. Please double check and provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ), -rr20f_024 as ( - select +rr20f_024 AS ( + SELECT organization, - "RR20F-024: Local Funds rounded to thousand" as name_of_check, - CASE WHEN MOD(CAST(ROUND(Local_Funds_2023) AS INT),1000) = 0 AND Local_Funds_2023 IS NOT NULL + "RR20F-024: Local Funds rounded to thousand" AS name_of_check, + CASE WHEN MOD(CAST(ROUND(Local_Funds_This_Year) AS INT),1000) = 0 AND Local_Funds_This_Year IS NOT NULL THEN "Fail" ELSE "Pass" - END as check_status, - CONCAT("2023 = ", CAST(ROUND(Local_Funds_2023) AS STRING)) as value_checked, - CASE WHEN MOD(CAST(ROUND(Local_Funds_2023) AS INT),1000) = 0 AND Local_Funds_2023 IS NOT NULL + END AS check_status, + CONCAT("{{this_year}} = ", CAST(ROUND(Local_Funds_This_Year) AS STRING)) AS value_checked, + CASE WHEN MOD(CAST(ROUND(Local_Funds_This_Year) AS INT),1000) = 0 AND Local_Funds_This_Year IS NOT NULL THEN "Local Funds are rounded to the nearest thousand, but should be reported as exact values. Please double check and provide a narrative justification." ELSE "" END AS description, - "" as Agency_Response, + "" AS Agency_Response, CURRENT_TIMESTAMP() AS date_checked - from {{ ref('int_ntd_rr20_financial_specific_funds') }} + FROM {{ ref('int_ntd_rr20_financial_specific_funds') }} ) SELECT * FROM rr20f_070 diff --git a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql index 6569f2d4d5..c1477fa4ef 100644 --- a/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql +++ b/warehouse/models/mart/ntd_validation/fct_ntd_rr20_service_checks.sql @@ -6,39 +6,34 @@ {% set frpt_threshold = 0.25 %} {% set rs_threshold = 0.15 %} {% set tph_threshold = 0.3 %} -{% set start_datetime = run_started_at %} -{% set start_date = start_datetime.strftime("%Y-%m-%d") %} -{% set this_year = start_datetime.strftime("%Y") %} -{% set last_year1 = start_datetime - modules.datetime.timedelta(days=365) %} -{% set last_year = last_year1.strftime("%Y") %} +{% set start_date = run_started_at.strftime("%Y-%m-%d") %} +{% set this_year = run_started_at.year %} +{% set last_year = this_year - 1 %} - - -WITH -rr20f_005 as ( +WITH rr20f_005 as ( select organization, "RR20F-005: Cost per Hour change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (cph_{{this_year}} IS NULL OR cph_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (cph_this_year IS NULL OR cph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (cph_{{last_year}} IS NULL OR cph_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (cph_last_year IS NULL OR cph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(cph_{{this_year}}) - ROUND(cph_{{last_year}}) / ROUND(cph_{{last_year}})) >= {{cph_threshold}} THEN "Fail" + WHEN ABS(ROUND(cph_this_year) - ROUND(cph_last_year) / ROUND(cph_last_year)) >= {{cph_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(cph_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(cph_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(cph_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(cph_last_year,1) AS STRING), "chg = ", - CAST(ROUND((ROUND(cph_{{this_year}},1) - ROUND(cph_{{last_year}},1))/ABS(ROUND(cph_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(cph_this_year,1) - ROUND(cph_last_year,1))/ABS(ROUND(cph_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (cph_{{this_year}} IS NULL OR cph_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (cph_this_year IS NULL OR cph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for cost per hr.") - WHEN (cph_{{last_year}} IS NULL OR cph_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (cph_last_year IS NULL OR cph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for cost per hr.") - WHEN ABS(ROUND(cph_{{this_year}}) - ROUND(cph_{{last_year}}) / ROUND(cph_{{last_year}})) >= {{cph_threshold}} + WHEN ABS(ROUND(cph_this_year) - ROUND(cph_last_year) / ROUND(cph_last_year)) >= {{cph_threshold}} THEN CONCAT("The cost per hour for this mode has changed from last year by > = ", {{cph_threshold}} * 100, "%, please provide a narrative justification.") @@ -48,30 +43,31 @@ rr20f_005 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_146 as ( select organization, "RR20F-146: Miles per Vehicle change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (mpv_{{this_year}} IS NULL OR mpv_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (mpv_this_year IS NULL OR mpv_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (mpv_{{last_year}} IS NULL OR mpv_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (mpv_last_year IS NULL OR mpv_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(mpv_{{this_year}}) - ROUND(mpv_{{last_year}}) / ROUND(mpv_{{last_year}})) >= {{mpv_threshold}} THEN "Fail" + WHEN ABS(ROUND(mpv_this_year) - ROUND(mpv_last_year) / ROUND(mpv_last_year)) >= {{mpv_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(mpv_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(mpv_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(mpv_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(mpv_last_year,1) AS STRING), "chg = ", - CAST(ROUND((ROUND(mpv_{{this_year}},1) - ROUND(mpv_{{last_year}},1))/ABS(ROUND(mpv_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(mpv_this_year,1) - ROUND(mpv_last_year,1))/ABS(ROUND(mpv_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (mpv_{{this_year}} IS NULL OR mpv_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (mpv_this_year IS NULL OR mpv_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for miles per vehicle.") - WHEN (mpv_{{last_year}} IS NULL OR mpv_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (mpv_last_year IS NULL OR mpv_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for miles per vehicle.") - WHEN ABS(ROUND(mpv_{{this_year}}) - ROUND(mpv_{{last_year}}) / ROUND(mpv_{{last_year}})) >= {{mpv_threshold}} + WHEN ABS(ROUND(mpv_this_year) - ROUND(mpv_last_year) / ROUND(mpv_last_year)) >= {{mpv_threshold}} THEN CONCAT("The miles per vehicle for this mode has changed from last year by >= ", {{mpv_threshold}} * 100, "%, please provide a narrative justification.") @@ -81,42 +77,43 @@ rr20f_146 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_179 as ( select organization, "RR20F-179: Missing Service Data check" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN ((vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0) OR (vrh_{{this_year}} IS NULL OR vrh_{{this_year}} = 0) OR - (upt_{{this_year}} IS NULL OR upt_{{this_year}} = 0) OR (voms_{{this_year}} IS NULL OR voms_{{this_year}} = 0)) + CASE WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR + (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Fail" - WHEN ((vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) OR (vrh_{{last_year}} IS NULL OR vrh_{{last_year}} = 0) OR - (upt_{{last_year}} IS NULL OR upt_{{last_year}} = 0) OR (voms_{{last_year}} IS NULL OR voms_{{last_year}} = 0)) + WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR + (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Fail" - WHEN ((vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0) OR (vrh_{{this_year}} IS NULL OR vrh_{{this_year}} = 0) OR - (upt_{{this_year}} IS NULL OR upt_{{this_year}} = 0) OR (voms_{{this_year}} IS NULL OR voms_{{this_year}} = 0)) + WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR + (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Didn't run" - WHEN ((vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) OR (vrh_{{last_year}} IS NULL OR vrh_{{last_year}} = 0) OR - (upt_{{last_year}} IS NULL OR upt_{{last_year}} = 0) OR (voms_{{last_year}} IS NULL OR voms_{{last_year}} = 0)) + WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR + (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Didn't run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " Service data: VRM=", vrm_{{this_year}}, " VRH=", vrh_{{this_year}}, " UPT=", upt_{{this_year}}, - " VOMS=", voms_{{this_year}}) as value_checked, - CASE WHEN ((vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0) OR (vrh_{{this_year}} IS NULL OR vrh_{{this_year}} = 0) OR - (upt_{{this_year}} IS NULL OR upt_{{this_year}} = 0) OR (voms_{{this_year}} IS NULL OR voms_{{this_year}} = 0)) + CONCAT({{this_year}}, " Service data: VRM=", vrm_this_year, " VRH=", vrh_this_year, " UPT=", upt_this_year, + " VOMS=", voms_this_year) as value_checked, + CASE WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR + (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "One or more service data fields are are missing for Vehicle Revenue Miles, Vehicle Revenue Hours, Unlinked Passenger Trips, or VOMS. Please revise your submission to report it if the transit unit operated revenue service during the fiscal year." - WHEN ((vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) OR (vrh_{{last_year}} IS NULL OR vrh_{{last_year}} = 0) OR - (upt_{{last_year}} IS NULL OR upt_{{last_year}} = 0) OR (voms_{{last_year}} IS NULL OR voms_{{last_year}} = 0)) + WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR + (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "One or more service data fields are are missing for Vehicle Revenue Miles, Vehicle Revenue Hours, Unlinked Passenger Trips, or VOMS. Please revise your submission to report it if the transit unit operated revenue service during the fiscal year." - WHEN ((vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0) OR (vrh_{{this_year}} IS NULL OR vrh_{{this_year}} = 0) OR - (upt_{{this_year}} IS NULL OR upt_{{this_year}} = 0) OR (voms_{{this_year}} IS NULL OR voms_{{this_year}} = 0)) + WHEN ((vrm_this_year IS NULL OR vrm_this_year = 0) OR (vrh_this_year IS NULL OR vrh_this_year = 0) OR + (upt_this_year IS NULL OR upt_this_year = 0) OR (voms_this_year IS NULL OR voms_this_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ". Did not yet check for missing service data.") - WHEN ((vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) OR (vrh_{{last_year}} IS NULL OR vrh_{{last_year}} = 0) OR - (upt_{{last_year}} IS NULL OR upt_{{last_year}} = 0) OR (voms_{{last_year}} IS NULL OR voms_{{last_year}} = 0)) + WHEN ((vrm_last_year IS NULL OR vrm_last_year = 0) OR (vrh_last_year IS NULL OR vrh_last_year = 0) OR + (upt_last_year IS NULL OR upt_last_year = 0) OR (voms_last_year IS NULL OR voms_last_year = 0)) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ". Did not yet check for missing service data.") ELSE "Pass" @@ -125,28 +122,29 @@ rr20f_179 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_139 as ( select organization, "RR20F-139: Vehicle Revenue Miles (VRM) % change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(vrm_{{this_year}}) - ROUND(vrm_{{last_year}}) / ROUND(vrm_{{last_year}})) >= {{vrm_threshold}} THEN "Fail" + CASE WHEN (vrm_this_year IS NULL OR vrm_this_year = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" + WHEN (vrm_last_year IS NULL OR vrm_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" + WHEN ABS(ROUND(vrm_this_year) - ROUND(vrm_last_year) / ROUND(vrm_last_year)) >= {{vrm_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(vrm_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(vrm_last_year,1) AS STRING), "chg = ", - CAST(ROUND((ROUND(vrm_{{this_year}},1) - ROUND(vrm_{{last_year}},1))/ABS(ROUND(vrm_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(vrm_this_year,1) - ROUND(vrm_last_year,1))/ABS(ROUND(vrm_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (vrm_{{this_year}} IS NULL OR vrm_{{this_year}} = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (vrm_this_year IS NULL OR vrm_this_year = 0 ) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (vrm_{{last_year}} IS NULL OR vrm_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (vrm_last_year IS NULL OR vrm_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(vrm_{{this_year}}) - ROUND(vrm_{{last_year}}) / ROUND(vrm_{{last_year}})) >= {{vrm_threshold}} + WHEN ABS(ROUND(vrm_this_year) - ROUND(vrm_last_year) / ROUND(vrm_last_year)) >= {{vrm_threshold}} THEN CONCAT("The annual vehicle revenue miles for this mode has changed from last year by >= ", {{vrm_threshold}} * 100, "%, please provide a narrative justification.") @@ -156,28 +154,29 @@ rr20f_139 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_008 as ( select organization, "RR20F-139: Fare Revenue Per Trip change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (frpt_{{this_year}} IS NULL OR frpt_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (frpt_{{last_year}} IS NULL OR frpt_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(frpt_{{this_year}}) - ROUND(frpt_{{last_year}}) / ROUND(frpt_{{last_year}})) >= {{frpt_threshold}} THEN "Fail" + CASE WHEN (frpt_this_year IS NULL OR frpt_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" + WHEN (frpt_last_year IS NULL OR frpt_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" + WHEN ABS(ROUND(frpt_this_year) - ROUND(frpt_last_year) / ROUND(frpt_last_year)) >= {{frpt_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(frpt_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(frpt_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(frpt_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(frpt_last_year,1) AS STRING), "chg = ", - CAST(ROUND((ROUND(frpt_{{this_year}},1) - ROUND(frpt_{{last_year}},1))/ABS(ROUND(frpt_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(frpt_this_year,1) - ROUND(frpt_last_year,1))/ABS(ROUND(frpt_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (frpt_{{this_year}} IS NULL OR frpt_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (frpt_this_year IS NULL OR frpt_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (frpt_{{last_year}} IS NULL OR frpt_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (frpt_last_year IS NULL OR frpt_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(frpt_{{this_year}}) - ROUND(frpt_{{last_year}}) / ROUND(frpt_{{last_year}})) >= {{frpt_threshold}} + WHEN ABS(ROUND(frpt_this_year) - ROUND(frpt_last_year) / ROUND(frpt_last_year)) >= {{frpt_threshold}} THEN CONCAT("The fare revenues per unlinked passenger trip for this mode has changed from last year by >= ", {{frpt_threshold}} * 100, "%, please provide a narrative justification.") @@ -187,30 +186,31 @@ rr20f_008 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_137 as ( select organization, "RR20F-139: Revenue Speed change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (rev_speed_{{this_year}} IS NULL OR rev_speed_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (rev_speed_this_year IS NULL OR rev_speed_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (rev_speed_{{last_year}} IS NULL OR rev_speed_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (rev_speed_last_year IS NULL OR rev_speed_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(rev_speed_{{this_year}}) - ROUND(rev_speed_{{last_year}}) / ROUND(rev_speed_{{last_year}})) >= {{rs_threshold}} THEN "Fail" + WHEN ABS(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)) >= {{rs_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(rev_speed_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(rev_speed_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(rev_speed_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(rev_speed_last_year,1) AS STRING), "chg = ", - CAST(ROUND(ROUND(rev_speed_{{this_year}}) - ROUND(rev_speed_{{last_year}}) / ROUND(rev_speed_{{last_year}})*100, 1) AS STRING), + CAST(ROUND(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)*100, 1) AS STRING), "%" ) as value_checked, - CASE WHEN (rev_speed_{{this_year}} IS NULL OR rev_speed_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (rev_speed_this_year IS NULL OR rev_speed_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (rev_speed_{{last_year}} IS NULL OR rev_speed_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (rev_speed_last_year IS NULL OR rev_speed_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(rev_speed_{{this_year}}) - ROUND(rev_speed_{{last_year}}) / ROUND(rev_speed_{{last_year}})) >= {{rs_threshold}} + WHEN ABS(ROUND(rev_speed_this_year) - ROUND(rev_speed_last_year) / ROUND(rev_speed_last_year)) >= {{rs_threshold}} THEN CONCAT("The revenue speed, the avg speed of your vehicles while in revenue service, for this mode has changed from last year by >= ", {{rs_threshold}} * 100, "%, please provide a narrative justification.") @@ -220,31 +220,32 @@ rr20f_137 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_154 as ( select organization, "RR20F-154: Trips per Hour change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (tph_{{this_year}} IS NULL OR tph_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (tph_this_year IS NULL OR tph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (tph_{{last_year}} IS NULL OR tph_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (tph_last_year IS NULL OR tph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN ABS(ROUND(tph_{{this_year}}) - ROUND(tph_{{last_year}}) / ROUND(tph_{{last_year}})) >= {{tph_threshold}} + WHEN ABS(ROUND(tph_this_year) - ROUND(tph_last_year) / ROUND(tph_last_year)) >= {{tph_threshold}} THEN "Fail" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(tph_{{this_year}},1) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(tph_{{last_year}},1) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(tph_this_year,1) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(tph_last_year,1) AS STRING), "chg = ", - CAST(ROUND((ROUND(tph_{{this_year}},1) - ROUND(tph_{{last_year}},1))/ABS(ROUND(tph_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(tph_this_year,1) - ROUND(tph_last_year,1))/ABS(ROUND(tph_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (tph_{{this_year}} IS NULL OR tph_{{this_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + CASE WHEN (tph_this_year IS NULL OR tph_this_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, ".") - WHEN (tph_{{last_year}} IS NULL OR tph_{{last_year}} = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (tph_last_year IS NULL OR tph_last_year = 0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, ".") - WHEN ABS(ROUND(tph_{{this_year}}) - ROUND(tph_{{last_year}}) / ROUND(tph_{{last_year}})) >= {{tph_threshold}} + WHEN ABS(ROUND(tph_this_year) - ROUND(tph_last_year) / ROUND(tph_last_year)) >= {{tph_threshold}} THEN CONCAT("The calculated trips per hour for this mode has changed from last year by >= ", {{tph_threshold}} * 100, "%, please provide a narrative justification.") @@ -254,33 +255,34 @@ rr20f_154 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_171 as ( select organization, "RR20F-171: Vehicles of Maximum Service (VOMS) change" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (ROUND(voms_{{this_year}}) =0 AND ROUND(voms_{{last_year}}) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) - OR (ROUND(voms_{{this_year}}) !=0 AND ROUND(voms_{{this_year}}) IS NOT NULL AND ROUND(voms_{{last_year}}) = 0) + CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + OR (ROUND(voms_this_year) !=0 AND ROUND(voms_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "Fail" - WHEN (voms_{{this_year}} IS NULL OR voms_{{this_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN (voms_this_year IS NULL OR voms_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (voms_{{last_year}} IS NULL OR voms_{{last_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (voms_last_year IS NULL OR voms_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(voms_{{this_year}}) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(voms_{{last_year}}) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(voms_this_year) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(voms_last_year) AS STRING), "chg = ", - CAST(ROUND((ROUND(voms_{{this_year}},1) - ROUND(voms_{{last_year}},1))/ABS(ROUND(voms_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(voms_this_year,1) - ROUND(voms_last_year,1))/ABS(ROUND(voms_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (ROUND(voms_{{this_year}}) =0 AND ROUND(voms_{{last_year}}) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) - OR (ROUND(voms_{{this_year}}) !=0 AND ROUND(voms_{{this_year}}) IS NOT NULL AND ROUND(voms_{{last_year}}) = 0) + CASE WHEN (ROUND(voms_this_year) =0 AND ROUND(voms_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + OR (ROUND(voms_this_year) !=0 AND ROUND(voms_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "The Vehicles of Maximum Service (VOMS) for this mode has changed either to or from 0 compared to last year, please provide a narrative justification." - WHEN (voms_{{this_year}} IS NULL OR voms_{{this_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN (voms_this_year IS NULL OR voms_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, " for VOMS.") - WHEN (voms_{{last_year}} IS NULL OR voms_{{last_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (voms_last_year IS NULL OR voms_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, " for VOMS.") ELSE "" END AS description, @@ -288,33 +290,34 @@ rr20f_171 as ( CURRENT_TIMESTAMP() AS date_checked from {{ ref('int_ntd_rr20_service_3ratios_wide') }} ), + rr20f_143 as ( select organization, "RR20F-143: Vehicle Revenue Miles (VRM) change from zero" as name_of_check, mode, {{this_year}} as year_of_data, - CASE WHEN (ROUND(vrm_{{this_year}}) =0 AND ROUND(vrm_{{last_year}}) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) - OR (ROUND(vrm_{{this_year}}) !=0 AND ROUND(vrm_{{this_year}}) IS NOT NULL AND ROUND(voms_{{last_year}}) = 0) + CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + OR (ROUND(vrm_this_year) !=0 AND ROUND(vrm_this_year) IS NOT NULL AND ROUND(voms_last_year) = 0) THEN "Fail" - WHEN (vrm_{{this_year}} IS NULL OR vrm_{{this_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN (vrm_this_year IS NULL OR vrm_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN "Did Not Run" - WHEN (vrm_{{last_year}} IS NULL OR vrm_{{last_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (vrm_last_year IS NULL OR vrm_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN "Did Not Run" ELSE "Pass" END as check_status, - CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_{{this_year}}) AS STRING), - ", ", {{last_year}}, " = ", CAST(ROUND(vrm_{{last_year}}) AS STRING), + CONCAT({{this_year}}, " = ", CAST(ROUND(vrm_this_year) AS STRING), + ", ", {{last_year}}, " = ", CAST(ROUND(vrm_last_year) AS STRING), "chg = ", - CAST(ROUND((ROUND(vrm_{{this_year}},1) - ROUND(vrm_{{last_year}},1))/ABS(ROUND(vrm_{{this_year}},1)) * 100,1) AS STRING), + CAST(ROUND((ROUND(vrm_this_year,1) - ROUND(vrm_last_year,1))/ABS(ROUND(vrm_this_year,1)) * 100,1) AS STRING), "%" ) as value_checked, - CASE WHEN (ROUND(vrm_{{this_year}}) =0 AND ROUND(vrm_{{last_year}}) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) - OR (ROUND(vrm_{{this_year}}) !=0 AND ROUND(vrm_{{this_year}}) IS NOT NULL AND ROUND(vrm_{{last_year}}) = 0) + CASE WHEN (ROUND(vrm_this_year) =0 AND ROUND(vrm_last_year) != 0 AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) > CAST(CONCAT({{this_year}}, "-10-31") AS DATE)) + OR (ROUND(vrm_this_year) !=0 AND ROUND(vrm_this_year) IS NOT NULL AND ROUND(vrm_last_year) = 0) THEN "The Vehicle Revenue Miles (VRM) for this mode has changed either to or from 0 compared to last year, please provide a narrative justification." - WHEN (vrm_{{this_year}} IS NULL OR vrm_{{this_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) + WHEN (vrm_this_year IS NULL OR vrm_this_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{this_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{this_year}}, " for VOMS.") - WHEN (vrm_{{last_year}} IS NULL OR vrm_{{last_year}}=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) + WHEN (vrm_last_year IS NULL OR vrm_last_year=0) AND PARSE_DATE('%Y', CAST({{start_date}} AS STRING)) < CAST(CONCAT({{last_year}}, "-10-31") AS DATE) THEN CONCAT("No data but this check was run before the NTD submission due date in ",{{last_year}}, " for VOMS.") ELSE "" END AS description, diff --git a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_exp_by_mode.sql b/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_exp_by_mode.sql deleted file mode 100644 index 2c6bcd5a38..0000000000 --- a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_exp_by_mode.sql +++ /dev/null @@ -1,6 +0,0 @@ ---- One-time data ingest of 2022 data, whose pattern which will not be repeated in the future ---- We pull these tables in to use them in later int and fct models --- TODO: enumerate columns -SELECT -- noqa: AM04 - * -FROM `cal-itp-data-infra.blackcat_raw.2022_rr20_expenses_by_mode` diff --git a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_financial.sql b/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_financial.sql deleted file mode 100644 index 6afc02d872..0000000000 --- a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_financial.sql +++ /dev/null @@ -1,6 +0,0 @@ ---- One-time data ingest of 2022 data, whose pattern which will not be repeated in the future ---- We pull these tables in to use them in later int and fct models --- TODO: enumerate columns -SELECT -- noqa: AM04 - * -FROM `cal-itp-data-infra.blackcat_raw.2022_rr20_financials__2` diff --git a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_service.sql b/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_service.sql deleted file mode 100644 index 770028f71c..0000000000 --- a/warehouse/models/staging/ntd_validation/stg_ntd_2022_rr20_service.sql +++ /dev/null @@ -1,6 +0,0 @@ ---- One-time data ingest of 2022 data, whose pattern which will not be repeated in the future ---- We pull these tables in to use them in later int and fct models --- TODO: enumerate columns -SELECT -- noqa: AM04 - * -FROM `cal-itp-data-infra.blackcat_raw.2022_rr20_service_data` diff --git a/warehouse/models/staging/ntd_validation/stg_ntd_rr20_urban_tribal.sql b/warehouse/models/staging/ntd_validation/stg_ntd_rr20_urban_tribal.sql index 96f3813a04..e6ee6c8bd7 100644 --- a/warehouse/models/staging/ntd_validation/stg_ntd_rr20_urban_tribal.sql +++ b/warehouse/models/staging/ntd_validation/stg_ntd_rr20_urban_tribal.sql @@ -12,5 +12,4 @@ SELECT ntdreportingrr20_urban_tribal_data.Description as description, ntdreportingrr20_urban_tribal_data.LastModifiedDate as last_modified_date FROM {{ source('ntd_report_validation', 'all_ntdreports') }}, --- `cal-itp-data-infra-staging.external_blackcat.all_ntdreports` UNNEST(`ntdreportingrr20_urban_tribal_data`) as `ntdreportingrr20_urban_tribal_data`