diff --git a/.github/workflows/production-load-monthly.yaml b/.github/workflows/production-load-monthly.yaml index 859f1b4e..7b590179 100644 --- a/.github/workflows/production-load-monthly.yaml +++ b/.github/workflows/production-load-monthly.yaml @@ -15,4 +15,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - name: Run the jobs - run: jobs/load_monthly \ No newline at end of file + run: | + jobs/replicate_dra + jobs/load_monthly \ No newline at end of file diff --git a/db/v0.5.4/migrate.sh b/db/v0.5.4/migrate.sh new file mode 100755 index 00000000..7a7154ee --- /dev/null +++ b/db/v0.5.4/migrate.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euxo pipefail + +# update DRA table +psql $DATABASE_URL -v ON_ERROR_STOP=1 -f sql/dra.sql + +# note version +psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "update bcfishpass.db_version set tag = '${PWD##*/}'" \ No newline at end of file diff --git a/db/v0.5.4/sql/dra.sql b/db/v0.5.4/sql/dra.sql new file mode 100644 index 00000000..bc8e05b9 --- /dev/null +++ b/db/v0.5.4/sql/dra.sql @@ -0,0 +1,81 @@ +-- drop non public columns from dra transport line +BEGIN; + +ALTER TABLE whse_basemapping.transport_line + drop column create_integration_session_id, + drop column create_integration_date, + drop column modify_integration_session_id, + drop column modify_integration_date, + drop column create_partner_org_id, + drop column create_partner_org, + drop column modify_partner_org_id, + drop column modify_partner_org, + drop column custodian_partner_org_id, + drop column z_value_derived_ind, + drop column deactivation_date, + drop column transport_line_divided_code, + drop column travel_direction_code, + drop column speed_limit, + drop column left_number_of_lanes, + drop column right_number_of_lanes, + drop column under_construction_ind, + drop column virtual_ind, + drop column disaster_route_ind, + drop column truck_route_ind, + drop column left_locality_id, + drop column left_locality, + drop column right_locality_id, + drop column right_locality, + drop column left_regional_district_id, + drop column right_regional_district_id, + drop column structured_name_1_id, + drop column structured_name_2_id, + drop column structured_name_3_id, + drop column structured_name_4_id, + drop column structured_name_5_id, + drop column structured_name_6_id, + drop column structured_name_6, + drop column structured_name_7_id, + drop column structured_name_7, + drop column highway_route_2, + drop column highway_route_3, + drop column industry_name_1, + drop column industry_name_2, + drop column industry_name_3, + drop column single_house_number, + drop column left_house_num_scheme_code, + drop column from_left_house_number, + drop column to_left_house_number, + drop column right_house_num_scheme_code, + drop column from_right_house_number, + drop column to_right_house_number, + drop column lane_restriction_code, + drop column access_restriction_code, + drop column from_traffic_impactor_code, + drop column to_traffic_impactor_code, + drop column from_left_turn_time_code, + drop column from_centre_turn_time_code, + drop column from_right_turn_time_code, + drop column to_left_turn_time_code, + drop column to_centre_turn_time_code, + drop column to_right_turn_time_code, + drop column from_vehicle_max_weight_kg, + drop column to_vehicle_max_weight_kg, + drop column from_vehicle_max_width_metre, + drop column to_vehicle_max_width_metre, + drop column from_vehicle_max_height_metre, + drop column to_vehicle_max_height_metre, + drop column ministry_of_transport_id, + drop column ministry_of_transport_name, + drop column integration_notes, + drop column excluded_rules, + drop column demographic_ind, + drop column extended_data, + drop column ministry_of_transport_data, + drop column from_navigation_rules, + drop column along_navigation_rules, + drop column to_navigation_rules, + drop column from_transport_node_point_id, + drop column to_transport_node_point_id; + +COMMIT; \ No newline at end of file diff --git a/jobs/load_monthly b/jobs/load_monthly index 7f11b9ee..7f1bb639 100755 --- a/jobs/load_monthly +++ b/jobs/load_monthly @@ -25,13 +25,7 @@ bcdata bc2pg -r whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp bcdata bc2pg -r whse_tantalis.ta_conservancy_areas_svw bcdata bc2pg -r whse_tantalis.ta_park_ecores_pa_svw -# DRA -mkdir -p tmp -# ** -# ** note temp dra data cache, to be replaced with new download location or a bc2pg download ** -# ** -#curl ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ -# -o tmp/dgtl_road_atlas.gdb.zip +# load DRA from bchamp bucket (with public portion of source transport_line schema) $PSQL -c "drop table if exists bcdata.transport_line" # in case of any failed loads ogr2ogr \ -f PostgreSQL \ @@ -40,10 +34,46 @@ ogr2ogr \ -lco GEOMETRY_NAME=geom \ -lco FID=transport_line_id \ -nln bcdata.transport_line \ - /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \ + /vsizip//vsicurl/https://nrs.objectstore.gov.bc.ca/bchamp/transport_line.gpkg.zip \ TRANSPORT_LINE $PSQL -c "truncate whse_basemapping.transport_line; - insert into whse_basemapping.transport_line select * from bcdata.transport_line" + insert into whse_basemapping.transport_line + ( + transport_line_id, + custodian_partner_org, + capture_date, + data_capture_method_code, + total_number_of_lanes, + structured_name_1, + structured_name_2, + structured_name_3, + structured_name_4, + structured_name_5, + highway_route_1, + highway_exit_number, + transport_line_type_code, + transport_line_surface_code, + transport_line_structure_code, + geom + ) + select + transport_line_id, + custodian_partner_org, + capture_date, + data_capture_method_code, + total_number_of_lanes, + structured_name_1, + structured_name_2, + structured_name_3, + structured_name_4, + structured_name_5, + highway_route_1, + highway_exit_number, + transport_line_type_code, + transport_line_surface_code, + transport_line_structure_code, + geom + from bcdata.transport_line" $PSQL -c "drop table bcdata.transport_line" # parcel fabric, renaming download so it can be read without unzipping diff --git a/jobs/replicate_dra b/jobs/replicate_dra new file mode 100755 index 00000000..eb17d540 --- /dev/null +++ b/jobs/replicate_dra @@ -0,0 +1,47 @@ +#!/bin/bash +set -euxo pipefail + +# We do not use the official public DRA data because it no longer matches our db schema +# Replicate public portion of password protected DRA distribution to publicly accessible file +# (requires $DRAPWD) + +# download +curl \ + -o /tmp/transport_line.gdb.zip \ + https://nrs.objectstore.gov.bc.ca/itqlyp/GBA/PROVINCE/transport_line.gdb.zip + +# extract password protected zipfile +unzip \ + -P $DRAPWD \ + -d /tmp \ + -o \ + /tmp/transport_line.gdb.zip + +# post public portion of dataset to bchamp object storage +ogr2ogr \ + -f GPKG \ + /tmp/transport_line.gpkg.zip \ + /tmp/transport_line.gdb \ + -sql "select + TRANSPORT_LINE_ID, + CUSTODIAN_PARTNER_ORG, + CAPTURE_DATE, + DATA_CAPTURE_METHOD_CODE, + TOTAL_NUMBER_OF_LANES, + STRUCTURED_NAME_1, + STRUCTURED_NAME_2, + STRUCTURED_NAME_3, + STRUCTURED_NAME_4, + STRUCTURED_NAME_5, + HIGHWAY_ROUTE_1, + HIGHWAY_EXIT_NUMBER, + TRANSPORT_LINE_TYPE_CODE, + TRANSPORT_LINE_SURFACE_CODE, + TRANSPORT_LINE_STRUCTURE_CODE, + GEOMETRY + from TRANSPORT_LINE" + +aws s3 cp /tmp/transport_line.gpkg.zip s3://bchamp --acl public-read + +rm /tmp/transport_line.gdb.zip +rm /tmp/transport_line.gpkg.zip \ No newline at end of file