fix #496 (#595)

db has been updated, good to merge
smnorris · Nov 7, 2024 · 2c01a52 · 2c01a52
1 parent c7e7d27
commit 2c01a52
Show file tree

Hide file tree

Showing 5 changed files with 178 additions and 10 deletions.
diff --git a/.github/workflows/production-load-monthly.yaml b/.github/workflows/production-load-monthly.yaml
@@ -15,4 +15,6 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run the jobs
-        run: jobs/load_monthly
+        run: |
+          jobs/replicate_dra
+          jobs/load_monthly
diff --git a/db/v0.5.4/migrate.sh b/db/v0.5.4/migrate.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euxo pipefail
+
+# update DRA table
+psql $DATABASE_URL -v ON_ERROR_STOP=1 -f sql/dra.sql
+
+# note version
+psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "update bcfishpass.db_version set tag = '${PWD##*/}'"
diff --git a/db/v0.5.4/sql/dra.sql b/db/v0.5.4/sql/dra.sql
@@ -0,0 +1,81 @@
+-- drop non public columns from dra transport line
+BEGIN;
+
+ALTER TABLE whse_basemapping.transport_line 
+  drop column create_integration_session_id,
+  drop column create_integration_date,
+  drop column modify_integration_session_id,
+  drop column modify_integration_date,
+  drop column create_partner_org_id,
+  drop column create_partner_org,
+  drop column modify_partner_org_id,
+  drop column modify_partner_org,
+  drop column custodian_partner_org_id,
+  drop column z_value_derived_ind,
+  drop column deactivation_date,
+  drop column transport_line_divided_code,
+  drop column travel_direction_code,
+  drop column speed_limit,
+  drop column left_number_of_lanes,
+  drop column right_number_of_lanes,
+  drop column under_construction_ind,
+  drop column virtual_ind,
+  drop column disaster_route_ind,
+  drop column truck_route_ind,
+  drop column left_locality_id,
+  drop column left_locality,
+  drop column right_locality_id,
+  drop column right_locality,
+  drop column left_regional_district_id,
+  drop column right_regional_district_id,
+  drop column structured_name_1_id,
+  drop column structured_name_2_id,
+  drop column structured_name_3_id,
+  drop column structured_name_4_id,
+  drop column structured_name_5_id,
+  drop column structured_name_6_id,
+  drop column structured_name_6,
+  drop column structured_name_7_id,
+  drop column structured_name_7,
+  drop column highway_route_2,
+  drop column highway_route_3,
+  drop column industry_name_1,
+  drop column industry_name_2,
+  drop column industry_name_3,
+  drop column single_house_number,
+  drop column left_house_num_scheme_code,
+  drop column from_left_house_number,
+  drop column to_left_house_number,
+  drop column right_house_num_scheme_code,
+  drop column from_right_house_number,
+  drop column to_right_house_number,
+  drop column lane_restriction_code,
+  drop column access_restriction_code,
+  drop column from_traffic_impactor_code,
+  drop column to_traffic_impactor_code,
+  drop column from_left_turn_time_code,
+  drop column from_centre_turn_time_code,
+  drop column from_right_turn_time_code,
+  drop column to_left_turn_time_code,
+  drop column to_centre_turn_time_code,
+  drop column to_right_turn_time_code,
+  drop column from_vehicle_max_weight_kg,
+  drop column to_vehicle_max_weight_kg,
+  drop column from_vehicle_max_width_metre,
+  drop column to_vehicle_max_width_metre,
+  drop column from_vehicle_max_height_metre,
+  drop column to_vehicle_max_height_metre,
+  drop column ministry_of_transport_id,
+  drop column ministry_of_transport_name,
+  drop column integration_notes,
+  drop column excluded_rules,
+  drop column demographic_ind,
+  drop column extended_data,
+  drop column ministry_of_transport_data,
+  drop column from_navigation_rules,
+  drop column along_navigation_rules,
+  drop column to_navigation_rules,
+  drop column from_transport_node_point_id,
+  drop column to_transport_node_point_id;
+
+COMMIT;
diff --git a/jobs/load_monthly b/jobs/load_monthly
@@ -25,13 +25,7 @@ bcdata bc2pg -r whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp
 bcdata bc2pg -r whse_tantalis.ta_conservancy_areas_svw
 bcdata bc2pg -r whse_tantalis.ta_park_ecores_pa_svw
 
-# DRA
-mkdir -p tmp
-# **
-# ** note temp dra data cache, to be replaced with new download location or a bc2pg download **
-# **
-#curl ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \
-#  -o tmp/dgtl_road_atlas.gdb.zip
+# load DRA from bchamp bucket (with public portion of source transport_line schema)
 $PSQL -c "drop table if exists bcdata.transport_line" # in case of any failed loads
 ogr2ogr \
   -f PostgreSQL \
@@ -40,10 +34,46 @@ ogr2ogr \
   -lco GEOMETRY_NAME=geom \
   -lco FID=transport_line_id \
   -nln bcdata.transport_line \
-  /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \
+  /vsizip//vsicurl/https://nrs.objectstore.gov.bc.ca/bchamp/transport_line.gpkg.zip \
   TRANSPORT_LINE
 $PSQL -c "truncate whse_basemapping.transport_line;
-          insert into whse_basemapping.transport_line select * from bcdata.transport_line"
+          insert into whse_basemapping.transport_line
+          (
+            transport_line_id,
+            custodian_partner_org,
+            capture_date,
+            data_capture_method_code,
+            total_number_of_lanes,
+            structured_name_1,
+            structured_name_2,
+            structured_name_3,
+            structured_name_4,
+            structured_name_5,
+            highway_route_1,
+            highway_exit_number,
+            transport_line_type_code,
+            transport_line_surface_code,
+            transport_line_structure_code,
+            geom
+          )
+          select
+            transport_line_id,
+            custodian_partner_org,
+            capture_date,
+            data_capture_method_code,
+            total_number_of_lanes,
+            structured_name_1,
+            structured_name_2,
+            structured_name_3,
+            structured_name_4,
+            structured_name_5,
+            highway_route_1,
+            highway_exit_number,
+            transport_line_type_code,
+            transport_line_surface_code,
+            transport_line_structure_code,
+            geom
+      from bcdata.transport_line"
 $PSQL -c "drop table bcdata.transport_line"
 
 # parcel fabric, renaming download so it can be read without unzipping

diff --git a/jobs/replicate_dra b/jobs/replicate_dra
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -euxo pipefail
+
+# We do not use the official public DRA data because it no longer matches our db schema
+# Replicate public portion of password protected DRA distribution to publicly accessible file
+# (requires $DRAPWD)
+
+# download
+curl \
+  -o /tmp/transport_line.gdb.zip \
+  https://nrs.objectstore.gov.bc.ca/itqlyp/GBA/PROVINCE/transport_line.gdb.zip
+
+# extract password protected zipfile
+unzip \
+  -P $DRAPWD \
+  -d /tmp \
+  -o \
+  /tmp/transport_line.gdb.zip
+
+# post public portion of dataset to bchamp object storage
+ogr2ogr \
+  -f GPKG \
+  /tmp/transport_line.gpkg.zip \
+  /tmp/transport_line.gdb \
+  -sql "select
+     TRANSPORT_LINE_ID,
+     CUSTODIAN_PARTNER_ORG,
+     CAPTURE_DATE,
+     DATA_CAPTURE_METHOD_CODE,
+     TOTAL_NUMBER_OF_LANES,
+     STRUCTURED_NAME_1,
+     STRUCTURED_NAME_2,
+     STRUCTURED_NAME_3,
+     STRUCTURED_NAME_4,
+     STRUCTURED_NAME_5,
+     HIGHWAY_ROUTE_1,
+     HIGHWAY_EXIT_NUMBER,
+     TRANSPORT_LINE_TYPE_CODE,
+     TRANSPORT_LINE_SURFACE_CODE,
+     TRANSPORT_LINE_STRUCTURE_CODE,
+     GEOMETRY
+  from TRANSPORT_LINE"
+
+aws s3 cp /tmp/transport_line.gpkg.zip s3://bchamp --acl public-read
+
+rm /tmp/transport_line.gdb.zip
+rm /tmp/transport_line.gpkg.zip