diff --git a/data/README.md b/data/README.md index 99458cda..220eff34 100644 --- a/data/README.md +++ b/data/README.md @@ -33,6 +33,10 @@ Currently, controls the barrier status of natural barriers (gradient, falls, sub NOTE - this table will only be used to identify modelled barriers known to be passable (gradient, subsurface flow) once bcfishpass incorporates CABD falls +## user_cabd_dams_exclusions.csv + +List of CABD dams to exclude from analysis (generally due to incorrect location or incorrect passability status). +Use as temporary fix and submit any location or passability status to CWF for fix in source. ## user_falls.csv diff --git a/data/user_cabd_dams_exclusions.csv b/data/user_cabd_dams_exclusions.csv new file mode 100644 index 00000000..4f0c9fb4 --- /dev/null +++ b/data/user_cabd_dams_exclusions.csv @@ -0,0 +1,8 @@ +cabd_id,reviewer_name,review_date,source,notes +30b88f1b-dc21-4b42-8daa-d4cebae24142,SN,2023-01-01,not noted, +3ca692b8-37cf-44e8-a783-2a315ec83102,SN,2023-01-01,not noted, +ba5fe3eb-7bbe-45c1-b301-555872387c16,SN,2023-01-01,not noted, +8a6b10fa-0d4f-4c45-857c-764d7e8028f8,SN,2023-01-01,not noted, +48478e95-e063-4df6-a047-6aaf6087011b,SN,2023-01-01,not noted, +e8e4bd88-c3c9-407c-a7a0-15c6c51704fd,SN,2023-01-01,not noted,dam may or may not be a barrier but location was incorrect at time of review +6a792d8f-b9c5-44a4-a260-0f06c3b20821,SN,2023-01-01,not noted,dam may or may not be a barrier but location gets matched to Salmon River \ No newline at end of file diff --git a/db/schemas/schemas.sql b/db/schemas/schemas.sql index 7e13607f..6485cc9d 100644 --- a/db/schemas/schemas.sql +++ b/db/schemas/schemas.sql @@ -1,2 +1,3 @@ create schema if not exists bcfishpass; +create schema if not exists cabd; create schema if not exists temp; \ No newline at end of file diff --git a/db/tables/cabd.sql b/db/tables/cabd.sql new file mode 100644 index 00000000..5154535d --- /dev/null +++ b/db/tables/cabd.sql @@ -0,0 +1,139 @@ +-- dams +drop table if exists cabd.dams cascade; + +create table cabd.dams ( + cabd_id text , + use_pollution_code integer , + maintenance_last date , + use_fish text , + height_m double precision , + use_irrigation text , + catchment_area_skm double precision , + facility_name_en text , + use_invasivespecies_code text , + use_electricity text , + waterbody_name_fr text , + last_modified date , + use_pollution text , + updates_pending boolean , + assess_schedule text , + dam_name_fr text , + facility_name_fr text , + waterbody_name_en text , + generating_capacity_mwh double precision , + use_floodcontrol_code integer , + upstream_linear_km text , + reservoir_present boolean , + use_other_code integer , + length_m double precision , + lake_control text , + condition_code integer , + reservoir_name_fr text , + removed_year text , + passability_status_code integer , + passability_status_note text , + latitude double precision , + federal_flow_req text , + size_class text , + dam_name_en text , + use_electricity_code integer , + turbine_type text , + reservoir_depth_m double precision , + province_territory_code text , + up_passage_type text , + passability_status text , + use_irrigation_code integer , + down_passage_route text , + url text , + complete_level_code integer , + construction_year integer , + spillway_capacity text , + reservoir_name_en text , + construction_material_code integer , + ownership_type text , + municipality text , + turbine_number text , + up_passage_type_code integer , + operating_status_code integer , + function_code integer , + use_floodcontrol text , + spillway_type_code integer , + hydro_peaking_system boolean , + expected_end_of_life text , + longitude double precision , + complete_level text , + provincial_compliance_status text , + use_fish_code integer , + spillway_type text , + use_recreation text , + use_recreation_code integer , + use_analysis boolean , + structure_type_code integer , + avg_rate_of_discharge_ls double precision , + nhn_watershed_name text , + use_code integer , + lake_control_code integer , + structure_type text , + use_navigation_code integer , + federal_compliance_status text , + use_supply_code integer , + size_class_code integer , + use_supply text , + provincial_flow_req text , + maintenance_next text , + operating_status text , + turbine_type_code integer , + use_invasivespecies text , + nhn_watershed_id text , + use_conservation text , + degree_of_regulation_pc double precision , + owner text , + construction_material text , + comments text , + dam_condition text , + down_passage_route_code text , + ownership_type_code integer , + use_other text , + reservoir_area_skm double precision , + feature_type text , + function_name text , + operating_notes text , + storage_capacity_mcm double precision , + province_territory text , + dam_use text , + use_conservation_code text , + use_navigation text , + datasource_url text , + geom geometry(Point,4326) +); + +-- waterfalls +drop table if exists cabd.waterfalls cascade; + +create table cabd.waterfalls ( + cabd_id text , + fall_name_fr text , + fall_height_m double precision , + complete_level text , + passability_status_code integer , + comments text , + updates_pending boolean , + fall_name_en text , + latitude double precision , + passability_status text , + municipality text , + url text , + use_analysis boolean , + feature_type text , + complete_level_code integer , + waterbody_name_en text , + province_territory text , + province_territory_code text , + waterbody_name_fr text , + last_modified date , + nhn_watershed_name text , + nhn_watershed_id text , + longitude double precision , + datasource_url text , + geom geometry(Point,4326) +); \ No newline at end of file diff --git a/db/tables/user.sql b/db/tables/user.sql index d6b7f381..0056f5cd 100644 --- a/db/tables/user.sql +++ b/db/tables/user.sql @@ -84,6 +84,20 @@ create table bcfishpass.user_barriers_definite_control primary key (blue_line_key, downstream_route_measure) ); +-- -------------- +-- USER CABD EXCLUSIONS +-- +-- ids of cabd dams to exclude +-- -------------- +drop table if exists bcfishpass.user_cabd_dams_exclusions; +create table bcfishpass.user_cabd_dams_exclusions ( + cabd_id text, + reviewer_name text, + review_date date, + source text, + notes text, + primary key (cabd_id) +); -- -------------- -- USER FALLS diff --git a/db/views/01_dams_vw.sql b/db/views/01_dams_vw.sql new file mode 100644 index 00000000..1aa106b0 --- /dev/null +++ b/db/views/01_dams_vw.sql @@ -0,0 +1,125 @@ +-- reference CABD dams to FWA stream network + +drop materialized view if exists bcfishpass.dams_vw cascade; + +create materialized view bcfishpass.dams_vw as +with cabd as ( + select + d.cabd_id as dam_id, + st_transform(d.geom, 3005) as geom + from cabd.dams d + -- exclude any dam noted in user exclusion table + left outer join bcfishpass.user_cabd_dams_exclusions x on d.cabd_id = x.cabd_id + where x.cabd_id is null +), + +nearest AS +( + select + pt.dam_id, + str.linear_feature_id, + str.blue_line_key, + str.wscode_ltree, + str.localcode_ltree, + str.watershed_group_code, + str.geom, + st_distance(str.geom, pt.geom) as distance_to_stream, + st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure + from cabd pt + cross join lateral ( + select + linear_feature_id, + blue_line_key, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from whse_basemapping.fwa_stream_networks_sp str + where str.localcode_ltree is not null + and not str.wscode_ltree <@ '999' + order by str.geom <-> pt.geom + limit 1 + ) as str + where st_distance(str.geom, pt.geom) <= 65 +), + + -- ensure only one feature returned, and interpolate the geom on the stream +cabd_pts as +( + select distinct on (n.dam_id) + n.dam_id, + n.linear_feature_id, + n.blue_line_key, + n.downstream_route_measure, + n.wscode_ltree, + n.localcode_ltree, + n.distance_to_stream, + n.watershed_group_code, + cabd.dam_name_en, + cabd.height_m, + cabd.owner, + cabd.dam_use, + cabd.operating_status, + cabd.passability_status_code, + + ((st_dump(ST_Force2D(st_locatealong(n.geom, n.downstream_route_measure)))).geom)::geometry(Point, 3005) AS geom + FROM nearest n + inner join cabd.dams cabd on n.dam_id = cabd.cabd_id + order by dam_id, distance_to_stream +), + +-- placeholders for major USA dams not present in CABD are stored in user_barriers_anthropogenic +usa as +( + select + (a.user_barrier_anthropogenic_id + 1200000000)::text as dam_id, + s.linear_feature_id, + a.blue_line_key, + a.downstream_route_measure, + s.wscode_ltree, + s.localcode_ltree, + 0 as distance_to_stream, + s.watershed_group_code, + a.barrier_name, + st_force2d((st_dump(st_locatealong(s.geom, a.downstream_route_measure))).geom) as geom + from bcfishpass.user_barriers_anthropogenic a + inner join whse_basemapping.fwa_stream_networks_sp s + on a.blue_line_key = s.blue_line_key + AND ROUND(a.downstream_route_measure::numeric) >= ROUND(s.downstream_route_measure::numeric) + AND ROUND(a.downstream_route_measure::numeric) < ROUND(s.upstream_route_measure::numeric) + where a.barrier_type = 'DAM' +) + +select * from cabd_pts +union all +select + dam_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + distance_to_stream, + watershed_group_code, + barrier_name as dam_name_en, + null as height_m, + null as owner, + null as dam_use, + null as operating_status, + null as passability_status_code, + geom +from usa; + +create unique index on bcfishpass.dams_vw (dam_id); +create index on bcfishpass.dams_vw using gist (geom); + +drop view if exists bcfishpass.dams_not_matched_to_streams; +create view bcfishpass.dams_not_matched_to_streams as +select + a.cabd_id, + a.dam_name_en +from cabd.dams a +left outer join bcfishpass.dams_vw b +on a.cabd_id::text = b.dam_id +where b.dam_id is null +order by a.cabd_id; \ No newline at end of file diff --git a/model/01_access/Makefile b/model/01_access/Makefile index d417c97f..35f6363e 100644 --- a/model/01_access/Makefile +++ b/model/01_access/Makefile @@ -74,9 +74,16 @@ test: # ------ # DAMS # ------ -.make/dams: dams/dams.sh dams/sql/dams.sql +.make/dams: mkdir -p .make - cd dams; ./dams.sh + $(PSQL) -c "truncate cabd.dams" + ogr2ogr -f PostgreSQL \ + "PG:$(DATABASE_URL)" \ + -append \ + -nln cabd.dams \ + "https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ + OGRGeoJSON + $(PSQL) -c "refresh materialized view bcfishpass.dams_vw" touch $@ # ------ diff --git a/model/01_access/dams/README.md b/model/01_access/dams/README.md deleted file mode 100644 index edf2e9ae..00000000 --- a/model/01_access/dams/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Dams - -BC dam locations and barrier status have been compiled from many sources by the Canadian Wildlife Federation into the [Canadian Aquatic Barrier Database (CABD)](https://aquaticbarriers.ca). Dam features for bcfishpass are taken directly from the CABD. - -Also included are temporary placeholders for downstream USA dams that are barriers to fish passage (cannot currently be matched to FWA streams) - - -## Usage - -Download CABD dams and match the points to the nearest FWA stream (within 50m): - - ./dams.sh - -## Output tables - -CABD tables are as [documented](https://cabd-docs.netlify.app/docs_user/docs_user_data_catalogue.html) - -`bcfishpass.dams` holds only the FWA/linear referencing information associated with a dam, plus the geometry of the feature snapped to the nearest FWA stream (for more information about dams, join this table back to `cabd.dams` or refer directly to the CABD) - - Table "bcfishpass.dams" - Column | Type | Collation | Nullable | Default - --------------------------+----------------------+-----------+----------+--------- - dam_id | character varying | | not null | - linear_feature_id | bigint | | | - blue_line_key | integer | | | - downstream_route_measure | double precision | | | - wscode_ltree | ltree | | | - localcode_ltree | ltree | | | - distance_to_stream | double precision | | | - watershed_group_code | text | | | - geom | geometry(Point,3005) | | | - Indexes: - "dams_pkey" PRIMARY KEY, btree (dam_id) - "dams_blue_line_key_downstream_route_measure_key" UNIQUE CONSTRAINT, btree (blue_line_key, downstream_route_measure) - "dams_blue_line_key_idx" btree (blue_line_key) - "dams_geom_idx" gist (geom) - "dams_linear_feature_id_idx" btree (linear_feature_id) - "dams_localcode_ltree_idx" gist (localcode_ltree) - "dams_localcode_ltree_idx1" btree (localcode_ltree) - "dams_watershed_group_code_idx" btree (watershed_group_code) - "dams_wscode_ltree_idx" gist (wscode_ltree) - "dams_wscode_ltree_idx1" btree (wscode_ltree) \ No newline at end of file diff --git a/model/01_access/dams/dams.sh b/model/01_access/dams/dams.sh deleted file mode 100755 index e8db618e..00000000 --- a/model/01_access/dams/dams.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -# --------------------- -# download CABD dams and match to FWA streams -# --------------------- - -PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" -DAMS_URL="https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" - -# load dams -$PSQL -c "create schema if not exists cabd" -ogr2ogr -f PostgreSQL \ - "PG:$DATABASE_URL" \ - -overwrite \ - -s_srs EPSG:4326 \ - -t_srs EPSG:3005 \ - -lco GEOMETRY_NAME=geom \ - -nln cabd.dams \ - $DAMS_URL \ - OGRGeoJSON - -# create bcfishpass.dams - matching the dams to streams -$PSQL -f sql/dams.sql \ No newline at end of file diff --git a/model/01_access/dams/sql/dams.sql b/model/01_access/dams/sql/dams.sql deleted file mode 100644 index 51e0cf65..00000000 --- a/model/01_access/dams/sql/dams.sql +++ /dev/null @@ -1,165 +0,0 @@ --- reference CABD dams to FWA stream network - -alter table cabd.dams alter column cabd_id type uuid using cabd_id::uuid; - -DROP TABLE IF EXISTS bcfishpass.dams; - -CREATE TABLE bcfishpass.dams -(dam_id text primary key, - linear_feature_id bigint, - blue_line_key integer, - downstream_route_measure double precision, - wscode_ltree ltree, - localcode_ltree ltree, - distance_to_stream double precision, - watershed_group_code text, - geom geometry(Point, 3005), - UNIQUE (blue_line_key, downstream_route_measure) -); - - -INSERT INTO bcfishpass.dams -( - dam_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - distance_to_stream, - watershed_group_code, - geom -) - -WITH nearest AS -( - SELECT - pt.cabd_id::text as dam_id, - str.linear_feature_id, - str.wscode_ltree, - str.localcode_ltree, - str.blue_line_key, - str.waterbody_key, - str.length_metre, - ST_Distance(str.geom, pt.geom) as distance_to_stream, - str.watershed_group_code, - str.downstream_route_measure as downstream_route_measure_str, - ( - ST_LineLocatePoint( - st_linemerge(str.geom), - ST_ClosestPoint(str.geom, pt.geom) - ) - * str.length_metre - ) + str.downstream_route_measure AS downstream_route_measure, - st_linemerge(str.geom) as geom_str - FROM cabd.dams pt - CROSS JOIN LATERAL - (SELECT - linear_feature_id, - wscode_ltree, - localcode_ltree, - blue_line_key, - waterbody_key, - length_metre, - downstream_route_measure, - watershed_group_code, - geom - FROM whse_basemapping.fwa_stream_networks_sp str - WHERE str.localcode_ltree IS NOT NULL - AND NOT str.wscode_ltree <@ '999' - ORDER BY str.geom <-> pt.geom - LIMIT 1) as str - WHERE ST_Distance(str.geom, pt.geom) <= 65 - -- exclude CABD features that may not have been fixed in source - AND pt.cabd_id NOT IN ( - 'e8e4bd88-c3c9-407c-a7a0-15c6c51704fd', -- seton (location issues) - '6a792d8f-b9c5-44a4-a260-0f06c3b20821', -- salmon/merton (location issues) - '222c3ccd-00de-4b7c-b0ff-ec282c060adb' -- Docee River counting fence is not a barrier - ) -) - -SELECT - dam_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - distance_to_stream, - watershed_group_code, - ST_Force2D( - ST_LineInterpolatePoint(geom_str, - ROUND( - CAST( - (downstream_route_measure - - downstream_route_measure_str) / length_metre AS NUMERIC - ), - 5) - ) - )::geometry(Point, 3005) AS geom -FROM nearest -ON CONFLICT DO NOTHING; - --- load placeholders for USA dams -insert into bcfishpass.dams - ( - dam_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - distance_to_stream, - watershed_group_code, - geom -) - -select - (a.user_barrier_anthropogenic_id + 1200000000)::text as dam_id, - s.linear_feature_id, - a.blue_line_key, - a.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - 0 as distance_to_stream, - s.watershed_group_code, - st_force2d((st_dump(st_locatealong(s.geom, a.downstream_route_measure))).geom) as geom -from bcfishpass.user_barriers_anthropogenic a -inner join whse_basemapping.fwa_stream_networks_sp s -on a.blue_line_key = s.blue_line_key -AND ROUND(a.downstream_route_measure::numeric) >= ROUND(s.downstream_route_measure::numeric) -AND ROUND(a.downstream_route_measure::numeric) < ROUND(s.upstream_route_measure::numeric) -where a.barrier_type = 'DAM'; - -CREATE INDEX ON bcfishpass.dams (linear_feature_id); -CREATE INDEX ON bcfishpass.dams (blue_line_key); -CREATE INDEX ON bcfishpass.dams (watershed_group_code); -CREATE INDEX ON bcfishpass.dams USING GIST (wscode_ltree); -CREATE INDEX ON bcfishpass.dams USING BTREE (wscode_ltree); -CREATE INDEX ON bcfishpass.dams USING GIST (localcode_ltree); -CREATE INDEX ON bcfishpass.dams USING BTREE (localcode_ltree); -CREATE INDEX ON bcfishpass.dams USING GIST (geom); - - --- patch CABD barrier status for these likely passable features -update cabd.dams -set passability_status_code = 3 -where cabd_id in ( -'30b88f1b-dc21-4b42-8daa-d4cebae24142', -'3ca692b8-37cf-44e8-a783-2a315ec83102', -'ba5fe3eb-7bbe-45c1-b301-555872387c16', -'8a6b10fa-0d4f-4c45-857c-764d7e8028f8', -'48478e95-e063-4df6-a047-6aaf6087011b' -); - --- report on dams that do not get matched to FWA streams -drop view if exists bcfishpass.dams_not_matched_to_streams; -create view bcfishpass.dams_not_matched_to_streams as -select - a.cabd_id, - a.dam_name_en -from cabd.dams a -left join bcfishpass.dams b -on a.cabd_id::text = b.dam_id -where b.dam_id is null -order by a.cabd_id; \ No newline at end of file diff --git a/model/01_access/sql/load_crossings.sql b/model/01_access/sql/load_crossings.sql index 8c50373c..95056802 100644 --- a/model/01_access/sql/load_crossings.sql +++ b/model/01_access/sql/load_crossings.sql @@ -410,7 +410,7 @@ select s.stream_order, s.stream_magnitude, (st_dump(st_locatealong(s.geom, d.downstream_route_measure))).geom as geom -from bcfishpass.dams d +from bcfishpass.dams_vw d inner join whse_basemapping.fwa_stream_networks_sp s on d.linear_feature_id = s.linear_feature_id inner join cabd.dams cabd on d.dam_id = cabd.cabd_id::text @@ -467,7 +467,7 @@ select s.stream_order, s.stream_magnitude, (st_dump(st_locatealong(s.geom, d.downstream_route_measure))).geom as geom -from bcfishpass.dams d +from bcfishpass.dams_vw d inner join whse_basemapping.fwa_stream_networks_sp s on d.linear_feature_id = s.linear_feature_id inner join bcfishpass.user_barriers_anthropogenic ba on d.blue_line_key = ba.blue_line_key diff --git a/model/01_access/sql/load_streams.sql b/model/01_access/sql/load_streams.sql index ba02a6f9..f5679306 100644 --- a/model/01_access/sql/load_streams.sql +++ b/model/01_access/sql/load_streams.sql @@ -39,8 +39,8 @@ SELECT s.stream_magnitude, s.feature_code, ua.upstream_area_ha, - s.stream_order_parent, - s.stream_order_max, + op.stream_order_parent, + om.stream_order_max, p.map_upstream, cw.channel_width, mad.mad_m3s, @@ -50,6 +50,8 @@ LEFT OUTER JOIN whse_basemapping.fwa_streams_watersheds_lut l ON s.linear_feature_id = l.linear_feature_id LEFT OUTER JOIN whse_basemapping.fwa_watersheds_upstream_area ua -- use left join in case the lookup has not been updated with latest FWA data ON l.watershed_feature_id = ua.watershed_feature_id +left outer join whse_basemapping.fwa_stream_networks_order_parent op on s.blue_line_key = op.blue_line_key +left outer join whse_basemapping.fwa_stream_networks_order_max om on s.blue_line_key_50k = om.blue_line_key LEFT OUTER JOIN bcfishpass.mean_annual_precip p ON s.wscode_ltree = p.wscode_ltree AND s.localcode_ltree = p.localcode_ltree LEFT OUTER JOIN bcfishpass.channel_width cw ON s.linear_feature_id = cw.linear_feature_id LEFT OUTER JOIN bcfishpass.discharge mad ON s.linear_feature_id = mad.linear_feature_id