Skip to content

Commit

Permalink
tidy dams data flow, support latest fwa schema (#457)
Browse files Browse the repository at this point in the history 
* clean up dams data flow

* add dam view file

* add dam notes, discard unused indexes

* add cabd exclusion table

* modify streams table creation to use updated fwa schema
  • Loading branch information
@smnorris
smnorris authored Feb 20, 2024
1 parent cf9444f commit 4773b48
Show file tree
Hide file tree
Showing 12 changed files with 306 additions and 237 deletions.
4 changes: 4 additions & 0 deletions data/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ Currently, controls the barrier status of natural barriers (gradient, falls, sub
NOTE - this table will only be used to identify modelled barriers known to be passable
(gradient, subsurface flow) once bcfishpass incorporates CABD falls

## user_cabd_dams_exclusions.csv

List of CABD dams to exclude from analysis (generally due to incorrect location or incorrect passability status).
Use as temporary fix and submit any location or passability status to CWF for fix in source.

## user_falls.csv

Expand Down
8 changes: 8 additions & 0 deletions data/user_cabd_dams_exclusions.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
cabd_id,reviewer_name,review_date,source,notes
30b88f1b-dc21-4b42-8daa-d4cebae24142,SN,2023-01-01,not noted,
3ca692b8-37cf-44e8-a783-2a315ec83102,SN,2023-01-01,not noted,
ba5fe3eb-7bbe-45c1-b301-555872387c16,SN,2023-01-01,not noted,
8a6b10fa-0d4f-4c45-857c-764d7e8028f8,SN,2023-01-01,not noted,
48478e95-e063-4df6-a047-6aaf6087011b,SN,2023-01-01,not noted,
e8e4bd88-c3c9-407c-a7a0-15c6c51704fd,SN,2023-01-01,not noted,dam may or may not be a barrier but location was incorrect at time of review
6a792d8f-b9c5-44a4-a260-0f06c3b20821,SN,2023-01-01,not noted,dam may or may not be a barrier but location gets matched to Salmon River
1 change: 1 addition & 0 deletions db/schemas/schemas.sql
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
create schema if not exists bcfishpass;
create schema if not exists cabd;
create schema if not exists temp;
139 changes: 139 additions & 0 deletions db/tables/cabd.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
-- dams
drop table if exists cabd.dams cascade;

create table cabd.dams (
cabd_id text ,
use_pollution_code integer ,
maintenance_last date ,
use_fish text ,
height_m double precision ,
use_irrigation text ,
catchment_area_skm double precision ,
facility_name_en text ,
use_invasivespecies_code text ,
use_electricity text ,
waterbody_name_fr text ,
last_modified date ,
use_pollution text ,
updates_pending boolean ,
assess_schedule text ,
dam_name_fr text ,
facility_name_fr text ,
waterbody_name_en text ,
generating_capacity_mwh double precision ,
use_floodcontrol_code integer ,
upstream_linear_km text ,
reservoir_present boolean ,
use_other_code integer ,
length_m double precision ,
lake_control text ,
condition_code integer ,
reservoir_name_fr text ,
removed_year text ,
passability_status_code integer ,
passability_status_note text ,
latitude double precision ,
federal_flow_req text ,
size_class text ,
dam_name_en text ,
use_electricity_code integer ,
turbine_type text ,
reservoir_depth_m double precision ,
province_territory_code text ,
up_passage_type text ,
passability_status text ,
use_irrigation_code integer ,
down_passage_route text ,
url text ,
complete_level_code integer ,
construction_year integer ,
spillway_capacity text ,
reservoir_name_en text ,
construction_material_code integer ,
ownership_type text ,
municipality text ,
turbine_number text ,
up_passage_type_code integer ,
operating_status_code integer ,
function_code integer ,
use_floodcontrol text ,
spillway_type_code integer ,
hydro_peaking_system boolean ,
expected_end_of_life text ,
longitude double precision ,
complete_level text ,
provincial_compliance_status text ,
use_fish_code integer ,
spillway_type text ,
use_recreation text ,
use_recreation_code integer ,
use_analysis boolean ,
structure_type_code integer ,
avg_rate_of_discharge_ls double precision ,
nhn_watershed_name text ,
use_code integer ,
lake_control_code integer ,
structure_type text ,
use_navigation_code integer ,
federal_compliance_status text ,
use_supply_code integer ,
size_class_code integer ,
use_supply text ,
provincial_flow_req text ,
maintenance_next text ,
operating_status text ,
turbine_type_code integer ,
use_invasivespecies text ,
nhn_watershed_id text ,
use_conservation text ,
degree_of_regulation_pc double precision ,
owner text ,
construction_material text ,
comments text ,
dam_condition text ,
down_passage_route_code text ,
ownership_type_code integer ,
use_other text ,
reservoir_area_skm double precision ,
feature_type text ,
function_name text ,
operating_notes text ,
storage_capacity_mcm double precision ,
province_territory text ,
dam_use text ,
use_conservation_code text ,
use_navigation text ,
datasource_url text ,
geom geometry(Point,4326)
);

-- waterfalls
drop table if exists cabd.waterfalls cascade;

create table cabd.waterfalls (
cabd_id text ,
fall_name_fr text ,
fall_height_m double precision ,
complete_level text ,
passability_status_code integer ,
comments text ,
updates_pending boolean ,
fall_name_en text ,
latitude double precision ,
passability_status text ,
municipality text ,
url text ,
use_analysis boolean ,
feature_type text ,
complete_level_code integer ,
waterbody_name_en text ,
province_territory text ,
province_territory_code text ,
waterbody_name_fr text ,
last_modified date ,
nhn_watershed_name text ,
nhn_watershed_id text ,
longitude double precision ,
datasource_url text ,
geom geometry(Point,4326)
);
14 changes: 14 additions & 0 deletions db/tables/user.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,20 @@ create table bcfishpass.user_barriers_definite_control
primary key (blue_line_key, downstream_route_measure)
);

-- --------------
-- USER CABD EXCLUSIONS
--
-- ids of cabd dams to exclude
-- --------------
drop table if exists bcfishpass.user_cabd_dams_exclusions;
create table bcfishpass.user_cabd_dams_exclusions (
cabd_id text,
reviewer_name text,
review_date date,
source text,
notes text,
primary key (cabd_id)
);

-- --------------
-- USER FALLS
Expand Down
125 changes: 125 additions & 0 deletions db/views/01_dams_vw.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
-- reference CABD dams to FWA stream network

drop materialized view if exists bcfishpass.dams_vw cascade;

create materialized view bcfishpass.dams_vw as
with cabd as (
select
d.cabd_id as dam_id,
st_transform(d.geom, 3005) as geom
from cabd.dams d
-- exclude any dam noted in user exclusion table
left outer join bcfishpass.user_cabd_dams_exclusions x on d.cabd_id = x.cabd_id
where x.cabd_id is null
),

nearest AS
(
select
pt.dam_id,
str.linear_feature_id,
str.blue_line_key,
str.wscode_ltree,
str.localcode_ltree,
str.watershed_group_code,
str.geom,
st_distance(str.geom, pt.geom) as distance_to_stream,
st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure
from cabd pt
cross join lateral (
select
linear_feature_id,
blue_line_key,
wscode_ltree,
localcode_ltree,
watershed_group_code,
geom
from whse_basemapping.fwa_stream_networks_sp str
where str.localcode_ltree is not null
and not str.wscode_ltree <@ '999'
order by str.geom <-> pt.geom
limit 1
) as str
where st_distance(str.geom, pt.geom) <= 65
),

-- ensure only one feature returned, and interpolate the geom on the stream
cabd_pts as
(
select distinct on (n.dam_id)
n.dam_id,
n.linear_feature_id,
n.blue_line_key,
n.downstream_route_measure,
n.wscode_ltree,
n.localcode_ltree,
n.distance_to_stream,
n.watershed_group_code,
cabd.dam_name_en,
cabd.height_m,
cabd.owner,
cabd.dam_use,
cabd.operating_status,
cabd.passability_status_code,

((st_dump(ST_Force2D(st_locatealong(n.geom, n.downstream_route_measure)))).geom)::geometry(Point, 3005) AS geom
FROM nearest n
inner join cabd.dams cabd on n.dam_id = cabd.cabd_id
order by dam_id, distance_to_stream
),

-- placeholders for major USA dams not present in CABD are stored in user_barriers_anthropogenic
usa as
(
select
(a.user_barrier_anthropogenic_id + 1200000000)::text as dam_id,
s.linear_feature_id,
a.blue_line_key,
a.downstream_route_measure,
s.wscode_ltree,
s.localcode_ltree,
0 as distance_to_stream,
s.watershed_group_code,
a.barrier_name,
st_force2d((st_dump(st_locatealong(s.geom, a.downstream_route_measure))).geom) as geom
from bcfishpass.user_barriers_anthropogenic a
inner join whse_basemapping.fwa_stream_networks_sp s
on a.blue_line_key = s.blue_line_key
AND ROUND(a.downstream_route_measure::numeric) >= ROUND(s.downstream_route_measure::numeric)
AND ROUND(a.downstream_route_measure::numeric) < ROUND(s.upstream_route_measure::numeric)
where a.barrier_type = 'DAM'
)

select * from cabd_pts
union all
select
dam_id,
linear_feature_id,
blue_line_key,
downstream_route_measure,
wscode_ltree,
localcode_ltree,
distance_to_stream,
watershed_group_code,
barrier_name as dam_name_en,
null as height_m,
null as owner,
null as dam_use,
null as operating_status,
null as passability_status_code,
geom
from usa;

create unique index on bcfishpass.dams_vw (dam_id);
create index on bcfishpass.dams_vw using gist (geom);

drop view if exists bcfishpass.dams_not_matched_to_streams;
create view bcfishpass.dams_not_matched_to_streams as
select
a.cabd_id,
a.dam_name_en
from cabd.dams a
left outer join bcfishpass.dams_vw b
on a.cabd_id::text = b.dam_id
where b.dam_id is null
order by a.cabd_id;
11 changes: 9 additions & 2 deletions model/01_access/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,16 @@ test:
# ------
# DAMS
# ------
.make/dams: dams/dams.sh dams/sql/dams.sql
.make/dams:
mkdir -p .make
cd dams; ./dams.sh
$(PSQL) -c "truncate cabd.dams"
ogr2ogr -f PostgreSQL \
"PG:$(DATABASE_URL)" \
-append \
-nln cabd.dams \
"https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \
OGRGeoJSON
$(PSQL) -c "refresh materialized view bcfishpass.dams_vw"
touch $@

# ------
Expand Down
42 changes: 0 additions & 42 deletions model/01_access/dams/README.md
View file

This file was deleted.

Loading

0 comments on commit 4773b48

Please sign in to comment.