Skip to content

Commit

Permalink
create a procedure for each parquet datatype
Browse files Browse the repository at this point in the history
  • Loading branch information
philerooski committed Jul 22, 2024
1 parent 2202e0f commit 1d37ce9
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 8 deletions.
24 changes: 24 additions & 0 deletions snowflake/objects/database/recover/schema/parquet/deploy.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
/*
Create a parquet schema (if it doesn't yet exist) and deploy all child objects.
Jinja templating variables:
git_branch - The name of the git branch from which we are deploying.
*/
CREATE SCHEMA IF NOT EXISTS parquet;
USE SCHEMA parquet;
Expand All @@ -22,3 +25,24 @@ EXECUTE IMMEDIATE
);
EXECUTE IMMEDIATE
FROM './table/deploy.sql';
EXECUTE IMMEDIATE
$$
BEGIN
IF ('{{ git_branch }}' = 'main') THEN
-- Our procedures will reference the prod stage
EXECUTE IMMEDIATE
FROM './procedure/deploy.sql'
USING (
stage_name => $parquet_prod_stage_name,
file_format => $parquet_file_format_name
);
ELSE
EXECUTE IMMEDIATE
FROM './procedure/deploy.sql'
USING (
stage_name => $parquet_dev_stage_name,
file_format => $parquet_file_format_name
);
END IF;
END;
$$;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
A stored procedure which copies data from a named stage into a table.
A stored procedure which copies Parquet data from a named stage into a table.
Because of limitations in how we can pass variables to stage names,
this procedure is specific to a stage location. That is, we cannot
Expand All @@ -8,11 +8,12 @@
to use a specific stage location.
Jinja templating variables:
stage_name - The name of the stage where we copy data from
stage_path - The location within the stage where our data is
file_format - The name of the file format object used during copy
datatype - The datatype which our stage location refers to.
stage_name - The name of the stage where our data exists.
stage_path - The location within the stage where our data exists.
file_format - The name of the file format object used during copy.
*/
CREATE OR REPLACE PROCEDURE copy_into_table_from_stage(
CREATE OR REPLACE PROCEDURE copy_into_table_from_{{ datatype }}_parquet_stage(
target_table VARCHAR
)
RETURNS TABLE ()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,61 @@
/*
Deploy all PROCEDURE objects
This script does nothing (yet) because all of our procuedures are stage-specific.
See limitations of specifying variables in stage names in
`copy_into_table_from_stage.sql`.
Jinja templating variables:
stage_name - The name of the stage where our data exists.
file_format - The name of the file format object used by the
`copy_into_table_from_stage.sql` procedure.
*/

WITH AS PROCEDURE ()
RETURNS VARCHAR
LANGUAGE SQL
AS
$$
DECLARE
parquet_datatypes ARRAY := [
'enrolledparticipants_customfields_symptoms',
'enrolledparticipants_customfields_treatments',
'enrolledparticipants',
'fitbitactivitylogs',
'fitbitdailydata',
'fitbitdevices',
'fitbitecg',
'fitbitecg_waveformsamples',
'fitbitintradaycombined',
'fitbitrestingheartrates',
'fitbitsleeplogs',
'fitbitsleeplogs_sleeplogdetails',
'googlefitsamples',
'healthkitv2activitysummaries',
'healthkitv2electrocardiogram',
'healthkitv2electrocardiogram_subsamples',
'healthkitv2heartbeat',
'healthkitv2heartbeat_subsamples',
'healthkitv2samples',
'healthkitv2statistics',
'healthkitv2workouts_events',
'healthkitv2workouts',
'symptomlog',
'symptomlog_value_symptoms',
'symptomlog_value_treatments'
];
datatype VARCHAR;
dataset_name VARCHAR;
BEGIN
FOR i in 0 to array_size(:parquet_datatypes)-1 DO
datatype := GET(:parquet_datatypes, :i);
dataset_name := CONCAT('dataset_', :datatype);
-- Create a stored procedure which uses this data type's stage location
EXECUTE IMMEDIATE
FROM './copy_into_table_from_stage.sql'
USING (
datatype => :datatype,
stage_name => '{{ stage_name }}',
stage_path => :dataset_name,
file_format => '{{ file_format }}'
);
END FOR;
END;
$$
CALL create_procedure_for_each_parquet_table();

0 comments on commit 1d37ce9

Please sign in to comment.