seeds update

first staging models

.gitignore

@@ -1,3 +1,4 @@
+workspace/
 .user.yml
 profiles.yml
 target/

.sqlfluff

@@ -0,0 +1,2 @@
+[sqlfluff]
+dialect = ansi

README.md

@@ -1,10 +1,48 @@
-Welcome to your new dbt project!
+### Format sql files
 
-### Using the starter project
+`pip install sqlfluff`
 
-Try running the following commands:
-- dbt run
-- dbt test
+```
+sqlfluff fix --dialect ansi ./models
+```
+
+
+### Setup the project's DB
+
+1. Locate the profiles.yaml File
+The profiles.yaml file is typically located in the .dbt directory in the user's home folder. If it doesn’t exist, the developer will need to create it. The location should be:
+
+On Unix-based systems (Linux, MacOS): ~/.dbt/profiles.yaml
+On Windows: %USERPROFILE%\.dbt\profiles.yaml
+
+2. Create or Modify the profiles.yaml File :
+
+```yaml
+crispy_dataprep:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      threads: 1 
+      host: hostname
+      port: 5432
+      user: username
+      pass: password
+      dbname: dbname
+      schema: schema_name
+```
+
+3. Initialize the data sources : 
+
+`dbt seed --full-refresh`
+
+### DBT commands
+
+```sh
+dbt run
+dbt test 
+dbt docs generate && dbt docs serve
+```
 
 
 ### Resources:

dbt_project.yml

@@ -33,5 +33,5 @@ clean-targets:         # directories to be removed by `dbt clean`
 models:
   crispy_dataprep:
     # Config indicated by + and applies to all files under models/example/
-    example:
+    staging:
       +materialized: view

models/staging/capacity_factors_data/staging_capfac_ipr.sql

@@ -0,0 +1,25 @@
+SELECT
+    variable_class,
+    sub_variable_class_1,
+    sub_variable_class_2,
+    scenario,
+    region,
+    sector,
+    units,
+    year,
+    value
+FROM {{ ref('raw_capacity_factors_IPR2021') }}
+
+UNION ALL
+
+SELECT
+    scenario,
+    region,
+    variable_class,
+    sub_variable_class_1,
+    sub_variable_class_2,
+    sector,
+    units,
+    year,
+    value
+FROM {{ ref('raw_capacity_factors_IPR2023') }}

models/staging/price_data_long_data/staging_price_ipr.sql

@@ -0,0 +1,25 @@
+SELECT
+    scenario,
+    region,
+    variable_class,
+    sub_variable_class_1,
+    NULL as sub_variable_class_2,  -- Placeholder for the extra column
+    NULL as sector,                -- Placeholder for the extra column    
+    units,
+    year,
+    value
+FROM {{ ref('raw_price_data_long_IPR2021') }}
+
+UNION ALL
+
+SELECT
+    scenario,
+    region,
+    variable_class,
+    sub_variable_class_1,
+    sub_variable_class_2,
+    sector,
+    units,
+    year,
+    value
+FROM {{ ref('raw_price_data_long_IPR2023') }}

models/staging/scenario_analysis_input_data/staging_scenario_ipr.sql

@@ -0,0 +1,25 @@
+SELECT
+    scenario,
+    region,
+    sector,
+    units,
+    variable_class,
+    sub_variable_class_1,
+    sub_variable_class_2,
+    year,
+    value
+FROM {{ ref('ipr_Scenarios_AnalysisInput') }}
+
+UNION ALL
+
+SELECT
+    scenario,
+    region,
+    variable_class,
+    sub_variable_class_1,
+    sub_variable_class_2,
+    sector,
+    units,
+    year,
+    value
+FROM {{ ref('ipr2023_Scenarios_AnalysisInput') }}

seeds/capacity_factors_data/preprocessed_capacity_factors_GEM_Steel.csv

@@ -1,8 +1,8 @@
-year,technology,value
+year,technology,value
 2021,Crude Steel ,0.592794894
 2021,BOF Steel,0.622848035
 2021,EAF Steel,0.636920856
 2021,OHF Steel,0.926829268
 2021,Iron,0.62425383
 2021,BF,0.675152043
-2021,DRI,0.461395189
+2021,DRI,0.461395189