Merge pull request #49 from TeoMeWhy/feat/ibge

Feat/ibge

databases/create.sql

@@ -1,7 +1,19 @@
 -- Databricks notebook source
--- DBTITLE 1,Olist
-CREATE DATABASE IF NOT EXISTS bronze.olist;
-CREATE DATABASE IF NOT EXISTS silver.olist;
+-- DBTITLE 1,DataSUS
+CREATE DATABASE IF NOT EXISTS bronze.datasus;
+CREATE DATABASE IF NOT EXISTS silver.datasus;
+
+-- COMMAND ----------
+
+-- DBTITLE 1,Dota
+CREATE DATABASE IF NOT EXISTS bronze.dota;
+CREATE DATABASE IF NOT EXISTS silver.dota;
+
+-- COMMAND ----------
+
+-- DBTITLE 1,IBGE
+CREATE DATABASE IF NOT EXISTS bronze.ibge;
+CREATE DATABASE IF NOT EXISTS silver.ibge;
 
 -- COMMAND ----------
 
@@ -17,12 +29,6 @@ CREATE DATABASE IF NOT EXISTS silver.pizza_query;
 
 -- COMMAND ----------
 
--- DBTITLE 1,DataSUS
-CREATE DATABASE IF NOT EXISTS bronze.datasus;
-CREATE DATABASE IF NOT EXISTS silver.datasus;
-
--- COMMAND ----------
-
--- DBTITLE 1,Dota
-CREATE DATABASE IF NOT EXISTS bronze.dota;
-CREATE DATABASE IF NOT EXISTS silver.dota;
+-- DBTITLE 1,Olist
+CREATE DATABASE IF NOT EXISTS bronze.olist;
+CREATE DATABASE IF NOT EXISTS silver.olist;

src/02.bronze/ibge/ingestao_municipios.py

@@ -0,0 +1,10 @@
+# Databricks notebook source
+path = "/mnt/datalake/ibge/municipios_brasileiros.csv"
+
+df = (spark.read
+           .csv(path, header=True, sep=";"))
+
+(df.write
+   .format("delta")
+   .mode("overwrite")
+   .saveAsTable("bronze.ibge.municipios_brasileiros"))

src/03.silver/ibge/municipios_brasileiros.sql

@@ -0,0 +1,9 @@
+-- Databricks notebook source
+DROP TABLE IF EXISTS silver.ibge.municipios_brasileiros;
+
+CREATE TABLE IF NOT EXISTS silver.ibge.municipios_brasileiros AS (
+
+  SELECT *
+  FROM bronze.ibge.municipios_brasileiros
+
+);