From 3d0f214b403718d2693ebea8760f73f0cb16fbab Mon Sep 17 00:00:00 2001 From: DanKronstal-NSIT <133252144+DanKronstal-NSIT@users.noreply.github.com> Date: Fri, 10 May 2024 14:34:21 -0600 Subject: [PATCH] Notebook update to correct from gold to silver Current target is the Gold lakehouse, but it should be the Silver one, which is more appropriately used as source for this notebook. --- .../03 Silver to Gold layer_ Medallion Architecture.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabric/artifacts/fabricnotebooks/03 Silver to Gold layer_ Medallion Architecture.ipynb b/fabric/artifacts/fabricnotebooks/03 Silver to Gold layer_ Medallion Architecture.ipynb index 61f3136cef..9e648fa950 100644 --- a/fabric/artifacts/fabricnotebooks/03 Silver to Gold layer_ Medallion Architecture.ipynb +++ b/fabric/artifacts/fabricnotebooks/03 Silver to Gold layer_ Medallion Architecture.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"markdown","id":"686164a4-05d8-4c4f-b152-17f0ff438a78","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["# Transform data from Silver (cleansed and conformed data) to Gold 'curated business-level tables' layer\n","##### Once the data is available in open delta parquet format, we can do the transformations and aggregations on the data using Spark or sql compute. In this notebook we showcase how we reach the gold layer of data which is curated for direct consumption in the Power BI reports. \n","##### These gold layer data products can also be made available to other departments without the need to copy this data anywhere."]},{"cell_type":"markdown","id":"8d42e8ea-e272-49ac-a963-31cbc1818eab","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["![Medallion Architecture](https://fabricddib.blob.core.windows.net/notebookimage/MedallionArchitecture.png)"]},{"cell_type":"markdown","id":"13fa527b-3412-44ca-b76a-17b8ce453267","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Spark configurations"]},{"cell_type":"code","execution_count":4,"id":"9ec6a227-f9a7-42d6-88ff-596ae8d299df","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"6a405f37-77d9-44b3-b800-caf9f1e59a49","queued_time":"2023-09-11T20:17:26.5860144Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["spark.conf.set(\"sprk.sql.parquet.vorder.enabled\", \"true\")\n","spark.conf.set(\"spark.microsoft.delta.optimizeWrite.enabled\", \"true\")\n","spark.conf.set(\"spark.microsoft.delta.optimizeWrite.binSize\", \"1073741824\")"]},{"cell_type":"markdown","id":"22ed4d48-ad0b-4768-b119-c37ee3623e6e","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Load silver layer delta tables in Spark Dataframe\n","##### First we created shortcuts to the delta tables we created in the silver layer lakehouse earlier. Now those tables are directly mounted (synched) and available for the consumption here in the gold layer lakehouse. We can read them in Spark DataFrames, then perform transformation and aggregations operations. "]},{"cell_type":"code","execution_count":2,"id":"e37117ee-b5b9-4403-9a6e-c92e0eb3d143","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"1f757dd0-701d-47be-979f-f1fdfb1e26f4","queued_time":"2023-09-11T21:30:44.4989328Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["df_fact_sale = spark.read.table(\"#LAKEHOUSE_GOLD#.fact_sales\") \n","df_dimension_date = spark.read.table(\"#LAKEHOUSE_GOLD#.dimension_date\")\n","df_dimension_product = spark.read.table(\"#LAKEHOUSE_GOLD#.dimension_product\")\n","df_fact_campaigndata= spark.read.table(\"#LAKEHOUSE_GOLD#.fact_campaigndata\")"]},{"cell_type":"markdown","id":"c62a551d-5a3d-4a6c-8fc9-277c06595e7d","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Aggregated Table: Total Sales By Product\n","##### Here we are creating an aggregate table from the facts and dimension tables using the PySpark. We also have the option of using a SQL syntax which we will explore in the next cell."]},{"cell_type":"code","execution_count":3,"id":"102358da-5e5e-4e6a-99e0-00d000cdc4e1","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"afea9456-f2c4-4c47-87e8-90645e0f22b7","queued_time":"2023-09-11T20:21:51.3073779Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["sale_by_date_product = df_fact_sale.alias(\"sale\") \\\n",".join(df_dimension_date.alias(\"date\"), df_fact_sale.TransactionDate == df_dimension_date.DateValue, \"inner\") \\\n",".join(df_dimension_product.alias(\"product\"), df_fact_sale.ProductId == df_dimension_product.Products_ID, \"inner\") \\\n",".select(\"date.DateValue\", \"date.MonthName\",\"product.Name\", \"product.Category\", \"sale.TotalAmount\", \"sale.ProfitAmount\")\\\n",".groupBy(\"date.DateValue\", \"date.MonthName\", \"product.Name\", \"product.Category\")\\\n",".sum(\"sale.TotalAmount\", \"sale.ProfitAmount\")\\\n",".withColumnRenamed(\"sum(TotalAmount)\", \"SumOfTotalAmount\")\\\n",".withColumnRenamed(\"sum(ProfitAmount)\", \"SumOfProfit\")\\\n",".orderBy(\"date.DateValue\", \"product.Name\")\n","sale_by_date_product.write.mode(\"overwrite\").format(\"delta\").option(\"overwriteSchema\", \"true\").save(\"Tables/aggregate_sale_by_product\")"]},{"cell_type":"markdown","id":"2ae00b83-0696-422b-bdc0-4acac63d9599","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Aggregated Table: Total Campaign Revenue By Product Category"]},{"cell_type":"code","execution_count":4,"id":"de97bad2-efc4-4924-84f1-af821770ceef","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"138c1516-46c7-4f05-8ee2-a0feffa70ca6","queued_time":"2023-09-11T21:32:10.9724787Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["revenue_by_campaign_product = df_fact_campaigndata.alias(\"factcampdata\") \\\n",".join(df_dimension_product.alias(\"product\"), df_fact_campaigndata.ProductCategory == df_dimension_product.Category, \"inner\") \\\n",".select(\"product.Category\", \"factcampdata.Revenue\",\"factcampdata.Campaign_Name\", \"factcampdata.Revenue_Target\")\\\n",".groupBy( \"product.Category\",\"factcampdata.Campaign_Name\")\\\n",".sum(\"factcampdata.Revenue\", \"factcampdata.Revenue_Target\")\\\n",".withColumnRenamed(\"sum(Revenue_Target)\", \"TotalRevenueTarget\")\\\n",".withColumnRenamed(\"sum(Revenue)\", \"TotalRevenue\")\\\n",".withColumnRenamed(\"Category\", \"ProductCategory\")\\\n",".withColumnRenamed(\"Campaign_Name\", \"CampaignName\")\\\n",".orderBy(\"factcampdata.Campaign_Name\")\n","revenue_by_campaign_product.write.mode(\"overwrite\").format(\"delta\").option(\"overwriteSchema\", \"true\").saveAsTable(\"total_campaigns_revenue_by_product\")"]},{"cell_type":"markdown","id":"fabf7d83-fda7-4053-a09a-1d36bc2ec83b","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Temporary view\n","##### Here is an example of aggregation using the sql syntax. Based on the experties of your data citizen you can choose your own syntax and operate in parallel on thesame lakehouse with real time tracking feature that enables parallel collaboration and allows you to see where your collegues are at in the same notebook."]},{"cell_type":"code","execution_count":11,"id":"63a44dc7-6001-467d-a3a5-488f853314cd","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"sparksql"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"fde38a2f-df64-4b63-b1b2-54f64a5ba95b","queued_time":"2023-09-11T21:34:50.466958Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.synapse.sparksql-result+json":{"data":[],"schema":{"fields":[],"type":"struct"}},"text/plain":[""]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["%%sql\n","CREATE OR REPLACE Temporary VIEW total_sale_by_customer\n","AS\n","SELECT\n","\tDE.FirstName, DE.LastName\n","\t,SUM(FS.TotalAmount) SumOfTotalExcludingTax\n","\t,SUM(FS.ProfitAmount) SumOfTotalProfit\n","FROM _LAKEHOUSE_GOLD_.fact_sales FS\n","INNER JOIN _LAKEHOUSE_GOLD_.dimension_Customer DE ON FS.CustomerId = DE.Id\n","GROUP BY DE.FirstName, DE.LastName\n","ORDER BY DE.FirstName ASC\n"]},{"cell_type":"markdown","id":"d19cc9d2-b50b-4a0f-940f-c94d57d3e89a","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### View Results From Table 'total_sale_by_customer'"]},{"cell_type":"code","execution_count":null,"id":"cf212f1c-fa99-43a1-a49f-d89def8e6f0b","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"sparksql"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"3f8e1383-6b93-48ea-a1e1-0a7ae0b691a3","queued_time":"2023-09-11T21:34:56.772032Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.synapse.sparksql-result+json":{"data":[],"schema":{"fields":[{"metadata":{},"name":"FirstName","nullable":true,"type":"string"},{"metadata":{},"name":"LastName","nullable":true,"type":"string"},{"metadata":{},"name":"SumOfTotalExcludingTax","nullable":true,"type":"double"},{"metadata":{},"name":"SumOfTotalProfit","nullable":true,"type":"double"}],"type":"struct"}},"text/plain":[""]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["%%sql\n","SELECT * FROM total_sale_by_customer"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"host":{".message-hub":{"messages":[{"from":"0a248563-b244-40a6-a6c1-77363d5043b0","lts":603770,"payload":{},"seq":1,"type":"trident.lakehouse-changed"},{"from":"0a248563-b244-40a6-a6c1-77363d5043b0","lts":615791,"payload":{},"seq":2,"type":"trident.lakehouse-changed"}],"nextSeq":3},".virtual-root":{"trident":{"lakehouse":{"default_lakehouse":"e00e0b23-f168-4fd3-8594-55332e9c065e","default_lakehouse_name":"lakehouseGold","default_lakehouse_workspace_id":"a5271330-478b-4f25-b793-db6ec1971842","known_lakehouses":[{"id":"e00e0b23-f168-4fd3-8594-55332e9c065e"}]}}},"synapse_widget":{"state":{},"token":"0409ae3e-e0ef-4456-beac-667a61ce63b7"}},"language":"python","ms_spell_check":{"ms_spell_check_language":"en"}},"notebook_environment":{},"nteract":{"version":"nteract-front-end@1.0.0"},"save_output":true,"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{},"enableDebugMode":false}},"synapse_widget":{"state":{},"version":"0.1"},"trident":{"lakehouse":{"default_lakehouse":"e00e0b23-f168-4fd3-8594-55332e9c065e","default_lakehouse_name":"lakehouseGold","default_lakehouse_workspace_id":"a5271330-478b-4f25-b793-db6ec1971842","known_lakehouses":[{"id":"e00e0b23-f168-4fd3-8594-55332e9c065e"}]}},"widgets":{}},"nbformat":4,"nbformat_minor":5} +{"cells":[{"cell_type":"markdown","id":"686164a4-05d8-4c4f-b152-17f0ff438a78","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["# Transform data from Silver (cleansed and conformed data) to Gold 'curated business-level tables' layer\n","##### Once the data is available in open delta parquet format, we can do the transformations and aggregations on the data using Spark or sql compute. In this notebook we showcase how we reach the gold layer of data which is curated for direct consumption in the Power BI reports. \n","##### These gold layer data products can also be made available to other departments without the need to copy this data anywhere."]},{"cell_type":"markdown","id":"8d42e8ea-e272-49ac-a963-31cbc1818eab","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["![Medallion Architecture](https://fabricddib.blob.core.windows.net/notebookimage/MedallionArchitecture.png)"]},{"cell_type":"markdown","id":"13fa527b-3412-44ca-b76a-17b8ce453267","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Spark configurations"]},{"cell_type":"code","execution_count":4,"id":"9ec6a227-f9a7-42d6-88ff-596ae8d299df","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"6a405f37-77d9-44b3-b800-caf9f1e59a49","queued_time":"2023-09-11T20:17:26.5860144Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["spark.conf.set(\"sprk.sql.parquet.vorder.enabled\", \"true\")\n","spark.conf.set(\"spark.microsoft.delta.optimizeWrite.enabled\", \"true\")\n","spark.conf.set(\"spark.microsoft.delta.optimizeWrite.binSize\", \"1073741824\")"]},{"cell_type":"markdown","id":"22ed4d48-ad0b-4768-b119-c37ee3623e6e","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Load silver layer delta tables in Spark Dataframe\n","##### First we created shortcuts to the delta tables we created in the silver layer lakehouse earlier. Now those tables are directly mounted (synched) and available for the consumption here in the gold layer lakehouse. We can read them in Spark DataFrames, then perform transformation and aggregations operations. "]},{"cell_type":"code","execution_count":2,"id":"e37117ee-b5b9-4403-9a6e-c92e0eb3d143","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"1f757dd0-701d-47be-979f-f1fdfb1e26f4","queued_time":"2023-09-11T21:30:44.4989328Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["df_fact_sale = spark.read.table(\"#LAKEHOUSE_SILVER#.fact_sales\") \n","df_dimension_date = spark.read.table(\"#LAKEHOUSE_SILVER#.dimension_date\")\n","df_dimension_product = spark.read.table(\"#LAKEHOUSE_SILVER#.dimension_product\")\n","df_fact_campaigndata= spark.read.table(\"#LAKEHOUSE_SILVER#.fact_campaigndata\")"]},{"cell_type":"markdown","id":"c62a551d-5a3d-4a6c-8fc9-277c06595e7d","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Aggregated Table: Total Sales By Product\n","##### Here we are creating an aggregate table from the facts and dimension tables using the PySpark. We also have the option of using a SQL syntax which we will explore in the next cell."]},{"cell_type":"code","execution_count":3,"id":"102358da-5e5e-4e6a-99e0-00d000cdc4e1","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"afea9456-f2c4-4c47-87e8-90645e0f22b7","queued_time":"2023-09-11T20:21:51.3073779Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["sale_by_date_product = df_fact_sale.alias(\"sale\") \\\n",".join(df_dimension_date.alias(\"date\"), df_fact_sale.TransactionDate == df_dimension_date.DateValue, \"inner\") \\\n",".join(df_dimension_product.alias(\"product\"), df_fact_sale.ProductId == df_dimension_product.Products_ID, \"inner\") \\\n",".select(\"date.DateValue\", \"date.MonthName\",\"product.Name\", \"product.Category\", \"sale.TotalAmount\", \"sale.ProfitAmount\")\\\n",".groupBy(\"date.DateValue\", \"date.MonthName\", \"product.Name\", \"product.Category\")\\\n",".sum(\"sale.TotalAmount\", \"sale.ProfitAmount\")\\\n",".withColumnRenamed(\"sum(TotalAmount)\", \"SumOfTotalAmount\")\\\n",".withColumnRenamed(\"sum(ProfitAmount)\", \"SumOfProfit\")\\\n",".orderBy(\"date.DateValue\", \"product.Name\")\n","sale_by_date_product.write.mode(\"overwrite\").format(\"delta\").option(\"overwriteSchema\", \"true\").save(\"Tables/aggregate_sale_by_product\")"]},{"cell_type":"markdown","id":"2ae00b83-0696-422b-bdc0-4acac63d9599","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Aggregated Table: Total Campaign Revenue By Product Category"]},{"cell_type":"code","execution_count":4,"id":"de97bad2-efc4-4924-84f1-af821770ceef","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"138c1516-46c7-4f05-8ee2-a0feffa70ca6","queued_time":"2023-09-11T21:32:10.9724787Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"}],"source":["revenue_by_campaign_product = df_fact_campaigndata.alias(\"factcampdata\") \\\n",".join(df_dimension_product.alias(\"product\"), df_fact_campaigndata.ProductCategory == df_dimension_product.Category, \"inner\") \\\n",".select(\"product.Category\", \"factcampdata.Revenue\",\"factcampdata.Campaign_Name\", \"factcampdata.Revenue_Target\")\\\n",".groupBy( \"product.Category\",\"factcampdata.Campaign_Name\")\\\n",".sum(\"factcampdata.Revenue\", \"factcampdata.Revenue_Target\")\\\n",".withColumnRenamed(\"sum(Revenue_Target)\", \"TotalRevenueTarget\")\\\n",".withColumnRenamed(\"sum(Revenue)\", \"TotalRevenue\")\\\n",".withColumnRenamed(\"Category\", \"ProductCategory\")\\\n",".withColumnRenamed(\"Campaign_Name\", \"CampaignName\")\\\n",".orderBy(\"factcampdata.Campaign_Name\")\n","revenue_by_campaign_product.write.mode(\"overwrite\").format(\"delta\").option(\"overwriteSchema\", \"true\").saveAsTable(\"total_campaigns_revenue_by_product\")"]},{"cell_type":"markdown","id":"fabf7d83-fda7-4053-a09a-1d36bc2ec83b","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["## Temporary view\n","##### Here is an example of aggregation using the sql syntax. Based on the experties of your data citizen you can choose your own syntax and operate in parallel on thesame lakehouse with real time tracking feature that enables parallel collaboration and allows you to see where your collegues are at in the same notebook."]},{"cell_type":"code","execution_count":11,"id":"63a44dc7-6001-467d-a3a5-488f853314cd","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"sparksql"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"fde38a2f-df64-4b63-b1b2-54f64a5ba95b","queued_time":"2023-09-11T21:34:50.466958Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.synapse.sparksql-result+json":{"data":[],"schema":{"fields":[],"type":"struct"}},"text/plain":[""]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["%%sql\n","CREATE OR REPLACE Temporary VIEW total_sale_by_customer\n","AS\n","SELECT\n","\tDE.FirstName, DE.LastName\n","\t,SUM(FS.TotalAmount) SumOfTotalExcludingTax\n","\t,SUM(FS.ProfitAmount) SumOfTotalProfit\n","FROM _LAKEHOUSE_SILVER_.fact_sales FS\n","INNER JOIN _LAKEHOUSE_SILVER_.dimension_Customer DE ON FS.CustomerId = DE.Id\n","GROUP BY DE.FirstName, DE.LastName\n","ORDER BY DE.FirstName ASC\n"]},{"cell_type":"markdown","id":"d19cc9d2-b50b-4a0f-940f-c94d57d3e89a","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### View Results From Table 'total_sale_by_customer'"]},{"cell_type":"code","execution_count":null,"id":"cf212f1c-fa99-43a1-a49f-d89def8e6f0b","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"sparksql"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":null,"execution_start_time":null,"livy_statement_state":null,"parent_msg_id":"3f8e1383-6b93-48ea-a1e1-0a7ae0b691a3","queued_time":"2023-09-11T21:34:56.772032Z","session_id":null,"session_start_time":null,"spark_jobs":null,"spark_pool":null,"state":"waiting","statement_id":null},"text/plain":["StatementMeta(, , , Waiting, )"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.synapse.sparksql-result+json":{"data":[],"schema":{"fields":[{"metadata":{},"name":"FirstName","nullable":true,"type":"string"},{"metadata":{},"name":"LastName","nullable":true,"type":"string"},{"metadata":{},"name":"SumOfTotalExcludingTax","nullable":true,"type":"double"},{"metadata":{},"name":"SumOfTotalProfit","nullable":true,"type":"double"}],"type":"struct"}},"text/plain":[""]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["%%sql\n","SELECT * FROM total_sale_by_customer"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"host":{".message-hub":{"messages":[{"from":"0a248563-b244-40a6-a6c1-77363d5043b0","lts":603770,"payload":{},"seq":1,"type":"trident.lakehouse-changed"},{"from":"0a248563-b244-40a6-a6c1-77363d5043b0","lts":615791,"payload":{},"seq":2,"type":"trident.lakehouse-changed"}],"nextSeq":3},".virtual-root":{"trident":{"lakehouse":{"default_lakehouse":"e00e0b23-f168-4fd3-8594-55332e9c065e","default_lakehouse_name":"lakehouseGold","default_lakehouse_workspace_id":"a5271330-478b-4f25-b793-db6ec1971842","known_lakehouses":[{"id":"e00e0b23-f168-4fd3-8594-55332e9c065e"}]}}},"synapse_widget":{"state":{},"token":"0409ae3e-e0ef-4456-beac-667a61ce63b7"}},"language":"python","ms_spell_check":{"ms_spell_check_language":"en"}},"notebook_environment":{},"nteract":{"version":"nteract-front-end@1.0.0"},"save_output":true,"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{},"enableDebugMode":false}},"synapse_widget":{"state":{},"version":"0.1"},"trident":{"lakehouse":{"default_lakehouse":"e00e0b23-f168-4fd3-8594-55332e9c065e","default_lakehouse_name":"lakehouseGold","default_lakehouse_workspace_id":"a5271330-478b-4f25-b793-db6ec1971842","known_lakehouses":[{"id":"e00e0b23-f168-4fd3-8594-55332e9c065e"}]}},"widgets":{}},"nbformat":4,"nbformat_minor":5}