Skip to content

Commit

Permalink
[Notebooks] Update hunting notebooks (#235)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhzhao8888
zhzhao8888 authored Sep 21, 2023
1 parent fe2e1ed commit 7873abb
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 63 deletions.
77 changes: 22 additions & 55 deletions ...rio-notebooks/Hunting-Notebooks/Hunting-AutomatedDataQueryAndIngestionToCustomTable.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,20 @@
"resource_group_name_source = \"\"\r\n",
"workspace_name_source = \"\"\r\n",
"workspace_id_source = \"\"\r\n",
"workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)"
"workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)\r\n"
],
"outputs": [],
"execution_count": null,
"metadata": {
"tags": [
"parameters"
]
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
Expand All @@ -87,7 +93,6 @@
"location = \"\"\r\n",
"workspace_name = ''\r\n",
"workspace_resource_id = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id, resource_group_name, workspace_name)\r\n",
"\r\n",
"data_collection_endpoint_name = \"\"\r\n",
"data_collection_rule_name = \"\"\r\n",
"custom_table_name = \"\"\r\n",
Expand Down Expand Up @@ -245,7 +250,7 @@
{
"cell_type": "code",
"source": [
"# Use Dror's test LA table\r\n",
"# Use test LA table, set 24 hours looking back\r\n",
"query_template = \"let t1 = SecurityAlert | extend ent = parse_json(Entities)| extend ip = tostring(ent[0]['Address']) | project-keep TimeGenerated, ip; let t2 = CommonSecurityLog | where TimeGenerated > ago({0}{1}) and TimeGenerated <= ago({2}{1}) | project ip = DestinationIP; t1 | join kind=innerunique t2 on ip\"\r\n",
"lookback_start = '24'\r\n",
"\r\n",
Expand All @@ -266,25 +271,6 @@
}
}
},
{
"cell_type": "code",
"source": [
"df_final"
],
"outputs": [],
"execution_count": null,
"metadata": {
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
"cell_type": "markdown",
"source": [
Expand All @@ -301,7 +287,7 @@
{
"cell_type": "code",
"source": [
"# Calling Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
"# Call Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
"# For different environments, such as national clouds, you may need to use different root_url, please contact with your admins.\r\n",
"# It can be ---.azure.us, ---.azure.microsoft.scloud, ---.azure.eaglex.ic.gov, etc.\r\n",
"def call_mdti_api_for_read(token, resource):\r\n",
Expand Down Expand Up @@ -347,9 +333,9 @@
{
"cell_type": "code",
"source": [
"# Calling MDTI API, hosts as example\r\n",
"#header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
"#response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
"# Call MDTI API, hosts as example\r\n",
"header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
"response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
],
"outputs": [],
"execution_count": null,
Expand All @@ -368,8 +354,9 @@
{
"cell_type": "code",
"source": [
"#sample_data = pd.read_json('abfs://modsynapsefiles/synapse/workspaces/modsynapse/mdti_host.json', typ='series').to_dict()\r\n",
"#df_host = pd.DataFrame(sample_data, index=[0])"
"# Merge data\r\n",
"df_final.loc[df_final['ip'].str.startswith('23.'), 'Fact'] = response_mdti_host.json()[\"registrar\"]\r\n",
"df_merged = df_final.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'IP', 'Fact': 'Fact'})[['TimeGenerated', 'IP', 'Fact']]"
],
"outputs": [],
"execution_count": null,
Expand All @@ -388,29 +375,7 @@
{
"cell_type": "code",
"source": [
"# Data process\r\n",
"#df_merged = pd.merge(df_final, df_host[['id','firstSeenDateTime','registrar']], left_on='ip', right_on='id', how=\"outer\")\r\n",
"#df_merged = df_merged.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'Url', 'registrar': 'Fact'})[['TimeGenerated', 'Url', 'Fact']]\r\n",
"#df_merged = df_merged.fillna(numpy.nan).replace([numpy.nan], [None])"
],
"outputs": [],
"execution_count": null,
"metadata": {
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
"cell_type": "code",
"source": [
"#df_merged = df_final"
"#df_merged"
],
"outputs": [],
"execution_count": null,
Expand Down Expand Up @@ -465,7 +430,9 @@
" index_block = len(df) // partition_number\r\n",
"\r\n",
" list_df = [df[i:i+index_block] for i in range(0,df.shape[0],index_block)]\r\n",
" return list_df"
" return list_df\r\n",
" else:\r\n",
" return [df]"
],
"outputs": [],
"execution_count": null,
Expand Down
78 changes: 70 additions & 8 deletions ...g-Notebooks/Scheduled_Hunting-AutomatedDataQueryAndMDTIAPIAndIngestionToCustomTable.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
"resource_group_name_source = \"\"\r\n",
"workspace_name_source = \"\"\r\n",
"workspace_id_source = \"\"\r\n",
"workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)"
"workspace_resource_id_source = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id_source, resource_group_name_source, workspace_name_source)\r\n"
],
"outputs": [],
"execution_count": null,
Expand All @@ -84,14 +84,13 @@
"cell_type": "code",
"source": [
"# User input for Log Analytics workspace for data ingestion\r\n",
"resource_group_name = \"\"\r\n",
"location = \"\"\r\n",
"workspace_name = \"\"\r\n",
"tenant_id = \"\"\r\n",
"subscription_id = \"\"\r\n",
"workspace_id = \"\"\r\n",
"resource_group_name = \"\"\r\n",
"location = \"\"\r\n",
"workspace_name = ''\r\n",
"workspace_resource_id = \"/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{2}\".format(subscription_id, resource_group_name, workspace_name)\r\n",
"\r\n",
"data_collection_endpoint_name = \"\"\r\n",
"data_collection_rule_name = \"\"\r\n",
"custom_table_name = \"\"\r\n",
Expand Down Expand Up @@ -249,7 +248,7 @@
{
"cell_type": "code",
"source": [
"# Use Dror's test LA table\r\n",
"# Use test LA table\r\n",
"query_template = \"let t1 = SecurityAlert | extend ent = parse_json(Entities)| extend ip = tostring(ent[0]['Address']) | project-keep TimeGenerated, ip; let t2 = CommonSecurityLog | where TimeGenerated > ago({0}{1}) and TimeGenerated <= ago({2}{1}) | project ip = DestinationIP; t1 | join kind=innerunique t2 on ip\"\r\n",
"lookback_start = '24'\r\n",
"\r\n",
Expand Down Expand Up @@ -286,7 +285,7 @@
{
"cell_type": "code",
"source": [
"# Calling Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
"# Call Microsoft MDTI API for List, the same template can be used for calling other Azure REST APIs with different parameters.\r\n",
"# For different environments, such as national clouds, you may need to use different root_url, please contact with your admins.\r\n",
"# It can be ---.azure.us, ---.azure.microsoft.scloud, ---.azure.eaglex.ic.gov, etc.\r\n",
"def call_mdti_api_for_read(token, resource):\r\n",
Expand Down Expand Up @@ -325,6 +324,67 @@
}
}
},
{
"cell_type": "code",
"source": [
"# Call MDTI API, hosts as example\r\n",
"header_token_value = \"Bearer {}\".format(get_token_for_graph())\r\n",
"response_mdti_host = call_mdti_api_for_read(header_token_value, \"hosts('www.microsoft.com')\")"
],
"outputs": [],
"execution_count": null,
"metadata": {
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
"cell_type": "code",
"source": [
"# Merge data\r\n",
"df_final.loc[df_final['ip'].str.startswith('23.'), 'Fact'] = response_mdti_host.json()[\"registrar\"]\r\n",
"df_merged = df_final.rename(columns = {'TimeGenerated': 'TimeGenerated', 'ip': 'IP', 'Fact': 'Fact'})[['TimeGenerated', 'IP', 'Fact']]"
],
"outputs": [],
"execution_count": null,
"metadata": {
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
"cell_type": "code",
"source": [
"#df_merged"
],
"outputs": [],
"execution_count": null,
"metadata": {
"jupyter": {
"source_hidden": false,
"outputs_hidden": false
},
"nteract": {
"transient": {
"deleting": false
}
}
}
},
{
"cell_type": "markdown",
"source": [
Expand Down Expand Up @@ -364,7 +424,9 @@
" index_block = len(df) // partition_number\r\n",
"\r\n",
" list_df = [df[i:i+index_block] for i in range(0,df.shape[0],index_block)]\r\n",
" return list_df"
" return list_df\r\n",
" else:\r\n",
" return [df]"
],
"outputs": [],
"execution_count": null,
Expand Down

0 comments on commit 7873abb

Please sign in to comment.