generate_readme.nbconvert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "latter-teddy",
   "metadata": {},
   "source": [
    "\n",
    "# Generate Readme with up to date list of xRunBooks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "421e4c6e-d5ef-4d53-9d36-f352426c4d87",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Input"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e84f7e80-dda2-4569-96dd-5abaaed2c73a",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "sitting-directory",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:20.966275Z",
     "iopub.status.busy": "2023-05-12T17:04:20.965745Z",
     "iopub.status.idle": "2023-05-12T17:04:23.825994Z",
     "shell.execute_reply": "2023-05-12T17:04:23.824909Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\r\n",
      "Collecting GitPython\r\n",
      "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 KB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hCollecting gitdb<5,>=4.0.1\r\n",
      "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 KB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hCollecting smmap<6,>=3.0.1\r\n",
      "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\r\n",
      "Installing collected packages: smmap, gitdb, GitPython\r\n",
      "Successfully installed GitPython-3.1.31 gitdb-4.0.10 smmap-5.0.0\r\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import requests\n",
    "import urllib.parse\n",
    "import copy\n",
    "from pathlib import Path\n",
    "import markdown\n",
    "import nbformat\n",
    "from nbconvert import MarkdownExporter\n",
    "from papermill.iorw import (\n",
    "    load_notebook_node,\n",
    "    write_ipynb,\n",
    ")\n",
    "try:\n",
    "    from git import Repo\n",
    "except:\n",
    "    !pip install GitPython\n",
    "    from git import Repo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "agricultural-contest",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "guided-edgar",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:23.830598Z",
     "iopub.status.busy": "2023-05-12T17:04:23.829762Z",
     "iopub.status.idle": "2023-05-12T17:04:23.836565Z",
     "shell.execute_reply": "2023-05-12T17:04:23.835758Z"
    },
    "tags": []
   },
   "output": {},
   "outputs": [],
   "source": [
    "# README variables\n",
    "readme_template = \"README_template.md\"\n",
    "readme = \"README.md\"\n",
    "replace_var = \"[[DYNAMIC_LIST]]\"\n",
    "badge_var = \"[[BADGE]]\"\n",
    "\n",
    "# welcome variables\n",
    "#this is a TODO\n",
    "#welcome_template = \"Welcome_template.ipynb\"\n",
    "#welcome = \"Welcome.ipynb\"\n",
    "#replace_var_quote = f'\"[[DYNAMIC_LIST]]\",\\n'\n",
    "\n",
    "\n",
    "# Others\n",
    "current_file = '.'\n",
    "notebook_ext = '.ipynb'\n",
    "github_url_base = 'https://github.com/unskript/Awesome-CloudOps-Automation/tree/master'\n",
    "local_OSS_url = 'http://127.0.0.1:8888/lab/tree'\n",
    "#fix these!\n",
    "github_download_url = 'https://raw.githubusercontent.com/unskript/Awesome-CloudOps-Automation/master/'\n",
    "unSkript_logo ='https://storage.googleapis.com/unskript-website/assets/favicon.png'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-declaration",
   "metadata": {},
   "source": [
    "### Get files list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36c9011e-5f51-4779-8062-a627503100e1",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:23.840663Z",
     "iopub.status.busy": "2023-05-12T17:04:23.840285Z",
     "iopub.status.idle": "2023-05-12T17:04:24.399232Z",
     "shell.execute_reply": "2023-05-12T17:04:24.398348Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChatGPT Get Handle is missing categories\n",
      "Get Datadog Handle is missing categories\n",
      "Get GCP Handle is missing categories\n",
      "Get Github Handle is missing categories\n",
      "Get Grafana Handle is missing categories\n",
      "Get Jenkins Handle is missing categories\n",
      "Get Jira SDK Handle is missing categories\n",
      "Get Kafka Producer Handle is missing categories\n",
      "Get Kubernetes Handle is missing categories\n",
      "Get MongoDB Handle is missing categories\n",
      "Get MS-SQL Handle is missing categories\n",
      "Get MySQL Handle is missing categories\n",
      "Netbox Get Handle is missing categories\n",
      "Nomad Get Handle is missing categories\n",
      "Get Pingdom Handle is missing categories\n",
      "Get PostgreSQL Handle is missing categories\n",
      "Get Prometheus handle is missing categories\n",
      "Get Redis Handle is missing categories\n",
      "Get REST handle is missing categories\n",
      "Get SSH handle is missing categories\n",
      "Get Slack SDK Handle is missing categories\n",
      "Get Stripe Handle is missing categories\n",
      "Get terraform handle is missing categories\n",
      "487\n"
     ]
    }
   ],
   "source": [
    "repo = Repo('.')\n",
    "branch =  repo.active_branch\n",
    "list_of_dir = f\"https://api.github.com/repos/unskript/Awesome-CloudOps-Automation/git/trees/{branch}?recursive=1\"\n",
    "r_gh = requests.get(list_of_dir).json().get(\"tree\")\n",
    "#print(branch)\n",
    "notebooks = []\n",
    "actions=[]\n",
    "actionCount = 0\n",
    "runBookCount = 0\n",
    "\n",
    "main_readme_chart = \"\"\n",
    "runbook_list=\"\"\n",
    "runbook_connector = \"\"\n",
    "runbook_connector_list = []\n",
    "\n",
    "action_list = \"\"\n",
    "action_connector = \"\"\n",
    "action_connector_list = []\n",
    "\n",
    "connector_readme_connector =''\n",
    "connector_readme = ''\n",
    "hasRunbook = False\n",
    "hasAction = False\n",
    "\n",
    "if r_gh is not None:\n",
    "    for file in r_gh:\n",
    "\n",
    "        #only look at files we care about - ignore some of the directories that we don't need to scan\n",
    "        if \".github\" not in file.get(\"path\") and \".gitignore\" not in file.get(\"path\") and \"templates\" not in file.get(\"path\") and \"/\" in file.get(\"path\")and \"__init__\" not in file.get(\"path\") and \"custom\" not in file.get(\"path\")and \"unskript-ctl\" not in file.get(\"path\"):\n",
    "            #runbooks are .ipynb, and actions are .py\n",
    "            if file.get(\"path\").endswith(\".ipynb\") or file.get(\"path\").endswith(\".py\"):\n",
    "                temp = file.get(\"path\").split(\"/\")\n",
    "                if temp == -1:\n",
    "                    data = {\n",
    "                        \"root\": None,\n",
    "                        \"filename\": file.get(\"path\")\n",
    "                    }\n",
    "                    notebooks.append(data)\n",
    "                    \n",
    "                else:\n",
    "                    isAction = False\n",
    "                    last_folder = \"\"\n",
    "                    file_name = temp[-1]\n",
    "                    filename_string = file_name[0:file_name.find(\".\")]\n",
    "\n",
    "                    temp.pop()\n",
    "                    path = \"\"\n",
    "                    for folder in temp:\n",
    "                        path = path + folder +\"/\"\n",
    "                        last_folder += \"/\" + folder\n",
    "                    \n",
    "                    # to be an action, the file must have the 2nd directory be lego, and there must be 3 layers of directory\n",
    "                    if len(temp) ==3 and temp[1] ==\"legos\":\n",
    "                        isAction = True\n",
    "                    #testing\n",
    "                    \n",
    "                    \n",
    "                    \n",
    "                    #JSON data\n",
    "                    filename_json = Path(path +\"/\"+filename_string+ \".json\")\n",
    "                    jsonData = json.loads(filename_json.read_text())\n",
    "                    \n",
    "                    \n",
    "                    ##we now have a path.. but only really need the root folder\n",
    "                    ## different ways to generate fior action vs runbook\n",
    "                    if isAction:\n",
    "                        actionCount += 1\n",
    "                        #this is an action folder\n",
    "                        #find first slash\n",
    "                        firstslash = last_folder.find(\"/\",1)\n",
    "                        root = last_folder[1:firstslash]\n",
    "                        name = jsonData['action_title']\n",
    "                        description = jsonData['action_description']\n",
    "                        if 'action_categories' in jsonData:\n",
    "                            categories = jsonData['action_categories']\n",
    "                        else:\n",
    "                            print(f\"{name} is missing categories\")\n",
    "                        polling = jsonData['action_supports_poll']\n",
    "                        iteration = jsonData['action_supports_iteration']\n",
    "                        #not the python file - but the readme\n",
    "                        github_url = f\"{github_url_base}{last_folder}/README.md\"   \n",
    "                        \n",
    "                        \n",
    "                    else:\n",
    "                        runBookCount+=1\n",
    "                        #root folder for notebooks\n",
    "                        root = last_folder[1:]\n",
    "                        name = jsonData['name']\n",
    "                        description = jsonData['description']\n",
    "                        categories = jsonData['categories']\n",
    "                        github_url = github_url_base+\"/\"+file.get(\"path\")\n",
    "                    \n",
    "                    data = {\n",
    "                        \"root\": root,\n",
    "                        \"filename\": file_name,\n",
    "                        \"name\": name,\n",
    "                        \"description\": description,\n",
    "                        \"categories\":categories,\n",
    "                        \"github_url\": github_url\n",
    "                    }\n",
    "                    \n",
    "                    if isAction:\n",
    "                        data['type'] = \"Action\"\n",
    "                        data['polling'] = polling\n",
    "                        data['iteration']=iteration\n",
    "                        actions.append(data)\n",
    "                    else:\n",
    "                        data['type'] = \"RunBook\"\n",
    "                        local_url = local_OSS_url+\"/\"+file.get(\"path\")\n",
    "                        data['local_url'] = local_url  \n",
    "                        notebooks.append(data)\n",
    "                        \n",
    "                        \n",
    "                    #generate the list of runbooks for tha main readme\n",
    "                    if not isAction:\n",
    "                        main_readme_chart += f\"|{root} |[{name}]({github_url}) | [Open in Browser]({local_url}) | \\n\"\n",
    "    \n",
    "                    #generate the runbook list page\n",
    "                    if not isAction:\n",
    "                        #have we created a category yet?\n",
    "                        if runbook_connector == \"\":\n",
    "                            runbook_connector = root\n",
    "                            runbook_connector_list.append(runbook_connector)\n",
    "                            runbook_list += f\"\\n# {runbook_connector}\\n\"\n",
    "                        #same category, or new one\n",
    "                        if runbook_connector != root:\n",
    "                            # new category\n",
    "                            runbook_connector = root\n",
    "                            runbook_connector_list.append(runbook_connector)\n",
    "                            runbook_list += f\"\\n# {runbook_connector}\\n\"\n",
    "                            \n",
    "                            \n",
    "                        #now add in each runbook\n",
    "                        runbook_list += f\"* [{name}]({github_url}): {description}\\n\"\n",
    "            \n",
    "                    #generate the action list page\n",
    "                    if isAction:\n",
    "                        #have we created a category yet?\n",
    "                        if action_connector == \"\":\n",
    "                            action_connector = root\n",
    "                            action_connector_list.append(action_connector)\n",
    "                            action_list += f\"\\n# {action_connector}\\n\"\n",
    "                        #same category, or new one\n",
    "                        if action_connector != root:\n",
    "                            # new category\n",
    "                            action_connector = root\n",
    "                            action_connector_list.append(action_connector)\n",
    "                            action_list += f\"\\n# {action_connector}\\n\"\n",
    "                            \n",
    "                        #now add in each Action\n",
    "                        action_list += f\"* [{name}]({github_url}): {description}\\n\"\n",
    "                    \n",
    "                    \n",
    "\n",
    "                    #generate the readme for each connector\n",
    "                    #have we created a category yet?\n",
    "                    if connector_readme_connector == \"\":\n",
    "                        connector_readme_connector = root\n",
    "                        connector_readme = ''\n",
    "                        hasRunbook = False\n",
    "                        hasAction = False\n",
    "                    #same category, or new one\n",
    "                    if connector_readme_connector != root:\n",
    "                        # starting a new readme\n",
    "                        #first let's save the old one:\n",
    "                        #print(connector_readme)\n",
    "                        readme_file = f\"{connector_readme_connector}/README.md\"\n",
    "                        f  = open(readme_file, \"w+\")\n",
    "                        f.write(connector_readme)\n",
    "                        f.close()\n",
    "                        #now start building the new readme\n",
    "                        connector_readme_connector = root\n",
    "                        connector_readme = ''\n",
    "                        hasRunbook = False\n",
    "                        hasAction = False\n",
    "\n",
    "                    if data['type'] ==\"RunBook\":\n",
    "                        if not hasRunbook:\n",
    "                            connector_readme += f'# {root} RunBooks\\n'\n",
    "                            hasRunbook = True\n",
    "                        connector_readme += f\"* [{name}]({github_url}): {description}\\n\"\n",
    "                    if data['type'] ==\"Action\":\n",
    "                        if not hasAction:\n",
    "                            connector_readme += f'\\n# {root} Actions\\n'\n",
    "                            hasAction = True\n",
    "                        connector_readme += f\"* [{name}]({github_url}): {description}\\n\"\n",
    "    \n",
    "print(actionCount)\n",
    "#print(action_list)\n",
    "#print(action_connector_list)\n",
    "#print(runbook_connector_list)\n",
    "#print(runBookCount, actionCount)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e492845d",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.407934Z",
     "iopub.status.busy": "2023-05-12T17:04:24.407433Z",
     "iopub.status.idle": "2023-05-12T17:04:24.414946Z",
     "shell.execute_reply": "2023-05-12T17:04:24.414227Z"
    }
   },
   "outputs": [],
   "source": [
    "## generate category list and category url list for runbooks\n",
    "\n",
    "notebook_categories = {}\n",
    "notebook_category_urls = {}\n",
    "\n",
    "\n",
    "for notebook in notebooks:\n",
    "    #print(notebook)\n",
    "    #print(notebook['root'], notebook['name'])\n",
    "    if len(notebook['categories'])>0:\n",
    "        #print(notebook['categories'])\n",
    "        for category in notebook['categories']:\n",
    "            if not category in notebook_categories:\n",
    "                notebook_categories[category] = []\n",
    "            notebook_categories[category].append(notebook)\n",
    "            category_name = category[14:]\n",
    "            notebook_category_urls[category_name] = Path(f\"runbook_{category_name}.md\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "68d91e36",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.419947Z",
     "iopub.status.busy": "2023-05-12T17:04:24.418609Z",
     "iopub.status.idle": "2023-05-12T17:04:24.426325Z",
     "shell.execute_reply": "2023-05-12T17:04:24.425696Z"
    }
   },
   "outputs": [],
   "source": [
    "##headers for al of the runbook pages \n",
    "category_header1 =\"|\"\n",
    "category_header2 =\"|\"\n",
    "category_table =\"|\"\n",
    "\n",
    "connector_header1 =\"|\"\n",
    "connector_header2 =\"|\"\n",
    "connector_table =\"|\"\n",
    "counter =0\n",
    "for connector in runbook_connector_list:\n",
    "    counter+= 1\n",
    "    if counter <3:\n",
    "        connector_header1 += f\" |\"\n",
    "        connector_header2 += f\" ---|\"\n",
    "    connector_table += f\" [{connector}](xRunBook_List.md#{connector}) |\"      \n",
    "    if counter%3 ==0:\n",
    "        #start a new row of category\n",
    "        connector_table += f\"\\n |\"\n",
    "        \n",
    "connector_markdown_table = f\"{connector_header1} \\n {connector_header2} \\n {connector_table} \\n\\n\"\n",
    "\n",
    "#print(notebook_category_urls)\n",
    "counter =0\n",
    "for categoryname in notebook_category_urls:\n",
    "    counter+= 1\n",
    "    if counter <3:\n",
    "        category_header1 += f\" |\"\n",
    "        category_header2 += f\" ---|\"\n",
    "    category_table += f\" [{categoryname}]({notebook_category_urls[categoryname]}) |\"\n",
    "    if counter%3 ==0:\n",
    "        #start a new row of category\n",
    "        category_table += f\"\\n |\"    \n",
    "category_markdown_table = f\"{category_header1} \\n {category_header2} \\n {category_table} \\n\"\n",
    "#this builds the Runbook list page\n",
    "#lets try it without the categories.  There is \"too much\" category... not enough runbook on ther pages\n",
    "#runbook_list = f\"# RunBook Connectors:\\n {connector_markdown_table} \\n# RunBook Categories:\\n {category_markdown_table} \\n\\n {runbook_list}\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6bfed614",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.431206Z",
     "iopub.status.busy": "2023-05-12T17:04:24.429886Z",
     "iopub.status.idle": "2023-05-12T17:04:24.443003Z",
     "shell.execute_reply": "2023-05-12T17:04:24.442338Z"
    }
   },
   "outputs": [],
   "source": [
    "## generate category lists from actions\n",
    "\n",
    "action_categories = {}\n",
    "action_category_urls = {}\n",
    "\n",
    "\n",
    "for action in actions:\n",
    "    #print(action)\n",
    "    if len(action['categories'])>0:\n",
    "        #print(notebook['categories'])\n",
    "        for category in action['categories']:\n",
    "            if not category in action_categories:\n",
    "                action_categories[category] = []\n",
    "            action_categories[category].append(action)\n",
    "            category_name = category[14:]\n",
    "            action_category_urls[category_name] = Path(f\"action_{category_name}.md\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d25256b7",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.447974Z",
     "iopub.status.busy": "2023-05-12T17:04:24.446686Z",
     "iopub.status.idle": "2023-05-12T17:04:24.456213Z",
     "shell.execute_reply": "2023-05-12T17:04:24.455604Z"
    }
   },
   "outputs": [],
   "source": [
    "##headers for al of the Action pages \n",
    "category_header1 =\"|\"\n",
    "category_header2 =\"|\"\n",
    "category_table =\"|\"\n",
    "\n",
    "action_header1 =\"|\"\n",
    "action_header2 =\"|\"\n",
    "action_table =\"|\"\n",
    "gitbooklist = \"\"\n",
    "counter =0\n",
    "for action in action_connector_list:\n",
    "    counter+= 1\n",
    "    if counter <3:\n",
    "        action_header1 += f\" |\"\n",
    "        action_header2 += f\" ---|\"\n",
    "    action_url = f\"action_{action.upper()}.md\"\n",
    "    action_url= action_url.replace(\"KUBERNETES\", \"K8S\")\n",
    "    action_url = action_url.replace(\"_MONGO.\", \"_MONGODB.\")\n",
    "    action_url= action_url.replace(\"action_ELASTICSEARCH\", \"action_ES\")\n",
    "    action_table += f\" [{action}]({action_url}) |\"\n",
    "    #print(action_url)\n",
    "    gitbooklist += f\"      * [{action}](action_url) \\n\"\n",
    "    if counter%3 ==0:\n",
    "        #start a new row of category\n",
    "        action_table += f\"\\n |\"\n",
    "\n",
    "action_connector_markdown_table = f\"{action_header1} \\n {action_header2} \\n {action_table} \\n\\n\"\n",
    "#print(action_connector_markdown_table)\n",
    "counter = 0\n",
    "for categoryname in action_category_urls:\n",
    "    category_printed = categoryname\n",
    "    counter+= 1\n",
    "    if counter <3:\n",
    "        category_header1 += f\" |\"\n",
    "        category_header2 += f\" ---|\"\n",
    "    category_url = str(action_category_urls[categoryname])\n",
    "    category_url = category_url.replace(\"KUBERNETES\", \"K8S\")\n",
    "    action_url = action_url.replace(\"_MONGO.\", \"_MONGODB.\")\n",
    "    #category_url = category_url.replace(\"POSTGRES\", \"POSTGRESQL\")\n",
    "    category_table += f\" [{category_printed}]({category_url}) |\"\n",
    "    gitbooklist += f\"      * [{category_printed}](lists/{action_category_urls[categoryname]})\\n\"\n",
    "    if counter%3 ==0:\n",
    "        #start a new row of category\n",
    "        category_table += f\"\\n |\"\n",
    "\n",
    "action_category_markdown_table = f\"{category_header1} \\n {category_header2} \\n {category_table} \\n\"\n",
    "\n",
    "\n",
    "#print(action_connector_markdown_table)\n",
    "action_list =f\"# Actions By Connector:\\n{action_connector_markdown_table} \\n # Actions By Category: \\n{action_category_markdown_table}\\n\\n\\n\\n \"   \n",
    "#print(action_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4f238d53",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.461018Z",
     "iopub.status.busy": "2023-05-12T17:04:24.459745Z",
     "iopub.status.idle": "2023-05-12T17:04:24.472692Z",
     "shell.execute_reply": "2023-05-12T17:04:24.472044Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "#generate pages for each Runbook category page\n",
    "category_name =\"\"\n",
    "category_listing = \"\"\n",
    "for category in notebook_categories:\n",
    "    #category change - we need a new header\n",
    "    if category_name ==\"\":\n",
    "        #new category\n",
    "        category_name = category[14:]\n",
    "        category_listing = f\"# RunBook Connectors:\\n {connector_markdown_table} \\n# RunBook Categories:\\n {category_markdown_table} \\n# Runbooks in {category_name.replace('_', ' ')}\\n\"\n",
    "        #all of the categories was too much. Pulling this for now\n",
    "        category_listing = \"\"\n",
    "    elif category_name != category:\n",
    "        # we have finished off a category\n",
    "        #save the oldcategory\n",
    "        #print(category_name, category_listing)\n",
    "        categoryList_filename = f\"lists/{notebook_category_urls[category_name]}\"\n",
    "        f  = open(categoryList_filename, \"w+\")\n",
    "        f.write(category_listing)\n",
    "        f.close()\n",
    "        category_name = category[14:]\n",
    "        category_listing = f\"# RunBook Connectors:\\n {connector_markdown_table} \\n# RunBook Categories:\\n {category_markdown_table}\\n # Runbooks in {category_name.replace('_', ' ')}\\n\"\n",
    "        #all of the categories was too much. Pulling this for now\n",
    "        category_listing = \"\"\n",
    "    #print(notebook_categories[category])\n",
    "    for runbook in notebook_categories[category]:\n",
    "        category_listing += f\"* {runbook['root']} [{runbook['name']}]({runbook['github_url']}): {runbook['description']}\\n\"\n",
    "#finished loop -wrote last category\n",
    "#print(category_listing)\n",
    "f  = open(categoryList_filename, \"w+\")\n",
    "f.write(category_listing)\n",
    "f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "81ea848a",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.477765Z",
     "iopub.status.busy": "2023-05-12T17:04:24.476494Z",
     "iopub.status.idle": "2023-05-12T17:04:24.529900Z",
     "shell.execute_reply": "2023-05-12T17:04:24.529091Z"
    }
   },
   "outputs": [],
   "source": [
    "# generate all of the action category pages \n",
    "category_name =\"\"\n",
    "category_listing = \"\"\n",
    "for category in action_categories:\n",
    "    if category_name ==\"\":\n",
    "        #new category\n",
    "        \n",
    "        category_name = category[14:]\n",
    "        if category_name ==\"ELASTICSEARCH\":\n",
    "            category_name ==\"ES\"\n",
    "        elif category_name ==\"KUBERNETES\":\n",
    "            category_name ==\"K8S\"\n",
    "        \n",
    "        elif category_name ==\"MONGO\":\n",
    "            print(\"MONGO\")\n",
    "            category_name ==\"MONGODB\"\n",
    "        category_listing = f\"# Actions in the {category_name.replace('_', ' ')} category\\n\"\n",
    "        #all of the categories was too much. Pulling this for now\n",
    "        category_listing = \"\"        \n",
    "    elif category_name != category:\n",
    "        # we have finished off a category\n",
    "        #save the oldcategory\n",
    "        #print(category_name, category_listing)\n",
    "        #print(category)\n",
    "        #print(category_listing)\n",
    "        #place the links at the bottom\n",
    "        #category_listing += f\"\\n# Actions By Connector:\\n{action_connector_markdown_table} \\n # Actions By Category: \\n{action_category_markdown_table} \\n\\n\"\n",
    "        categoryList_filename = f\"lists/{action_category_urls[category_name]}\"\n",
    "        f  = open(categoryList_filename, \"w+\")\n",
    "        f.write(category_listing)\n",
    "        f.close()\n",
    "        category_name = category[14:]\n",
    "        if category_name ==\"ELASTICSEARCH\":\n",
    "            category_name ==\"ES\"\n",
    "        elif category_name ==\"KUBERNETES\":\n",
    "            category_name ==\"K8S\"\n",
    "        elif category_name ==\"MONGO\":\n",
    "            print(\"mongo\")\n",
    "            category_name ==\"MONGODB\"\n",
    "        category_listing = \"\" #f\"# Actions in the {category_name.replace('_', ' ')} category\\n\"\n",
    "    \n",
    "    #print(category_listing)\n",
    "    for action in action_categories[category]:\n",
    "        # removing **{action['root']}**: from each listing\n",
    "        category_listing += f\"* [{action['name']}]({action['github_url']}): {action['description']}\\n\\n\"\n",
    "# last category is compelted when loop ends\n",
    "\n",
    "#category_listing += f\"\\n# Actions By Connector:\\n{action_connector_markdown_table} \\n # Actions By Category: \\n{action_category_markdown_table} \\n\\n\"\n",
    "categoryList_filename = f\"lists/{action_category_urls[category_name]}\"\n",
    "f  = open(categoryList_filename, \"w+\")\n",
    "f.write(category_listing)\n",
    "f.close()\n",
    "        \n",
    "        \n",
    "#print(action_categories)\n",
    "#print(action_category_urls)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aboriginal-responsibility",
   "metadata": {},
   "source": [
    "### Preview the generated list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88e075a7-8341-45af-a250-c1594c004579",
   "metadata": {},
   "source": [
    "### Generate readme for github repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "younger-consensus",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.535754Z",
     "iopub.status.busy": "2023-05-12T17:04:24.534275Z",
     "iopub.status.idle": "2023-05-12T17:04:24.541944Z",
     "shell.execute_reply": "2023-05-12T17:04:24.541322Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Open README template\n",
    "template = open(readme_template).read()\n",
    "\n",
    "# Replace var to get list of templates in markdown format\n",
    "template = template.replace(replace_var, main_readme_chart)\n",
    "\n",
    "#create the action and runbook badges\n",
    "actionImage = f'https://img.shields.io/static/v1?label=ActionCount&message={actionCount}&color=green'\n",
    "runbookImage = f'https://img.shields.io/static/v1?label=xRunBookCount&message={runBookCount}&color=orange'\n",
    "actionbadge = f\"<img src={actionImage}>\"\n",
    "runbookbadge =  f\"<img src={runbookImage}>\"\n",
    "\n",
    "\n",
    "#insert the badges on the readme\n",
    "badgeCode = actionbadge + runbookbadge\n",
    "template =  template.replace(badge_var, badgeCode)\n",
    "\n",
    "# Save README\n",
    "f  = open(readme, \"w+\")\n",
    "f.write(template)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "4c1ed44c",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2a95cba3-027c-4a57-8bfa-2ee1e9053bb7",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.546467Z",
     "iopub.status.busy": "2023-05-12T17:04:24.544956Z",
     "iopub.status.idle": "2023-05-12T17:04:24.551141Z",
     "shell.execute_reply": "2023-05-12T17:04:24.550347Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "#generate a page of all the actions that are available\n",
    "ac_list= readme = \"lists/Action_list.md\"\n",
    "# Save README\n",
    "f  = open(ac_list, \"w+\")\n",
    "f.write(action_list)\n",
    "f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "70db7c84-57ad-4a92-a60b-9f20b60b5329",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.554996Z",
     "iopub.status.busy": "2023-05-12T17:04:24.554699Z",
     "iopub.status.idle": "2023-05-12T17:04:24.559737Z",
     "shell.execute_reply": "2023-05-12T17:04:24.559083Z"
    }
   },
   "outputs": [],
   "source": [
    "#generate a page of all the Runbooks that are available\n",
    "rb_list = readme = \"lists/xRunBook_list.md\"\n",
    "# Save README\n",
    "f  = open(rb_list, \"w+\")\n",
    "f.write(runbook_list)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee479c83",
   "metadata": {
    "credentialsJson": {},
    "execution": {
     "iopub.execute_input": "2023-05-12T17:04:24.564042Z",
     "iopub.status.busy": "2023-05-12T17:04:24.562668Z",
     "iopub.status.idle": "2023-05-12T17:04:24.571948Z",
     "shell.execute_reply": "2023-05-12T17:04:24.571034Z"
    }
   },
   "outputs": [],
   "source": [
    "#create JSON files that can be used t build badges for the website showing our Runbook and action counts\n",
    "\n",
    "import json\n",
    "#store Runbookcount json (to create shield on webpage)\n",
    "rb_list = readme = \".github/images/runbookShield.json\"\n",
    "runBookCount = str(runBookCount)\n",
    "json1 = {\"schemaVersion\": 1,\"label\": \"RunBook Count\",\"message\": runBookCount,\"color\": \"orange\"}\n",
    "\n",
    "# Save README\n",
    "\n",
    "with open(rb_list, \"w\") as outfile:\n",
    "    json.dump(json1, outfile)\n",
    "\n",
    "\n",
    "#store action count json (to create shield on webpage)\n",
    "rb_list = readme = \".github/images/actionShield.json\"\n",
    "actionCount = str(actionCount)\n",
    "json2 = {\"schemaVersion\": 1,\"label\": \"Action Count\",\"message\": actionCount,\"color\": \"green\"}\n",
    "\n",
    "# Save README\n",
    "with open(rb_list, \"w\") as outfile:\n",
    "    json.dump(json2, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "076fbbd0",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0041b5a5",
   "metadata": {
    "credentialsJson": {}
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af012c24",
   "metadata": {
    "credentialsJson": {}
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27801143",
   "metadata": {
    "credentialsJson": {}
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}