Will Thomson's Modified Fbref Scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Looking for [chromedriver 83.0.4103.39 mac64] driver in cache \n",
      "[WDM] - File found in cache by path [/Users/keesvanhemmen/.wdm/drivers/chromedriver/83.0.4103.39/mac64/chromedriver]\n"
     ]
    }
   ],
   "source": [
    "# Returns a DataFrame containing all of fbref's StatsBomb data for Men's European competition\n",
    "def scrape_europe(path_to_chromedriver=\"\", league_names=['epl', 'laliga', 'bundesliga', 'seriea', 'ligue1', 'championsleague','europaleague']):\n",
    "    # imports necessary libraries\n",
    "    ## some must be downloaded\n",
    "    from selenium import webdriver\n",
    "    import pandas as pd\n",
    "    import numpy as np\n",
    "    from balaban.utils import get_col_dtype\n",
    "    from selenium.webdriver.chrome.options import Options\n",
    "    from selenium import webdriver\n",
    "    from webdriver_manager.chrome import ChromeDriverManager\n",
    "\n",
    "    # Creates the arrays we will iterate through\n",
    "    years = np.array([\"/2017-2018-\", \"/2018-2019-\", \"\"])\n",
    "    top_5_league_2018 = np.array([\"1631/\", \"1652/\", \"1634/\", \"1632/\", \"1640/\", \"\", \"\"])\n",
    "    top_5_league_2019 = np.array([\"1889/\", \"1886/\", \"2109/\", \"2104/\", \"1896/\", \"2102/\", \"2103/\"])\n",
    "    top_5_league_2020 = np.array([\"\", \"\", \"\", \"\", \"\", \"\", \"\"])\n",
    "    top_5_league_nums = np.array(['9', '12', '20', '11', '13', '8', '19'])\n",
    "    top_5_league_names = np.array(['Premier-League', 'La-Liga', 'Bundesliga', 'Serie-A', 'Ligue-1', \"Champions-League\", \"Europa-League\"])\n",
    "    league_codes = ['epl', 'laliga', 'bundesliga', 'ligue1', 'seriea', 'championsleague', 'europaleague']\n",
    "    league_names = list(league_names) if (type(league_names) is np.ndarray) or (\n",
    "            type(league_names) is tuple) else league_names\n",
    "    league_names = [league_names] if type(league_names) is str else league_names\n",
    "\n",
    "    # Argument validation for the optional league inputs\n",
    "    try:\n",
    "        league_matches = np.isin(league_codes, league_names)\n",
    "    except:\n",
    "        raise ValueError(\n",
    "            \"league_names should be a list of characters. Available options are 'epl', 'laliga', 'bundesliga',  \"\n",
    "            \"'seriea', 'ligue1', 'championsleague','europaleague'\")\n",
    "\n",
    "    if np.sum(league_matches) == 0:\n",
    "        raise ValueError(\n",
    "            \"league_names should be a list of characters. Available options are 'epl', 'laliga', 'bundesliga',  \"\n",
    "            \"'seriea', 'ligue1', 'championsleague','europaleague'\")\n",
    "    elif np.sum(league_matches) != len(league_names):\n",
    "        raise ValueError(\n",
    "            \"league_names contains strings that weren't matched. Available options are 'epl', 'laliga', 'bundesliga', \"\n",
    "            \" 'seriea', 'ligue1', 'championsleague','europaleague'\")\n",
    "\n",
    "    # Creates the Chromedriver connection\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    chrome_options.add_argument('--headless')\n",
    "    chrome_options.add_argument('--no-sandbox')\n",
    "    chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "    # Finds / Downloads ChromeDriver for you\n",
    "    browser =   webdriver.Chrome(ChromeDriverManager().install())\n",
    "\n",
    "    # Makes lists out of the arrays\n",
    "    years = list(years)\n",
    "    year_nums_2018 = list(top_5_league_2018)\n",
    "    year_nums_2019 = list(top_5_league_2019)\n",
    "    year_nums_2020 = list(top_5_league_2020)\n",
    "    top_5_league_names = list(top_5_league_names)\n",
    "    top_5_league_nums = list(top_5_league_nums)\n",
    "\n",
    "    # The pages we will iterate through\n",
    "    categories = ['passing', 'shooting', 'misc','possession','defense','gca','passing_types']\n",
    "    # The DataFrame we will return\n",
    "    all_players_df = pd.DataFrame()\n",
    "    for year in years:\n",
    "        # The DataFrame for this year we will append to all_players_df\n",
    "        year_df = pd.DataFrame()\n",
    "        \n",
    "        # Sets the leaguecode, which is actually a year-league code indicating the pages to access, based on year\n",
    "        if year == \"/2017-2018-\":\n",
    "            leaguecode = year_nums_2018\n",
    "        elif year == \"/2018-2019-\":\n",
    "            leaguecode = year_nums_2019\n",
    "        else:\n",
    "            leaguecode = year_nums_2020\n",
    "        for lnum, lnam, ynum in zip(top_5_league_nums, top_5_league_names, leaguecode):\n",
    "            # There is no data for this year and these competitions, so we do not attempt to access it\n",
    "            if (year == \"/2017-2018-\" and (lnam == \"Champions-League\" or lnam == \"Europa-League\")):\n",
    "                continue\n",
    "            \n",
    "            for category in categories:\n",
    "                # the current year has a different url structure\n",
    "                if year == \"\":\n",
    "                    url = 'https://fbref.com/en/comps/' + lnum + '/' + ynum + category + '/' + year + lnam + '-Stats'\n",
    "                else:\n",
    "                    url = 'https://fbref.com/en/comps/' + lnum + '/' + ynum + category + year + lnam + '-Stats'\n",
    "                # Uses the webdriver to retrieve the URL\n",
    "                browser.get(url)\n",
    "                # We operate differently upon our first page for each league/year because we only want certain fields once\n",
    "                if category == 'passing':\n",
    "                    # Creates a df from the table on fbref's site\n",
    "                    my_table = browser.find_element_by_id('div_stats_passing')\n",
    "                    my_table = my_table.find_element_by_xpath(\"table\")\n",
    "                    df = pd.read_html(my_table.get_attribute('outerHTML'))[0]\n",
    "                    # Creates singular column names from hierarchical column names to prevent column name redundancy\n",
    "                    col_names = np.array(\n",
    "                        [('Unnamed:' not in val[0]) * (val[0] + ': ') + val[1] for val in df.columns.values])\n",
    "                    # Creates an array of the columns we want / 0 is Rk, 6 is Birth Year, we keep the rest\n",
    "                    tmp = np.array(df)[:, np.r_[np.arange(1, 6), np.arange(7, df.shape[1] - 1)]]\n",
    "                    # Creates a league column with lnam as the league name\n",
    "                    tmp = np.c_[np.tile(lnam, tmp.shape[0]), tmp]\n",
    "                    # Sets column names\n",
    "                    tmp_col_names = np.r_[np.array('League'), col_names[np.r_[np.arange(1, 6), np.arange(7, df.shape[1] - 1)]]]\n",
    "                    # Makes a new dataframe out of the cleaned data\n",
    "                    league_df = pd.DataFrame(tmp)\n",
    "                    # Sets the column names\n",
    "                    league_df.columns = tmp_col_names\n",
    "                    # Drops the rows where Player is the Player name (redundant column names)\n",
    "                    league_df = league_df[league_df['Player'] != 'Player']\n",
    "                    # Casts all the columns to uniform datatypes\n",
    "                    league_df = league_df.astype(league_df.apply(get_col_dtype).to_dict())\n",
    "                    # Sets player as the index\n",
    "                    league_df.index = league_df['Player']\n",
    "                    # Cleans nation name\n",
    "                    league_df[\"Nation\"]= league_df['Nation'].str.split(\" \").str[1]\n",
    "                else:\n",
    "                    my_table = browser.find_element_by_id('div_stats_'+category)\n",
    "                    my_table = my_table.find_element_by_xpath(\"table\")\n",
    "                    df = pd.read_html(my_table.get_attribute('outerHTML'))[0]\n",
    "                    col_names = np.array(\n",
    "                        [('Unnamed:' not in val[0]) * (val[0] + ': ') + val[1] for val in df.columns.values])\n",
    "                    tmp = np.array(df)[:, np.r_[1, np.arange(8, df.shape[1] - 1)]]\n",
    "                    tmp_df = pd.DataFrame(tmp)\n",
    "                    tmp_df.columns = col_names[np.r_[1, np.arange(8, df.shape[1] - 1)]]\n",
    "                    tmp_df = tmp_df[tmp_df['Player'] != 'Player']\n",
    "                    tmp_df = tmp_df.astype(tmp_df.apply(get_col_dtype).to_dict())\n",
    "                    tmp_df = tmp_df.set_index(list(tmp_df)[0])\n",
    "                    league_df = pd.concat([league_df, tmp_df], axis=1, sort=False)\n",
    "            # Adds the data from the given league to the data for the given year\n",
    "            year_df = pd.concat([year_df, league_df])\n",
    "        # Creates a year column in the df\n",
    "        if year != \"\":\n",
    "            year_df[\"year\"] = year[1:-1]\n",
    "        else:\n",
    "            year_df[\"year\"] = \"2019-2020\"\n",
    "        # Adds the year to the final df\n",
    "        all_players_df = pd.concat([all_players_df, year_df])\n",
    "    return all_players_df\n",
    "\n",
    "df = scrape_europe(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"fbrefEurope.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}