Evaluate_power_vs_motion_activity_analysed_master.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up the dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for reading and validating data\n",
    "import emeval.input.spec_details as eisd\n",
    "import emeval.input.phone_view as eipv\n",
    "import emeval.input.eval_view as eiev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualization helpers\n",
    "import emeval.viz.phone_view as ezpv\n",
    "import emeval.viz.eval_view as ezev\n",
    "import emeval.viz.geojson as ezgj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analytics results\n",
    "import emeval.metrics.baseline_segmentation as embs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For plots\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.collections import PatchCollection\n",
    "from matplotlib.patches import Rectangle\n",
    "%matplotlib inline\n",
    "\n",
    "import IPython.display as ipyd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For maps\n",
    "import folium\n",
    "import branca.element as bre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For easier debugging while working on modules\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.options.display.float_format = '{:.6f}'.format\n",
    "import arrow\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "THIRTY_MINUTES = 30 * 60\n",
    "TEN_MINUTES = 10 * 60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The spec\n",
    "\n",
    "The spec defines what experiments were done, and over which time ranges. Once the experiment is complete, most of the structure is read back from the data, but we use the spec to validate that it all worked correctly. The spec also contains the ground truth for the legs. Here, we read the spec for the trip to UC Berkeley."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASTORE_URL = \"http://cardshark.cs.berkeley.edu\"\n",
    "AUTHOR_EMAIL = \"shankari@eecs.berkeley.edu\"\n",
    "sd_la = eisd.SpecDetails(DATASTORE_URL, AUTHOR_EMAIL, \"unimodal_trip_car_bike_mtv_la\")\n",
    "sd_sj = eisd.SpecDetails(DATASTORE_URL, AUTHOR_EMAIL, \"car_scooter_brex_san_jose\")\n",
    "sd_ucb = eisd.SpecDetails(DATASTORE_URL, AUTHOR_EMAIL, \"train_bus_ebike_mtv_ucb\")\n",
    "sd_ucb_reroute = eisd.SpecDetails(DATASTORE_URL, AUTHOR_EMAIL, \"train_bus_ebike_sm_reroute_mtv_ucb\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The views\n",
    "\n",
    "There are two main views for the data - the phone view and the evaluation view. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phone view\n",
    "\n",
    "In the phone view, the phone is primary, and then there is a tree that you can traverse to get the data that you want. Traversing that tree typically involves nested for loops; here's an example of loading the phone view and traversing it. You can replace the print statements with real code. When you are ready to check this in, please move the function to one of the python modules so that we can invoke it more generally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(eipv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pv_la = eipv.PhoneView(sd_la)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_sj = eipv.PhoneView(sd_sj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb = eipv.PhoneView(sd_ucb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb_reroute = eipv.PhoneView(sd_ucb_reroute)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emeval.analysed.phone_view as eapv\n",
    "importlib.reload(eapv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "av_la = eapv.create_analysed_view(pv_la, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\")\n",
    "av_sj = eapv.create_analysed_view(pv_sj, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\")\n",
    "av_ucb = eapv.create_analysed_view(pv_ucb, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\")\n",
    "av_ucb_reroute = eapv.create_analysed_view(pv_ucb_reroute, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eapv.create_analysed_view(pv_la, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ANDROID_MODE_MAP = {0: \"AUTOMOTIVE\", 1: \"CYCLING\", 2: \"WALKING\", 3: \"STATIONARY\"}\n",
    "ANDROID_MAP_FN = lambda t: ANDROID_MODE_MAP[t[\"zzbhB\"]]\n",
    "\n",
    "def IOS_MAP_FN(t):\n",
    "    t_series = pd.Series(t)\n",
    "    all_true = t_series[t_series == True].index.tolist()\n",
    "    if len(all_true) == 1:\n",
    "        return all_true[0].upper()\n",
    "    else:\n",
    "        # Do something more sophisticated here?\n",
    "        return \"INVALID\"\n",
    "\n",
    "MAP_FNS = {\"android\": ANDROID_MAP_FN, \"ios\": IOS_MAP_FN}\n",
    "TRANSITION_FNS = {\"android\": embs.get_transition_mask_android, \"ios\": embs.get_transition_mask_ios}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tradeoff_entries(pv):\n",
    "    tradeoff_entry_list = []\n",
    "    for phone_os, phone_map in pv.map().items():\n",
    "        print(15 * \"=*\")\n",
    "        print(phone_os, phone_map.keys())\n",
    "        for phone_label, phone_detail_map in phone_map.items():\n",
    "            print(4 * ' ', 15 * \"-*\")\n",
    "            print(4 * ' ', phone_label, phone_detail_map.keys())\n",
    "            if \"control\" in phone_detail_map[\"role\"]:\n",
    "                print(\"Ignoring %s phone %s since they are always on\" % (phone_detail_map[\"role\"], phone_label))\n",
    "                continue\n",
    "            # this spec does not have any calibration ranges, but evaluation ranges are actually cooler\n",
    "            for r in phone_detail_map[\"evaluation_ranges\"]:\n",
    "                print(8 * ' ', 30 * \"=\")\n",
    "                print(8 * ' ',r.keys())\n",
    "                print(8 * ' ',r[\"trip_id\"], r[\"eval_common_trip_id\"], r[\"eval_role\"], len(r[\"evaluation_trip_ranges\"]))\n",
    "                bcs = r[\"battery_df\"][\"battery_level_pct\"]\n",
    "                delta_battery = bcs.iloc[0] - bcs.iloc[-1]\n",
    "                print(\"Battery starts at %d, ends at %d, drain = %d\" % (bcs.iloc[0], bcs.iloc[-1], delta_battery))\n",
    "                for tr in r[\"evaluation_trip_ranges\"]:\n",
    "                    matching_section_map = embs.find_matching_segments(tr[\"evaluation_section_ranges\"], \n",
    "                                                                      \"trip_id\",\n",
    "                                                                       [sr[\"data\"] for sr in tr[\"sensed_section_ranges\"]])\n",
    "                    print(\"For trip %s, found matching ranges %s\" % (tr[\"trip_id\"], matching_section_map))\n",
    "                    for section in tr[\"evaluation_section_ranges\"]:\n",
    "                        section_gt_leg = pv.spec_details.get_ground_truth_for_leg(tr[\"trip_id_base\"],\n",
    "                                                                                  section[\"trip_id_base\"])\n",
    "                        if section_gt_leg[\"type\"] == \"WAITING\":\n",
    "                            print(\"Skipping WAITING section %s %s with potential partway transitions\" %\n",
    "                                  (tr[\"trip_id\"], section[\"trip_id\"]))\n",
    "                            continue\n",
    "                        print(12 * ' ',section[\"trip_id\"], section[\"trip_id_base\"], tr[\"trip_id\"])\n",
    "                        sensed_section_range = matching_section_map[section[\"trip_id\"]]\n",
    "                        results = embs.get_count_start_end_ts_diff(section, sensed_section_range)\n",
    "                        tradeoff_entry = {\"phone_os\": phone_os, \"phone_label\": phone_label,\n",
    "                                      \"timeline\": pv.spec_details.curr_spec[\"id\"],\n",
    "                                      \"range_id\": r[\"trip_id\"],\n",
    "                                     \"run\": r[\"trip_run\"], \"duration\": r[\"duration\"],\n",
    "                                     \"role\": r[\"eval_role_base\"], \"battery_drain\": delta_battery,\n",
    "                                     \"section_count\": len(tr[\"sensed_section_ranges\"]),\n",
    "                                     \"trip_id\": tr[\"trip_id\"],\n",
    "                                     \"section_id\": section[\"trip_id\"],\n",
    "                                     \"section_type\": section_gt_leg[\"type\"]}\n",
    "                        tradeoff_entry.update(results)\n",
    "                        tradeoff_entry_list.append(tradeoff_entry)\n",
    "\n",
    "    return tradeoff_entry_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# We are not going to look at battery life at the evaluation trip level; we will end with evaluation range\n",
    "# since we want to capture the overall drain for the timeline\n",
    "tradeoff_entries_list = []\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_la))\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_sj))\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_ucb))\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_ucb_reroute))\n",
    "tradeoff_df = pd.DataFrame(tradeoff_entries_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add in other entries to the dataframe to allow us to plot better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's merge the reroutes\n",
    "tradeoff_df.timeline.replace(\"train_bus_ebike_sm_reroute_mtv_ucb\", \"train_bus_ebike_mtv_ucb\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r2q_map = {\"power_control\": 0, \"HAMFDC\": 1, \"MAHFDC\": 2, \"HAHFDC\": 3, \"accuracy_control\": 4}\n",
    "q2r_map = {0: \"power\", 1: \"HAMFDC\", 2: \"MAHFDC\", 3: \"HAHFDC\", 4: \"accuracy\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a number so that can get the plots to come out in order\n",
    "tradeoff_df[\"quality\"] = tradeoff_df.role.apply(lambda r: r2q_map[r])\n",
    "tradeoff_df[\"count_diff\"] = tradeoff_df[[\"count\"]] - 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Timeline + section count variations\n",
    "\n",
    "We should ideally have only one transition in every TRAVEL section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"timeline=='unimodal_trip_car_bike_mtv_la' & run == 1 & role == 'HAMFDC'\").section_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=2,ncols=3,figsize=(9,6), sharex=False, sharey=False)\n",
    "timeline_list = [\"train_bus_ebike_mtv_ucb\", \"car_scooter_brex_san_jose\", \"unimodal_trip_car_bike_mtv_la\"]\n",
    "for i, tl in enumerate(timeline_list):\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'android'\").boxplot(ax = ax_array[0][i], column=[\"count_diff\"], by=[\"quality\"])\n",
    "    ax_array[0][i].set_title(tl)\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[1][i], column=[\"count_diff\"], by=[\"quality\"])\n",
    "    ax_array[1][i].set_title(\"\")\n",
    "    # tradeoff_df.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[2][i], column=[\"visit_reports\"], by=[\"quality\"])\n",
    "    # ax_array[2][i].set_title(\"\")\n",
    "\n",
    "    # print(android_ax_returned.shape, ios_ax_returned.shape)\n",
    "\n",
    "for i, ax in enumerate(ax_array[0]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "for i, ax in enumerate(ax_array[1]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[1]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[2]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[0][0].set_ylabel(\"Difference in section counts (android)\")\n",
    "ax_array[1][0].set_ylabel(\"Difference in section counts (ios)\")\n",
    "# ax_array[2][0].set_ylabel(\"Difference in visit reports (ios)\")\n",
    "ifig.suptitle(\"Section count differences v/s configured quality over multiple timelines\")\n",
    "# ifig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs.find_section_transitions(pv_la.map()[\"android\"][\"ucb-sdb-android-2\"][\"evaluation_ranges\"][3][\"evaluation_trip_ranges\"][0][\"motion_activity_df\"].query(embs.ANDROID_VALID_QUERY_NO_STILL), embs.TRANSITION_FNS[\"android\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_count_with_errors(ax_array, phone_os):\n",
    "    for i, (tl, trip_gt) in enumerate(timeline_trip_gt.items()):\n",
    "        ax_array[i].bar(0, trip_gt)\n",
    "        for q in range(1,4):\n",
    "            curr_df = tradeoff_df.query(\"timeline == @tl & phone_os == @phone_os & quality == @q\")\n",
    "            print(\"%s %s %s values = %s %s %s\" % (phone_os, tl, q2r_map[q], curr_df.section_count.min(), curr_df.section_count.mean(), curr_df.section_count.max()))\n",
    "            lower_error = curr_df.section_count.mean() - curr_df.section_count.min()\n",
    "            upper_error = curr_df.section_count.max() - curr_df.section_count.mean()\n",
    "            ax_array[i].bar(x=q, height=curr_df.section_count.mean(),\n",
    "                            yerr=[[lower_error], [upper_error]])\n",
    "            print(\"%s %s %s errors = %s %s %s\" % (phone_os, tl, q2r_map[q], lower_error, curr_df.section_count.mean(), upper_error))\n",
    "            ax_array[i].set_title(tl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=2,ncols=3,figsize=(9,4.5), sharex=False, sharey=False)\n",
    "section_count = lambda pv: sum([len(t[\"legs\"]) for t in pv.spec_details.curr_spec[\"evaluation_trips\"]])\n",
    "timeline_trip_gt = {\"train_bus_ebike_mtv_ucb\": section_count(pv_ucb),\n",
    "                    \"car_scooter_brex_san_jose\": section_count(pv_sj),\n",
    "                    \"unimodal_trip_car_bike_mtv_la\": section_count(pv_la)}\n",
    "\n",
    "plot_count_with_errors(ax_array[0], \"android\")\n",
    "plot_count_with_errors(ax_array[1], \"ios\")\n",
    "\n",
    "for ax in ax_array[0]:\n",
    "    ax.set_xticks(range(0,4))\n",
    "    ax.set_xticklabels([q2r_map[r] for r in range(0,4)])\n",
    "    ax.set_yticks(range(0,tradeoff_df.section_count.max(),3))\n",
    "    \n",
    "for ax in ax_array[1]:\n",
    "    ax.set_xticks(range(0,4))\n",
    "    ax.set_xticklabels([q2r_map[r] for r in range(0,4)])\n",
    "    ax.set_yticks(range(0,tradeoff_df.section_count.max(),3))\n",
    "    \n",
    "ax_array[0,0].set_ylabel(\"nTrips (android)\")\n",
    "ax_array[1,0].set_ylabel(\"nTrips (ios)\")\n",
    "    \n",
    "ifig.tight_layout(pad=0.85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_of_battery_phones = tradeoff_df.query(\"timeline=='train_bus_ebike_mtv_ucb' & role=='HAHFDC' & trip_id=='berkeley_to_mtv_SF_express_bus_0' & phone_os == 'android'\")\n",
    "for i in out_of_battery_phones.index:\n",
    "    tradeoff_df.loc[i,\"end_diff_mins\"] = float('nan')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Timeline + section count variations (TRAVEL sections only)\n",
    "\n",
    "We should ideally have only one transition in every TRAVEL section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"timeline=='unimodal_trip_car_bike_mtv_la' & run == 1 & role == 'HAMFDC'\").section_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=2,ncols=3,figsize=(9,6), sharex=False, sharey=False)\n",
    "timeline_list = [\"train_bus_ebike_mtv_ucb\", \"car_scooter_brex_san_jose\", \"unimodal_trip_car_bike_mtv_la\"]\n",
    "for i, tl in enumerate(timeline_list):\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'android'\").boxplot(ax = ax_array[0][i], column=[\"count_diff\"], by=[\"quality\"])\n",
    "    ax_array[0][i].set_title(tl)\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[1][i], column=[\"count_diff\"], by=[\"quality\"])\n",
    "    ax_array[1][i].set_title(\"\")\n",
    "    # tradeoff_df.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[2][i], column=[\"visit_reports\"], by=[\"quality\"])\n",
    "    # ax_array[2][i].set_title(\"\")\n",
    "\n",
    "    # print(android_ax_returned.shape, ios_ax_returned.shape)\n",
    "\n",
    "for i, ax in enumerate(ax_array[0]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "for i, ax in enumerate(ax_array[1]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[1]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[2]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[0][0].set_ylabel(\"Difference in section counts (android)\")\n",
    "ax_array[1][0].set_ylabel(\"Difference in section counts (ios)\")\n",
    "# ax_array[2][0].set_ylabel(\"Difference in visit reports (ios)\")\n",
    "ifig.suptitle(\"Section count differences v/s configured quality over multiple timelines\")\n",
    "# ifig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"count_diff < 0 & section_type == 'TRAVEL' & end_diff_mins > 0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ipyd.display(tradeoff_df.query(\"count_diff > 0 & section_type == 'TRAVEL'\").section_id.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start-end results\n",
    "\n",
    "#### Overall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=1,ncols=4,figsize=(12,3), sharex=False, sharey=True)\n",
    "tradeoff_df.query(\"phone_os == 'android' & start_diff_mins\").boxplot(ax = ax_array[0], column=[\"start_diff_mins\"], by=[\"quality\"])\n",
    "ax_array[0].set_title(\"start time (android)\")\n",
    "tradeoff_df.query(\"phone_os == 'android' & start_diff_mins\").boxplot(ax = ax_array[1], column=[\"end_diff_mins\"], by=[\"quality\"])\n",
    "ax_array[1].set_title(\"end time (android)\")\n",
    "tradeoff_df.query(\"phone_os == 'ios' & start_diff_mins\").boxplot(ax = ax_array[2], column=[\"start_diff_mins\"], by=[\"quality\"])\n",
    "ax_array[2].set_title(\"start_time (ios)\")\n",
    "tradeoff_df.query(\"phone_os == 'ios' & start_diff_mins\").boxplot(ax = ax_array[3], column=[\"end_diff_mins\"], by=[\"quality\"])\n",
    "ax_array[3].set_title(\"end_time (ios)\")\n",
    "\n",
    "    # print(android_ax_returned.shape, ios_ax_returned.shape)\n",
    "\n",
    "ax_array[0].set_xticklabels([q2r_map[int(t.get_text())] for t in ax_array[0].get_xticklabels()])\n",
    "ax_array[1].set_xticklabels([q2r_map[int(t.get_text())] for t in ax_array[1].get_xticklabels()])\n",
    "ax_array[2].set_xticklabels([q2r_map[int(t.get_text())] for t in ax_array[2].get_xticklabels()])\n",
    "ax_array[3].set_xticklabels([q2r_map[int(t.get_text())] for t in ax_array[3].get_xticklabels()])\n",
    "\n",
    "for ax in ax_array:\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[1].text(0.55,25,\"Excluding trips where battery ran out\")\n",
    "\n",
    "ax_array[0].set_ylabel(\"Diff (mins)\")\n",
    "# ifig.suptitle(\"Section start end accuracy v/s configured quality\")\n",
    "ifig.suptitle(\"\")\n",
    "# ifig.tight_layout(pad=1.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Timeline specific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=4,ncols=3,figsize=(10,10), sharex=False, sharey=True)\n",
    "timeline_list = [\"train_bus_ebike_mtv_ucb\", \"car_scooter_brex_san_jose\", \"unimodal_trip_car_bike_mtv_la\"]\n",
    "for i, tl in enumerate(timeline_list):\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'android' & start_diff_mins < 30\").boxplot(ax = ax_array[0][i], column=[\"start_diff_mins\"], by=[\"quality\"])\n",
    "    ax_array[0][i].set_title(tl)\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'android' & end_diff_mins < 30\").boxplot(ax = ax_array[1][i], column=[\"end_diff_mins\"], by=[\"quality\"])\n",
    "    ax_array[1][i].set_title(\"\")\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'ios' & start_diff_mins < 30\").boxplot(ax = ax_array[2][i], column=[\"start_diff_mins\"], by=[\"quality\"])\n",
    "    ax_array[2][i].set_title(\"\")\n",
    "    tradeoff_df.query(\"timeline == @tl & phone_os == 'ios' & end_diff_mins < 30\").boxplot(ax = ax_array[3][i], column=[\"end_diff_mins\"], by=[\"quality\"])\n",
    "    ax_array[3][i].set_title(\"\")\n",
    "\n",
    "    # print(android_ax_returned.shape, ios_ax_returned.shape)\n",
    "\n",
    "for ax in ax_array[0]:\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "    \n",
    "for ax in ax_array[1]:\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[1,0].text(0.55,25,\"Excluding trips where battery ran out\")\n",
    "\n",
    "for ax in ax_array[2]:\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "for ax in ax_array[3]:\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[0][0].set_ylabel(\"Start time diff (android)\")\n",
    "ax_array[1][0].set_ylabel(\"End time diff (android)\")\n",
    "ax_array[2][0].set_ylabel(\"Start time diff (ios)\")\n",
    "ax_array[3][0].set_ylabel(\"End time diff (ios)\")\n",
    "ifig.suptitle(\"Section start end accuracy (mins) v/s configured quality over multiple timelines\")\n",
    "\n",
    "# ifig.tight_layout(pad=2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb.spec_details.curr_spec.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Anomaly checks\n",
    "\n",
    "We can clearly see that there are several outliers with the start/end timestamps for the sections. Let us explore these in greater detail and see if we can find any patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fmt = lambda ts: arrow.get(ts).to(\"America/Los_Angeles\")\n",
    "\n",
    "\n",
    "def check_outlier(eval_range, trip_idx, section_id):\n",
    "    eval_trip = eval_range[\"evaluation_trip_ranges\"][trip_idx]\n",
    "    eval_range[\"motion_activity_df\"][\"fmt_time\"] = eval_range[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_trip[\"motion_activity_df\"][\"fmt_time\"] = eval_trip[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_section = [s for s in eval_trip[\"evaluation_section_ranges\"] if s[\"trip_id\"] == section_id][0]\n",
    "    print(fmt(eval_section[\"start_ts\"]), \"->\", fmt(eval_section[\"end_ts\"]))\n",
    "    print([(fmt(ssr[\"data\"][\"start_ts\"]), fmt(ssr[\"data\"][\"end_ts\"])) for ssr in eval_trip[\"sensed_section_ranges\"]])\n",
    "    match = embs.find_matching_segments(eval_trip[\"evaluation_section_ranges\"], \"trip_id\",\n",
    "                                        [sr[\"data\"] for sr in eval_trip[\"sensed_section_ranges\"]])[section_id]\n",
    "    print(match)\n",
    "    print([(fmt(cm[\"start_ts\"]), fmt(cm[\"end_ts\"])) for cm in match[\"match\"]])\n",
    "    print(\"trip activity head\")\n",
    "    ipyd.display(eval_trip[\"motion_activity_df\"].head(n=3))\n",
    "    print(\"trip activity tail\")\n",
    "    ipyd.display(eval_trip[\"motion_activity_df\"].tail(n=3))\n",
    "    trip_end_ts = eval_trip[\"end_ts\"]\n",
    "    print(\"post-trip end activity head\")\n",
    "    ipyd.display(eval_range[\"motion_activity_df\"].query(\"@trip_end_ts <= ts <= @trip_end_ts + 30 * 60\").head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sections which don't max out but which have large start/end\n",
    "\n",
    "- all the walk_start trips are because we only started tracking after the walk start section was complete. There was too short a walk section at the beginning\n",
    "- the `tt_*` ones are likely to be similar since they are short\n",
    "- the others (`walk_downtown_urban_canyon_0`, `commuter_rail_aboveground_0`, `ebike_bikeshare_urban_long_0`) are investigated in greater detail below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"25 < end_diff_mins < 30\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### walk_start\n",
    "\n",
    "too short, does not have sufficient transitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_sj.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][0], 0, \"walk_start_0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### commuter_rail_aboveground_0\n",
    "\n",
    "too many transitions = flip flop. The GT is `08:31` -> `09:12` but the closest matching transition is from `08:34` to `08:46`. And in fact, we get one `walking` entry at around `2019-07-26T08:46:43.940675-07:00` which breaks up an otherwise consistent section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(pv_ucb.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][2], 0, \"commuter_rail_aboveground_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arrow.get(\"2019-07-26T08:31:56.065814-07:00\").timestamp, arrow.get(\"2019-07-26T09:12:01.181978-07:00\").timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][2][\"evaluation_trip_ranges\"][0][\"motion_activity_df\"].query(\"1564155116 <= ts <= 1564157521\").query(embs.IOS_VALID_QUERY_NO_STILL).loc[50:60]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### walk_downtown_urban_canyon_0\n",
    "\n",
    "no transition at end, bleeds over to the light rail. GT is `17:51` to `18:00`, closest range is `17:51` to `18:28`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(pv_ucb.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][2], 2, \"walk_downtown_urban_canyon_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arrow.get(\"2019-07-26T18:00:00-07:00\").timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][2][\"evaluation_trip_ranges\"][2][\"motion_activity_df\"].query(\"1564189200 <= ts <= 1564190904\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ebike_bikeshare_urban_long\n",
    "\n",
    "no transition at end, bleeds over to the express_bus. GT is `16:36` to `16:56`, first range is `16:37` to `17:23`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(pv_ucb.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][1], 2, \"ebike_bikeshare_urban_long_0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sections which do max out\n",
    "\n",
    "This has essentially one entry which is because we get no points at all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"start_diff_mins == 30 & end_diff_mins == 30 & section_id != 'walk_start_0' & section_id != 'walk_end_0'\").groupby(\"section_id\").section_id.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "check_outlier(pv_ucb.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0], 1, \"walk_urban_university_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    print(pv_ucb.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0][\"evaluation_trip_ranges\"][i][\"motion_activity_df\"].ts.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# check_outlier(pv_ucb.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][2], 1, \"walk_urban_university_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r = pv_ucb.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][2]\n",
    "# tr = r[\"evaluation_trip_ranges\"][1]\n",
    "# trip_end_ts = tr[\"end_ts\"]\n",
    "# trip_ma_df = tr[\"motion_activity_df\"]\n",
    "# extended_ma_df = r[\"motion_activity_df\"].query(\"@trip_end_ts <= ts <= @trip_end_ts + 30 * 60\")\n",
    "# ma_df = pd.concat([trip_ma_df, extended_ma_df], axis=\"index\")\n",
    "# curr_trip_section_transitions = embs.find_section_transitions(ma_df.query(VALID_QUERIES_NO_STILL[\"android\"]),\n",
    "#                                                             TRANSITION_FNS[\"android\"])\n",
    "# still_section_transitions = extended_ma_df.query(STILL_ENTRIES[\"android\"])\n",
    "# if len(still_section_transitions) > 0:\n",
    "#     curr_trip_section_transitions = curr_trip_section_transitions.append(still_section_transitions.iloc[0])\n",
    "# ipyd.display(curr_trip_section_transitions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### suburb_bicycling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"start_diff_mins == 30 & end_diff_mins == 30 & section_id == 'suburb_bicycling_0'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "check_outlier(av_la.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][4], 1, \"suburb_bicycling_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = av_la.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][4]\n",
    "tr = r[\"evaluation_trip_ranges\"][1]\n",
    "print([(fmt(sr[\"start_ts\"]), fmt(sr[\"end_ts\"])) for sr in r[\"evaluation_trip_ranges\"]])\n",
    "print([(sr[\"data\"][\"start_fmt_time\"], sr[\"data\"][\"end_fmt_time\"]) for sr in r[\"sensed_trip_ranges\"]])\n",
    "ipyd.display(r[\"transition_df\"][[\"currState\",\"transition\", \"fmt_time\"]])\n",
    "pvr = pv_la.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][4]\n",
    "gt_leg = sd_la.get_ground_truth_for_leg(\"suburb_bicycling\", \"suburb_bicycling\"); print(gt_leg[\"id\"])\n",
    "curr_map = ezgj.get_map_for_geojson(sd_ucb.get_geojson_for_leg(gt_leg), name=\"ground_truth\")\n",
    "name_err_time = lambda lr: \"%d: %s\" % (lr[\"index\"], sd_ucb.fmt(lr[\"ts\"], \"MM-DD HH:mm:ss\"))\n",
    "gt_16k = lambda lr: (1567272304 <= lr[\"ts\"] <= 1567272305) or (1567277181 <= lr[\"ts\"] <= 1567277184)\n",
    "folium.GeoJson(ezgj.get_geojson_for_loc_df(pvr[\"location_df\"], color=\"red\"), name=\"sensed_values\").add_to(curr_map)\n",
    "ezgj.get_fg_for_loc_df(pvr[\"location_df\"], name=\"sensed_points\", color=\"red\", popupfn=name_err_time, stickyfn=gt_16k).add_to(curr_map)\n",
    "folium.LayerControl().add_to(curr_map)\n",
    "curr_map\n",
    "# print([(sr[\"data\"][\"start_fmt_time\"], sr[\"data\"][\"end_fmt_time\"]) for sr in av_la.map()['ios']['ucb-sdb-ios-3'][\"sensed_section_ranges\"]])\n",
    "# print([(sr[\"data\"][\"start_fmt_time\"], sr[\"data\"][\"end_fmt_time\"]) for sr in tr[\"sensed_section_ranges\"]])\n",
    "# ma_df = tr[\"motion_activity_df\"]\n",
    "# we may get some transitions after the trip ends \n",
    "# let's expand the activity range to account for that\n",
    "# trip_end_ts = tr[\"end_ts\"]\n",
    "# ma_df = pd.concat([ma_df, \n",
    "#             r[\"motion_activity_df\"].query(\"@trip_end_ts <= ts <= @trip_end_ts + 30 * 60\")],\n",
    "#             axis=\"index\")\n",
    "# embs.find_section_transitions(ma_df.query(IOS_VALID_QUERY_WITH_STILL), get_transition_mask_ios).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    print(pv_la.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][i][\"evaluation_trip_ranges\"][1][\"motion_activity_df\"].ts.count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### walk_downtown_urban_canyon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"start_diff_mins == 30 & end_diff_mins == 30 & section_id == 'walk_downtown_urban_canyon_0'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "check_outlier(pv_ucb.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][0], 2, \"walk_downtown_urban_canyon_0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(ems)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r = pv_ucb.map()['ios']['ucb-sdb-ios-2'][\"evaluation_ranges\"][0]\n",
    "# tr = r[\"evaluation_trip_ranges\"][2]\n",
    "# sr = tr[\"evaluation_section_ranges\"][5]; print(sr[\"trip_id\"])\n",
    "# embs.find_matching_segments(tr[\"evaluation_section_ranges\"], \"trip_id\", tr[\"sensed_section_ranges\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r = pv_la.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0]\n",
    "# tr = r[\"evaluation_trip_ranges\"][1]\n",
    "# ma_df = tr[\"motion_activity_df\"]\n",
    "# we may get some transitions after the trip ends \n",
    "# let's expand the activity range to account for that\n",
    "# trip_end_ts = tr[\"end_ts\"]\n",
    "# ma_df = pd.concat([ma_df, \n",
    "#             r[\"motion_activity_df\"].query(\"@trip_end_ts <= ts <= @trip_end_ts + 30 * 60\")],\n",
    "#             axis=\"index\")\n",
    "# embs.find_section_transitions(ma_df.query(IOS_VALID_QUERY_WITH_STILL), get_transition_mask_ios).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    print(pv_la.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][i][\"evaluation_trip_ranges\"][1][\"motion_activity_df\"].ts.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}