Step 2 - Build Events Page With Meta Details.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json, pprint, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EVENT_NAME       = \"2017 Hurricane Nate\"\n",
    "html_output_file = \"/home/anderstj/tweetsonamap.com/docs/evacuation/events/nate.html\"\n",
    "input_directory  = \"/data/chime/geo2/NEW/NATE/\"\n",
    "\n",
    "ROOT             = \"https://s3-us-west-1.amazonaws.com/movement-derivation/cdn/NEW/NATE/\"\n",
    "USER_URL = \"../index.html?user={0}&root=\"+ROOT"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "EVENT_NAME       = \"2012 Hurricane Sandy: Red Hook (Contextual Plus)\"\n",
    "html_output_file = \"/home/anderstj/tweetsonamap.com/docs/evacuation/events/redhook.html\"\n",
    "input_directory  = \"/data/chime/geo2/NEW/redhook-plus/\"\n",
    "\n",
    "ROOT             = \"https://s3-us-west-1.amazonaws.com/movement-derivation/cdn/NEW/REDHOOK-PLUS/\"\n",
    "USER_URL = \"../index.html?user={0}&root=\"+ROOT"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "EVENT_NAME       = \"2017 Hurricane Irma: Geotagged for inland flooding\"\n",
    "html_output_file = \"/home/anderstj/tweetsonamap.com/docs/evacuation/events/irma-inland.html\"\n",
    "input_directory  = \"/data/chime/geo2/NEW/IRMA-INLAND/\"\n",
    "\n",
    "ROOT             = \"https://s3-us-west-1.amazonaws.com/movement-derivation/cdn/NEW/IRMA-INLAND/\"\n",
    "USER_URL = \"../index.html?user={0}&root=\"+ROOT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Go run this code:\n",
    "\n",
    "```\n",
    "aws --profile=epic s3 cp --recursive --acl=public-read /data/chime/geo2/NEW/contextual-stream-test s3://movement-derivation/cdn/NEW/contextual-stream-test\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_user_metas(input_directory):\n",
    "    users = [x for x in os.listdir(input_directory) if x.endswith('-meta.geojson') ]\n",
    "    print(\"Found {0} metadata files\".format(len(users)))\n",
    "    return users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1411 metadata files\n"
     ]
    }
   ],
   "source": [
    "users = load_user_metas(input_directory)\n",
    "home_locations = {\"type\":\"FeatureCollection\",\"features\":[]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rankUser(u):\n",
    "    #Read the metadata file and do something\n",
    "    score = 0;\n",
    "    clustered_tweets = 0;\n",
    "    \n",
    "    this_user = json.load(open(input_directory + u))\n",
    "    \n",
    "    handle = u[0:-13]\n",
    "    \n",
    "    home_loc = False;\n",
    "    \n",
    "    clusters = [x for x in this_user['features'] if x['properties'].get('clusterCenter',False)]\n",
    "\n",
    "    for cluster in clusters:\n",
    "        if cluster['properties'].get('likelyHome',False):\n",
    "            score += 1;\n",
    "            home_loc = cluster;\n",
    "        elif  cluster['properties'].get('likelyWork',False):\n",
    "            score += 1;\n",
    "        clustered_tweets += cluster['properties']['tweetCount']\n",
    "        \n",
    "    if home_loc:\n",
    "        home_loc['properties'] = {\n",
    "            'handle': handle,\n",
    "            'link' : USER_URL.format(handle),\n",
    "            'totalClusteredTweets': clustered_tweets,\n",
    "            'totalClusters': len(clusters)\n",
    "        }\n",
    "        home_locations['features'].append(home_loc)\n",
    "    \n",
    "    return (score, len(clusters), clustered_tweets, handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score | #Clusters | #Tweets | User\n",
      "0        1        4           QueenTriniLisa\n",
      "2        5        26           NotableBirdsLA\n",
      "1        18        85           tmj_bhm_hrta\n",
      "0        1        15           TrendsNicaragua\n",
      "0        1        5           ChefdavidDavid\n",
      "0        1        24           _TuscaloosaAL\n",
      "1        2        6           2LiterLuke\n",
      "0        1        46           trendinaliaAR\n",
      "0        1        3           ultabeautyjobs\n",
      "0        1        26           liveatgreendoor\n"
     ]
    }
   ],
   "source": [
    "print(\"Score | #Clusters | #Tweets | User\")\n",
    "for u in users[:10]:\n",
    "    score, clusterCount, tweetCount, handle = rankUser(u)\n",
    "    print(\"{0}        {1}        {2}           {3}\".format(score, clusterCount, tweetCount, handle))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write out HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeTableRow(handle, tweetCount, clusterCount):\n",
    "    string = \"\"\"<tr>\n",
    "    <td><a target=\"_blank\" class=\"link\" href=\"{3}\">{0}</a></td>\n",
    "    <td>{1}</td>\n",
    "    <td>{2}</td>\n",
    "    </tr>\"\"\".format(handle, tweetCount, clusterCount, USER_URL.format(handle,ROOT))\n",
    "    return [string, tweetCount]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48 344 1019\n"
     ]
    }
   ],
   "source": [
    "twos = []\n",
    "ones = []\n",
    "zeroes = []\n",
    "for u in users:\n",
    "    try:\n",
    "        score, clusterCount, tweetCount, handle = rankUser(u)\n",
    "\n",
    "        if score==2:\n",
    "            twos.append(makeTableRow(handle, tweetCount, clusterCount))\n",
    "\n",
    "        if score==1:\n",
    "            ones.append(makeTableRow(handle, tweetCount, clusterCount))\n",
    "\n",
    "        if score==0:\n",
    "            zeroes.append(makeTableRow(handle, tweetCount, clusterCount))\n",
    "    except:\n",
    "        print(\"error on \"+u)\n",
    "print(len(twos), len(ones), len(zeroes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "home_locations['features'] = home_locations['features']\n",
    "\n",
    "CONTENT = \"\"\n",
    "CONTENT += \"<h4 class='txt-h4'>Users with calculated home & work location</h4>\"\n",
    "CONTENT += \"<table>\\n<thead><tr><th>User</th><th>Clustered Tweets</th><th># Clusters</th><tr></thead>\\n<tbody>\"\n",
    "for row in sorted(twos, key=lambda x: -x[1]):\n",
    "    CONTENT += row[0]+\"\\n\"\n",
    "CONTENT += \"</tbody></table>\"\n",
    "\n",
    "CONTENT += \"<h4 class='txt-h4'>Users with home or work location determined</h4>\"\n",
    "CONTENT += \"<table>\\n<thead><tr><th>User</th><th>Clustered Tweets</th><th># Clusters</th><tr></thead>\\n<tbody>\"\n",
    "for row in sorted(ones, key=lambda x: -x[1]):\n",
    "    CONTENT += row[0]+\"\\n\"\n",
    "CONTENT += \"</tbody></table>\"\n",
    " \n",
    "CONTENT += \"<h4 class='txt-h4'>Users with neither clustered home or work location</h4>\"\n",
    "CONTENT += \"<table>\\n<thead><tr><th>User</th><th>Clustered Tweets</th><th># Clusters</th><tr></thead>\\n<tbody>\"\n",
    "for row in sorted(zeroes, key=lambda x: -x[1]):\n",
    "    CONTENT += row[0]+\"\\n\"\n",
    "CONTENT += \"</tbody></table>\"\n",
    "CONTENT += \"<script type='text/javascript'>\"\n",
    "CONTENT += \"var homeLocations = \" + json.dumps(home_locations)\n",
    "CONTENT += \"</script>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEMPLATE = open('/home/anderstj/tweetsonamap.com/_templates/event_page.html_template','r').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(html_output_file,'w') as outFile:\n",
    "    outFile.write(TEMPLATE.format(EVENT_NAME, EVENT_NAME, CONTENT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}