QL_Sched_update_rpi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7fcf0ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "import time as tm\n",
    "#df = pd.read_csv ('simulation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ab32ce75",
   "metadata": {},
   "outputs": [],
   "source": [
    "HL = 5 #history length\n",
    "def SenDySched(Te,CL):\n",
    "    Tes = np.sort(Te)\n",
    "    #print(Tes)\n",
    "    Tsp = np.min(Tes)\n",
    "    #print(Tsp)\n",
    "    L1 = Tsp\n",
    "    L2 = len(Tes)\n",
    "    for i in range(L1):\n",
    "        for j in Tes:\n",
    "            n = np.ceil(j/Tsp)\n",
    "            thresh = j + CL\n",
    "            #print(j,n,Tsp,n*Tsp,thresh)\n",
    "            if n*Tsp > thresh:\n",
    "                Tsp -= 1\n",
    "                break\n",
    "    if Tsp > 1:\n",
    "        return Tsp\n",
    "    else:\n",
    "        print(\"alg. failed\")  \n",
    "\n",
    "def gen_rand_Te(HL):\n",
    "    none = random.sample(range(50, 200), HL*HL+1)\n",
    "    microwave = random.sample(range(30, 300), HL)\n",
    "    kettle = random.sample(range(50, 240), HL)\n",
    "    faucet = random.sample(range(15, 120), HL)\n",
    "    wdisposer = random.sample(range(10, 60), HL)\n",
    "    vfan = random.sample(range(200, 600), HL)\n",
    "    events_Te = [none,microwave,kettle,faucet,wdisposer,vfan]\n",
    "    return events_Te\n",
    "#print(none,microwave,kettle,faucet,wdisposer,vfan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "319be654",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Check_Tsp(Te,CL,Tsp):\n",
    "    Tes = np.sort(Te)\n",
    "    #print(Tes)\n",
    "    #print(Tsp)\n",
    "    valid = False\n",
    "    for t in Tes:\n",
    "        #print(t)\n",
    "        n = np.ceil(t/Tsp)\n",
    "        #thresh = t + cl\n",
    "        if n*Tsp-t <= CL:\n",
    "            valid = True\n",
    "            \n",
    "            #print(n,n*Tsp,t,cl,'no')\n",
    "            #print(\"Tsp not valid\")\n",
    "        else:\n",
    "            valid = False\n",
    "            break\n",
    "            #print(n,n*Tsp,t,cl,'yes')\n",
    "            #print(\"Tsp is valid for CL=\",cl) \n",
    "        #print(valid)\n",
    "    return valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0d5d66af",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_timeline():\n",
    "    events_seq = []\n",
    "    seq_periods = []\n",
    "    xo = random.randint(1, 5)\n",
    "    yo = random.randint(0, 4)\n",
    "    for none in events_Te[0]:\n",
    "        events_seq.append([0])\n",
    "        seq_periods.append([none])\n",
    "        x = random.randint(1, 5)\n",
    "        y = random.randint(0, 4)\n",
    "        if x == xo:\n",
    "            x = random.randint(1, 5)\n",
    "        xo = x\n",
    "        events_seq.append([x])\n",
    "        seq_periods.append([events_Te[x][y]])\n",
    "    events_seq.append([0])\n",
    "    seq_periods.append([random.randint(50, 200)])\n",
    "    return events_seq,seq_periods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c0729184",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sim_plot(sim,eclass):    \n",
    "    plot_a = []\n",
    "    L = len(sim)\n",
    "    for i in range(L):\n",
    "        for j in range(sim[i]):\n",
    "            c = eclass[i]\n",
    "            #print(j,c)\n",
    "            plot_a.append([int(c)])\n",
    "    return plot_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9638329",
   "metadata": {},
   "outputs": [],
   "source": [
    "class train_Qsched:\n",
    "    def __init__(self, Time,events,Tsp,CL,position,n_actions,test):\n",
    "        #self.size = size\n",
    "        self.position = position\n",
    "        self.event_att = 0\n",
    "        self.Time = Time\n",
    "        self.length = len(Time)\n",
    "        self.events = events\n",
    "        self.Tsp = Tsp\n",
    "        self.Tspold = Tsp\n",
    "        self.CL = CL\n",
    "        self.reward = 0\n",
    "        self.Tideal = 0\n",
    "        self.num_classes = 6\n",
    "        self.cur_event = [0]\n",
    "        self.pre_event = [0]\n",
    "        self.event_states = 2\n",
    "        self.num_states = self.num_classes*n_actions\n",
    "        #self.num_states = self.num_classes*self.event_states\n",
    "        self.num_actions = n_actions#0\n",
    "        self.increment = 1 #or 5 or 10\n",
    "        self.event_changed = 0 #0=no, 1=yes\n",
    "        self.w1 = 1\n",
    "        self.w2 = 1\n",
    "        self.step = 0\n",
    "        self.done = False\n",
    "        self.test = test\n",
    "    def get_qtable(self):\n",
    "        self.qtable = np.zeros([self.num_states, self.num_actions])\n",
    "        return self.qtable\n",
    "    \n",
    "    def update_qtable(self, new_qtable):\n",
    "        self.qtable = new_qtable\n",
    "        \n",
    "    def reset_position(self):\n",
    "        self.position = 0\n",
    "        \n",
    "    def take_action(self, action):\n",
    "        Tsp = action+1\n",
    "        #print(Tsp)\n",
    "        reward = 0\n",
    "        inc = self.increment\n",
    "        done = self.done\n",
    "        self.pre_event = self.events[self.position]\n",
    "        self.step += 1\n",
    "        self.position += Tsp\n",
    "        #print(self.position)\n",
    "        self.Tsp = Tsp\n",
    "        #print(self.cur_event)\n",
    "        cl = self.CL[self.cur_event[0]]\n",
    "        if self.position + Tsp > self.length: \n",
    "            done = True\n",
    "            #print(\"done\",self.position,self.length)\n",
    "            self.position = self.length - 1\n",
    "            \n",
    "        Tideal = self.find_Tideal(self.events,self.position)\n",
    "        #print(Tsp,self.Tspold)\n",
    "        if Tsp >= Tideal:\n",
    "            if Tsp - Tideal <= cl:\n",
    "                reward = 50\n",
    "            else:\n",
    "                reward = -50#*self.w1\n",
    "        #elif Tsp < Tideal:\n",
    "        #    if Tideal - Tsp < cl:\n",
    "        #        reward = 0\n",
    "        #    else:\n",
    "        #        reward = -10*self.w2 \n",
    "            #reward = -5\n",
    "            \n",
    "            #returnhere\n",
    "\n",
    "        else:\n",
    "            if Tsp >= self.Tspold:\n",
    "                reward = 30\n",
    "            else: \n",
    "                reward = -30         \n",
    "        if self.events[self.position] == self.cur_event:\n",
    "            self.event_changed = 0\n",
    "            #reward += -5\n",
    "        else:\n",
    "            self.cur_event = self.events[self.position] \n",
    "            self.event_changed = 1\n",
    "            #reward += 50\n",
    "            #print(self.cur_event,self.event_changed)\n",
    "        #print(reward)\n",
    "        #next_state = self.encode0(self.cur_event[0],self.event_changed)\n",
    "        i = (self.cur_event[0])*self.num_actions\n",
    "        #next_Tsp = np.max(np.where(self.qtable==self.qtable[i:i+self.num_actions-1].max()))#[0][0]#-i\n",
    "        #next_Tsp = np.where(self.qtable[i:i+self.num_actions]==self.qtable[i:i+self.num_actions].max())[0][0]\n",
    "        next_Tsp = random.randint(0, self.num_actions-1)\n",
    "        #print(self.cur_event[0],Tsp,next_Tsp)\n",
    "        if self.test:\n",
    "            next_state = self.encode(self.cur_event[0],Tsp)\n",
    "        else:\n",
    "            next_state = self.encode(self.cur_event[0],Tsp)#next_Tsp)\n",
    "        \n",
    "\n",
    "        \n",
    "        self.Tspold = Tsp\n",
    "        '''\n",
    "        print(\"event cathced\")\n",
    "        cposition_arr.append([position])\n",
    "        cevent_hist.append(event_cur)\n",
    "        correct += 1\n",
    "        print(\"Ideal Tsp:\", find_Tideal(events,position,CL))\n",
    "        else:\n",
    "        print(\"missed!!\")\n",
    "        missed += 1\n",
    "        xposition_arr.append([position])\n",
    "        xevent_hist.append(event_cur)\n",
    "        event_cur = events[position]\n",
    "        '''\n",
    "        return next_state, reward, done, self.cur_event[0]\n",
    "    \n",
    "    def get_info(self):\n",
    "        return self.position,self.pre_event,self.event_changed\n",
    "    \n",
    "    def encode0(self, eclass,changed):\n",
    "        if changed == 0:\n",
    "            i = eclass\n",
    "        else:\n",
    "            i = eclass+self.num_classes\n",
    "        return i\n",
    "    \n",
    "    def encode(self, eclass,Tsp):\n",
    "        # (5) 5, 5, 4\n",
    "        i = (eclass)*self.num_actions#*eclass\n",
    "    \n",
    "        #print(i)\n",
    "        #i *= changed+1\n",
    "        ##i *= 6\n",
    "        '''\n",
    "        if changed == 0:\n",
    "            i *= 1\n",
    "        else:\n",
    "            i *= 2\n",
    "            i += self.num_actions\n",
    "        '''\n",
    "        i += Tsp - 1\n",
    "        return i\n",
    "    \n",
    "    def find_Tideal(self, events,position):\n",
    "        new_position = position\n",
    "        #print(\"position:\",position)\n",
    "        current_event = events[position]\n",
    "        #print(current_event)\n",
    "        #l = len(events)-position\n",
    "        #cl = CL[current_event[0]]\n",
    "        #Tideal = 0\n",
    "        while current_event == events[new_position] and (new_position < self.length - 1):\n",
    "            new_position += 1\n",
    "        Tideal = new_position-position\n",
    "        #print(Tideal,self.length)\n",
    "        return Tideal# + cl/2\n",
    "    \n",
    "    def print_info(self):\n",
    "        print(\"current Tsp:\",self.Tsp)\n",
    "        print(\"current Position:\",self.position)\n",
    "        print(\"current event changed?\",self.event_changed)\n",
    "        \n",
    "def encode(eclass,Tsp):\n",
    "    # (5) 5, 5, 4\n",
    "    i = (eclass)*50#*eclass\n",
    "    \n",
    "    print(i)\n",
    "    #i *= changed+1\n",
    "    ##i *= 6\n",
    "\n",
    "    i += Tsp - 1 \n",
    "    #i *= 1\n",
    "    return i\n",
    "def encode0(eclass,changed):\n",
    "    # (5) 5, 5, 4\n",
    "    if changed == 0:\n",
    "        i = eclass\n",
    "    else:\n",
    "        i = eclass+6\n",
    "    return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aef2ff56",
   "metadata": {},
   "outputs": [],
   "source": [
    "events_Te = np.load(\"events_Te.npy\",allow_pickle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6a74f1ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 2 0 4 0 1 0 3 0 2 0 1 0 4 0 3 0 1 0 4 0 2 0 5 0 4 0 3 0 5 0 3 0 5 0 1 0\n",
      " 2 0 4 0 1 0 2 0 1 0 3 0 5 0 3 0]\n",
      "[111 158  73  30 198 138  78 116 114 169 192 294  75  32 134 117  65 138\n",
      " 188  30 107  98  58 407 155  55 151 117 100 493 113  30  67 560  88 172\n",
      "  82 169  51  55  55 172 186 158 141 138 109 117  85 324  97  84 151]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "events_seq,seq_periods = gen_timeline()\n",
    "tn_events = sim_plot(np.hstack(seq_periods),np.hstack(events_seq))\n",
    "l = len(tn_events)\n",
    "times = np.linspace(0, l, num=l, endpoint=True)\n",
    "print(np.hstack(events_seq))\n",
    "print(np.hstack(seq_periods))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "165d6171",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.010911226272583008\n"
     ]
    }
   ],
   "source": [
    "#time to update class-based sched\n",
    "st = tm.time()\n",
    "l = len(events_Te)\n",
    "CL = [5,12,15,5,8,15]\n",
    "num_CLs = 10\n",
    "event_CL = []\n",
    "event_Tsp = []\n",
    "for i in range(l):\n",
    "    event_Tsp.append([])\n",
    "    #CL = np.min(events_Te[i])\n",
    "    #CLs = np.sort(random.sample(range(1, CL+1), num_CLs))\n",
    "    #event_CL.append(np.sort(CLs))\n",
    "    #for j in range(num_CLs):\n",
    "    event_Tsp[i].append(SenDySched(events_Te[i],CL[i]))\n",
    "#print(\"Random selected CLs for each event:\",event_CL)\n",
    "cb_time = tm.time()-st\n",
    "print(cb_time)\n",
    "\n",
    "lines = (\"Updating CLass-based_sched time = \", str(cb_time))\n",
    "with open('sched_updating_time.txt', 'w') as f:\n",
    "    for line in lines:\n",
    "        f.write(line)\n",
    "    f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "94064e27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 2 0 4 0 1 0 3 0 2 0 1 0 4 0 3 0 1 0 4 0 2 0 5 0 4 0 3 0 5 0 3 0 5 0 1 0\n",
      " 2 0 4 0 1 0 2 0 1 0 3 0 5 0 3 0]\n",
      "[111 158  73  30 198 138  78 116 114 169 192 294  75  32 134 117  65 138\n",
      " 188  30 107  98  58 407 155  55 151 117 100 493 113  30  67 560  88 172\n",
      "  82 169  51  55  55 172 186 158 141 138 109 117  85 324  97  84 151]\n",
      "Training finished.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "thresh_list = [0.1,0.01,0.001,0.0001]\n",
    "upd_times = []\n",
    "q_tables = []\n",
    "for thresh in thresh_list:\n",
    "    gen_timeline()\n",
    "    print(np.hstack(events_seq))\n",
    "    print(np.hstack(seq_periods))\n",
    "    q_table = np.load('q_table.npy')\n",
    "    CL = [5,12,15,5,8,15]\n",
    "    Tsp = random.randint(0,10)\n",
    "    position = 0\n",
    "    n_actions = 100\n",
    "\n",
    "    #%%time\n",
    "    \"\"\"Training the agent\"\"\"\n",
    "\n",
    "    import random\n",
    "    from IPython.display import clear_output\n",
    "    position = 0\n",
    "    sttime = tm.time()\n",
    "    ql_sched = train_Qsched(times,tn_events,Tsp,CL,position,n_actions,False)\n",
    "    # Hyperparameters\n",
    "    alpha = 0.3#1\n",
    "    gamma = 0.3#6\n",
    "    epsilon = 0.1#1\n",
    "    penalties = 0\n",
    "    rewards = 0\n",
    "    # For plotting metrics\n",
    "    all_epochs = []\n",
    "    all_penalties = []\n",
    "    MAXitra = 50000\n",
    "    changed = 0\n",
    "    #thresh = 0.01#005#01\n",
    "    conv_counter = 0\n",
    "    pre_pen = 0\n",
    "    cur_pen = 0\n",
    "    for i in range(1, MAXitra):\n",
    "        state = 0#random.randint(0,11)\n",
    "        epochs, reward, = 0, 0\n",
    "        done = False\n",
    "        #print(\"OK\")\n",
    "        while not done:\n",
    "            if random.uniform(0, 1) < epsilon:\n",
    "                action = random.randint(0, n_actions-1) # Explore action space\n",
    "            else:\n",
    "                action = np.argmax(q_table[state]) # Exploit learned values\n",
    "            #print(f\"Action: {action}\")\n",
    "            #print(\"OKact\")\n",
    "            next_state, reward, done, cur_event = ql_sched.take_action(action)\n",
    "            #print(done)\n",
    "            #ql_sched.print_info()\n",
    "            old_value = q_table[state, action]\n",
    "            next_max = np.max(q_table[next_state])\n",
    "            #print(next_state)\n",
    "            new_value = (1 - alpha) * old_value + alpha * (reward + gamma * next_max)\n",
    "            q_table[state, action] = new_value\n",
    "            #clear_output(wait=True)\n",
    "            #print(\"OKqta\")\n",
    "            if reward <= -10:\n",
    "                penalties += 1\n",
    "\n",
    "            if reward >= 10:\n",
    "                rewards += 1\n",
    "\n",
    "            #print(\"OKrew\")\n",
    "            epochs += 1\n",
    "            #clear_output(wait=True)\n",
    "            #with np.printoptions(threshold=sys.maxsize):\n",
    "            #print(q_table)\n",
    "            #print(\"state\",state)\n",
    "            ##print(\"next_state\",next_state)\n",
    "            pre_pen = cur_pen\n",
    "\n",
    "            if state != next_state:\n",
    "                changed += 1\n",
    "            #print(\"OKstate\")\n",
    "            if i % 100 == 0:\n",
    "                clear_output(wait=True)\n",
    "\n",
    "                print(f\"Episode: {i}\")\n",
    "                print(f\"Action: {action}\")\n",
    "                print(f\"reward: {reward}\")\n",
    "                print(\"pens: \", penalties/i)\n",
    "                print(\"r: \", rewards/i)\n",
    "                print(f\"current state: {state}, next state: {next_state}, CHANGED: {changed}\")\n",
    "                print(\"conv counter:\", conv_counter)\n",
    "                ql_sched.print_info()\n",
    "\n",
    "\n",
    "\n",
    "            state = next_state\n",
    "            ql_sched.update_qtable(q_table)\n",
    "        cur_pen = penalties/i    \n",
    "        #print(\"conv counter:\", conv_counter)\n",
    "        #print(\"pre_pen = cur_pen\",  pre_pen - cur_pen)\n",
    "        if abs(pre_pen - cur_pen) <= thresh:\n",
    "            conv_counter += 1\n",
    "        else:\n",
    "            conv_counter = 0\n",
    "        if conv_counter >= 10:\n",
    "            break    \n",
    "        ql_sched.reset_position()\n",
    "        '''\n",
    "        if i % 100 == 0:\n",
    "            clear_output(wait=True)\n",
    "            print(f\"Episode: {i}\")\n",
    "            print(f\"Action: {action}\")\n",
    "            print(f\"reward: {reward}\")\n",
    "            ql_sched.print_info()\n",
    "        '''\n",
    "    print(\"Training finished.\\n\")\n",
    "    upd_times.append(tm.time()-sttime)\n",
    "    q_tables.append(q_table)\n",
    "    #print(endtime)\n",
    "    #lines = (\"Updating QL_sched with thresh = \", str(thresh),\" is = \", str(endtime))\n",
    "    #with open('sched_updating_time.txt', 'w') as f:\n",
    "    #    for line in lines:\n",
    "    #        f.write(line)\n",
    "    #    f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e1789a1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = (\"Updating CLass-based_sched time = \", str(cb_time))\n",
    "\n",
    "with open('sched_updating_time.txt', 'w') as f:\n",
    "    for line in lines:\n",
    "        f.write(line)\n",
    "    f.write('\\n')\n",
    "    for i in range(len(thresh_list)):\n",
    "        lines = (\"Updating QL_sched with thresh = \", str(thresh_list[i]),\" is = \", str(upd_times[i]),'\\n')\n",
    "        for line in lines:\n",
    "            f.write(line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70801c29",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}