data_creation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4b7fe8c1-2f4d-43c6-8d1c-77f36412a62f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "from statsmodels.tsa.arima.model import ARIMA\n",
    "# from dart import ts_spikes\n",
    "from statsmodels.stats.diagnostic import het_arch\n",
    "from statsmodels.tsa.ar_model import AutoReg\n",
    "from statsmodels.tsa.seasonal import STL\n",
    "from scipy.optimize import curve_fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2a0d150a-73a9-4c46-8a99-ffcff8a8c8d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "GRID_SIZE = 50\n",
    "NUMBER = 1000\n",
    "NUMBER_OF_GROUPS = 10\n",
    "NUMBER_OF_TILES = NUMBER // NUMBER_OF_GROUPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b8458c2-6733-4eed-be21-f7c930eb3d96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_data(df,grid_size, number_of_tiles, number):\n",
    "    '''Creates the data from a given array'''\n",
    "    df = df.sort_values(by = ['latitude', 'longitude']).reset_index(drop=True)\n",
    "    df['temporal_group'] = np.tile(np.arange(number) // (number_of_tiles), grid_size * grid_size)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "404728b5-d668-4f78-ac4e-d5e33202c474",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "latitude = np.repeat(range(GRID_SIZE),GRID_SIZE * NUMBER)\n",
    "longitude = np.tile(np.arange(GRID_SIZE),GRID_SIZE * NUMBER) \n",
    "power = np.random.random(GRID_SIZE*GRID_SIZE*NUMBER)\n",
    "df = pd.DataFrame({'latitude' : latitude, 'longitude' : longitude, 'power' :  power})\n",
    "df = create_data(df, GRID_SIZE, NUMBER_OF_TILES, NUMBER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c8ec0673-f553-4fe7-ad12-d27a5127cc6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "latitude  longitude\n",
       "0         0            9\n",
       "          1            9\n",
       "          2            9\n",
       "          3            9\n",
       "          4            9\n",
       "                      ..\n",
       "49        45           9\n",
       "          46           9\n",
       "          47           9\n",
       "          48           9\n",
       "          49           9\n",
       "Name: temporal_group, Length: 2500, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['latitude', 'longitude'])['temporal_group'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "84b8793d-37e6-4ccc-b04f-0f281ea287cf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2534448/1435448137.py:123: FutureWarning: the 'maxlag'' keyword is deprecated, use 'nlags' instead.\n",
      "  lag, arch_stat, p_value, f_test = het_arch(x, maxlag=10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"21\" halign=\"left\">power</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>var</th>\n",
       "      <th>skew</th>\n",
       "      <th>x_acf1</th>\n",
       "      <th>diff1_acf1</th>\n",
       "      <th>diff2_acf1</th>\n",
       "      <th>x_acf10</th>\n",
       "      <th>diff1_acf10</th>\n",
       "      <th>diff2_acf10</th>\n",
       "      <th>entropy</th>\n",
       "      <th>...</th>\n",
       "      <th>nonlinear</th>\n",
       "      <th>linearity</th>\n",
       "      <th>curvature</th>\n",
       "      <th>x_pacf5</th>\n",
       "      <th>diff1_pacf5</th>\n",
       "      <th>diff2_pacf5</th>\n",
       "      <th>lumpiness</th>\n",
       "      <th>stability</th>\n",
       "      <th>arch_stat</th>\n",
       "      <th>trend</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>0.480224</td>\n",
       "      <td>0.083460</td>\n",
       "      <td>0.076767</td>\n",
       "      <td>-0.037106</td>\n",
       "      <td>-0.511272</td>\n",
       "      <td>-0.673846</td>\n",
       "      <td>1.006728</td>\n",
       "      <td>1.273023</td>\n",
       "      <td>1.503124</td>\n",
       "      <td>364.853856</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.912568</td>\n",
       "      <td>-0.282875</td>\n",
       "      <td>-0.892816</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.656494</td>\n",
       "      <td>0.052160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.506503</td>\n",
       "      <td>0.082065</td>\n",
       "      <td>0.001629</td>\n",
       "      <td>0.052060</td>\n",
       "      <td>-0.486049</td>\n",
       "      <td>-0.669547</td>\n",
       "      <td>1.015216</td>\n",
       "      <td>1.253957</td>\n",
       "      <td>1.505451</td>\n",
       "      <td>362.453901</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.042335</td>\n",
       "      <td>-0.240502</td>\n",
       "      <td>-0.902749</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.146294</td>\n",
       "      <td>0.060540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.488894</td>\n",
       "      <td>0.087774</td>\n",
       "      <td>0.047771</td>\n",
       "      <td>0.010518</td>\n",
       "      <td>-0.512061</td>\n",
       "      <td>-0.682145</td>\n",
       "      <td>1.005324</td>\n",
       "      <td>1.276243</td>\n",
       "      <td>1.528818</td>\n",
       "      <td>355.343115</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.041523</td>\n",
       "      <td>-0.257595</td>\n",
       "      <td>-0.894067</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.939799</td>\n",
       "      <td>0.073461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.496995</td>\n",
       "      <td>0.084856</td>\n",
       "      <td>0.007271</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>-0.487072</td>\n",
       "      <td>-0.654608</td>\n",
       "      <td>1.009520</td>\n",
       "      <td>1.251086</td>\n",
       "      <td>1.466608</td>\n",
       "      <td>357.517898</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.083152</td>\n",
       "      <td>-0.313164</td>\n",
       "      <td>-0.907293</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.357230</td>\n",
       "      <td>0.079207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.497579</td>\n",
       "      <td>0.084707</td>\n",
       "      <td>0.005146</td>\n",
       "      <td>-0.016521</td>\n",
       "      <td>-0.520909</td>\n",
       "      <td>-0.673413</td>\n",
       "      <td>1.010381</td>\n",
       "      <td>1.282173</td>\n",
       "      <td>1.492492</td>\n",
       "      <td>358.453335</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.076921</td>\n",
       "      <td>-0.314825</td>\n",
       "      <td>-0.933823</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.372098</td>\n",
       "      <td>0.083073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">49</th>\n",
       "      <th>45</th>\n",
       "      <td>0.510430</td>\n",
       "      <td>0.083715</td>\n",
       "      <td>-0.033505</td>\n",
       "      <td>0.032233</td>\n",
       "      <td>-0.480992</td>\n",
       "      <td>-0.640338</td>\n",
       "      <td>1.011867</td>\n",
       "      <td>1.251169</td>\n",
       "      <td>1.433355</td>\n",
       "      <td>357.862872</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.111075</td>\n",
       "      <td>-0.233906</td>\n",
       "      <td>-0.893584</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.199607</td>\n",
       "      <td>0.079097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.505560</td>\n",
       "      <td>0.086940</td>\n",
       "      <td>-0.031735</td>\n",
       "      <td>-0.006437</td>\n",
       "      <td>-0.515921</td>\n",
       "      <td>-0.678583</td>\n",
       "      <td>1.008451</td>\n",
       "      <td>1.277575</td>\n",
       "      <td>1.506224</td>\n",
       "      <td>352.704758</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.987168</td>\n",
       "      <td>-0.301243</td>\n",
       "      <td>-0.913955</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.661778</td>\n",
       "      <td>0.062250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.513288</td>\n",
       "      <td>0.084247</td>\n",
       "      <td>-0.055424</td>\n",
       "      <td>-0.043909</td>\n",
       "      <td>-0.503063</td>\n",
       "      <td>-0.660203</td>\n",
       "      <td>1.016531</td>\n",
       "      <td>1.275182</td>\n",
       "      <td>1.476668</td>\n",
       "      <td>355.996286</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.894528</td>\n",
       "      <td>-0.324548</td>\n",
       "      <td>-0.914972</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.170467</td>\n",
       "      <td>0.050416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.485466</td>\n",
       "      <td>0.080335</td>\n",
       "      <td>0.065583</td>\n",
       "      <td>-0.001047</td>\n",
       "      <td>-0.487571</td>\n",
       "      <td>-0.668763</td>\n",
       "      <td>1.014526</td>\n",
       "      <td>1.264626</td>\n",
       "      <td>1.518362</td>\n",
       "      <td>369.269242</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.925504</td>\n",
       "      <td>-0.256119</td>\n",
       "      <td>-0.876133</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.228770</td>\n",
       "      <td>0.047034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.490147</td>\n",
       "      <td>0.084658</td>\n",
       "      <td>0.012641</td>\n",
       "      <td>-0.026176</td>\n",
       "      <td>-0.492013</td>\n",
       "      <td>-0.659094</td>\n",
       "      <td>1.005871</td>\n",
       "      <td>1.245872</td>\n",
       "      <td>1.461891</td>\n",
       "      <td>360.404851</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.884036</td>\n",
       "      <td>-0.341345</td>\n",
       "      <td>-0.906595</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.960807</td>\n",
       "      <td>0.052488</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2500 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       power                                           \\\n",
       "                        mean       var      skew    x_acf1 diff1_acf1   \n",
       "latitude longitude                                                      \n",
       "0        0          0.480224  0.083460  0.076767 -0.037106  -0.511272   \n",
       "         1          0.506503  0.082065  0.001629  0.052060  -0.486049   \n",
       "         2          0.488894  0.087774  0.047771  0.010518  -0.512061   \n",
       "         3          0.496995  0.084856  0.007271  0.023810  -0.487072   \n",
       "         4          0.497579  0.084707  0.005146 -0.016521  -0.520909   \n",
       "...                      ...       ...       ...       ...        ...   \n",
       "49       45         0.510430  0.083715 -0.033505  0.032233  -0.480992   \n",
       "         46         0.505560  0.086940 -0.031735 -0.006437  -0.515921   \n",
       "         47         0.513288  0.084247 -0.055424 -0.043909  -0.503063   \n",
       "         48         0.485466  0.080335  0.065583 -0.001047  -0.487571   \n",
       "         49         0.490147  0.084658  0.012641 -0.026176  -0.492013   \n",
       "\n",
       "                                                                             \\\n",
       "                   diff2_acf1   x_acf10 diff1_acf10 diff2_acf10     entropy   \n",
       "latitude longitude                                                            \n",
       "0        0          -0.673846  1.006728    1.273023    1.503124  364.853856   \n",
       "         1          -0.669547  1.015216    1.253957    1.505451  362.453901   \n",
       "         2          -0.682145  1.005324    1.276243    1.528818  355.343115   \n",
       "         3          -0.654608  1.009520    1.251086    1.466608  357.517898   \n",
       "         4          -0.673413  1.010381    1.282173    1.492492  358.453335   \n",
       "...                       ...       ...         ...         ...         ...   \n",
       "49       45         -0.640338  1.011867    1.251169    1.433355  357.862872   \n",
       "         46         -0.678583  1.008451    1.277575    1.506224  352.704758   \n",
       "         47         -0.660203  1.016531    1.275182    1.476668  355.996286   \n",
       "         48         -0.668763  1.014526    1.264626    1.518362  369.269242   \n",
       "         49         -0.659094  1.005871    1.245872    1.461891  360.404851   \n",
       "\n",
       "                    ...                                                      \\\n",
       "                    ... nonlinear linearity curvature   x_pacf5 diff1_pacf5   \n",
       "latitude longitude  ...                                                       \n",
       "0        0          ...      None      None      None  0.912568   -0.282875   \n",
       "         1          ...      None      None      None  1.042335   -0.240502   \n",
       "         2          ...      None      None      None  1.041523   -0.257595   \n",
       "         3          ...      None      None      None  1.083152   -0.313164   \n",
       "         4          ...      None      None      None  1.076921   -0.314825   \n",
       "...                 ...       ...       ...       ...       ...         ...   \n",
       "49       45         ...      None      None      None  1.111075   -0.233906   \n",
       "         46         ...      None      None      None  0.987168   -0.301243   \n",
       "         47         ...      None      None      None  0.894528   -0.324548   \n",
       "         48         ...      None      None      None  0.925504   -0.256119   \n",
       "         49         ...      None      None      None  0.884036   -0.341345   \n",
       "\n",
       "                                                                        \n",
       "                   diff2_pacf5 lumpiness stability arch_stat     trend  \n",
       "latitude longitude                                                      \n",
       "0        0           -0.892816      None      None  0.656494  0.052160  \n",
       "         1           -0.902749      None      None  0.146294  0.060540  \n",
       "         2           -0.894067      None      None  0.939799  0.073461  \n",
       "         3           -0.907293      None      None  0.357230  0.079207  \n",
       "         4           -0.933823      None      None  0.372098  0.083073  \n",
       "...                        ...       ...       ...       ...       ...  \n",
       "49       45          -0.893584      None      None  0.199607  0.079097  \n",
       "         46          -0.913955      None      None  0.661778  0.062250  \n",
       "         47          -0.914972      None      None  0.170467  0.050416  \n",
       "         48          -0.876133      None      None  0.228770  0.047034  \n",
       "         49          -0.906595      None      None  0.960807  0.052488  \n",
       "\n",
       "[2500 rows x 22 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating aggregations\n",
    "aggregated_dict = { \n",
    "                    'power' : [\n",
    "                                'mean',\n",
    "                                'var',\n",
    "                                # slope,\n",
    "                                # last_CSO_value,\n",
    "                                'skew',\n",
    "                                x_acf1,\n",
    "                                diff1_acf1,\n",
    "                                diff2_acf1,\n",
    "                                # e_acf1,\n",
    "                                x_acf10,\n",
    "                                diff1_acf10,\n",
    "                                diff2_acf10,\n",
    "                                # e_acf10,\n",
    "                                entropy,\n",
    "                                crossing_points,\n",
    "                                flat_spots,\n",
    "                                nonlinear,\n",
    "                                linearity,\n",
    "                                curvature,\n",
    "                                x_pacf5,\n",
    "                                diff1_pacf5,\n",
    "                                diff2_pacf5,\n",
    "                                lumpiness,\n",
    "                                stability,\n",
    "                                arch_stat,\n",
    "                                trend\n",
    "                            ]\n",
    "                    }\n",
    "aggregated_data = df.groupby(['latitude', 'longitude']).agg(aggregated_dict)\n",
    "aggregated_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "896bce34-f1c6-4931-bb94-eac109347f51",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "_,group = next(iter(df.groupby(['temporal_group', 'latitude', 'longitude'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6f7706c1-191c-41d4-9715-fe520cdcd3a4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function __main__.diff1_acf1(x)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff1_acf1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "a717a056-6f7f-474e-986e-d29a23b39221",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Different aggregate functions\n",
    "def difference_time_series(x):\n",
    "    ''''''\n",
    "    return x.diff().dropna()\n",
    "\n",
    "\n",
    "def x_acf1(x):\n",
    "    '''The autocorrelation function of the series'''\n",
    "    return acf(x, nlags=1)[1]\n",
    "\n",
    "\n",
    "def diff1_acf1(x):\n",
    "    '''The autocorrelation function of the differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    return acf(first_difference, nlags=1)[1]\n",
    "\n",
    "\n",
    "def diff2_acf1(x):\n",
    "    '''The autocorrelation function of the second-order differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    second_difference = difference_time_series(first_difference)\n",
    "    return acf(second_difference, nlags=1)[1]\n",
    "\n",
    "\n",
    "def e_acf1(x):\n",
    "    '''The autocorrelation function of the residuals'''\n",
    "    return acf(data - x.rolling(window=2).mean().dropna(), nlags=1)[1]\n",
    "\n",
    "\n",
    "def x_acf10(x):\n",
    "    '''The sum of squares of the first 10 autocorrelation coefficients for series'''\n",
    "    return sum(acf(x, nlags=10)**2)\n",
    "\n",
    "def diff1_acf10(x):\n",
    "    '''The sum of squares of the first 10 autocorrelation coefficients for the differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    return sum(acf(first_difference, nlags=10)**2)\n",
    "\n",
    "def diff2_acf10(x):\n",
    "    '''The sum of squares of the first 10 autocorrelation coefficients for second-order the differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    second_difference = difference_time_series(first_difference)\n",
    "    return sum(acf(second_difference, nlags=10)**2)\n",
    "\n",
    "\n",
    "def e_acf10(x,window_size=2):\n",
    "    '''The sum of squares of the first 10 autocorrelation coefficients of the residuals'''\n",
    "    # Calculate the rolling mean\n",
    "    rolling_mean = data_pd.rolling(window=window_size).mean().dropna()\n",
    "\n",
    "    # Calculate the residuals by subtracting the rolling mean from the original data\n",
    "    residuals = data[:-window_size+1] - rolling_mean.values\n",
    "\n",
    "    # Compute the autocorrelation function for the residuals\n",
    "    e_acf = acf(residuals, nlags=10)\n",
    "\n",
    "    # Calculate the sum of squares of the first 10 autocorrelation coefficients\n",
    "    e_acf10 = np.sum(np.square(e_acf[:10]))\n",
    "\n",
    "    return e_acf10\n",
    "\n",
    "\n",
    "def entropy(x):\n",
    "    '''The spectral entropy is the Shannon entropy'''\n",
    "    return -np.sum(x * np.log2(x))\n",
    "\n",
    "\n",
    "def crossing_points(x):\n",
    "    '''The number of times the temporal data-set crosses the median line'''\n",
    "    return len(np.where(np.diff((x > x.median())))[0])\n",
    "\n",
    "\n",
    "def flat_spots(x):\n",
    "    '''The temporal dataset is divided into ten equally blocks then the largest runlength represents the value of flat_spots'''\n",
    "    pass\n",
    "\n",
    "\n",
    "def nonlinear(x):\n",
    "    '''The nonlinearity is estimated from a modified Teräsvirta’s test'''\n",
    "    pass\n",
    "\n",
    "\n",
    "def linearity(x):\n",
    "    ''' The strength of curvature are estimated from the coefficients of the orthogonal quadratic regression.'''\n",
    "    pass\n",
    "\n",
    "\n",
    "def curvature(x):\n",
    "    ''''''\n",
    "    pass\n",
    "\n",
    "\n",
    "def x_pacf5(x):\n",
    "    '''Thee sum of the first 5 partial autocorrelation coefficients of the series'''\n",
    "    return np.sum(pacf(x, nlags=5)[:5])\n",
    "\n",
    "\n",
    "def diff1_pacf5(x):\n",
    "    ''''The sum of the first 5 partial autocorrelation coefficients of the differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    return np.sum(pacf(first_difference, nlags=5)[:5])\n",
    "    \n",
    "\n",
    "def diff2_pacf5(x):\n",
    "    '''The sum of the first 5 partial autocorrelation coefficients of the second-order differenced series'''\n",
    "    first_difference = difference_time_series(x)\n",
    "    second_difference = difference_time_series(first_difference)\n",
    "    return np.sum(pacf(second_difference, nlags=5)[:5])\n",
    "\n",
    "\n",
    "def lumpiness(x):\n",
    "    '''Temporal dataset is divided into non-overlapping windoes the variance of the mean of the tiled windows'''\n",
    "    pass\n",
    "\n",
    "\n",
    "def stability(x):\n",
    "    '''Temporal dataset is divided into non-overlapping windoes the variance of the variance of the tiled windows'''\n",
    "    pass\n",
    "\n",
    "\n",
    "def arch_stat(x):\n",
    "    '''Stability based on Lagrange Multiplier Test'''\n",
    "    lag, arch_stat, p_value, f_test = het_arch(x, nlags=10)\n",
    "    return arch_stat\n",
    "\n",
    "\n",
    "def trend(x):\n",
    "    '''The strength of the trend is found from Seasonal-Trend decomposition using LOESS'''\n",
    "    stl = STL(x, period=12)\n",
    "    res = stl.fit()\n",
    "\n",
    "    # calculate trend strength measure\n",
    "    trend_var, total_var = np.var(res.trend), np.var(x)\n",
    "    return trend_var / total_var\n",
    "\n",
    "\n",
    "def spike(x):\n",
    "    '''Variance of the leave-one-out variances of the residuals'''\n",
    "    model = ARIMA(x, order=(1,1,1))\n",
    "    model_fit = model.fit()\n",
    "\n",
    "    # calculate residuals\n",
    "    residuals = pd.Series(model_fit.resid)\n",
    "\n",
    "    # calculate variance of residuals\n",
    "    residuals_var = np.var(residuals)\n",
    "\n",
    "    # calculate leave-one-out variances of residuals\n",
    "    loocv_variances = []\n",
    "    for i in range(len(residuals)):\n",
    "        # remove one observation from the data\n",
    "        data_loo = x.drop(data.index[i])\n",
    "        # fit ARIMA model to the leave-one-out data\n",
    "        model_loo = ARIMA(data_loo, order=(1,1,1))\n",
    "        model_fit_loo = model_loo.fit()\n",
    "        # calculate residuals of the leave-one-out model\n",
    "        residuals_loo = pd.Series(model_fit_loo.resid)\n",
    "        # calculate variance of residuals of the leave-one-out model\n",
    "        residuals_var_loo = np.var(residuals_loo)\n",
    "        # add variance to the list\n",
    "        loocv_variances.append(residuals_var_loo)\n",
    "\n",
    "    # calculate spike as the variance of the leave-one-out variances of residuals\n",
    "    spike = np.var(loocv_variances)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}