Chapter3.Rmd

---
title: "Chapter 3"
author: "Julin Maloof"
date: "1/11/2021"
output: html_document
---

## Installations

NOT WORKING set up python environment, only has to be done once 
```{r, eval=FALSE}
virtualenv_remove("r-python3.8")
virtualenv_create("r-python3.8", python="/usr/local/opt/python@3.8/bin/python3")
```

NOT WORKING
```{r, eval=FALSE}
use_virtualenv("r-python3.8")
install_keras(method="virtualenv", envname="r-python3.8")
```

GIVING UP AND TRYING CONDA METHOD

```{r, eval=FALSE}
#only needs to be done once
install.packages("keras")
install.packages("reticulate")
library(keras)
library(reticulate)
#use_condaenv("tf")
conda_create(packages="python=3.6")
use_condaenv("r-reticulate")
install_keras(method="conda", envname="r-reticulate")
```

## New installation usiing new mac tensor flow

But this actually runs slower on at least Boston dataset.  So, use conda insall, above.
```{bash, eval=FALSE}
 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/apple/tensorflow_macos/master/scripts/download_and_install.sh)"
 
var/folders/xr/9cbydt955pj42zfq6mc_y5g40000gn/T/tmp.ggNkh5wb/tensorflow_macos/install_venv.sh --python=/usr/local/opt/python@3.8/bin/python3 /Users/jmaloof/.virtualenvs/tensorflow_macos_venv/

. "/Users/jmaloof/.virtualenvs/tensorflow_macos_venv/bin/activate"

pip install keras
pip install h5py pyyaml requests Pillow scipy
```


## load libraries

don't use this one
```{r, eval=FALSE}
library(keras)
library(reticulate)
use_virtualenv("~/.virtualenvs/tensorflow_macos_venv/", required = TRUE )
py_module_available("tensorflow") 
library(tensorflow)
```
above is slower on old MacBook processing Boston data set.  Use the conda install instead.


## alternate load libraries:
use this one
```{r}
library(keras)
use_condaenv("r-reticulate")
```


## Reuters data set

get the data
```{r}
reuters <- dataset_reuters(num_words = 10000)
c(c(train_data, train_labels), c(test_data, test_labels)) %<-% reuters
```

```{r}
length(train_data)
length(test_data)
```

vectorize data
```{r}
vectorize_sequences <- function(sequences, dimension = 10000) {
  results <- matrix(0, nrow = length(sequences), ncol = dimension)
  for (i in 1:length(sequences))
    results[i, sequences[[i]]] <- 1
  results
}

x_train <- vectorize_sequences(train_data)            
x_test <- vectorize_sequences(test_data)              
```

vectorize labels
```{r}
one_hot_train_labels <- to_categorical(train_labels)
dim(one_hot_train_labels)
one_hot_test_labels <- to_categorical(test_labels)
```

model
```{r}
model <- keras_model_sequential() %>%
  layer_dense(units = 64, activation = "relu", input_shape = c(10000)) %>%
  layer_dense(units = 64, activation = "relu") %>%
  layer_dense(units = 46, activation = "softmax")
```

```{r}
model %>% compile(
  optimizer = "rmsprop",
  loss = "categorical_crossentropy",
  metrics = c("accuracy")
)
```

validation set:
```{r}
val_indices <- 1:1000

x_val <- x_train[val_indices,]
partial_x_train <- x_train[-val_indices,]

y_val <- one_hot_train_labels[val_indices,]
partial_y_train = one_hot_train_labels[-val_indices,]
```

run it
```{r}
system.time(
  history <- model %>% fit(
    partial_x_train,
    partial_y_train,
    epochs = 20,
    batch_size = 512,
    validation_data = list(x_val, y_val)
  )
)
```

```{r}
plot(history)
```
retrain
```{r}
model <- keras_model_sequential() %>%
  layer_dense(units = 64, activation = "relu", input_shape = c(10000)) %>%
  layer_dense(units = 64, activation = "relu") %>%
  layer_dense(units = 46, activation = "softmax")

model %>% compile(
  optimizer = "rmsprop",
  loss = "categorical_crossentropy",
  metrics = c("accuracy")
)

history <- model %>% fit(
  partial_x_train,
  partial_y_train,
  epochs = 9,
  batch_size = 512,
  validation_data = list(x_val, y_val)
)

results <- model %>% evaluate(x_test, one_hot_test_labels)
```

```{r}
results
```

## Boston data set

get the data
```{r}
dataset <- dataset_boston_housing()
c(c(train_data, train_targets), c(test_data, test_targets)) %<-% dataset
```

```{r}
str(train_data)
str(test_data)
```

```{r}
str(train_targets)
```

scale and standardize

```{r}
mean <- apply(train_data, 2, mean)                                  
std <- apply(train_data, 2, sd)
train_data <- scale(train_data, center = mean, scale = std)         
test_data <- scale(test_data, center = mean, scale = std)
```

```{r}
build_model <- function() {                                
  model <- keras_model_sequential() %>%
    layer_dense(units = 64, activation = "relu",
                input_shape = dim(train_data)[[2]]) %>%
    layer_dense(units = 64, activation = "relu") %>%
    layer_dense(units = 1)
  model %>% compile(
    optimizer = "rmsprop",
    loss = "mse",
    metrics = c("mae")
  )
}
```

```{r}
k <- 4
indices <- sample(1:nrow(train_data))
folds <- cut(indices, breaks = k, labels = FALSE)

num_epochs <- 100
all_scores <- c()
for (i in 1:k) {
  cat("processing fold #", i, "\n")
  
  val_indices <- which(folds == i, arr.ind = TRUE)                     
  val_data <- train_data[val_indices,]
  val_targets <- train_targets[val_indices]
  partial_train_data <- train_data[-val_indices,]                      
  partial_train_targets <- train_targets[-val_indices]
  
  model <- build_model()                                               
  
  print(
    system.time(#52.229   7.077  40.545  royce-2, with apple tensor flow
                #46.309   6.561  36.486  royce-2, with conda tensor flow
      model %>% fit(partial_train_data, partial_train_targets,             
                    epochs = num_epochs, batch_size = 1, verbose = 0)
    )
  )
  
  results <- model %>% evaluate(val_data, val_targets, verbose = 0)    
  #all_scores <- c(all_scores, results$mean_absolute_error)
  all_scores <- c(all_scores, results["mae"])
}
```

```{r}
all_scores
mean(all_scores)
```


save each epoch:
```{r}
num_epochs <- 500
all_mae_histories <- NULL
for (i in 1:k) {
  cat("processing fold #", i, "\n")
  
  val_indices <- which(folds == i, arr.ind = TRUE)
  val_data <- train_data[val_indices,]
  val_targets <- train_targets[val_indices]
  
  partial_train_data <- train_data[-val_indices,]
  partial_train_targets <- train_targets[-val_indices]
  
  model <- build_model()  
  
  print(
    system.time( #329.849  36.252 278.699 
      history <- model %>% fit(                                    
        partial_train_data, partial_train_targets,
        validation_data = list(val_data, val_targets),
        epochs = num_epochs, batch_size = 1, verbose = 0
      )
    )
  )
  
  mae_history <- history$metrics$val_mae
  all_mae_histories <- rbind(all_mae_histories, mae_history)
}
```

```{r}
average_mae_history <- data.frame(
  epoch = seq(1:ncol(all_mae_histories)),
  validation_mae = apply(all_mae_histories, 2, mean)
)
```

```{r}
library(ggplot2)
ggplot(average_mae_history, aes(x = epoch, y = validation_mae)) + geom_line()
```

```{r}
model <- build_model()
model %>% fit(train_data, train_targets,                    
          epochs = 80, batch_size = 16, verbose = 0)
result <- model %>% evaluate(test_data, test_targets)
```

```{r}
result
```

## OJ

```{r}
library(tidyverse)
library(ISLR)
data(OJ)
?OJ
```

```{r}
OJ
```

```{r}
OJ %>% select(StoreID, STORE) # these are redundant
```

because store is categorical we need to turn that into a series of dummy variables.
```{r}
store_cat <- OJ %>% select(StoreID) %>%
  mutate(row=1:nrow(.),data=1) %>%
  pivot_wider(id_cols = row, names_from=StoreID, values_from=data, values_fill=0, names_prefix="Store")

store_cat
```

```{r}
OJ <- OJ %>% select(-StoreID, -STORE, -Store7) %>% cbind(store_cat)
OJ
```

### split into test and train and other formatting

```{r}
set.seed(111820)
train <- sample(1:nrow(OJ), size = 800)
oj.train <- OJ[train,]
oj.test <- OJ[-train,]
```

```{r}
oj.train.label <- oj.train %>% select(Purchase) %>% mutate(Purchase=ifelse(Purchase=="CH", 0, 1)) %>% pull(Purchase)
oj.test.label <- oj.test %>% select(Purchase) %>% mutate(Purchase=ifelse(Purchase=="CH", 0, 1)) %>% pull(Purchase)
```

scale it
```{r}
oj.train <- oj.train %>% select(-Purchase)
oj.test <- oj.test %>% select(-Purchase)

oj.mean <- apply(oj.train, 2, mean)
oj.std <- apply(oj.test, 2, sd)

oj.train <- scale(oj.train, center=oj.mean, scale=oj.std)
oj.test <- scale(oj.test, center=oj.mean, scale=oj.std)
```


### validation set

```{r}
set.seed(12312)
val <- sample(1:nrow(oj.train), size = 100)

oj.train.val <- oj.train[val,]
oj.train.label.val <- oj.train.label[val]

oj.train.part <- oj.train[-val,]
oj.train.label.part <- oj.train.label[-val]

```


### set up and run the model!

```{r}
model <- keras_model_sequential() %>% 
  layer_dense(units = 64, activation = "relu", input_shape = ncol(oj.train) ) %>% 
  layer_dense(units = 16, activation = "relu") %>% 
  layer_dense(units = 1, activation = "sigmoid")


model %>% compile(
  optimizer = "rmsprop",
  loss = "binary_crossentropy",
  metrics = c("accuracy")
)

system.time(history <- model %>% fit(
  oj.train.part,
  oj.train.label.part,
  epochs = 100,
  batch_size = 256,
  validation_data = list(oj.train.val, oj.train.label.val),
  verbose=0 # delete this if you want real-time plots
))
```

```{r}
plot(history)
```

60 seems good, re do with 60 epochs and full training:

```{r}
model <- keras_model_sequential() %>% 
  layer_dense(units = 64, activation = "relu", input_shape = ncol(oj.train) ) %>% 
  layer_dense(units = 16, activation = "relu") %>% 
  layer_dense(units = 1, activation = "sigmoid")


model %>% compile(
  optimizer = "rmsprop",
  loss = "binary_crossentropy",
  metrics = c("accuracy")
)

system.time(history <- model %>% fit(
  oj.train,
  oj.train.label,
  epochs = 100,
  batch_size = 256,
  verbose=0
))
```


```{r}
plot(history)
```


```{r}
results <- model %>% evaluate(oj.test, oj.test.label)

results
```

Accuracy is 84.8%, which is a little bit better than the trees, although I also modified the input data.