Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Gpu batch #35

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Gpu batch #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e2d398a
Push batch on cuda
Jeanselme Jan 27, 2021
c87ece7
Correction on cuda arg
Jeanselme Jan 27, 2021
089e0c5
RNN and CNN with cuda
Jeanselme Jan 27, 2021
a9cdbd6
Update loss for cuda
Jeanselme Jan 27, 2021
bd45a8f
Merge branch 'master' into GPU_Batch
Jeanselme Feb 8, 2021
37ab640
Allow loss test on gpu
Jeanselme Feb 9, 2021
e9dbd3e
Loss .item()
Jeanselme Feb 9, 2021
1c9850a
Avoid repeated code
Jeanselme Feb 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 40 additions & 12 deletions dsm/dsm_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,16 @@ class DSMBase():
"""Base Class for all DSM models"""

def __init__(self, k=3, layers=None, distribution="Weibull",
temp=1000., discount=1.0):
temp=1000., discount=1.0, cuda=False):
self.k = k
self.layers = layers
self.dist = distribution
self.temp = temp
self.discount = discount
self.fitted = False

self.cuda = cuda
self.torch_model = None

def _gen_torch_model(self, inputdim, optimizer, risks):
"""Helper function to return a torch model."""
Expand All @@ -71,6 +74,12 @@ def _gen_torch_model(self, inputdim, optimizer, risks):
optimizer=optimizer,
risks=risks)

def cpu(self):
self.cuda = False
if self.torch_model:
self.torch_model = self.torch_model.cpu()
return self

def fit(self, x, t, e, vsize=0.15, val_data=None,
iters=1, learning_rate=1e-3, batch_size=100,
elbo=True, optimizer="Adam", random_state=100):
Expand Down Expand Up @@ -122,13 +131,18 @@ def fit(self, x, t, e, vsize=0.15, val_data=None,

maxrisk = int(np.nanmax(e_train.cpu().numpy()))
model = self._gen_torch_model(inputdim, optimizer, risks=maxrisk)

if self.cuda:
model = model.cuda()

model, _ = train_dsm(model,
x_train, t_train, e_train,
x_val, t_val, e_val,
n_iter=iters,
lr=learning_rate,
elbo=elbo,
bs=batch_size)
bs=batch_size,
cuda=self.cuda)

self.torch_model = model.eval()
self.fitted = True
Expand Down Expand Up @@ -163,15 +177,22 @@ def compute_nll(self, x, t, e):
x_val, t_val, e_val = x_val,\
_reshape_tensor_with_nans(t_val),\
_reshape_tensor_with_nans(e_val)

if self.cuda:
x_val, t_val, e_val = x_val.cuda(), t_val.cuda(), e_val.cuda()

loss = 0
for r in range(self.torch_model.risks):
loss += float(losses.conditional_loss(self.torch_model,
x_val, t_val, e_val, elbo=False,
risk=str(r+1)).detach().numpy())
loss += float(losses.conditional_loss(self.torch_model,
x_val, t_val, e_val, elbo=False,
risk=str(r+1)).detach().cpu().numpy())
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dont think this needs to be detached and put to CPU. torch doesnt track this variable.

return loss

def _prepocess_test_data(self, x):
return torch.from_numpy(x)
data = torch.from_numpy(x)
if self.cuda:
data = data.cuda()
return data

def _prepocess_training_data(self, x, t, e, vsize, val_data, random_state):

Expand Down Expand Up @@ -204,7 +225,6 @@ def _prepocess_training_data(self, x, t, e, vsize, val_data, random_state):
return (x_train, t_train, e_train,
x_val, t_val, e_val)


def predict_mean(self, x, risk=1):
r"""Returns the mean Time-to-Event \( t \)

Expand Down Expand Up @@ -339,12 +359,14 @@ class DeepRecurrentSurvivalMachines(DSMBase):
"""

def __init__(self, k=3, layers=None, hidden=None,
distribution="Weibull", temp=1000., discount=1.0, typ="LSTM"):
distribution="Weibull", temp=1000., discount=1.0, typ="LSTM",
cuda=False):
super(DeepRecurrentSurvivalMachines, self).__init__(k=k,
layers=layers,
distribution=distribution,
temp=temp,
discount=discount)
discount=discount,
cuda=cuda)
self.hidden = hidden
self.typ = typ
def _gen_torch_model(self, inputdim, optimizer, risks):
Expand All @@ -361,7 +383,10 @@ def _gen_torch_model(self, inputdim, optimizer, risks):
risks=risks)

def _prepocess_test_data(self, x):
return torch.from_numpy(_get_padded_features(x))
data = torch.from_numpy(_get_padded_features(x))
if self.cuda:
data = data.cuda()
return data

def _prepocess_training_data(self, x, t, e, vsize, val_data, random_state):
"""RNNs require different preprocessing for variable length sequences"""
Expand Down Expand Up @@ -413,13 +438,16 @@ class DeepConvolutionalSurvivalMachines(DSMBase):
"""

def __init__(self, k=3, layers=None, hidden=None,
distribution="Weibull", temp=1000., discount=1.0, typ="ConvNet"):
distribution='Weibull', temp=1000., discount=1.0, typ='ConvNet',
cuda=False):
super(DeepConvolutionalSurvivalMachines, self).__init__(k=k,
distribution=distribution,
temp=temp,
discount=discount)
discount=discount,
cuda=cuda)
self.hidden = hidden
self.typ = typ

def _gen_torch_model(self, inputdim, optimizer, risks):
"""Helper function to return a torch model."""
return DeepConvolutionalSurvivalMachinesTorch(inputdim,
Expand Down
6 changes: 3 additions & 3 deletions dsm/losses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ def _weibull_cdf(model, x, t_horizon, risk='1'):
k_ = shape
b_ = scale

t_horz = torch.tensor(t_horizon).double()
t_horz = torch.tensor(t_horizon, device=x.device).double()
t_horz = t_horz.repeat(shape.shape[0], 1)

cdfs = []
Expand Down Expand Up @@ -338,7 +338,7 @@ def _lognormal_cdf(model, x, t_horizon, risk='1'):
k_ = shape
b_ = scale

t_horz = torch.tensor(t_horizon).double()
t_horz = torch.tensor(t_horizon, device=x.device).double()
t_horz = t_horz.repeat(shape.shape[0], 1)

cdfs = []
Expand Down Expand Up @@ -375,7 +375,7 @@ def _normal_cdf(model, x, t_horizon, risk='1'):
k_ = shape
b_ = scale

t_horz = torch.tensor(t_horizon).double()
t_horz = torch.tensor(t_horizon, device=x.device).double()
t_horz = t_horz.repeat(shape.shape[0], 1)

cdfs = []
Expand Down
18 changes: 12 additions & 6 deletions dsm/utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ def pretrain_dsm(model, t_train, e_train, t_valid, e_valid,
valid_loss = 0
for r in range(model.risks):
valid_loss += unconditional_loss(premodel, t_valid, e_valid, str(r+1))
valid_loss = valid_loss.detach().cpu().numpy()
valid_loss = valid_loss.item()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets use float

costs.append(valid_loss)
#print(valid_loss)

if np.abs(costs[-1] - oldcost) < thres:
patience += 1
if patience == 3:
Expand Down Expand Up @@ -112,7 +112,7 @@ def train_dsm(model,
x_train, t_train, e_train,
x_valid, t_valid, e_valid,
n_iter=10000, lr=1e-3, elbo=True,
bs=100):
bs=100, cuda=False):
"""Function to train the torch instance of the model."""

logging.info('Pretraining the Underlying Distributions...')
Expand All @@ -132,6 +132,10 @@ def train_dsm(model,
lr=1e-2,
thres=1e-4)

if cuda:
x_valid, t_valid_, e_valid_ = x_valid.cuda(),\
t_valid_.cuda(), e_valid_.cuda()

for r in range(model.risks):
model.shape[str(r+1)].data.fill_(float(premodel.shape[str(r+1)]))
model.scale[str(r+1)].data.fill_(float(premodel.scale[str(r+1)]))
Expand All @@ -154,6 +158,10 @@ def train_dsm(model,
tb = t_train[j*bs:(j+1)*bs]
eb = e_train[j*bs:(j+1)*bs]


if cuda:
xb, tb, eb = xb.cuda(), tb.cuda(), eb.cuda()

if xb.shape[0] == 0:
continue

Expand All @@ -166,7 +174,6 @@ def train_dsm(model,
_reshape_tensor_with_nans(eb),
elbo=elbo,
risk=str(r+1))
#print ("Train Loss:", float(loss))
loss.backward()
optimizer.step()

Expand All @@ -179,8 +186,7 @@ def train_dsm(model,
elbo=False,
risk=str(r+1))

valid_loss = valid_loss.detach().cpu().numpy()
costs.append(float(valid_loss))
costs.append(valid_loss.item())
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets use float()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.item() automatically puts on cpu if necessary and cast it

dics.append(deepcopy(model.state_dict()))

if costs[-1] >= oldcost:
Expand Down