From 59e69641e4e47859d3f771fd9da6c650e048dd69 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Sun, 24 Dec 2023 14:22:39 -0800 Subject: [PATCH 01/32] Correted Model Save Path for Method-1 LSTM --- Code/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/main.py b/Code/main.py index 776dfaa..e6a3d58 100644 --- a/Code/main.py +++ b/Code/main.py @@ -65,7 +65,7 @@ def main(): model_save_path_ae = '../Models/Method1/model_autoencoder_m1.pth' trainer_autoencoder_baseline = Trainer(model_autoencoder, loss_mse, model_save_path_ae) print('Baseline AutoEncoder Trainer Initialized.') - model_save_path_lstm = '../Models/model_lstm_m1.pth' + model_save_path_lstm = '../Models/Method1/model_lstm_m1.pth' trainer_lstm_baseline = Trainer(model_lstm, loss_mse, model_save_path_lstm) print('Baseline LSTM Trainer Initialized.') From 3edb9e93d57ec21d56a0696bdbf7f15cbc2a1ebd Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Sun, 24 Dec 2023 15:02:56 -0800 Subject: [PATCH 02/32] Updated LSTM --- Code/data.py | 12 +++++++----- Code/lstm_model.py | 3 ++- Code/main.py | 13 ++++++------- Code/training.py | 9 +++++++-- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Code/data.py b/Code/data.py index 3defdea..5efeace 100644 --- a/Code/data.py +++ b/Code/data.py @@ -65,15 +65,17 @@ def get_autoencoder_batches(self,val_split): return train_loader, val_loader # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) - def get_lstm_batches(self): + def get_lstm_batches(self,val_split): # Add an extra dimension to the grayscale images tensor greyscale_image_sequence = self.grayscale_images.unsqueeze(0) # Split the sequence into training and validation sets - greyscale_image_sequence_train = greyscale_image_sequence[:, 1::2] # All odd-indexed images for Training - greyscale_image_sequence_val = greyscale_image_sequence # All images for Validation of Interpolated Frames + num_frames = greyscale_image_sequence.size(1) + split_idx = int(num_frames * val_split) + greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] + greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] # Create TensorDatasets - train_data = TensorDataset(greyscale_image_sequence_train) - val_data = TensorDataset(greyscale_image_sequence_val) + train_data = TensorDataset(greyscale_image_sequence_train[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_train[:, 1:self.grayscale_images.size(0):2]) + val_data = TensorDataset(greyscale_image_sequence_val[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_val[:, 1:self.grayscale_images.size(0):2]) # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index de29179..c130ea5 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -52,7 +52,8 @@ def forward(self, input_tensor, cur_state): def init_hidden(self, batch_size, image_size): # Initialize the hidden state and cell state to zeros - height, width = image_size + # height, width = image_size + height, width = int(image_size[0]), int(image_size[1]) return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) diff --git a/Code/main.py b/Code/main.py index e6a3d58..be4a99c 100644 --- a/Code/main.py +++ b/Code/main.py @@ -21,7 +21,6 @@ image_height = 400 image_width = 600 batch_size = 4 -val_split = 0.2 def main(): @@ -41,13 +40,13 @@ def main(): print('Importing Loss Functions Complete.') # Initialize AutoEncoder Model and Import Dataloader (Training, Validation) - data_autoencoder_train, data_autoencoder_val = dataset.get_autoencoder_batches(val_split) + data_autoencoder_train, data_autoencoder_val = dataset.get_autoencoder_batches(val_split=0.2) print('AutoEncoder Model Data Initialized.') model_autoencoder = Grey2RGBAutoEncoder() print('AutoEncoder Model Initialized.') # Initialize LSTM Model and Import Image Sequences (Training, Validation) - grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches() + grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches(val_split=0.8) print('LSTM Model Data Initialized.') C = 1 hidden_size = 64 @@ -93,14 +92,14 @@ def main(): ''' # Method-1 try: - epochs = 5 + epochs = 1 print('M1 AutoEncoder Training Start.') model_autoencoder_m1 = trainer_autoencoder_baseline.train_autoencoder(epochs, data_autoencoder_train, data_autoencoder_val) print('M1 AutoEncoder Training Complete.') except Exception as e: print(f"M1 AutoEncoder Training Error : \n{e}") try: - epochs = 5 + epochs = 1 print('M1 LSTM Training Start.') model_lstm_m1 = trainer_lstm_baseline.train_lstm(epochs, n_interpolate_frames, grey_sequence_train, grey_sequence_val) print('M1 LSTM Training Complete.') @@ -109,7 +108,7 @@ def main(): # Method-2 try: - epochs = 5 + epochs = 1 print('M2 AutoEncoder Training Start.') model_autoencoder_m2 = trainer_autoencoder_m2.train_autoencoder(epochs, data_autoencoder_train, data_autoencoder_val) print('M2 AutoEncoder Training Complete.') @@ -119,7 +118,7 @@ def main(): # Method-3 try: - epochs = 5 + epochs = 1 print('M3 LSTM Training Start.') model_lstm_m3 = trainer_lstm_m3.train_lstm(epochs, n_interpolate_frames, grey_sequence_train, grey_sequence_val) print('M3 LSTM Training Complete.') diff --git a/Code/training.py b/Code/training.py index 7f436ac..e85a0f9 100644 --- a/Code/training.py +++ b/Code/training.py @@ -71,8 +71,13 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): # Validation Loss Calculation self.model.eval() # Set the Model to Evaluation Mode with torch.no_grad(): - val_loss = sum(self.loss_function(self.model(sequence, n_interpolate_frames), sequence).item() for sequence in val_data) # Compute Total Validation Loss - val_loss /= len(val_data) # Compute Average Validation Loss + val_loss = 0.0 + for sequence, target in val_data: + sequence = sequence.to(self.device) + target = target.to(self.device) + output = self.model(sequence, n_interpolate_frames) + val_loss += self.loss_function(output, target).item() # Calculate loss only on interpolated frames + val_loss /= len(val_data) # Print the epoch number and the validation loss print(f'Epoch : {epoch}, Validation Loss : {val_loss}') # If the current validation loss is lower than the best validation loss, save the model From b59b497bda0847e887184a2900e7507327c04208 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Sun, 24 Dec 2023 17:54:34 -0800 Subject: [PATCH 03/32] Tensor Dimention Mismatch Error --- Code/lstm_model.py | 13 ++++++++----- Code/main.py | 5 +++++ Code/training.py | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index c130ea5..9464616 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -51,9 +51,10 @@ def forward(self, input_tensor, cur_state): return h_next, c_next def init_hidden(self, batch_size, image_size): - # Initialize the hidden state and cell state to zeros - # height, width = image_size - height, width = int(image_size[0]), int(image_size[1]) + # Ensure that image_size is a tuple of (height, width) + if not isinstance(image_size, tuple) or not len(image_size) == 2: + raise ValueError("image_size must be a tuple of (height, width)") + height, width = image_size return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) @@ -80,13 +81,14 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): def forward(self, x, n): # Get the batch size, sequence length, and image height and width from the input batch_size, seq_len, _, h, w = x.shape + image_height, image_width = h, w # Initialize the input for the first layer and the lists of hidden and cell states layer_input = x hidden_states = [] cell_states = [] # Initialize the hidden and cell states for each layer for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h, w)) + h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (image_height, image_width)) hidden_states.append(h) cell_states.append(c) # Initialize the list of interpolated sequences with the first frame @@ -103,7 +105,8 @@ def forward(self, x, n): hidden_states[layer], cell_states[layer] = self.conv_lstm_cells[layer](current_frame, (hidden_states[layer], cell_states[layer])) current_frame = hidden_states[layer] # Predict the next frame and store the next hidden and cell states - next_frame_pred, (h_n, c_n) = self.conv_lstm_cells[layer](next_frame, (hidden_states[layer], cell_states[layer])) + h_n, c_n = self.conv_lstm_cells[layer](next_frame, (hidden_states[layer], cell_states[layer])) + next_frame_pred = h_n hidden_states_next, cell_states_next = h_n, c_n # Generate `n` interpolated frames between the current frame and the predicted next frame for j in range(1, n + 1): diff --git a/Code/main.py b/Code/main.py index be4a99c..e87db4c 100644 --- a/Code/main.py +++ b/Code/main.py @@ -12,6 +12,7 @@ # Import Necessary Libraries import os +import traceback # Define Working Directories grayscale_dir = '../Dataset/Greyscale' @@ -98,6 +99,7 @@ def main(): print('M1 AutoEncoder Training Complete.') except Exception as e: print(f"M1 AutoEncoder Training Error : \n{e}") + traceback.print_exc() try: epochs = 1 print('M1 LSTM Training Start.') @@ -105,6 +107,7 @@ def main(): print('M1 LSTM Training Complete.') except Exception as e: print(f"M1 LSTM Training Error : \n{e}") + traceback.print_exc() # Method-2 try: @@ -114,6 +117,7 @@ def main(): print('M2 AutoEncoder Training Complete.') except Exception as e: print(f"M2 AutoEncoder Training Error : \n{e}") + traceback.print_exc() # Method-2 LSTM == Method-1 LSTM, no need to train again # Method-3 @@ -124,6 +128,7 @@ def main(): print('M3 LSTM Training Complete.') except Exception as e: print(f"M3 LSTM Training Error : \n{e}") + traceback.print_exc() # Method-3 AutoEncoder == Method-1 AutoEncoder, no need to train again diff --git a/Code/training.py b/Code/training.py index e85a0f9..7dbebb8 100644 --- a/Code/training.py +++ b/Code/training.py @@ -44,6 +44,7 @@ def train_autoencoder(self, epochs, train_loader, val_loader): # Validation Loss Calculation self.model.eval() # Set the Model to Evaluation Mode with torch.no_grad(): # Disable gradient computation + val_loss = 0.0 val_loss = sum(self.loss_function(self.model(input.to(self.device)), target.to(self.device)).item() for input, target in val_loader) # Compute Total Validation Loss val_loss /= len(val_loader) # Compute Average Validation Loss # Print the epoch number and the validation loss From a8686ab7919c61eb9ced92d9a9036b90184c9871 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Sun, 24 Dec 2023 21:08:23 -0800 Subject: [PATCH 04/32] Experiment Code for LSTM --- Code/data.py | 49 ++++++++++++++++++++++---------- Code/lstm_model.py | 69 ++++++++++++++++------------------------------ Code/training.py | 53 +++++++++++++++++++++++------------ 3 files changed, 93 insertions(+), 78 deletions(-) diff --git a/Code/data.py b/Code/data.py index 5efeace..1c6ef02 100644 --- a/Code/data.py +++ b/Code/data.py @@ -65,18 +65,39 @@ def get_autoencoder_batches(self,val_split): return train_loader, val_loader # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) - def get_lstm_batches(self,val_split): - # Add an extra dimension to the grayscale images tensor - greyscale_image_sequence = self.grayscale_images.unsqueeze(0) - # Split the sequence into training and validation sets - num_frames = greyscale_image_sequence.size(1) - split_idx = int(num_frames * val_split) - greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] - greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] - # Create TensorDatasets - train_data = TensorDataset(greyscale_image_sequence_train[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_train[:, 1:self.grayscale_images.size(0):2]) - val_data = TensorDataset(greyscale_image_sequence_val[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_val[:, 1:self.grayscale_images.size(0):2]) - # Create DataLoaders - train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) - val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) + def get_lstm_batches(self, val_split): + # Assumption: self.grayscale_images is a tensor with shape [num_samples, channels, height, width] + + # Calculate the number of frames for training and validation + num_frames = self.grayscale_images.size(0) + train_frames = int(num_frames * (1 - val_split)) + val_frames = num_frames - train_frames + + # Split the frames into training and validation sets + train_frames_tensor = self.grayscale_images[:train_frames] + val_frames_tensor = self.grayscale_images[-val_frames:] + + # Create sequences (8 frames per sequence here) for training and validation + # The goal is to predict 7 intermediate frames from 8 input frames + input_seq_len = 8 + target_seq_len = 7 + + train_seqs = train_frames_tensor.unfold(0, input_seq_len, 1) + train_targets = train_seqs.roll(-1, dims=0)[:, :target_seq_len] # Shift temporal sequence by one step + + val_seqs = val_frames_tensor.unfold(0, input_seq_len, 1) + val_targets = val_seqs.roll(-1, dims=0)[:, :target_seq_len] # Shift temporal sequence by one step + + # Make sure there is no overlap between training and validation targets + if train_seqs[-1].equals(val_seqs[0]): + val_seqs = val_seqs[1:] + val_targets = val_targets[1:] + + # Create tensor datasets and data loaders + train_dataset = TensorDataset(train_seqs, train_targets) + val_dataset = TensorDataset(val_seqs, val_targets) + + train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=self.batch_size) + return train_loader, val_loader diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 9464616..3bf799d 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -13,7 +13,6 @@ class ConvLSTMCell(nn.Module): def __init__(self, input_channels, hidden_channels, kernel_size, bias=True): super(ConvLSTMCell, self).__init__() - # Store the number of input channels, hidden channels, kernel size, and bias self.input_channels = input_channels self.hidden_channels = hidden_channels self.kernel_size = kernel_size @@ -24,48 +23,36 @@ def __init__(self, input_channels, hidden_channels, kernel_size, bias=True): else: raise ValueError("kernel_size must be an int or a tuple of two ints") self.bias = bias - # Define a convolutional layer that takes both the input and the hidden state as input self.conv = nn.Conv2d(in_channels=self.input_channels + self.hidden_channels, - out_channels=4 * self.hidden_channels, # Four sets of filters for the LSTM gates + out_channels=4 * self.hidden_channels, kernel_size=self.kernel_size, padding=self.padding, bias=self.bias) + def forward(self, input_tensor, cur_state): - # Unpack the current state h_cur, c_cur = cur_state - # Concatenate the input tensor and the current hidden state along the channel dimension - combined = torch.cat([input_tensor, h_cur], dim=1) - # Apply the convolutional layer + combined = torch.cat([input_tensor, h_cur], dim=1) combined_conv = self.conv(combined) - # Split the convolutional output into four parts for the LSTM gates cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_channels, dim=1) - # Compute the values for the input, forget, output gates and cell state i = torch.sigmoid(cc_i) f = torch.sigmoid(cc_f) o = torch.sigmoid(cc_o) g = torch.tanh(cc_g) - # Compute the next cell state and hidden state c_next = f * c_cur + i * g h_next = o * torch.tanh(c_next) - # Return the next hidden state and cell state return h_next, c_next def init_hidden(self, batch_size, image_size): - # Ensure that image_size is a tuple of (height, width) - if not isinstance(image_size, tuple) or not len(image_size) == 2: - raise ValueError("image_size must be a tuple of (height, width)") height, width = image_size return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) - + # Frame Interpolation model using ConvLSTM cells class FrameInterpolationLSTM(nn.Module): def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): super(FrameInterpolationLSTM, self).__init__() - # Store the hidden dimension and number of layers self.hidden_dim = hidden_dim self.num_layers = num_layers - # Define a list of ConvLSTM cells self.conv_lstm_cells = nn.ModuleList([ ConvLSTMCell(input_channels=(input_dim if i == 0 else hidden_dim), hidden_channels=hidden_dim, @@ -73,56 +60,46 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): bias=True) for i in range(num_layers) ]) - # Define a pointwise convolutional layer to map the LSTM output to the desired output feature space self.conv = nn.Conv2d(in_channels=hidden_dim, - out_channels=input_dim, # Assuming we want to generate grayscale images - kernel_size=(1, 1), # pointwise convolution + out_channels=input_dim, + kernel_size=(1, 1), padding=0) + def forward(self, x, n): - # Get the batch size, sequence length, and image height and width from the input batch_size, seq_len, _, h, w = x.shape - image_height, image_width = h, w - # Initialize the input for the first layer and the lists of hidden and cell states layer_input = x hidden_states = [] cell_states = [] - # Initialize the hidden and cell states for each layer + for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (image_height, image_width)) + h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h, w)) hidden_states.append(h) cell_states.append(c) - # Initialize the list of interpolated sequences with the first frame + interpolated_sequences = [x[:, :1]] - # Loop over each time step for t in range(seq_len - 1): - # Get the current frame and the next frame current_frame = layer_input[:, t] next_frame = layer_input[:, t + 1] - interpolations = [current_frame] - # Loop over each layer + interpolations = [current_frame.unsqueeze(1)] + for layer in range(self.num_layers): - # Update the current frame and the hidden and cell states - hidden_states[layer], cell_states[layer] = self.conv_lstm_cells[layer](current_frame, (hidden_states[layer], cell_states[layer])) - current_frame = hidden_states[layer] - # Predict the next frame and store the next hidden and cell states - h_n, c_n = self.conv_lstm_cells[layer](next_frame, (hidden_states[layer], cell_states[layer])) - next_frame_pred = h_n - hidden_states_next, cell_states_next = h_n, c_n - # Generate `n` interpolated frames between the current frame and the predicted next frame + h_cur, c_cur = hidden_states[layer], cell_states[layer] + h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) + current_frame = h_next[:, -1] + + h_n, c_n = self.conv_lstm_cells[layer](next_frame.unsqueeze(1), (h_cur, c_cur)) + next_frame_pred = h_n[:, -1] + hidden_states[layer], cell_states[layer] = h_n, c_n + for j in range(1, n + 1): alpha = j / float(n + 1) interpolated_frame = (1 - alpha) * current_frame + alpha * next_frame_pred interpolated_frame = F.relu(self.conv(interpolated_frame)) interpolations.append(interpolated_frame) - # Append the true next frame and update the hidden and cell states for the next true frame interpolations.append(next_frame) - for layer in range(self.num_layers): - hidden_states[layer], cell_states[layer] = hidden_states_next[layer], cell_states_next[layer] - # Concatenate the interpolations along the time dimension and append them to the list of interpolated sequences + seq_interpolations = torch.cat(interpolations, dim=1) interpolated_sequences.append(seq_interpolations) - # Concatenate all sequences (original and interpolated) along the time dimension + full_sequence = torch.cat(interpolated_sequences, dim=1) - # Return the full sequence - return full_sequence - + return full_sequence \ No newline at end of file diff --git a/Code/training.py b/Code/training.py index 7dbebb8..a0217a2 100644 --- a/Code/training.py +++ b/Code/training.py @@ -58,32 +58,49 @@ def train_autoencoder(self, epochs, train_loader, val_loader): def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): min_val_loss = float('inf') # Initialize the minimum validation loss to infinity - # Loop over the number of epochs + for epoch in range(epochs): - self.model.train() # Set the model to training mode + self.model.train() # Set the model to training mode + train_loss = 0.0 + # Training Loop - for sequence, target in train_data: - self.optimizer.zero_grad() # Reset the gradients accumulated from the previous iteration - sequence, target = sequence.to(self.device), target.to(self.device) # Moved both to the device - output = self.model(sequence, n_interpolate_frames) - loss = self.loss_function(output, target) # Compute Training Loss - loss.backward() # Backward Pass - self.optimizer.step() # Update Model Parameters + for sequences, targets in train_data: + self.optimizer.zero_grad() # Reset the gradients accumulated from the previous iteration + sequences = sequences.to(self.device) + targets = targets.to(self.device) + outputs = self.model(sequences, n_interpolate_frames) + # Assuming the outputs and targets are of shape [batch_size, seq_len, channels, height, width] + # Compute Training Loss only on the interpolated frames (not on the original frames) + loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) + loss.backward() # Backward Pass + self.optimizer.step() # Update Model Parameters + train_loss += loss.item() + + train_loss /= len(train_data) + print(f'Epoch : {epoch}, Training Loss : {train_loss}') + # Validation Loss Calculation - self.model.eval() # Set the Model to Evaluation Mode + self.model.eval() # Set the Model to Evaluation Mode + val_loss = 0.0 + with torch.no_grad(): - val_loss = 0.0 - for sequence, target in val_data: - sequence = sequence.to(self.device) - target = target.to(self.device) - output = self.model(sequence, n_interpolate_frames) - val_loss += self.loss_function(output, target).item() # Calculate loss only on interpolated frames + for sequences, targets in val_data: + sequences = sequences.to(self.device) + targets = targets.to(self.device) + outputs = self.model(sequences, n_interpolate_frames) + # Compute Validation Loss only on interpolated frames (not on the original frames) + loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) + val_loss += loss.item() + val_loss /= len(val_data) + # Print the epoch number and the validation loss print(f'Epoch : {epoch}, Validation Loss : {val_loss}') + # If the current validation loss is lower than the best validation loss, save the model if val_loss < min_val_loss: - min_val_loss = val_loss # Update the best validation loss - self.save_model() # Save the model + min_val_loss = val_loss # Update the best validation loss + self.save_model() # Save the model + # Return the Trained Model return self.model \ No newline at end of file From e9c57f9cda7c6497a6cfeb605c356e4908a7990b Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Sun, 24 Dec 2023 21:12:49 -0800 Subject: [PATCH 05/32] Fixed Dimension Issue with LSTM Data --- Code/data.py | 51 +++++++++++++++------------------------------------ 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/Code/data.py b/Code/data.py index 1c6ef02..56edf14 100644 --- a/Code/data.py +++ b/Code/data.py @@ -65,39 +65,18 @@ def get_autoencoder_batches(self,val_split): return train_loader, val_loader # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) - def get_lstm_batches(self, val_split): - # Assumption: self.grayscale_images is a tensor with shape [num_samples, channels, height, width] - - # Calculate the number of frames for training and validation - num_frames = self.grayscale_images.size(0) - train_frames = int(num_frames * (1 - val_split)) - val_frames = num_frames - train_frames - - # Split the frames into training and validation sets - train_frames_tensor = self.grayscale_images[:train_frames] - val_frames_tensor = self.grayscale_images[-val_frames:] - - # Create sequences (8 frames per sequence here) for training and validation - # The goal is to predict 7 intermediate frames from 8 input frames - input_seq_len = 8 - target_seq_len = 7 - - train_seqs = train_frames_tensor.unfold(0, input_seq_len, 1) - train_targets = train_seqs.roll(-1, dims=0)[:, :target_seq_len] # Shift temporal sequence by one step - - val_seqs = val_frames_tensor.unfold(0, input_seq_len, 1) - val_targets = val_seqs.roll(-1, dims=0)[:, :target_seq_len] # Shift temporal sequence by one step - - # Make sure there is no overlap between training and validation targets - if train_seqs[-1].equals(val_seqs[0]): - val_seqs = val_seqs[1:] - val_targets = val_targets[1:] - - # Create tensor datasets and data loaders - train_dataset = TensorDataset(train_seqs, train_targets) - val_dataset = TensorDataset(val_seqs, val_targets) - - train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True) - val_loader = DataLoader(val_dataset, batch_size=self.batch_size) - - return train_loader, val_loader + def get_lstm_batches(self,val_split): + # Add an extra dimension to the grayscale images tensor + greyscale_image_sequence = self.grayscale_images.unsqueeze(0) + # Split the sequence into training and validation sets + num_frames = greyscale_image_sequence.size(1) + split_idx = int(num_frames * val_split) + greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] + greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] + # Create TensorDatasets + train_data = TensorDataset(greyscale_image_sequence_train[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_train[:, 1:self.grayscale_images.size(0):2]) + val_data = TensorDataset(greyscale_image_sequence_val[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_val[:, 1:self.grayscale_images.size(0):2]) + # Create DataLoaders + train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) + val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) + return train_loader, val_loader \ No newline at end of file From d73105690f778bdd0bb20aadd0dbe3cec7b0cdc7 Mon Sep 17 00:00:00 2001 From: Siddharth Kekre Date: Mon, 25 Dec 2023 07:50:20 +0000 Subject: [PATCH 06/32] Sync Code Across Devices --- Code/data.py | 3 ++- Code/lstm_model.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Code/data.py b/Code/data.py index 56edf14..da12c40 100644 --- a/Code/data.py +++ b/Code/data.py @@ -79,4 +79,5 @@ def get_lstm_batches(self,val_split): # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) - return train_loader, val_loader \ No newline at end of file + return train_loader, val_loader + diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 3bf799d..6380a5e 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -43,7 +43,8 @@ def forward(self, input_tensor, cur_state): return h_next, c_next def init_hidden(self, batch_size, image_size): - height, width = image_size + # height, width = image_size + height, width = image_size[0], image_size[1] return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) From 97f730cdb1dea4ff54fee34de4d08002fcbc96e7 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 00:14:52 -0800 Subject: [PATCH 07/32] Attempt to fix LSTM --- Code/data.py | 9 +++++---- Code/lstm_model.py | 9 +++++---- Code/main.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Code/data.py b/Code/data.py index 56edf14..32c9459 100644 --- a/Code/data.py +++ b/Code/data.py @@ -11,7 +11,8 @@ from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True import torch -from torch.utils.data import DataLoader, TensorDataset +from torch.utils.data import DataLoader, TensorDataset, Dataset + class Dataset: def __init__(self, grayscale_dir, rgb_dir, image_size, batch_size): @@ -65,7 +66,7 @@ def get_autoencoder_batches(self,val_split): return train_loader, val_loader # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) - def get_lstm_batches(self,val_split): + def get_lstm_batches(self, val_split): # Add an extra dimension to the grayscale images tensor greyscale_image_sequence = self.grayscale_images.unsqueeze(0) # Split the sequence into training and validation sets @@ -74,8 +75,8 @@ def get_lstm_batches(self,val_split): greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] # Create TensorDatasets - train_data = TensorDataset(greyscale_image_sequence_train[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_train[:, 1:self.grayscale_images.size(0):2]) - val_data = TensorDataset(greyscale_image_sequence_val[:, 0:self.grayscale_images.size(0):2], greyscale_image_sequence_val[:, 1:self.grayscale_images.size(0):2]) + train_data = TensorDataset(greyscale_image_sequence_train[:, ::2], greyscale_image_sequence_train[:, 1::2]) + val_data = TensorDataset(greyscale_image_sequence_val[:, ::2], greyscale_image_sequence_val[:, 1::2]) # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 3bf799d..c3c8af6 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -43,9 +43,10 @@ def forward(self, input_tensor, cur_state): return h_next, c_next def init_hidden(self, batch_size, image_size): - height, width = image_size - return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), - torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) + # height, width = image_size + height, width = image_size[0], image_size[1] + return (torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device), + torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device)) # Frame Interpolation model using ConvLSTM cells class FrameInterpolationLSTM(nn.Module): @@ -72,7 +73,7 @@ def forward(self, x, n): cell_states = [] for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h, w)) + h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h.item(), w.item())) hidden_states.append(h) cell_states.append(c) diff --git a/Code/main.py b/Code/main.py index e87db4c..2588fdc 100644 --- a/Code/main.py +++ b/Code/main.py @@ -47,7 +47,7 @@ def main(): print('AutoEncoder Model Initialized.') # Initialize LSTM Model and Import Image Sequences (Training, Validation) - grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches(val_split=0.8) + grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches(val_split=0.2) print('LSTM Model Data Initialized.') C = 1 hidden_size = 64 From 8562c201a7a4097e1f3ca9297d733df8cf6e6ae1 Mon Sep 17 00:00:00 2001 From: Siddharth Kekre Date: Mon, 25 Dec 2023 08:28:12 +0000 Subject: [PATCH 08/32] Change int values in LSTM Hidden State --- Code/lstm_model.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 1a215c8..81335f0 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -45,13 +45,8 @@ def forward(self, input_tensor, cur_state): def init_hidden(self, batch_size, image_size): # height, width = image_size height, width = image_size[0], image_size[1] -<<<<<<< HEAD - return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), - torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) -======= return (torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device)) ->>>>>>> 97f730cdb1dea4ff54fee34de4d08002fcbc96e7 # Frame Interpolation model using ConvLSTM cells class FrameInterpolationLSTM(nn.Module): @@ -78,7 +73,7 @@ def forward(self, x, n): cell_states = [] for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h.item(), w.item())) + h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h, w)) hidden_states.append(h) cell_states.append(c) From 98970c5d677c93860c0d35169caea6763ce30e1e Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 00:45:52 -0800 Subject: [PATCH 09/32] Changed Dimensions for Image Size --- Code/lstm_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 81335f0..44254f3 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -43,10 +43,10 @@ def forward(self, input_tensor, cur_state): return h_next, c_next def init_hidden(self, batch_size, image_size): - # height, width = image_size - height, width = image_size[0], image_size[1] - return (torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device), - torch.zeros(batch_size, self.hidden_channels, *image_size, device=self.conv.weight.device)) + height, width = image_size[2], image_size[3] + return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), + torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) + # Frame Interpolation model using ConvLSTM cells class FrameInterpolationLSTM(nn.Module): @@ -73,7 +73,7 @@ def forward(self, x, n): cell_states = [] for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, (h, w)) + h, c = self.conv_lstm_cells[i].init_hidden(batch_size, x.shape) hidden_states.append(h) cell_states.append(c) From f7577f2529ae6d64315dcacbee3b63ec009bb3d7 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:26:12 -0800 Subject: [PATCH 10/32] Fixed LSTM Input-HiddenState Tensor Dimension Mismatch Error --- Code/data.py | 4 ++-- Code/lstm_model.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/data.py b/Code/data.py index 0bdfa1d..bb83271 100644 --- a/Code/data.py +++ b/Code/data.py @@ -75,8 +75,8 @@ def get_lstm_batches(self, val_split): greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] # Create TensorDatasets - train_data = TensorDataset(greyscale_image_sequence_train[:, ::2], greyscale_image_sequence_train[:, 1::2]) - val_data = TensorDataset(greyscale_image_sequence_val[:, ::2], greyscale_image_sequence_val[:, 1::2]) + train_data = TensorDataset(greyscale_image_sequence_train[:, ::2].squeeze(dim=0), greyscale_image_sequence_train[:, 1::2].squeeze(dim=0)) + val_data = TensorDataset(greyscale_image_sequence_val[:, ::2].squeeze(dim=0), greyscale_image_sequence_val[:, 1::2].squeeze(dim=0)) # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 44254f3..302bb5f 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -81,7 +81,7 @@ def forward(self, x, n): for t in range(seq_len - 1): current_frame = layer_input[:, t] next_frame = layer_input[:, t + 1] - interpolations = [current_frame.unsqueeze(1)] + interpolations = [current_frame] for layer in range(self.num_layers): h_cur, c_cur = hidden_states[layer], cell_states[layer] From 894baa70544ef3e41b506cfaa729b418e24ffb99 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:35:14 -0800 Subject: [PATCH 11/32] Changed LSTM DataLoader --- Code/data.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Code/data.py b/Code/data.py index bb83271..db6a84e 100644 --- a/Code/data.py +++ b/Code/data.py @@ -67,18 +67,23 @@ def get_autoencoder_batches(self,val_split): # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) def get_lstm_batches(self, val_split): - # Add an extra dimension to the grayscale images tensor greyscale_image_sequence = self.grayscale_images.unsqueeze(0) - # Split the sequence into training and validation sets num_frames = greyscale_image_sequence.size(1) split_idx = int(num_frames * val_split) + # Ensure even number of frames for splitting into pairs. + split_idx = split_idx if split_idx % 2 == 0 else split_idx - 1 greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] - # Create TensorDatasets - train_data = TensorDataset(greyscale_image_sequence_train[:, ::2].squeeze(dim=0), greyscale_image_sequence_train[:, 1::2].squeeze(dim=0)) - val_data = TensorDataset(greyscale_image_sequence_val[:, ::2].squeeze(dim=0), greyscale_image_sequence_val[:, 1::2].squeeze(dim=0)) + # Ensure the same length for both odd and even splits. + even_indices = range(0, split_idx, 2) + odd_indices = range(1, split_idx, 2) + train_data = TensorDataset(greyscale_image_sequence_train[:, even_indices].squeeze(dim=0), + greyscale_image_sequence_train[:, odd_indices].squeeze(dim=0)) + even_indices = range(0, greyscale_image_sequence_val.size(1), 2) + odd_indices = range(1, greyscale_image_sequence_val.size(1), 2) + val_data = TensorDataset(greyscale_image_sequence_val[:, even_indices].squeeze(dim=0), + greyscale_image_sequence_val[:, odd_indices].squeeze(dim=0)) # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) return train_loader, val_loader - From b75a7676fb54618f599789da753446ddae3b9ffa Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:42:41 -0800 Subject: [PATCH 12/32] Shape Mismatch LSTM --- Code/lstm_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 302bb5f..9c3ad81 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -67,7 +67,7 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): padding=0) def forward(self, x, n): - batch_size, seq_len, _, h, w = x.shape + batch_size, seq_len, _, (h, w) = x.shape layer_input = x hidden_states = [] cell_states = [] From db6ee563501824f158c53c22a8d0470e85f149ea Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:45:18 -0800 Subject: [PATCH 13/32] ... --- Code/lstm_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 9c3ad81..525d986 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -67,7 +67,7 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): padding=0) def forward(self, x, n): - batch_size, seq_len, _, (h, w) = x.shape + batch_size, seq_len, _, _ = x.shape layer_input = x hidden_states = [] cell_states = [] From 4500cf5d639333f144bae235ea83e567a67883ed Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:58:41 -0800 Subject: [PATCH 14/32] Normalized Values in MaxEnt Calculation --- Code/losses.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Code/losses.py b/Code/losses.py index e8c8b4b..3e6d51e 100644 --- a/Code/losses.py +++ b/Code/losses.py @@ -22,8 +22,12 @@ def __init__(self, alpha=0.5): def forward(self, output, target): mse_loss = self.mse(output, target) # Compute MSE Loss - entropy = -torch.sum(target * torch.log(output + 1e-8), dim=-1).mean() # Compute Entropy - composite_loss = self.alpha * mse_loss + (1 - self.alpha) * entropy # Compute Composite Loss + # Normalize the output tensor along the last dimension to represent probabilities + output_normalized = torch.softmax(output, dim=-1) + # Compute Entropy + entropy = -torch.sum(target * torch.log(output_normalized + 1e-8), dim=-1).mean() + # Compute Composite Loss + composite_loss = self.alpha * mse_loss + (1 - self.alpha) * entropy return composite_loss ''' From 58e8a347806e7e6228a5856c379150cca0712afe Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 01:59:09 -0800 Subject: [PATCH 15/32] Updated Model Architecture for LSTM --- Code/data.py | 19 +++++++++---------- Code/lstm_model.py | 5 +++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Code/data.py b/Code/data.py index db6a84e..4cfed0c 100644 --- a/Code/data.py +++ b/Code/data.py @@ -67,22 +67,21 @@ def get_autoencoder_batches(self,val_split): # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) def get_lstm_batches(self, val_split): - greyscale_image_sequence = self.grayscale_images.unsqueeze(0) - num_frames = greyscale_image_sequence.size(1) + num_frames = self.grayscale_images.size(0) split_idx = int(num_frames * val_split) # Ensure even number of frames for splitting into pairs. split_idx = split_idx if split_idx % 2 == 0 else split_idx - 1 - greyscale_image_sequence_train = greyscale_image_sequence[:, :split_idx, :, :, :] - greyscale_image_sequence_val = greyscale_image_sequence[:, split_idx:, :, :, :] + grayscale_images_train = self.grayscale_images[:split_idx] + grayscale_images_val = self.grayscale_images[split_idx:] # Ensure the same length for both odd and even splits. even_indices = range(0, split_idx, 2) odd_indices = range(1, split_idx, 2) - train_data = TensorDataset(greyscale_image_sequence_train[:, even_indices].squeeze(dim=0), - greyscale_image_sequence_train[:, odd_indices].squeeze(dim=0)) - even_indices = range(0, greyscale_image_sequence_val.size(1), 2) - odd_indices = range(1, greyscale_image_sequence_val.size(1), 2) - val_data = TensorDataset(greyscale_image_sequence_val[:, even_indices].squeeze(dim=0), - greyscale_image_sequence_val[:, odd_indices].squeeze(dim=0)) + train_data = TensorDataset(grayscale_images_train[even_indices], + grayscale_images_train[odd_indices]) + even_indices = range(0, len(grayscale_images_val), 2) + odd_indices = range(1, len(grayscale_images_val), 2) + val_data = TensorDataset(grayscale_images_val[even_indices], + grayscale_images_val[odd_indices]) # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 525d986..d4bb839 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -88,7 +88,7 @@ def forward(self, x, n): h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) current_frame = h_next[:, -1] - h_n, c_n = self.conv_lstm_cells[layer](next_frame.unsqueeze(1), (h_cur, c_cur)) + h_n, c_n = self.conv_lstm_cells[layer](next_frame.unsqueeze(1), (h_next, c_next)) next_frame_pred = h_n[:, -1] hidden_states[layer], cell_states[layer] = h_n, c_n @@ -103,4 +103,5 @@ def forward(self, x, n): interpolated_sequences.append(seq_interpolations) full_sequence = torch.cat(interpolated_sequences, dim=1) - return full_sequence \ No newline at end of file + return full_sequence + From c65ce7891183efff93fa155c2e105de2cf06a9dd Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 20:58:30 -0800 Subject: [PATCH 16/32] Changed File Names for Dataset --- Dataset/Greyscale/{Greyscale (1).tif => Moon (1).tif} | 0 Dataset/Greyscale/{Greyscale (10).tif => Moon (10).tif} | 0 Dataset/Greyscale/{Greyscale (11).tif => Moon (11).tif} | 0 Dataset/Greyscale/{Greyscale (12).tif => Moon (12).tif} | 0 Dataset/Greyscale/{Greyscale (13).tif => Moon (13).tif} | 0 Dataset/Greyscale/{Greyscale (14).tif => Moon (14).tif} | 0 Dataset/Greyscale/{Greyscale (15).tif => Moon (15).tif} | 0 Dataset/Greyscale/{Greyscale (16).tif => Moon (16).tif} | 0 Dataset/Greyscale/{Greyscale (2).tif => Moon (2).tif} | 0 Dataset/Greyscale/{Greyscale (3).tif => Moon (3).tif} | 0 Dataset/Greyscale/{Greyscale (4).tif => Moon (4).tif} | 0 Dataset/Greyscale/{Greyscale (5).tif => Moon (5).tif} | 0 Dataset/Greyscale/{Greyscale (6).tif => Moon (6).tif} | 0 Dataset/Greyscale/{Greyscale (7).tif => Moon (7).tif} | 0 Dataset/Greyscale/{Greyscale (8).tif => Moon (8).tif} | 0 Dataset/Greyscale/{Greyscale (9).tif => Moon (9).tif} | 0 Dataset/RGB/{RGB (1).tif => Moon (1).tif} | 0 Dataset/RGB/{RGB (10).tif => Moon (10).tif} | 0 Dataset/RGB/{RGB (11).tif => Moon (11).tif} | 0 Dataset/RGB/{RGB (12).tif => Moon (12).tif} | 0 Dataset/RGB/{RGB (13).tif => Moon (13).tif} | 0 Dataset/RGB/{RGB (14).tif => Moon (14).tif} | 0 Dataset/RGB/{RGB (15).tif => Moon (15).tif} | 0 Dataset/RGB/{RGB (16).tif => Moon (16).tif} | 0 Dataset/RGB/{RGB (2).tif => Moon (2).tif} | 0 Dataset/RGB/{RGB (3).tif => Moon (3).tif} | 0 Dataset/RGB/{RGB (4).tif => Moon (4).tif} | 0 Dataset/RGB/{RGB (5).tif => Moon (5).tif} | 0 Dataset/RGB/{RGB (6).tif => Moon (6).tif} | 0 Dataset/RGB/{RGB (7).tif => Moon (7).tif} | 0 Dataset/RGB/{RGB (8).tif => Moon (8).tif} | 0 Dataset/RGB/{RGB (9).tif => Moon (9).tif} | 0 32 files changed, 0 insertions(+), 0 deletions(-) rename Dataset/Greyscale/{Greyscale (1).tif => Moon (1).tif} (100%) rename Dataset/Greyscale/{Greyscale (10).tif => Moon (10).tif} (100%) rename Dataset/Greyscale/{Greyscale (11).tif => Moon (11).tif} (100%) rename Dataset/Greyscale/{Greyscale (12).tif => Moon (12).tif} (100%) rename Dataset/Greyscale/{Greyscale (13).tif => Moon (13).tif} (100%) rename Dataset/Greyscale/{Greyscale (14).tif => Moon (14).tif} (100%) rename Dataset/Greyscale/{Greyscale (15).tif => Moon (15).tif} (100%) rename Dataset/Greyscale/{Greyscale (16).tif => Moon (16).tif} (100%) rename Dataset/Greyscale/{Greyscale (2).tif => Moon (2).tif} (100%) rename Dataset/Greyscale/{Greyscale (3).tif => Moon (3).tif} (100%) rename Dataset/Greyscale/{Greyscale (4).tif => Moon (4).tif} (100%) rename Dataset/Greyscale/{Greyscale (5).tif => Moon (5).tif} (100%) rename Dataset/Greyscale/{Greyscale (6).tif => Moon (6).tif} (100%) rename Dataset/Greyscale/{Greyscale (7).tif => Moon (7).tif} (100%) rename Dataset/Greyscale/{Greyscale (8).tif => Moon (8).tif} (100%) rename Dataset/Greyscale/{Greyscale (9).tif => Moon (9).tif} (100%) rename Dataset/RGB/{RGB (1).tif => Moon (1).tif} (100%) rename Dataset/RGB/{RGB (10).tif => Moon (10).tif} (100%) rename Dataset/RGB/{RGB (11).tif => Moon (11).tif} (100%) rename Dataset/RGB/{RGB (12).tif => Moon (12).tif} (100%) rename Dataset/RGB/{RGB (13).tif => Moon (13).tif} (100%) rename Dataset/RGB/{RGB (14).tif => Moon (14).tif} (100%) rename Dataset/RGB/{RGB (15).tif => Moon (15).tif} (100%) rename Dataset/RGB/{RGB (16).tif => Moon (16).tif} (100%) rename Dataset/RGB/{RGB (2).tif => Moon (2).tif} (100%) rename Dataset/RGB/{RGB (3).tif => Moon (3).tif} (100%) rename Dataset/RGB/{RGB (4).tif => Moon (4).tif} (100%) rename Dataset/RGB/{RGB (5).tif => Moon (5).tif} (100%) rename Dataset/RGB/{RGB (6).tif => Moon (6).tif} (100%) rename Dataset/RGB/{RGB (7).tif => Moon (7).tif} (100%) rename Dataset/RGB/{RGB (8).tif => Moon (8).tif} (100%) rename Dataset/RGB/{RGB (9).tif => Moon (9).tif} (100%) diff --git a/Dataset/Greyscale/Greyscale (1).tif b/Dataset/Greyscale/Moon (1).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (1).tif rename to Dataset/Greyscale/Moon (1).tif diff --git a/Dataset/Greyscale/Greyscale (10).tif b/Dataset/Greyscale/Moon (10).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (10).tif rename to Dataset/Greyscale/Moon (10).tif diff --git a/Dataset/Greyscale/Greyscale (11).tif b/Dataset/Greyscale/Moon (11).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (11).tif rename to Dataset/Greyscale/Moon (11).tif diff --git a/Dataset/Greyscale/Greyscale (12).tif b/Dataset/Greyscale/Moon (12).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (12).tif rename to Dataset/Greyscale/Moon (12).tif diff --git a/Dataset/Greyscale/Greyscale (13).tif b/Dataset/Greyscale/Moon (13).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (13).tif rename to Dataset/Greyscale/Moon (13).tif diff --git a/Dataset/Greyscale/Greyscale (14).tif b/Dataset/Greyscale/Moon (14).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (14).tif rename to Dataset/Greyscale/Moon (14).tif diff --git a/Dataset/Greyscale/Greyscale (15).tif b/Dataset/Greyscale/Moon (15).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (15).tif rename to Dataset/Greyscale/Moon (15).tif diff --git a/Dataset/Greyscale/Greyscale (16).tif b/Dataset/Greyscale/Moon (16).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (16).tif rename to Dataset/Greyscale/Moon (16).tif diff --git a/Dataset/Greyscale/Greyscale (2).tif b/Dataset/Greyscale/Moon (2).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (2).tif rename to Dataset/Greyscale/Moon (2).tif diff --git a/Dataset/Greyscale/Greyscale (3).tif b/Dataset/Greyscale/Moon (3).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (3).tif rename to Dataset/Greyscale/Moon (3).tif diff --git a/Dataset/Greyscale/Greyscale (4).tif b/Dataset/Greyscale/Moon (4).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (4).tif rename to Dataset/Greyscale/Moon (4).tif diff --git a/Dataset/Greyscale/Greyscale (5).tif b/Dataset/Greyscale/Moon (5).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (5).tif rename to Dataset/Greyscale/Moon (5).tif diff --git a/Dataset/Greyscale/Greyscale (6).tif b/Dataset/Greyscale/Moon (6).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (6).tif rename to Dataset/Greyscale/Moon (6).tif diff --git a/Dataset/Greyscale/Greyscale (7).tif b/Dataset/Greyscale/Moon (7).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (7).tif rename to Dataset/Greyscale/Moon (7).tif diff --git a/Dataset/Greyscale/Greyscale (8).tif b/Dataset/Greyscale/Moon (8).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (8).tif rename to Dataset/Greyscale/Moon (8).tif diff --git a/Dataset/Greyscale/Greyscale (9).tif b/Dataset/Greyscale/Moon (9).tif similarity index 100% rename from Dataset/Greyscale/Greyscale (9).tif rename to Dataset/Greyscale/Moon (9).tif diff --git a/Dataset/RGB/RGB (1).tif b/Dataset/RGB/Moon (1).tif similarity index 100% rename from Dataset/RGB/RGB (1).tif rename to Dataset/RGB/Moon (1).tif diff --git a/Dataset/RGB/RGB (10).tif b/Dataset/RGB/Moon (10).tif similarity index 100% rename from Dataset/RGB/RGB (10).tif rename to Dataset/RGB/Moon (10).tif diff --git a/Dataset/RGB/RGB (11).tif b/Dataset/RGB/Moon (11).tif similarity index 100% rename from Dataset/RGB/RGB (11).tif rename to Dataset/RGB/Moon (11).tif diff --git a/Dataset/RGB/RGB (12).tif b/Dataset/RGB/Moon (12).tif similarity index 100% rename from Dataset/RGB/RGB (12).tif rename to Dataset/RGB/Moon (12).tif diff --git a/Dataset/RGB/RGB (13).tif b/Dataset/RGB/Moon (13).tif similarity index 100% rename from Dataset/RGB/RGB (13).tif rename to Dataset/RGB/Moon (13).tif diff --git a/Dataset/RGB/RGB (14).tif b/Dataset/RGB/Moon (14).tif similarity index 100% rename from Dataset/RGB/RGB (14).tif rename to Dataset/RGB/Moon (14).tif diff --git a/Dataset/RGB/RGB (15).tif b/Dataset/RGB/Moon (15).tif similarity index 100% rename from Dataset/RGB/RGB (15).tif rename to Dataset/RGB/Moon (15).tif diff --git a/Dataset/RGB/RGB (16).tif b/Dataset/RGB/Moon (16).tif similarity index 100% rename from Dataset/RGB/RGB (16).tif rename to Dataset/RGB/Moon (16).tif diff --git a/Dataset/RGB/RGB (2).tif b/Dataset/RGB/Moon (2).tif similarity index 100% rename from Dataset/RGB/RGB (2).tif rename to Dataset/RGB/Moon (2).tif diff --git a/Dataset/RGB/RGB (3).tif b/Dataset/RGB/Moon (3).tif similarity index 100% rename from Dataset/RGB/RGB (3).tif rename to Dataset/RGB/Moon (3).tif diff --git a/Dataset/RGB/RGB (4).tif b/Dataset/RGB/Moon (4).tif similarity index 100% rename from Dataset/RGB/RGB (4).tif rename to Dataset/RGB/Moon (4).tif diff --git a/Dataset/RGB/RGB (5).tif b/Dataset/RGB/Moon (5).tif similarity index 100% rename from Dataset/RGB/RGB (5).tif rename to Dataset/RGB/Moon (5).tif diff --git a/Dataset/RGB/RGB (6).tif b/Dataset/RGB/Moon (6).tif similarity index 100% rename from Dataset/RGB/RGB (6).tif rename to Dataset/RGB/Moon (6).tif diff --git a/Dataset/RGB/RGB (7).tif b/Dataset/RGB/Moon (7).tif similarity index 100% rename from Dataset/RGB/RGB (7).tif rename to Dataset/RGB/Moon (7).tif diff --git a/Dataset/RGB/RGB (8).tif b/Dataset/RGB/Moon (8).tif similarity index 100% rename from Dataset/RGB/RGB (8).tif rename to Dataset/RGB/Moon (8).tif diff --git a/Dataset/RGB/RGB (9).tif b/Dataset/RGB/Moon (9).tif similarity index 100% rename from Dataset/RGB/RGB (9).tif rename to Dataset/RGB/Moon (9).tif From 19c77f0ebc636019ad9bf7433fce10aa721b4a4b Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 20:59:06 -0800 Subject: [PATCH 17/32] Made AutoEncoder easier to Scale in terms of Layers --- Code/autoencoder_model.py | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Code/autoencoder_model.py b/Code/autoencoder_model.py index 4ad54bf..1ec557d 100644 --- a/Code/autoencoder_model.py +++ b/Code/autoencoder_model.py @@ -11,38 +11,39 @@ class Grey2RGBAutoEncoder(nn.Module): def __init__(self): super(Grey2RGBAutoEncoder, self).__init__() - ''' # Define the Encoder The Encoder consists of 4 Convolutional layers with ReLU activation function Encoder takes 1-Chanel Grayscale image (1 channel) as input and outputs High-Dimentional-Representation ''' - self.encoder = nn.Sequential( - nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1), - nn.ReLU(), - nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1), - nn.ReLU(), - nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1), - nn.ReLU(), - nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=1), - nn.ReLU() - ) + self.encoder = self._make_layers([1, 64, 128, 256, 512]) ''' # Define the Decoder The Decoder consists of 4 Transpose Convolutional layers with ReLU activation function Decoder takes High-Dimentional-Representation as input and outputs 3-Chanel RGB image + The last layer uses a Sigmoid activation function instead of ReLU ''' - self.decoder = nn.Sequential( - nn.ConvTranspose2d(512, 256, kernel_size=3, stride=1, padding=1, output_padding=0), - nn.ReLU(), - nn.ConvTranspose2d(256, 128, kernel_size=3, stride=1, padding=1, output_padding=0), - nn.ReLU(), - nn.ConvTranspose2d(128, 64, kernel_size=3, stride=1, padding=1, output_padding=0), - nn.ReLU(), - nn.ConvTranspose2d(64, 3, kernel_size=3, stride=1, padding=1, output_padding=0), - nn.Sigmoid() - ) + self.decoder = self._make_layers([512, 256, 128, 64, 3], decoder=True) + + # Helper function to create the encoder or decoder layers. + def _make_layers(self, channels, decoder=False): + layers = [] + for i in range(len(channels) - 1): + ''' + For each pair of consecutive values in the channels list, a Convolutional or Transposed Convolutional layer is created. + The number of input channels is the first value, and the number of output channels is the second value. + A ReLU activation function is added after each Convolutional layer. + ''' + if decoder: + layers += [nn.ConvTranspose2d(channels[i], channels[i+1], kernel_size=3, stride=1, padding=1), + nn.ReLU(inplace=True)] + else: + layers += [nn.Conv2d(channels[i], channels[i+1], kernel_size=3, stride=1, padding=1), + nn.ReLU(inplace=True)] + if decoder: + layers[-1] = nn.Sigmoid() # Replace last ReLU with Sigmoid for decoder + return nn.Sequential(*layers) # The forward pass takes an input image, passes it through the encoder and decoder, and returns the output image def forward(self, x): From 7bc5bac6342457aeb3b4f94224c8d7e72094ec7c Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 20:59:30 -0800 Subject: [PATCH 18/32] Updated Method to Calculate MSE --- Code/losses.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Code/losses.py b/Code/losses.py index 3e6d51e..003cf67 100644 --- a/Code/losses.py +++ b/Code/losses.py @@ -8,6 +8,7 @@ # Import Necessary Libraries import torch import torch.nn as nn +import torch.nn.functional as F from pytorch_msssim import SSIM ''' @@ -18,10 +19,9 @@ class LossMEP(nn.Module): def __init__(self, alpha=0.5): super(LossMEP, self).__init__() self.alpha = alpha # Weighting factor for the loss - self.mse = nn.MSELoss() # Mean Squared Error loss def forward(self, output, target): - mse_loss = self.mse(output, target) # Compute MSE Loss + mse_loss = F.mse_loss(output, target) # Compute MSE Loss using functional API # Normalize the output tensor along the last dimension to represent probabilities output_normalized = torch.softmax(output, dim=-1) # Compute Entropy @@ -35,12 +35,8 @@ def forward(self, output, target): - Maximum Likelihood Principle ''' class LossMSE(nn.Module): - def __init__(self): - super(LossMSE, self).__init__() - self.mse = nn.MSELoss() # Mean Squared Error loss - def forward(self, output, target): - likelihood_loss = self.mse(output, target) # Compute MSE loss + likelihood_loss = F.mse_loss(output, target) # Compute MSE loss using functional API return likelihood_loss ''' @@ -51,11 +47,9 @@ def forward(self, output, target): class SSIMLoss(nn.Module): def __init__(self, data_range=1, size_average=True): super(SSIMLoss, self).__init__() - self.data_range = data_range # The range of the input image (usually 1.0 or 255) - self.size_average = size_average # If True, the SSIM of all windows are averaged # Initialize SSIM module - self.ssim_module = SSIM(data_range=self.data_range, size_average=self.size_average) + self.ssim_module = SSIM(data_range=data_range, size_average=size_average) def forward(self, img1, img2): ssim_value = self.ssim_module(img1, img2) # Compute SSIM - return 1 - ssim_value # Return loss \ No newline at end of file + return 1 - ssim_value # Return loss From 0fcd98eda6ad08fc7cc970c7261f2b564969803d Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 21:00:32 -0800 Subject: [PATCH 19/32] Updated Input-Target Image Sequences for LSTM Model --- Code/lstm_model.py | 60 +++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index d4bb839..55f60e2 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -9,20 +9,18 @@ import torch.nn as nn import torch.nn.functional as F -# Convolutional LSTM cell for processing sequences of images. +# Define Convolutional LSTM Cell class ConvLSTMCell(nn.Module): def __init__(self, input_channels, hidden_channels, kernel_size, bias=True): super(ConvLSTMCell, self).__init__() + # Initialize input, hidden channels, kernel size, and bias self.input_channels = input_channels self.hidden_channels = hidden_channels self.kernel_size = kernel_size - if isinstance(kernel_size, int): - self.padding = kernel_size // 2 - elif isinstance(kernel_size, tuple): - self.padding = (kernel_size[0] // 2, kernel_size[1] // 2) - else: - raise ValueError("kernel_size must be an int or a tuple of two ints") self.bias = bias + # Calculate padding based on kernel size + self.padding = kernel_size // 2 if isinstance(kernel_size, int) else (kernel_size[0] // 2, kernel_size[1] // 2) + # Define convolutional layer self.conv = nn.Conv2d(in_channels=self.input_channels + self.hidden_channels, out_channels=4 * self.hidden_channels, kernel_size=self.kernel_size, @@ -30,78 +28,96 @@ def __init__(self, input_channels, hidden_channels, kernel_size, bias=True): bias=self.bias) def forward(self, input_tensor, cur_state): + # Split current state into hidden and cell states h_cur, c_cur = cur_state + # Concatenate input tensor and hidden state along channel dimension combined = torch.cat([input_tensor, h_cur], dim=1) + # Apply convolution combined_conv = self.conv(combined) + # Split convolution output into four parts for LSTM gates cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_channels, dim=1) + # Calculate LSTM gate values i = torch.sigmoid(cc_i) f = torch.sigmoid(cc_f) o = torch.sigmoid(cc_o) g = torch.tanh(cc_g) + # Calculate next cell and hidden states c_next = f * c_cur + i * g h_next = o * torch.tanh(c_next) + # Return next hidden and cell states return h_next, c_next def init_hidden(self, batch_size, image_size): + # Get height and width of the image height, width = image_size[2], image_size[3] + # Initialize hidden and cell states to zeros return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) - -# Frame Interpolation model using ConvLSTM cells +# Define LSTM Model Architecture class FrameInterpolationLSTM(nn.Module): def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): super(FrameInterpolationLSTM, self).__init__() + # Initialize hidden dimension, number of layers, and ConvLSTM cells self.hidden_dim = hidden_dim self.num_layers = num_layers self.conv_lstm_cells = nn.ModuleList([ ConvLSTMCell(input_channels=(input_dim if i == 0 else hidden_dim), - hidden_channels=hidden_dim, - kernel_size=kernel_size, - bias=True) - for i in range(num_layers) - ]) + hidden_channels=hidden_dim, + kernel_size=kernel_size, + bias=True) + for i in range(num_layers)]) + # Define a convolutional layer self.conv = nn.Conv2d(in_channels=hidden_dim, out_channels=input_dim, kernel_size=(1, 1), padding=0) def forward(self, x, n): + # Get batch size and sequence length from input batch_size, seq_len, _, _ = x.shape layer_input = x hidden_states = [] cell_states = [] - + # Initialize hidden and cell states for each layer for i in range(self.num_layers): h, c = self.conv_lstm_cells[i].init_hidden(batch_size, x.shape) hidden_states.append(h) cell_states.append(c) - + # Initialize list to store interpolated sequences interpolated_sequences = [x[:, :1]] for t in range(seq_len - 1): + # Get current and next frame current_frame = layer_input[:, t] next_frame = layer_input[:, t + 1] interpolations = [current_frame] - + # Pass through each ConvLSTM layer for layer in range(self.num_layers): h_cur, c_cur = hidden_states[layer], cell_states[layer] h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) current_frame = h_next[:, -1] - + # Predict next frame h_n, c_n = self.conv_lstm_cells[layer](next_frame.unsqueeze(1), (h_next, c_next)) next_frame_pred = h_n[:, -1] hidden_states[layer], cell_states[layer] = h_n, c_n - + # Generate interpolated frames for j in range(1, n + 1): alpha = j / float(n + 1) interpolated_frame = (1 - alpha) * current_frame + alpha * next_frame_pred interpolated_frame = F.relu(self.conv(interpolated_frame)) interpolations.append(interpolated_frame) interpolations.append(next_frame) - + # Concatenate interpolated frames seq_interpolations = torch.cat(interpolations, dim=1) interpolated_sequences.append(seq_interpolations) - + # Process the last frame + current_frame = layer_input[:, -1] + for layer in range(self.num_layers): + h_cur, c_cur = hidden_states[layer], cell_states[layer] + h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) + current_frame = h_next[:, -1] + hidden_states[layer], cell_states[layer] = h_next, c_next + interpolated_sequences.append(current_frame.unsqueeze(1)) + # Concatenate all interpolated sequences full_sequence = torch.cat(interpolated_sequences, dim=1) return full_sequence - From a013e837aca85a420d23f8e46cce22cd40a8417e Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Mon, 25 Dec 2023 21:01:40 -0800 Subject: [PATCH 20/32] Experimental Code : Needs Review --- Code/data.py | 113 +++++++++++++++++++++++------------------------ Code/main.py | 4 +- Code/training.py | 6 ++- 3 files changed, 62 insertions(+), 61 deletions(-) diff --git a/Code/data.py b/Code/data.py index 4cfed0c..72b191e 100644 --- a/Code/data.py +++ b/Code/data.py @@ -5,84 +5,81 @@ ''' # Import Necessary Libraries -import os -import numpy as np from PIL import Image from PIL import ImageFile -ImageFile.LOAD_TRUNCATED_IMAGES = True -import torch -from torch.utils.data import DataLoader, TensorDataset, Dataset +from pathlib import Path +from torch.utils.data import DataLoader, Dataset, random_split +import torchvision.transforms as transforms +# Allow loading of truncated images +ImageFile.LOAD_TRUNCATED_IMAGES = True -class Dataset: - def __init__(self, grayscale_dir, rgb_dir, image_size, batch_size): - self.grayscale_dir = grayscale_dir # Directory for grayscale images - self.rgb_dir = rgb_dir # Directory for RGB images +# Define a custom dataset class +class CustomDataset(Dataset): + def __init__(self, grayscale_dir, rgb_dir, image_size, batch_size, valid_exts=['.tif', '.tiff']): + # Initialize directory paths and parameters + self.grayscale_dir = Path(grayscale_dir) # Directory for grayscale images + self.rgb_dir = Path(rgb_dir) # Directory for RGB images self.image_size = image_size # Size to which images will be resized self.batch_size = batch_size # Batch size for training - ''' - Load Greyscale and RGB images from respective directories - Store All Images of the Directory in a Normalized NumPy arrays - Convert the NumPy arrays to PyTorch Tensors - ''' - self.grayscale_images = self.load_images_to_tensor(self.grayscale_dir) - self.rgb_images = self.load_images_to_tensor(self.rgb_dir) + self.valid_exts = valid_exts # Valid file extensions + # Get list of valid image filenames + self.filenames = [f for f in self.grayscale_dir.iterdir() if f.suffix in self.valid_exts] + # Define transformations: resize and convert to tensor + self.transform = transforms.Compose([ + transforms.Resize(self.image_size), + transforms.ToTensor()]) + + # Return the total number of images + def __len__(self): + return len(self.filenames) + + # Get a single item from the dataset + def __getitem__(self, idx): + # Get paths for grayscale and RGB images + grayscale_path = self.filenames[idx] + rgb_path = self.rgb_dir / grayscale_path.name + # Open images + grayscale_img = Image.open(grayscale_path) + rgb_img = Image.open(rgb_path) + # Apply transformations + grayscale_img = self.transform(grayscale_img) + rgb_img = self.transform(rgb_img) + # Return transformed images + return grayscale_img, rgb_img - # Function to load images, resize and export as NumPy array - def load_images_to_tensor(self, directory): - images = [] - # Loop through all files in the directory - for filename in os.listdir(directory): - # If the file is an image file - if filename.endswith('.tif') or filename.endswith('.tiff'): - img_path = os.path.join(directory, filename) - img = Image.open(img_path) - # Resize the image - img = img.resize(self.image_size) - # Append the normalized image to the list - img_array = np.array(img).astype('float32') / 255.0 - # Add an extra dimension for grayscale images - if len(img_array.shape) == 2: - img_array = np.expand_dims(img_array, axis=-1) - images.append(img_array) - # Return the PyTorch Tensor {with shape [m, C, H, W]} created from of NumPy Array of Images - images = np.array(images) - images = torch.tensor(images, dtype=torch.float32).permute(0, 3, 1, 2) - return images - - # Function to get batches of input-target pairs from data (This Functionality is for AutoEncoder Component of the Program) - def get_autoencoder_batches(self,val_split): - # Create a Dataset from the Tensors - dataset = TensorDataset(self.grayscale_images, self.rgb_images) + # Get batches for autoencoder training + def get_autoencoder_batches(self, val_split): # Calculate the number of samples to include in the validation set - val_size = int(val_split * len(dataset)) - train_size = len(dataset) - val_size + val_size = int(val_split * len(self)) + train_size = len(self) - val_size # Split the dataset into training and validation sets - train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size]) + train_dataset, val_dataset = random_split(self, [train_size, val_size]) # Create dataloaders for the training and validation sets train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=self.batch_size, shuffle=True) # Return the training and validation dataloaders return train_loader, val_loader - # Function to get batches of original_sequence-interpolated_sequence from data (This Functionality is for LSTM Component of the Program) + # Get batches for LSTM training def get_lstm_batches(self, val_split): - num_frames = self.grayscale_images.size(0) + # Calculate the split index + num_frames = len(self) split_idx = int(num_frames * val_split) - # Ensure even number of frames for splitting into pairs. + # Ensure even number of frames for splitting into pairs split_idx = split_idx if split_idx % 2 == 0 else split_idx - 1 - grayscale_images_train = self.grayscale_images[:split_idx] - grayscale_images_val = self.grayscale_images[split_idx:] - # Ensure the same length for both odd and even splits. - even_indices = range(0, split_idx, 2) + # Split grayscale images into training and validation sets + grayscale_images_train = self[:split_idx] + grayscale_images_val = self[split_idx:] + # Ensure the same length for both odd and even splits odd_indices = range(1, split_idx, 2) - train_data = TensorDataset(grayscale_images_train[even_indices], - grayscale_images_train[odd_indices]) - even_indices = range(0, len(grayscale_images_val), 2) - odd_indices = range(1, len(grayscale_images_val), 2) - val_data = TensorDataset(grayscale_images_val[even_indices], - grayscale_images_val[odd_indices]) + even_indices = range(0, split_idx, 2) + # Create training and validation data + train_data = [grayscale_images_train[i] for i in odd_indices] + val_data = [grayscale_images_val[i] for i in even_indices] # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) + # Return the training and validation dataloaders return train_loader, val_loader + diff --git a/Code/main.py b/Code/main.py index 2588fdc..aa8fc1d 100644 --- a/Code/main.py +++ b/Code/main.py @@ -69,7 +69,7 @@ def main(): trainer_lstm_baseline = Trainer(model_lstm, loss_mse, model_save_path_lstm) print('Baseline LSTM Trainer Initialized.') - # Method 2 : Composite Loss for AutoEncoder and Mean Squared Error Loss for LSTM + # Method 2 : Composite Loss (MSE + MaxEnt) for AutoEncoder and Mean Squared Error Loss for LSTM os.makedirs('../Models/Method2', exist_ok=True) # Creating Directory for Model Saving model_save_path_ae = '../Models/Method2/model_autoencoder_m2.pth' trainer_autoencoder_m2 = Trainer(model_autoencoder, loss_mep, model_save_path_ae) @@ -83,7 +83,7 @@ def main(): trainer_lstm_m3 = Trainer(model_lstm, loss_ssim, model_save_path_lstm) print('Method-3 LSTM Trainer Initialized.') - # Method 4 : Proposed Method : Composite Loss for AutoEncoder and SSIM Loss for LSTM + # Method 4 : Proposed Method : Composite Loss (MSE + MaxEnt) for AutoEncoder and SSIM Loss for LSTM print('Method-4 AutoEncoder == Method-2 AutoEncoder') print('Method-4 LSTM == Method-3 LSTM') diff --git a/Code/training.py b/Code/training.py index a0217a2..10c6d5c 100644 --- a/Code/training.py +++ b/Code/training.py @@ -69,6 +69,8 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): sequences = sequences.to(self.device) targets = targets.to(self.device) outputs = self.model(sequences, n_interpolate_frames) + # sequences.requires_grad_() # Ensure gradients are required for the input to ConvLSTM + # targets.requires_grad_(False) # Ensure targets do not require gradients # Assuming the outputs and targets are of shape [batch_size, seq_len, channels, height, width] # Compute Training Loss only on the interpolated frames (not on the original frames) loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) @@ -89,7 +91,9 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): targets = targets.to(self.device) outputs = self.model(sequences, n_interpolate_frames) # Compute Validation Loss only on interpolated frames (not on the original frames) - loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) + predicted_interpolated_frames = outputs[:, 1:-1].reshape(-1, *outputs.shape[2:]) # Reshape to (B * seq_len, C, H, W) + true_interpolated_frames = targets[:, 1:-1].reshape(-1, *targets.shape[2:]) # Same reshaping for targets + loss = self.loss_function(predicted_interpolated_frames, true_interpolated_frames) val_loss += loss.item() val_loss /= len(val_data) From f13db99dd15fe27a73240964baaa10523a1108f4 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 01:28:53 -0800 Subject: [PATCH 21/32] Changed Dataset Class Name --- Code/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/main.py b/Code/main.py index aa8fc1d..0468e9f 100644 --- a/Code/main.py +++ b/Code/main.py @@ -4,7 +4,7 @@ ''' # Importing Custom Modules -from data import Dataset +from data import CustomDataset from lstm_model import FrameInterpolationLSTM from autoencoder_model import Grey2RGBAutoEncoder from losses import LossMSE, LossMEP, SSIMLoss @@ -28,7 +28,7 @@ def main(): # Initialize Dataset Object (PyTorch Tensors) print('Loading Dataset Initiated.') try: - dataset = Dataset(grayscale_dir, rgb_dir, (image_height, image_width), batch_size) + dataset = CustomDataset(grayscale_dir, rgb_dir, (image_height, image_width), batch_size) print('Loading Dataset Completed.') except Exception as e: print(f"Loading Dataset In-Complete : \n{e}") From e768e8d5008cd8baaeaf7968ef20bc4dc1b6e1fc Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 01:31:43 -0800 Subject: [PATCH 22/32] Fixed Data Type Mismatch --- Code/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Code/data.py b/Code/data.py index 72b191e..966e72a 100644 --- a/Code/data.py +++ b/Code/data.py @@ -34,8 +34,10 @@ def __init__(self, grayscale_dir, rgb_dir, image_size, batch_size, valid_exts=[' def __len__(self): return len(self.filenames) - # Get a single item from the dataset + # Get a single item or a slice from the dataset def __getitem__(self, idx): + if isinstance(idx, slice): + return [self[i] for i in range(*idx.indices(len(self)))] # Get paths for grayscale and RGB images grayscale_path = self.filenames[idx] rgb_path = self.rgb_dir / grayscale_path.name From dde4c9bd19acf8aefe093512f373a9dd3ca99cf1 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 01:57:10 -0800 Subject: [PATCH 23/32] Added Debugging Logs --- Code/data.py | 8 ++++++++ Code/lstm_model.py | 1 + Code/main.py | 4 ++++ Code/training.py | 7 +++++++ 4 files changed, 20 insertions(+) diff --git a/Code/data.py b/Code/data.py index 966e72a..a0c3cae 100644 --- a/Code/data.py +++ b/Code/data.py @@ -59,6 +59,10 @@ def get_autoencoder_batches(self, val_split): train_dataset, val_dataset = random_split(self, [train_size, val_size]) # Create dataloaders for the training and validation sets train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True) + print("Sample from autoencoder training data:") + for sample in train_loader: + print(f'Input shape: {sample[0].shape}, Target shape: {sample[1].shape}') + break # Just print the first sample and break val_loader = DataLoader(val_dataset, batch_size=self.batch_size, shuffle=True) # Return the training and validation dataloaders return train_loader, val_loader @@ -81,6 +85,10 @@ def get_lstm_batches(self, val_split): val_data = [grayscale_images_val[i] for i in even_indices] # Create DataLoaders train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) + print("Sample from LSTM training data:") + for sample in train_loader: + print(f'Input sequence shape: {sample[0].shape}, Target sequence shape: {sample[1].shape}') + break # Just print the first sample and break val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) # Return the training and validation dataloaders return train_loader, val_loader diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 55f60e2..e7c1eca 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -33,6 +33,7 @@ def forward(self, input_tensor, cur_state): # Concatenate input tensor and hidden state along channel dimension combined = torch.cat([input_tensor, h_cur], dim=1) # Apply convolution + print(f'Input tensor shape: {input_tensor.shape}, Hidden state shape: {h_cur.shape}') combined_conv = self.conv(combined) # Split convolution output into four parts for LSTM gates cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_channels, dim=1) diff --git a/Code/main.py b/Code/main.py index 0468e9f..e6d325b 100644 --- a/Code/main.py +++ b/Code/main.py @@ -45,6 +45,8 @@ def main(): print('AutoEncoder Model Data Initialized.') model_autoencoder = Grey2RGBAutoEncoder() print('AutoEncoder Model Initialized.') + print('AutoEncoder Model Summary:') + print(model_autoencoder) # Initialize LSTM Model and Import Image Sequences (Training, Validation) grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches(val_split=0.2) @@ -56,6 +58,8 @@ def main(): kernel_size = (3, 3) model_lstm = FrameInterpolationLSTM(C, hidden_size, kernel_size, num_layers) print('LSTM Model Initialized.') + print('LSTM Model Summary:') + print(model_lstm) ''' Initialize Trainer Objects diff --git a/Code/training.py b/Code/training.py index 10c6d5c..0a03b59 100644 --- a/Code/training.py +++ b/Code/training.py @@ -36,8 +36,10 @@ def train_autoencoder(self, epochs, train_loader, val_loader): # Training Loop for input, target in train_loader: # Input - Grayscale Image, Target - RGB Image input, target = input.to(self.device), target.to(self.device) + print(f'Input shape: {input.shape}, Target shape: {target.shape}') output = self.model(input) # Forward Pass loss = self.loss_function(output, target) # Compute Training Loss + print(f'Epoch: {epoch}, Training loss: {loss.item()}') self.optimizer.zero_grad() # Zero gradients to prepare for Backward Pass loss.backward() # Backward Pass self.optimizer.step() # Update Model Parameters @@ -60,11 +62,13 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): min_val_loss = float('inf') # Initialize the minimum validation loss to infinity for epoch in range(epochs): + print(f'Epoch: {epoch}, Training') self.model.train() # Set the model to training mode train_loss = 0.0 # Training Loop for sequences, targets in train_data: + print(f'Input sequence shape: {sequences.shape}, Target sequence shape: {targets.shape}') self.optimizer.zero_grad() # Reset the gradients accumulated from the previous iteration sequences = sequences.to(self.device) targets = targets.to(self.device) @@ -74,6 +78,7 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): # Assuming the outputs and targets are of shape [batch_size, seq_len, channels, height, width] # Compute Training Loss only on the interpolated frames (not on the original frames) loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) + print(f'Train loss: {loss.item()}') loss.backward() # Backward Pass self.optimizer.step() # Update Model Parameters train_loss += loss.item() @@ -87,6 +92,7 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): with torch.no_grad(): for sequences, targets in val_data: + print(f'Epoch: {epoch}, Validation') sequences = sequences.to(self.device) targets = targets.to(self.device) outputs = self.model(sequences, n_interpolate_frames) @@ -97,6 +103,7 @@ def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): val_loss += loss.item() val_loss /= len(val_data) + print(f'Validation loss: {val_loss}') # Print the epoch number and the validation loss print(f'Epoch : {epoch}, Validation Loss : {val_loss}') From 76614ce02970947226eba7f7311e42f296e3582c Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 02:30:23 -0800 Subject: [PATCH 24/32] Updated gett_lstm_batches --- Code/data.py | 80 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/Code/data.py b/Code/data.py index a0c3cae..404c696 100644 --- a/Code/data.py +++ b/Code/data.py @@ -68,28 +68,62 @@ def get_autoencoder_batches(self, val_split): return train_loader, val_loader # Get batches for LSTM training - def get_lstm_batches(self, val_split): - # Calculate the split index - num_frames = len(self) - split_idx = int(num_frames * val_split) - # Ensure even number of frames for splitting into pairs - split_idx = split_idx if split_idx % 2 == 0 else split_idx - 1 - # Split grayscale images into training and validation sets - grayscale_images_train = self[:split_idx] - grayscale_images_val = self[split_idx:] - # Ensure the same length for both odd and even splits - odd_indices = range(1, split_idx, 2) - even_indices = range(0, split_idx, 2) - # Create training and validation data - train_data = [grayscale_images_train[i] for i in odd_indices] - val_data = [grayscale_images_val[i] for i in even_indices] - # Create DataLoaders - train_loader = DataLoader(train_data, batch_size=self.batch_size, shuffle=True) - print("Sample from LSTM training data:") - for sample in train_loader: - print(f'Input sequence shape: {sample[0].shape}, Target sequence shape: {sample[1].shape}') - break # Just print the first sample and break - val_loader = DataLoader(val_data, batch_size=self.batch_size, shuffle=True) - # Return the training and validation dataloaders + def get_lstm_batches(self, val_split, n=1): + # Calculate the number of samples to include in the validation set + val_size = int(val_split * len(self)) + train_size = len(self) - val_size + + # Split the dataset into training and validation sets + all_indices = list(range(len(self))) + train_indices = all_indices[:train_size] + val_indices = all_indices[train_size:] + + # Function to generate input-target sequence pairs + def generate_sequences(indices): + input_sequences = [] # To store input sequences (odd-indexed frames) + target_sequences = [] # To store target sequences (full frame sequences) + + for start_idx in range(0, len(indices) - 1, 2): # Step by 2 to get odd-indexed frames + end_idx = start_idx + 2 * n + 1 # Calculate end index of the sequence + if end_idx > len(indices): + break # If the end index goes beyond the dataset size, stop adding sequences + + # Extract the input sequence (odd-indexed frames) + input_seq_indices = indices[start_idx:end_idx:2] # Every second frame (odd) + input_seq = [self[i][0] for i in input_seq_indices] # Select grayscale image only + input_sequences.append(torch.stack(input_seq)) + + # Extract the target sequence (full frames, including intermediate frames) + target_seq_indices = indices[start_idx:end_idx] # All frames in the range + target_seq = [self[i][0] for i in target_seq_indices] # Select grayscale image only + target_sequences.append(torch.stack(target_seq)) + + return torch.stack(input_sequences), torch.stack(target_sequences) + + # Generate training and validation sequences + train_input_seqs, train_target_seqs = generate_sequences(train_indices) + val_input_seqs, val_target_seqs = generate_sequences(val_indices) + + # Create custom Dataset for the LSTM sequences + class LSTMDataset(Dataset): + def __init__(self, input_seqs, target_seqs): + self.input_seqs = input_seqs + self.target_seqs = target_seqs + + def __len__(self): + return len(self.input_seqs) + + def __getitem__(self, idx): + return self.input_seqs[idx], self.target_seqs[idx] + + # Instantiate the custom Dataset objects + train_dataset = LSTMDataset(train_input_seqs, train_target_seqs) + val_dataset = LSTMDataset(val_input_seqs, val_target_seqs) + + # Create DataLoaders for the LSTM datasets + train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True) + val_loader = DataLoader(val_dataset, batch_size=self.batch_size, shuffle=True) + + # Return the training and validation DataLoaders return train_loader, val_loader From 7b820e24471950140aeacff4667a997d0cb7f23d Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 02:32:21 -0800 Subject: [PATCH 25/32] Fixed Imported Libraries --- Code/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/data.py b/Code/data.py index 404c696..80daa9a 100644 --- a/Code/data.py +++ b/Code/data.py @@ -10,6 +10,7 @@ from pathlib import Path from torch.utils.data import DataLoader, Dataset, random_split import torchvision.transforms as transforms +import torch # Allow loading of truncated images ImageFile.LOAD_TRUNCATED_IMAGES = True From 5307406fadf0ae60313eb92bf3184cddb575c191 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 02:36:18 -0800 Subject: [PATCH 26/32] Accounted for dimention change of target tensor --- Code/lstm_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index e7c1eca..92e0372 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -76,7 +76,7 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): def forward(self, x, n): # Get batch size and sequence length from input - batch_size, seq_len, _, _ = x.shape + batch_size, seq_len, chanel, height, width = x.shape layer_input = x hidden_states = [] cell_states = [] From edb440d510db86d7a54cf2d7cc6f33a3bf11ea4b Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 16:24:54 -0800 Subject: [PATCH 27/32] Attempt to Fix LSTM, Failed, Sync Code to Devices --- Code/data.py | 55 ++++++++---------- Code/lstm_model.py | 139 ++++++++++++++++----------------------------- Code/main.py | 29 +++++----- Code/training.py | 89 ++++++++++++++--------------- 4 files changed, 131 insertions(+), 181 deletions(-) diff --git a/Code/data.py b/Code/data.py index 80daa9a..7fd9092 100644 --- a/Code/data.py +++ b/Code/data.py @@ -71,39 +71,30 @@ def get_autoencoder_batches(self, val_split): # Get batches for LSTM training def get_lstm_batches(self, val_split, n=1): # Calculate the number of samples to include in the validation set - val_size = int(val_split * len(self)) - train_size = len(self) - val_size + val_size = int(val_split * (len(self) // 2)) # Half of sequences because we use every second image. + train_size = (len(self) // 2) - val_size - # Split the dataset into training and validation sets - all_indices = list(range(len(self))) - train_indices = all_indices[:train_size] - val_indices = all_indices[train_size:] - - # Function to generate input-target sequence pairs - def generate_sequences(indices): - input_sequences = [] # To store input sequences (odd-indexed frames) - target_sequences = [] # To store target sequences (full frame sequences) - - for start_idx in range(0, len(indices) - 1, 2): # Step by 2 to get odd-indexed frames - end_idx = start_idx + 2 * n + 1 # Calculate end index of the sequence - if end_idx > len(indices): - break # If the end index goes beyond the dataset size, stop adding sequences - - # Extract the input sequence (odd-indexed frames) - input_seq_indices = indices[start_idx:end_idx:2] # Every second frame (odd) - input_seq = [self[i][0] for i in input_seq_indices] # Select grayscale image only - input_sequences.append(torch.stack(input_seq)) - - # Extract the target sequence (full frames, including intermediate frames) - target_seq_indices = indices[start_idx:end_idx] # All frames in the range - target_seq = [self[i][0] for i in target_seq_indices] # Select grayscale image only - target_sequences.append(torch.stack(target_seq)) - - return torch.stack(input_sequences), torch.stack(target_sequences) - - # Generate training and validation sequences - train_input_seqs, train_target_seqs = generate_sequences(train_indices) - val_input_seqs, val_target_seqs = generate_sequences(val_indices) + # Get indices for the odd (input) and even (target) frames. + odd_indices = list(range(0, len(self), 2)) + even_indices = list(range(1, len(self), 2)) + + # Split the dataset indices into training and validation subsets + train_odd_indices = odd_indices[:train_size] + val_odd_indices = odd_indices[train_size:] + + train_even_indices = even_indices[:train_size] + val_even_indices = even_indices[train_size:] + + # Define a helper function to extract sequences by indices + def extract_sequences(indices): + return [self[i] for i in indices] + + # Use the helper function to create training and validation sets + train_input_seqs = torch.stack(extract_sequences(train_odd_indices)) + train_target_seqs = torch.stack(extract_sequences(train_even_indices)) + + val_input_seqs = torch.stack(extract_sequences(val_odd_indices)) + val_target_seqs = torch.stack(extract_sequences(val_even_indices)) # Create custom Dataset for the LSTM sequences class LSTMDataset(Dataset): diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 92e0372..491ced7 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -9,116 +9,77 @@ import torch.nn as nn import torch.nn.functional as F -# Define Convolutional LSTM Cell class ConvLSTMCell(nn.Module): - def __init__(self, input_channels, hidden_channels, kernel_size, bias=True): + def __init__(self, input_dim, hidden_dim, kernel_size, bias=True): super(ConvLSTMCell, self).__init__() - # Initialize input, hidden channels, kernel size, and bias - self.input_channels = input_channels - self.hidden_channels = hidden_channels + self.input_dim = input_dim + self.hidden_dim = hidden_dim self.kernel_size = kernel_size + self.padding = kernel_size // 2 self.bias = bias - # Calculate padding based on kernel size - self.padding = kernel_size // 2 if isinstance(kernel_size, int) else (kernel_size[0] // 2, kernel_size[1] // 2) - # Define convolutional layer - self.conv = nn.Conv2d(in_channels=self.input_channels + self.hidden_channels, - out_channels=4 * self.hidden_channels, - kernel_size=self.kernel_size, - padding=self.padding, - bias=self.bias) + + self.conv = nn.Sequential( + nn.Conv2d(in_channels=self.input_dim + self.hidden_dim, + out_channels=4 * self.hidden_dim, + kernel_size=self.kernel_size, + padding=self.padding, + bias=self.bias), + nn.BatchNorm2d(4 * self.hidden_dim) + ) def forward(self, input_tensor, cur_state): - # Split current state into hidden and cell states h_cur, c_cur = cur_state - # Concatenate input tensor and hidden state along channel dimension combined = torch.cat([input_tensor, h_cur], dim=1) - # Apply convolution - print(f'Input tensor shape: {input_tensor.shape}, Hidden state shape: {h_cur.shape}') combined_conv = self.conv(combined) - # Split convolution output into four parts for LSTM gates - cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_channels, dim=1) - # Calculate LSTM gate values + cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_dim, dim=1) i = torch.sigmoid(cc_i) f = torch.sigmoid(cc_f) o = torch.sigmoid(cc_o) g = torch.tanh(cc_g) - # Calculate next cell and hidden states + c_next = f * c_cur + i * g h_next = o * torch.tanh(c_next) - # Return next hidden and cell states + return h_next, c_next def init_hidden(self, batch_size, image_size): - # Get height and width of the image - height, width = image_size[2], image_size[3] - # Initialize hidden and cell states to zeros - return (torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device), - torch.zeros(batch_size, self.hidden_channels, height, width, device=self.conv.weight.device)) + height, width = image_size[0], image_size[1] + return (torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device), + torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device)) -# Define LSTM Model Architecture -class FrameInterpolationLSTM(nn.Module): +class ConvLSTM(nn.Module): def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): - super(FrameInterpolationLSTM, self).__init__() - # Initialize hidden dimension, number of layers, and ConvLSTM cells + super(ConvLSTM, self).__init__() self.hidden_dim = hidden_dim self.num_layers = num_layers - self.conv_lstm_cells = nn.ModuleList([ - ConvLSTMCell(input_channels=(input_dim if i == 0 else hidden_dim), - hidden_channels=hidden_dim, - kernel_size=kernel_size, - bias=True) - for i in range(num_layers)]) - # Define a convolutional layer - self.conv = nn.Conv2d(in_channels=hidden_dim, - out_channels=input_dim, - kernel_size=(1, 1), - padding=0) - def forward(self, x, n): - # Get batch size and sequence length from input - batch_size, seq_len, chanel, height, width = x.shape - layer_input = x - hidden_states = [] - cell_states = [] - # Initialize hidden and cell states for each layer - for i in range(self.num_layers): - h, c = self.conv_lstm_cells[i].init_hidden(batch_size, x.shape) - hidden_states.append(h) - cell_states.append(c) - # Initialize list to store interpolated sequences - interpolated_sequences = [x[:, :1]] - for t in range(seq_len - 1): - # Get current and next frame - current_frame = layer_input[:, t] - next_frame = layer_input[:, t + 1] - interpolations = [current_frame] - # Pass through each ConvLSTM layer + layers = [] + for i in range(num_layers): + layers.append(ConvLSTMCell(input_dim if i == 0 else hidden_dim, + hidden_dim, kernel_size)) + + self.conv_lstm_layers = nn.ModuleList(layers) + + def forward(self, input_tensor): + h, c = self.init_hidden(input_tensor.size(0), (input_tensor.size(3), input_tensor.size(4))) + + internal_state = [] + outputs = [] + for timestep in range(input_tensor.size(1)): + x = input_tensor[:, timestep, :, :, :] for layer in range(self.num_layers): - h_cur, c_cur = hidden_states[layer], cell_states[layer] - h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) - current_frame = h_next[:, -1] - # Predict next frame - h_n, c_n = self.conv_lstm_cells[layer](next_frame.unsqueeze(1), (h_next, c_next)) - next_frame_pred = h_n[:, -1] - hidden_states[layer], cell_states[layer] = h_n, c_n - # Generate interpolated frames - for j in range(1, n + 1): - alpha = j / float(n + 1) - interpolated_frame = (1 - alpha) * current_frame + alpha * next_frame_pred - interpolated_frame = F.relu(self.conv(interpolated_frame)) - interpolations.append(interpolated_frame) - interpolations.append(next_frame) - # Concatenate interpolated frames - seq_interpolations = torch.cat(interpolations, dim=1) - interpolated_sequences.append(seq_interpolations) - # Process the last frame - current_frame = layer_input[:, -1] - for layer in range(self.num_layers): - h_cur, c_cur = hidden_states[layer], cell_states[layer] - h_next, c_next = self.conv_lstm_cells[layer](current_frame.unsqueeze(1), (h_cur, c_cur)) - current_frame = h_next[:, -1] - hidden_states[layer], cell_states[layer] = h_next, c_next - interpolated_sequences.append(current_frame.unsqueeze(1)) - # Concatenate all interpolated sequences - full_sequence = torch.cat(interpolated_sequences, dim=1) - return full_sequence + lstm_cell = self.conv_lstm_layers[layer] + h, c = lstm_cell(x, (h, c)) + internal_state.append((h, c)) + + outputs.append(h) + internal_state = [] # Reset the state for the next timestep + + outputs = torch.stack(outputs, dim=1) + return outputs + + def init_hidden(self, batch_size, image_size): + init_states = [] + for i in range(self.num_layers): + init_states.append(self.conv_lstm_layers[i].init_hidden(batch_size, image_size)) + return tuple(init_states) diff --git a/Code/main.py b/Code/main.py index e6d325b..ad08de4 100644 --- a/Code/main.py +++ b/Code/main.py @@ -5,14 +5,16 @@ # Importing Custom Modules from data import CustomDataset -from lstm_model import FrameInterpolationLSTM +from lstm_model import ConvLSTM from autoencoder_model import Grey2RGBAutoEncoder from losses import LossMSE, LossMEP, SSIMLoss from training import Trainer + # Import Necessary Libraries import os import traceback +import torch # Define Working Directories grayscale_dir = '../Dataset/Greyscale' @@ -49,17 +51,16 @@ def main(): print(model_autoencoder) # Initialize LSTM Model and Import Image Sequences (Training, Validation) - grey_sequence_train, grey_sequence_val = dataset.get_lstm_batches(val_split=0.2) + train_loader, val_loader = dataset.get_lstm_batches(val_split=0.2) print('LSTM Model Data Initialized.') - C = 1 - hidden_size = 64 - num_layers = 3 - n_interpolate_frames = 1 # Number of intermediate frames to interpolate - kernel_size = (3, 3) - model_lstm = FrameInterpolationLSTM(C, hidden_size, kernel_size, num_layers) + input_dim = 1 # Grayscale images have 1 input channel + hidden_dim = 64 + kernel_size = 3 + num_layers = 3 # Number of ConvLSTM layers in the model + lstm_model = ConvLSTM(input_dim, hidden_dim, kernel_size, num_layers) print('LSTM Model Initialized.') print('LSTM Model Summary:') - print(model_lstm) + print(lstm_model) ''' Initialize Trainer Objects @@ -70,7 +71,8 @@ def main(): trainer_autoencoder_baseline = Trainer(model_autoencoder, loss_mse, model_save_path_ae) print('Baseline AutoEncoder Trainer Initialized.') model_save_path_lstm = '../Models/Method1/model_lstm_m1.pth' - trainer_lstm_baseline = Trainer(model_lstm, loss_mse, model_save_path_lstm) + lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) + trainer_lstm_baseline = Trainer(lstm_model, loss_mse, lstm_optimizer, model_save_path_lstm) print('Baseline LSTM Trainer Initialized.') # Method 2 : Composite Loss (MSE + MaxEnt) for AutoEncoder and Mean Squared Error Loss for LSTM @@ -84,7 +86,8 @@ def main(): os.makedirs('../Models/Method3', exist_ok=True) # Creating Directory for Model Saving print('Method-3 AutoEncoder == Method-1 AutoEncoder') model_save_path_lstm = '../Models/Method3/model_lstm_m3.pth' - trainer_lstm_m3 = Trainer(model_lstm, loss_ssim, model_save_path_lstm) + lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) + trainer_lstm_m3 = Trainer(lstm_model, loss_ssim, lstm_optimizer, model_save_path_lstm) print('Method-3 LSTM Trainer Initialized.') # Method 4 : Proposed Method : Composite Loss (MSE + MaxEnt) for AutoEncoder and SSIM Loss for LSTM @@ -107,7 +110,7 @@ def main(): try: epochs = 1 print('M1 LSTM Training Start.') - model_lstm_m1 = trainer_lstm_baseline.train_lstm(epochs, n_interpolate_frames, grey_sequence_train, grey_sequence_val) + model_lstm_m1 = trainer_lstm_baseline.train_lstm(epochs, train_loader, val_loader) print('M1 LSTM Training Complete.') except Exception as e: print(f"M1 LSTM Training Error : \n{e}") @@ -128,7 +131,7 @@ def main(): try: epochs = 1 print('M3 LSTM Training Start.') - model_lstm_m3 = trainer_lstm_m3.train_lstm(epochs, n_interpolate_frames, grey_sequence_train, grey_sequence_val) + model_lstm_m3 = trainer_lstm_m3.train_lstm(epochs, train_loader, val_loader) print('M3 LSTM Training Complete.') except Exception as e: print(f"M3 LSTM Training Error : \n{e}") diff --git a/Code/training.py b/Code/training.py index 0a03b59..3020517 100644 --- a/Code/training.py +++ b/Code/training.py @@ -58,60 +58,55 @@ def train_autoencoder(self, epochs, train_loader, val_loader): # Return the Trained Model return self.model - def train_lstm(self, epochs, n_interpolate_frames, train_data, val_data): - min_val_loss = float('inf') # Initialize the minimum validation loss to infinity + def train_lstm(self, epochs, train_loader, val_loader): + best_val_loss = float('inf') for epoch in range(epochs): - print(f'Epoch: {epoch}, Training') - self.model.train() # Set the model to training mode + self.model.train() train_loss = 0.0 - - # Training Loop - for sequences, targets in train_data: - print(f'Input sequence shape: {sequences.shape}, Target sequence shape: {targets.shape}') - self.optimizer.zero_grad() # Reset the gradients accumulated from the previous iteration - sequences = sequences.to(self.device) - targets = targets.to(self.device) - outputs = self.model(sequences, n_interpolate_frames) - # sequences.requires_grad_() # Ensure gradients are required for the input to ConvLSTM - # targets.requires_grad_(False) # Ensure targets do not require gradients - # Assuming the outputs and targets are of shape [batch_size, seq_len, channels, height, width] - # Compute Training Loss only on the interpolated frames (not on the original frames) - loss = self.loss_function(outputs[:, 1:-1], targets[:, 1:-1]) - print(f'Train loss: {loss.item()}') - loss.backward() # Backward Pass - self.optimizer.step() # Update Model Parameters + + # Training loop + for input_seqs, target_seqs in train_loader: + input_seqs = input_seqs.to(self.device) + target_seqs = target_seqs.to(self.device) + + # Perform forward pass + self.optimizer.zero_grad() + outputs = self.model(input_seqs) + + # Compute loss; assuming that we are using an unsupervised learning approach for now + loss = self.loss_function(outputs, target_seqs) train_loss += loss.item() - - train_loss /= len(train_data) - print(f'Epoch : {epoch}, Training Loss : {train_loss}') - # Validation Loss Calculation - self.model.eval() # Set the Model to Evaluation Mode + # Backward pass and optimization + loss.backward() + self.optimizer.step() + + # Average the loss over all batches and print + train_loss /= len(train_loader.dataset) + print(f'Epoch [{epoch+1}/{epochs}], Loss: {train_loss:.4f}') + + # Validation loop val_loss = 0.0 - + self.model.eval() with torch.no_grad(): - for sequences, targets in val_data: - print(f'Epoch: {epoch}, Validation') - sequences = sequences.to(self.device) - targets = targets.to(self.device) - outputs = self.model(sequences, n_interpolate_frames) - # Compute Validation Loss only on interpolated frames (not on the original frames) - predicted_interpolated_frames = outputs[:, 1:-1].reshape(-1, *outputs.shape[2:]) # Reshape to (B * seq_len, C, H, W) - true_interpolated_frames = targets[:, 1:-1].reshape(-1, *targets.shape[2:]) # Same reshaping for targets - loss = self.loss_function(predicted_interpolated_frames, true_interpolated_frames) + for input_seqs, target_seqs in val_loader: + input_seqs, target_seqs = input_seqs.to(self.device), target_seqs.to(self.device) + + # Perform validation forward pass + outputs = self.model(input_seqs) + + # Compute loss; assuming that we are using an unsupervised learning approach for now + loss = self.loss_function(outputs, target_seqs) val_loss += loss.item() - - val_loss /= len(val_data) - print(f'Validation loss: {val_loss}') - - # Print the epoch number and the validation loss - print(f'Epoch : {epoch}, Validation Loss : {val_loss}') - # If the current validation loss is lower than the best validation loss, save the model - if val_loss < min_val_loss: - min_val_loss = val_loss # Update the best validation loss - self.save_model() # Save the model - - # Return the Trained Model + val_loss /= len(val_loader.dataset) + print(f'Validation Loss: {val_loss:.4f}') + + # Save model with best validation loss + if val_loss < best_val_loss: + best_val_loss = val_loss + self.save_model() + print('Best model saved with validation loss: {:.4f}'.format(val_loss)) + return self.model \ No newline at end of file From 624ffe05e9a6b5bb7eae045c1a0a66b04e509cad Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 21:54:47 -0800 Subject: [PATCH 28/32] Fixed Grayscale only for LSTM --- Code/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/data.py b/Code/data.py index 7fd9092..59634ea 100644 --- a/Code/data.py +++ b/Code/data.py @@ -87,7 +87,7 @@ def get_lstm_batches(self, val_split, n=1): # Define a helper function to extract sequences by indices def extract_sequences(indices): - return [self[i] for i in indices] + return [self[i][0] for i in indices] # Only return the grayscale images, not the tuples # Use the helper function to create training and validation sets train_input_seqs = torch.stack(extract_sequences(train_odd_indices)) From 6583f6461314bf96a8dee3f2e018c0a061d3b771 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 21:55:12 -0800 Subject: [PATCH 29/32] Fixed LSTM Training Initialization Parameters --- Code/main.py | 4 ++-- Code/training.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Code/main.py b/Code/main.py index ad08de4..5a20413 100644 --- a/Code/main.py +++ b/Code/main.py @@ -72,7 +72,7 @@ def main(): print('Baseline AutoEncoder Trainer Initialized.') model_save_path_lstm = '../Models/Method1/model_lstm_m1.pth' lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) - trainer_lstm_baseline = Trainer(lstm_model, loss_mse, lstm_optimizer, model_save_path_lstm) + trainer_lstm_baseline = Trainer(model=lstm_model, loss_function=loss_mse, optimizer=lstm_optimizer, model_save_path=model_save_path_lstm) print('Baseline LSTM Trainer Initialized.') # Method 2 : Composite Loss (MSE + MaxEnt) for AutoEncoder and Mean Squared Error Loss for LSTM @@ -87,7 +87,7 @@ def main(): print('Method-3 AutoEncoder == Method-1 AutoEncoder') model_save_path_lstm = '../Models/Method3/model_lstm_m3.pth' lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) - trainer_lstm_m3 = Trainer(lstm_model, loss_ssim, lstm_optimizer, model_save_path_lstm) + trainer_lstm_m3 = Trainer(model=lstm_model, loss_function=loss_ssim, optimizer=lstm_optimizer, model_save_path=model_save_path_lstm) print('Method-3 LSTM Trainer Initialized.') # Method 4 : Proposed Method : Composite Loss (MSE + MaxEnt) for AutoEncoder and SSIM Loss for LSTM diff --git a/Code/training.py b/Code/training.py index 3020517..5f300be 100644 --- a/Code/training.py +++ b/Code/training.py @@ -13,7 +13,7 @@ # Define Training Class class Trainer(): - def __init__(self, model, loss_function, model_save_path): + def __init__(self, model, loss_function, optimizer=None, model_save_path=None): # Define the device self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Define the model and move it to the device @@ -21,7 +21,7 @@ def __init__(self, model, loss_function, model_save_path): # Define the loss function self.loss_function = loss_function # Define the optimizer - self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.001) + self.optimizer = optimizer if optimizer is not None else torch.optim.Adam(self.model.parameters(), lr=0.001) # Define the path to save the model self.model_save_path = model_save_path From 05c94924386d3350bcb7a897a6cf34ba0a0a8f25 Mon Sep 17 00:00:00 2001 From: Siddharth Kekre Date: Wed, 27 Dec 2023 06:04:43 +0000 Subject: [PATCH 30/32] Fix 1 --- Code/lstm_model.py | 2 +- Code/main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index 491ced7..c7d4059 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -61,7 +61,7 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): self.conv_lstm_layers = nn.ModuleList(layers) def forward(self, input_tensor): - h, c = self.init_hidden(input_tensor.size(0), (input_tensor.size(3), input_tensor.size(4))) + h, c = self.init_hidden(input_tensor.size(0), (-2, -1)) # Get the last two dimensions internal_state = [] outputs = [] diff --git a/Code/main.py b/Code/main.py index 5a20413..b3af5c4 100644 --- a/Code/main.py +++ b/Code/main.py @@ -68,7 +68,7 @@ def main(): # Method 1 : Baseline : Mean Squared Error Loss for AutoEncoder and LSTM os.makedirs('../Models/Method1', exist_ok=True) # Creating Directory for Model Saving model_save_path_ae = '../Models/Method1/model_autoencoder_m1.pth' - trainer_autoencoder_baseline = Trainer(model_autoencoder, loss_mse, model_save_path_ae) + trainer_autoencoder_baseline = Trainer(model=model_autoencoder, loss_function=loss_mse, optimizer=torch.optim.Adam(model_autoencoder.parameters()), model_save_path=model_save_path_ae) print('Baseline AutoEncoder Trainer Initialized.') model_save_path_lstm = '../Models/Method1/model_lstm_m1.pth' lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) @@ -78,7 +78,7 @@ def main(): # Method 2 : Composite Loss (MSE + MaxEnt) for AutoEncoder and Mean Squared Error Loss for LSTM os.makedirs('../Models/Method2', exist_ok=True) # Creating Directory for Model Saving model_save_path_ae = '../Models/Method2/model_autoencoder_m2.pth' - trainer_autoencoder_m2 = Trainer(model_autoencoder, loss_mep, model_save_path_ae) + trainer_autoencoder_m2 = Trainer(model=model_autoencoder, loss_function=loss_mep, optimizer=torch.optim.Adam(model_autoencoder.parameters()), model_save_path=model_save_path_ae) print('Method-2 AutoEncoder Trainer Initialized.') print('Method-2 LSTM == Method-1 LSTM') From 2a99816d1668fa5798ce60966cff3ca605d1ca77 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 22:12:20 -0800 Subject: [PATCH 31/32] Updated Hidden State Initializations --- Code/lstm_model.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Code/lstm_model.py b/Code/lstm_model.py index c7d4059..e1bba53 100644 --- a/Code/lstm_model.py +++ b/Code/lstm_model.py @@ -42,8 +42,13 @@ def forward(self, input_tensor, cur_state): return h_next, c_next + # def init_hidden(self, batch_size, image_size): + # height, width = image_size[0], image_size[1] + # return (torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device), + # torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device)) def init_hidden(self, batch_size, image_size): - height, width = image_size[0], image_size[1] + height, width = image_size # Unpack the image size tuple + # Initialize hidden and cell states with non-negative dimensions return (torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device), torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv[0].weight.device)) @@ -61,7 +66,8 @@ def __init__(self, input_dim, hidden_dim, kernel_size, num_layers): self.conv_lstm_layers = nn.ModuleList(layers) def forward(self, input_tensor): - h, c = self.init_hidden(input_tensor.size(0), (-2, -1)) # Get the last two dimensions + # h, c = self.init_hidden(input_tensor.size(0), (-2, -1)) # Get the last two dimensions + h, c = self.init_hidden(input_tensor.size(0), (input_tensor.size(2), input_tensor.size(3))) internal_state = [] outputs = [] From abc695bd96e9f426de2d4ab83a9e4a0baf104351 Mon Sep 17 00:00:00 2001 From: iSiddharth20 Date: Tue, 26 Dec 2023 22:12:43 -0800 Subject: [PATCH 32/32] Fixed Trainer Initilization Parameter Order --- Code/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/main.py b/Code/main.py index b3af5c4..082c50f 100644 --- a/Code/main.py +++ b/Code/main.py @@ -68,7 +68,7 @@ def main(): # Method 1 : Baseline : Mean Squared Error Loss for AutoEncoder and LSTM os.makedirs('../Models/Method1', exist_ok=True) # Creating Directory for Model Saving model_save_path_ae = '../Models/Method1/model_autoencoder_m1.pth' - trainer_autoencoder_baseline = Trainer(model=model_autoencoder, loss_function=loss_mse, optimizer=torch.optim.Adam(model_autoencoder.parameters()), model_save_path=model_save_path_ae) + trainer_autoencoder_baseline = Trainer(model=model_autoencoder, loss_function=loss_mse, optimizer=torch.optim.Adam(model_autoencoder.parameters(), lr=0.001), model_save_path=model_save_path_ae) print('Baseline AutoEncoder Trainer Initialized.') model_save_path_lstm = '../Models/Method1/model_lstm_m1.pth' lstm_optimizer = torch.optim.Adam(lstm_model.parameters(), lr=0.001) @@ -78,7 +78,7 @@ def main(): # Method 2 : Composite Loss (MSE + MaxEnt) for AutoEncoder and Mean Squared Error Loss for LSTM os.makedirs('../Models/Method2', exist_ok=True) # Creating Directory for Model Saving model_save_path_ae = '../Models/Method2/model_autoencoder_m2.pth' - trainer_autoencoder_m2 = Trainer(model=model_autoencoder, loss_function=loss_mep, optimizer=torch.optim.Adam(model_autoencoder.parameters()), model_save_path=model_save_path_ae) + trainer_autoencoder_m2 = Trainer(model=model_autoencoder, loss_function=loss_mep, optimizer=torch.optim.Adam(model_autoencoder.parameters(), lr=0.001), model_save_path=model_save_path_ae) print('Method-2 AutoEncoder Trainer Initialized.') print('Method-2 LSTM == Method-1 LSTM')