sync remote changes

cgan_earth/code_files/networks.py

@@ -36,13 +36,13 @@ def __init__(self, g_dim, img_length, im_chan=3, hidden_dim=64):
             # Build the neural network
             self.gen = nn.Sequential(
                 self.make_gen_block(g_dim, hidden_dim * 8),
-                self.make_gen_block(hidden_dim * 8, hidden_dim * 8, Trans=False),
                 self.make_gen_block(hidden_dim * 8, hidden_dim * 8),
-                self.make_gen_block(hidden_dim * 8, hidden_dim * 8, Trans=False),
+                self.make_gen_block(hidden_dim * 8, hidden_dim * 8),
+                self.make_gen_block(hidden_dim * 8, hidden_dim * 8),
                 self.make_gen_block(hidden_dim * 8, hidden_dim * 8)
             )
 
-        def make_gen_block(self, input_channels, output_channels, kernel_size=4, stride=2, padding=2, Trans=True):
+        def make_gen_block(self, input_channels, output_channels, kernel_size=4, stride=2, padding=2):
             '''
             Function to return a sequence of operations corresponding to a generator block of DCGAN;
             a transposed convolution, a batchnorm (except in the final layer), and an activation.
@@ -52,18 +52,11 @@ def make_gen_block(self, input_channels, output_channels, kernel_size=4, stride=
                 kernel_size: the size of each convolutional filter, equivalent to (kernel_size, kernel_size)
                 stride: the stride of the convolution
             '''
-            if Trans:
-                return nn.Sequential(
-                    nn.ConvTranspose2d(input_channels, output_channels, kernel_size, stride, padding),
-                    nn.BatchNorm2d(output_channels),
-                    nn.ReLU(inplace=True)
-                )
-            else:
-                return nn.Sequential(
-                    nn.Conv2d(input_channels, output_channels, kernel_size, stride, padding),
-                    nn.BatchNorm2d(output_channels),
-                    nn.ReLU(inplace=True)
-                )
+            return nn.Sequential(
+                nn.ConvTranspose2d(input_channels, output_channels, kernel_size, stride, padding),
+                nn.BatchNorm2d(output_channels),
+                nn.ReLU(inplace=True)
+            )
 
         def forward(self, noise, labels, Training=True, ratio=2):
             '''
@@ -73,16 +66,10 @@ def forward(self, noise, labels, Training=True, ratio=2):
                 noise: a noise tensor with dimensions (n_samples, z_dim)
             '''
             x = torch.cat((noise.float(), labels.float()), 1)
-            for n, layer in enumerate(self.gen):
-                if n == 1 or n == 3:
-                    x = F.interpolate(x, size = (int(x.shape[-2]*3.6), int(x.shape[-1]*3.6)))
+            for layer in self.gen:
                 x = layer(x)
             # upsample to give output spatial size (img_length, img_length)
-            if Training:
-                up = F.interpolate(x, size = (x.shape[-2]+2, x.shape[-1]+2))
-            else:
-                up = x
-            return torch.sigmoid(self.final_conv(up))
+            return torch.sigmoid(self.final_conv(x))
 
 
     class Critic(nn.Module):
@@ -135,7 +122,10 @@ def forward(self, image, labels):
             '''
             x = torch.cat((image.float(), labels.float()), 1)
             for layer in self.crit:
+                print(x.shape)
                 x = layer(x)
+            print(x.shape)
+            print('all convs done')
             return x.view(len(x), -1)
 
     return Generator, Critic

cgan_earth/code_files/util.py

@@ -201,7 +201,7 @@ def test(path, labels, netG, n_classes, z_dim=64, lf=4, device='cpu', ratio=2):
         tifs.append(tif)
     return tifs, netG
 
-def roll_video(path, label, netG, n_classes, z_dim=64, lf=4, device='cpu', ratio=2, n_clips=30, step_size=1, original_noise=None, replace=False):
+def roll_video(path, label, netG, n_classes, z_dim=64, lf=4, device='cpu', ratio=2, n_clips=30, step_size=1, original_noise=None):
     '''
     Given an integer label, generate an array of images that roll through z and can be used to make a video
     Params:
@@ -244,8 +244,6 @@ def roll_video(path, label, netG, n_classes, z_dim=64, lf=4, device='cpu', ratio
             original_noise = add_noise_dim(random, original_noise, 3)
     else:
         max_len = original_noise.shape[-1]
-    if replace == True:
-        original_noise = replace_noise(original_noise, z_dim, lf, ratio, device)
 
     netG.eval()
     test_label = gen_labels(label, n_classes)[:, :, None, None]
@@ -337,12 +335,9 @@ def transit_video(label1, label2, n_classes, original_noise, netG, lf=4, ratio=2
             img = torch.multiply(img, 255).cpu().detach().numpy()
             for i in range(num_img):
                 if step_size < 1:
-                    if i == 0:
-                        # one z represents 32 pixels in the -1 dimension
-                        out = img[:, :, :, :img.shape[-1]-32]
-                    else:
-                        # currently only implemented for step_size 0.5
-                        out = img[:, :, :, 16:img.shape[-1]-16]
+                    # one z represents 32 pixels in the -1 dimension
+                    step_idx = int(i * step_size * 32)
+                    out = img[:, :, :, step_idx:img.shape[-1]-(32-step_idx)]
                 else:
                     out = img[:, :, :, :img.shape[-1]-32]
                 out = np.moveaxis(out, 1, -1)
@@ -370,6 +365,56 @@ def transit_video(label1, label2, n_classes, original_noise, netG, lf=4, ratio=2
             noise = roll_noise(original_noise, step, max_step, IntStep)
     return imgs, noise, netG
 
+def change_noise(label, original_noise, netG, n_classes, z_dim=64, lf=4, device='cpu', ratio=2, n_clips=30, step_size=1, value=0.01, method='add'):
+    max_len = original_noise.shape[-1]
+
+    test_label = gen_labels(label, n_classes)[:, :, None, None]
+
+    lbl = test_label.repeat(1, 1, lf, max_len).to(device)
+    imgs = np.array([])
+    noise = original_noise
+    step = 0.0
+    if step_size >= 1:
+        num_img = 1
+    else:
+        num_img = int(1/step_size)
+    for _ in tqdm(range(n_clips)):
+        with torch.no_grad():
+            img = netG(noise, lbl, Training=False, ratio=ratio).cuda()
+            img = torch.multiply(img, 255).cpu().detach().numpy()
+            for i in range(num_img):
+                if step_size < 1:
+                    # one z represents 32 pixels in the -1 dimension
+                    step_idx = int(i * step_size * 32)
+                    out = img[:, :, :, step_idx:img.shape[-1]-(32-step_idx)]
+                else:
+                    out = img[:, :, :, :img.shape[-1]-32]
+                out = np.moveaxis(out, 1, -1)
+                if imgs.shape[0] == 0:
+                    imgs = out
+                else:
+                    imgs = np.vstack((imgs, out))
+                step += step_size
+            max_step = lf*ratio-2
+            if max_len == lf*ratio:
+                IntStep = True
+            else:
+                IntStep = False
+            if step_size < 1:
+                step_idx = 1
+            else:
+                step_idx = step_size
+            if step > max_step:
+                step -= max_step
+            noise = roll_noise(noise, step_idx, max_step, IntStep)
+            if method == 'add':
+                noise = torch.sub(noise, value)
+            elif method == 'sub':
+                noise = torch.add(noise, value)
+            elif method == 'combined':
+                noise = vary_noise(noise, value/2, ratio=0.5)
+    return imgs, noise, netG
+
 def animate(path, imgs, fps=24):
     clip = ImageSequenceClip(list(imgs),fps=fps)
     clip.write_gif(path + '_demo1.gif')

cgan_earth/code_files/video.py

@@ -34,13 +34,21 @@ def roll_noise(original_noise, step, max_step, IntStep=True):
 
 def replace_noise(original_noise, z_dim, lf, ratio, device):
     new_noise = torch.zeros_like(original_noise)
-    # keep z0, z1
-    new_noise[:, :, :, :2] = original_noise[:, :, :, :2]
-    new_noise[:, :, :, -3:-1] = original_noise [:, :, :, -3:-1]
+    # keep z0, z1, ...
+    new_noise[:, :, :, :-1] = original_noise[:, :, :, :-1]
     # slot in new random noise
-    new_noise[:, :, :, 2:-3] = torch.randn(1, z_dim, lf, lf*ratio-4, device=device)
-    # repeat z2
-    new_noise[:, :, :, -1] = new_noise[:, :, :, 2]
+    new_noise[:, :, :, -1] = torch.randn(1, z_dim, lf, device=device)
+    return new_noise
+
+def vary_noise(original_noise, value, ratio):
+    new_noise = torch.zeros_like(original_noise)
+    for idx0 in range(original_noise.shape[0]):
+        for idx1 in range(original_noise.shape[1]):
+            old = original_noise[idx0][idx1].clone().flatten()
+            old[torch.randint(len(old), (int(len(old)*ratio),))] = torch.add(old[torch.randint(len(old), (int(len(old)*ratio),))], value)
+            new_noise[idx0][idx1] = old.reshape(original_noise.shape[-2], original_noise.shape[-1])
+    boolean = torch.eq(original_noise, new_noise)
+    new_noise[boolean] = torch.sub(new_noise[boolean], value)
     return new_noise
 
 def uniform_transit(label1_channel, label2_channel, cur_label, l_step_size):

cgan_earth/make_video.py

@@ -36,12 +36,13 @@
 
 # test1: forest, then transit to sea, then roll in sea
 # the speed currently must start with the lowest possible speed to make sure the noise has right dimension
-imgs1, noise, netG = roll_video(proj_path, sea_lbl, netG(z_dim+n_classes, img_length), n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=24*2, step_size=0.25)
-imgs2, noise, netG = transit_video(sea_lbl, forest_lbl, n_classes, noise, netG, lf=lf, ratio=ratio, device=device, step_size=0.25, z_step_size=0.1, l_step_size=0.1, transit_mode='uniform')
-imgs3, noise, netG = roll_video(proj_path, forest_lbl, netG, n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=24, step_size=0.25, original_noise=noise)
-imgs4, noise, netG = roll_video(proj_path, forest_lbl, netG, n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=24, step_size=0.25, original_noise=noise, replace=True)
+imgs1, noise, netG = roll_video(proj_path, sea_lbl, netG(z_dim+n_classes, img_length), n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=5, step_size=0.25)
+imgs2, noise, netG = transit_video(sea_lbl, forest_lbl, n_classes, noise, netG, lf=lf, ratio=ratio, device=device, step_size=0.25, z_step_size=0.1, l_step_size=0.2, transit_mode='uniform')
+imgs3, noise, netG = roll_video(proj_path, forest_lbl, netG, n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=15, step_size=0.25, original_noise=noise)
+imgs4, noise, netG = change_noise(forest_lbl, noise, netG, n_classes, z_dim, lf=lf, device=device, ratio=ratio, n_clips=30, step_size=0.25, value=0.02, method='combined')
 
 # concatenante the imgs together and make video
 imgs = np.vstack((imgs1, imgs2))
 imgs = np.vstack((imgs, imgs3))
+imgs = np.vstack((imgs, imgs4))
 animate(proj_path, imgs, fps=24)