Start using saner line breaks around fn defs and calls

reduce # of lines by passing multiple arguments per line than what
autopep8 does

Author: felker

plasma/models/loader.py

@@ -107,13 +107,8 @@ def training_batch_generator(self, shot_list):
                         num_so_far, num_total
             epoch += 1
 
-    def fill_training_buffer(
-            self,
-            Xbuff,
-            Ybuff,
-            end_indices,
-            shot,
-            is_first_fill=False):
+    def fill_training_buffer(self, Xbuff, Ybuff, end_indices, shot,
+                             is_first_fill=False):
         sig, res = self.get_signal_result_from_shot(shot)
         length = self.conf['model']['length']
         if is_first_fill:  # cut signal to random position
@@ -388,20 +383,16 @@ def fill_batch_queue(self, shot_list, queue):
 
     def training_batch_generator_process(self, shot_list):
         queue = mp.Queue()
-        proc = mp.Process(
-            target=self.fill_batch_queue, args=(
-                shot_list, queue))
+        proc = mp.Process(target=self.fill_batch_queue,
+                          args=(shot_list, queue))
         proc.start()
         while True:
             yield queue.get(True)
         proc.join()
         queue.close()
 
-    def load_as_X_y_list(
-            self,
-            shot_list,
-            verbose=False,
-            prediction_mode=False):
+    def load_as_X_y_list(self, shot_list, verbose=False,
+                         prediction_mode=False):
         """
         The method turns a ShotList into a set of equal-sized patches which
         contain a number of examples that is a multiple of the batch size.
@@ -696,12 +687,8 @@ def arange_patches(self, sig_patches, res_patches):
             y_list.append(y)
         return X_list, y_list
 
-    def arange_patches_single(
-            self,
-            sig_patches,
-            res_patches,
-            prediction_mode=False,
-            custom_batch_size=None):
+    def arange_patches_single(self, sig_patches, res_patches,
+                              prediction_mode=False, custom_batch_size=None):
         if prediction_mode:
             num_timesteps = self.conf['model']['pred_length']
             batch_size = self.conf['model']['pred_batch_size']

plasma/models/mpi_runner.py

@@ -54,8 +54,8 @@
 
 if backend == 'tf' or backend == 'tensorflow':
     if NUM_GPUS > 1:
-        os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(
-            MY_GPU)  # ,mode=NanGuardMode'
+        os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU)
+        # ,mode=NanGuardMode'
     os.environ['KERAS_BACKEND'] = 'tensorflow'
     import tensorflow as tf
     from keras.backend.tensorflow_backend import set_session
@@ -749,8 +749,8 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None):
     return y_prime_global, y_gold_global, disruptive_global
 
 
-def mpi_make_predictions_and_evaluate(
-        conf, shot_list, loader, custom_path=None):
+def mpi_make_predictions_and_evaluate(conf, shot_list, loader,
+                                      custom_path=None):
     y_prime, y_gold, disruptive = mpi_make_predictions(
         conf, shot_list, loader, custom_path)
     analyzer = PerformanceAnalyzer(conf=conf)
@@ -823,17 +823,9 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader,
         shot_list=shot_list_train)
 
     print("warmup {}".format(warmup_steps))
-    mpi_model = MPIModel(
-        train_model,
-        optimizer,
-        comm,
-        batch_generator,
-        batch_size,
-        lr=lr,
-        warmup_steps=warmup_steps,
-        num_batches_minimum=num_batches_minimum,
-        conf=conf
-    )
+    mpi_model = MPIModel(train_model, optimizer, comm, batch_generator,
+                         batch_size, lr=lr, warmup_steps=warmup_steps,
+                         num_batches_minimum=num_batches_minimum, conf=conf)
     mpi_model.compile(
         conf['model']['optimizer'],
         clipnorm,
@@ -940,13 +932,11 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader,
 
             # tensorboard
             if backend != 'theano':
-                val_generator = partial(
-                    loader.training_batch_generator,
-                    shot_list=shot_list_validate)()
+                val_generator = partial(loader.training_batch_generator,
+                                        shot_list=shot_list_validate)()
                 val_steps = 1
-                tensorboard.on_epoch_end(
-                    val_generator, val_steps, int(
-                        round(e)), epoch_logs)
+                tensorboard.on_epoch_end(val_generator, val_steps,
+                                         int(round(e)), epoch_logs)
 
         print_unique("end epoch {} 0".format(e))
         stop_training = comm.bcast(stop_training, root=0)
@@ -972,11 +962,8 @@ def get_stop_training(callbacks):
 
 
 class TensorBoard(object):
-    def __init__(self, log_dir='./logs',
-                 histogram_freq=0,
-                 validation_steps=0,
-                 write_graph=True,
-                 write_grads=False):
+    def __init__(self, log_dir='./logs', histogram_freq=0, validation_steps=0,
+                 write_graph=True, write_grads=False):
         if K.backend() != 'tensorflow':
             raise RuntimeError('TensorBoard callback only works '
                                'with the TensorFlow backend.')

plasma/models/shallow_runner.py

@@ -71,12 +71,8 @@ def get_sample_probs(self, shot_list, num_samples):
             return val, val
         return sample_prob_d, sample_prob_nd
 
-    def load_shots(
-            self,
-            shot_list,
-            is_inference=False,
-            as_list=False,
-            num_samples=np.Inf):
+    def load_shots(self, shot_list, is_inference=False, as_list=False,
+                   num_samples=np.Inf):
         X = []
         Y = []
         Disr = []
@@ -153,7 +149,6 @@ def process(self, shot):
         return X, Y, disr
 
     def get_X(self, shot):
-
         use_signals = self.loader.conf['paths']['use_signals']
         sig_sample = shot.signals_dict[use_signals[0]]
         if len(shot.ttd.shape) == 1:
@@ -181,13 +176,8 @@ def get_Y(self, shot):
         offset = self.timesteps - 1
         return np.round(shot.ttd[offset:, 0]).astype(np.int)
 
-    def load_shot(
-            self,
-            shot,
-            is_inference=False,
-            sample_prob_d=1.0,
-            sample_prob_nd=1.0):
-
+    def load_shot(self, shot, is_inference=False, sample_prob_d=1.0,
+                  sample_prob_nd=1.0):
         X, Y, disr = self.process(shot)
 
         # cut shot ends if we are supposed to
@@ -200,10 +190,9 @@ def load_shot(
         if disr:
             sample_prob = sample_prob_d
         if sample_prob < 1.0:
-            indices = np.sort(
-                np.random.choice(np.array(range(len(Y))),
-                                 int(round(sample_prob*len(Y))),
-                                 replace=False))
+            indices = np.sort(np.random.choice(np.array(range(len(Y))),
+                                               int(round(sample_prob*len(Y))),
+                                               replace=False))
             X = X[indices]
             Y = Y[indices]
         return X, Y, disr

plasma/models/torch_runner.py

@@ -66,10 +66,8 @@ def __init__(self, n_scalars, n_profiles, profile_size, layer_sizes,
             self.conv_output_size = self.conv_output_size*layer_sizes[-1]
         self.linear_layers = []
 
-        print(
-            "Final feature size = {}".format(
-                self.n_scalars
-                + self.conv_output_size))
+        print("Final feature size = {}".format(self.n_scalars
+                                               + self.conv_output_size))
         self.linear_layers.append(
             nn.Linear(
                 self.conv_output_size
@@ -111,8 +109,8 @@ def forward(self, x):
 
 
 def calculate_conv_output_size(L_in, padding, dilation, stride, kernel_size):
-    return int(np.floor((L_in + 2*padding - dilation
-                         * (kernel_size-1) - 1)*1.0/stride + 1))
+    return int(np.floor(
+        (L_in + 2*padding - dilation * (kernel_size-1) - 1)*1.0/stride + 1))
 
 
 class Chomp1d(nn.Module):
@@ -125,51 +123,28 @@ def forward(self, x):
 
 
 class TemporalBlock(nn.Module):
-    def __init__(
-            self,
-            n_inputs,
-            n_outputs,
-            kernel_size,
-            stride,
-            dilation,
-            padding,
-            dropout=0.2):
+    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation,
+                 padding, dropout=0.2):
         super(TemporalBlock, self).__init__()
-        self.conv1 = weight_norm(
-            nn.Conv1d(
-                n_inputs,
-                n_outputs,
-                kernel_size,
-                stride=stride,
-                padding=padding,
-                dilation=dilation))
+        self.conv1 = weight_norm(nn.Conv1d(
+            n_inputs, n_outputs, kernel_size, stride=stride, padding=padding,
+            dilation=dilation))
         self.chomp1 = Chomp1d(padding)
         self.relu1 = nn.ReLU()
         self.dropout1 = nn.Dropout2d(dropout)
 
-        self.conv2 = weight_norm(
-            nn.Conv1d(
-                n_outputs,
-                n_outputs,
-                kernel_size,
-                stride=stride,
-                padding=padding,
-                dilation=dilation))
+        self.conv2 = weight_norm(nn.Conv1d(
+            n_outputs, n_outputs, kernel_size, stride=stride, padding=padding,
+            dilation=dilation))
         self.chomp2 = Chomp1d(padding)
         self.relu2 = nn.ReLU()
         self.dropout2 = nn.Dropout2d(dropout)
 
-        self.net = nn.Sequential(
-            self.conv1,
-            self.chomp1,
-            self.relu1,
-            self.dropout1,
-            self.conv2,
-            self.chomp2,
-            self.relu2,
-            self.dropout2)
-        self.downsample = nn.Conv1d(
-            n_inputs, n_outputs, 1) if n_inputs != n_outputs else None
+        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1,
+                                 self.dropout1, self.conv2, self.chomp2,
+                                 self.relu2, self.dropout2)
+        self.downsample = (nn.Conv1d(n_inputs, n_outputs, 1)
+                           if n_inputs != n_outputs else None)
         self.relu = nn.ReLU()
         self.init_weights()
 
@@ -196,11 +171,8 @@ def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
             dilation_size = 2 ** i
             in_channels = num_inputs if i == 0 else num_channels[i-1]
             out_channels = num_channels[i]
-            layers += [TemporalBlock(in_channels,
-                                     out_channels,
-                                     kernel_size,
-                                     stride=1,
-                                     dilation=dilation_size,
+            layers += [TemporalBlock(in_channels, out_channels, kernel_size,
+                                     stride=1, dilation=dilation_size,
                                      padding=(kernel_size-1) * dilation_size,
                                      dropout=dropout)]
 
@@ -211,19 +183,11 @@ def forward(self, x):
 
 
 class TCN(nn.Module):
-    def __init__(
-            self,
-            input_size,
-            output_size,
-            num_channels,
-            kernel_size,
-            dropout):
+    def __init__(self, input_size, output_size, num_channels, kernel_size,
+                 dropout):
         super(TCN, self).__init__()
-        self.tcn = TemporalConvNet(
-            input_size,
-            num_channels,
-            kernel_size,
-            dropout=dropout)
+        self.tcn = TemporalConvNet(input_size, num_channels, kernel_size,
+                                   dropout=dropout)
         self.linear = nn.Linear(num_channels[-1], output_size)
 #         self.sig = nn.Sigmoid()
 
@@ -278,7 +242,6 @@ def __init__(self, module, batch_first=False):
         self.batch_first = batch_first
 
     def forward(self, x):
-
         if len(x.size()) <= 2:
             return self.module(x)
 
@@ -301,8 +264,7 @@ def forward(self, x):
 
 def build_torch_model(conf):
     dropout = conf['model']['dropout_prob']
-# dim = 10
-
+    # dim = 10
     # lin = nn.Linear(input_size,intermediate_dim)
     n_scalars, n_profiles, profile_size = get_signal_dimensions(conf)
     # dim = n_scalars + n_profiles*profile_size
@@ -445,9 +407,7 @@ def train_epoch(model, data_gen, optimizer, loss_fn):
 
 
 def train(conf, shot_list_train, shot_list_validate, loader):
-
     np.random.seed(1)
-
     # data_gen = ProcessGenerator(partial(
     # loader.training_batch_generator_full_shot_partial_reset,shot_list
     # = shot_list_train)())