Clip learning rate for the case of very large number of workers: needed to run > ~1800 GPUs on Titan

Author: ASvyatkovskiy

plasma/models/mpi_runner.py

@@ -150,9 +150,9 @@ def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=No
     self.epoch = 0
     self.model = model
     self.optimizer = optimizer
-    self.lr = lr
-    self.DUMMY_LR = 0.1
     self.max_lr = 0.1
+    self.lr = lr if (lr < self.max_lr) else self.max_lr
+    self.DUMMY_LR = 0.1
     self.comm = comm
     self.batch_size = batch_size
     self.batch_iterator_func = batch_iterator()