PBS setup for Titan

Author: ASvyatkovskiy

examples/pbs.cmd

@@ -0,0 +1,49 @@
+#!/bin/bash
+#PBS -A FUS117
+#PBS -l walltime=0:05:00
+#PBS -l nodes=2
+##PBS -l procs=1
+##PBS -l gres=atlas1%atlas2
+
+
+#Cannot see home folder, will just hang until wall limit
+export HOME=/ccs/proj/fus117/
+cd $HOME/PPPL/plasma-python/examples
+
+source $MODULESHOME/init/bash
+module switch PrgEnv-pgi PrgEnv-gnu
+
+module load cudatoolkit
+export LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64:$LIBRARY_PATH
+
+#This block is CuDNN module
+export LD_LIBRARY_PATH=$HOME/cuda/lib64:$LD_LIBRARY_PATH
+export LIBRARY_PATH=$HOME/cuda/lib64:$LIBRARY_PATH
+export LDFLAGS=$LDFLAGS:$HOME/cuda/lib64
+export INCLUDE=$INCLUDE:$HOME/cuda/include
+export CPATH=$CPATH:$HOME/cuda/include
+export FFLAGS=$FFLAGS:$HOME/cuda/include
+export LOCAL_LDFLAGS=$LOCAL_LDFLAGS:$HOME/cuda/lib64
+export LOCAL_INCLUDE=$LOCAL_INCLUDE:$HOME/cuda/include
+export LOCAL_CFLAGS=$LOCAL_CFLAGS:$HOME/cuda/include
+export LOCAL_FFLAGS=$LOCAL_FFLAGS:$HOME/cuda/include
+export LOCAL_CXXFLAGS=$LOCAL_CXXFLAGS:$HOME/cuda/include
+
+#This sets new home and Anaconda module
+export PATH=$HOME/anaconda2/bin:$PATH
+export LD_LIBRARY_PATH=$HOME/anaconda2/lib:$LD_LIBRARY_PATH
+source activate PPPL
+
+PYTHON=`which python`
+echo $PYTHON
+
+#pygpu backend
+#export CPATH=$CPATH:~/.local/include
+#export LIBRARY_PATH=$LIBRARY_PATH:~/.local/lib
+#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/.local/lib
+
+export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+export MPICH_RDMA_ENABLED_CUDA=1
+
+rm $HOME/tigress/alexeys/model_checkpoints/*
+aprun -n2 -N1 $PYTHON mpi_learn.py