working d3d download script

Author: +Julian Kates-Harbeck

data/d3d_signals.py

@@ -29,14 +29,14 @@
 r'\bol_l15_p', #Radiated Power core
 r'\bol_l03_p', #Radiated Power Edge
 'bmspinj', #Total Beam Power
-'bmstinj', #Total Beam Torque
-'pcechpwrf'] #Total ECH Power
+'bmstinj',] #Total Beam Torque
+#'pcechpwrf'] #Total ECH Power Not always on!
 
 signal_paths = ['d3d/' + path for path in signal_paths]
 
 ### 1D EFIT signals ###
 signal_paths += [
-'AOT/EQU.te', #electron temperature profile vs rho (uniform mapping over time)
+'AOT/EQU.t_e', #electron temperature profile vs rho (uniform mapping over time)
 'AOT/EQU.dens_e'] #electron density profile vs rho (uniform mapping over time)
 
 #make into list of lists format to be consistent with jet_signals.py

data/gadata.py

@@ -18,10 +18,11 @@ def __init__(self,signal,shot,tree=None,connection=None,nomds=False):
 		self.yunits = ''
 		self.rank = -1
 		self.connection = connection
+		
 
 		## Retrieve Data 
 		t0 =  time.time()
-		found = False
+		self.found = False
 
 		# Create the MDSplus connection (thin) if not passed in  
 		if self.connection is None:
@@ -31,6 +32,7 @@ def __init__(self,signal,shot,tree=None,connection=None,nomds=False):
 		if nomds == False:
 			#first try, retrieve directly from tree and tag
 			try:     
+				#print 'trying direct using tree and tag'
 				if tree != None:
 					tag = self.signal
 					fstree 	= tree
@@ -48,41 +50,44 @@ def __init__(self,signal,shot,tree=None,connection=None,nomds=False):
    					self.xunits = self.connection.get('units(dim_of(_s))').data()
 				if self.rank > 1:
 					self.ydata 	= self.connection.get('dim_of(_s,1)').data()
-                    self.yunits = self.connection.get('units_of(dim_of(_s,1))').data()
+                			self.yunits = self.connection.get('units_of(dim_of(_s,1))').data()
 					if self.yunits == '' or self.yunits == ' ':
  						self.yunits     = self.connection.get('units(dim_of(_s,1))').data()
-				found = True
+				#print 'zdata: ' + str(self.zdata)
+				self.found = True
 
 				# MDSplus seems to return 2-D arrays transposed.  Change them back.
-				if numpy.ndim(self.zdata) == 2: self.zdata = numpy.transpose(self.zdata)
-				if numpy.ndim(self.ydata) == 2: self.ydata = numpy.transpose(self.ydata)
-				if numpy.ndim(self.xdata) == 2: self.xdata = numpy.transpose(self.xdata)
+				#if numpy.ndim(self.zdata) == 2: self.zdata = numpy.transpose(self.zdata)
+				#if numpy.ndim(self.ydata) == 2: self.ydata = numpy.transpose(self.ydata)
+				#if numpy.ndim(self.xdata) == 2: self.xdata = numpy.transpose(self.xdata)
 
-            except Exception,e:
+            		except Exception,e:
             	#node not found
 #				print '   Signal not in MDSplus: %s' % (signal,) 
 				pass
 
 			# Retrieve data from PTDATA if node not found
-	        if not found:
-                self.zdata = self.connection.get('_s = ptdata2("'+signal+'",'+str(shot)+')')
-                if len(self.zdata) != 1:
-                    self.xdata = self.connection.get('dim_of(_s)')
-                    self.rank = 1
-                    found = True
+	        if not self.found:
+			#print 'Trying ptdata: %s' % (signal,)
+                	self.zdata = self.connection.get('_s = ptdata2("'+signal+'",'+str(shot)+')')
+                	if len(self.zdata) != 1:
+                    		self.xdata = self.connection.get('dim_of(_s)')
+                    		self.rank = 1
+                    		self.found = True
 
 			# Retrieve data from Pseudo-pointname if not in ptdata
-			if not found:
-	#			print '   Signal not in PTDATA: %s' % (signal,) 
-				self.zdata = self.connection.get('_s = pseudo("'+signal+'",'+str(shot)+')')
-				if len(self.zdata) != 1:
-					self.xdata = self.connection.get('dim_of(_s)')
-					self.rank = 1
-				        found = True
+		if not self.found:
+			#print '   Signal not in PTDATA: %s' % (signal,) 
+			self.zdata = self.connection.get('_s = pseudo("'+signal+'",'+str(shot)+')')
+			if len(self.zdata) != 1:
+				self.xdata = self.connection.get('dim_of(_s)')
+				self.rank = 1
+				self.found = True
 
-			if not found: 
+		if not self.found:  #this means the signal wasn't found
+			pass
 	#			print '   Signal not in pseudo-pointname: %s' % (signal,) 
-				print "   No such signal: %s" % (signal,)
+			#print "   No such signal: %s" % (signal,)
 
-	        print '   GADATA Retrieval Time : ',time.time() - t0
-			return
+	        #print '   GADATA Retrieval Time : ',time.time() - t0
+		return

data/get_mdsplus_data.py

@@ -6,7 +6,7 @@
 '''
 from __future__ import print_function
 from MDSplus import *
-from pylab import *
+#from pylab import *
 import numpy as np
 import sys
 import multiprocessing as mp
@@ -21,7 +21,7 @@
 
 #print("Importing numpy version"+np.__version__)
 
-prepath = '/cscratch/share/frnn'
+prepath = '/cscratch/share/frnn/'
 shot_numbers_path = 'shot_lists/'
 save_path = 'signal_data/'
 machine = 'd3d'
@@ -43,7 +43,9 @@
 #	shot_numbers_files = ['shotlist_JaysonBarr_disrupt.txt']
 	shot_numbers_files = ['d3d_short_clear.txt']# ,'d3d_clear.txt', 'd3d_disrupt.txt']
 	server_path = 'atlas.gat.com'
-	import d3d_signals
+	from d3d_signals import signal_paths	
+	import itertools
+	signal_paths = list(itertools.chain.from_iterable(signal_paths))
 #	signal_paths = ['PINJ','IP','Q95','DENSITY','WMHD'] #,'PRAD'] #PRAD returns a 2D xdata
 #       Recommended signals from DIII-D
 	# signal_paths = ['efsli','ipsip','efsbetan','efswmhd','nssampn1l','nssfrqn1l',
@@ -127,6 +129,14 @@ def get_tree_and_tag(path):
 	tag = '\\' + spl[1]
 	return tree,tag
 
+def get_tree_and_tag_no_backslash(path):
+	spl = path.split('/')
+	tree = spl[0]
+	tag = spl[1]
+	return tree,tag
+
+
+
 
 def format_save_path(prepath,signal_path,shot_num):
 	return prepath + signal_path  + '/{}.txt'.format(shot_num)
@@ -161,8 +171,11 @@ def save_shot(shot_num_queue,c,signal_paths,save_prepath,machine):
 							print('Signal {}, shot {} missing. Filling with zeros'.format(signal_path,shot_num))
 							time,data = create_missing_value_filler()
 					elif machine == 'd3d':
+						tree,tag = get_tree_and_tag_no_backslash(signal_path)
 						try:
-							ga1 = gadata.gadata('{}'.format(signal_path),shot_num,tree='d3d',connection=c)
+							ga1 = gadata.gadata('{}'.format(tag),shot_num,tree=tree,connection=c)
+							if not ga1.found:
+								raise
 #							ga1 = gadata.gadata('\\{}'.format(signal_path),shot_num,tree='d3d',connection=c)
 							data = ga1.zdata
 							time = ga1.xdata					
@@ -171,7 +184,7 @@ def save_shot(shot_num_queue,c,signal_paths,save_prepath,machine):
 							print('Signal {}, shot {} missing. Filling with zeros'.format(signal_path,shot_num))
 							time,data = create_missing_value_filler()
 
-					data_two_column = vstack((time,data)).transpose()
+					data_two_column = np.vstack((np.atleast_2d(time),np.atleast_2d(data))).transpose()
 					try: #can lead to race condition
 						mkdirdepth(save_path_full)
 					except OSError, e:
@@ -181,7 +194,7 @@ def save_shot(shot_num_queue,c,signal_paths,save_prepath,machine):
 					    else:
 					        # Our target dir exists as a file, or different error, reraise the error!
 					        raise
-					savetxt(save_path_full,data_two_column,fmt = '%f %f')
+					np.savetxt(save_path_full,data_two_column,fmt = '%.5e')#fmt = '%f %f')
 					print('.',end='')
 			    except:
 					print('Could not save shot {}, signal {}'.format(shot_num,signal_path))