#! /usr/bin/python
"""
#+
# NAME:
#	mag_noaa
# PURPOSE:
#	Downloads daily-updated photospheric magnetic maps from NOAA and
#	run's it through X's HCSS code to make a viable source surface map.
# CALLING SEQUENCE:
#	mag_noaa.py <source> <destination> --ftp --sync --gzip --observatory=observatory
#	as cron job:
#	bash --login -c "mag_noaa.py"
# OPTIONAL INPUTS:
#	<source>	source of photospheric magnetic field maps
#			This is either a directory containing wso*.dat maps,
#			or a text file with a list of fully-qualified names
#			of wso*.dat files.
#			Default: $DAT/map/<observatory>
#	<destination>	destination directory for source surface maps
#	--ftp		synchronizes files on NOAA ftp server with local files.
#			If -ftp is set then <source>=$DAT/map/wso_noaa and
#			<destination>=$DAT/map/wso_noaa/hcss are used, unless
#			-source and/or -destination are specified.
#	--sync		only processes photospheric maps that do not have
#			a matching source surface file in <destination>
#	--gzip		gzip the output files in <destination>
#	--observatory <observatory>
#			observatory; must be one of wso, nso, mwo, nso_ktpk
#			default is nso_ktpk (NOAA Solis data)
# CALLS:
#	mirror, xhcs
# PROCEDURE:
#	Currently  magnetic source surface map at 15 solar radii are made.
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS)
#	OCT-2003, Paul Hick (UCSD/CASS)
#		Added mag_noaa_badfiles to exclude files stored in the text
#		file 'badfiles_'+obs+'.txt' located in the destination directory.
#	NOV-2003, Paul Hick (UCSD/CASS)
#		Added a check to detect when a photospheric map downloaded from
#		NOAA overwrites an existing file. In this case the source surface
#		file in the destination directory (produced from the overwritten
#		file) is deleted to make sure that the new file is processed by
#		Xuepus program.
#	OCT-2005, Paul Hick (UCSD/CASS)
#		Modified default for destination directory to subdirectory
#		hcss in source directory.
#	JUN-2006, Paul Hick (UCSD/CASS)
#		nso_ktpk are now downloaded from remote subdirectory 2006
#	FEB-2006, Paul Hick (UCSD/CASS)
#		nso_ktpk are now downloaded from remote subdirectory 2007
#	JUN-2013, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#		Substantial rewrite to accomodate Solis files from
#		Gong pipeline.
#-
"""

import sys, os, re

def mag_noaa_nameonly ( path_name_ext ):		# Returns name only (no dir, no ext)
	path, name_ext = os.path.split( path_name_ext )
	name, ext = os.path.splitext( name_ext )
	if ext == '.gz': name_ext = name		# Drop .gz (if present)
	name, ext = (os.path.splitext(name_ext))	# Drop extension ('.dat' or '.fits')
	return name

def mag_noaa_diff ( prefix_lst, postfix_lst, lst, prefix_ref, postfix_ref, ref ):

	#+
	# NAME:
	#	mag_noaa_diff
	# PURPOSE:
	#	Looks for entries in array 'lst' that are not present in array 'ref'
	# INPUTS:
	#	prefix_lst
	#	postfix_lst
	#	lst
	#	prefix_ref
	#	postfix_ref
	#	ref
	# OUTPUTS:
	#	lst_modified
	# PROCEDURE:
	#	'lst' contains filenames with names (after stripping extensions)
	#	match prefix_lst at the start and and postfix_lst at the end
	#	of the name.
	#
	#	'ref' contains filenames with names (after stripping extensions)
	#	match prefix_ref at the start and and postfix_ref at the end
	#	of the name.
	#
	#	The center part of the filename in between the prefix and postfix is
	#	used to determine a match: if the center part of a file in 'lst'
	#	matches the center part of a file in 'ref' it is dropped from the
	#	the return list.
	#
	#	The center part of the filenames is a Carrington rotation
	#	and a Carrington longitude in the form NNNN_LLL with NNNN a 4-digit
	#	Carrinton rotation and LLL a 3-digit longitude.
	#	'lst' is a list of solar surface synoptic maps.
	#	'ref' is a list of hcss synoptic maps.
	#
	#	If after removing pre- and postfixes from the filenames the
	#	rotation/longitude in an 'lst' entry is NOT present in the 'ref'
	#	list than the 'lst' entry represents a solar surface map that
	#	has not yet been run through Xuepus hcss program.
	#-

	ref_nameonly = [ mag_noaa_nameonly(f) for f in ref ]

	regex = re.compile('^'+prefix_lst+'(.+)'+postfix_lst+'$')
	newlst = []
	for f in lst:
		match = regex.search( mag_noaa_nameonly( f ) )
		if match and ref_nameonly.count( prefix_ref+match.group(1)+postfix_ref ) == 0:
			newlst.append( f )

	return newlst

def mag_noaa_obs ( prefix, postfix, lst ):	# Pick up file names with right pre- and postfix
	regex = re.compile('^'+prefix+'.+'+postfix+'$')
	return [ l for l in lst if regex.search( mag_noaa_nameonly(l) ) ]

def mag_noaa_badfiles( destin, obs, lst ):

	badlist = os.path.join(destin,'badfiles_'+obs+'.txt')

	if os.path.exists( badlist ):
		ref = (open(badlist, 'r')).read().split()
		ref_nameonly = [ mag_noaa_nameonly( f ) for f in ref ]
		newlst = [ f for f in lst if ref_nameonly.count( mag_noaa_nameonly( f ) ) == 0 ]
	else:
		newlst = lst

	return newlst

if __name__ == '__main__':

	from time		import gmtime, strftime
	from mirror		import mirror
	from tiny_bits	import say, hide_env
	from optparse	import OptionParser

	version = '0.00'
	usage = "  %prog <source> <destination>\n"	+ \
		"\t<source>       directory with solar surface synoptic maps, or\n" + \
		"\t               name of file with fully qualified names of surface maps\n" + \
		"\t               (default: %s)\n"%os.path.join( os.environ['DAT'],'map','<observatory>' ) + \
		"\t<destination>  directory with hcss synoptic maps\n" + \
		"\t               (default: %s)\n"%os.path.join( '<source>', 'hcss' )

	parser = OptionParser(usage=usage,version=version)

	parser.add_option('-v', '--verbose',
		dest	= 'verbose'		,
		action	= 'store_true'	,
		default	= False			,
		help	= 'verbose output'
	)

	parser.add_option('-o', '--observatory',
		dest	= 'obs'	,
		action	= 'store'		,
		type	= 'string'		,
		default	= 'nso_ktpk'		,
		help	= 'name of observatory (default: nso_ktpk)'
    )

	parser.add_option('', '--ftp',
		dest	= 'ftp'		,
		action	= 'store_true'	,
		default	= False			,
		help	= 'download by ftp using mirror.pl'
	)

	parser.add_option('', '--gzip',
		dest	= 'gzip'		,
		action	= 'store_true'	,
		default	= False			,
		help	= 'gzip'
	)

	parser.add_option('', '--sync',
		dest	= 'sync'		,
		action	= 'store_true'	,
		default	= False			,
		help	= 'only run xhcss for solar surface maps without matching hcss file)'
	)

	now = strftime('%Y%m%d', gmtime())
	parser.add_option('', '--now',
		dest	= 'now'			,
		action	= 'store'		,
		type	= 'string'		,
		default	= now			,
		help	= 'day for which data should be downloaded in format YYYYMMDD (default:%s)'%now
	)

	options, args = parser.parse_args()

	say = say((os.path.splitext( (os.path.split(sys.argv[0]))[1] ))[0],options.verbose,timetag="%m/%d %H:%M")

	obs  = options.obs
	ftp  = options.ftp
	gzip = options.gzip
	sync = options.sync or ftp
	now  = options.now

	import tempfile
	tempfile.tempdir = os.environ['TUB']

	info = {
		'wso_noaa':			# New Fits files from NOAA
			{
				'remote_site'	: 'helios.sec.noaa.gov',
				'remote_dir'	: os.path.join('/pub','lmayer','WSA','synoptic','WSO','daily'),
				'remote_prefix' : 'wso'	,
				'remote_postfix': ''	,
				'local_prefix'  : obs	,
				'local_postfix' : ''	,
			},

		'wso_arge':			# Old .dat files from NOAA
			{
		  		'remote_site'	: ''		,
		  		'remote_dir'	: ''		,
				'get_patt'		: ''		,
				'remote_prefix' : 'wso'		,
				'remote_postfix': ''		,
				'local_prefix'  : 'wso_noaa',	# Merge output with wso_noaa
				'local_postfix' : ''		,
			},
		'nso_ktpk_legacy':
			{
				'remote_site'	: 'solis.nso.edu'	,
				'remote_dir'	: os.path.join('/synoptic','level3','vsm','merged','carr-daily','2013'),
				'get_patt'		: '^svsm_m11lr_B3_cr.*.fts.gz$'	,
				'remote_prefix' : 'svsm_m11lr_B3_cr'			,
				'remote_postfix': ''		,
				'local_prefix'  : obs		,
				'local_postfix' : ''		,
			},
		'nso_ktpk':
			{
				'remote_site'	: 'solis.nso.edu'	,
				'remote_dir'	: os.path.join('/pubkeep','v7g', now[0:6], 'kcv7g%s'%now[2:8]),
				'get_patt'		: '^kcv7g%st....c.*_syn-mas_dim-180.fits.gz$'%now[2:8],
				'remote_prefix' : 'kcv7g......t....c',
				'remote_postfix': '_syn-mas_dim-180',
				'local_prefix'  : obs		,
				'local_postfix' : ''		,
			},
		'nso_noaa':
			{
		  		'remote_site'	: ''		,
		  		'remote_dir'	: ''		,
				'get_patt'		: ''		,
				'remote_prefix' : 'nso'		,
				'remote_postfix': ''		,
				'local_prefix'  : obs		,
				'local_postfix' : ''		,
			},
		'mwo_noaa':
			{
		  		'remote_site'	: ''		,
		  		'remote_dir'	: ''		,
				'get_patt'		: ''		,
				'remote_prefix' : 'mwo'		,
				'remote_postfix': ''		,
				'local_prefix'  : obs		,
				'local_postfix' : ''		,
			},
#		'nso_gong':
#			{
#				'remote_site'	: 'gong2.nso.edu'	,
#				'remote_dir'	: os.path.join('/QR/bqs', now[0:6], 'mrbqs%s'%now[2:8]),
#				'get_patt'		: '^mrbqs%st(05|11|17|23)..c.*.fits.gz'%(now[2:8]) ,
#				'remote_prefix' : 'mrbqs......t....c',
#				'remote_postfix': ''		,
#				'local_prefix'  : obs		,
#				'local_postfix' : ''		,
#			},
                'nso_gong':
			{
                    'remote_site'	: 'nispdata.nso.edu'	,
				'remote_dir'	: os.path.join('/QR/bqs', now[0:6], 'mrbqs%s'%now[2:8]),
				'get_patt'	    : '^mrbqs%st(05|11|17|23)..c.*.fits.gz'%(now[2:8]) ,
				'remote_prefix' : 'mrbqs......t....c',
				'remote_postfix': ''		,
				'local_prefix'  : obs		,
				'local_postfix' : ''		,
			},
		}

	source = args[1] if len(args) > 1 else os.path.join( os.environ['DAT'],'map',obs )
	destin = args[2] if len(args) > 2 else os.path.join( source,'hcss' )
#        print(info[obs]['remote_site'],info[obs]['remote_dir'],info[obs]['get_patt'])
	if ftp:
		# Mirror the data directory remote_dir locally in source
		# This keeps remote and local directory identical

		new_list = mirror(
			{
		  		'job'				: obs,
		  		'local_dir'			: source		,
		  		'remote_site'		: info[obs]['remote_site']	,
		  		'remote_dir'		: info[obs]['remote_dir' ]	,
				'remote_password'	: 'bvjackson@ucsd.edu',
		  		'recursive'			: 'false'	,
				'get_patt'			: info[obs]['get_patt'	 ]	,
				'local_ignore'		: 'hcss'	,
                'passive_ftp'       : 'true',
			}
		)

		dst_list = os.listdir( destin )

		# Occasionally we download a file that overwrites a previous photospheric map.
		# We test for this by looking for a source surface file in destin with the same name.
		# If this happens, delete the source surface file to make sure that the new file
		# gets processed by Xuepus program

		tmp_list = [ mag_noaa_nameonly( f ) for f in dst_list ]

		regex = re.compile('^'+info[obs]['remote_prefix']+'(.*)'+info[obs]['remote_postfix'])
		for f in new_list:
			match = regex.search( mag_noaa_nameonly( f ) )
			if match:
				tmp = info[obs]['local_prefix']+match.group(1)+info[obs]['local_postfix']
				if tmp_list.count(tmp) != 0:
					say.say( 'existing file downloaded: '+f )
					tmp = os.path.join( destin, dst_list[tmp_list.index(tmp)] )
					say.say( 'removing source surface file: '+hide_env(tmp) )
					os.remove( tmp )

		# Put a copy of the date file in destination

		#datefile = os.path.join(source, 'datefile_'+obs+'.dat')
		#print 'copy '+datefile+' --> '+destin
		#os.spawnlp( os.P_WAIT, 'cp', 'cp', datefile, destin )

	if os.path.isfile( source ):	# Read list of daily files from file in first cmd line argument

		new_list = (open(source,'r' )).read().split()

	elif os.path.isdir(source): 	# Fully-qualified file names

		new_list = [ os.path.join(source,f) for f in os.listdir(source) if os.path.isfile(os.path.join(source,f)) ]

	else:
		say.die( 'invalid source specified, %s'%source )
	# Pick up all photospheric maps that do not have a matching map
	# in the source surface directory.
	if sync:
		new_list = mag_noaa_diff(
			info[obs]['remote_prefix'], info[obs]['remote_postfix'],
				mag_noaa_obs(info[obs]['remote_prefix'], info[obs]['remote_postfix'], new_list			) ,
			info[obs]['local_prefix' ], info[obs]['local_postfix' ],
				mag_noaa_obs(info[obs]['local_prefix' ], info[obs]['local_postfix' ], os.listdir(destin))
		)
	new_list = mag_noaa_badfiles( destin, obs, new_list )
	if len(new_list) == 0:
		if obs in ('nso_ktpk','nso_gong'):
			say.done('no new photospheric maps for %s on %s'%(obs,now))
		else:
			say.done('no new photospheric maps for %s'%obs)

	new_list.sort()

	main_pro = tempfile.mktemp('.pro')
	tmp_list = tempfile.mktemp('.txt')

	# Write main IDL program that runs xhcs.pro. Arguments are the destination
	# directory for the source surface files and the file containing the
	# fully-qualified names of photospheric maps.

	lines = [
		'xhcs, /silent, "%s"'%obs+
			', destination="%s"'	%destin			+
			', list="%s"'			%tmp_list		+
			', remote_prefix="%s"'	%info[obs]['remote_prefix' ]	+
			', remote_postfix="%s"'	%info[obs]['remote_postfix']	+
			', local_prefix="%s"'	%info[obs]['local_prefix'  ]	+
			', local_postfix="%s"'	%info[obs]['local_postfix' ]	+
			', gzip=%d'			%gzip					,
		'tmp = do_file(/delete, "'+tmp_list+'", /silent)'	,
		'tmp = do_file(/delete, "'+main_pro+'", /silent)'	,
		'exit' ]
			 				# Write main program
	(open(main_pro,'w')).write( '\n'.join( lines )+'\n' )

	# Write the input file with a list of photospheric maps

	new_list.reverse()
	(open(tmp_list,'w')).write( '\n'.join( new_list )+'\n' )

	# Process the newly arrived files
	# Note that FTS reader is needed (readfits.pro)

	os.spawnlp( os.P_WAIT, 'idl', 'idl', '-quiet', main_pro )

	#if obs == 'wso':
	#	os.spawnlp( os.P_WAIT, 'idl', 'idl', '-quiet', main_pro )
	#else:
	#	os.spawnlp( os.P_WAIT, 'csh', 'csh', '-c', 'sswidl '+main_pro )

	sys.exit()