#! /usr/bin/python

import os, sys, tempfile, base64
tempfile.tempdir = os.environ['TUB']
from tiny import hide_env, is_there, start, run_cmd, args, count_instances

#+
# NAME:
#	l1a_near
# PURPOSE:
#	Function to update / retrieve list of already processed L1A files
# CALLING SEQUENCE:
#	near_list = l1a_near( buf_files )
# INPUTS:
#	buf_files		list of file names to be added to the list of already
#					processed L1A files stored in file 'file_list'
#					(this may be an empty list)
# OUTPUTS:
#	near_list		list of processed L1A files currently
# PROCEDURE:
#	The file names (name and type only) of all processed files are stored
#	in the ascii file 'file_list
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#-

def l1a_near( buf_files ):

	global file_list

	say = 'l1a_near, '

	# Get list of l1a files already processed locally
	# The names in 'file_list' do NOT include the directory

	if os.path.exists(file_list):
		near_list = open(file_list, 'r').read()
		near_list = near_list.split()
	else:
		near_list = []

	if len(buf_files) == 0:

		print say+'%d local L1A files'%len(near_list)

	else:

		# Add the names in buf_files to near_list and update 'file_list'

		for buf_file in buf_files:

			tmp_file,drop = os.path.splitext(buf_file)
			if drop != '.gz': tmp_file = buf_file	# Drop .gz if it is present

			drop,tmp_file = os.path.split(tmp_file)	# Drop directory
			near_list.append( tmp_file.lower() )

		near_list.sort()

		# Remove duplicate entries from near_list before writing the list back to disk.

		new_files = []
		for buf_file in near_list:
			if new_files.count ( buf_file ) == 0:
				new_files.append ( buf_file )
			else:
				print say+'duplicate '+buf_file

		near_list = new_files

		open(file_list, 'w').write('\n'.join(near_list)+'\n')

	return near_list

#+
# NAME:
#	l1a_redo
# PURPOSE:
#	Check list of available files against list of files to be reprocessed.
# CALLING SEQUENCE:
#	list = l1a_redo( byftp, buf_files, redo_list, negate )
# INPUTS:
#	byftp		if set, then buf_files is a list of *.buf.gz files (obtained from ftp nlist cmd)
#				if not set, then buf_files is a list of *.buf files available locally.
#
#	buf_files	byftp set: list of *.buf.gz files (file names, plus extension only)
#				byftp not set: list of *.buf files (fully-qualified file names)
#
#	redo_list	list of *.buf files (only file name, plus extension)
#
#	negate		negate set: return subset of buf_files NOT present in redo_list
#				negate NOT set: return subset of buf_files also present in redo_list
# OUTPUTS:
#	list		list of files will be a subset of buf_files.
# PROCEDURE:
#	Compares entries in buf_files against entries in redo_list.
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#-

def l1a_redo( byftp, buf_files, redo_list, negate ):

	say = 'l1a_redo, '

	redo_files = []

	for buf_file in buf_files:

		if byftp:

			# File names obtained from the ftp nlist files are gzipped and may contain
			# uppercase letters (in particular L1A instead of l1a). Strip the gz extension
			# and change to lowercase

			tmp_file,drop = os.path.splitext( buf_file )
			if drop != '.gz': tmp_file = buf_file

		else:		# Drop directory (if present), only retain file name and extension

			drop,tmp_file = os.path.split( buf_file )

		tmp_file = tmp_file.lower()
		count = redo_list.count( tmp_file )
		
		if negate:

			if count == 0:
				redo_files.append( buf_file )

		else:

			if count == 0:
				print say+buf_file+' skipped'
			else:
				redo_files.append( buf_file )

	if byftp:
		print say+'%d remote L1A files to process'%len(redo_files)
	else:
		print say+'%d local L1A files to process'%len(redo_files)

	return redo_files

#+
# NAME:
#	l1a_gunzip
# PURPOSE:
#	Unzip list of *.buf.gz files (usually files downloaded from Hanscom)
# CALLING SEQUENCE:
#	buf_files = l1a_gunzip(gz_list, ldir, redo_list)
# INPUTS:
#	gz_list		list of *.buf.gz files (only file name, plus extension)
#	ldir		directory where *.buf.gz files are located
#	redo_list	only entries in gz_list also on redo_list are processed
#				(if redo_list is empty all files in gz_list are processed)
# OUTPUTS:
#	buf_files	list of successfully unzipped *.buf files
#				(fully-qualified file names, located in ldir)
# PROCEDURE:
#	Loops over all files in far_list, spawning gunzip for each file individually.
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#-

def l1a_gunzip(gz_list, ldir, redo_list):

	say = 'l1a_gunzip, '

	buf_files = []		# Stores *.buf files (return array)

	for gz_file in gz_list:

		buf_file,ext = os.path.splitext(gz_file)
		gunzip = ext == '.gz'
		if gunzip: buf_file,ext = os.path.splitext(buf_file)

		if ext == '.buf':				# Check for .buf

			if len(redo_list) == 0 or redo_list.count( gz_file ) != 0:
				buf_file = os.path.join( ldir, gz_file )
 
				print say+hide_env(buf_file)
				if gunzip:
					sts = os.spawnlp( os.P_WAIT, 'gunzip', 'gunzip', '-f', buf_file )
					buf_file,ext = os.path.splitext(buf_file)

				if os.path.isfile( buf_file ):	# Check whether unzipped file exists
					buf_files.append( buf_file )

	return buf_files						# Return list of unzipped *.buf files
#+
# NAME:
#	l1a_summary
# PURPOSE:
# CALLING SEQUENCE:
#	status = l1a_summary( wait )
# INPUTS:
# OUTPUTS:
#	(none)
# PROCEDURE:
# MODIFICATION HISTORY:
#	JUL-2003, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#-

def l1a_summary( wait ):

	doy_summary = os.path.join( os.environ['HOME'],'l1a_doy_summary.txt' )

	status = os.path.exists(doy_summary)
	
	if status:

		doy = open(doy_summary,'r').read()

		doy = doy.split('\n')				# Last day summarized
		doy = int(doy[0])+1					# Add one
		doy = '%(doy)d' % vars()			# Convert to string

		main_pro = tempfile.mktemp('.pro')

		# Write the main IDL program. This is just a single line calling smei_frm_summary,
		# with a second line to delete the main IDL program after completion.

		open(main_pro,'w').write(		\
			'smei_frm_summary, timeset(doy='+doy+',hr=[0,24]), dest=getenv("TUB"), /silent\n'	+ \
			'tmp = do_file(/delete, "'+main_pro+'", /silent)\n'									+ \
			'exit\n'	\
		)

		# Run the IDL program

		os.spawnlp( wait, 'idl', 'idl', '-quiet', main_pro )

		open(doy_summary, 'w').write(doy+'\n')		# Update doy_summary file

	return status


#+
# NAME:
#	l1a_idl
# PURPOSE:
#	Unpack *.buf files writing separate files for each frame
# CALLING SEQUENCE:
#	l1a_idl( sys_argv, wait, buf_files, ddir)
# INPUTS:
#	buf_files	list of *.buf files or *.buf.gz (fully-qualified names)
#				Must have at least one element.
#	ddir		destination directory
# OUTPUTS:
#	(none)
# PROCEDURE:
# > Creates main program for starting up the IDL procedure smei_buf
# >	Writes ascii file with input information for smei_buf,
# > Then spawns the main IDL program
# > Both the IDL main program and the ascii file will be deleted by smei_buf
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS)
#	FEB-2004, Paul Hick (UCSD/CASS)
#		Added -noclones keyword
#	MAR-2005, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#		Added -nic keyword. By default Fits files are written to disk.
#-

def l1a_mtime( f1, f2 ):

	p1 = (os.stat(f1))[8]
	p2 = (os.stat(f2))[8]

	if p1 > p2:
		sts = 1
	elif p1 == p2:
		sts = 0
	else:
		sts = -1

	return sts

def l1a_idl( sys_argv, wait, buf_files, ddir ):

	say = 'l1a_idl, '

	# Type of data to be extracted (default: full frames)

	nodups = is_there('-nodups', sys_argv)

	if is_there('-nowrite', sys_argv):
		overwrite = 2
	else:
		overwrite = is_there('-overwrite', sys_argv)

	zipdir	= is_there('-zipdir'  , sys_argv)
	remove	= is_there('-remove'  , sys_argv)
	noclones= is_there('-noclones', sys_argv)
	silent	= is_there('-silent'  , sys_argv)
	mtime	= is_there('-mtime'   , sys_argv)
	nic		= is_there('-nic'	 , sys.argv)

	moveto  = start('-move=', sys.argv )
	if moveto != '':
		if not os.path.isdir(moveto):
			print 'non-existent directory:', moveto
			moveto = ''

	# List of email addresses to notify of completion (default: none)

	email  = start('-email=', sys_argv)
	if email == '':
		email = 'no_email'

	main_pro = tempfile.mktemp('.pro')
	tmp_list = tempfile.mktemp('.txt')

	# Write the main IDL program to do the unpacking. This program calls
	# smei_buf with the name of the input file tmp_list as argument

	open(main_pro,'w').write(		\
		'smei_buf, "'+tmp_list+'"\n'						+ \
		'tmp = do_file(/delete, "'+tmp_list+'", /silent)\n'	+ \
		'tmp = do_file(/delete, "'+main_pro+'", /silent)\n'	+ \
		'exit\n'					\
	)

	# Write the input file for smei_buf

	iu = open( tmp_list, 'w' )

	iu.write( ddir +'\n' )			# Destination directory
	iu.write( email+'\n' )			# Email addresses to notify of completion

	# nodups=0: Don't remove duplicate frames
	# nodups=1: Remove duplicate frames

	iu.write( ['keep','remove'][nodups]+'\n' )

	# overwrite=0: Don't overwrite existing frames
	# overwrite=1: Overwrite existing frames

	iu.write( ['skip existing','overwrite existing','do not write'][overwrite]+'\n' )

	iu.write( ['raw' ,'gzip'  ][zipdir  ]+'\n' )	# When leaving subdirectory, zip all frame files

	if moveto == '':
		iu.write( ['keep','remove'][remove  ]+'\n' )# Delete *.buf files after processing
	else:
		iu.write( moveto+'\n' )

	iu.write( ['keep','remove'][noclones]+'\n' )	# Skip clone frame
	iu.write( ['loud','silent'][silent  ]+'\n' )	# Level of silence (0,1)
	iu.write( ['.fts','.nic'  ][nic	    ]+'\n' )	# Nic of Fits files

	# List of *.buf files to be processed (fully-qualified) names

	if mtime:
		buf_files.sort( l1a_mtime )			
	else:
		buf_files.sort()

	iu.write( '\n'.join(buf_files)+'\n' )

	iu.close()

	# Process the newly arrived files (the IDL program gets its input
	# from the file tmp_list.

	os.spawnlp( wait, 'idl', 'idl', '-quiet', main_pro )

	return

#+
# NAME:
#	l1a_ftp
# PURPOSE:
#	Download .buf.gz files
# CALLING SEQUENCE:
#	buf_files = l1a_ftp( sys_argv, ldir, redo_list, all_l1a, ddir, unblock )
# INPUTS:
#	sys_argv	command line list sys.argv
#				contains the info to log into the ftp site
#	ldir		local directory where to put the downloaded files
#	redo_list	list of l1a files to process
#				Only used if len(redo_list) != 0
#	all_l1a		if set, process all *.buf files
#				if not set, only *.buf files not processed locally yet will be
#				downloaded.
# OUTPUTS:
#	buf_files	list *.buf.gz to be processed further
#				if detach is set then the return value buf_files is an empty list.
# CALLS:
#	l1a_near, l1a_redo, l1a_idl
# PROCEDURE:
# > The presence of ftp-specific keywords -detach, -sparseft and -maxftp is tested
#	for inside this routine.
# MODIFICATION HISTORY:
#	APR-2003, Paul Hick (UCSD/CASS)
#	FEB-2004, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#		Local L1A file names are now in lower case (l1a* instead of L1A*)
#-

def l1a_ftp_modtime( site, folder, far_list ):

	ftp_script = tempfile.mktemp('.ftp')

	# Write the ftp script

	iu = open( ftp_script, 'w' )
	iu.write('cd '+folder+'\n')
	for x in far_list: iu.write('modtime '+x+'\n' )
	iu.write('quit\n' )
	iu.close()

	# This depends on having an entry in .netrc

	lst = run_cmd( 'cat '+ftp_script+' | ftp '+site, 1 )

	os.remove( ftp_script )

	lst = lst.split('\n')
	gmt = []
	for x in lst:

		# x should be in the form
		#	L1A_2005_058_063623_032056.buf.gz       02/27/2005 12:20:09 GMT
		# Occassionally the file timestamp cannot be retrieved and a line
		#	"550 L1A_...buf.gz: No such file or directory"
		# shows up. We try to eliminate those here by only accepting lines
		# starting with L1A_. This also elimates the trailing empty entry.
  
		if x.find('L1A_') == 0:
			y = x.split('\011')
			y.reverse()
			y[0] = y[0][6:10]+'/'+y[0][3:5]+'/'+y[0][0:2]+y[0][10:19]
			gmt.append(y)

 	gmt.sort()

	lst = []
	for x in gmt:
		 lst.append(x[1])

	return lst


def l1a_ftp( sys_argv, ldir, redo_list, all_l1a, ddir, unblock ):

	global file_list

	say = 'l1a_ftp, '

	use_sftp = is_there('-sftp'  , sys.argv)
	if use_sftp:
		kblob = start('-kblob=', sys_argv)
		if kblob == '':
			kblob = 64				# Read in blocks of 64 kbytes
		else:
			kblob = int(kblob)

	argv = args( sys_argv )

	buf_files = []

	# Check whether info is available in .netrc file. If not then all info must
	# be input through the command line.

	netrc = os.path.join( os.environ['HOME'], '.netrc' )

	username = ''
	if os.path.exists( netrc ):
		machines = (open(netrc,'r')).read()

		machines = machines.split('\n')
		for m in machines:
			part = m.split()
			if part.count(argv[1]) != 0:
				username = part[3]
				password = part[5]

	if username == '':

		if len(argv) != 5:
			print say+'specify remote info (site, dir [, user and password])'
			return buf_files

		username = argv[3]
		password = argv[4]

	print say+'remote site: '+username+'<*****>@'+argv[1]+':'+argv[2]

	if not unblock:
		if count_instances ( os.path.split(argv[0])[1]+' '+argv[1], 1 ) >= 2:
			sys.exit()

	if use_sftp:

		import paramiko
		import time

		keyfile = os.path.join(os.environ['HOME'],'.ssh','known_hosts')
		iu = open(keyfile, 'r')

		for line in iu:
			keylist = line.split(' ')
			if len(keylist) != 3:
				continue
			hosts, keytype, key = keylist
			if hosts.find(argv[1]) == 0:
				break
		else:
			print say+'no key for '+argv[1]
			iu.close()
			sys.exit()

		iu.close()
		key = paramiko.RSAKey(data=base64.decodestring(key))

		#passwd  = ''
		#keyfile = os.path.join(os.environ['HOME'], '.ssh','id_dsa')
		#key = paramiko.DSSKey.from_private_key_file(keyfile, passwd)
		ftp = paramiko.Transport(argv[1])
		ftp.connect(hostkey=key, username=username, password=password)
		#ftp.connect(username=username, pkey=key)
		#ftp.close()

		sftp = ftp.open_session()
		sftp.exec_command('cd '+argv[2]+'; ls *.buf')
		far_list = sftp.makefile().read()
		sftp.close()
		ftp.close()

		far_list = far_list.split('\n')
		far_list = far_list[0:len(far_list)-1]

	else:

		from ftplib import FTP

		# Connect to remote site and pull down list of l1a files
		# These should all be l1a*.buf.gz files

		ftp = FTP(argv[1],username,password)# Open connection

		ftp.cwd(argv[2])					# CD to data directory
		far_list = ftp.nlst()				# Get list of files
		ftp.close()							# Close connection

	# Only download *.buf.gz files (in the past sometimes VCID telemetry
	# files have ended up in the level1a directory.

	near_list = []
	for far_gzip in far_list:
		buf,ext = os.path.splitext(far_gzip)
		if ext == '.gz':
			buf,ext = os.path.splitext(buf)
			ext = ext+'.gz'

		# The file must be a buf.gz file with name
		# L1A_2003_123_123456_123456.buf.gz, i.e. 33 chars
		# or a .buf file (30 chars)

		if ext == '.buf.gz' and len(far_gzip) == 33:
			near_list.append(far_gzip)
		elif ext == '.buf' and len(far_gzip) == 30:
			near_list.append(far_gzip)
		else:
			print 'not downloading '+far_gzip

	far_list = near_list
	if len(redo_list) != 0:
		far_list = l1a_redo( 1, far_list, redo_list, 0 )

	if not all_l1a:
		near_list = l1a_near( [] )
		far_list  = l1a_redo( 1, far_list, near_list, 1 )

	if len(far_list) == 0:

		print say+'no remote L1A files left to download'

		# Nothing to download, so lets run the summary program

		#if not l1a_summary( os.P_NOWAIT ):
		#	print say+'no summary plot made'

	else:			# Test for ftp-specific keywords

		detach	  = is_there('-detach'	, sys_argv)
		maxftp	  = start   ('-maxftp='  , sys_argv)

		if maxftp == '': maxftp = '5'
		maxftp = eval(maxftp)

		far_list.sort()

		if len(far_list) > maxftp:
			far_list = far_list[0:maxftp]

		mtime = is_there('-mtime' , sys_argv)

		if mtime:
			far_list = l1a_ftp_modtime( argv[1], argv[2], far_list )

		# Opening an sftp connection once here should work, I think,
		# but it never worked very well. With Paramiko 1.0 only two L1A
		# files could be downloaded safely in one haul; with Paramiko 1.4
		# every other file would fail to download.

		#if use_sftp:
		#	#ftp = paramiko.Transport(argv[1])
		#	#ftp.connect(hostkey=key, username=username, password=password)
		#	#sftp = paramiko.SFTP.from_transport(ftp)
		#else:
		#	ftp = FTP(argv[1],username,password)	# Open connection (again)
		#	ftp.cwd(argv[2])						# CD to data directory

		if not use_sftp:
			ftp = FTP(argv[1],username,password)	# Open connection (again)
			ftp.cwd(argv[2])						# CD to data directory

		for far_gzip in far_list:

			tt = time.time()
			near_gzip = os.path.join(ldir,far_gzip.lower())

			try:
				if use_sftp:

					# Open new sftp connection for each L1A file.
					# Ugly, but it seems to work.

					ftp = paramiko.Transport(argv[1])
					ftp.connect(hostkey=key, username=username, password=password)
					sftp = ftp.open_sftp_client()

					iu = sftp.open( os.path.join(argv[2],far_gzip), 'r', -1 )
					sz_far = iu.stat().st_size	# Remote file size

					print say+far_gzip+' -> '+hide_env(ldir)+' (%d bytes)'%sz_far

					if kblob == 0:				# Works but is quite slow

						data = iu.read()
						iu.close()

						ftp.close()

						sz_near = len(data)
						(open(near_gzip,'wb')).write(data)

					else:						# 64 kB appears to be about optimal

						blob = kblob*1024		# Read in blocks of 'blob' bytes
						sz_near = 0
						ju = open(near_gzip,'wb')
						data = iu.read(blob)
						while len(data) > 0:	# Read until there is nothing left
							ju.write(data)
							sz_near = sz_near+len(data)
							data = iu.read(blob)
						ju.close()				# Close local file
						iu.close()				# Close remote file

						ftp.close()				# Close sftp connection

					# Compare local and remote file size. Reject the downloaded
					# file if they do not match.

					if sz_near == sz_far:
						tt = time.time()-tt
						sz_near = sz_near/1024.0
						sts = 'sftp complete %.2f MB, %.2f hours, %.2f kB/s'%	\
							(sz_near/1024.0,tt/3600.0,sz_near/tt)
					else:
						sts = 'local and remote size do not match, '+far_gzip
				else:

					print say+far_gzip+' -> '+hide_env(ldir)

					sts = ftp.retrbinary('RETR '+far_gzip, open(near_gzip,'wb').write)

			except:

				if use_sftp:
					ftp.close()					# Close sftp connection

				# Error downloading far_gzip
				# Variable 'sts' is not defined here

				print 'exception on '+far_gzip
				os.remove( near_gzip )

				# Add file to list of locally processed files to avoid
				# downloading over and over again.

				near_list = l1a_near( [near_gzip] )

				print	'\n\n\n'+	\
						'=================== WARNING =================\n'+	\
						near_gzip+'\n'+								\
						'Download interrupted. To force another download\n'+\
						'remove entry from '+file_list+'\n'+				\
						'=============================================\n\n\n'

			else:
				print say+sts

				if	(use_sftp and							\
						(									\
						sts.find('sftp complete') == 0	\
						)									\
					) or									\
					(not use_sftp and						\
						(									\
						sts == '226 Transfer complete.' or	\
						sts == '226 File send OK.'			\
						)									\
					):

					if detach:
						near_gzip = [near_gzip]
						near_list = l1a_near( near_gzip )
						l1a_idl( sys_argv, os.P_NOWAIT, near_gzip, ddir )
					else:
						near_gzip = os.path.split(near_gzip)[1]
						buf_files.append(near_gzip)

				else:
					os.remove( near_gzip )

		if not use_sftp:
			ftp.close()				# Close connection (again)

		if detach: print say+'detached unpack finished'

	return buf_files

def l1a_mirror( sys_argv, ldir, redo_list, all_l1a ):

	global file_list

	say = 'l1a_mirror, '

	argv = args( sys_argv )

	buf_files = []

	far_list = os.listdir( argv[2] )

	# Only download *.buf.gz files (in the past sometimes VCID telemetry
	# files have ended up in the level1a directory.

	near_list = []
	for far_gzip in far_list:
		buf,ext = os.path.splitext(far_gzip)
		if ext == '.gz':
			buf,ext = os.path.splitext(buf)
			ext = ext+'.gz'

		# The file must be a buf.gz file with name
		# L1A_2003_123_123456_123456.buf.gz, i.e. 33 chars
		# or a .buf file (30 chars)

		if ext == '.buf.gz' and len(far_gzip) == 33:
			near_list.append(far_gzip)
		elif ext == '.buf' and len(far_gzip) == 30:
			near_list.append(far_gzip)
		else:
			print 'not processing '+far_gzip

	far_list = near_list
	if len(redo_list) != 0:
		far_list = l1a_redo( 1, far_list, redo_list, 0 )

	if not all_l1a:
		near_list = l1a_near( [] )
		far_list  = l1a_redo( 1, far_list, near_list, 1 )

	if len(far_list) == 0:

		print say+'no L1A files left to process'

	else:

		maxftp = start('-maxftp=', sys_argv)

		if maxftp == '': maxftp = '5'
		maxftp = eval(maxftp)

		far_list.sort()

		if len(far_list) > maxftp:
			far_list = far_list[0:maxftp]

		for far_gzip in far_list:
			near_gzip = far_gzip.lower()
			tmp = run_cmd( 'cp -vp '+os.path.join(argv[2],far_gzip)+' '+os.path.join(ldir,near_gzip), 0 )
			buf_files.append(near_gzip)

	return buf_files

#+
# NAME:
#	l1a
# PURPOSE:
#	Script for processing the SMEI L1A data
# CATEGORY:
#	smei/gen/python
# CALLING SEQUENCE:
#	l1a.py <site> <rdir> <user> <password>
# INPUTS:
#	If a remote server is specified an attempt at downloading by (s)ftp is made:
#
#	<site>			remote server
#	<rdir>			directory on remote site with L1A files
#	<user>			remote user
#	<password>		password
# OPTIONAL INPUT PARAMETERS:
#	-source=source	default: /mnt/cdrom
#					source directory to look for *.buf or *.buf.gz files.
#					For ftp this is also the directory where downloaded
#					*.buf.gz are put (write access to the source directory
#					is needed)
#					If L1A files are retrieved from a local L1A mirror
#					(-mirror is set) then this is the directory where the
#					L1A files taken from the mirror are copied to.
#	-destination=destination
#					destination directory for CCD frames:
#					default: $SMEIDB
#
#	-gunzip			look for gzipped *.buf.gz files and unzip them,
#					instead of looking for *.buf files
#	-maxftp=maxftp	maximum number of l1a*.buf.gz files downloaded by ftp,
#					or copied from local L1A mirror (if -mirror is set)
#
#	-nounpack		only download and unzip L1A files. Don't unpack them.
#					The names of successfully downloaded and unzipped files are
#					WILL BE ADDED to the list of locally processed L1A files.
#	-detach			only used for ftp downloads.
#					If set then an L1A file is unpacked immediately after
#					download is complete, in parallel with the download of the
#					next L1A file. If not set, all L1A files are downloaded
#					first, and are then unpacked in a single operation.
#
#	-l1a=l1a		name of specific L1A file to be processed
#	-all_l1a		by default only L1A files are processed that are not yet
#					listed in file 'file_list' (under the assumption that
#					these have already been dealt with). Set this keyword to
#					unconditionally process L1A files.
#	-file_list=file_list
#					fully-qualified name of file with names of already processed
#					L1A files. The file must exist already, so if necessary
#					create an empty one before running this script.
#	-nowrite		suppress writing of CCD frames (useful for testing only)
#	-overwrite		force ovewriting of existing CCD frames.
#					By default frames are not written to drive if a file for the
#					frame already exists. This is currently the primary control
#					for picking up either the first or second telemetry dump
#					(setting -overwrite speeds up processing by a factor 2).
#	-nodups			if set then an attempt is made to remove duplicate frames
#					before starting to write individual frames.
#	-zipdir			zip CCD frames when leaving subdirectory
#	-remove			delete *.buf files after processing
#	-move=move		move *.buf files to directory 'move' after processing
#	-noclones		skip clone frames (see href=smei_buf_prep=)
#	-unblock		skips test for running instances of l1a.py in ftp mode.
#	-nic			write .nic files instead of .fts files (OBSOLETE)
#	-kblob			buffer size for reading remote L1A files for sftp.
#					default is 64 kB. If set to zero the file is downloaded
#					with a single read (this is very slow for some reason).
#	-sftp			download using sftp instead of ftp.
#	-mirror			looks for L1A files in locally maintained mirror of
#					remote L1A files (instead of downloading by ftp or sftp)
# OUTPUTS:
#	Updated data in <destination>
# CALLS:
#	l1a_near, l1a_ftp, l1a_idl, l1a_redo, l1a_gunzip
# SEE ALSO:
#	smei_buf
# PROCEDURE:
# >	Only one instance of the script is allowed to run.
# >	The list of L1A files found locally or (when using ftp) on the remote site
#	are compared against the list of l1a file names in file 'file_list'
#	Only files not on this local list will be downloaded. Override this
#	behaviour using keyword all_l1a.
# MODIFICATION HISTORY:
#	MAR-2003, Paul Hick (UCSD/CASS)
#	APR-2003, Paul Hick (UCSD/CASS)
#		Added -overwrite, -nodups, -zipdir, -remove keywords.
#	OCT-2003, Paul Hick (UCSD/CASS)
#		Added -alive keyword.
#	FEB-2004, Paul Hick (UCSD/CASS)
#		Added -nounpack keyword.
#		Added -noclones keyword to l1a_idl
#		Added -move keyword
#	MAR-2004, Paul Hick (UCSD/CASS)
#		Added keyword 'file_list' instead of hardcoding file name $HOME/l1a_files.txt.
#	FEB-2005, Paul Hick (UCSD/CASS)
#		Removed -sparseftp option (never used it).
#		Added -sftp switch to use sftp instead of plain ftp (requires
#		packages paramiko and pycrypto).
#		Removed all references to the module string.
#	SEP-2005, Paul Hick (UCSD/CASS)
#		After installing Paramiko 1.4 on 64-bit AMD (running Fedora Core)
#		download would fail for every other file (first downloaded OK,
#		second failed, third OK, etc.). Modified code to open a new
#		ssh session for each download. Also introduced kblob keyword.
#	DEC-2005, Paul Hick (UCSD/CASS)
#		Added keyword -mirror to be able to process L1A files from a
#		locally-maintained mirror of the L1A data base.
#	FEB-2008, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#		Added check for existence of env var $L1A.
#		Added check to prevent two copies of l1a.py to run at the
#		same time.
#-

if __name__ == '__main__':

	name = os.path.split(sys.argv[0])[1]
	say  = name+', '

	if not os.environ.has_key('L1A'):
		print say+'env var L1A not defined'
		sys.exit()

	# =========
	# Here we check for problems with the Promise board.
	# Make sure that the SMEI drives are accessible by checking that
	# $SMEIDB1/2003_033 is a valid directory.

	if is_there('-alive', sys.argv):

		if os.path.isdir(os.path.join(os.environ['SMEIDB1'],'2003_033')) and	\
		   os.path.isdir(os.path.join(os.environ['SMEIDB2'],'2003_180')):

			print say+'SMEI drives available; all is well'

		else:

			print say+'SMEI drives not found; cold boot required'

			email = start('-email=', sys.argv)
			if email != "":
				cmd = 'echo "Cold reboot required" | mail '+	\
					email+' -s"SMEI drives on '+os.environ["HOSTNAME"]+	\
					' apparently off-line"'
				status = run_cmd( cmd, 0 )

		sys.exit()

	file_list = start('-file_list=', sys.argv)
	if file_list == '':
		print say+'no list of already-processed l1a files available'
		sys.exit()

	if not os.path.exists(file_list):
		print say+'file does not exist:', hide_env(file_list)
		sys.exit()

	# Source directory for the L1A files (default: /mnt/cdrom)
	# This is also the directory where the L1A files are downloaded.

	ldir = start('-source=', sys.argv)
	if ldir == '':
		ldir = '/mnt/cdrom'

	bymirror = is_there('-mirror', sys.argv)
	if bymirror:
		byftp = False

		if count_instances('l1a.py -source='+os.environ['L1A_GET'],1) > 1:
			sys.exit()
	else:
		argv = args( sys.argv )
		byftp = len( argv ) > 1

		if byftp:
			if count_instances('l1a.py '+argv[1],1) > 1:
				sys.exit()

	# Destination directory for the individual frames (default: $SMEIDB)
	# Frames are put in a subdirectory of this directory.

	ddir = start('-destination=', sys.argv)
	if ddir == '':
		ddir = os.environ['SMEIDB']

	if ddir != "SMEIDB?":
		if not os.path.isdir(ddir):
			print say+'non-existent directory, '+hide_env(ddir)
			sys.exit()

	# Explicit list of L1A files to be processed

	redo_list = start('-l1a=', sys.argv)
	if redo_list == '':
		redo_list == []
	else:
		redo_list = redo_list.split(',')

	# Downloaded files are gzipped, so we need to unzip.
	# The first command line arg (if present, should be the remote site)

	gunzip	= is_there('-gunzip'  , sys.argv)
	all_l1a	= is_there('-all_l1a' , sys.argv)
	nounpack= is_there('-nounpack', sys.argv)
	unblock = is_there('-unblock' , sys.argv)

	# Get list of l1a files already processed locally
	# The names in file 'file_list' do NOT include the directory

	if byftp:				# Download *.buf.gz files to ldir and unzip them
		buf_files = l1a_ftp( sys.argv, ldir, redo_list, all_l1a, ddir, unblock )
		buf_files = l1a_gunzip( buf_files, ldir, [] )

	elif bymirror:
		buf_files = l1a_mirror( sys.argv, ldir, redo_list, all_l1a )
		buf_files = l1a_gunzip( buf_files, ldir, [] )

	elif gunzip:			# Unzip all *.buf.gz files in ldir
		buf_files = l1a_gunzip( os.listdir(ldir), ldir, redo_list ) 

	else:					# Find all *.buf files (probably on /mnt/cdrom)
		buf_files = []

		for buf_file in os.listdir(ldir):
			if os.path.splitext(buf_file)[1] == '.buf':
				if len(redo_list) == 0 or redo_list.count(buf_file) != 0:
					buf_files.append( os.path.join( ldir, buf_file) )

	if not all_l1a:
		near_list = l1a_near( [] )
		buf_files = l1a_redo( 0, buf_files, near_list, 1 )

	if len(buf_files) == 0:
		print say+'no *.buf files in '+ldir

	else:
		near_list = l1a_near( buf_files )# Add to list of locally processed l1a files
		if nounpack:
			moveto = start( '-move=', sys.argv )
			if moveto != '':
				if os.path.isdir(moveto):
					for buf_file in buf_files:	
						os.rename( buf_file, os.path.join(moveto,(os.path.split(buf_file))[1]) )
				else:
					print 'non-existent directory:', moveto
		else:
			l1a_idl( sys.argv, os.P_WAIT, buf_files, ddir)

	print say+'========== ALL DONE =========='

	sys.exit()