;+
; NAME:
;	vu_select
; PURPOSE:
;	Find selection of tomography files (nv3d* or wson*)
; CALLING SEQUENCE:
	FUNCTION vu_select, files	, $
		files_		= files_	, $
		path		= path		, $
		filter		= filter	, $
		pick		= pick		, $
		marker		= marker	, $
		highmark	= highmark	, $
		one_mark	= one_mark	, $
		check		= check		, $
		allow_nrad_changes=allow_nrad_changes, $
		prefixlen	= prefixlen	, $
		tt_range	= tt_range	, $
		tt_nfile	= tt_nfile	, $
		rounds		= rounds	, $
		nohdr		= nohdr		, $
		count		= count		, $
		rots		= rots		, $
		imark		= imark		, $
		read_all	= read_all	, $
		ff		= ff		, $
		silent		= silent	, $
		nosort		= nosort	, $
		_extra		= _extra
; INPUTS:
;	files				array; type: string
;							list of files. Alternative ways of specifying a set
;							of files involve keywords path, filter and pick.
; OPTIONAL INPUT PARAMETERS:
;	path=path			scalar; type: string; default: $NAGOYA/slow/raw
;							directory in which to search for files
;	filter=filter		scalar; type: string; default: nv3d*.*
;							file filter (incl. wildcard) to search for files
;	/pick				IDL dialog_pickfile function is called to let user
;							select files. If not set then FindAllFiles is used.
;	/nohdr				if set no headers are read. Instead the Carrington times
;							from the file names are returned. (If /nohder is
;							not set these times are available in return keyword
;							'rots'
;	prefixlen=prefixlen	scalar; type: integer; default: 4
;							length of filename prefix. This is 4 for the normal
;							tomography output files (nv3d, nv3f, wson, nson).
;							If set to 3 vu_select can also be used to extract
;							times from e.g. ea_ and e3_ files (make sure
;							to set /nohdr or there will be trouble !!!)
;	/check				passed to vu_header
;							This keyword is not used if /nohdr is set
;	/allow_nrad_changes	passed to vu_header
;							Only used if keyword /check is set
;
;	tt_range=tt_range	array[1] or array[2]; type: time structure or float
;							if array[1] only ea* files prior to tt_range are used
;							if array[2] only ea* files between tt_range[0] and
;								tt_range[1] will be used
;							tt_range can be specified as time structure or as
;								modified Carrington variable.
;	tt_nfile=tt_nfile	scalar; type: integer
;							max. # files selected
;							only the most recent tt_nfile files are used.
;	/read_all			if set than the selected files are read, and the result
;							is returned in keyword ff (href=vu_read= keywords,
;							such as /get_roi and /silent are also permitted)
;	/marker				if set, only headers of files with a marker in the file
;							name (made by the time-dependent tomography program)
;							are returned. Keyword needs to be set only to
;							separate marker files from non-marker files. If the
;							file selection consist of either marker files only,
;							or non-marker files only, then this keyword is
;							ignored (i.e. all files are read regardles of the
;							/marker setting).
;	/highmark			if set, then only files with the highest marker value,
;							i.e. files from the most recent time-dependent
;							tomography run are selected.
;
;	rounds=rounds		scalar; type: integer; default: none
;							Only affects the return argument 'rots'.
;							The Carrington variables in the headers typically
;							refer to UT times at integer hours. This is the case
;							for the raw time-dependent files (each 6 hours) and
;							for the final files for both corotating and
;							time-dependent tomography (every 2 and every 6
;							hours, respectively). The imprecision in the
;							Carrington times causes deviations of several
;							minutes. This is corrected by converting to UT time,
;							rounding to the specified time (rounds=300 should
;							do), and then convert back to Carrington variable.
;							Procedure forecast_map uses this to try to get an
;							exact match between current system time (rounded to
;							an integer hour) and a matrix time to avoid
;							unnecessary interpolation in the time dimension.
; OUTPUTS:
;	Result				array[count]; type: structure
;							header information for all files
;
;						Only if /nohdr is set:
;						array[count]; type: double
;							Carrington times extracted from file names
;
;							Entries are sorted chronologically. Entries with the
;							same times (from time-dependent tomography) are also
;							sorted on marker value.
; OPTIONAL OUTPUTS:
;	count=count			scalar; type: integer
;							number of valid tomography files found
;	ff=ff				array[n,l,m,2,k]; type: float
;							3D heliospheric arrays (only if /read_all is set)
;
;	rots=rots			array; type: time structure
;							times extracted from file names converted
;							from Carrington variable to UT time. 
;	imark=imark			array; type: integer
;							Marker values extracted from file name
;							(only if /marker is set)
; INCLUDE:
	@compile_opt.pro							; On error, return to caller
; EXTERNAL:
;	vu_header__define
; CALLS:
;	InitVar, SetFileSpec, GetFileSpec, FindAllFiles, SuperArray, IsTime
;	TimeOp, TimeGet, Carrington, BadValue, vu_header, vu_read, boost
;	IsBadTime, hide_env, IsType, vu_get, vu_set, vu_prefix, vu_check
; PROCEDURE:
;	Keywords path, filter and pick are used only if argument 'files' is not
;	specified.
;
;	The tomographic data are stored in files with names having the form
;	abc1234.6789_00001, where 1234.678 indicates a modified Carrington variable,
;	and 00001 is a counter for the number of files in rotation 1234.
;
;	Only files with names of the specified form are put in the returned list of
;	headers. If one of the keywords tt_range and/or tt_nfile are specified then
;	the list of files is further reduced
;
;	If tt_range is specified then only files prior to tt_range[0] are used
;	(if tt_range has only one elements) or all files between tt_range[0] and
;	tt_range[1] are used (if tt_range has two elements).
;
;	After tt_range has been applied a group of files remain. If tt_nfile is
;	specified then only the most recent tt_nfile files are used.
;	If tt_nfile is not specified then all files are returned.
; MODIFICATION HISTORY:
;	MAR-2001, Paul Hick (UCSD/CASS)
;	JUL-2002, Paul Hick (UCSD/CASS)
;		Added keyword imark to return marker values if keyword /marker is set.
;		Some modifications for increased efficiency. Simplified explicit
;		do-loop required by limitation of IDL 'string' function to 1024
;		-element string arrays. Modified call to flt_string.
;	SEP-2002, Paul Hick (UCSD/CASS)
;		Added keyword 'prefixlen' to be able to extract times from the
;		file names for ea_ and e3_ files.
;
;		The time-dependent tomography produces multiple reconstructions for the
;		same times, differentiated by different marker values. However, the
;		Fortran tomography program does not always use completely identical
;		times due to inherent imprecision in the calculation of Carrington
;		times. Differences should be no bigger than 0.0001 Carrington rotation.
;		These differences are eliminated now by using the mean time for all
;		entries with different marker values (but referring to the same time).
;
;		Replaced call to IDL findfile function to a call to FindAllFiles
;		(including _extra keyword). This allows recursive searches.
;	OCT-2002, Paul Hick (UCSD/CASS)
;		Added keyword rounds
;	OCT-2011, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
;		Added keyword /highmark
;-

InitVar, silent, 0
;InitVar, marker, /key  ; vu_check differentiates between marker=0 and marker undefined!

IF IsType(files_, /defined) THEN files = files_

nofiles = IsType(files, /undefined)

InitVar, nohdr	 , /key
InitVar, read_all, /key
InitVar, prefixlen, 4	; Length of file name prefix (nv3d, nv3f)
InitVar, highmark, /key
InitVar, nosort	 , /key

len_marker = 5		; Length of marker in file type
len_fract  = 4		; Length of Carrington fraction in file type

CASE nohdr OF
0: empty_hdr = {vu_header}
1: empty_hdr = IsBadTime(/get)
ENDCASE

CASE nofiles OF
0: BEGIN

	count = n_elements(files)
	str_prefix = hide_env(files[0])+'+'

END
1: BEGIN

	InitVar, pick, /key
	InitVar, path, filepath(root=getenv('NAGOYA'),subdir='slow','raw')
	InitVar, filter, vu_prefix(/silent,_extra=_extra)+'*.*'

	CASE pick OF
	0: files = FindAllFiles(filter, path=path, _extra=_extra)
	1: files = dialog_pickfile(path=path, filter=filter, /multiple, /read)
	ENDCASE

	str_prefix = hide_env(filepath(root=path, filter))

	IF files[0] EQ '' THEN BEGIN					; No files found
		str_message = ', no files '+(['found','selected'])[pick]
		goto, error_exit
	ENDIF

	count = n_elements(files)

END
ENDCASE


; Check the file names for inconsistencies.

IF NOT vu_check(files, rots, imark, $
	count		= count		, $
	marker		= marker	, $
	prefixlen	= prefixlen	, $
	str_message	= str_message	, $
	silent		= silent) THEN goto, error_exit

IF marker THEN BEGIN

	IF highmark THEN BEGIN		; Keep files with highest marker

		tmp = where( imark EQ max(imark), count )
		say, threshold=0, silent=silent, $
			'selected files for highest marker'+strcompress(max(imark))

		files = files[tmp]
		rots  = rots [tmp]
		imark = imark[tmp]

	ENDIF ELSE IF IsType(one_mark,/defined) THEN BEGIN

		tmp = where( imark EQ one_mark[0], count )
		IF count EQ 0 THEN BEGIN
			say, threshold=2, silent=silent, 'no files found for marker'+strcompress(one_mark[0])
			goto, error_exit
		ENDIF
		say, threshold=0, silent=silent, $
			'selected files for marker'+strcompress(one_mark[0])

		files = files[tmp]
		rots  = rots [tmp]
		imark = imark[tmp]

	ENDIF

ENDIF

say, threshold=0, silent=silent, $
	[str_prefix, str_message+'; '+strcompress(count,/rem)+' left']

IF NOT nosort THEN BEGIN
	tmp = sort(files)					; Puts files in chronological order
	files = files[tmp]
	rots  = rots [tmp]

	IF marker THEN BEGIN
		imark = imark[tmp]

		; Ugly, but necessary, I think:
		; For time-dependent tomography, files for a given time exist with different marker values.
		; The Carrington variable associated with the given time may not be exactly the same everywhere
		; however, so for each time an additional sort operation is needed to put the different
		; marker values in proper order.

		xc_error = 0.0002d0				; ~ 8 minutes

		; Collect a list of all different times in 'unique'

		i = -1
		REPEAT BEGIN
			IF i EQ -1 THEN rest = indgen(count) ELSE rest = rest[tmp]
			boost, unique, rest[0]			; Add unique time at index rest[0]
			i = i+1
			tmp = where(rots[rest] LT rots[unique[i]]-xc_error OR rots[rest] GT rots[unique[i]]+xc_error)
		ENDREP UNTIL tmp[0] EQ -1

		; Sort on marker values

		FOR i=0,n_elements(unique)-1 DO BEGIN
			same = where(rots[unique[i]]-xc_error LE rots AND rots LE rots[unique[i]]+xc_error)
			IF same[0] NE -1 THEN BEGIN		; Same times (should be a contiguous set of indices)
				tmp = imark[same]			; Should be all different marker values
				tmp = same[sort(tmp)]		; Put marker values in ascending order

				; The elements in rots[tmp] should be almost identical (differing by no more
				; than 0.0002d0. We force them to be identical by using the mean.

				files[same] = files[tmp]
				rots [same] = mean(rots[tmp])
				imark[same] = imark[tmp]
			ENDIF
		ENDFOR

	ENDIF
ENDIF

IF nohdr THEN BEGIN
	IF count NE n_elements(files) THEN BEGIN
		whatis, count, n_elements(files)
		say, threshold=10, silent=silent, 'should not have happened'
	ENDIF
	count = n_elements(files)

	; FUDGE A BIT (see discussion of keyword rounds):

	rots = Carrington(rots)
	IF IsType(rounds,/defined) THEN rots = TimeGet(rots, TimeUnit(/sec), roundt=rounds)

	IF NOT nofiles AND count EQ 1 THEN BEGIN
		files = files[0]
		rots  = rots [0]
		IF marker THEN imark = imark[0]
	ENDIF
	say, threshold=0, silent=silent, $
		str_prefix+', got '+strcompress(count,/rem)+' names'

	RETURN, rots					; Return all times from file names without reading header
ENDIF

; Do NOT add the /sort_time keyword in vu_header. This screws up the ordering when /marker is set.

hdrs = vu_header(files, check=check, allow_nrad_changes=allow_nrad_changes, count=count)

IF count EQ 0 THEN goto, error_exit

; The 'rots' array values for each time-dependent run were replaced by the average
; for that run. Make sure that hdrs array contain the same time.

IF marker THEN hdrs = vu_set(hdrs, uttime=rots)

ntt = n_elements(tt_range)
IF ntt NE 0 THEN BEGIN

	times = vu_get(hdrs, /uttime)

	CASE IsTime(tt_range) OF
	0: trange = Carrington(tt_range)
	1: trange = tt_range
	ENDCASE

	CASE ntt OF
	1: BEGIN
		tmp  = TimeOp(/subtract, times, trange[0], sign=sign)
		sign = where(sign LT 0, count)
		IF count EQ 0 THEN BEGIN
			CASE nofiles OF
			0: str_message = ', no file(s) before '+TimeGet(trange[0], /ymd)
			1: str_message = ', no '+filter+' file(s) found in '+path+' before '+TimeGet(trange[0], /ymd)
			ENDCASE
			goto, error_exit
		ENDIF
	END
	2: BEGIN
		tmp  = TimeOp(/subtract, times, trange[0], sign=sign1)
		tmp  = TimeOp(/subtract, times, trange[1], sign=sign2)
		sign = where(sign1 GT 0 AND sign2 LT 0, count)
		IF count EQ 0 THEN BEGIN
			CASE nofiles OF
			0: str_message = ', no file(s) found in between '+	$
				TimeGet(trange[0], /ymd)+' and '+TimeGet(trange[1], /ymd)
			1: str_message = ', no '+filter+' file(s) found in '+path+' between '+	$
				TimeGet(trange[0], /ymd)+' and '+TimeGet(trange[1], /ymd)
			ENDCASE
			goto, error_exit
		ENDIF
	END
	ENDCASE

	hdrs  = hdrs [sign]
	files = files[sign]
	rots  = rots [sign]
	IF marker THEN imark = imark[sign]
ENDIF

IF IsType(tt_nfile,/defined) THEN BEGIN			; IF # FILES SPECIFIED
	IF count GT tt_nfile THEN BEGIN
		hdrs  = hdrs [count-tt_nfile:count-1]
		files = files[count-tt_nfile:count-1]	; Keep only last tt_nfile files
		rots  = rots [count-tt_nfile:count-1]
		imark = imark[count-tt_nfile:count-1]
		count = tt_nfile
	ENDIF
ENDIF

CASE read_all OF
0: str_message = ', got '+strcompress(count,/rem)+' headers'
1: ff = vu_read(hdrs, silent=silent, _extra=_extra, count=count)
ENDCASE

IF NOT nofiles AND count EQ 1 THEN BEGIN
	files = files[0]
	rots  = rots [0]
	IF marker THEN imark = imark[0]
ENDIF

IF IsType(str_message, /defined) THEN	$
	say, threshold=0, silent=silent, str_prefix+str_message


rots = vu_get(hdrs, /uttime)

IF IsType(rounds, /defined) THEN BEGIN

	; FUDGE A BIT (see discussion of keyword rounds):

	rots = TimeGet(rots, roundt=rounds, TimeUnit(/sec))
	hdrs = vu_set(hdrs, uttime=rots)
ENDIF

tmp = TimeOp(/subtract,rots,vu_get(hdrs,/uttime),TimeUnit(/day)) NE 0
IF total(tmp) NE 0 THEN BEGIN
	FOR i=0,count-1 DO	$
		IF tmp[0] THEN $
			print, hdrs[i].file, TimeGet(vu_get(hdrs[i],/uttime),/ymd),	$
				TimeGet(rots[i], /ymd), TimeOp(/subtract,vu_get(hdrs[i],/uttime),rots[i],TimeUnit(/min)),' min'
	message, 'should not have happened'
ENDIF

RETURN, hdrs

error_exit:

count = 0

IF NOT nofiles AND count EQ 1 THEN BEGIN
	files = files[0]
	IF IsType(rots ,/defined) THEN rots  = rots [0]
	IF IsType(imark,/defined) THEN imark = imark[0]		; ????
ENDIF

CASE IsType(str_message, /defined) OF
0: str_prefix = str_prefix+' no additional error info'
1: str_prefix = str_prefix+str_message
ENDCASE

say, threshold=2, silent=silent, str_prefix

RETURN, empty_hdr  &  END