;+
; NAME:
;	smei_mkbase
; PURPOSE:
;	Used to run smeidb_base and smei_hdr-update
;	for whole days of frames in $SSWDB_SMEI
; CALLING SEQUENCE:
	PRO smei_mkbase, $
		camera		= camera	  , $
		lastdays	= lastdays	  , $
		days_to_cal	= days_to_cal , $
		ignore_block= ignore_block, $
		force		= force		  , $
		test		= test
; INCLUDE:
	@compile_opt.pro
; RESTRICTIONS:
;	Status files are written to $SSWDB_SMEI on the smei server
;	over ssh. If this procedure is run as part of a cronjob
;	then login without password to the smei server should
;	be possible
; PROCEDURE:
; MODIFICATION HISTORY:
;	MAR-2009, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
;	JUN-2010, John Clover (UCSD/CASS)
;		Increasing default threshold for camera 2
;		ped_median_excess from 6 to 8.
;       SEP-2010, John Clover (UCSD/CASS)
;               Changed defaults for camera 1 and camera 2
;               ped_median_excess           8            8
;               dark_cut                   14           14
;               dark_max_ratio           1.05         1.05
;               These likely still need tweaking
;-

InitVar, camera  , [3,2,1]
InitVar, lastdays, 0
InitVar, days_to_cal, 10
InitVar, force, /key
InitVar, test, /key

IF NOT test AND getenv('HOSTNAME') NE 'smeidb.smei.ucsd.edu' THEN BEGIN
	message, /info, 'run only on SMEI data server (smeidb.smei.ucsd.edu)'
	RETURN
ENDIF

IF smei_blocked(ignore_block=ignore_block) THEN	$
	RETURN

; Set directories for status files in $SSWDB_SMEI

lst_dir = smei_filepath(mode='list')
sts_dir = smei_filepath(mode='sts')
cal_dir = smei_filepath(mode='cal')

uday = TimeUnit(/day)
usec = TimeUnit(/sec)

last_tle = TimeGet(/_ydoy, smei_last_tle(), upto=uday)
ndoy_lag = -3

destroyvar, sts_done		; Redundant

FOR icam=0,n_elements(camera)-1 DO BEGIN

	print

	cam_int = camera[icam]
	cam_str = strcompress(cam_int,/rem)

	; The smeidb_base calls are stored in base_daily

	base_daily = filepath(root=lst_dir,'c'+cam_str+'base_daily')

	; Find all c*sts files in $SSWDB_SMEI/cat/sts

	sts_files = FindAllFiles('c'+cam_str+'sts_*.txt',paths=sts_dir, count=count)

	IF count EQ 0 THEN	$	; No files; do next camera
		continue

	CASE 1 OF

	lastdays LT 0: last = count

	lastdays EQ 0: BEGIN		; Look for last cal pattern

		last = -ndoy_lag

		FOR ists=count-1,0,-1 DO BEGIN

			sts_file = sts_files[ists]

			IF txt_read(sts_file,txt,/silent) NE 1 THEN BEGIN
				last = 0
				break
			ENDIF

			cstr = 'cal       :'
			i = where( strpos(txt,cstr) EQ 0 )
			cal_ok = fix( strmid( txt[i], strlen(cstr) ) )

			if cal_ok then	$
				message, /info, hide_env(sts_file)+', cal pattern'

			cstr = 'base      :'
			i = where( strpos(txt,cstr) EQ 0 )
			base_ok = fix( strmid( txt[i], strlen(cstr) ) )

			IF NOT base_ok THEN  $
				continue

			last = count-ists-1
			message, /info, hide_env(sts_file)+', last day with base done'

			break

		ENDFOR  

		message, /info, 'process '+strjoin(strcompress([last,count],/rem),'/')+' sts files'

	END
        
	ELSE: last = lastdays < count
        
	ENDCASE

	destroyvar, ff_update
	ndoy_lag <= -last

	print

	FOR ists=(count-last) > 0,count-1 DO BEGIN	; Loop over all selected days

		sts_file = sts_files[ists]
		sts_hide = hide_env(sts_file)
		sts_time = smei_filename(sts_file)

		; Do not run smei_base past the last tle

		IF TimeGet(/_ydoy,sts_time,upto=uday) GE last_tle THEN BEGIN
			message, /info, sts_hide+', last tle is on '+last_tle+' -> skip'
			break				; Remaining days are also past last_tlm	
		ENDIF

		; Read the status file in $SSWDB_SMEI/cat/sts

		IF txt_read(sts_file,txt,/silent) NE 1 THEN	$
			continue

		cstr = 'frames    :'	; Look for record with 'frames' in it
		nframes = long( strmid( txt[where( strpos(txt,cstr) EQ 0 )], strlen(cstr) ) )

		IF nframes EQ 0 THEN BEGIN
			message, /info, sts_hide+', no frames'
			continue	; No frames; skip
		ENDIF

		cstr = 'base      :'
		base_ok = fix( strmid( txt[where( strpos(txt,cstr) EQ 0 )], strlen(cstr) ) )

		IF base_ok THEN	BEGIN
			message, /info, sts_hide+', base already done'
			continue	; smeidb_base already run; skip
		ENDIF

		; Found a doy where base hase has not been done yet.
		; Must be the start of unbased frames.

		; Go back to previous sts files until one is found
		; with the 'cal' flag set (i.e. a calibration pattern exists)

		jsts = ists
		cal_ok = 0		; Redundant?
		REPEAT BEGIN
			IF txt_read(sts_files[jsts],jtxt,/silent) THEN BEGIN 
				cstr = 'cal       :'
				cal_ok = fix( strmid( jtxt[where( strpos(jtxt,cstr) EQ 0 )], strlen(cstr) ) )
				IF NOT cal_ok THEN jsts -= 1
			ENDIF
		ENDREP UNTIL cal_ok OR jsts LT 0

		tt = smei_filename(sts_files[ists])
		uu = smei_filename(sts_files[jsts])

		; Do not base this day if cal pattern too far away

		IF TimeOp(tt,uu,uday) GT days_to_cal THEN BEGIN
			message, /info, hide_env(sts_files[jsts])+', cal pattern too far away'
			continue
		ENDIF

		message, /info, sts_hide+', '+hide_env(sts_files[jsts])+' is previous cal pattern'

		; Run smeidb_base from beginning to end of day

		tt  = [tt,TimeOp(/add,tt,TimeSet(/diff,1,uday))]
		ctt = TimeGet(/_ydoy,tt,upto=usec)

		; Set up smeidb_base call for science mode

		science_mode = ([2,2,1])[cam_int-1]

		CASE cam_int OF
		1: thresholds = ' -ped_median_excess=8 -dark_cut=14 -dark_max_ratio=1.05'
		2: thresholds = ' -ped_median_excess=8 -dark_cut=14 -dark_max_ratio=1.05'
		3: thresholds = ' -ped_median_excess=8 -dark_cut=66.66 -ndark_min=400'
		ENDCASE

		cmd = 'smeidb_base'			+ $
			' -camera='+cam_str		+ $
			' -mode='  +strcompress(science_mode,/rem)+ $
			' -start=' +ctt[0]		+ $
			' -stop='  +ctt[1]		+ $
			' -destination=SMEIDB?'	+ $
			thresholds				+ $
			(['',' -force'])[force]

		; Run command and store command in 'base_daily'

		out = filepath(root=getenv('TUB'),'mkbase_c'+cam_str+'m'+strcompress(science_mode,/rem)+'_'+ctt[0])
		cmdout = cmd+' > '+out

		print, cmdout

		IF test THEN continue

		spawn, cmdout
		spawn, 'echo "'+cmd+'" | ssh soft@smei "cat - >> '+base_daily+'"'

		; Set up smeidb_base call for engineering mode

		CASE cam_int OF
		1: thresholds = ' '
		2: thresholds = ' '
		3: thresholds = ' -ped_median_excess=20'
		ENDCASE

		cmd = 'smeidb_base'			+ $
			' -camera='+cam_str		+ $
			' -mode=0'				+ $
			' -start=' +ctt[0]		+ $
			' -stop='  +ctt[1]		+ $
			' -destination=SMEIDB?'	+ $
			thresholds				+ $
			(['',' -force'])[force]

		; Run command and store command in 'base_daily'

		out = filepath(root=getenv('TUB'),'mkbase_c'+cam_str+'m0_'+ctt[0])
		cmdout = cmd+' > '+out

		print, cmdout
		spawn, cmdout
		spawn, 'echo "'+cmd+'" | ssh soft@smei "cat - >> '+base_daily+'"'

		; Collect the tt[0] (start of day) times of the days processed
		; This is used later to update the hdr and png data bases on $SSWDB_SMEI,
		; and to update the "base" and "hdr" flags in the status files.
		; This is done for each camera separately.

		boost, ff_update, sts_file

		; Collect all sts_files updated for all three cameras collectively.
		; This is used below after running smei_orbit_stats.

		boost, sts_done, sts_file

	ENDFOR

	; If /test is set then ff_update is not defined

	ndays = n_elements(ff_update)

	IF ndays GT 0 THEN BEGIN

		IF cam_int EQ 3 THEN BEGIN
			tt = [smei_filename(ff_update[0]),TimeOp(/add,smei_filename(ff_update[ndays-1]),TimeSet(/diff,1,uday))]
			smei_frm_smoothdark, tt, silent=1
		ENDIF

		FOR ists=0,ndays-1 DO BEGIN
			sts_file = ff_update[ists]
			sts_time = smei_filename(sts_file)

			smei_hdr_update, sts_time, camera=cam_int, silent=1, plot=2

			; Update the status file in $SSWDB_SMEI/cat/sts: flags "base" and "hdr"

			IF txt_read(sts_file,txt,/silent) THEN BEGIN
				txt[ where( strpos(txt,'base      :') EQ 0 ) ] = 'base      :         1'
				txt[ where( strpos(txt,'hdr       :') EQ 0 ) ] = 'hdr       :         1'
				message, /info, hide_env(sts_file)+', set "base" and "hdr" field'
				txt = strjoin(txt,string(10B))  ; 10B = new line
				spawn, 'echo "'+txt+'" | ssh soft@smei "cat - > '+sts_file+'"'
			ENDIF
		ENDFOR

	ENDIF

ENDFOR

IF test THEN return

CASE IsType(sts_done,/defined) OF
0: message, /info, '"summary" field nowhere updated'
1: BEGIN			; Update orbit stats and status files in $SSWDB_SMEI/cat/sts: flags "summary"

	smei_orbit_stats, /update, ndoy_lag=ndoy_lag, mode=-1, silent=2

	FOR ists=0,n_elements(sts_done)-1 DO BEGIN
		sts_file = sts_done[ists]
		IF txt_read(sts_file,txt,/silent) THEN BEGIN
			txt[ where( strpos(txt,'summary   :') EQ 0 ) ] = 'summary   :         1'
			message, /info, hide_env(sts_file)+', set "summary" field'
			txt = strjoin(txt,string(10B))  ; 10B = new line
			spawn, 'echo "'+txt+'" | ssh soft@smei "cat - > '+sts_file+'"'
		ENDIF
	ENDFOR
END
ENDCASE

RETURN  &  END