#!/bin/bash

#+
# NAME: 
#	sync_ips_realtime
# PURPOSE:
#	Download latest realtime gvalues from STELab from
#		ftp://ftp.stelab.nagoya-u.ac.jp/pub/vlist/rt/gvalue_rt1.dat, Currently this link is not working
#        ftp://ftp.isee.nagoya-u.ac.jp/pub/vlist/rt/gvalue_rt1.dat, is the new link
#	compare to daily file in $NAGOYA/daily and wget the file
#	if it is different
# CALLING SEQUENCE:
#	sync_ips_realtime <email-addresses>
# INPUTS:
#	email-addresses		list of email addressed that will receive
#						a listing of new data
# OUTPUTS:
#	exit code:
#	0	new data; run tomography
#	2	no new data; no need to run tomography
#	1	error downloading from Nagoya or other processing error
# PROCEDURE:
#	New data points are collected in $NAGOYA/daily/nagoya.<year>
#
# Nagoya format:
# SOURCE   YRMNDY    UT DIST HLA  HLO GLA  GLO CARR    V  ER G-VALUE   RA(B1950) DC(B1950) RA(J2000) DC(J2000)
# 1245-19  151231 21.00 0.98  -2   10  -6  285 2172 -999-999 1.17374    12 45 44 -19 42 58  12 48 23 -19 59 19
#
# Morelia format:
# Date     MidObsUT Dur. Site Freq BW Source   Size  RA-J2000 Dec-J2000 Limb  Dist.   Lat.   PA   Elong   Vel.  V-err g-value g-err Method     Vel.  V-err g-value g-err Method
# 20151231 20:59:52  2.7 STEL  327 10 1245-19  -999  12 48 23 -19 59 19   W   208.15  -6.3  249.2  80.9   -999   -999   -999   -999 3-St. CC   -999   -999  1.174  0.212 1-St. PS
#
# MODIFICATION HISTORY:
#	APR-2013, Paul Hick (UCSD/CASS; pphick@ucsd.edu)
#	    Fixed bug: source list not calculated for new yearly file
#-

__CALLER__=`basename $0`
__TIME_TAG__="%m/%d %H:%M"

. get_options.sh

# Set defaults for input arguments

#REMOTE_HOST="https://urldefense.com/v3/__ftp://ftp2021.isee.nagoya-u.ac.jp:20021/pub/vlist/__;!!Mih3wA!QcXWGBBxr5nE0A-ER8GeETs2GomRq3W2MQ2ZiT0P-zlX9_KPcUopVRn2K7SsEQ$"
#REMOTE_HOST=ftp://ftp.isee.nagoya-u.ac.jp
#REMOTE_HOST="https://urldefense.com/v3/__http://stsw1.isee.nagoya-u.ac.jp/vlist__;!!Mih3wA!WSbEOiwvwpEP7nrMSURAk_5MA7DbdhqKcayHfidWymyfkUPOCKWDzOezofpN3HznHw$"
REMOTE_HOST="https://stsw1.isee.nagoya-u.ac.jp"
#REMOTE_HOST=ftp://ftp.stelab.nagoya-u.ac.jp
#REMOTE_DIR=/pub/vlist/rt
#REMOTE_DIR=/vlist/rt
REMOTE_DIR=/vlist
REMOTE_FILE=gvalue_rt1.dat

DATA_DIR=$NAGOYA/daily
SOURCES_DIR=$NAGOYA/sources

DATA_FILE_TEMPLATE=nagoya.YYYY
SOURCES_FILE_TEMPLATE=nagoyasources.YYYY

# Position first char, and number of chars

POS_NAME="1-9"
POS_YEAR="10-11"
POS_RA="71-78"
POS_DEC="79-88"

__MESSAGE__="Version 0.01 --- Paul Hick (UCSD/CAIDA; pphick@ucsd.edu) --- 07-Jan-2016"
__OPTION__=(	\
	[--remote-host]=REMOTE_HOST							\
	[--remote-dir]=REMOTE_DIR							\
	[--remote-file]=REMOTE_FILE							\
	[--data-directory]=DATA_DIR							\
	[--sources-directory]=SOURCES_DIR					\
	[--data-file-template]=DATA_FILE_TEMPLATE			\
	[--sources-file-template]=SOURCES_FILE_TEMPLATE		\
	[--pos-year]=POS_YEAR								\
	[--pos-name]=POS_NAME								\
	[--pos-ra]=POS_RA									\
	[--pos-dec]=POS_DEC									\
)
__DESCRIPTION__=( \
	[--remote-host]="Remote host"						\
	[--remote-dir]="Remote directory"					\
	[--remote-file]="Remote file"						\
	[--data-directory]="Local directory for data file"	\
	[--sources-directory]="Directory for sources file"	\
	[--data-file-template]="Data file template containing YYYY"			\
	[--sources-file-template]="Sources file template containing YYYY"	\
	[--pos-year]="Position year"						\
	[--pos-name]="Position source name"					\
	[--pos-ra]="Position right ascencion"				\
	[--pos-dec]="Position declination"					\
)

get_options $*
unset get_options

if [[ $REMOTE_FILE =~ YYYY ]]; then
	current_year=`date --utc +%Y`
	REMOTE_FILE=${REMOTE_FILE/YYYY/$current_year}
fi

tiny_say "check for new IPS data: $REMOTE_HOST$REMOTE_DIR/$REMOTE_FILE"

# Dowload the data from Nagoya. Abort on error.

get="wget"
if [ $VERBOSE -eq 0 ]; then
	get+=" -q"
fi
if [[ $REMOTE_FILE = 'gvalue_rt1.dat' ]]; then
    REMOTE_DIR=/vlist/rt
fi
$get -N -P $TUB $REMOTE_HOST$REMOTE_DIR/$REMOTE_FILE
if [ $? -ne 0 ]; then
	tiny_yell "$get -N -P $TUB $REMOTE_HOST$REMOTE_DIR/$REMOTE_FILE"
	tiny_die "wget failure, error code $?"
fi

# Cut the year out of the last line (most recent g-value)
# in the downloaded file

new_datafile="$TUB/$REMOTE_FILE"

# Extract the year from the last line in the downloaded file.
# Presumably this is an IPS observation.
# In the Nagoya files the data is stored at position then in the
# format YYMMDD (so the year must have 2000 added to it).
# In the Morelia files the date is at the start of the record
# in format YYYYMMDD.

year=`tail -n 1 $new_datafile | cut -c$POS_YEAR`
if [ -z "$year" -o  "$year" = "YR" -o "$year" = "Date" ]; then
	tiny_die "$new_datafile is empty"
elif [ $year -le 0 ]; then
	tiny_die "got bad year '$year' from new data file"
elif [ $year -lt 100 ]; then
	((year+=2000))
fi

# ${string/substring/replacement}
old_datafile="$DATA_DIR/${DATA_FILE_TEMPLATE/YYYY/$year}"
sources_file="$SOURCES_DIR/${SOURCES_FILE_TEMPLATE/YYYY/$year}"

exit_code=0
if [ -e $old_datafile ]; then	# Yearly file exists

	# Seems like pretty much every g-level gets recalculated, so
	# not much point looking at the difference.

	grep -iv "SOURCE" $new_datafile | diff $old_datafile - > /dev/null
	if [ $? -eq 0 ]; then					# No new data
		tiny_say "no new Nagoya IPS data available"
		exit_code=2
	elif [ $DRYRUN -eq 1 ]; then
		grep -iv "SOURCE" $new_datafile | comm -13 $old_datafile -
	elif [ -n "$__ARGV__" ]; then
		grep -iv "SOURCE" $new_datafile | comm -13 $old_datafile - | mail -s"New STELab realtime g-values" $__ARGV__
	fi
else				# Yearly file does not exist yet: start new year	
	tiny_say "start new '$old_datafile' from downloaded file"
fi

if [ $exit_code -eq 0 ]; then
	if [ $DRYRUN -eq 0 ]; then

		# Overwrite $old_datafile.

		grep -iv "SOURCE" $new_datafile > $old_datafile

		# Determine the sources for the year so far
		# - s: first 9 chars is source name
		# - r: right ascension is 9 chars starting at position 70
		# - d: declination is 9 chars starting at 79
		# - perl is used to move the minus sign on decs between -1 and 0 degrees
		#   (probably not needed anymore: in IDL getnagoyasources.pro can handle this)
		# - sort and uniq are used to retain only one line per source (assumes coordinates do not change!)
		# - output is written to $sources_file

		#cut -c$POS_NAME,$POS_RA,$POS_DEC $old_datafile | sort | uniq | perl -nle's/ -00 00 /  00 00-/; s/ -00 /  00-/; print "$_        327"' > $sources_file.2
		cut -c$POS_NAME,$POS_RA,$POS_DEC $old_datafile | sort | uniq | awk '{print $0"        327"}' > $sources_file.2
		#  | awk '{print $0"        327"}'   perl -nle'print "$_        327"'

		cp -v $sources_file.2 $sources_file

	fi
fi

exit $exit_code