C+ C NAME: C dailyips C PURPOSE: C Sort daily IPS files into yearly files C CATEGORY: C IPS C CALLING SEQUENCE: program dailyips C C dailyips -day=day_dir -year=year_dir (full mode) C dailyips file-list -year=year_dir (update mode) C INPUTS: C In 'full mode' daily IPS files are read from $DAT/nagoya/ipsrt C In 'update mode' daily files whose names are read from 'file-list' are read C OUTPUTS: C New yearly files ('full mode') or updated yearly files ('update mode') C C NOTE: an exit code of 1 indicates that new yearly files have been created C or that records have been added to the yearly files. C This is currently used by the Linux script 'sync_daily_ips' to decide C whether or not the tomography program should be rerun. C CALLS: C cInt2Str, ForeignArg, itrim, LocFirst, LocFirstLen, bOpenFile, iArrI4ValuePresent C iCheckDirectory, iGetFileSpec, iSetFileSpec, iFourDigitYear, iFilePath, IndexChar C iSearch, Int2Str, Str2Str, Say, DailyIPS_UCSD C INCLUDE: include 'dirspec.h' include 'openfile.h' include 'filparts.h' C RESTRICTIONS: C PROCEDURE: C > Daily files files are located in $DAT/nagoya/ipsrt unless another directory C is specified on the command line using argument -day_dir C > Yearly files are assumed to be located in the same directory as the daily files C unless another directory is specified on the command line using argument -year_dir C C > The yearly files will be sorted into chronological order. C > FULL MODE: C If no foreign input is provided then the program will look in the directory C $DAT/nagoya/ipsrt for daily IPS files, and process all into yearly IPS files. C $DAT should be defined as a logical (VMS) or should be present in LOGFIL.TXT. C The yearly files are put in the same directory as the daily files. C Note that on Unix, Linux and NT this could mean that old version are C overwritten. C > UPDATE MODE: C If foreign input (command line argument) is provided it is assumed to be C a file containing a list of names of daily IPS files to be integrated into C the existing data base of yearly files. C The file names for the daily files should be fully-qualified file names C (including directory, which will be the same for all the daily files). C If no directory is specified then the program will look in the current C directory for daily and yearly files, so in that case the program MUST be C run from the data directory containing the IPS files. C MODIFICATION HISTORY: C SEP-1999, Paul Hick (UCSD/CASS) C SEP-2002, Paul Hick (UCSD/CASS) C Adapted to handle more than one scintillation index C OCT-2002, Paul Hick (UCSD/CASS) C Added option to specify locations of daily and yearly files C as command line arguments. (The sync_daily_ips script now uses C the Perl script 'mirror' to keep track of the VLIST_UCSD files, C so the yearly files had to be moved to another directory). C C Fixed minor bug in detecting duplicate records. Sometimes duplicate C records were put in the yearly files. C NOV-2002, Paul Hick (UCSD/CASS; pphick@ucsd.edu) C Rearranged format of output yearly files. Now each record ends with C pairs of g-level and scint index values in format F8.5,E10.3. C The first of these are the 'best values' (the g-level of this pair C is what is currently used in the tomography programs. Then follow C pairs for individual IPS stations (currently Kiso and Fuji). C The VLIST_UCSD files contain only the scint indices for the two C IPS stations. This program insert a leading pair of 'best values' C (zero g-level with scint index from the first IPS station) and C inserts columns for the g-level for each station (again zero g-level). C Fujiki's nagoya_glevel.pro calculates the g-level values. C- !------- ! nL_VLIST length of VLIST_UCSD files excl scint indices ! nL_GLEVEL number of chars in g-level values (F8.5) ! nL_SCINDX number of chars in scint index (E10.3) ! The VLIST_UCSD files have one or more scint indices at the end of the record in E10.3. ! The max # of indices is set by nMaxSites. parameter (nL_VLIST = 58) parameter (nL_GLEVEL= 8) parameter (nL_SCINDX= 10) ! # chars in scint index parameter (nL_BLOCK = nL_GLEVEL+nL_SCINDX) parameter (nMaxSites= 2) ! Max # of scint indices in VLIST_UCSD files parameter (nReclIn = nL_VLIST+ nMaxSites *nL_SCINDX) ! Record length of input file (bytes) parameter (nReclOut = nL_VLIST+(1+nMaxSites)*nL_BLOCK ) ! Record length of output file (bytes) parameter (nRec = 10000) character cRec(nRec)*(nReclOut) character cStrOut *(nReclOut) character cTmp(nRec)*(FIL__LENGTH) character cNew(100 )*(FIL__LENGTH) integer indx(nRec) character cDaily * 10 /'VLIST_UCSD'/ ! File name for daily files character cYearly* 7 /'nagoya.'/ ! File name for yearly files character cSub(2)* 6 /'nagoya','ipsrt'/ ! Directory off $DAT character cPrefix /'A'/ character cSearch *(FIL__LENGTH) character cFile *(FIL__LENGTH) character cDayDir *(FIL__LENGTH) character cYearDir*(FIL__LENGTH) character cStr *(FIL__LENGTH) character cHideLogical*(FIL__LENGTH) character cSay*8 /'DailyIPS'/ integer nYears(10) integer iReclOut(2) /1000000,0/ character cInt2Str*14 integer Str2Str logical bOpenFile parameter (nVar=2) character cVar(nVar)*(FIL__LENGTH) character cArg*512 iVar = nVar cVar(1) = 'quiet' call ForeignArg(' ',iVar,cVar,cArg) ! Check for foreign input !------- ! Check the command line for the location of the yearly files. If not ! specified they should be located in the same directory as the daily files. cYearDir = ' ' cStr = cSwitch//'year=' I = itrim(cStr) L = LocFirst(cStr(:I),cArg) if (L .ne. 0) then L = L+I I = L-1+LocFirstLen(cSwitch,cArg(L:)) cYearDir = cArg(L:I-1) if (iCheckDirectory(cYearDir) .eq. 0) call Say(cSay,'E',cYearDir,'does not exist') endif cDayDir = ' ' cStr = cSwitch//'day=' I = itrim(cStr) L = LocFirst(cStr(:I),cArg) if (L .ne. 0) then L = L+I I = L-1+LocFirstLen(cSwitch,cArg(L:)) cDayDir = cArg(L:I-1) if (iCheckDirectory(cDayDir) .eq. 0) call Say(cSay,'E',cDayDir,'does not exist') endif if (iVar .ne. 0) then ! cVar(1) is assumed to be a file name cSearch = cVar(1) !------- ! Open the file specified on the command line; if the open fails bOpenFile terminates ! the program (so the continue statement never gets executed). The exit code of bOpenFile ! should be 'E' (=2). The sync_daily_ips script uses this to terminate. I = 0 ! Record length shouldn't matter iAct = OPN__TEXT+OPN__READONLY+OPN__TRYINPUT+OPN__ONEPASS+OPN__STOP+OPN__NOMESSAGE if (bOpenFile(iAct,iU,cSearch,I)) continue call Say(cSay,'I','Process','daily IPS files in '//cSearch) ! The file is assumed to be a list of daily files. Read the names into cNew. iNew = 0 read (iU,'(A)',iostat=iErr) cStr ! Read file names of daily IPS files do while (iErr .eq. 0) iNew = iNew+1 cNew(iNew) = cStr read (iU,'(A)',iostat=iErr) cStr end do iRec = 0 do I=1,iNew ! Read content of new daily files call DailyIPS_UCSD(1,cNew(I),iRec,nRec,cRec,cTmp,1,iTb,iTe,nL_SCINDX,nReclIn,nReclOut,iReclOut,cStrOut) end do !------- ! If the new files are empty (iRec = 0), then exit with error code 'I' (=3). ! The sync_daily_ips script uses this to terminate. if (iRec .eq. 0) call Say(cSay,'I','Stop','no update required') iRecDaily = iRec ! # recs read from daily files I = 0 I = Int2Str(iRec,cStr(I+1:)) I = Str2Str(' new records to IPS yearly files',cStr(I+1:)) call Say(cSay,'I','Add',cStr) nY = 1 ! Year of first record nYears(nY) = iFourDigitYear(cRec(1)(iTb:)) do I=2,iRec ! Collect all years occurring iYr = iFourDigitYear(cRec(I)(iTb:)) ! .. in the daily files (usually only one year) do J=1,nY if (iArrI4ValuePresent(nY,nYears,iYr) .eq. 0) then nY = nY+1 if (nY .gt. 10) call Say(cSay,'E','Error','oops') nYears(nY) = iYr end if end do end do !------- ! Pick up directory where daily files are located. ! Note that this overrides command line input. I = iSetFileSpec(cNew(1)) ! Determine directory of 1st daily file I = iGetFileSpec(0,FIL__DIRECTORY,cDayDir) !------- ! If the directory for the yearly files was not specified on the command ! line, set it to the directory where the daily files are located. if (itrim(cYearDir) .eq. 0) cYearDir = cDayDir !------- ! The content of the yearly files is appended to the cRec array do I=1,nY ! Read yearly data files that need updating J = iFilePath(cYearDir,0,' ',cYearly//cInt2Str(nYears(nY)),cFile) call Say(cSay,'I','Update','yearly file '//cHideLogical(cFile)) call DailyIPS_UCSD(1,cFile,iRec,nRec,cRec,cTmp,0,iTb,iTe,nL_BLOCK,nReclIn,nReclOut,iReclOut,cStrOut) end do !------ ! Add the prefix 'A' to the names of the yearly files to make sure ! that they are considered 'earlier' than a daily 'VLIST_UCSD' file, ! i.e. if records with identical source and time are found than the ! record from the VLIST_UCSD file is retained and the record from ! the yearly file is ditched. iPrefix = 1 if (itrim(cPrefix) .ne. 0) iPrefix = 2 do I=iRecDaily+1,iRec J = iSetFileSpec(cTmp(I)) J = iGetFileSpec(0,FIL__DIRECTORY,cTmp(I)) J = J+Str2Str(cPrefix,cTmp(I)(J+1:)) ! Insert prefix J = J+iGetFileSpec(FIL__NAME,0,cTmp(I)(J+1:)) end do I = 0 I = Int2Str(iRec-iRecDaily,cStr(I+1:)) I = Str2Str(' records from IPS yearly files',cStr(I+1:)) call Say(cSay,'I','Read',cStr) else call Say(cSay,'I','Process','all daily IPS files') !------- ! Use directory $DAT/nagoya/ipsrt if no command line arg was given. if (itrim(cDayDir) .eq. 0) I = iFilePath(cEnvi//'DAT',2,cSub,' ',cDayDir) !------- ! If the directory for the yearly files was not specified on the command ! line, set it to the directory where the daily files are located. if (itrim(cYearDir) .eq. 0) cYearDir = cDayDir !------- ! Set up wildcard to search for daily files in cDayDir I = iFilePath(cDayDir,0,' ',cDaily//'_*',cSearch) !------- ! Read in all VLIST_UCSD_* files from cDayDir ! The first record in each file is skipped (header record) iStat = 1 iRec = 0 do while (iSearch(iStat,cSearch,cFile) .eq. 1) ! Loop over all daily files iStat = 0 call DailyIPS_UCSD(0,cFile,iRec,nRec,cRec,cTmp,1,iTb,iTe,nL_SCINDX,nReclIn,nReclOut,iReclOut,cStrOut) end do I = 0 I = Int2Str(iRec,cStr(I+1:)) I = Str2Str(' records from IPS daily files',cStr(I+1:)) call Say(cSay,'I','Read',cStr) end if !------- ! If no data are available at this point (iRec = 0), then exit with error code ! 'I' (=3). The sync_daily_ips script uses this to terminate. if (iRec .eq. 0) call Say(cSay,'I','Stop','no records available') !------- ! Each record starts with an IPS source name, followed by a column for the ! time of observation. The sorting collects all observations for each ! source, and sorts observations for each source into chronological order. call IndexChar(1,iRec,1,nRec,cRec,indx) !------- ! Loop through all records searching for duplicate entries (same source, ! same time of observation, but from a different VLIST_UCSD file). These ! should be neighbouring entries in the sorted list. ! If a duplicate entry is found, check the files of origin and discard ! the record from the older file. I = 1 J = 1 K = 50 do while (J .lt. iRec) J = J+1 nI = indx(I) nJ = indx(J) if (cRec(nJ)(:iTe) .eq. cRec(nI)(:iTe)) then ! Duplicate source and time found if (cTmp(nJ) .eq. cTmp(nI)) then ! Compare files of origin cFile = cTmp(nJ) call Say(cSay,'E','Same','#.#.#duplicate data in#'//cFile(iPrefix:)//'#'//cRec(nI)(:K)//' ...') else if (cTmp(nJ) .lt. cTmp(nI)) then cFile = cTmp(nJ) call Say(cSay,'W','Ditch','#.#.#duplicate data in#'// & cFile(iPrefix:)//'#'//cRec(nJ)(:K)//' ...'// & '#more recent version in#'//cTmp(nI)(iPrefix:)//'#'//cRec(nI)(:K)//' ...') cRec(nJ) = ' ' else cFile = cTmp(nI) call Say(cSay,'W','Ditch','#.#.#duplicate data in#'// & cFile(iPrefix:)//'#'//cRec(nI)(:K)//' ...'// & '#more recent version in#'//cTmp(nJ)(iPrefix:)//'#'//cRec(nJ)(:K)//' ...') cRec(nI) = ' ' I = J end if cFile = cFile(iPrefix:) else I = J end if end do !------- ! Leftadjust the cRec array, so that all remaining records are at the ! start of the array J = 0 do I=1,iRec if (cRec(I) .ne. ' ') then J = J+1 if (J .ne. I) cRec(J) = cRec(I) end if end do I = 0 I = I+Str2Str('#.#.#' ,cStr(I+1:))+1 I = I+Int2Str(iRec-J ,cStr(I+1:)) I = I+Str2Str('/' ,cStr(I+1:)) I = I+Int2Str(iRec ,cStr(I+1:)) I = I+Str2Str(' records ;' ,cStr(I+1:))+1 I = I+Int2Str(J ,cStr(I+1:)) I = I+Str2Str(' records remaining#.#.' ,cStr(I+1:)) call Say(cSay,'W','Ditched',cStr) iRec = J ! Updated # records !------- ! Min and max record lengths are stored in iReclOut(1) and iReclOut(2). ! Check min and max record lengths if they are not the same they should ! differ by a multiple of nL_BLOCK. Pad the shorter records by adding ! g-level/scint index pairs at the end. if (iReclOut(1) .ne. iReclOut(2)) then do I=1,iRec iRecl = itrim(cRec(I)) nPresent = (iRecl -nL_VLIST)/nL_BLOCK ! # scint indices in record nRequired = (iReclOut(2)-nL_VLIST)/nL_BLOCK ! # scint indices needed if (nRequired .gt. nPresent) then !------- ! If there is only one scint index present, append a copy ! Can't happen for daily files. ! Maybe could happen when adding to an old yearly file?. if (nPresent .eq. 1) then cRec(I) = cRec(I)(:iRecl)//cRec(I)(iRecl-nL_BLOCK+1:iRecl) nPresent = nPresent+1 end if !------ ! If there still are more scint indices needed append zeros. do J=1,nRequired-nPresent cRec(I) = cRec(I)(:itrim(cRec(I)))//' 0.00000 0.000E+00' end do end if end do end if !------- ! Records for each year are written to separate output files. This ! requires the remaining records to be sorted into chronological order ! (disregarding the source name). Extract the time substring (e.g. ! '990823 1.85') from each record, and sort them. do I=1,iRec cTmp(I) = cRec(I)(iTb:iTe) end do call IndexChar(1,iRec,1,nRec,cTmp,indx) !------- ! Write the records in chronological order, starting a new file when ! the year indicator (two first characters of substring changes). iAct = OPN__TEXT+OPN__TRYINPUT+OPN__ONEPASS+OPN__UNKNOWN+OPN__RECLBYTE iU = FIL__NOUNIT iFile = iFilePath(cYearDir,0,' ',cYearly,cFile) nYr = -1 do I=1,iRec nI = indx(I) iYr = iFourDigitYear(cTmp(nI)) if (iYr .ne. nYr) then if (iU .ne. FIL__NOUNIT) then iU = iFreeLun(iU) K = 0 K = K+Int2Str(J,cStr(K+1:)) K = K+Str2Str(' records written',cStr(K+1:)) call Say(cSay,'I',cHideLogical(cSearch),cStr) end if nYr = iYr cSearch = cFile(:iFile)//cInt2Str(nYr) if (bOpenFile(iAct,iU,cSearch,iReclOut(2))) J = 0 end if if (iU .ne. FIL__NOUNIT) then write (iU,'(A)') cRec(nI)(:itrim(cRec(nI))) J = J+1 end if end do K = 0 K = K+Int2Str(J,cStr(K+1:)) K = K+Str2Str(' records written',cStr(K+1:)) call Say(cSay,'I',cHideLogical(cSearch),cStr) !------- ! 'Stop' is needed to force the proper exit code call Say(cSay,'S','Stop','updated yearly IPS data files') end C+ C NAME: C DailyIPS_UCSD C PURPOSE: C Reads content of IPS data file, and appends it to the input arrays C CALLING SEQUENCE: subroutine DailyIPS_UCSD(iParse,cName,iRec,nRec,cRec,cFil,iTime,iTb,iTe,nBlock,nReclIn,nReclOut,iReclOut,cStrOut) C INPUTS: C iParse integer 0: do not parse the input file name C 1: parse the input file name C cName character*(*) Name of IPS data file to be read C iRec integer # records in input/output arrays already filled with data C (the current file is appended starting at iRec+1) C nRec integer Max. # records stored in input/output arrays. C iTime integer 0: file does not contain header (yearly file) C 1: file contains header (daily VLIST_UCSD file) C nBlock integer daily file: # chars used for scint index (10) C yearly file: # chars used for g-level/scint index pair (10+8=18) C nReclIn integer Max. allowed record length for input records C nReclOut integer Max. allowed record length for output records C cStrOut character*(*) Scratch string C OUTPUTS: C iRec integer # records containing valid data C cRec(nRec) character*(*) Records read from file C cFil(nRec) character*(*) File name from which each record was read C iTb integer start of time substring in data records C iTe integer end of time substring in data records C iReclOut(2) integer Max. length of records read from file C INCLUDE: include 'openfile.h' C CALLS: C bOpenFile, LocFirst, LocLast iFreeLun, Str2Flt, Str2Flt_Mask, Str2Flt_Exp C Str2Flt_Format, itrim, Say, iFreeLun C MODIFICATION HISTORY: C SEP-1999, Paul Hick (UCSD/CASS) C SEP-2002, Paul Hick (UCSD/CASS; pphick@ucsd.edu) C Modified to handle multiple scint indices in the incoming VLIST_UCSD files C The header in the new format is slightly different: it has SCINDX-A instead C of SC-INDX starting in a different column. Instead of using the position of C this string to determine the position of the scint index, now the first data C record is used directly assuming that the scint indices are written in E10.3 C (i.e. 9 chars preceeded by a space). C In addition to inserting a column for g-levels (set to zero), a copy of the C first scint indices is inserted, and now is the first scint index in each C record. This extra column can later be used to derive a 'best' scint index C from the indices from multiple sites. C- integer iParse character cName*(*) integer iRec integer nRec character cRec(nRec)*(*) character cFil(nRec)*(*) integer iTime integer iTb integer iTe integer nBlock integer nReclIn integer nReclOut integer iReclOut(2) character cStrOut*(*) character cWarn(0:1) /'W','E'/ character cSay*12 /'DailyIPS_UCSD'/ logical bOpenFile character cFmt*256 parameter (nVec=16) real rVec(nVec) iAct = OPN__TEXT+OPN__READONLY+OPN__TRYINPUT+OPN__ONEPASS if (iParse .eq. 0) iAct = iAct+OPN__NOPARSE !------- ! A header record is only present on the daily files (same on all files). ! Not on the yearly files. iRecl = 0 ! Record length should not matter if (bOpenFile(iAct,iU,cName,iRecl)) then read (iU,'(A)',iostat=iErr) cStrOut ! Read first record of file if (iErr .eq. 0) then iTb = LocFirst('YRMNDY', cStrOut) ! Position of time substring iTe = LocFirst('UT' , cStrOut)+1 ! Position of last char of UT ! Read first data record if (iErr .eq. 0 .and. iTb .ne. 0 .and. iTe .ne. 1) read (iU,'(A)',iostat=iErr) cStrOut !------- ! The time substring is detected using three assumptions: ! The IPS source starts the record with 10 chars (actually 9 chars + 1 space) ! The year, month, date are specified as a single integer (I6) ! The time of day is specified as a floating point number (F6.2) iTb = 10 ! 6-digit YMD starts at char 10 call Str2Flt_Mask('000000000') ! Mask IPS source name of 9 chars call Str2Flt_Exp(.TRUE.) I = nVec call Str2Flt(cStrOut,I,rVec) call Str2Flt_Format(cFmt) I = nVec call Str2Flt(cFmt,I,rVec) iTe = iTb-1+nint(rVec(2))+int(rVec(3)) ! Should be 21 !------- ! The IPS data (scint indices or g-level/scint index pairs) are stored at the end of ! the record in groups of nBlock chars. We assume that all numbers are separated by ! a space. Determine # scint indices by counting spaces nBlock chars apart starting ! at the end of the record. if (iErr .eq. 0) then nSite = -1 I = itrim(cStrOut)+1 iSI = I+nBlock do while (iSI-I .eq. nBlock) ! Always true at first pass nSite = nSite+1 iSI = I ! Space at the start of the 1st scint index I = LocLast(' ',cStrOut(:iSI-1)) !------- ! For daily files (iTime = 1) insert a zero g-level before the scint index ! For yearly files skip to the preceeding space if the space just located ! is less than nBlock chars before the last one (this should skip past the ! the space between g-level and scint index. if (iTime .eq. 0 .and. iSI-I .lt. nBlock) I = LocLast(' ',cStrOut(:I-1)) end do if (nSite .eq. 0) call Say(cSay,cWarn(iTime),cName,'no scintillation index found') end if end if !------- ! iSI is now the space preceeding the first scint index in the daily files (iTime = 1), or ! the space at the start of the first g-level value in the yearly files. do while (iErr .eq. 0) !------- ! For the daily VLIST_UCSD files, zero g-values are inserted for each of the ! 'nSite' scint indices, and in addition a copy of the first g-level/scint index ! pair is inserted before the pairs just created. if (iTime .eq. 1) then do I=nSite-1,0,-1 cStrOut(iSI+I*nBlock:) = ' 0.00000'//cStrOut(iSI+I*nBlock:) end do cStrOut(iSI:) = cStrOut(iSI:iSI+8+nBlock-1)//cStrOut(iSI:) end if I = itrim(cStrOut) iReclOut(1) = min(iReclOut(1), I) iReclOut(2) = max(iReclOut(2), I) iRec = iRec+1 ! Record counter cRec(iRec) = cStrOut ! Record content cFil(iRec) = cName ! File of origin read (iU,'(A)',iostat=iErr) cStrOut end do iU = iFreeLun(iU) end if return end C+ C NAME: C iFourDigitYear C CALLING SEQUENCE: function iFourDigitYear(cYr) C CALLS: C Str2Flt C- character cYr*(*) iYr = 1 call Str2Flt(cYr(:2),iYr,rVec) iYr = nint(rVec) if (iYr .gt. 70) iYr = 1900+iYr if (iYr .le. 70) iYr = 2000+iYr iFourDigitYear = iYr return end