#!/usr/bin/python # MODULE: # remove_stars.py # DESCRIPTION: # This Python script is a means of automating star subtractions in IDL. # In other words, this script allows batch processing of sky maps. The # parameters of this script are specified via command line options. # # The process of this script's execution is straightforward: # # 1. Read in any command line options specified by the user. # A. Check for invalid command line options. # B. Modify the parameters of the skymap processing routines # based on which options were specified. # 2. Display the parameters of the skymap processing to the user. # 3. If a file containing star names was specified by the user via # a command line option, then read in the list of star names. # 4. List the contents of the unprocessed skymap directory (hereafter # referred to as SKY) and the processed skymap directory (hereafter # referred to as EQU). # A. Store the skymap files in SKY and EQU in two hashtables, # one for SKY and one for EQU, using the standard SMEI date # portions of their filenames as keys. # B. For each date key "k" in the SKY hashtable and for a list # "l" of strings representing IDL commands: # i. If there does not exist a key "e" in the EQU hashtable # such that k equals e, then create an IDL command to # process the file corresponding to k and store the # command in l. # ii. If there exists a key "e" in the EQU hashtable such # that k equals e, then if the modification timestamp of # the file corresponding to k is greater than the # modification timestamp of the file corresponding to e, # then create an IDL command to process the file # corresponding to k and store the command in l. # C. Write l to an IDL batch script file. # 5. If the user did not request that the processing not occur, # spawn a process running the IDL batch script file. # 6. If the user did not request that the IDL batch script file not # be deleted, delete the IDL batch script file. # # These steps are explained in detail below: # # 1. Command line options begin with a hyphen and are separated by # whitespace. For example: # # % ./remove_stars.py -noprocess -noscientific # # Has two command line options: -noprocess and -noscientific. # Command line options are case-insensitive. # # A. Invalid command line options are options that do not have # leading hyphens or that do not match any of the acceptable # command line options. # # B. Executing remove_stars.py without command line options # results in the default behavior, which is as follows: # # + Unprocessed skymaps are located in the camera # subdirectories of $SMEISKY0/sky. # + Processed skymaps are located in the camera # subdirectories of $SMEISKY0/equ. # + Newly processed skymaps are written to # $SMEISKY0/equ (what I call the "destination # directory"). # + Skymaps from cameras 1, 2, and 3 are processed. # + Both engineering and scientific mode skymaps are # processed. # + Skymaps of all dates are processed. # + All stars in the stars catalog are subtracted during # processing in IDL. # + The IDL batch script file is named # $SMEISKY0/equ/remove_stars123.pro. # # If the user provides command line options, the default # behavior is modified. # # 2. If remove_stars.py runs according to the default behavior, then # only a few parameters are displayed to the user on standard # output. If additional command line options are provided, then # they are displayed as well. # # 3. There is a special command line option (see below) that specifies # a path to a file containing star names. If this is provided # by the user, then the star names are read from the file and # stored in a string. The string is used later in the creation # of IDL commands to constrain the IDL star subtraction # (smei_star_remove) to only remove stars found in the star names # file. # # 4. A list of strings "l" is created to hold IDL commands (which # are created in steps 4Bi and 4Bii below). I will label the # directory where unprocessed skymaps are held "SKY" and the # directory where processed skymaps are held "EQU" for the # documentation below. # # A. The skymap files are expected to be in the REX form # c[123]sky_\d\d\d\d_\d\d\d_\d\d\d\d\d\d.fts.gz or # c[123]equ_\d\d\d\d_\d\d\d_\d\d\d\d\d\d.fts.gz (the former # for skymaps found in SKY and the latter for skymaps found # in EQU). For each skymap file "x" in SKY, the standard # SMEI date portion (characters indicies 6-20) of the filename # are used as key "k" for x in the hashtable associated with # SKY (which I will label "hS"). For each skymap file "y" # in EQU, the standard date portion (character indicies 6-20) # of the filename are used as key "e" for y in the hashtable # associated with EQU (which I will label "hE"). If the # user provides a starting SMEI date, an ending SMEI date, # or both, then only skymap files with dates falling within # the range specified by the user are placed in the hashtables. # # B. Loop through all of the keys in hS. Call each individual # key "k", the value to which k maps in hS "hS[k]", and # the value to which k maps in hE "hE[k]". # # i. If there does not exist some key "e" in hE such that # k equals e, then create a new IDL command to call # the IDL star subtraction routine (smei_star_remove) # on the file hS[k] and add the command to l. # # ii. If there exists some key "e" in hE such that k equals e, # then if the modification timestamp of the file hS[k] # is greater than the modification timestamp of the file # hE[e] (that is, file hS[k] is more recent than file # hE[e]), then create a new IDL command to call the # IDL star subtraction routine (smei_star_remove) on the # file hS[k] and add the command to l. # # C. A text file (the name of which can be modified by the user # via particular command line options; see below) is created # in the destination directory ($SMEISKY0/equ by default; # can be modified by the user via particular command line # options) and the IDL commands in l are written to it. # # 5. spawnlpe is used to execute the IDL batch script created in # step 4C. The IDL batch script instructs IDL to write the # newly-created processed skymaps to the appropriate camera/mode # subdirectory of the destination directory ($SMEISKY0/equ by # default; can be modified by the user via particular command # line options). This processing behavior can be prevented # by the user by providing the command line option -noprocess, # which terminates remove_stars.py after step 4C but before this # step. # # 6. The IDL batch script is deleted via unlink. The deletion can # be prevented by the user by providing the command line option # -nodelete, which terminates remove_stars.py after step 5 but # before this step. # # SKY, EQU, and the destination directory are expected to contain the # subdirectories c1, c1m0, c2, c2m0, c3, and c3m0 (subdirectories ending # in "m0" contain engineering mode skymaps; the others contain scientific # mode skymaps). If a particular subdirectory is not found in # either SKY or EQU during step 4A, then skymaps of the camera and # mode corresponding to the nonexistent directory are not processed (e.g., # if c1m0 is not found in EQU, then camera 1 engineering mode skymaps # are not processed) and a message indicating the failure to locate # the missing directory is displayed to the user on standard output. # If the destination directory does not contain a particular subdirectory, # then the subdirectory is created in the destination directory between # steps 5 and 6 to ensure that newly-created processed skymaps are # written to disk. # # remove_stars.py is executed via the command line as follows: # # % remove_stars.py [Options] # # The following is a list of valid command line options: # # -cameras=cameranums # Instructs remove_stars.py to only search for and process sky # maps of one of the specified camera numbers. This list of # camera numbers does not need separators, but it must not have # spaces. Non-numeric characters in the list are discarded. This # list of camera numbers also modifies the name of the IDL batch # script file by adding the numbers to the file name (for example, # if the user provides the option "-cameras=12", then [assuming # the prefix remains unchanged by the user] the IDL batch script # will be named "remove_stars12.pro"). # Default: 123 # # -dest=dest_dir # Sets the directory where the output IDL batch script will be # written. This is also the directory where newly-created # processed skymaps (ones generated by smei_star_remove in IDL) # are placed (actually, they are placed in the subdirectories # based on camera number and mode). # Default: $SMEISKY0/equ # # -equdir=equ_dir # Sets the directory where processed skymap files are to be # located. The skymaps found in this directory are compared to # the skymaps found in the directory specified by -skydir (see # below) in steps 4A and 4B. # Default: $SMEISKY0/equ # # -filename=prefix # Sets the prefix of the IDL batch script file. This is # not meant to be a path, as the path to the directory where the # IDL batch script will be written is specified by the -dest # option. remove_stars.py will automatically append the camera # numbers used to conduct the search and a .PRO extension to the # prefix specified here. # Default: remove_stars # # -help # Prints a usage message to the user on standard output. # # -nodelete # Instructs remove_stars.py to skip step 6 (that is, the IDL # batch script is not deleted after it is run). This is # useful if the user wishes to keep a record of which skymaps # were processed (and how they were processed). Note that the # -noprocess option implies -nodelete. # # -noengineering # Instructs remove_stars.py to not process engineering mode # skymaps. Engineering mode skymaps are processed by default. # # -noprocess # Instructs remove_stars.py to terminate before step 5 is # executed (that is, the program terminates after the IDL batch # script is created but before it is executed). The IDL batch # script is not deleted (that is, -noprocess implies -nodelete). # # -noscientific # Instructs remove_stars.py to not process scientific mode # skymaps. Scientific mode sky maps are processed by default. # # -skydir=skymap_dir # Sets the base directory where unprocessed skymaps are to be # located. # Default: $SMEISKY0/sky # # -starsfile=sfile # Forces the IDL star subtraction routine (smei_star_remove) to # remove stars found in the file sfile. sfile can be an absolute # or relative path to a text file containing, one per line, the # names of stars to subtract when skymaps are processed by # remove_stars.py. Only stars named in this file are subtracted. # The default behavior is to use all stars in the current # star catalog for star subtractions. # # -startdate=start # Limits the star removal by adding a lower SMEI date boundary # (inclusive). start must be a date formatted as a standard SMEI # date (YYYY_DOY_hhmmss). Only skymaps with SMEI dates at or # later than start will be processed. This boundary date will # apply to skymaps of all cameras and modes. The default behavior # of remove_stars.py is to not have a lower bound on skymap dates. # # -stopdate=stop # Limits the star removal by adding an upper SMEI date boundary # (inclusive). stop must be a date formatted as a standard SMEI # date (YYYY_DOY_hhmmss). Only skymaps with SMEI dates at or # earlier than stop will be processed. This boundary date will # apply to skymaps of all cameras and modes. The default behavior # of remove_stars.py is to not have an upper bound on skymap # dates. # # LANGUAGE: # Python 2.4.3 # REVISION HISTORY: # April 12, 2007 (Jordan T. Vaughan, jtvaugha@ucsd.edu) # + Finalized module code for v1.00 # + spawnlpe call fixed by John Clover # + Added documentation # ########### # Imports # ########### import os # Imported for filesystem access and manipulation. import sys # Imported for access to argv. import re # Imported for regular expression matching. ############# # Variables # ############# # Engineering/scientific mode constants. engineeringMode = 1 scientificMode = 2 # Basic variables that may be modified through command line options # The string containing the path of the sky directory (where unsubtracted # skymaps are kept). skymapdir = os.path.expanduser(os.path.expandvars("$SMEISKY0/sky")) # The string containing the path of the equ directory (where star-subtracted # skymaps are kept). The skymaps in this directory are compared to the # skymaps in skymapdir based on filename and timestamp to see which skymaps # in skymapdir should be processed. equfiledir = os.path.expanduser(os.path.expandvars("$SMEISKY0/equ")) # The string containing the path of the directory to which processed skymaps # will be written. destdir = os.path.expanduser(os.path.expandvars("$SMEISKY0/equ")) # The string representing the extension of the IDL script that will be # generated by this program to process skymaps. filenameExtension = '.pro' # The string representing the prefix of the name of the IDL script that # will be generated by this program to process skymaps. filename = 'remove_stars' # Boolean flag determining whether or not skymaps should be processed by # the generated IDL script. noprocess = False # Boolean flag determining whether or not the generated IDL script should # be deleted after it is executed. nodelete = False # List containing integers denoting which cameras will be processed. cameras = [1, 2, 3] # String representing the starting date of the skymap processing in standard # SMEI date format. startDate = '' # String representing the ending date of the skymap processing in standard # SMEI date format. stopDate = '' # Boolean flag determining whether or not engineering mode skymaps should be # processed. ignoreEngineeringMode = False # Boolean flag determining whether or not scientific mode skymaps should be # processed. ignoreScientificMode = False # List containing numbers indicating which modes (scientific or engineering) # should be processed. This should contain only a combination of the # engineering/scientific mode constants given above. modesToProcess = [] # String containing strings representing the names of stars to remove from # skymaps during processing. These stars are read in from the file represented # by listOfStarsToRemoveFile. This string is passed to the IDL star removal # routine as the list of stars to remove from skymaps. listOfStarsToRemove = '' # The path of a text file containing the names of stars to remove from skymaps # during processing. listOfStarsToRemoveFile = '' # Tuple of subdirectories within skymapdir and equfiledir to search. subdirectories = ("c1", "c1m0", "c2", "c2m0", "c3", "c3m0") # Hashtable relating each subdirectory name to a camera number. subdirectorycameras = {"c1": 1, "c1m0": 1, "c2": 2, "c2m0": 2, "c3": 3, "c3m0": 3} # Hashtable relating each subdirectory to a mode. Each key is set to # either engineeringMode or scientificMode. subdirectoryModes = {'c1': scientificMode, 'c1m0': engineeringMode, 'c2': scientificMode, 'c2m0': engineeringMode, 'c3': scientificMode, 'c3m0': engineeringMode} # Regular expression patterns for file formats and the standard SMEI date. SMEIDateRE = '\\d\\d\\d\\d_\\d\\d\\d_\\d\\d\\d\\d\\d\\d' skymapre = "c[123]sky_" + SMEIDateRE + ".fts.gz" equfilere = "c[123]equ_" + SMEIDateRE + ".fts.gz" # Offsets and sizes for the SMEI date sections of the sky map and star # subtraction file names. skymapdateoffset = 6 equfiledateoffset = 6 datesize = 15 cameranumberoffset = 1 ############# # Functions # ############# # FUNCTION: # printUsageMessage # DESCRIPTION: # Prints a usage message to standard output. # PARAMETERS: # None # RETURN VALUE: # None # EXCEPTIONS: # None # def printUsageMessage(): print """ remove_stars.py, v1.0 (build 4-12-2007) by Jordan T. Vaughan (jtvaugha@ucsd.edu) This Python script is a means of automating star subtractions in IDL. In other words, this script allows batch processing of sky maps. The parameters of this script are specified via command line options. The process of this script's execution is straightforward: 1. Read in any command line options specified by the user. A. Check for invalid command line options. B. Modify the parameters of the skymap processing routines based on which options were specified. 2. Display the parameters of the skymap processing to the user. 3. If a file containing star names was specified by the user via a command line option, then read in the list of star names. 4. List the contents of the unprocessed skymap directory (hereafter referred to as SKY) and the processed skymap directory (hereafter referred to as EQU). A. Store the skymap files in SKY and EQU in two hashtables, one for SKY and one for EQU, using the standard SMEI date portions of their filenames as keys. B. For each date key "k" in the SKY hashtable and for a list "l" of strings representing IDL commands: i. If there does not exist a key "e" in the EQU hashtable such that k equals e, then create an IDL command to process the file corresponding to k and store the command in l. ii. If there exists a key "e" in the EQU hashtable such that k equals e, then if the modification timestamp of the file corresponding to k is greater than the modification timestamp of the file corresponding to e, then create an IDL command to process the file corresponding to k and store the command in l. C. Write l to an IDL batch script file. 5. If the user did not request that the processing not occur, spawn a process running the IDL batch script file. 6. If the user did not request that the IDL batch script file not be deleted, delete the IDL batch script file. This is a convenience script that is meant to automate star subtraction for skymaps. This script is best used in a cron job. A more detailed explanation of the function of remove_stars.py, including a full explanation of the execution procedure (denoted by step numbers), can be found in the module documentation comments of remove_stars.py. These can be viewed with any text editor. Usage: %% remove_stars.py [Options] Options: -cameras=cameranums Instructs remove_stars.py to only search for and process sky maps of one of the specified camera numbers. This list of camera numbers does not need separators, but it must not have spaces. Non-numeric characters in the list are discarded. This list of camera numbers also modifies the name of the IDL batch script file by adding the numbers to the file name (for example, if the user provides the option "-cameras=12", then [assuming the prefix remains unchanged by the user] the IDL batch script will be named "remove_stars12.pro"). Default: 123 -dest=dest_dir Sets the directory where the output IDL batch script will be written. This is also the directory where newly-created processed skymaps (ones generated by smei_star_remove in IDL) are placed (actually, they are placed in the subdirectories based on camera number and mode). Default: $SMEISKY0/equ -equdir=equ_dir Sets the directory where processed skymap files are to be located. The skymaps found in this directory are compared to the skymaps found in the directory specified by -skydir (see below) in steps 4A and 4B. Default: $SMEISKY0/equ -filename=prefix Sets the prefix of the IDL batch script file. This is not meant to be a path, as the path to the directory where the IDL batch script will be written is specified by the -dest option. remove_stars.py will automatically append the camera numbers used to conduct the search and a .PRO extension to the prefix specified here. Default: remove_stars -help Prints a usage message to the user on standard output. -nodelete Instructs remove_stars.py to skip step 6 (that is, the IDL batch script is not deleted after it is run). This is useful if the user wishes to keep a record of which skymaps were processed (and how they were processed). Note that the -noprocess option implies -nodelete. -noengineering Instructs remove_stars.py to not process engineering mode skymaps. Engineering mode skymaps are processed by default. -noprocess Instructs remove_stars.py to terminate before step 5 is executed (that is, the program terminates after the IDL batch script is created but before it is executed). The IDL batch script is not deleted (that is, -noprocess implies -nodelete). -noscientific Instructs remove_stars.py to not process scientific mode skymaps. Scientific mode sky maps are processed by default. -skydir=skymap_dir Sets the base directory where unprocessed skymaps are to be located. Default: $SMEISKY0/sky -starsfile=sfile Forces the IDL star subtraction routine (smei_star_remove) to remove stars found in the file sfile. sfile can be an absolute or relative path to a text file containing, one per line, the names of stars to subtract when skymaps are processed by remove_stars.py. Only stars named in this file are subtracted. The default behavior is to use all stars in the current star catalog for star subtractions. -startdate=start Limits the star removal by adding a lower SMEI date boundary (inclusive). start must be a date formatted as a standard SMEI date (YYYY_DOY_hhmmss). Only skymaps with SMEI dates at or later than start will be processed. This boundary date will apply to skymaps of all cameras and modes. The default behavior of remove_stars.py is to not have a lower bound on skymap dates. -stopdate=stop Limits the star removal by adding an upper SMEI date boundary (inclusive). stop must be a date formatted as a standard SMEI date (YYYY_DOY_hhmmss). Only skymaps with SMEI dates at or earlier than stop will be processed. This boundary date will apply to skymaps of all cameras and modes. The default behavior of remove_stars.py is to not have an upper bound on skymap dates. Examples: %% remove_stars.py Compares the skymaps in $SMEISKY0/sky against the processed skymaps files in $SMEISKY0/equ (that is, in their subdirectories) and writes the results of the comparison as IDL batch commands to the IDL batch script file $SMEISKY0/equ/remove_stars123.pro. The script is run and later deleted. %% remove_stars.py -help Prints this lovely and helpful usage message to standard output. %% remove_stars.py -skydir=/a/skymap/dir -equdir=/an/equ/dir -dest=/home/soft/temp -filename=test.pro Compares the skymaps in /a/skymap/dir against the processed skymaps in /an/equ/dir (that is, in their subdirectories) and writes the results of the comparison to /home/soft/temp/test.pro, which is run as an IDL batch script. /home/soft/temp/test.pro is deleted after the script terminates. %% remove_stars.py -noprocess -cameras=12 Same as the first example, except the IDL batch script is not run, nor is it deleted. Only skymaps corresponding to camera 1 or camera 2 are added to the IDL batch script. The name of the batch script is changed to remove_stars12.pro as well. %% remove_stars.py -noengineering -noscientific No sky maps will be processed, for this call does not allow either engineering or scientific mode skymaps to be processed. %% remove_stars.py -startdate=2003_001_000000 -stopdate=2004_001_000000 Processes both engineering and scientific mode skymaps for all three cameras, but only skymaps that fall between the dates 2003_001_000000 and 2004_001_000000, inclusive (that is, all sky maps in the year 2003), are processed. Regarding File Names: The directories specified by the -skydir (which I will label "SKY" for short), -equdir (which I will label "EQU" for short), and -dest (which I will label "destination directory" for short) options are expected to contain the subdirectories c1, c1m0, c2, c2m0, c3, and c3m0 (subdirectories ending in "m0" contain engineering mode skymaps; the others contain scientific mode skymaps). If a particular subdirectory is not found in either SKY or EQU, then skymaps of the camera and mode corresponding to the nonexistent directory are not processed (e.g., if c1m0 is not found in EQU, then camera 1 engineering mode skymaps are not processed) and a message indicating the failure to locate the missing directory is displayed to the user on standard output. If the destination directory does not contain a particular subdirectory, then the subdirectory is created in the destination directory to ensure that newly-created processed skymaps are written to disk.\n""" ################ # MAIN PROGRAM # ################ # Flags used for indicating whether or not certain options were specified by # the user. This is used to prevent repetitions of command line options. skydirgiven = False equdirgiven = False destdirgiven = False filenamegiven = False camerasgiven = False # Which arguments were specified? for x in sys.argv[1:]: arg = x.lower() # Did the user request a usage message? if arg == "-help": printUsageMessage() sys.exit() # Did the user specify the sky map base directory? elif arg.startswith("-skydir="): if skydirgiven: print "ERROR: -skydir specified more than once." sys.exit() skymapdir = os.path.expanduser(os.path.expandvars(arg[8:])) if len(skymapdir) == 0: print "ERROR: No sky map directory specified." sys.exit() if not os.path.exists(skymapdir): print "ERROR: Sky map directory does not exist -- " + skymapdir sys.exit() if not os.path.isdir(skymapdir): print "ERROR: Sky map directory is not a directory -- " + skymapdir sys.exit() skydirgiven = True # Did the user specify the star subtraction file base directory? elif arg.startswith("-equdir="): if equdirgiven: print "ERROR: -equdir specified more than once." sys.exit() equfiledir = os.path.expanduser(os.path.expandvars(arg[8:])) if len(equfiledir) == 0: print "ERROR: No star subtraction file directory specified." sys.exit() if not os.path.exists(equfiledir): print ("ERROR: Star subtraction file directory does not " + "exist -- " + equfiledir) sys.exit() if not os.path.isdir(equfiledir): print ("ERROR: Star subtraction file directory is not a " + "directory -- " + equfiledir) sys.exit() equdirgiven = True # Did the user specify the output (destination) directory? elif arg.startswith("-dest="): if destdirgiven: print "ERROR: -dest specified more than once." sys.exit() destdir = os.path.expanduser(os.path.expandvars(arg[6:])) if len(destdir) == 0: print "ERROR: No destination directory specified." sys.exit() if not os.path.exists(destdir): print "ERROR: Destination directory does not exist -- " + destdir sys.exit() if not os.path.isdir(destdir): print "ERROR: Destination directory is not a directory -- "+destdir sys.exit() destdirgiven = True # Did the user specify the name of the output file? elif arg.startswith("-filename="): if filenamegiven: print "ERROR: -filename specified more than once." sys.exit() filename = arg[10:] if len(filename) == 0: print "ERROR: No output file name specified." sys.exit() filenamegiven = True # Did the user specify to not process the sky maps that need stars removed? elif arg == "-noprocess": if noprocess: print "ERROR: -noprocess specified more than once." sys.exit() noprocess = True # Did the user specify to not delete the IDL batch script after it is # processed? elif arg == "-nodelete": if nodelete: print "ERROR: -nodelete specified more than once." sys.exit() nodelete = True # Did the user specify certain camera numbers? elif arg.startswith("-cameras="): if camerasgiven: print "ERROR: -cameras specified more than once." sys.exit() camearsgiven = True if not arg[9:]: print "ERROR: No camera numbers specified for -cameras." sys.exit() cameras = [] for x in arg[9:]: if x.isdigit() and int(x) >= 1 and int(x) <= 3: cameras.append(int(x)) if not cameras: print "ERROR: Illegal comma-separated camera number list: "+arg[9:] sys.exit() # Did the user want to ignore engineering mode maps? elif arg == '-noengineering': if ignoreEngineeringMode: print 'ERROR: -noengineering specified more than once.' sys.exit() ignoreEngineeringMode = True # Did the user want to ignore scientific mode maps? elif arg == '-noscientific': if ignoreScientificMode: print 'ERROR: -noscientific specified more than once.' sys.exit() ignoreScientificMode = True # Did the user want to specify a lower date bound? elif arg.startswith('-startdate='): if startDate: print 'ERROR: -startdate specified more than once.' sys.exit() startDate = arg[11:] if not startDate: print 'ERROR: No starting SMEI date specified for -startdate.' sys.exit() if not re.search(SMEIDateRE, startDate): print ('ERROR: Argument provided for -startdate is not in ' + 'standard SMEI format.') sys.exit() # Did the user want to specify an upper date bound? elif arg.startswith('-stopdate='): if stopDate: print 'ERROR: -stopdate specified more than once.' sys.exit() stopDate = arg[10:] if not stopDate: print 'ERROR: No stopping SMEI date specified for -stopdate.' sys.exit() if not re.search(SMEIDateRE, stopDate): print ('ERROR: Argument provided for -stopdate is not in ' + 'standard SMEI format.') sys.exit() # Did the user want to specify a file to read star names from (to limit # star removal to the given stars)? # New in version 1.4. elif arg.startswith('-starsfile='): if listOfStarsToRemoveFile: print 'ERROR: -starsfile specified more than once.' sys.exit() listOfStarsToRemoveFile = os.path.expanduser(os.path.expandvars( arg[11:])) if not listOfStarsToRemoveFile: print 'ERROR: No stars list file specified for -starsfile.' sys.exit() if not os.path.exists(listOfStarsToRemoveFile): print 'ERROR: File '+ listOfStarsToRemoveFile +' does not exist.' sys.exit() if not os.path.isfile(listOfStarsToRemoveFile): print 'ERROR: File '+ listOfStarsToRemoveFile +' is not a file.' sys.exit() # Unrecognized command line option. else: print "ERROR: Command line option not recognized -- " + x sys.exit() # Create the output script name. for number in cameras: filename = filename + str(number) filename = filename + filenameExtension # Set up the list of sky map modes to be processed. if not ignoreEngineeringMode: modesToProcess.append(engineeringMode) if not ignoreScientificMode: modesToProcess.append(scientificMode) # Inform the user of the search parameters. print print "Sky map base directory: " + skymapdir print "Star subtraction base directory: " + equfiledir print "Base destination directory: " + destdir print "Script file name: " + filename print "Cameras to be processed: " + str(cameras) if listOfStarsToRemoveFile: print "List of stars to remove: " + listOfStarsToRemoveFile if not ignoreEngineeringMode: print "Engineering mode maps will be processed." if not ignoreScientificMode: print "Scientific mode maps will be processed." if startDate: print "Lower date bound: " + startDate if stopDate: print "Upper date bound: " + stopDate if noprocess: print "The IDL batch script will not be run." elif nodelete: print "The IDL batch script will not be deleted." print # If a stars list was provided, open the file and get the names of the stars. # New in version 1.4. if listOfStarsToRemoveFile: try: listOfStarsToRemoveFile = file(listOfStarsToRemoveFile, 'r') except IOError: print 'ERROR: Could not open list of stars file for reading.' sys.exit() listOfStarsToRemove = ", stars_list=[" for starNameForList in listOfStarsToRemoveFile: starNameForList = starNameForList.strip() if starNameForList: listOfStarsToRemove = listOfStarsToRemove + ('"' + starNameForList + '",') listOfStarsToRemove = listOfStarsToRemove[:-1] + "]" listOfStarsToRemoveFile.close() # Go through the subdirectories and compare the number of sky maps with the # number of star subtraction files based on date. Make a note of the # differences. outputentries = [] for subdir in subdirectories: # Are we supposed to process this subdirectory (i.e. does this subdirectory # correspond to a camera number that should be processed)? if not (subdirectorycameras.has_key(subdir) and (subdirectorycameras[subdir] in cameras)): continue # Does this subdirectory correspond to a mode that we are supposed to # process (i.e. did the user want to process this subdirectory's mode)? if not subdirectoryModes[subdir] in modesToProcess: continue # Do the subdirectories exist, and are they directories? skysubdir = os.path.join(skymapdir, subdir) equsubdir = os.path.join(equfiledir, subdir) if not os.path.exists(skysubdir): print "ERROR: Subdirectory " + skysubdir + " does not exist. Skipping." continue if not os.path.exists(equsubdir): print "ERROR: Subdirectory " + equsubdir + " does not exist. Skipping." continue if not os.path.isdir(skysubdir): print ("ERROR: Subdirectory " + skysubdir + " is not a directory. " + "Skipping.") continue if not os.path.isdir(equsubdir): print ("ERROR: Subdirectory " + equsubdir + " is not a directory. " + "Skipping.") continue # Obtain file listings from both directories. print "Descending into subdirectory " + subdir skymaps = os.listdir(skysubdir) equfiles = os.listdir(equsubdir) skymaps.sort() equfiles.sort() print " Files and directories in " + skysubdir + ": " +str(len(skymaps)) print " Files and directories in " + equsubdir + ": " +str(len(equfiles)) # Extract the date portions from the file names and stash them away. skydates = {} equdates = {} for skyfile in skymaps: if not os.path.isfile(os.path.join(skysubdir, skyfile)): continue if not re.match(skymapre, skyfile.lower()): continue if startDate and (skyfile[skymapdateoffset: skymapdateoffset+datesize] < startDate): continue if stopDate and (skyfile[skymapdateoffset: skymapdateoffset+datesize] > stopDate): continue skydates[skyfile[skymapdateoffset: skymapdateoffset + datesize]] = ( skyfile) for equfile in equfiles: if not os.path.isfile(os.path.join(equsubdir, equfile)): continue if not re.search(equfilere, equfile.lower()): continue if startDate and (equfile[equfiledateoffset:equfiledateoffset+datesize] < startDate): continue if stopDate and (equfile[equfiledateoffset: equfiledateoffset+datesize] > stopDate): continue equdates[equfile[equfiledateoffset: equfiledateoffset+datesize]] = ( equfile) # Now, compare sky map dates with star subtraction file dates. Stash the # ones that differ in a list. previousentrycount = len(outputentries) for skydate in skydates.keys(): # If the unprocessed skymap does not have a counterpart in the # directory containing the processed (star-subtracted) skymaps, then # add it to the list of skymaps to be processed. if not equdates.has_key(skydate): outputentries.append( "smei_star_remove, '" + os.path.abspath(os.path.join(skysubdir, skydates[skydate])) + "', destination='" + os.path.abspath(os.path.join(destdir, subdir)) + "', camera=" + skydates[skydate][cameranumberoffset: cameranumberoffset + 1] + ", overwrite=1, bkgnd_radius=[1.15,1.32], bkgnd_count=20" #+ ", /use_bkgnd_plane" + listOfStarsToRemove) # If two files do not differ by their date, then check their # modification timestamps. If the unprocessed skymap was modified # later than the processed (star-subtracted) skymap, then add it # to the list of skymaps to be processed. else: if (os.path.getmtime(os.path.join(skysubdir, skydates[skydate])) > os.path.getmtime(os.path.join(equsubdir, equdates[skydate]))): outputentries.append( "smei_star_remove, '" + os.path.abspath(os.path.join(skysubdir,skydates[skydate]))+ "', destination='" + os.path.abspath(os.path.join(destdir, subdir)) + "', camera=" + skydates[skydate][cameranumberoffset: cameranumberoffset + 1] + ", overwrite=1, bkgnd_radius=[1.15,1.32], bkgnd_count=20" #+ ", /use_bkgnd_plane" + listOfStarsToRemove) # Display how many skymaps are slated for star subtraction. print (" " + str(len(outputentries) - previousentrycount) + " entries " + "added to the IDL script") # Are there any sky maps to process? if len(outputentries) == 0: print "\nNo sky maps to process. IDL batch script not written.\n" sys.exit() # Write the list of IDL commands (the list of skymaps to process) to the # IDL batch script file. outputentries.sort() scriptfilepath = os.path.join(destdir, filename) print ("\nWriting script file " + scriptfilepath + " (" + str(len(outputentries)) + " entries)") outputentries.append("exit") outputfile = file(scriptfilepath, "w") if not outputfile: print "ERROR: Could not write to " + scriptfilepath sys.exit() for entry in outputentries: outputfile.write(entry + "\n") outputfile.close() # If the user requested that we skip the star removal process, end here. if noprocess: print "\nSkipping IDL batch script run." print "\nDone!" sys.exit() # Ensure that the subdirectories exist in the destination directory. for subdir in subdirectories: destsubdir = os.path.join(destdir, subdir) if not os.path.exists(destsubdir): print "WARNING: Directory " + destsubdir + " does not exist. " + ( "Creating...") os.mkdir(destsubdir) if not os.path.exists(destsubdir): print ("ERROR: Directory " + destsubdir + " could not be created. Aborting.") sys.exit() elif not os.path.isdir(destsubdir): print "ERROR: Path " + destsubdir + " is not a directory. Aborting." sys.exit() # Run spawnlp (not system) to open IDL and run the batch script. print "\nRunning IDL batch script " + os.path.abspath(scriptfilepath) + ".\n" os.spawnlpe(os.P_WAIT, "idl", "idl", scriptfilepath, os.environ) # Delete the IDL batch script unless -nodelete was specified by the user. if not nodelete: print "\nDeleting IDL batch script " + scriptfilepath os.unlink(scriptfilepath) else: print "\nSkipping IDL batch script deletion step" # Done! print "\nDone!"