#!/usr/bin/python #+ # NAME: # remove_stars.py # PURPOSE: # Compares the contents of the subdirectories of $SMEISKY0/sky # against the contents of the subdirectories of $SMEISKY0/equ # and writes the names of the files present in $SMEISKY0/sky # but not present in $SMEISKY0/equ to a text file. These are # written as IDL commands to remove stars from the sky maps. # PROCEDURE: # PYTHON-VERSION: 2.4.2 # VERSION: 1.3 # OS: Fedora Core 5 (Linux) # MODIFICATION HISTORY: # JUN-2006, Jordan T. Vaughan (jvaughan) # JUL-2006, Jordan T. Vaughan # Revision # AUG-2006, Paul Hick (UCSD/CASS; pphick@ucsd.edu) # Added check for unmatched subtracted (equ) skymap. # IDL main program is now created in $TUB and is given a random prefix. #- ########### # Imports # ########### import os # Imported for filesystem access and manipulation. import sys # Imported for access to argv. import re # Imported for regular expression matching. from tiny import hide_env import tempfile tempfile.tempdir = os.environ['TUB'] ############# # Variables # ############# smeisky0 = os.environ['SMEISKY0'] # Basic variables that may be modified through command line options skymapdir = os.path.join(smeisky0,'sky') equmapdir = os.path.join(smeisky0,'equ') destdir = os.path.join(smeisky0,'equ') filename = 'remove_stars.pro' noprocess = False nodelete = False cameras = [1, 2, 3] # Tuple of subdirectories to search. subdirectories = ("c1", "c1m0", "c2", "c2m0", "c3", "c3m0") # Hashtable relating each subdirectory to a camera number. subdirectorycameras = {"c1": 1, "c1m0": 1, "c2": 2, "c2m0": 2, "c3": 3, "c3m0": 3} # Regular expression patterns for file formats. skymapre = "c[123]sky_\\d\\d\\d\\d_\\d\\d\\d_\\d\\d\\d\\d\\d\\d.fts.gz" equmapre = "c[123]equ_\\d\\d\\d\\d_\\d\\d\\d_\\d\\d\\d\\d\\d\\d.fts.gz" # Offsets and sizes for the SMEI date sections of the sky map and star # subtraction file names. skymapdateoffset = 6 equmapdateoffset = 6 datesize = 15 cameranumberoffset = 1 ############# # Functions # ############# # FUNCTION: printUsageMessage # DESCRIPTION: Resets most variables to their default values (specified in the # variables section above) and prints a usage message to standard # output. This should only be called when an error occurs that # requires that a usage message be displayed to the user or when # the user specifies the "-help" command line option. # PARAMETERS: None # RETURN: None # EXCEPTIONS: None def printUsageMessage(): # Reset all variables to their defaults. skymapdir = os.path.join(smeisky0,'sky') equmapdir = os.path.join(smeisky0,'equ') destdir = os.path.join(smeisky0,'equ') filename = 'remove_stars.pro' noprocess = False nodelete = False # Print a usage message. print ("""\nCompares the number and dates of sky maps against the number and dates of star subtraction files and writes an IDL batch script to process sky maps that do not have stars removed. Sky maps with SMEI dates that do not have corresponding processed sky maps (ones with stars removed) or sky maps with a modification timestamp dated earlier than their corresponding processed sky maps (ones with stars removed) are slated for star removal in the generated IDL batch script. Usage: %% remove_stars.py [Options] Options: -help Prints this helpful usage message to standard output. -skydir=sky_map_dir Sets the base directory where sky maps are to be located to sky_map_dir. [default=%s] -equdir=equ_dir Sets the base directory where star subtraction files are to be located to equ_dir. [default=%s] -dest=dest_dir Sets the directory where the output IDL script will be written. This is also the directory where newly processed sky maps (ones generated by smei_star_remove) are placed, though the fresh sky maps are technically placed in the destination directory's subdirectories based on camera number and mode. [default=%s] -filename=file_name Sets the name of the IDL script file. This is not meant to be a path, as the path is specified by the -dest option. The extension of the filename should be .pro or any other extension that IDL will associate with IDL batch scripts. [default=%s] -nodelete Instructs remove_stars.py to not delete the IDL batch script after it is written and processed. This is useful if you wish to keep a record of which skymaps were processed by a previous run of remove_stars.py. The -noprocess option implies -nodelete. -noprocess Forces remove_stars.py to skip the sky map processing stage (i.e. running the IDL batch script) and leaves the IDL batch script intact (normally the batch script is deleted when the IDL interpreter terminates). -cameras=cameranums Instructs remove_stars.py to only search for and process sky maps with one of the specified camera numbers. This list of camera numbers does not need separators, but it must not have spaces. Non-numeric characters in the list are discarded. [default=123] Examples: %% remove_stars.py Compares the sky maps in %s against the star subtraction files in %s (that is, in their subdirectories) and writes the results of the comparison as IDl batch commands to the IDL batch script file %s%s. The script is run and later deleted. %% remove_stars.py -help Prints this lovely and helpful usage message to standard output. %% remove_stars.py -skydir=/a/skymap/dir -equdir=/an/equ/dir -dest=/home/soft/temp -filename=test.pro Compares the sky maps in /a/skymap/dir against the star subtraction files in /an/equ/dir (that is, in their subdirectories) and writes the results of the comparison to /home/soft/temp/test.pro, which is run as an IDL batch script. /home/soft/temp/test.pro is deleted after the script terminates. %% remove_stars.py -noprocess -cameras=12 Same as the first example, except the IDL batch script is not run, nor is it deleted. Only sky maps corresponding to camera 1 or camera 2 are added to the IDL batch script. Regarding File Names: Sky map files and star subtraction files are compared based on the standard SMEI date components of their file names. Standard SMEI format for dates is YYYY_DOY_hhmmss. If a sky map file with a certain date is located in one of the subdirectories of the sky map base directory but is not located in the equivalent subdirectory of the star subtraction (equ) base directory, then the difference is noted in the IDL script file. Star subtraction files that exist without equivalent sky map files are not noted in the outputted text file. In addition, sky map files with modification timestamps earlier than those of their equivalent star subtracted sky maps (if they exist) are also slated for processing by smei_star_remove in the IDL script. Sky maps have the format c[123]sky_YYYY_DOY_hhmmss.fts.gz. Star subtraction files have the format c[123]equ_YYYY_DOY_hhmmss.fts.gz. Subdirectories searched: The subdirectories c1, c1m0, c2, c2m0, c3, and c3m0 are expected to reside in both the sky map base directory and the star subtraction base directory (specified by the -skydir and -equdir options, respectively). If a subdirectory (e.g. c1) is located in only one base directory or in neither base directory, then it is ignored (results for the subdirectory will not appear in the outputted text file). However, if the destination directory does not contain one of the subdirectories, then the subdirectory is created so that star subtracted sky maps generated by smei_star_remove can be written.""" % (skymapdir, equmapdir, destdir, filename, skymapdir, equmapdir, destdir, filename)) ################ # MAIN PROGRAM # ################ # Flags used for indicating whether or not certain options were specified by # the user. This is used to prevent repetitions of command line options. skydirgiven = False equdirgiven = False destdirgiven = False filenamegiven = False camerasgiven = False # Which arguments were specified? for x in sys.argv[1:]: arg = x.lower() # Did the user request a usage message? if arg == "-help": printUsageMessage() sys.exit() # Did the user specify the sky map base directory? elif arg.startswith("-skydir="): if skydirgiven: print "ERROR: -skydir specified more than once." sys.exit() skymapdir = os.path.expanduser(os.path.expandvars(arg[8:])) if len(skymapdir) == 0: print "ERROR: No sky map directory specified." sys.exit() if not os.path.exists(skymapdir): print "ERROR: Sky map directory does not exist -- %s"%hide_env(skymapdir) sys.exit() if not os.path.isdir(skymapdir): print "ERROR: Sky map directory is not a directory -- %s"%hide_env(skymapdir) sys.exit() skydirgiven = True # Did the user specify the star subtraction file base directory? elif arg.startswith("-equdir="): if equdirgiven: print "ERROR: -equdir specified more than once." sys.exit() equmapdir = os.path.expanduser(os.path.expandvars(arg[8:])) if len(equmapdir) == 0: print "ERROR: No star subtraction file directory specified." sys.exit() if not os.path.exists(equmapdir): print "ERROR: Star subtraction file directory does not exist -- %s"%hide_env(equmapdir) sys.exit() if not os.path.isdir(equmapdir): print "ERROR: Star subtraction file directory is not a directory -- %s"%hide_env(equmapdir) sys.exit() equdirgiven = True # Did the user specify the output (destination) directory? elif arg.startswith("-dest="): if destdirgiven: print "ERROR: -dest specified more than once." sys.exit() destdir = os.path.expanduser(os.path.expandvars(arg[6:])) if len(destdir) == 0: print "ERROR: No destination directory specified." sys.exit() if not os.path.exists(destdir): print "ERROR: Destination directory does not exist -- %s"%hide_env(destdir) sys.exit() if not os.path.isdir(destdir): print "ERROR: Destination directory is not a directory -- %s"%hide_env(destdir) sys.exit() destdirgiven = True # Did the user specify the name of the output file? elif arg.startswith("-filename="): if filenamegiven: print "ERROR: -filename specified more than once." sys.exit() filename = arg[10:] if len(filename) == 0: print "ERROR: No output file name specified." sys.exit() filenamegiven = True # Did the user specify to not process the sky maps that need stars removed? elif arg == "-noprocess": if noprocess: print "ERROR: -noprocess specified more than once." sys.exit() noprocess = True # Did the user specify to not delete the IDL batch script after it is # processed? elif arg == "-nodelete": if nodelete: print "ERROR: -nodelete specified more than once." sys.exit() nodelete = True # Did the user specify certain camera numbers? elif arg.startswith("-cameras="): if camerasgiven: print "ERROR: -cameras specified more than once." sys.exit() camearsgiven = True if not arg[9:]: print "ERROR: No camera numbers specified for -cameras." sys.exit() cameras = [] for x in arg[9:]: if x.isdigit(): cameras.append(int(x)) if not cameras: print "ERROR: Illegal comma-separated camera number list: "+arg[9:] sys.exit() # Unrecognized command line option. else: print "ERROR: Command line option not recognized -- %s"%x sys.exit() outputfile = tempfile.mktemp('_'+filename) # Inform the user of the search parameters. print print "Sky map base directory: %s"%hide_env(skymapdir) print "Star subtraction base directory: %s"%hide_env(equmapdir) print "Destination directory: %s"%hide_env(destdir ) print "Script file name: %s"%hide_env(outputfile) print "Cameras to be processed: %s"%str(cameras) # Go through the subdirectories and compare the number of sky maps with the # number of star subtraction files based on date. Make a note of the # differences. outputentries = [] for subdir in subdirectories: # Are we supposed to process this subdirectory (i.e. does this subdirectory # correspond to a camera number that should be processed)? if not (subdirectorycameras.has_key(subdir) and (subdirectorycameras[subdir] in cameras)): continue # Do the subdirectories exist, and are they directories? skysubdir = os.path.join(skymapdir, subdir) equsubdir = os.path.join(equmapdir, subdir) if not os.path.exists(skysubdir): print "ERROR: Directory %s does not exist. Skipping."%hide_env(skysubdir) continue if not os.path.exists(equsubdir): print "ERROR: Directory %s does not exist. Skipping."%hide_env(equsubdir) continue if not os.path.isdir(skysubdir): print "ERROR: Directory %s is not a directory. Skipping."%hide_env(skysubdir) continue if not os.path.isdir(equsubdir): print "ERROR: Directory %s is not a directory. Skipping."%hide_env(equsubdir) continue # Obtain file listings from both directories. print "\nDescending into subdirectory %s"%hide_env(subdir) skymaps = os.listdir(skysubdir) equmaps = os.listdir(equsubdir) skymaps.sort() equmaps.sort() # Check all skymaps with stars still in it. skydates = {} # Used to collect dates for skymap in skymaps: full_name = os.path.join(skysubdir, skymap) # Make sure it is a file if not os.path.isfile(full_name): continue # Make sure it fits the skymap filename template if not re.match(skymapre, skymap.lower()): continue # Extract date field YYYY_DOY_hhmmss key = skymap[skymapdateoffset: skymapdateoffset+datesize] skydates[key] = skymap print "Unsubtracted skymaps in %s: %d"%(hide_env(skysubdir),len(skydates)) # Check all skymaps where stars already have been subtracted equdates = {} # Used to collect dates for equmap in equmaps: full_name = os.path.join(equsubdir, equmap) # Make sure it is a file if not os.path.isfile(full_name): continue # Make sure it fits the skymap filename template if not re.match(equmapre, equmap.lower()): continue # Extract date field YYYY_DOY_hhmmss key = equmap[equmapdateoffset: equmapdateoffset+datesize] # Make sure that there is a matching unsubtracted skymap if not skydates.has_key(key): print "WARNING: Unmatched subtracted skymap, %s"%hide_env(full_name) equdates[key] = equmap print " Subtracted skymaps in %s: %d"%(hide_env(equsubdir),len(equdates)) # Now, compare skymap dates with equmap dates. Stash the # differences in a list. count_write = 0 count_overwrite = 0 for skydate in skydates.keys(): # Loop over all skymap dates if not equdates.has_key(skydate): # If there is no matching equdate count_write = count_write+1 # There is no equmap with the same date as skymap: # Set the skymap up for star subtraction outputentries.append( "smei_star_remove,'"+os.path.join(skysubdir, skydates[skydate]) +"',"+ " destination='" +os.path.join(destdir, subdir) +"',"+ " camera=" +skydates[skydate][cameranumberoffset: cameranumberoffset+1]) else: # There is an equmap matching the date of the skymap. # Compare file time stamps. if (os.path.getmtime(os.path.join(skysubdir, skydates[skydate])) > os.path.getmtime(os.path.join(equsubdir, equdates[skydate]))): count_overwrite = count_overwrite+1 # Skymap is more recent than equmap. # Set the skymap up for star subtraction. # Remember to set the /overwrite keyword on smei_star_remove outputentries.append( "smei_star_remove,'"+os.path.join(skysubdir, skydates[skydate]) +"',"+ " destination='" +os.path.join(destdir, subdir) +"',"+ " camera=" +skydates[skydate][cameranumberoffset: cameranumberoffset+1]+","+ " /overwrite") print "Unsubtracted skymaps without matching subtracted map: %d"%count_write print "Unsubtracted skymaps more recent than subtracted map: %d"%count_overwrite # Are there any sky maps to process? if len(outputentries) == 0: print "\nNo sky maps to process. IDL batch script not written.\n" sys.exit() # Ensure that the subdirectories exist in the destination directory. for subdir in subdirectories: destsubdir = os.path.join(destdir, subdir) if not os.path.exists(destsubdir): print "WARNING: Directory %s does not exist. Creating..."%hide_env(destsubdir) os.mkdir(destsubdir) if not os.path.exists(destsubdir): print "ERROR: Directory %s could not be created. Aborting."%hide_env(destsubdir) sys.exit() elif not os.path.isdir(destsubdir): print "ERROR: Path %s is not a directory. Aborting."%hide_env(destsubdir) sys.exit() # Add IDL exit statement outputentries.append("exit") # Write the list of IDL calls to smei_star_remove into outputfile print "\nWriting IDL program %s (%d entries)"%(hide_env(outputfile),len(outputentries)) (open(outputfile,'w')).write('\n'.join(outputentries)+'\n') if not noprocess: # Version 1.1 code: run spawnlp instead of system print "\nRunning IDL batch script %s.\n"%hide_env(outputfile) os.spawnlp(os.P_WAIT, "idl", "idl", outputfile) # Delete the IDL batch script. # (Modified in version 1.2 to account for the -nodelete option.) if nodelete: print "\nSkipping IDL batch script deletion step" else: print "\nDeleting IDL batch script %s"%hide_env(outputfile) os.unlink(outputfile) # Done! print "\nDone!"