;+
; NAME:
;	fileset_sizes
; PURPOSE:
; CALLING SEQUENCE:
	PRO fileset_sizes, data_file, $
		file		= file			, $
		xysize		= xysize		, $
		silent		= silent		, $
		time_ago	= time_ago		, $
		start_time	= start_time	, $
		stop_time	= stop_time		, $
		bin_width	= bin_width		, $
		charsize	= charsize		, $
		unlabeled	= unlabeled		, $
		refresh		= refresh		, $
		show_gaps	= show_gaps		, $
		cdf			= cdf			, $
		ratio		= ratio			, $
		title		= title			, $
		_extra		= _extra
; INPUT:
;	data_file		fully-qualified name with the list of file
;					names on which the plot are based.
;					Default: telescope_duty_cycle.txt
;	file			filename on which names of graphics output
;					files are based.
; OPTIONAL INPUTS:
;
;	/cdf
;
;	start_time=start_time
;					start time for graphs; default: earliest file
;	stop_time=stop_time
;					stop time of graphs; default: latest file
;	time_ago=time_ago
;					instead of start_time, specify a time range:
;					start_time then becomes stop_time-time_ago
;
;	/unlabeled		omit labeling
;	/show_gaps		used for duty cycle plot to emphasize times were
;					no data are present: these time ranges are explicitly
;					erased with a final polyfill. This can have the
;					unwelcome side effect that very small gaps (less
;					then one pixel) are shown as a full pixel wide.
;	charsize=charsize
;					character size for labeling
; INCLUDE:
	@compile_opt.pro
; CALLS:
;	InitVar, TimeUnit, GetFileSpec, txt_read
;	TimeGet, TimeSet, strposn, IsType, IsTime
;	set_page, get_page, plotcurves
; MODIFICATION HISTORY:
;	MAR-2017, started adding documentation
;-

InitVar, bin_width	, 1
InitVar, silent	 	, 0
InitVar, charsize	, 1
InitVar, unlabeled	, /key
InitVar, refresh	, /key
InitVar, show_gaps	, /key
InitVar, cdf		, /key
InitVar, ratio		, /key
InitVar, title		, 'some fileset'

uday = TimeUnit(/day)
usec = TimeUnit(/second)
sec_per_day = 86400LL

InitVar, data_file, who_am_i(/procedure)+'.txt'
fileset = GetFileSpec(data_file,upto='name')
IF GetFileSpec(data_file,part='type') EQ '.gz' THEN fileset = GetFileSpec(fileset,upto='name')
save_file = fileset+'.idl.sav'

; Check for the save file. If not present, then make sure
; refresh=0, forcing (re)reading of list of files.

IF NOT refresh THEN BEGIN
	IF (file_search(save_file))[0] EQ '' THEN BEGIN
		IF silent LT 1 THEN message, /info, "IDL save file not found, '"+save_file+"'"
		refresh = 1
	ENDIF
ENDIF

;/data/topology/ark/data/prefix-probing/2018/05/yyz-ca.20180506.1525572000.warts.gz 115270652 359330367 748521
;./2018-05-04/net44.1525471200.pcap.gz 68660259860 183521461721
;net44.1524247200.pcap.gz.enc 62902857600 62902857600

IF refresh THEN BEGIN

	IF silent LT 1 THEN message, /info, data_file
	IF NOT txt_read(data_file,file_name,silent=silent) THEN RETURN

	nfile = n_elements(file_name)
	compressed_size		= lon64arr(nfile)
	uncompressed_size	= lon64arr(nfile)
	traces				= lon64arr(nfile)
	times				= strarr(nfile)	;lon64arr(nfile)

	FOR i=0L,nfile-1 DO BEGIN					; Loop over all filenames
		tmp = strtok(file_name[i],/extract)
		n = n_elements(tmp)
		file_name  [i] = tmp[0]		; Filename
		compressed_size[i] = tmp[1]	; Ondisk size
		IF n GT 2 THEN IF tmp[2] NE tmp[1] THEN uncompressed_size[i] = tmp[2]
		IF n GT 3 THEN traces[i] = tmp[3]
	ENDFOR

	; prefix-probing : zrh2-ch.20181204.1543888800.warts.gz
	; v4 traceroute  : daily.l7.t1.c006947.20180829.yyc-ca.warts.gz
	;		   aal-dk.team-probing.c006951.20180916.warts.gz
	; v6 traceroute  : topo-v6.l8.20181205.1544026262.dca3-us.warts.gz
	; telescope      : net44.1543978800.pcap.gz
	; flowtuple      : ucsd-nt.1536534000.flowtuple.cors.gz
	; dos            : ucsd-nt.1536534000.dos.cors.gz

	n = fileset EQ 'ark-ipv4-prefix-probing' ? 2 : fileset EQ 'ark-ipv4-traceroute' ? 3 : fileset EQ 'ark-ipv6-traceroute' ? 3 : 1

	file_name = GetFileSpec(file_name,part='name')
	FOR i=0L,nfile-1 DO BEGIN
		tmp = strtok(file_name[i],'.',/extract)
		;times[i] = tmp[ n+(fileset EQ 'ark-ipv4-traceroute')*(tmp[0] EQ 'daily') ]
		times[i] = tmp[ n+(tmp[0] EQ 'daily') ]
	ENDFOR

	IF fileset EQ 'ark-ipv4-traceroute' THEN times = TimeGet(TimeSplit(times,format='YYYYMNDD'),/unix)
	times = long64(times)

	save, nfile, file_name, compressed_size, uncompressed_size, traces, times, filename=save_file

ENDIF ELSE BEGIN

	IF silent LT 1 THEN message, /info, save_file
	restore, save_file

ENDELSE

IF total(uncompressed_size	) EQ 0 THEN destroyvar, uncompressed_size
IF total(traces   			) EQ 0 THEN destroyvar, traces

compressed_only = IsType(uncompressed_size,/undefined)
IF ratio AND compressed_only THEN BEGIN
	message, /info, 'no full file size available'
	RETURN
ENDIF

first_time = min(times)
last_time  = max(times) + sec_per_day

; Set the start and stop time for the graph from keywords
; time_ago, start_time and stop_time

IF IsType(time_ago,/defined) THEN BEGIN
	InitVar, stop_time, last_time
	IF IsType(stop_time,/string) THEN stop_time = TimeSet(stop_time)
	IF IsTime(stop_time) THEN stop_time = TimeGet(stop_time,/unix,/scalar)
	IF IsTime(time_ago) THEN time_ago = TimeGet(time_ago,/unix,/scalar) ELSE time_ago *= sec_per_day
	start_time = stop_time-time_ago
ENDIF ELSE IF IsType(start_time,/defined) THEN BEGIN
	IF IsType(start_time,/string) THEN start_time = TimeSet(start_time)
	IF IsTime(start_time) THEN start_time = TimeGet(start_time,/unix,/scalar)
	InitVar, stop_time, last_time
	IF IsType(stop_time,/string) THEN stop_time = TimeSet(stop_time)
	IF IsTime(stop_time) THEN stop_time = TimeGet(stop_time,/unix,/scalar)
ENDIF ELSE IF IsType(stop_time,/defined) THEN BEGIN
	IF IsType(stop_time,/string) THEN stop_time = TimeSet(stop_time)
	IF IsTime(stop_time) THEN stop_time = TimeGet(stop_time,/unix,/scalar)
	InitVar, start_time, first_time
	IF IsType(start_time,/string) THEN start_time = TimeSet(start_time)
	IF IsTime(start_time) THEN start_time = TimeGet(start_time,/unix,/scalar)
ENDIF ELSE BEGIN
	start_time = first_time
	stop_time  = last_time
ENDELSE

; Shift start_time to beginning of day
; Shift stop_time to end of day

start_time = TimeGet(TimeGet(TimeSet(unix=start_time),uday,/bot),/unix,/scalar)
stop_time  = TimeGet(TimeGet(TimeSet(unix=stop_time ),uday,/eot),/unix,/scalar)

trange = [start_time,stop_time]+0.025*(stop_time-start_time)*[-1,1]

IF silent LT 2 THEN BEGIN
	message,/info,'Data time range '+strjoin(TimeGet(/ymd,TimeSet(unix=[first_time,last_time]), upto=usec),' to ')
	message,/info,'Axis time range '+strjoin(TimeGet(/ymd,TimeSet(unix=trange                ), upto=usec),' to ')
	message,/info,'Plot time range '+strjoin(TimeGet(/ymd,TimeSet(unix=[start_time,stop_time]), upto=usec),' to ')
ENDIF

ctable = 12
color = [14,8,15]*16
color_rasp = 4*16

InitVar, file, 'fileset_sizes.eps'
is_eps = GetFileSpec(file,part='type') EQ '.eps'

time_range = TimeGet(TimeSet(unix=start_time),format='YYYY Mon DD')+' to '+TimeGet(TimeSet(unix=stop_time),format='YYYY Mon DD')

ndays = 1LL+(stop_time-start_time)/(bin_width*sec_per_day)
tunix = start_time+bin_width*sec_per_day*l64indgen(ndays)
tplot = TimeSet(unix=tunix)

daily_compressed_size = lon64arr(ndays)
IF NOT compressed_only THEN daily_uncompressed_size = lon64arr(ndays)

FOR i=0L,nfile-1 DO BEGIN
	tmp = min(abs(tunix-times[i]),n)
	IF tmp LE 43200*bin_width THEN BEGIN
		daily_compressed_size  [n] += compressed_size  [i]
		IF NOT compressed_only THEN daily_uncompressed_size[n] += uncompressed_size[i]
	ENDIF
ENDFOR

i = where(daily_compressed_size NE 0, n)
IF n NE 0 THEN BEGIN
	tday = [1,1]#tunix[i]+[0,sec_per_day]#replicate(1,n)
	n = merge_ranges(tday,merge=tday)
ENDIF

InitVar, xysize, [8,6]*80, set=xysize_
xysize_ = long( xysize_*(is_eps ? 1.0/100.0*3 : 1) )
scale_factor = is_eps ? 10 : 1

IF ratio THEN BEGIN
	IF compressed_only THEN BEGIN
		message, /info, 'no full file size available'
		RETURN
	ENDIF

	yratio = double(daily_uncompressed_size)/double(daily_compressed_size)
	i = where(1-finite(yratio))
	IF i[0] NE -1 THEN yratio[i] = 0

	y = yratio[sort(yratio)]
	n = long(0.99*n_elements(yratio)) < (n_elements(yratio)-1)
	ymax = y[n]

	ytitle = 'Inflation ratio'
	yrange = [-0.05,1]*ceil( ymax )

	;yrange[0] >= 0
	;yrange[1] <= 6

	set_page, $
		/new, $
		file			= file		, $
		xysize			= xysize_	, $
		ctable  		= ctable	, $
		silent			= silent	, $
		scale_factor	= scale_factor, $
		_extra			= _extra

	TimeXAxis, TimeSet(unix=trange), $
		/noxtitle			, $
		silent	= silent	, $
		charsize= charsize	, $
		ymargin = [3, 5]	, $
		/exact				, $
		_extra	= _extra

	plotbars, tplot, yratio , $
		/oplot				, $
		/newyaxis			, $
		yrange	= yrange	, $
		low_margin= 0		, $
		ystyle	= 1			, $
		ytitle  = ytitle	, $
		silent	= silent	, $
		shade	= color[0]	, $
		/noline				, $
		charsize= charsize	, $
		_extra	= _extra

	IF NOT unlabeled THEN BEGIN
		;x0 = !x.crange
		y0 = !y.crange
		;dxval = 0.075*(x0[1]-x0[0])
		dyval = 0.075*(y0[1]-y0[0])
		xyouts, mean(!x.crange), !y.crange[1]+1.6*dyval, align=0.5, title + ' ('+ytitle+')', charsize=charsize, _extra=_extra
		xyouts, mean(!x.crange), !y.crange[1]+0.6*dyval, align=0.5, time_range, charsize=charsize, _extra=_extra
	ENDIF

ENDIF ELSE BEGIN

	IF compressed_only THEN BEGIN
		ymax = max( daily_compressed_size )
		ymax = (daily_compressed_size[sort(daily_compressed_size)])[long(0.99*n_elements(daily_compressed_size)) < (n_elements(daily_compressed_size)-1)]
		y = cdf ? total( daily_compressed_size ) : ymax
	ENDIF ELSE BEGIN
		ymax = max( daily_uncompressed_size )
		ymax = (daily_uncompressed_size[sort(daily_uncompressed_size)])[long(0.99*n_elements(daily_uncompressed_size)) < (n_elements(daily_uncompressed_size)-1)]
		y = cdf ? total( daily_uncompressed_size ) : ymax
	ENDELSE

	unit = floor(alog10(y))
	kunit = unit/3
	runit = unit-3*kunit
	post = (['','10 ','100 '])[runit]+(['','k','M','G','T','P','E','Z','Y'])[kunit]+'B'

	unit = 10.0^unit
	yrange = [-0.05,1]*ceil( y/unit )

	ytitle = 'Size, '+post+(cdf ? '' : ' per day')

	set_page, $
		_extra			= _extra	, $
		file			= file		, $
		xysize			= xysize_	, $
		ctable  		= ctable	, $
		/new						, $
		silent			= silent	, $
		scale_factor	= scale_factor

	TimeXAxis, TimeSet(unix=trange), $
		/noxtitle			, $
		silent	= silent	, $
		charsize= charsize	, $
		ymargin = [3, 5]	, $
		/exact				, $
		_extra	= _extra

	IF compressed_only THEN BEGIN

		plotbars, tplot, daily_compressed_size/unit , $
			cdf		= cdf		, $
			/oplot				, $
			/newyaxis			, $
			yrange	= yrange	, $
			low_margin= 0		, $
			ystyle	= 1			, $
			ytitle  = ytitle	, $
			silent	= silent	, $
			shade	= color[2*compressed_only]	, $
			/noline				, $
			charsize= charsize	, $
			_extra	= _extra

	ENDIF ELSE BEGIN

		plotbars, tplot, daily_uncompressed_size/unit , $
			cdf		= cdf		, $
			/oplot				, $
			/newyaxis			, $
			yrange	= yrange	, $
			low_margin= 0		, $
			ystyle	= 1			, $
			ytitle  = ytitle	, $
			silent	= silent	, $
			shade	= color[2*compressed_only]	, $
			/noline				, $
			charsize= charsize	, $
			_extra	= _extra

		plotbars, tplot, daily_compressed_size/unit, $
			cdf		= cdf		, $
			/oplot				, $
			low_margin= 0		, $
			silent	= silent	, $
			shade	= color[2]	, $
			/noline				, $
			_extra	= _extra

	ENDELSE

	IF NOT unlabeled THEN BEGIN
		x0 = !x.crange
		y0 = !y.crange
		dxval = 0.075*(x0[1]-x0[0])
		dyval = 0.075*(y0[1]-y0[0])

		x = x0[0]+dxval+[0,1,1,0]*dxval
		y = y0[1]+(-0.10+0.02*[0,0,1,1])*(y0[1]-y0[0])
		IF NOT compressed_only THEN BEGIN
			polyfill, x, y, color=color[0]
			xyouts, x[1]+15,y[1], 'uncompressed', charsize=charsize, _extra=_extra

			y -= 0.05*(y0[1]-y0[0])
			polyfill, x, y, color=color[2]
			xyouts, x[1]+15,y[1], 'compressed', charsize=charsize, _extra=_extra
		ENDIF

		xyouts, mean(!x.crange), !y.crange[1]+1.6*dyval, align=0.5, title+' ('+ytitle+')', charsize=charsize, _extra=_extra
		xyouts, mean(!x.crange), !y.crange[1]+0.6*dyval, align=0.5, time_range, charsize=charsize, _extra=_extra
	ENDIF

ENDELSE

get_page, $
	silent	= silent	, $
	_extra	= _extra

RETURN  &  END