new routines for time averaging of cycle files

gridded/da/analysis/daily_fluxes.py

+#!/usr/bin/env python
+# daily_fluxes.py
+
+"""
+Author : peters 
+
+Revision History:
+File created on 20 Dec 2012.
+
+"""
+import sys
+sys.path.append('../../')
+import os
+import sys
+import shutil
+import datetime
+
+def daily_avg(rundir,avg):
+    """ Function to create a set of daily files in a folder, needed to make longer term means """
+    
+    if avg not in ['transcom','olson']:
+        raise IOError,'Choice of averaging invalid'
+    if no os.path.exists(rundir):
+        raise IOError,'rundir requested (%s) does not exist, exiting...'%rundir
+
+    weekdir = os.path.join(rundir + 'data_%s_weekly'%avg)
+    files  = os.listdir(weekdir)
+    
+    daydir = os.path.join(rundir + 'data_%s_daily'%avg)
+    if not os.path.exists(daydir):
+        print "Creating new output directory " + daydir
+        os.makedirs(daydir)
+
+    files = [f for f in files if '-' in f]
+
+    fileinfo = {}
+    for filename in files:
+        date=datetime.datetime.strptime(filename.split('.')[-2],'%Y-%m-%d')
+        fileinfo[filename] = date
+    
+    dt = fileinfo[files[1]] - fileinfo[files[0]]
+
+    for k,v in fileinfo.iteritems():
+        cycle_file = os.path.join(weekdir,k)
+        for i in range(dt.days):
+            daily_file = os.path.join(daydir,'%s_fluxes.%s.nc'%(avg,(v+datetime.timedelta(days=i)).strftime('%Y-%m-%d')))
+            if not os.path.lexists(daily_file):
+                os.symlink(cycle_file,daily_file)
+                print daily_file,cycle_file
+
+
+if __name__ == "__main__":
+
+    rundir  = "/Storage/CO2/ivar/zeus_output/ICDC9/ctdas_co2c13_terdisbcb_manscale/analysis/"
+
+    try:
+        avg=sys.argv[1]
+    except:
+        avg='transcom'
+
+    daily_avg(rundir,avg)
+

gridded/da/analysis/monthly_fluxes.py

+#!/usr/bin/env python
+# daily_fluxes.py
+
+"""
+Author : peters 
+
+Revision History:
+File created on 20 Dec 2012.
+
+"""
+
+import os
+import sys
+import datetime
+from dateutil.relativedelta import relativedelta
+import subprocess
+
+def monthly_avg(rundir,avg):
+    """ Function to average a set of files in a folder from daily to monthly means """
+    
+    if avg not in ['transcom','olson']:
+        raise IOError,'Choice of averaging invalid'
+    if no os.path.exists(rundir):
+        raise IOError,'rundir requested (%s) does not exist, exiting...'%rundir
+
+    daydir = rundir + 'data_%s_daily'%avg
+    files  = os.listdir(daydir)
+    monthdir = os.path.join(rundir,'data_%s_monthly'%avg)
+    if not os.path.exists(monthdir):
+        print "Creating new output directory " + monthdir
+        os.makedirs(monthdir)
+        sd = datetime.datetime(1999,1,1)
+    else: 
+        files_monthly = os.listdir(monthdir)
+        files_monthly.sort()
+        fm = files_monthly[-1]
+        #sd = datetime.datetime(int(fm[9:13]),int(fm[14:16]),1)
+        sd = datetime.datetime.strptime(fm.split('.')[-2],'%Y-%m')
+    print sd
+            
+    files = [f for f in files if '-' in f]
+
+    fileinfo = {}
+    for filename in files:
+        date=datetime.datetime.strptime(filename.split('.')[-2],'%Y-%m-%d')
+        fileinfo[filename] = date
+
+    years = [d.year for d in fileinfo.values()]
+    months = set([d.month for d in fileinfo.values()])
+    
+    if sd > datetime.datetime(1999,2,1):
+        sd = sd + relativedelta(months=+1)
+    else: sd = datetime.datetime(min(years),1,1)
+    ed = datetime.datetime(max(years)+1,1,1)
+
+    while sd < ed: 
+
+        nd = sd + relativedelta(months=+1)
+        
+        avg_files = [os.path.join(daydir,k) for k,v in fileinfo.iteritems() if v < nd and v >= sd]
+        print sd, nd , len(avg_files)
+       
+        if len(avg_files) > 0: 
+            command = ['ncra']+ avg_files + [os.path.join(monthdir,'%s_fluxes.%s.nc'%(avg,sd.strftime('%Y-%m')))]
+        
+            status = subprocess.check_call(command)
+
+        sd = nd
+
+if __name__ == "__main__":
+
+    rundir = "/Storage/CO2/ivar/zeus_output/ICDC9/ctdas_co2c13_terdisbcbmanscale_neweps449_test/analysis/"
+    try:
+        avg=sys.argv[1]
+    except:
+        avg='transcom'

gridded/da/analysis/yearly_fluxes.py

+#!/usr/bin/env python
+# daily_fluxes.py
+
+"""
+Author : peters 
+
+Revision History:
+File created on 20 Dec 2012.
+
+"""
+
+import os
+import sys
+import datetime
+from dateutil.relativedelta import relativedelta
+import subprocess
+
+def yearly_avg(rundir,avg):
+    """ Function to average a set of files in a folder from monthly to yearly means """
+
+    if avg not in ['transcom','olson']:
+        raise IOError,'Choice of averaging invalid'
+    if no os.path.exists(rundir):
+        raise IOError,'rundir requested (%s) does not exist, exiting...'%rundir
+
+    monthdir = rundir + 'data_%s_monthly'%avg
+    files  = os.listdir(monthdir)
+    yeardir = os.path.join(rundir,'data_%s_yearly'%avg)
+    if not os.path.exists(yeardir):
+        print "Creating new output directory " + yeardir
+        os.makedirs(yeardir)
+        sd = datetime.datetime(1999,1,1)
+    else: 
+        files_yearly = os.listdir(monthdir)
+        files_yearly.sort()
+            
+    files = [f for f in files if '-' in f]
+
+    fileinfo = {}
+    for filename in files:
+        date=datetime.datetime.strptime(filename.split('.')[-2],'%Y-%m')
+        fileinfo[filename] = date
+
+    years = set([d.year for d in fileinfo.values()])
+    
+    sd = datetime.datetime(min(years),1,1)
+    ed = datetime.datetime(max(years)+1,1,1)
+
+    while sd < ed: 
+
+        nd = sd + relativedelta(years=+1)
+        
+        avg_files = [os.path.join(monthdir,k) for k,v in fileinfo.iteritems() if v < nd and v >= sd]
+        print sd, nd , len(avg_files)
+       
+        if len(avg_files) > 0 : 
+            command = ['ncra']+ avg_files + [os.path.join(yeardir,'%s_fluxes.%s.nc'%(avg,sd.strftime('%Y')))]
+        
+            status = subprocess.check_call(command)
+
+        sd = nd
+
+
+
+if __name__ == "__main__":
+
+    
+    rundir = "/Storage/CO2/ivar/zeus_output/ICDC9/ctdas_co2c13_terdisbcbmanscale_neweps449_test/analysis/"
+
+    try:
+        avg=sys.argv[1]
+    except:
+        avg='transcom'
+
+    yearly_avg(rundir,avg)
+