working version 11. june

da/cosmo/base_optimizer.py

@@ -49,8 +49,8 @@ class Optimizer(object):
     def setup(self, dims):
         self.nlag = dims[0]
         self.nmembers = dims[1]
-#        self.nparams = dims[2]
-        self.nparams = 181
+        self.nparams = dims[2]
+#        self.nparams = 181
         self.nobs = dims[3]
         self.create_matrices()
 
@@ -80,6 +80,7 @@ class Optimizer(object):
         # localization of obs
         self.may_localize = np.zeros(self.nobs, bool)
         # rejection of obs
+        #self.may_reject = np.ones(self.nobs, bool)
         self.may_reject = np.zeros(self.nobs, bool)
         # flags of obs
         self.flags = np.zeros(self.nobs, int)
@@ -314,7 +315,6 @@ class Optimizer(object):
         for n in range(self.nobs):
 
             # Screen for flagged observations (for instance site not found, or no sample written from model)
-
             if self.flags[n] != 0:
                 logging.debug('Skipping observation (%s,%i) because of flag value %d' % (self.sitecode[n], self.obs_ids[n], self.flags[n]))
                 continue
@@ -324,13 +324,12 @@ class Optimizer(object):
             res = self.obs[n] - self.Hx[n]
 
             if self.may_reject[n]:
+                print('may reject')
                 threshold = self.rejection_threshold * np.sqrt(self.R[n])
                 if np.abs(res) > threshold:
                     logging.debug('Rejecting observation (%s,%i) because residual (%f) exceeds threshold (%f)' % (self.sitecode[n], self.obs_ids[n], res, threshold))
                     self.flags[n] = 2
                     continue
-                else:
-                    logging.debug('Taking observation (%s,%i) as residual (%f) doesnt exceed threshold (%f)' % (self.sitecode[n], self.obs_ids[n], res, threshold))
 
             logging.debug('Proceeding to assimilate observation %s, %i' % (self.sitecode[n], self.obs_ids[n]))
 

da/cosmo/covariances.py

@@ -29,6 +29,7 @@ import logging
 import numpy as np
 #from da.cosmo.statevector_uniform import StateVector, EnsembleMember
 #from da.cosmo.statevector_read_from_output import StateVector, EnsembleMember
+#from da.cosmo.statevector_mean import StateVector, EnsembleMember
 from da.cosmo.statevector import StateVector, EnsembleMember
 
 import da.tools.io4 as io

da/cosmo/pipeline.py

@@ -376,7 +376,8 @@ def invert(dacycle, statevector, optimizer):
     logging.info(header + "starting invert" + footer)
     dims = (int(dacycle['time.nlag']),
                   int(dacycle['da.optimizer.nmembers']),
-                  int(dacycle.dasystem['nparameters']),
+                  int(dacycle['nparameters']),
+                  #int(dacycle.dasystem['nparameters']),
                   statevector.nobs)
 
     if 'opt.algorithm' not in dacycle.dasystem:

da/cosmo/statevector.py

@@ -292,7 +292,8 @@ class StateVector(object):
  #           rands_bg = np.random.uniform(low=-0.05, high=0.05, size=1)
 #            rands = np.random.randn(self.nparams-1)
  #           rands_bg = np.random.randn(1)*1E-4     # x variance(1E-4)
-            rands = np.random.normal(loc=0.0, scale=1, size=self.nparams-1)
+#            rands = np.random.normal(loc=0.0, scale=0.4, size=self.nparams-1)
+            rands = np.random.normal(loc=0.0, scale=1000, size=self.nparams-1)
             #rands = np.random.normal(loc=0.0, scale=0.7, size=self.nparams-1)
             rands_bg = np.random.normal(loc=0.0, scale=0.05, size=1)  #*1E-4     # x variance(1E-4)
            # rands_bg = np.random.normal(loc=0.0, scale=1E-4, size=1)  #*1E-4     # x variance(1E-4)

da/cosmo/statevector_read_from_output.py

@@ -293,9 +293,10 @@ class StateVector(object):
         ifile = Dataset('/scratch/snx3000/parsenov/octe/optimizer.20130401.nc', mode='r')
 #        par = ifile.variables['lambda'][:]
         par = ifile.variables['statevectordeviations_prior'][:]
-        par = par.reshape(181,2,21)
+        par = par.reshape(2,181,21) + 1
+        #par = par.reshape(181,2,21) + 1
+#        par = par + np.ones(shape=(181,2,21))
 #        par = par + np.ones(shape=(2,40,181))
-        par = par + np.ones(shape=(181,2,21))
         # Create members 1:nmembers and add to ensemble_members list
         for member in range(1, self.nmembers):
 #            rands = np.random.uniform(low=-1., high=1., size=self.nparams-1)
@@ -310,7 +311,7 @@ class StateVector(object):
    #         newmember.param_values = (np.hstack((randsC.flatten(),rands_bg)) + newmean).ravel()    #THIS ONE
     #        newmember.param_values[newmember.param_values<0.] = 0.
      #       newmember.param_values[newmember.param_values>2.] = 2.
-            newmember.param_values = par[:, lag, member]
+            newmember.param_values = par[lag, :, member]
 #            newmember.param_values = par[lag, member, :]
 #            if member==1:
 #                print(par[:, lag, member])

template.py

@@ -36,6 +36,7 @@ from da.cosmo.obspack_globalviewplus2 import ObsPackObservations
 from da.cosmo.optimizer import CO2Optimizer
 #from da.cosmo.observationoperator_parallel import ObservationOperator   # does not fully work
 from da.cosmo.observationoperator_octe import ObservationOperator
+#from da.cosmo.observationoperator_read_from_files import ObservationOperator
 #from da.cosmo.observationoperator import ObservationOperator
 #from da.cosmo.expand_fluxes import save_weekly_avg_1x1_data, save_weekly_avg_state_data, save_weekly_avg_tc_data, save_weekly_avg_ext_tc_data
 #from da.analysis.expand_molefractions import write_mole_fractions