Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
CTDAS
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Container registry
Model registry
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Tsurata, Aki
CTDAS
Commits
fc26c338
Commit
fc26c338
authored
13 years ago
by
Peters, Wouter
Browse files
Options
Downloads
Patches
Plain Diff
replaced dummy variable by status, can be handled if ever needed
parent
5dac45bf
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
da/tools/pipeline.py
+36
-36
36 additions, 36 deletions
da/tools/pipeline.py
with
36 additions
and
36 deletions
da/tools/pipeline.py
+
36
−
36
View file @
fc26c338
...
...
@@ -34,27 +34,27 @@ def EnsembleSmootherPipeline(DaCycle,PlatForm, DaSystem, Samples,StateVector,Obs
msg
=
header
+
"
Initializing current cycle
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
JobStart
(
DaCycle
,
DaSystem
,
PlatForm
,
StateVector
,
Samples
,
ObsOperator
)
status
=
JobStart
(
DaCycle
,
DaSystem
,
PlatForm
,
StateVector
,
Samples
,
ObsOperator
)
msg
=
header
+
"
starting PrepareState
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
PrepareState
(
DaCycle
,
StateVector
)
status
=
PrepareState
(
DaCycle
,
StateVector
)
msg
=
header
+
"
starting SampleState
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
SampleState
(
DaCycle
,
Samples
,
StateVector
,
ObsOperator
)
status
=
SampleState
(
DaCycle
,
Samples
,
StateVector
,
ObsOperator
)
msg
=
header
+
"
starting Invert
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
Invert
(
DaCycle
,
StateVector
,
Optimizer
)
status
=
Invert
(
DaCycle
,
StateVector
,
Optimizer
)
msg
=
header
+
"
starting Advance
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
Advance
(
DaCycle
,
Samples
,
StateVector
,
ObsOperator
)
status
=
Advance
(
DaCycle
,
Samples
,
StateVector
,
ObsOperator
)
msg
=
header
+
"
starting SaveAndSubmit
"
+
footer
;
logging
.
info
(
msg
)
dummy
=
SaveAndSubmit
(
DaCycle
,
StateVector
)
status
=
SaveAndSubmit
(
DaCycle
,
StateVector
)
msg
=
"
Cycle finished...exiting pipeline
"
;
logging
.
info
(
msg
)
...
...
@@ -63,15 +63,15 @@ def EnsembleSmootherPipeline(DaCycle,PlatForm, DaSystem, Samples,StateVector,Obs
def
JobStart
(
DaCycle
,
DaSystem
,
DaPlatForm
,
StateVector
,
Samples
,
ObsOperator
):
"""
Set up the job specific directory structure and create an expanded rc-file
"""
dummy
=
DaSystem
.
Initialize
()
status
=
DaSystem
.
Initialize
()
dummy
=
DaSystem
.
Validate
()
status
=
DaSystem
.
Validate
()
DaCycle
.
DaSystem
=
DaSystem
DaCycle
.
DaPlatForm
=
DaPlatForm
dummy
=
DaCycle
.
Initialize
()
status
=
DaCycle
.
Initialize
()
StateVector
.
DaCycle
=
DaCycle
# also embed object in StateVector so it can access cycle information for I/O etc
...
...
@@ -90,7 +90,7 @@ def PrepareState(DaCycle, StateVector):
# with new values for each week. After we have constructed the StateVector, it will be propagated by one cycle length so it is ready to be used
# in the current cycle
dummy
=
StateVector
.
Initialize
()
status
=
StateVector
.
Initialize
()
if
not
DaCycle
[
'
time.restart
'
]:
...
...
@@ -100,7 +100,7 @@ def PrepareState(DaCycle, StateVector):
for
n
in
range
(
0
,
nlag
):
date
=
DaCycle
[
'
time.start
'
]
+
datetime
.
timedelta
(
days
=
(
n
+
0.5
)
*
int
(
DaCycle
[
'
time.cycle
'
]))
cov
=
StateVector
.
GetCovariance
(
date
)
dummy
=
StateVector
.
MakeNewEnsemble
(
n
+
1
,
cov
)
status
=
StateVector
.
MakeNewEnsemble
(
n
+
1
,
cov
)
else
:
...
...
@@ -114,14 +114,14 @@ def PrepareState(DaCycle, StateVector):
# Now propagate the ensemble by one cycle to prepare for the current cycle
dummy
=
StateVector
.
Propagate
()
status
=
StateVector
.
Propagate
()
# Finally, also write the StateVector to a file so that we can always access the a-priori information
savedir
=
DaCycle
[
'
dir.output
'
]
filename
=
os
.
path
.
join
(
savedir
,
'
savestate.nc
'
)
dummy
=
StateVector
.
WriteToFile
(
filename
)
# write prior info because StateVector.Isoptimized == False for now
status
=
StateVector
.
WriteToFile
(
filename
)
# write prior info because StateVector.Isoptimized == False for now
return
None
...
...
@@ -134,7 +134,7 @@ def SampleState(DaCycle,Samples,StateVector, ObservationOperator):
# (i) The transport model restart data which holds the background mixing ratios. This is needed to run the model one cycle forward
# (ii) The random numbers (or the seed for the random number generator) so we can recreate the ensembles if needed
#
dummy
= DaCycle.MoveSaveData(io_option='store',save_option='partial',filter=[])
#
status
= DaCycle.MoveSaveData(io_option='store',save_option='partial',filter=[])
#msg = "All restart data have been copied to the save/tmp directory for future use" ; logging.debug(msg)
nlag
=
int
(
DaCycle
[
'
time.nlag
'
])
...
...
@@ -144,7 +144,7 @@ def SampleState(DaCycle,Samples,StateVector, ObservationOperator):
msg
=
header
+
"
.....Ensemble Kalman Filter at lag %d
"
%
(
int
(
lag
)
+
1
,)
;
logging
.
info
(
msg
)
dummy
=
SampleOneCycle
(
DaCycle
,
Samples
,
StateVector
,
ObservationOperator
,
lag
)
status
=
SampleOneCycle
(
DaCycle
,
Samples
,
StateVector
,
ObservationOperator
,
lag
)
# Optionally, post-processing of the model output can be added that deals for instance with
...
...
@@ -160,9 +160,9 @@ def SampleOneCycle(DaCycle,Samples,StateVector, ObservationOperator,lag):
if
lag
==
0
:
dummy
=
ObservationOperator
.
GetInitialData
()
status
=
ObservationOperator
.
GetInitialData
()
dummy
=
DaCycle
.
SetSampleTimes
(
lag
)
status
=
DaCycle
.
SetSampleTimes
(
lag
)
startdate
=
DaCycle
[
'
time.sample.start
'
]
enddate
=
DaCycle
[
'
time.sample.end
'
]
...
...
@@ -178,13 +178,13 @@ def SampleOneCycle(DaCycle,Samples,StateVector, ObservationOperator,lag):
# Implement something that writes the ensemble member parameter info to file, or manipulates them further into the
# type of info needed in your transport model
dummy
=
StateVector
.
WriteMembersToFile
(
lag
+
1
)
status
=
StateVector
.
WriteMembersToFile
(
lag
+
1
)
dummy
=
Samples
.
Initialize
()
status
=
Samples
.
Initialize
()
dummy
=
Samples
.
Validate
()
status
=
Samples
.
Validate
()
dummy
=
Samples
.
AddObs
()
status
=
Samples
.
AddObs
()
filename
=
Samples
.
WriteSampleInfo
()
...
...
@@ -194,7 +194,7 @@ def SampleOneCycle(DaCycle,Samples,StateVector, ObservationOperator,lag):
# Run the observation operator
dummy
=
RunForecastModel
(
DaCycle
,
ObservationOperator
)
status
=
RunForecastModel
(
DaCycle
,
ObservationOperator
)
# Add the observation operator restart+output files to the general Output+RestartFileList, only after 'Advance'
# Same logic for observation files
...
...
@@ -209,7 +209,7 @@ def SampleOneCycle(DaCycle,Samples,StateVector, ObservationOperator,lag):
# Add model-data mismatch to all samples, this *might* use output from the ensemble in the future??
dummy
=
Samples
.
AddModelDataMismatch
()
status
=
Samples
.
AddModelDataMismatch
()
msg
=
"
Added Model Data Mismatch to all samples
"
;
logging
.
debug
(
msg
)
...
...
@@ -222,7 +222,7 @@ def SampleOneCycle(DaCycle,Samples,StateVector, ObservationOperator,lag):
# one file per member, some logic needs to be included to merge all files!!!
filename
=
os
.
path
.
join
(
ObservationOperator
.
outputdir
,
'
flask_output.%s.nc
'
%
DaCycle
[
'
time.sample.stamp
'
])
dummy
=
Samples
.
AddSimulations
(
filename
)
status
=
Samples
.
AddSimulations
(
filename
)
# Give each member a model sample by first copying all samples, and then selecting the data for Member #n
...
...
@@ -253,29 +253,29 @@ def Invert(DaCycle, StateVector, Optimizer ):
int
(
DaCycle
.
DaSystem
[
'
nparameters
'
]),
StateVector
.
nobs
,
)
dummy
=
Optimizer
.
Initialize
(
dims
)
status
=
Optimizer
.
Initialize
(
dims
)
dummy
=
Optimizer
.
StateToMatrix
(
StateVector
)
status
=
Optimizer
.
StateToMatrix
(
StateVector
)
dummy
=
Optimizer
.
WriteDiagnostics
(
DaCycle
,
StateVector
,
type
=
'
prior
'
)
status
=
Optimizer
.
WriteDiagnostics
(
DaCycle
,
StateVector
,
type
=
'
prior
'
)
dummy
=
Optimizer
.
SetLocalization
(
'
None
'
)
status
=
Optimizer
.
SetLocalization
(
'
None
'
)
if
not
DaCycle
.
DaSystem
.
has_key
(
'
opt.algorithm
'
):
msg
=
"
There was no minimum least squares algorithm specified in the DA System rc file (key : opt.algorithm)
"
;
logging
.
info
(
msg
)
msg
=
"
...using serial algorithm as default...
"
;
logging
.
info
(
msg
)
dummy
=
Optimizer
.
SerialMinimumLeastSquares
()
status
=
Optimizer
.
SerialMinimumLeastSquares
()
elif
DaCycle
.
DaSystem
[
'
opt.algorithm
'
]
==
'
serial
'
:
msg
=
"
Using the serial minimum least squares algorithm to solve ENKF equations
"
;
logging
.
info
(
msg
)
dummy
=
Optimizer
.
SerialMinimumLeastSquares
()
status
=
Optimizer
.
SerialMinimumLeastSquares
()
elif
DaCycle
.
DaSystem
[
'
opt.algorithm
'
]
==
'
bulk
'
:
msg
=
"
Using the bulk minimum least squares algorithm to solve ENKF equations
"
;
logging
.
info
(
msg
)
dummy
=
Optimizer
.
BulkMinimumLeastSquares
()
status
=
Optimizer
.
BulkMinimumLeastSquares
()
dummy
=
Optimizer
.
MatrixToState
(
StateVector
)
status
=
Optimizer
.
MatrixToState
(
StateVector
)
dummy
=
Optimizer
.
WriteDiagnostics
(
DaCycle
,
StateVector
,
type
=
'
optimized
'
)
status
=
Optimizer
.
WriteDiagnostics
(
DaCycle
,
StateVector
,
type
=
'
optimized
'
)
StateVector
.
isOptimized
=
True
...
...
@@ -300,12 +300,12 @@ def SaveAndSubmit( DaCycle, StateVector):
savedir
=
DaCycle
[
'
dir.output
'
]
filename
=
os
.
path
.
join
(
savedir
,
'
savestate.nc
'
)
dummy
=
StateVector
.
WriteToFile
(
filename
)
status
=
StateVector
.
WriteToFile
(
filename
)
DaCycle
.
RestartFileList
.
extend
(
[
filename
]
)
# write optimized info because StateVector.Isoptimized == False for now
dummy
=
DaCycle
.
Finalize
()
status
=
DaCycle
.
Finalize
()
return
None
...
...
@@ -331,7 +331,7 @@ def RunForecastModel(DaCycle,ObsOperator):
status
=
ObsOperator
.
Run
()
dummy
=
ObsOperator
.
SaveData
()
status
=
ObsOperator
.
SaveData
()
return
status
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment