From 44507563e15908287e0b23c2010a7d2facd0ac40 Mon Sep 17 00:00:00 2001
From: Remco Kok <rdkok@int3.local.snellius.surf.nl>
Date: Tue, 16 Nov 2021 15:15:26 +0100
Subject: [PATCH] Added snellius platform file and changed obs_gvp_co2 to
read-only.
---
da/observations/obs_gvp_co2.py | 2 +-
da/platform/snellius.py | 183 +++++++++++++++++++++++++++++++++
2 files changed, 184 insertions(+), 1 deletion(-)
create mode 100644 da/platform/snellius.py
diff --git a/da/observations/obs_gvp_co2.py b/da/observations/obs_gvp_co2.py
index 0701b728..6bb25dcf 100755
--- a/da/observations/obs_gvp_co2.py
+++ b/da/observations/obs_gvp_co2.py
@@ -95,7 +95,7 @@ class ObsPackObservations(Observations):
# Step 1: Read list of available site files in package
infile = os.path.join(self.obspack_dir, 'summary', '%s_dataset_summary.txt' % (self.obspack_id,))
- f = open(infile, 'r+', encoding="utf-8")
+ f = open(infile, 'r', encoding="utf-8")
lines = f.readlines()
f.close()
diff --git a/da/platform/snellius.py b/da/platform/snellius.py
new file mode 100644
index 00000000..4ef04249
--- /dev/null
+++ b/da/platform/snellius.py
@@ -0,0 +1,183 @@
+"""CarbonTracker Data Assimilation Shell (CTDAS) Copyright (C) 2017 Wouter Peters.
+Users are recommended to contact the developers (wouter.peters@wur.nl) to receive
+updates of the code. See also: http://www.carbontracker.eu.
+
+This program is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software Foundation,
+version 3. This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this
+program. If not, see <http://www.gnu.org/licenses/>."""
+#!/usr/bin/env python
+# cartesius.py
+
+"""
+Author : peters
+
+Revision History:
+File created on 06 Sep 2010.
+
+"""
+
+import logging
+import subprocess
+
+from da.platform.platform_baseclass import Platform
+
+std_joboptions = {'jobname':'test', 'jobaccount':'co2', 'jobtype':'serial', 'jobshell':'/bin/sh', 'depends':'', 'jobtime':'24:00:00', 'jobinput':'/dev/null', 'jobnodes':'1', 'jobtasks':'', 'modulenetcdf':'netCDF/4.8.0-iimpi-2021a', 'networkMPI':'','jobqueue': 'thin'}
+
+
+class SnelliusPlatform(Platform):
+ def __init__(self):
+ self.ID = 'snellius' # the identifier gives the platform name
+ self.version = '1.0' # the platform version used
+
+
+ def give_blocking_flag(self):
+ """
+ Returns a blocking flag, which is important if tm5 is submitted in a queue system. The python ctdas code is forced to wait before tm5 run is finished
+
+ -on Huygens: return "-s"
+ -on Maunaloa: return "" (no queue available)
+ -on Jet/Zeus: return
+ """
+ return ""
+
+ def give_queue_type(self):
+ """
+ Return a queue type depending whether your computer system has a queue system, or whether you prefer to run in the foreground.
+ On most large systems using the queue is mandatory if you run a large job.
+ -on Huygens: return "queue"
+ -on Maunaloa: return "foreground" (no queue available)
+ -on Jet/Zeus: return
+
+ """
+ return "foreground"
+
+ def get_job_template(self, joboptions={}, block=False):
+ """
+ Returns the job template for a given computing system, and fill it with options from the dictionary provided as argument.
+ The job template should return the preamble of a job that can be submitted to a queue on your platform,
+ examples of popular queuing systems are:
+ - SGE
+ - MOAB
+ - XGrid
+ -
+
+ A list of job options can be passed through a dictionary, which are then filled in on the proper line,
+ an example is for instance passing the dictionary {'account':'co2'} which will be placed
+ after the ``-A`` flag in a ``qsub`` environment.
+
+ An extra option ``block`` has been added that allows the job template to be configured to block the current
+ job until the submitted job in this template has been completed fully.
+ """
+
+ #template = """## \n"""+ \
+ # """## This is a set of dummy names, to be replaced by values from the dictionary \n"""+ \
+ # """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """+ \
+ # """## \n"""+ \
+ # """ \n"""+ \
+ # """#$ jobname \n"""+ \
+ # """#$ jobaccount \n"""+ \
+ # """#$ jobnodes \n"""+ \
+ # """#$ jobtime \n"""+ \
+ # """#$ jobshell \n"""+ \
+ # """\n"""+ \
+ # """source /usr/bin/sh\n"""+ \
+ # """module load python\n"""+ \
+ # """\n"""
+
+
+ template = """#!/bin/bash \n""" + \
+ """## \n""" + \
+ """## This is a set of dummy names, to be replaced by values from the dictionary \n""" + \
+ """## Please make your own platform specific template with your own keys and place it in a subfolder of the da package.\n """ + \
+ """## \n""" + \
+ """#SBATCH -J jobname \n""" + \
+ """#SBATCH -p jobqueue \n""" + \
+ """#SBATCH -n jobnodes \n""" + \
+ """#SBATCH -t jobtime \n""" + \
+ """#SBATCH -o joblog \n""" + \
+ """module load ctdas2022 \n""" + \
+ """\n"""
+
+ if 'depends' in joboptions:
+ template += """#$ -hold_jid depends \n"""
+
+ # First replace from passed dictionary
+ for k, v in list(joboptions.items()):
+ while k in template:
+ template = template.replace(k, v)
+
+ # Fill remaining values with std_options
+ for k, v in list(std_joboptions.items()):
+ while k in template:
+ template = template.replace(k, v)
+
+ return template
+
+
+ def submit_job(self, jobfile, joblog=None, block=False):
+ """ This method submits a jobfile to the queue, and returns the queue ID """
+
+
+ #cmd = ["llsubmit","-s",jobfile]
+ #msg = "A new task will be started (%s)"%cmd ; logging.info(msg)
+
+ if block:
+ cmd = ["salloc",'-n',std_joboptions['jobnodes'],'-t',std_joboptions['jobtime'], jobfile]
+ logging.info("A new task will be started (%s)" % cmd)
+ output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
+ logging.info(output)
+ print(('output', output))
+ jobid = output.split()[-1]
+ print(('jobid', jobid))
+ else:
+ cmd = ["sbatch", jobfile]
+ logging.info("A new job will be submitted (%s)" % cmd)
+ output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] ; logging.info(output)
+ jobid = output.split()[-1]
+
+ return jobid
+
+
+
+
+
+
+
+# jobid = output.split()[2]
+# retcode = output.split()[-1]
+#
+# #for huygens
+# print 'output', output
+# test = output.split()[3]
+# dummy, jobid =test.split('nl.')
+# jobid='%s%s' %('"',jobid)
+# submitmsg ='%s%s%s'%(output.split()[4],output.split()[5],output.split()[6])
+# if submitmsg=='hasbeensubmitted.':
+# retcode=2
+# print 'retcode',submitmsg,retcode
+# return retcode
+#
+# def KillJob(self,jobid):
+# """ This method kills a running job """
+#
+# output = subprocess.Popen(['qdel',jobid], stdout=subprocess.PIPE).communicate()[0] ; logging.info(output)
+#
+# return output
+#
+# def StatJob(self,jobid):
+# """ This method gets the status of a running job """
+# import subprocess
+#
+# #output = subprocess.Popen(['sgestat'], stdout=subprocess.PIPE).communicate()[0] ; logging.info(output)
+#
+# return ''
+
+
+
+if __name__ == "__main__":
+ pass
--
GitLab