Merge branch 'release/0.4.0'

c2682042 · Nauta, Lisanne · 77176b1c · 7ad5fc67 · c2682042 · c2682042
Commit c2682042 authored May 12, 2022 by Nauta, Lisanne
--- a/.Rprofile
+++ b/.Rprofile
+source("renv/activate.R")
--- a/.dockerignore
+++ b/.dockerignore
+node_modules
+soil_moisture_model/renv/library
+soil_moisture_model/renv/staging
\ No newline at end of file
--- a/.gitignore
+++ b/.gitignore
 node_modules
 *.env
 .vscode
-dist
\ No newline at end of file
+dist
+.Rproj.user
\ No newline at end of file
--- a/Dockerfile
+++ b/Dockerfile
-FROM node:12-buster
+FROM ubuntu:20.04 
+
 # instal node package dependencies
 RUN apt update
-RUN apt-get install -y libgdal-dev
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository ppa:ubuntugis/ppa
+# RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/'
+# intall R and udinits2
+RUN apt-get install -y r-base libudunits2-dev
+# install the ubuntu gis things
+RUN apt-get install -y gdal-bin libgdal-dev libgeos-dev libproj-dev
+# install for postgres
+RUN apt-get install -y libpq-dev postgis
+# install node and npm
+RUN apt-get install -y nodejs npm
 # Create app directory, this is in our container/in our image
-WORKDIR /waterapps/src/app
+WORKDIR /farmer-support-api/src/app

 # Install app dependencies
 # A wildcard is used to ensure both package.json AND package-lock.json are copied
@@ -12,7 +23,7 @@ COPY package*.json ./

 # install gdal seperately because of libgdal version mismatch in node gdal.
 RUN npm install
-RUN npm install gdal --build-from-source --shared_gdal
+# RUN npm install gdal --build-from-source --shared_gdal
 # If you are building your code for production
 # RUN npm ci --only=production

@@ -21,6 +32,11 @@ COPY . .

 RUN npm run build

+RUN Rscript -e "install.packages('renv',repos='http://cran.rstudio.com/')"
+RUN Rscript -e "renv::activate('./soil_moisture_model');renv::restore('./soil_moisture_model')"
+
+
 EXPOSE 3000
+ENV NODE_ENV=dev
 CMD [ "node", "dist/main" ]

--- a/package-lock.json
+++ b/package-lock.json
 {
  "name": "api",
-  "version": "0.0.1",
+  "version": "0.3.1",
  "lockfileVersion": 1,
  "requires": true,
  "dependencies": {
@@ -1577,6 +1577,12 @@
        "@types/range-parser": "*"
      }
    },
+    "@types/geojson": {
+      "version": "7946.0.8",
+      "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.8.tgz",
+      "integrity": "sha512-1rkryxURpr6aWP7R786/UQOkJ3PcpQiWkAXBmdWc7ryFWqN6a4xfK7BtjXvFBKO9LjQ+MWQSWxYeZX1OApnArA==",
+      "dev": true
+    },
    "@types/graceful-fs": {
      "version": "4.1.5",
      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz",

--- a/package.json
+++ b/package.json
 {
  "name": "api",
-  "version": "0.3.1",
+  "version": "0.4.0",
  "description": "",
  "author": "lisanne.nauta@wur.nl",
  "private": true,
@@ -10,8 +10,9 @@
    "build": "nest build",
    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
    "start": "nest start",
+    "start:dev:nowatch": "NODE_ENV=dev nest start",
    "start:dev": "NODE_ENV=dev nest start --watch",
-    "start:debug": "nest start --debug --watch",
+    "start:debug": "NODE_ENV=dev nest start --debug --watch",
    "start:stag": "NODE_ENV=stag node dist/main",
    "start:dev:azure-test": "NODE_ENV=azure-test nest start --watch",
    "start:prod": "NODE_ENV=prod node dist/main",
@@ -28,6 +29,7 @@
    "migration:generate:azure-test": "NODE_ENV=azure-test npm run typeorm -- migration:generate --config src/ormconfig -n",
    "migration:run:dev": "NODE_ENV=dev npm run typeorm -- migration:run --config src/ormconfig",
    "migration:run:azure-test": "NODE_ENV=azure-test npm run typeorm -- migration:run --config src/ormconfig",
+    "migration:run:azure-release": "NODE_ENV=azure-release npm run typeorm -- migration:run --config src/ormconfig",
    "migration:revert:dev": "NODE_ENV=dev npm run typeorm -- migration:revert --config src/ormconfig",
    "migration:revert:azure-test": "NODE_ENV=azure-test npm run typeorm -- migration:revert --config src/ormconfig"
  },
@@ -67,6 +69,7 @@
    "@nestjs/testing": "^8.0.0",
    "@types/cron": "^1.7.3",
    "@types/express": "^4.17.13",
+    "@types/geojson": "^7946.0.8",
    "@types/jest": "^27.0.1",
    "@types/node": "^16.0.0",
    "@types/passport-jwt": "^3.0.6",

--- a/releases/v0.4.0/pre_deployment.sh
+++ b/releases/v0.4.0/pre_deployment.sh
+#!/bin/bash
+
+d=`date +%Y%m%d`
+echo $d
+
+#) 1 backup production database
+
+pg_host=$1
+pg_user=$2
+release_name=v040
+release_db=waterapps_${release_name}
+backup_dir=/home/nauta/OneDriveWUR/projects/2021_wagrinova/data/databases/production/
+
+echo "Enter password for ${pg_host}"
+bash ./scripts/backup_database.sh ${pg_host} ${pg_user} ${backup_dir}
+
+echo "Enter password for ${pg_host}"
+read -p "Validate that backup has been created. Press enter to continue"
+
+# 2 create release database
+echo "Enter password for ${pg_host}"
+createdb --host=${pg_host} --username=postgres ${release_db}
+
+echo "Enter password for ${pg_host}"
+psql -a --host=${pg_host} --username=postgres -d ${release_db} -f ./scripts/enable_postgis.sql
+
+read -p "Validate that database has been created. Press enter to continue"
+
+#3 restore database
+echo "Enter password for ${pg_host}"
+bash ./scripts/restore_backup.sh ${pg_host} ${pg_user} ${release_db} ${backup_dir}/20220511_azure_waterapps.dump
+
+read -p "Validate that release database has been populated. Press enter to continue"
+
+# 4 run migrations
+npm run migration:run:azure-release 
+
+read -p "Validate that database migrations has been succeeded. Press enter to continue"
+
+# 5 upload raster data to soil database (requires gdal bindings)
+# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_fc /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/FC_HWSD.tif
+
+# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_pwp /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/PWP_HWSD.tif
+
+# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_tex_class /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/DROP_TEX_CLASS.tif
+
+read -p "Validate that raster data has been uploadded to the soil production database.Press enter to continue"
\ No newline at end of file
--- a/renv/.gitignore
+++ b/renv/.gitignore
+library/
+local/
+cellar/
+lock/
+python/
+staging/
--- a/renv/activate.R
+++ b/renv/activate.R
--- a/scripts/backup_database.sh
+++ b/scripts/backup_database.sh
+#!/bin/bash
+
+host=$1
+pg_user=$2
+backup_dir=$3
+d=`date +%Y%m%d`
+dump_file=${backup_dir}/${d}_azure_waterapps.dump
+
+pg_dump -Fc -v --host=${host} --username=${pg_user} --dbname=waterapps --schema=public -f ${dump_file}
\ No newline at end of file
--- a/scripts/enable_postgis.sql
+++ b/scripts/enable_postgis.sql
+CREATE EXTENSION IF NOT EXISTS postgis
+    SCHEMA public;
+
+CREATE EXTENSION IF NOT EXISTS postgis_raster
+    SCHEMA public;
\ No newline at end of file
--- a/scripts/restore_backup.sh
+++ b/scripts/restore_backup.sh
+#!/bin/bash
+
+host=$1
+pg_user=$2
+db=$3
+dump_file=$4
+
+pg_restore --dbname ${db} -U ${pg_user} -h ${host} ${dump_file} 
\ No newline at end of file
--- a/scripts/upload_raster.sh
+++ b/scripts/upload_raster.sh
+#!/bin/bash
+
+host=$1
+pg_user=$2
+db=$3
+table=$4
+raster_file=$5
+
+raster2pgsql -t 3600x3600 -d -C -M -I ${raster_file} ${table} | psql -d ${db} -U ${pg_user} -h ${host} -p 5432
\ No newline at end of file
--- a/soil_moisture_dry_run.sh
+++ b/soil_moisture_dry_run.sh
+#!/bin/bash
+# depends on GDAL and postgis installation. e.g. use conda-forge gdal
+
+fdt_str=$1
+pg_user=$2
+pg_password=$3
+pg_host=$4
+pg_port=$5
+pg_db=$6
+
+Rscript --vanilla -e "source('./soil_moisture_model/run.R');run('$1','$2','$3','$4','$5','$6')"
\ No newline at end of file
--- a/soil_moisture_model/renv.lock
+++ b/soil_moisture_model/renv.lock
+{
+  "R": {
+    "Version": "4.1.2",
+    "Repositories": [
+      {
+        "Name": "CRAN",
+        "URL": "https://cloud.r-project.org"
+      }
+    ]
+  },
+  "Packages": {
+    "DBI": {
+      "Package": "DBI",
+      "Version": "1.1.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "dcd1743af4336156873e3ce3c950b8b9",
+      "Requirements": []
+    },
+    "FNN": {
+      "Package": "FNN",
+      "Version": "1.1.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "b56998fff55e4a4b4860ad6e8c67e0f9",
+      "Requirements": []
+    },
+    "KernSmooth": {
+      "Package": "KernSmooth",
+      "Version": "2.23-20",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "8dcfa99b14c296bc9f1fd64d52fd3ce7",
+      "Requirements": []
+    },
+    "MASS": {
+      "Package": "MASS",
+      "Version": "7.3-54",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "0e59129db205112e3963904db67fd0dc",
+      "Requirements": []
+    },
+    "R.methodsS3": {
+      "Package": "R.methodsS3",
+      "Version": "1.8.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "4bf6453323755202d5909697b6f7c109",
+      "Requirements": []
+    },
+    "R.oo": {
+      "Package": "R.oo",
+      "Version": "1.24.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "5709328352717e2f0a9c012be8a97554",
+      "Requirements": [
+        "R.methodsS3"
+      ]
+    },
+    "R.utils": {
+      "Package": "R.utils",
+      "Version": "2.11.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "a7ecb8e60815c7a18648e84cd121b23a",
+      "Requirements": [
+        "R.methodsS3",
+        "R.oo"
+      ]
+    },
+    "R6": {
+      "Package": "R6",
+      "Version": "2.5.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "470851b6d5d0ac559e9d01bb352b4021",
+      "Requirements": []
+    },
+    "RPostgreSQL": {
+      "Package": "RPostgreSQL",
+      "Version": "0.7-3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "d1258c16d609c27a3ba022d50c054b34",
+      "Requirements": [
+        "DBI"
+      ]
+    },
+    "RPostgres": {
+      "Package": "RPostgres",
+      "Version": "1.4.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "bb757978bf74ef58557242b83e559a96",
+      "Requirements": [
+        "DBI",
+        "Rcpp",
+        "bit64",
+        "blob",
+        "hms",
+        "lubridate",
+        "plogr",
+        "withr"
+      ]
+    },
+    "Rcpp": {
+      "Package": "Rcpp",
+      "Version": "1.0.8.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "32e79b908fda56ee57fe518a8d37b864",
+      "Requirements": []
+    },
+    "abind": {
+      "Package": "abind",
+      "Version": "1.4-5",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "4f57884290cc75ab22f4af9e9d4ca862",
+      "Requirements": []
+    },
+    "bit": {
+      "Package": "bit",
+      "Version": "4.0.4",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "f36715f14d94678eea9933af927bc15d",
+      "Requirements": []
+    },
+    "bit64": {
+      "Package": "bit64",
+      "Version": "4.0.5",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "9fe98599ca456d6552421db0d6772d8f",
+      "Requirements": [
+        "bit"
+      ]
+    },
+    "blob": {
+      "Package": "blob",
+      "Version": "1.2.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "dc5f7a6598bb025d20d66bb758f12879",
+      "Requirements": [
+        "rlang",
+        "vctrs"
+      ]
+    },
+    "class": {
+      "Package": "class",
+      "Version": "7.3-19",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "1593b7beb067c8381c0d24e38bd778e0",
+      "Requirements": [
+        "MASS"
+      ]
+    },
+    "classInt": {
+      "Package": "classInt",
+      "Version": "0.4-3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "17bdfa3c51df4a6c82484d13b11fb380",
+      "Requirements": [
+        "KernSmooth",
+        "class",
+        "e1071"
+      ]
+    },
+    "cli": {
+      "Package": "cli",
+      "Version": "3.2.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "1bdb126893e9ce6aae50ad1d6fc32faf",
+      "Requirements": [
+        "glue"
+      ]
+    },
+    "codetools": {
+      "Package": "codetools",
+      "Version": "0.2-18",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "019388fc48e48b3da0d3a76ff94608a8",
+      "Requirements": []
+    },
+    "cpp11": {
+      "Package": "cpp11",
+      "Version": "0.4.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "fa53ce256cd280f468c080a58ea5ba8c",
+      "Requirements": []
+    },
+    "crayon": {
+      "Package": "crayon",
+      "Version": "1.5.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "8dc45fd8a1ee067a92b85ef274e66d6a",
+      "Requirements": []
+    },
+    "dplyr": {
+      "Package": "dplyr",
+      "Version": "1.0.8",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "ef47665e64228a17609d6df877bf86f2",
+      "Requirements": [
+        "R6",
+        "generics",
+        "glue",
+        "lifecycle",
+        "magrittr",
+        "pillar",
+        "rlang",
+        "tibble",
+        "tidyselect",
+        "vctrs"
+      ]
+    },
+    "e1071": {
+      "Package": "e1071",
+      "Version": "1.7-9",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "32885be243a29301c90d33db37c3aad8",
+      "Requirements": [
+        "class",
+        "proxy"
+      ]
+    },
+    "ellipsis": {
+      "Package": "ellipsis",
+      "Version": "0.3.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077",
+      "Requirements": [
+        "rlang"
+      ]
+    },
+    "fansi": {
+      "Package": "fansi",
+      "Version": "1.0.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "83a8afdbe71839506baa9f90eebad7ec",
+      "Requirements": []
+    },
+    "foreach": {
+      "Package": "foreach",
+      "Version": "1.5.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "618609b42c9406731ead03adf5379850",
+      "Requirements": [
+        "codetools",
+        "iterators"
+      ]
+    },
+    "gdalUtils": {
+      "Package": "gdalUtils",
+      "Version": "2.0.3.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "05ee75453c7c7cf6741a050af30927ba",
+      "Requirements": [
+        "R.utils",
+        "foreach",
+        "raster",
+        "rgdal",
+        "sp"
+      ]
+    },
+    "generics": {
+      "Package": "generics",
+      "Version": "0.1.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "177475892cf4a55865868527654a7741",
+      "Requirements": []
+    },
+    "glue": {
+      "Package": "glue",
+      "Version": "1.6.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "4f2596dfb05dac67b9dc558e5c6fba2e",
+      "Requirements": []
+    },
+    "gstat": {
+      "Package": "gstat",
+      "Version": "2.0-9",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "933584b85b80a3005af4be8220ff2c9f",
+      "Requirements": [
+        "FNN",
+        "lattice",
+        "sp",
+        "spacetime",
+        "zoo"
+      ]
+    },
+    "hms": {
+      "Package": "hms",
+      "Version": "1.1.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "5b8a2dd0fdbe2ab4f6081e6c7be6dfca",
+      "Requirements": [
+        "ellipsis",
+        "lifecycle",
+        "pkgconfig",
+        "rlang",
+        "vctrs"
+      ]
+    },
+    "intervals": {
+      "Package": "intervals",
+      "Version": "0.15.2",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "079e6e04c6aafe087aba89f93311d8ba",
+      "Requirements": []
+    },
+    "iterators": {
+      "Package": "iterators",
+      "Version": "1.0.14",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "8954069286b4b2b0d023d1b288dce978",
+      "Requirements": []
+    },
+    "lattice": {
+      "Package": "lattice",
+      "Version": "0.20-45",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "b64cdbb2b340437c4ee047a1f4c4377b",
+      "Requirements": []
+    },
+    "lifecycle": {
+      "Package": "lifecycle",
+      "Version": "1.0.1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "a6b6d352e3ed897373ab19d8395c98d0",
+      "Requirements": [
+        "glue",
+        "rlang"
+      ]
+    },
+    "lubridate": {
+      "Package": "lubridate",
+      "Version": "1.8.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "2ff5eedb6ee38fb1b81205c73be1be5a",
+      "Requirements": [
+        "cpp11",
+        "generics"
+      ]
+    },
+    "lwgeom": {
+      "Package": "lwgeom",
+      "Version": "0.2-8",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "68b7afe010f5ddc3d5a5c8113018bb39",
+      "Requirements": [
+        "Rcpp",
+        "sf",
+        "units"
+      ]
+    },
+    "magrittr": {
+      "Package": "magrittr",
+      "Version": "2.0.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "7ce2733a9826b3aeb1775d56fd305472",
+      "Requirements": []
+    },
+    "pillar": {
+      "Package": "pillar",
+      "Version": "1.7.0",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "51dfc97e1b7069e9f7e6f83f3589c22e",
+      "Requirements": [
+        "cli",
+        "crayon",
+        "ellipsis",
+        "fansi",
+        "glue",
+        "lifecycle",
+        "rlang",
+        "utf8",
+        "vctrs"
+      ]
+    },
+    "pkgconfig": {
+      "Package": "pkgconfig",
+      "Version": "2.0.3",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Hash": "01f28d4278f15c76cddbea05899c5d6f",
+      "Requirements": []
+    },
+    "plogr": {