Commit 666b9fdb authored by Nauta, Lisanne's avatar Nauta, Lisanne
Browse files

Merge branch 'release/0.7.0'

parents 946908fd 1dd18277
{
"name": "api",
"version": "0.6.2",
"version": "0.7.0",
"description": "",
"author": "lisanne.nauta@wur.nl",
"private": true,
......@@ -17,6 +17,7 @@
"start:dev:azure-test": "NODE_ENV=azure-test nest start --watch",
"start:dev:azure-release": "NODE_ENV=azure-release nest start --watch",
"start:debug:azure-release": "NODE_ENV=azure-release nest start --debug --watch",
"start:debug:azure-test": "NODE_ENV=azure-test nest start --debug --watch",
"start:prod": "NODE_ENV=prod node dist/main",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
......
#!/bin/bash
d=`date +%Y%m%d`
echo $d
#) 1 backup production database
d=`date +%Y%m%d%H`
pg_host=$1
pg_user=$2
release_name=v070
release_db=waterapps_${release_name}
backup_dir=/home/nauta/OneDriveWUR/projects/2021_wagrinova/data/databases/production
echo "Enter password for ${pg_host}"
bash ./scripts/backup_database.sh ${pg_host} ${pg_user} ${backup_dir}
echo "Enter password for ${pg_host}"
read -p "Validate that backup has been created. Press enter to continue"
# 2 create release database
echo "Enter password for ${pg_host}"
createdb --host=${pg_host} --username=postgres ${release_db}
#echo "Enter password for ${pg_host}"
psql -a --host=${pg_host} --username=postgres -d ${release_db} -f ./scripts/enable_postgis.sql
read -p "Validate that database has been created. Press enter to continue"
#3 restore database
echo "Enter password for ${pg_host}"
bash ./scripts/restore_backup.sh ${pg_host} ${pg_user} ${release_db} ${backup_dir}/20221122_azure_waterapps.dump
read -p "Validate that release database has been populated. Press enter to continue"
# 4 run migrations
npm run migration:run:azure-release
read -p "Validate that database migrations has been succeeded. Press enter to continue"
# 5 upload raster data to soil database (requires gdal bindings)
# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_fc /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/FC_HWSD.tif
# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_pwp /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/PWP_HWSD.tif
# bash ./scripts/upload_raster.sh ${pg_host} ${pg_user} waterapps_soil soil_tex_class /home/nauta/OneDriveWUR/projects/2021_wagrinova/data/hwsd/DROP_TEX_CLASS.tif
# read -p "Validate that raster data has been uploadded to the soil production database.Press enter to continue"
\ No newline at end of file
import { BadRequestException, Controller, Get, ParseIntPipe, Query, UseGuards } from '@nestjs/common';
import { ApiQuery, ApiResponse, ApiTags } from '@nestjs/swagger';
import { DateTime } from 'luxon';
import { UserRole } from 'src/auth/enums/user-role';
import UserRoleGuard from 'src/auth/guards/user-role.guard';
import { DataResponse } from 'src/data/models/data-response';
......@@ -19,24 +20,44 @@ export class ForecastController {
required: false
})
@ApiQuery({
name: "min_forcast_date",
name: "min_forecast_date",
type: Date,
description: "Optional variable. Use ISO string",
required: false
})
@ApiQuery({
name: "startdate",
type: Date,
description: "Optional startdate. Use ISO string",
required: false
})
@ApiResponse({type: DataResponse})
async getLatest(@Query('region_id', ParseIntPipe) regionId: number, @Query('min_forcast_date',) minForcastDateString?: Date ,@Query('variable_code') variableCode?: string){
async getLatest(@Query('region_id', ParseIntPipe) regionId: number,@Query('min_forecast_date',) minForecastDateString?: string ,@Query('variable_code') variableCode?: string ,@Query('startdate') startDateString?:string){
let variableCodes = null;
if(variableCode){
let startDate :Date = null;
if(variableCode!=null){
variableCodes = [variableCode]
}
let minForcastDate: Date = null;
if(minForcastDateString){
minForcastDate = new Date(minForcastDateString);
let minForcastDate :Date = null;
if(minForecastDateString!=null){
minForcastDate = DateTime.fromISO(minForecastDateString).toJSDate();
}
if(startDateString!=null){
const startDateTimeLocal = DateTime.fromISO(startDateString,{setZone:true})
let startDateTimeUTC = DateTime.fromISO(startDateString);
// cause we currently have only UTC daily aggregation there is a timeshift when requesting local daily aggregations.
if(startDateTimeLocal.offset >= 0){
startDateTimeUTC = startDateTimeUTC.startOf('day');
}
else{
startDateTimeUTC = startDateTimeUTC.endOf('day');
}
startDate = startDateTimeUTC.toJSDate();
}
let latestForecasts = [];
latestForecasts = await this.forecastService.findLatestForRegion(regionId,variableCodes, minForcastDate);
latestForecasts = await this.forecastService.findLatestForRegion(regionId,variableCodes, minForcastDate,startDate);
if(latestForecasts.length ==0 ){
return null;
......
......@@ -15,16 +15,21 @@ export class ForecastService {
}
findLatestForRegion(regionId:number, variableCodes: string[], minForcastDate?: Date): Promise<Forecast[]> {
findLatestForRegion(regionId:number, variableCodes?: string[], minForcastDate?: Date, startDate?:Date): Promise<Forecast[]> {
let query = {where: {regionid: regionId}} as FindManyOptions<Forecast>;
if(variableCodes){
if(variableCodes!=null && variableCodes.length>0){
query.where['variablecode'] = In(variableCodes);
}
if(startDate!=null){
query.where['datetime'] = MoreThanOrEqual(startDate)
}
let queryLatestForecastDate = {...query};
if(minForcastDate){
queryLatestForecastDate.where['forecastdatetime'] = MoreThanOrEqual(minForcastDate)
}
queryLatestForecastDate.order = {forecastdatetime: 'DESC', datetime:'ASC'};
queryLatestForecastDate.take =1;
......
import { Transform } from "class-transformer";
import { IsDateString, IsNumber, IsOptional } from "class-validator";
export class ForecastDataQuery {
@IsOptional()
@IsDateString()
startdate?:string;
@IsOptional()
@IsNumber()
@Transform(({ value }) => parseInt(value))
lead?:number;
}
import {MigrationInterface, QueryRunner} from "typeorm";
export class obsFdt1664202566034 implements MigrationInterface {
name = 'obsFdt1664202566034'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "observations" ADD "forecastdatetime" TIMESTAMP`);
await queryRunner.query(`ALTER TABLE "observations" ADD CONSTRAINT "UQ_ff756b3f8fd3dd83c0bfe44d9a6" UNIQUE ("datetime", "forecastdatetime", "locationid", "variablecode", "datasourcecode")`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "observations" DROP CONSTRAINT "UQ_ff756b3f8fd3dd83c0bfe44d9a6"`);
await queryRunner.query(`ALTER TABLE "observations" DROP COLUMN "forecastdatetime"`);
}
}
import { ApiProperty } from "@nestjs/swagger";
import { Type } from "class-transformer";
import { IsDate, IsNotEmpty } from 'class-validator';
import { Expose, Transform, Type } from "class-transformer";
import { IsDate, isDateString, IsISO8601, isISO8601, IsNotEmpty, IsOptional, IS_ISO8601 } from 'class-validator';
import { DateTime } from "luxon";
export class ObservationDto {
......@@ -11,10 +12,10 @@ export class ObservationDto {
value: number;
@IsNotEmpty()
@IsDate()
@Type(() => Date)
@IsISO8601()
//@Type(() => Date)
@ApiProperty()
dateTime: Date;
dateTime: string;
@IsNotEmpty()
@ApiProperty()
userId: number;
......@@ -30,6 +31,20 @@ export class ObservationDto {
@ApiProperty()
dataSourceCode: string;
@ApiProperty({
required:false
})
@IsISO8601()
@IsOptional()
forecastDateTime?: string;
@Expose({
toClassOnly:true
})
get tz(){
return DateTime.fromISO(this.dateTime,{setZone:true}).zoneName;
}
constructor() {}
......
......@@ -40,7 +40,7 @@ export class ObservationController {
@Post()
@UsePipes(new ValidationPipe({ transform: true }))
async create(@Body() observationDto: ObservationDto){
const dateTime = observationDto.dateTime;
const duplicate = await this.observationService.findUnique(observationDto);
if(duplicate){
return this.observationService.update(duplicate.id, observationDto).then(result => {
......@@ -153,8 +153,21 @@ export class ObservationController {
@ApiOkResponse({
type:DataResponse
})
@ApiQuery({
name:'minforecastdate',
type:Date,
required:false
})
@Get('local/data/count')
async getCountLocal(@Req() req: Request,@Query('lat',ParseFloatPipe) lat: number, @Query('lon', ParseFloatPipe) lon: number, @Query('startdate') startDate: string, @Query('enddate') endDate: string ,@Query('variable_code') variableCode: string,@Query('data_source_code') dataSourceCode: string){
async getCountLocal(
@Req() req: Request,
@Query('lat',ParseFloatPipe) lat: number,
@Query('lon', ParseFloatPipe) lon: number,
@Query('startdate') startDate: string,
@Query('enddate') endDate: string ,
@Query('variable_code') variableCode: string,
@Query('data_source_code') dataSourceCode: string,
@Query('minforecastdate') minForecastDate?:string){
const user:User = req.user as User;
const userRegion = await this.regionService.getById(user.regionid);
const locIds = await this.locationService.getByDistance([lon,lat],userRegion.forecastradius).then(locs => {
......@@ -169,9 +182,15 @@ export class ObservationController {
const startDateTime = DateTime.fromISO(startDate,{setZone:true});
// note we parse date in user timezone
const endDateTime = DateTime.fromISO(endDate,{setZone:true});
const tz = startDateTime.zoneName;
//let fdt = DateTime.now().setZone(tz).startOf('day').toUTC().toJSDate();
let fdt = null;
if(minForecastDate!=null){
fdt = DateTime.fromISO(minForecastDate).toJSDate();
}
// note toJSDate will parse to server time zone (UTC), because the user observations are stored in UTC (no time zone)
// note only supports daily aggregation
const items = await this.observationService.countByLocationId(locIds,startDateTime.toJSDate(), endDateTime.toJSDate(),variableCode,dataSourceCode,startDateTime.zoneName);
const items = await this.observationService.countByLocationId(locIds,startDateTime.toJSDate(), endDateTime.toJSDate(),variableCode,dataSourceCode,tz,fdt);
let res = DataResponse.fromObservationCount(items,variable,startDateTime,endDateTime);
return res;
......
import { Exclude, Transform } from "class-transformer";
import { Exclude, plainToClass, Transform } from "class-transformer";
import { DateTime } from "luxon";
import { DataSource } from "src/data-source/data-source.entity";
import { Location } from "src/location/location.entity";
import { User } from "src/user/user.entity";
import { Variable } from "src/variable/variable.entity";
import { Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
import { Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Unique } from "typeorm";
import { ObservationDto } from "./observation-dto";
@Entity('observations')
@Unique(['datetime', 'forecastdatetime','locationid','variablecode','datasourcecode'])
export class Observation {
@PrimaryGeneratedColumn()
id: number;
......@@ -14,6 +16,12 @@ export class Observation {
@Column({type: 'timestamp without time zone' })
datetime: Date;
@Column({
type: 'timestamp without time zone' ,
nullable: true
})
forecastdatetime?: Date;
@Column()
value: number;
......@@ -47,4 +55,17 @@ export class Observation {
@JoinColumn({name: "datasourcecode", referencedColumnName:"code"})
datasource: DataSource
static fromDto(dto:ObservationDto){
let observation = new Observation();
const d = DateTime.fromISO(dto.dateTime).toUTC();
observation.datetime = d.toJSDate();
observation.forecastdatetime = dto.forecastDateTime? DateTime.fromISO(dto.forecastDateTime).toUTC().toJSDate() : null;
observation.value = dto.value;
// add relation ids
observation.userid = dto.userId;
observation.variablecode = dto.variableCode;
observation.locationid = dto.locationId;
observation.datasourcecode = dto.dataSourceCode;
return observation;
}
}
......@@ -17,31 +17,13 @@ export class ObservationService {
create(dto: ObservationDto){
let observation = new Observation();
// add properties
const d = DateTime.fromJSDate(dto.dateTime).toUTC();
observation.datetime = d.toJSDate();
observation.value = dto.value;
// add relation ids
observation.userid = dto.userId;
observation.variablecode = dto.variableCode;
observation.locationid = dto.locationId;
observation.datasourcecode = dto.dataSourceCode;
let observation = Observation.fromDto(dto);
return this.repo.save(observation);
}
update(id, dto:ObservationDto){
let observation = new Observation();
// add properties
const d = DateTime.fromJSDate(dto.dateTime).toUTC();
observation.datetime = d.toJSDate();
observation.value = dto.value;
// add relation ids
observation.userid = dto.userId;
observation.variablecode = dto.variableCode;
observation.locationid = dto.locationId;
observation.datasourcecode = dto.dataSourceCode;
let observation = Observation.fromDto(dto);
return this.repo.update(id, observation)
}
......@@ -58,9 +40,10 @@ export class ObservationService {
return this.repo.findOne({where: {
locationid: observationDto.locationId,
datetime: observationDto.dateTime,
datetime: DateTime.fromISO(observationDto.dateTime) ,
variablecode: observationDto.variableCode,
datasourcecode: observationDto.dataSourceCode
datasourcecode: observationDto.dataSourceCode,
forecastdatetime: observationDto.forecastDateTime? DateTime.fromISO(observationDto.forecastDateTime) : null
}});
}
......@@ -93,18 +76,22 @@ export class ObservationService {
* @param variableCode - filter single variablecode
* @param datasourceCode - filter single datasource code
* @param tz - aggregate using following timezone
* @param minForcastDate - filter forecastdatetime greater or equal
* @returns
*/
countByLocationId(locIds: number[], startDate:Date, endDate:Date,variableCode:string, datasourceCode:string,tz:string){
countByLocationId(locIds: number[], startDate:Date, endDate:Date,variableCode:string, datasourceCode:string,tz:string,minForcastDate?:Date){
// count all observations.
// The observation datetime is truncated on day in the given timezone (e.g UTC+2. observation datetime 2022-01-02T22:00+0 -> 2022-01-03T00:00+0).
// Then truncated datetime is converted to given timezone (e.g. 2022-01-03T00:00+0 -> 2022-01-02T22:00+2)
const q = this.repo.createQueryBuilder().select('value')
let q = this.repo.createQueryBuilder().select('value')
.addSelect(`date_trunc('day',datetime at time zone '${tz}') at time zone '${tz}'`,'datetime')
.addSelect('COUNT(*)')
.leftJoinAndSelect("variables","variable",'variablecode = variable.code')
.where("locationid IN (:...ids )", { ids: locIds })
.andWhere(`datetime >= '${startDate.toISOString()}'::timestamp`).andWhere(`datetime <= '${endDate.toISOString()}'::timestamp`)
.where("locationid IN (:...ids )", { ids: locIds });
if(minForcastDate!=null){
q.andWhere(`forecastdatetime >= '${minForcastDate.toISOString()}'::timestamp`)
}
q.andWhere(`datetime >= '${startDate.toISOString()}'::timestamp`).andWhere(`datetime <= '${endDate.toISOString()}'::timestamp`)
.andWhere(`variablecode = '${variableCode}'`)
.andWhere(`datasourcecode = '${datasourceCode}'`)
.orderBy('datetime','ASC')
......
......@@ -241,16 +241,26 @@ export class LocalSoilMoistureForecastService {
}));
}
public getById(id:number){
const findOne = this.repo.findOne(id);
public getById(id:number,startDate?:Date, lead?:number){
let options : FindOneOptions = {};
const findOne = this.repo.findOne(id,options);
const orderSeries = from(findOne).pipe(map((f)=>{
const orderedSeries = f.series.sort((a,b)=>{
// sort by asceding datetimes to be able to slice the array
return a.datetime.getTime() - b.datetime.getTime();
});
f.series = orderedSeries;
return f;
if(startDate!=null){
const filteredSeries = f.series.filter((x)=> x.datetime.getTime() >= startDate.getTime());
f.series = filteredSeries;
}
if(lead!=null){
f.series = f.series.slice(0,lead);
}
return f;
}));
return lastValueFrom(orderSeries);
}
......
......@@ -19,6 +19,7 @@ import { Transform } from 'class-transformer';
import { CropFieldSoilMoistureForecastDto } from './soil-moisture-forecast/local-soil-moisture-forecast/dtos/crop-field-soil-moisture-forecast-dto';
import {Point} from 'gdal-next'
import { ListLocalForecastQuery } from './soil-moisture-forecast/local-soil-moisture-forecast/dtos/list-local-forecast-query';
import { ForecastDataQuery } from 'src/forecast/models/forecast-data-query';
const PG_UNIQUE_CONSTRAINT_VIOLATION = "23505";
......@@ -171,19 +172,36 @@ export class SoilMoistureController {
@ApiResponse({
type: DataResponse
})
@ApiQuery({
name:'startdate',
type:Date,
required:false,
example:'2022-10-01T00:00+02:00'
})
@ApiQuery({
name:'lead',
type:Number,
required:false
})
@UseGuards(UserRoleGuard(UserRole.User))
@Get('forecast-local/:id/data')
async getLocalForcastDataById(@Req() request : Request, @Param('id') id:number,@Query('lead') lead?:number){
const forecast = await this.soilMoistureForecastService.getById(id);
if(lead && lead < forecast.series.length){
forecast.series = forecast.series.slice(0,lead);
async getLocalForcastDataById(@Req() request : Request, @Param('id') id:number, @Query() query:ForecastDataQuery){
// the query startdate is parsed to server timezone (00:00+2 -> 22:00+0).
let startDateUTC :Date = null;
if(query.startdate!=null){
const startDateLocal = DateTime.fromISO(query.startdate, {setZone:true});
let startDateTimeUTC = DateTime.fromISO(query.startdate);
// cause we currently have only UTC daily aggregation there is a timeshift when requesting local aggregations.
if(startDateLocal.offset >= 0){
startDateTimeUTC = startDateTimeUTC.startOf('day');
}
else{
startDateTimeUTC = startDateTimeUTC.endOf('day');
}
startDateTimeUTC.toJSDate()
}
const forecast = await this.soilMoistureForecastService.getById(id,startDateUTC,query.lead);
const lon = forecast.cropfield.position.coordinates[0];
const lat = forecast.cropfield.position.coordinates[1];
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment