Skip to content
GitLab
Commit 9330891f authored by Nauta, Lisanne's avatar Nauta, Lisanne
Browse files

Merge branch 'release/0.3.0'

parents d09cb98a 3d30727a
Hide whitespace changes
Inline Side-by-side
package.json
View file @ 9330891f
{
"name": "api",
"version": "0.2.1",
"version": "0.3.0",
"description": "",
"author": "lisanne.nauta@wur.nl",
"private": true,
......
src/app.controller.ts
View file @ 9330891f
import { Controller, Get } from '@nestjs/common';
import { ApiSecurity } from '@nestjs/swagger';
import { readFileSync } from 'fs';
import { AppService } from './app.service';
@Controller()
......
src/data-provider/meteoblue/meteoblue.service.ts
View file @ 9330891f
......@@ -13,6 +13,7 @@ import {readFileSync} from 'fs';
import { Task } from 'src/task/task.entity';
import { User } from 'src/user/user.entity';
import { RegionService } from 'src/region/region.service';
import { Region } from 'src/region/region.entity';
const CONTEXT = "MeteoblueService"
......@@ -40,6 +41,7 @@ export class MeteoblueService {
//let data = JSON.parse(readFileSync("src/data-provider/meteoblue/meteoblue_response.json", {encoding: 'utf-8'})) as MeteoblueDataResponse
//let meteoblueData = Object.assign(new MeteoblueDataResponse(), data);
//return of(meteoblueData);
//const testError = throwError(()=> new BadRequestException() );
//const testSuccess= of(meteoblueData);
//const responses = [testError, testSuccess]
......@@ -62,7 +64,7 @@ export class MeteoblueService {
}))
}
getForecasts(location: Location): Observable<Forecast[]> {
/* getForecasts(location: Location): Observable<Forecast[]> {
Logger.log(`Get Meteoblue forecasts for location ${location.id}`, CONTEXT);
const geoJson = location.geometry;
const point : Point = Geometry.fromGeoJson(geoJson) as Point;
......@@ -71,6 +73,16 @@ export class MeteoblueService {
return this.getData(lat,lon).pipe(map((meteoblueResponse: MeteoblueDataResponse)=>{
return meteoblueResponse.toForecast(["precipitation","precipitation_probability","temperature_mean"], location.id);
}))
} */
getRegionForecasts(region: Region): Observable<Forecast[]>{
Logger.log(`Get Meteoblue forecasts for region ${region.name}`, CONTEXT);
const point : Point = Geometry.fromGeoJson(region.forecastlocation)
const lon = point.x;
const lat = point.y;
return this.getData(lat,lon).pipe(map((meteoblueResponse: MeteoblueDataResponse)=>{
return meteoblueResponse.toForecast(["precipitation","precipitation_probability","temperature_mean"], region.id);
}))
}
@OnEvent('user.created',{async:true})
......@@ -80,62 +92,40 @@ export class MeteoblueService {
}
if(!user.locationid){
throw Error("User requires location");
}
this.regionService.getById(user.regionid).then(userRegion=>{
this.forecastService.findLatestForRegion(userRegion).then(res=>{
let createForecasts : Forecast[] = [];
res.forEach((f)=>{
let newUserForecast = {
forecastdatetime: f.forecastdatetime,
locationid: user.locationid,
datetime: f.datetime,
variablecode: f.variablecode,
value: f.value,
datasourcecode:f.datasourcecode
} as Forecast
createForecasts.push(newUserForecast)
});
this.forecastService.createBulk(createForecasts);
})
})
}
}
/* Request forecast data for every user location and stores in database.
listens to event emitters from the task service.
*/
@OnEvent('task.run.meteoblue_store',{async:true, promisify:true})
@OnEvent('task.run.meteoblue_store_for_region',{async:true, promisify:true})
public async onRunMeteoblueForecastStore(task: Task){
Logger.log('Run event task.run.meteoblue_store', CONTEXT);
const locations = await this.userService.getLocations();
const regions = await this.regionService.getAll();
let errors = 0;
let success = 0;
let calls = locations.map((location) =>{
return this.getForecasts(location).pipe(
tap({
next: (forecast)=>{
Logger.log(`Successfully retrieved Meteoblue forecasts for ${location.id}.`, CONTEXT);
this.forecastService.createBulk(forecast).catch(err=>{
Logger.error(`Could not store Meteoblue forecasts for locationId ${location.id}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`ERROR: Could not store Meteoblue forecasts for locationId ${location.id}.`);
}).then(()=>{
Logger.log(`Successfully stored Meteoblue forecast for locationId ${location.id}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`SUCCES: Meteoblue forecast stored for locationId ${location.id}.`);
})
success++;
},
error: (err)=>{
Logger.error(`Meteoblue forecast request error for locationId ${location.id}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`ERROR: Meteoblue forecast request error for locationId ${location.id}. Error Message: ${err.response.data.error_message}.`)
errors++;
}
}), catchError((e)=>of(e))) // catch indivdual errors so the jorkjoin will not be stopped on singe fails.
});
let calls = regions.map(region => {
return this.getRegionForecasts(region).pipe(tap({
next: (forecast) => {
Logger.log(`Successfully retrieved Meteoblue forecasts for ${region.name}.`, CONTEXT);
this.forecastService.createBulk(forecast).catch(err=>{
Logger.error(`Could not store Meteoblue forecasts for region ${region.name}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`ERROR: Could not store Meteoblue forecasts for region ${region.name}.`);
}).then(()=>{
Logger.log(`Successfully stored Meteoblue forecast for region ${region.name}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`SUCCES: Meteoblue forecast stored for region ${region.name}.`);
})
success++;
},
error: (err)=>{
Logger.error(`Meteoblue forecast request error for region ${region.name}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`ERROR: Meteoblue forecast request error for region ${region.name}. Error Message: ${err.response.data.error_message}.`)
errors++;
}
}),catchError((e)=>of(e)))
})
// returns a promise of the combined calls. Waits for all calls to be finished.
return lastValueFrom(forkJoin(calls).pipe(tap({
......@@ -152,69 +142,11 @@ export class MeteoblueService {
this.eventEmitter.emit("task.log",task,`WARNING: error: ${errors} success: ${success}.`);
}
else{
this.eventEmitter.emit("task.log",task,"SUCCESS: All locations success.");
this.eventEmitter.emit("task.log",task,"SUCCESS: All region success.");
}
Logger.log("All Meteoblue forecast requests completed", CONTEXT);
}
})))
}
@OnEvent('task.run.meteoblue_store_for_region',{async: true,promisify:true})
async onRunMeteoblueForecastStoreForRegion(task:Task){
Logger.log('Run event task.run.meteoblue_store_for_region', CONTEXT);
let errors = 0;
// get all registerd regions
const regions = await this.regionService.getAll();
// create calls for rxjs forkjoin. forkjoin fires calls and waits for all to complete.
const calls = regions.map(region => {
const geoJson = region.forecastlocation;
const point : Point = Geometry.fromGeoJson(geoJson) as Point;
const lon = point.x;
const lat = point.y;
return this.getData(lat,lon).pipe(mergeMap(res =>{
return of(...region.users).pipe(mergeMap(user=>{
const userForecast = res.toForecast(["precipitation","precipitation_probability","temperature_mean"], user.location.id);
return from(this.forecastService.createBulk(userForecast)).pipe(tap({
next:() =>{
Logger.log(`Successfully stored Meteoblue forecast for locationId ${user.location.id}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`SUCCES: Meteoblue forecast stored for locationId ${user.location.id}.`);
},
error:() =>{
Logger.log(`Successfully stored Meteoblue forecast for locationId ${user.location.id}.`, CONTEXT);
this.eventEmitter.emit("task.log",task,`ERROR: Could not store Meteoblue forecasts for locationId ${user.location.id}.`);
errors++ ;
}
}))
}))
}),catchError(err=>{
Logger.error(err)
this.eventEmitter.emit("task.log",task,`ERROR: Meteoblue forecast request error for lat: ${lat} lon:${lon}. Error Message: ${err.response.data.error_message}.`)
errors++;
return of(err)
}))
});
const all = forkJoin(calls).pipe(tap({
error: (err) =>{
Logger.error(err);
throw Error();
},
complete: () =>{
if(errors > 0){
Logger.warn("One or more errors", CONTEXT);
this.eventEmitter.emit("task.log",task,`WARNING: error: ${errors}.`);
}
else{
this.eventEmitter.emit("task.log",task,"SUCCESS: All Meteoblue user forecasts success.");
}
Logger.log("All Meteoblue forecast requests completed", CONTEXT);
}
}));
return lastValueFrom(all);
}
}
src/data-provider/meteoblue/models/meteo-blue-data-response.ts
View file @ 9330891f
......@@ -14,7 +14,7 @@ export class MeteoblueDataResponse {
trend_day: MeteoblueTrendDay;
constructor(){}
public toForecast(variableKeys: string[], locatationId:number):Forecast[]{
public toForecast(variableKeys: string[], regionId:number):Forecast[]{
let forecasts : Forecast[] = [];
for(var iTime in this.trend_day.time){
......@@ -23,7 +23,7 @@ export class MeteoblueDataResponse {
datetime: DateTime.fromISO(this.trend_day.time[iTime],{zone:'UTC'}).toJSDate(),
forecastdatetime: DateTime.fromISO(this.metadata.modelrun_utc,{zone:'UTC'}).toJSDate(),
value: this.trend_day[varKey][iTime],
locationid: locatationId,
regionid: regionId,
variablecode: MeteoblueVariableMapping[varKey],
datasourcecode:"meteoblue_trend_day"
......
src/forecast/forecast.controller.ts
View file @ 9330891f
import { Controller, Get, ParseIntPipe, Query, UseGuards } from '@nestjs/common';
import { BadRequestException, Controller, Get, ParseIntPipe, Query, UseGuards } from '@nestjs/common';
import { ApiQuery, ApiResponse, ApiTags } from '@nestjs/swagger';
import { UserRole } from 'src/auth/enums/user-role';
import UserRoleGuard from 'src/auth/guards/user-role.guard';
......@@ -25,7 +25,7 @@ export class ForecastController {
required: false
})
@ApiResponse({type: DataResponse})
async getLatest(@Query('location_id', ParseIntPipe) locationId: number, @Query('min_forcast_date',) minForcastDateString?: Date ,@Query('variable_code') variableCode?: string){
async getLatest(@Query('region_id', ParseIntPipe) regionId: number, @Query('min_forcast_date',) minForcastDateString?: Date ,@Query('variable_code') variableCode?: string){
let variableCodes = null;
if(variableCode){
variableCodes = [variableCode]
......@@ -34,7 +34,10 @@ export class ForecastController {
if(minForcastDateString){
minForcastDate = new Date(minForcastDateString);
}
const latestForecasts = await this.forecastService.findLatest(locationId,variableCodes,minForcastDate);
let latestForecasts = [];
latestForecasts = await this.forecastService.findLatestForRegion(regionId,variableCodes, minForcastDate);
if(latestForecasts.length ==0 ){
return null;
}
......@@ -53,12 +56,12 @@ export class ForecastController {
})
@ApiResponse({type: DataResponse})
@Get('data')
async getData(@Query('location_id', ParseIntPipe) locationId: number, @Query('forecast_date') forecast_date: Date, @Query('variable_code') variableCode?: string){
async getData(@Query('region_id', ParseIntPipe) region_id: number,@Query('forecast_date') forecast_date: Date, @Query('variable_code') variableCode?: string){
let variableCodes = null;
if(variableCode){
variableCodes = [variableCode]
}
const latestForecasts = await this.forecastService.findLatest(locationId,variableCodes);
const latestForecasts = await this.forecastService.findLatestForRegion(region_id,variableCodes);
const data = new DataResponse(latestForecasts);
return data;
}
......
src/forecast/forecast.entity.ts
View file @ 9330891f
import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn, PrimaryGeneratedColumn } from "typeorm";
import { Location } from "src/location/location.entity";
import { Variable } from "src/variable/variable.entity";
import { Region } from "src/region/region.entity";
@Entity({name:'forecasts'})
export class Forecast {
......@@ -17,9 +18,16 @@ export class Forecast {
@PrimaryColumn({type: 'timestamp'})
forecastdatetime: Date;
@PrimaryColumn()
@Column({
nullable: true
})
locationid:number;
@PrimaryColumn({
default:1
})
regionid:number;
@PrimaryColumn()
variablecode:string;
......@@ -30,10 +38,14 @@ export class Forecast {
@JoinColumn({ name: "variablecode", referencedColumnName:'code'})
variable: Variable;
@ManyToOne(() => Location, {eager: true, cascade: true})
@ManyToOne(() => Location, {eager: false, cascade: true, nullable: true})
@JoinColumn({ name: "locationid", referencedColumnName:'id'})
location: Location;
@ManyToOne(() => Region, {eager: true, cascade: true})
@JoinColumn({name: "regionid", referencedColumnName:'id'})
region: Region
constructor(){}
......
src/forecast/forecast.service.ts
View file @ 9330891f
......@@ -6,7 +6,7 @@ import { TimeSeriesItem } from 'src/data/models/time-series-item';
import { Location } from 'src/location/location.entity';
import { Observation } from 'src/observation/observation.entity';
import { Region } from 'src/region/region.entity';
import { FindManyOptions, FindOneOptions, In, MoreThan, Repository } from 'typeorm';
import { FindManyOptions, FindOneOptions, In, MoreThan, MoreThanOrEqual, Repository } from 'typeorm';
import { Forecast } from './forecast.entity';
@Injectable()
......@@ -16,30 +16,35 @@ export class ForecastService {
}
findLatestForRegion(region:Region, dataSourceCode="meteoblue_trend_day"){
const locationIds = region.users.map(u=>{return u.locationid});
let queryLatestLoc = {
where:{
locationid: In(locationIds),
datasourcecode: dataSourceCode
},
order:{
forecastdatetime: "DESC"
}
} as FindOneOptions<Forecast>
return this.repo.findOne(queryLatestLoc).then(res=>{
return this.repo.find({
where:{
locationid: res.locationid,
forecastdatetime: res.forecastdatetime,
datasourcecode: dataSourceCode
}
})
findLatestForRegion(regionId:number, variableCodes: string[], minForcastDate?: Date){
let query = {where: {regionid: regionId}} as FindManyOptions<Forecast>;
if(variableCodes){
query.where['variablecode'] = In(variableCodes);
}
let queryLatestForecastDate = {...query};
if(minForcastDate){
queryLatestForecastDate.where = {
forecastdatetime: MoreThanOrEqual(minForcastDate)
}
}
queryLatestForecastDate.order = {forecastdatetime: 'DESC'};
queryLatestForecastDate.take =1;
return this.repo.findOne(queryLatestForecastDate).then((res)=>{
// no forcasts found for query
if(!res){
return [];
}
const fdt = res.forecastdatetime;
query.where["forecastdatetime"] = [fdt];
return this.repo.find(query);
});
}
findLatest(locationId:number, variableCodes?:string[], minForecastDate?: Date){
/* findLatest(locationId:number, variableCodes?:string[], minForecastDate?: Date){
let query = {where: {locationid: locationId}} as FindManyOptions<Forecast>;
if(variableCodes){
query.where['variablecode'] = In(variableCodes);
......@@ -67,7 +72,7 @@ export class ForecastService {
return this.repo.find(query);
});
}
} */
createBulk(forecast: Forecast[]){
return this.repo.save(forecast)
......
src/migrations/1643048613589-refactor_forecast_location.ts 0 → 100644
View file @ 9330891f
import {MigrationInterface, QueryRunner} from "typeorm";
export class refactorForecastLocation1643048613589 implements MigrationInterface {
name = 'refactorForecastLocation1643048613589'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "forecasts" ADD "regionid" integer NOT NULL DEFAULT '1'`);
await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_bbbf9c350e820b7eed4639cff63"`);
//await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36" PRIMARY KEY ("datetime", "forecastdatetime", "locationid", "variablecode", "regionid")`);
//await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36"`);
//await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_55da5a36b80056cb6cc0db61569" PRIMARY KEY ("datetime", "forecastdatetime", "regionid", "variablecode")`);
await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "FK_b0f528bb1a31db5f90854b771ac"`);
await queryRunner.query(`ALTER TABLE "forecasts" ALTER COLUMN "locationid" DROP NOT NULL`);
//await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36"`);
await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_55da5a36b80056cb6cc0db61569" PRIMARY KEY ("datetime", "forecastdatetime", "variablecode", "regionid")`);
await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "FK_b0f528bb1a31db5f90854b771ac" FOREIGN KEY ("locationid") REFERENCES "locations"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "FK_6a59bdf6782dacd019d24e10033" FOREIGN KEY ("regionid") REFERENCES "regions"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "FK_6a59bdf6782dacd019d24e10033"`);
await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "FK_b0f528bb1a31db5f90854b771ac"`);
await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_55da5a36b80056cb6cc0db61569"`);
//await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36" PRIMARY KEY ("datetime", "forecastdatetime", "locationid", "variablecode", "regionid")`);
await queryRunner.query(`ALTER TABLE "forecasts" ALTER COLUMN "locationid" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "FK_b0f528bb1a31db5f90854b771ac" FOREIGN KEY ("locationid") REFERENCES "locations"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
//await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_55da5a36b80056cb6cc0db61569"`);
//await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36" PRIMARY KEY ("datetime", "forecastdatetime", "locationid", "variablecode", "regionid")`);
//await queryRunner.query(`ALTER TABLE "forecasts" DROP CONSTRAINT "PK_a34cce0a64db3a453f9b4e7bc36"`);
await queryRunner.query(`ALTER TABLE "forecasts" ADD CONSTRAINT "PK_bbbf9c350e820b7eed4639cff63" PRIMARY KEY ("datetime", "forecastdatetime", "locationid", "variablecode")`);
await queryRunner.query(`ALTER TABLE "forecasts" DROP COLUMN "regionid"`);
}
}
src/observation/observation.controller.ts
View file @ 9330891f
import { Body, Controller, Get, ParseFloatPipe, Post, Query, UseGuards, UsePipes, ValidationPipe } from '@nestjs/common';
import { Body, ClassSerializerInterceptor, Controller, Get, ParseFloatPipe, Post, Query, UseGuards, UseInterceptors, UsePipes, ValidationPipe } from '@nestjs/common';
import { ApiQuery, ApiResponse, ApiTags } from '@nestjs/swagger';
import { Point, SpatialReference } from 'gdal-next';
import { DataResponse } from 'src/data/models/data-response';
......@@ -28,6 +28,7 @@ export class ObservationController {
}
@UseGuards(UserRoleGuard(UserRole.User))
@UseInterceptors(ClassSerializerInterceptor)
@Post()
@UsePipes(new ValidationPipe({ transform: true }))
async create(@Body() observationDto: ObservationDto){
......
src/observation/observation.entity.ts
View file @ 9330891f
import { Transform } from "class-transformer";
import { Exclude, Transform } from "class-transformer";
import { DateTime } from "luxon";
import { DataSource } from "src/data-source/data-source.entity";
import { Location } from "src/location/location.entity";
......@@ -33,10 +33,12 @@ export class Observation {
@JoinColumn({ name: "variablecode", referencedColumnName:'code'})
variable: Variable;
@Exclude()
@ManyToOne(()=> User, {eager: true})
@JoinColumn({ name: "userid", referencedColumnName:'id'})
user: User;
@Exclude()
@ManyToOne(()=> Location, {eager: true})
@JoinColumn({ name: "locationid", referencedColumnName:'id'})
location: Location;
......
src/region/region.controller.ts
View file @ 9330891f
import { ClassSerializerInterceptor, Controller, Get, UseInterceptors, UsePipes, ValidationPipe } from '@nestjs/common';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { ClassSerializerInterceptor, Controller, Get, Param, UseInterceptors, UsePipes, ValidationPipe } from '@nestjs/common';
import { ApiOkResponse, ApiParam, ApiTags } from '@nestjs/swagger';
import { Region } from './region.entity';
import { RegionService } from './region.service';
......@@ -21,4 +21,16 @@ export class RegionController {
return this.regionService.getAll();
}
@ApiParam({
name:'id',
})
@ApiOkResponse({
type: Region
})
@Get(':id')
@UseInterceptors(ClassSerializerInterceptor)
getById(@Param('id') id:number){
return this.regionService.getById(id)
}
}
src/user/user.entity.ts
View file @ 9330891f
......@@ -28,12 +28,15 @@ export class User{
phonenumber: string;
@Column()
@ApiProperty()
@RelationId((user:User)=> user.location)
locationid: number;
@ApiProperty()
@Column()
regionid:number;
@ApiProperty()
@Column({
type: 'enum',
enum: UserRole,
......@@ -47,8 +50,8 @@ export class User{
@JoinColumn({ name: "locationid", referencedColumnName:'id' })
location: Location;
@ManyToOne(() => Region, {lazy:true}) // specify inverse side as a second parameter
@ApiProperty()
@ManyToOne(() => Region, {}) // specify inverse side as a second parameter
@JoinColumn({ name: "regionid", referencedColumnName:'id' })
region: Promise<Region>
region: Region
}
\ No newline at end of file
src/user/user.service.ts
View file @ 9330891f
......@@ -4,7 +4,7 @@ import { Repository } from 'typeorm';
import { User } from './user.entity';
import { CreateUserDto } from './user.dto';
import { Location } from 'src/location/location.entity';
import { firstValueFrom, from, map } from 'rxjs';
import { firstValueFrom, from, map, switchMap } from 'rxjs';
import { EventEmitter2 } from '@nestjs/event-emitter';
import * as appInsights from 'applicationinsights'
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment