Skip to content

GitLab

Commit 04088c8b authored by Jorge S. Mendes de Jesus's avatar Jorge S. Mendes de Jesus
Browse files

New k8s implementation

parent 9fed521d
Pipeline #24745 failed with stage
in 18 seconds
Hide whitespace changes
Inline Side-by-side
.s2i/environment deleted 100644 → 0
View file @ 9fed521d
APP_MODULE=appWSM:app
Dockerfile
View file @ 04088c8b
FROM ubuntu:18.04 FROM python:3.9.5-slim-buster@sha256:076f9edf940d59446b98c242de0058054240b890a47d1dbed18560d12ec58228
LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/* # Required volume for monoliths e.g:
RUN mkdir -p /var/www/wsm.rest # docker volume create monoliths-volume
# docker build -t wsm.rest
# docker run -it -v /data/monoliths-volume:/data/monoliths .
WORKDIR /var/www/wsm.rest LABEL "maintainer"="jorge.mendesdejesus@isric.org"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV DEBIAN_FRONTEND noninteractive
ENV DB_HOST scomp1270.wurnet.nl
ENV DB_NAME isric
ENV DB_USER hass005
ENV DB_PASSWORD 1515NewPW
ENV DB_PORT 5479
ENV MONOLITH_PATH /data/monoliths
RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client build-essential libpq-dev tini
RUN mkdir -p /app
# Mount point with data
RUN mkdir -p /data/monoliths
WORKDIR /app
COPY . . COPY . .
RUN chmod +x configs/startup.sh
RUN pip install -r requirements.txt RUN pip install -r requirements.txt
ENTRYPOINT [ "configs/startup.sh" ] USER www-data
EXPOSE 8000
ENTRYPOINT ["/usr/bin/tini","-g","--"]
CMD ["gunicorn","--bind","0.0.0.0","--workers","2","--threads","4","--worker-class","eventlet", "--log-file=-", "wsgi:app"]
# build-essential \
#RUN apt-get update \
# && apt-get install -y --no-install-recommends \
# postgresql-client \
# build-essential \
# gdal-bin libgdal-dev libtbb2 tini
#WORKDIR /src
#RUN mkdir -p /soils
#COPY soils /soils/soils
#COPY requirements.txt /soils/requirements.txt
#COPY gunicorn.py /soils/gunicorn.py
#WORKDIR /soils
#RUN pip install -r requirements.txt
# Clean up
#RUN pip cache purge \
# && apt remove -y build-essential \
# && apt -y autoremove \
# && rm -rf /var/lib/apt/lists/*
#USER www-data
#EXPOSE 8000
#ENTRYPOINT ["/usr/bin/tini","-g","--"]
#CMD ["gunicorn","--config=./gunicorn.py", "soils:app"]
#FROM ubuntu:18.04
#LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
#RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/*
#RUN mkdir -p /var/www/wsm.rest
#WORKDIR /var/www/wsm.rest
#COPY . .
#RUN chmod +x configs/startup.sh
#RUN pip install -r requirements.txt
#ENTRYPOINT [ "configs/startup.sh" ]
EXPOSE 80 #EXPOSE 80
STOPSIGNAL SIGQUIT #STOPSIGNAL SIGQUIT
HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1 #HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1
CMD ["nginx", "-g", "daemon off;"] #CMD ["nginx", "-g", "daemon off;"]
################### Usage ##################### ################### Usage #####################
# Build it # Build it
...@@ -29,4 +97,4 @@ CMD ["nginx", "-g", "daemon off;"] ...@@ -29,4 +97,4 @@ CMD ["nginx", "-g", "daemon off;"]
# docker run -dit --env-file /path/to/envfile -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest # docker run -dit --env-file /path/to/envfile -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
# Declaring variables inline: # Declaring variables inline:
# docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric -e DB_USER=haas005 -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest # docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric -e DB_USER=haas005 -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
\ No newline at end of file
RESTServer.wsgi deleted 100644 → 0
View file @ 9fed521d
import sys,os
#sys.path.append('../')
sys.path.insert(0,os.path.abspath(os.path.dirname(__file__)))
from appWSM import app as application
appWSM.py
View file @ 04088c8b
#https://github.com/humiaozuzu/awesome-flask #https://github.com/humiaozuzu/awesome-flask
from flask import Flask,make_response from flask import Flask,make_response
from flask_restful import Api from flask_restful import Api
from flask_cors import CORS
from resources.meta import Monoliths,Monolith,MonolithRoot from resources.meta import Monoliths,Monolith,MonolithRoot
import logging,os.path import logging,os.path
app = Flask(__name__) app = Flask(__name__)
CORS(app, resources={r"/*": {'origins': '*'}})
api = Api(app) api = Api(app)
basePath=os.path.dirname(__file__) basePath=os.path.dirname(__file__)
file_handler = logging.FileHandler(os.path.join(basePath,'app.log')) #file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
app.logger.addHandler(file_handler) #app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO) #app.logger.setLevel(logging.INFO)
app.config["LOG_TYPE"] = os.environ.get("LOG_TYPE", "stream")
app.config["LOG_LEVEL"] = os.environ.get("LOG_LEVEL", "INFO")
#'/monoliths/<path:path>', #'/monoliths/<path:path>',
api.add_resource(Monolith,"/monoliths","/") api.add_resource(Monolith,"/monoliths","/")
......
common/filters.py
View file @ 04088c8b
...@@ -38,8 +38,6 @@ def filter_extra(fileList): ...@@ -38,8 +38,6 @@ def filter_extra(fileList):
[{u'JM-006-def.1per.jpg': {'profile_id': 63764, 'publication_year': 2006, 'classification': 'Cambisols'}}, {u'FR-011-def.1per.jpg': {'profile_id': 69755, 'publication_year': 2006, 'classification': 'Albeluvisols'}}, {u'IT-016-def.1per.jpg': None}] [{u'JM-006-def.1per.jpg': {'profile_id': 63764, 'publication_year': 2006, 'classification': 'Cambisols'}}, {u'FR-011-def.1per.jpg': {'profile_id': 69755, 'publication_year': 2006, 'classification': 'Albeluvisols'}}, {u'IT-016-def.1per.jpg': None}]
""" """
#connect to database #connect to database
#conn = psycopg2.connect(host="85.214.35.184",database="isric", user="jjesus", password="isric126")
#dictCur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
dictCur=get_db_connection() dictCur=get_db_connection()
#dictCur.execute("select * from rest.vw_isis_classification_profile_geom") #dictCur.execute("select * from rest.vw_isis_classification_profile_geom")
#dictCur.execute("select * from web_isis.vw_isis_profiles_with_classification_geo_img_rest") #dictCur.execute("select * from web_isis.vw_isis_profiles_with_classification_geo_img_rest")
......
common/secret.py
View file @ 04088c8b
...@@ -3,10 +3,10 @@ ...@@ -3,10 +3,10 @@
from os import environ from os import environ
# Get environment variables and set default if none specified: # Get environment variables and set default if none specified:
DB_HOST = environ.get("DB_HOST", "scomp1270.wurnet.nl") DB_HOST = environ.get("DB_HOST")
DB_NAME = environ.get("DB_NAME", "isric") DB_NAME = environ.get("DB_NAME")
DB_USER = environ.get("DB_USER", "haas005") DB_USER = environ.get("DB_USER")
DB_PASSWORD = environ.get("DB_PASSWORD", "1515NewPW") DB_PASSWORD = environ.get("DB_PASSWORD")
DB_PORT = environ.get("DB_PORT", "5479") DB_PORT = environ.get("DB_PORT")
DB={"host":DB_HOST,"database":DB_NAME, "user":DB_USER, "password":DB_PASSWORD,"port":DB_PORT} DB={"host":DB_HOST,"database":DB_NAME, "user":DB_USER, "password":DB_PASSWORD,"port":DB_PORT}
common/util.py
View file @ 04088c8b
...@@ -61,4 +61,4 @@ def get_url_params(testing=False,req=None): ...@@ -61,4 +61,4 @@ def get_url_params(testing=False,req=None):
if __name__ == "__main__": if __name__ == "__main__":
#print get_url_params(testing=True,req=[('country', [u'BR,ES']), ('limit', [u'15.3'])]) #print get_url_params(testing=True,req=[('country', [u'BR,ES']), ('limit', [u'15.3'])])
import doctest import doctest
doctest.testmod(verbose=1) doctest.testmod(verbose=1)
\ No newline at end of file
requirements.txt
View file @ 04088c8b
aniso8601==1.3.0 aniso8601==9.0.1
click==6.7 attrs==21.2.0
Flask==0.12.2 click==7.1.2
Flask-Cors==3.0.6 eventlet==0.30.2
Flask-RESTful==0.3.6 Flask==1.1.4
flask-restplus==0.10.1 Flask-RESTful==0.3.9
itsdangerous==0.24 flask-restplus==0.13.0
Jinja2==2.10 gunicorn==20.1.0
jsonschema==2.6.0 itsdangerous==1.1.0
MarkupSafe==1.0 Jinja2==2.11.3
natsort==5.1.1 jsonschema==3.2.0
psycopg2==2.7.3.2 MarkupSafe==2.0.1
python-dateutil==2.6.1 natsort==7.1.1
pytz==2017.3 psycopg2==2.9.1
six==1.11.0 pyrsistent==0.17.3
Werkzeug==0.13 pytz==2021.1
gunicorn==19.7.1 six==1.16.0
\ No newline at end of file Werkzeug==1.0.1
resources/meta.py
View file @ 04088c8b
#from flask_restful import Resource
from flask import current_app from flask import current_app
from flask import render_template,make_response,send_file from flask import render_template,make_response,send_file
# ISSUE: https://github.com/noirbizarre/flask-restplus/issues/777
import werkzeug
werkzeug.cached_property = werkzeug.utils.cached_property
from flask_restplus import Resource from flask_restplus import Resource
import json import json
import os.path,os import os,os.path
from natsort import natsorted from natsort import natsorted
from common.util import get_url_params from common.util import get_url_params
from common.filters import filter_countries,filter_limit,filter_extra,filter_classification from common.filters import filter_countries,filter_limit,filter_extra,filter_classification
from os import environ
class Logger(object): class Logger(object):
"""We cant get the logging object in the resources""" """We cant get the logging object in the resources"""
def info(self): def info(self):
app = current_app._get_current_object() app = current_app._get_current_object()
return app.logger return app.logger
class Monolith(Resource): class Monolith(Resource):
"""Class that return the monoliths available""" """Class that return the monoliths available"""
def __init__(self): def __init__(self):
self.logger = Logger().info() self.logger = Logger().info()
def get(self): def get(self):
#return 'You want path: %s' % path headers = {'Content-Type': 'text/html'}
headers = {'Content-Type': 'text/html'} return make_response(render_template('index.html'),200,headers)
#return render_template('index.html')
return make_response(render_template('index.html'),200,headers)
#return "Yes I have monoliths"
class MonolithRoot(Resource): class MonolithRoot(Resource):
"""Class that return the monoliths available""" """Class that return the monoliths available"""
def __init__(self): def __init__(self):
self.logger = Logger().info() self.logger = Logger().info()
def get(self): def get(self):
#return 'You want path: %s' % path headers = {'Content-Type': 'application/json'}
#headers = {'Content-Type': 'text/html'} monolithsPath = environ.get("MONOLITH_PATH")
#return render_template('index.html') listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
#return make_response(render_template('index.html'),200,headers)
headers = {'Content-Type': 'application/json'}
monolithsPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/" )
#listDir=[o for o in os.listdir("./monoliths/") if os.path.isdir("./monoliths/"+o)]
listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
response = json.dumps(listDir) response = json.dumps(listDir)
return make_response(response,200,headers) return make_response(response,200,headers)
class CloseBy(Resource): class CloseBy(Resource):
def __init__(self): def __init__(self):
...@@ -57,43 +53,37 @@ class CloseBy(Resource): ...@@ -57,43 +53,37 @@ class CloseBy(Resource):
class Monoliths(Resource): class Monoliths(Resource):
def __init__(self): def __init__(self):
self.logger = Logger().info() self.logger = Logger().info()
def get(self,path): def get(self,path):
monolithPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/",path ) monolithPath = os.path.join(environ.get("MONOLITH_PATH"),path)
#monolithPath=os.path.join("./monoliths/"+path) #monolithPath=os.path.join("./monoliths/"+path)
#self.logger.info("./monoliths/"+path) #self.logger.info("./monoliths/"+path)
#if os.path.isdir(relativePath) #if os.path.isdir(relativePath)
#http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES #http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES
if os.path.isdir(monolithPath): if os.path.isdir(monolithPath):
#send JSON with content #send JSON with content
#http://127.0.0.1:5000/monoliths/10per/jpg #http://127.0.0.1:5000/monoliths/10per/jpg
listFile=natsorted(os.listdir(monolithPath)) listFile = natsorted(os.listdir(monolithPath))
req=get_url_params()
req=get_url_params()
if req["country"]: if req["country"]:
listFile=filter_countries(req["country"],listFile) listFile=filter_countries(req["country"],listFile)
if req["classification"]: if req["classification"]:
listFile=filter_classification(req["classification"],listFile) listFile=filter_classification(req["classification"],listFile)
if req["limit"]: if req["limit"]:
listFile=filter_limit(req["limit"] ,listFile) listFile=filter_limit(req["limit"] ,listFile)
#PASOP THIS HAS TO BE THE LAST FILTER #PASOP THIS HAS TO BE THE LAST FILTER
if req["extra"]: if req["extra"]:
listFile=filter_extra(listFile) listFile=filter_extra(listFile)
headers = {'Content-Type': 'application/json'}
headers = {'Content-Type': 'application/json'} response = json.dumps(listFile)
response = json.dumps(listFile) return make_response(response,200,headers)
return make_response(response,200,headers) #return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg
#return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg else:
else: #send file
#send file return send_file(monolithPath)
return send_file(monolithPath)
#send_file
#return 'You want path: %s' % str(os.path.isdir(relativePath))
#return 1
\ No newline at end of file
tools/createTableImage.py 0 → 100644
View file @ 04088c8b
#select p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'');
#creating table CREATE TABLE rest.images(country_id character(2) NOT NULL,profile_id integer NOT NULL,profile_code character varying(80) NOT NULL, reference_soil_group character varying(30) NOT NULL,file_path character varying(250) NOT NULL, geom geometry);
import psycopg2
import psycopg2.extras
import glob, re,csv
SQLClassTable = """select distinct p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'')"""
tiffLocation="/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif"
fileList=glob.glob(tiffLocation)
#/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif
conn = psycopg2.connect("dbname='isric' user='jjesus' host='85.214.35.184' password='isric126'")
cur = conn.cursor(cursor_factory = psycopg2.extras.NamedTupleCursor)
cur.execute(SQLClassTable)
res = cur.fetchall()
#Record(country_id='FI', geom='0101000020E61000004F1BE8B4812E38406DA0D3063AB54E40', profile_id=69734, profile_code='FI002', classification='Podzols')
f1=open("/tmp/f1.csv","w")
w=csv.writer(f1)
w.writerow(("country_id","profile_id","profile_code","classification","path"))
for record in res:
profile_code=record.profile_code[0:2]+'-'+record.profile_code[2:5]
fileOK = [x for x in fileList if re.search(profile_code, x)]
if fileOK:
w.writerow([record.country_id,record.profile_id,record.profile_code,record.classification,str(fileOK)])
print "done"
#try:
# conn = psycopg2.connect("dbname='web_isis' user='jjesus' host='85.214.35.184' password='isric126'")
#except:
# print "I am unable to connect to the database"
tools/imageProcessing/GeomNoPics.csv 0 → 100644
View file @ 04088c8b
#Missing pictures, profiles on ISIS with coordinates but no pictures
AU-012
AU-016
AU-009
AU-030
AU-032
AU-010
AU-039
AU-011
AU-006
AU-029
AU-021
AU-008
AU-033
AU-018
AU-015
BE-003
BE-001
BE-001
BE-002
BJ-001
BR-013
CA-017
CA-014
CD-002
CD-001
CI-006
CI-007
CM-001
CN-016
CN-015
CN-011
CN-010
CN-050
CN-051
CN-048
CN-049
CN-018
CN-006
CN-005
CN-007
CN-032
CN-029
CN-025
CN-022
CN-024
CN-028
CN-026
CN-001
CN-035
CN-042
CN-037
CO-014
CO-018
CR-003
CU-004
CU-016
CU-015
DE-004
DK-003
DK-001
EC-020
EC-018
EC-001
FI-005
FI-003
FR-007
GL-002
GL-004
ID-023
ID-013
ID-025
IS-007
JM-001
JM-002
JM-003
JM-004
JM-007
JM-005
KE-018
KE-019
KE-026
KE-025
KE-035
KE-037
KE-002
KE-016
KE-005
KE-034
KE-023
KE-024
KE-038
KE-007
KE-022
KE-039
KE-042
KE-041
KE-043
KE-021
KE-046
KE-045
KE-036
KE-047
KE-050
KE-008
KE-055
KE-014
KE-012
KE-033
KE-031
KE-063
KE-032
KE-029
KE-011
KE-030
KE-028
LK-001
MZ-009
MZ-001
MZ-002
NA-006
NA-004
NA-008
NG-020
NG-017
NG-025
NG-022
NG-015
NI-011
NI-005
NI-004
NI-009
NI-008
NL-035
NL-005
NL-018
NZ-003
PE-012
PE-015
PE-013
PE-014
PE-017
PE-018
PE-011
RU-004
RU-010
RU-009
RU-011
RU-012
RU-013
RU-002
RU-015
RU-014
RW-002
RW-001
SK-001
SK-002
SK-006
SK-005
SK-004
SK-003
SK-007
TJ-001
US-005
US-006
US-007
US-027
UY-008
ZM-003
ZW-010
ZW-008