From e924dd8da72501785c064f81e008cb44d1f0500a Mon Sep 17 00:00:00 2001 From: Jorge de Jesus Date: Sat, 29 Dec 2018 11:12:20 +0100 Subject: [PATCH 1/3] PYWPS_PROCESS and PYWPS_CFG implementation --- .gitignore | 3 + .travis.yml | 8 ++- demo.py | 116 ++++++++++++++++++++++---------------- processes/total_length.py | 45 +++++++++++++++ pywps.cfg | 2 +- setup.py | 23 ++++---- tests/__init__.py | 4 +- tests/test_variables.py | 115 +++++++++++++++++++++++++++++++++++++ wsgi/pywps.wsgi | 60 +++++++++++++------- 9 files changed, 290 insertions(+), 86 deletions(-) create mode 100644 processes/total_length.py create mode 100644 tests/test_variables.py diff --git a/.gitignore b/.gitignore index e30a284..2b1e8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ pywps_demo.egg-info/ # docs build artifacts _build/ +/pywps_flask.egg-info/ +/src/ +/.pytest_cache/ diff --git a/.travis.yml b/.travis.yml index bbaa199..63db281 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.7" - - "3.4" + - "3.6" # Handle git submodules yourself git: @@ -14,6 +14,8 @@ install: - pip install -r requirements.txt before_script: - "python demo.py -d" + - python demo.py -a -d -script: "python -m unittest tests" +script: +# - find . -type f -name "*.py" -not -path "./src/*" -not -path "./build/*"| xargs flake8 + - python -m unittest tests diff --git a/demo.py b/demo.py index e3eff59..7bb609c 100755 --- a/demo.py +++ b/demo.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 # Copyright (c) 2016 PyWPS Project Steering Committee -# -# +# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# +# +# The above copyright notice and this permission +# notice shall be included in all copies +# or substantial portions of the Software. +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -22,50 +23,55 @@ # SOFTWARE. import os +import glob +import inspect +import importlib import flask - import pywps from pywps import Service -from processes.sleep import Sleep -from processes.ultimate_question import UltimateQuestion -from processes.centroids import Centroids -from processes.sayhello import SayHello -from processes.feature_count import FeatureCount -from processes.buffer import Buffer -from processes.area import Area -from processes.bboxinout import Box -from processes.jsonprocess import TestJson +# dynamically loading processes, +# using PYWPS_PROCESSES if not the load default processes on folder processes + +def get_processes(): + + PYWPS_PROCESSES = os.environ["PYWPS_PROCESSES"] if "PYWPS_PROCESSES" in os.environ else "./processes" + package_processes = os.path.basename(os.path.abspath(PYWPS_PROCESSES)) + modules = glob.glob(os.path.abspath(PYWPS_PROCESSES)+"/*.py") + # need relative modules with (.) + modules = ["." + os.path.basename(f)[:-3] + for f in modules if os.path.isfile(f) + and not f.endswith('__init__.py')] + + processes = [] + for module_name in modules: + module = importlib.import_module(module_name, package_processes) + # getting all member is module and + # then fiter for classes and pywps processes + process_classes = inspect.\ + getmembers(module, lambda member: inspect.isclass(member) + and member.__module__ == module.__name__) + for process_class in process_classes: + # [('Sleep', )] + processes.append(getattr(module, process_class[0]).__call__()) + + + # For the process dict on the home page + return processes app = flask.Flask(__name__) -processes = [ - FeatureCount(), - SayHello(), - Centroids(), - UltimateQuestion(), - Sleep(), - Buffer(), - Area(), - Box(), - TestJson() -] - -# For the process list on the home page - -process_descriptor = {} -for process in processes: - abstract = process.abstract - identifier = process.identifier - process_descriptor[identifier] = abstract - -# This is, how you start PyWPS instance -service = Service(processes, ['pywps.cfg']) - - @app.route("/") def hello(): + + process_descriptor = {} + + for process in get_processes(): + abstract = process.abstract + identifier = process.identifier + process_descriptor[identifier] = abstract + server_url = pywps.configuration.get_config_value("server", "url") request_url = flask.request.url return flask.render_template('home.html', request_url=request_url, @@ -75,8 +81,15 @@ def hello(): @app.route('/wps', methods=['GET', 'POST']) def wps(): + # Need to determine config files + def create_wps_app(): + config_files = [os.path.join(os.path.dirname(__file__), 'pywps.cfg')] + if 'PYWPS_CFG' in os.environ: + config_files.append(os.environ['PYWPS_CFG']) + service = Service(processes=get_processes(), cfgfiles=config_files) + return service - return service + return create_wps_app() @app.route('/outputs/'+'') @@ -105,7 +118,9 @@ def staticfile(filename): else: flask.abort(404) + if __name__ == "__main__": + import argparse parser = argparse.ArgumentParser( @@ -118,15 +133,18 @@ def staticfile(filename): ) parser.add_argument('-d', '--daemon', action='store_true', help="run in daemon mode") - parser.add_argument('-a','--all-addresses', - action='store_true', help="run flask using IPv4 0.0.0.0 (all network interfaces)," + - "otherwise bind to 127.0.0.1 (localhost). This maybe necessary in systems that only run Flask") + parser.add_argument('-a', '--all-addresses', action='store_true', + help="run flask using IPv4 0.0.0.0 " + + "(all network interfaces)," + + "otherwise bind to 127.0.0.1 (localhost)." + + "This maybe necessary in" + + "systems that only run Flask") args = parser.parse_args() - + if args.all_addresses: - bind_host='0.0.0.0' + bind_host = '0.0.0.0' else: - bind_host='127.0.0.1' + bind_host = '127.0.0.1' if args.daemon: pid = None @@ -137,8 +155,8 @@ def staticfile(filename): if (pid == 0): os.setsid() - app.run(threaded=True,host=bind_host) + app.run(threaded=True, host=bind_host) else: os._exit(0) else: - app.run(threaded=True,host=bind_host) + app.run(threaded=True, host=bind_host) diff --git a/processes/total_length.py b/processes/total_length.py new file mode 100644 index 0000000..03a203b --- /dev/null +++ b/processes/total_length.py @@ -0,0 +1,45 @@ +import os +from osgeo import ogr +from pywps import Process, ComplexInput, LiteralOutput, Format +from pywps.wpsserver import temp_dir + +import logging +LOGGER = logging.getLogger('PYWPS') +LOGGER.info('Adquired logger inside total_length.py') + +class TotalLength(Process): + """Process calculating area of given polygon + """ + def __init__(self): + inputs = [ComplexInput('layer', 'Layer', + [Format('application/gml+xml')])] + outputs = [LiteralOutput('total', 'Total', data_type='string')] + + super(TotalLength, self).__init__( + self._handler, + identifier='total_length', + title='Process Total Length', + abstract="""Process returns the total length of all lines in a submitted GML file""", + inputs=inputs, + outputs=outputs, + store_supported=True, + status_supported=True + ) + + def _handler(self, request, response): + with temp_dir() as tmp: + + input_gml = request.inputs['layer'][0].file + + driver = ogr.GetDriverByName("GML") + + dataSource = driver.Open(input_gml, 0) + layer = dataSource.GetLayer() + total = 0 + + LOGGER.info('Hola!! y que tal ???') + + for feature in layer: + total = total + feature.length + response.outputs['total'].data = str(total) + return response diff --git a/pywps.cfg b/pywps.cfg index 01be2c2..e0e1874 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -35,7 +35,7 @@ maxprocesses=10 parallelprocesses=2 [processing] -mode=docker +mode=default port_min=5050 port_max=5070 docker_img=container diff --git a/setup.py b/setup.py index 7c92362..087d261 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ # Copyright (c) -# -# +# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of the Software. +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -33,13 +33,13 @@ INSTALL_REQUIRES.append('pywps=='+VERSION) DESCRIPTION = ( -'''PyWPS is an implementation of the Web Processing Service standard from the + '''PyWPS is an implementation of the Web Processing Service standard from the Open Geospatial Consortium. PyWPS is written in Python. -PyWPS-Flask is an example service using the PyWPS server, distributed along -with a basic set of sample processes and sample configuration file. It's +PyWPS-Flask is an example service using the PyWPS server, distributed along +with a basic set of sample processes and sample configuration file. It's usually used for testing and development purposes. -''') + ''') KEYWORDS = 'PyWPS WPS OGC processing' @@ -67,7 +67,8 @@ 'version': VERSION, 'install_requires': INSTALL_REQUIRES, 'dependency_links': [ - 'git+https://github.com/lazaa32/pywps.git@pywps-'+VERSION+'#egg=pywps-'+VERSION + 'git+https://github.com/lazaa32/pywps.git@pywps-' + + VERSION + '#egg=pywps-' + VERSION ], 'packages': ['processes', 'tests'], 'scripts': ['demo.py'], diff --git a/tests/__init__.py b/tests/__init__.py index 038d3e4..9e76442 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,6 +6,7 @@ from tests import test_execute from tests import test_requests from tests import test_log +from tests import test_variables #from tests import test_exceptions def load_tests(loader=None, tests=None, pattern=None): @@ -14,7 +15,8 @@ def load_tests(loader=None, tests=None, pattern=None): test_describe.load_tests(), test_execute.load_tests(), test_requests.load_tests(), - test_log.load_tests() + test_log.load_tests(), + test_variables.load_tests() ]) if __name__ == "__main__": diff --git a/tests/test_variables.py b/tests/test_variables.py new file mode 100644 index 0000000..33e6801 --- /dev/null +++ b/tests/test_variables.py @@ -0,0 +1,115 @@ + +import sys +sys.path.insert(0,'../..') + +from demo import wps +from lxml import etree +from io import BytesIO +import re,os +import glob +from werkzeug.test import Client +from werkzeug.wrappers import BaseResponse +import unittest +import tempfile +import shutil + +class VariableTest(unittest.TestCase): + + def setUp(self): + """Strategy for testing PYWPS_PROCESSES is to create a dummy process folder with only 1 process + and test if the wps() function from demo.py picks up the variable path. + For testing PYWPS_CFG we create a new pywps_testing.cfg that will sent a 'TESTING' string on the output""" + + self.tmp_dir = tempfile.mkdtemp() + self.test_process = None + self.pywps_cfg_name = "pywps_testing.cfg" + self.testing_str = "TESTING" + + #remember unitests are called from top direcory + with open("pywps.cfg",'r') as cfg_file: + new_cfg_content = cfg_file.read().replace("PyWPS Demo server",self.testing_str) + with open(os.path.join(self.tmp_dir,self.pywps_cfg_name),'w') as new_file: + new_file.write(new_cfg_content) + + #make temporary dummy module + with open(os.path.join(self.tmp_dir,"__init__.py"), 'a'): + os.utime(os.path.join(self.tmp_dir,"__init__.py"), None) + + glob.glob("./processes/*.py") + process_list=glob.glob("./processes/*.py") + try: + process_list.remove("./processes/__init__.py") + except: + pass + + #no random process always the firs in list for consistency + self.test_process = process_list[0] + os.makedirs(os.path.join(self.tmp_dir,"processes")) + shutil.copyfile(self.test_process,os.path.join(self.tmp_dir,"processes",os.path.basename(self.test_process))) + + + def test_process_variable(self): + """Test PYWPS_PROCESSES""" + + os.environ["PYWPS_PROCESSES"]=os.path.join(self.tmp_dir,"processes") + service = wps() + c = Client(service, WPSBaseResponse) + resp = c.get('?request=GetCapabilities&service=wps') + + process_return=resp.get_process_identifers() + + assert len(process_return) == 1 + assert process_return[0] in self.test_process + + def test_config_variable(self): + """Test PYWPS_CFG""" + os.environ["PYWPS_CFG"]= os.path.join(self.tmp_dir,self.pywps_cfg_name) + service = wps() + c = Client(service, WPSBaseResponse) + resp = c.get('?request=GetCapabilities&service=wps') + title= resp.get_title() + assert title == self.testing_str + + def tearDown(self): + """Removing temporary folder""" + shutil.rmtree(self.tmp_dir) + +def load_tests(loader=None, tests=None, pattern=None): + if not loader: + loader = unittest.TestLoader() + suite_list = [ + loader.loadTestsFromTestCase(VariableTest), + ] + return unittest.TestSuite(suite_list) + + + + +class WPSBaseResponse(BaseResponse): + + def __init__(self, *args): + super(WPSBaseResponse, self).__init__(*args) + if re.match(r'text/xml(;\s*charset=.*)?', self.headers.get('Content-Type')): + #self.etree = etree.parse(BytesIO(self.get_data())) + self.etree = etree.parse(BytesIO(self.get_data())).getroot() + + + def get_title(self): + # v1.0.0 and v2.0.0 have same path + # Title is first child element after ServiceIdentification + title_path = "//*[local-name() = 'ServiceIdentification']/*[1]" + el_title = self.etree.xpath(title_path) + if el_title: + return el_title[0].text + else: + return None + + def get_process_identifers(self): + identifier_path = "//*[local-name() = 'Identifier']" + els_identifier=self.etree.xpath(identifier_path) + if els_identifier: + return [el.text for el in els_identifier] + else: + return None + + diff --git a/wsgi/pywps.wsgi b/wsgi/pywps.wsgi index 56628f8..f41f2a0 100644 --- a/wsgi/pywps.wsgi +++ b/wsgi/pywps.wsgi @@ -28,28 +28,46 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -__author__ = "Jachym Cepicky" +import os from pywps.app.Service import Service +from .processes import processes + +def create_app(cfgfiles=None): + config_files = [os.path.join(os.path.dirname(__file__), 'default.cfg')] + print(config_files) + if cfgfiles: + config_files.extend(cfgfiles) + if 'PYWPS_CFG' in os.environ: + config_files.append(os.environ['PYWPS_CFG']) + service = Service(processes=processes, cfgfiles=config_files) + return service + + +application = create_app() + + +#from pywps.app.Service import Service + # processes need to be installed in PYTHON_PATH -from processes.sleep import Sleep -from processes.ultimate_question import UltimateQuestion -from processes.centroids import Centroids -from processes.sayhello import SayHello -from processes.feature_count import FeatureCount -from processes.buffer import Buffer -from processes.area import Area - - -processes = [ - FeatureCount(), - SayHello(), - Centroids(), - UltimateQuestion(), - Sleep(), - Buffer(), - Area() -] - -application = Service(processes, ['pywps.cfg']) +#from processes.sleep import Sleep +#from processes.ultimate_question import UltimateQuestion +#from processes.centroids import Centroids +#from processes.sayhello import SayHello +#from processes.feature_count import FeatureCount +#from processes.buffer import Buffer +#from processes.area import Area + + +#processes = [ +# FeatureCount(), +# SayHello(), +# Centroids(), +# UltimateQuestion(), +# Sleep(), +# Buffer(), +# Area() +#] + +#application = Service(processes, ['pywps.cfg']) From 4151dd6352deab11327e64918c38466da0487f87 Mon Sep 17 00:00:00 2001 From: Jorge de Jesus Date: Sat, 29 Dec 2018 11:12:58 +0100 Subject: [PATCH 2/3] missing data? --- static/data/point.gfs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 static/data/point.gfs diff --git a/static/data/point.gfs b/static/data/point.gfs new file mode 100644 index 0000000..82dc96b --- /dev/null +++ b/static/data/point.gfs @@ -0,0 +1,20 @@ + + + point + point + + 1 + + 1 + 0.04854 + 0.04854 + 0.01369 + 0.01369 + + + id + id + Integer + + + From e3a7e016d190d90bf529c9e388dbeaa0833f1cb3 Mon Sep 17 00:00:00 2001 From: Jorge de Jesus Date: Sun, 30 Dec 2018 09:06:38 +0100 Subject: [PATCH 3/3] missing [-1] in assert tests --- point_buffer | 19 +++++++++++++++++++ point_buffer.gfs | 15 +++++++++++++++ tests/test_variables.py | 4 +++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 point_buffer create mode 100644 point_buffer.gfs diff --git a/point_buffer b/point_buffer new file mode 100644 index 0000000..44d219f --- /dev/null +++ b/point_buffer @@ -0,0 +1,19 @@ + + + + + -0.9514645979959721-0.986306232731747 + 1.0485354020040281.013693767268253 + + + + + + 1.04853540200403,0.013693767268253 1.0471649367586,-0.038642188974691 1.0430572973723,-0.0908346959994 1.03622374259917,-0.142740697771978 1.02668300273783,-0.194217923549506 1.0144612282931,-0.245125277834268 0.999591918299182,-0.295323227106694 0.98211582850123,-0.344674182277047 0.962080859646629,-0.393042875807547 0.939541926192396,-0.440296732471293 0.914560805788467,-0.486306232731747 0.887205969949452,-0.530945267746774 0.857552396378976,-0.57409148502422 0.825681363460999,-0.615626623781584 0.791680227481423,-0.655436839090605 0.755642183190576,-0.693413013918294 0.717666008362887,-0.729451058209141 0.677855793053866,-0.763452194188717 0.636320654296502,-0.795323227106694 0.593174437019056,-0.824976800677171 0.548535402004029,-0.852331636516185 0.502525901743576,-0.877312756920114 0.455272045079829,-0.899851690374347 0.40690335154933,-0.919886659228948 0.357552396378977,-0.9373627490269 0.30735444710655,-0.952232059020815 0.256447092821789,-0.964453833465552 0.204969867044261,-0.973994573326884 0.153063865271684,-0.98082812810002 0.100871358246974,-0.984935767486321 0.048535402004031,-0.986306232731747 -0.003800554238913,-0.984935767486321 -0.055993061263623,-0.980828128100021 -0.1078990630362,-0.973994573326885 -0.159376288813728,-0.964453833465553 -0.21028364309849,-0.952232059020816 -0.260481592370916,-0.937362749026902 -0.309832547541269,-0.91988665922895 -0.358201241071769,-0.899851690374349 -0.405455097735516,-0.877312756920117 -0.451464597995969,-0.852331636516187 -0.496103633010996,-0.824976800677173 -0.539249850288442,-0.795323227106697 -0.580784989045806,-0.763452194188721 -0.620595204354827,-0.729451058209144 -0.658571379182516,-0.693413013918298 -0.694609423473363,-0.655436839090609 -0.72861055945294,-0.615626623781588 -0.760481592370917,-0.574091485024224 -0.790135165941393,-0.530945267746778 -0.817490001780408,-0.486306232731752 -0.842471122184337,-0.440296732471299 -0.865010055638571,-0.393042875807552 -0.885045024493172,-0.344674182277053 -0.902521114291124,-0.2953232271067 -0.917390424285039,-0.245125277834274 -0.929612198729776,-0.194217923549512 -0.939152938591109,-0.142740697771984 -0.945986493364245,-0.090834695999407 -0.950094132750546,-0.038642188974697 -0.951464597995972,0.013693767268246 -0.950094132750546,0.06602972351119 -0.945986493364246,0.118222230535899 -0.939152938591111,0.170128232308477 -0.929612198729779,0.221605458086005 -0.917390424285042,0.272512812370766 -0.902521114291128,0.322710761643193 -0.885045024493177,0.372061716813546 -0.865010055638576,0.420430410344046 -0.842471122184344,0.467684267007793 -0.817490001780415,0.513693767268246 -0.790135165941401,0.558332802283273 -0.760481592370924,0.601479019560719 -0.728610559452948,0.643014158318084 -0.694609423473372,0.682824373627105 -0.658571379182526,0.720800548454794 -0.620595204354837,0.756838592745641 -0.580784989045817,0.790839728725218 -0.539249850288453,0.822710761643195 -0.496103633011006,0.852364335213672 -0.451464597995979,0.879719171052687 -0.405455097735526,0.904700291456617 -0.358201241071779,0.927239224910851 -0.309832547541279,0.947274193765452 -0.260481592370926,0.964750283563404 -0.2102836430985,0.979619593557319 -0.159376288813738,0.991841368002057 -0.107899063036209,1.00138210786339 -0.055993061263632,1.00821566263653 -0.003800554238922,1.01232330202283 0.048535402004022,1.01369376726825 0.100871358246967,1.01232330202283 0.153063865271677,1.00821566263653 0.204969867044254,1.00138210786339 0.256447092821783,0.99184136800206 0.307354447106545,0.979619593557322 0.357552396378972,0.964750283563408 0.406903351549325,0.947274193765456 0.455272045079825,0.927239224910855 0.502525901743572,0.904700291456622 0.548535402004026,0.879719171052693 0.593174437019053,0.852364335213678 0.636320654296499,0.822710761643202 0.677855793053864,0.790839728725225 0.717666008362885,0.756838592745648 0.755642183190575,0.720800548454801 0.791680227481422,0.682824373627112 0.825681363460999,0.643014158318091 0.857552396378975,0.601479019560726 0.887205969949452,0.55833280228328 0.914560805788467,0.513693767268253 0.939541926192396,0.467684267007799 0.962080859646629,0.420430410344052 0.98211582850123,0.372061716813552 0.999591918299182,0.322710761643199 1.0144612282931,0.272512812370772 1.02668300273783,0.22160545808601 1.03622374259917,0.170128232308481 1.0430572973723,0.118222230535904 1.0471649367586,0.066029723511194 1.04853540200403,0.013693767268253 + + + diff --git a/point_buffer.gfs b/point_buffer.gfs new file mode 100644 index 0000000..13446aa --- /dev/null +++ b/point_buffer.gfs @@ -0,0 +1,15 @@ + + + point_buffer + point_buffer + + 3 + + 1 + -0.95146 + 1.04854 + -0.98631 + 1.01369 + + + diff --git a/tests/test_variables.py b/tests/test_variables.py index 33e6801..36301f8 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -59,7 +59,7 @@ def test_process_variable(self): process_return=resp.get_process_identifers() assert len(process_return) == 1 - assert process_return[0] in self.test_process + assert process_return[0][-1] in self.test_process def test_config_variable(self): """Test PYWPS_CFG""" @@ -72,6 +72,8 @@ def test_config_variable(self): def tearDown(self): """Removing temporary folder""" + os.environ.pop("PYWPS_PROCESSES",None) + os.environ.pop("PYWPS_CFG",None) shutil.rmtree(self.tmp_dir) def load_tests(loader=None, tests=None, pattern=None):