Merge pull request bird-house#44 in OGC/testbed14-twitcher from proce…

…sses-not-visible to dynamic-wps-processes

* commit '1a574f1303b9e2896dfec74b218c443b6ec711b1':
  reapply Process.identifier 'pop'
  fix error expected failure
  fix duplicate test name
  more variable checking + job execute if visible + unittests for execute & visibility get/put
  owsexception testing json response
  delete process visibility filter
  unittest process deploy
  fixes for describe process visibility + json rendering + proceses unittests + typing
  wps 1.0 test execute success/error on public/private visibility
  config setup for wps testing
  many changes to allow testing wps 1.0 filtred by visibility
  public/private process in self
  more pywps configs and tests
  wps tests
  typing
  filter WPS 1.0 processes according to adapter (MagpieAdapter perms)

setup.cfg

@@ -6,7 +6,7 @@ addopts =
 python_files = test_*.py
 markers =
     demo: mark test as demo validation
-    functional: mark test as funtionality validation
+    functional: mark test as functionality validation
     online: mark test to need internet connection
     slow: mark test to be slow
 

twitcher/adapter/__init__.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, Text
+from typing import Dict, AnyStr
 from twitcher.adapter.default import DefaultAdapter, AdapterInterface
 
 LOGGER = logging.getLogger("TWITCHER")
@@ -19,7 +19,7 @@ def import_adapter(name):
 
 
 def adapter_factory(settings):
-    # type: (Dict[Text, Text]) -> AdapterInterface
+    # type: (Dict[AnyStr, AnyStr]) -> AdapterInterface
     """
     Creates an adapter with the interface of :class:`twitcher.adapter.AdapterInterface`.
     By default the twitcher.adapter.DefaultAdapter implementation will be used.

twitcher/datatype.py

@@ -9,7 +9,7 @@
 from datetime import timedelta
 # noinspection PyProtectedMember
 from logging import _levelNames, ERROR, INFO
-
+from typing import Any, AnyStr, Dict, List, Optional, Union
 from twitcher.utils import (
     now,
     now_secs,
@@ -21,7 +21,9 @@
 )
 from twitcher.exceptions import ProcessInstanceError
 from twitcher.processes import process_mapping
-from twitcher.processes.types import PACKAGE_PROCESSES, PROCESS_WPS
+from twitcher.processes.types import PROCESS_WITH_MAPPING, PROCESS_WPS
+# noinspection PyProtectedMember
+from twitcher.processes.wps_package import _wps2json_io
 from twitcher.status import job_status_values, STATUS_UNKNOWN
 from twitcher.visibility import visibility_values, VISIBILITY_PRIVATE
 from pywps import Process as ProcessWPS
@@ -468,83 +470,108 @@ def __setattr__(self, item, value):
 
     @property
     def id(self):
+        # type: (...) -> AnyStr
         return self['id']
 
     @property
     def identifier(self):
+        # type: (...) -> AnyStr
         return self.id
 
+    @identifier.setter
+    def identifier(self, value):
+        # type: (AnyStr) -> None
+        self['id'] = value
+
     @property
     def title(self):
+        # type: (...) -> AnyStr
         return self.get('title', self.id)
 
     @property
     def abstract(self):
+        # type: (...) -> AnyStr
         return self.get('abstract', '')
 
     @property
     def keywords(self):
+        # type: (...) -> List[AnyStr]
         return self.get('keywords', [])
 
     @property
     def metadata(self):
+        # type: (...) -> List[AnyStr]
         return self.get('metadata', [])
 
     @property
     def version(self):
+        # type: (...) -> Union[None, AnyStr]
         return self.get('version')
 
     @property
     def inputs(self):
+        # type: (...) -> Union[None, List[Dict[AnyStr, Any]]]
         return self.get('inputs')
 
     @property
     def outputs(self):
+        # type: (...) -> Union[None, List[Dict[AnyStr, Any]]]
         return self.get('outputs')
 
     @property
     def jobControlOptions(self):
+        # type: (...) -> Union[None, List[AnyStr]]
         return self.get('jobControlOptions')
 
     @property
     def outputTransmission(self):
+        # type: (...) -> Union[None, List[AnyStr]]
         return self.get('outputTransmission')
 
     @property
     def processDescriptionURL(self):
+        # type: (...) -> Union[None, AnyStr]
         return self.get('processDescriptionURL')
 
     @property
     def processEndpointWPS1(self):
+        # type: (...) -> Optional[AnyStr]
         return self.get('processEndpointWPS1')
 
     @property
     def executeEndpoint(self):
+        # type: (...) -> Union[None, AnyStr]
         return self.get('executeEndpoint')
 
     @property
     def owsContext(self):
+        # type: (...) -> Union[None, Dict[AnyStr, Any]]
         return self.get('owsContext')
 
     # wps, workflow, etc.
     @property
     def type(self):
-        return self.get('type')
+        # type: (...) -> AnyStr
+        return self.get('type', 'WPS')
 
     @property
     def package(self):
+        # type: (...) -> Union[None, Dict[AnyStr, Any]]
         return self.get('package')
 
     @property
     def payload(self):
+        # type: (...) -> Union[None, Dict[AnyStr, Any]]
         return self.get('payload')
 
     @property
     def visibility(self):
+        # type: (...) -> AnyStr
         return self.get('visibility', VISIBILITY_PRIVATE)
 
     @visibility.setter
     def visibility(self, visibility):
+        # type: (AnyStr) -> None
         if not isinstance(visibility, six.string_types):
             raise TypeError("Type `str` is required for `{}.visibility`".format(type(self)))
         if visibility not in visibility_values:
@@ -553,15 +580,18 @@ def visibility(self, visibility):
         self['visibility'] = visibility
 
     def __str__(self):
+        # type: (...) -> AnyStr
         return "Process <{0}> ({1})".format(self.identifier, self.title)
 
     def __repr__(self):
+        # type: (...) -> AnyStr
         cls = type(self)
         repr_ = dict.__repr__(self)
         return '{0}.{1}({2})'.format(cls.__module__, cls.__name__, repr_)
 
     @property
     def params(self):
+        # type: (...) -> Dict[AnyStr, Any]
         return {
             'identifier': self.identifier,
             'title': self.title,
@@ -582,7 +612,8 @@ def params(self):
 
     @property
     def params_wps(self):
-        """Values applicable to WPS Process __init__"""
+        # type: (...) -> Dict[AnyStr, Any]
+        """Values applicable to PyWPS Process __init__"""
         return {
             'identifier': self.identifier,
             'title': self.title,
@@ -597,19 +628,36 @@ def params_wps(self):
         }
 
     def json(self):
+        # type: (...) -> Dict[AnyStr, Any]
         return sd.Process().deserialize(self)
 
     def process_offering(self):
-        return sd.ProcessOffering().deserialize(ProcessOffering(self))
+        # type: (...) -> Dict[AnyStr, Any]
+        process_offering = {"process": self}
+        if self.version:
+            process_offering.update({"processVersion": self.version})
+        if self.jobControlOptions:
+            process_offering.update({"jobControlOptions": self.jobControlOptions})
+        if self.outputTransmission:
+            process_offering.update({"outputTransmission": self.outputTransmission})
+        return sd.ProcessOffering().deserialize(process_offering)
 
     def process_summary(self):
+        # type: (...) -> Dict[AnyStr, Any]
         return sd.ProcessSummary().deserialize(self)
 
     @staticmethod
     def from_wps(wps_process, **extra_params):
+        # type: (ProcessWPS, Any) -> Process
+        """
+        Converts a PyWPS Process into a `twitcher.datatype.Process` using provided parameters.
+        """
         assert isinstance(wps_process, ProcessWPS)
         process = wps_process.json
-        process.update({'type': wps_process.identifier, 'package': None, 'reference': None})
+        process_type = getattr(wps_process, 'type', wps_process.identifier)
+        process.update({'type': process_type, 'package': None, 'reference': None,
+                        'inputs': [_wps2json_io(i) for i in wps_process.inputs],
+                        'outputs': [_wps2json_io(o) for o in wps_process.outputs]})
         process.update(**extra_params)
         return Process(process)
 
@@ -618,22 +666,12 @@ def wps(self):
         if self.type == PROCESS_WPS:
             process_key = self.identifier
         if process_key not in process_mapping:
-            ProcessInstanceError("Unknown process `{}` in mapping".format(process_key))
-        if process_key in PACKAGE_PROCESSES:
+            ProcessInstanceError("Unknown process `{}` in mapping.".format(process_key))
+        if process_key in PROCESS_WITH_MAPPING:
             return process_mapping[process_key](**self.params_wps)
         return process_mapping[process_key]()
 
 
-class ProcessOffering(dict):
-    def __init__(self, process):
-        super(ProcessOffering, self).__init__({
-            "process": process,
-            "processVersion": process.version,
-            "jobControlOptions": process.jobControlOptions,
-            "outputTransmission": process.outputTransmission,
-        })
-
-
 class Quote(dict):
     """
     Dictionary that contains quote information.

twitcher/exceptions.py

@@ -3,6 +3,14 @@
 """
 
 
+class InvalidIdentifierValue(ValueError):
+    """
+    Error indicating that an id to be employed for following operations
+    is not considered as valid to allow further processed or usage.
+    """
+    pass
+
+
 class AccessTokenNotFound(Exception):
     """
     Error indicating that an access token could not be read from the
@@ -27,9 +35,17 @@ class ServiceRegistrationError(Exception):
     pass
 
 
+class ProcessNotAccessible(Exception):
+    """
+    Error indicating that a local WPS process exists but is not visible to retrieve
+    from the storage backend of an instance of :class:`twitcher.store.ProcessStore`.
+    """
+    pass
+
+
 class ProcessNotFound(Exception):
     """
-    Error indicating that a local WPS service could not be read from the
+    Error indicating that a local WPS process could not be read from the
     storage backend by an instance of :class:`twitcher.store.ProcessStore`.
     """
     pass

twitcher/execute.py

@@ -0,0 +1,25 @@
+EXECUTE_MODE_AUTO = 'auto'
+EXECUTE_MODE_ASYNC = 'async'
+EXECUTE_MODE_SYNC = 'sync'
+
+execute_mode_options = frozenset([
+    EXECUTE_MODE_AUTO,
+    EXECUTE_MODE_ASYNC,
+    EXECUTE_MODE_SYNC,
+])
+
+EXECUTE_RESPONSE_RAW = 'raw'
+EXECUTE_RESPONSE_DOCUMENT = 'document'
+
+execute_response_options = frozenset([
+    EXECUTE_RESPONSE_RAW,
+    EXECUTE_RESPONSE_DOCUMENT,
+])
+
+EXECUTE_TRANSMISSION_MODE_VALUE = 'value'
+EXECUTE_TRANSMISSION_MODE_REFERENCE = 'reference'
+
+execute_transmission_mode_options = frozenset([
+    EXECUTE_TRANSMISSION_MODE_VALUE,
+    EXECUTE_TRANSMISSION_MODE_REFERENCE,
+])

twitcher/namesgenerator.py

@@ -5,6 +5,7 @@
 `docker names-generator.go <https://github.com/docker/docker/blob/master/pkg/namesgenerator/names-generator.go>`_
 """
 
+from twitcher.exceptions import InvalidIdentifierValue
 import random
 import re
 
@@ -169,12 +170,12 @@ def get_sane_name(name, minlen=3, maxlen=None, assert_invalid=True, replace_inva
 
 def assert_sane_name(name, minlen=3, maxlen=None):
     if name is None:
-        raise ValueError('Invalid process name : {0}'.format(name))
+        raise InvalidIdentifierValue('Invalid process name : {0}'.format(name))
     name = name.strip()
     if '--' in name \
        or name.startswith('-') \
        or name.endswith('-') \
        or len(name) < minlen \
        or (maxlen is not None and len(name) > maxlen) \
        or not re.match("^[a-zA-Z0-9_\-]+$", name):
-        raise ValueError('Invalid process name : {0}'.format(name))
+        raise InvalidIdentifierValue('Invalid process name : {0}'.format(name))

twitcher/owsexceptions.py

@@ -6,16 +6,14 @@
 
 import json
 from string import Template
-
+from typing import AnyStr, Dict
+# noinspection PyPackageRequirements
 from zope.interface import implementer
-
 from webob import html_escape as _html_escape
-from webob.acceptparse import MIMEAccept
-
+from webob.acceptparse import create_accept_header
 from pyramid.interfaces import IExceptionResponse
 from pyramid.response import Response
 from pyramid.compat import text_type
-import os
 
 
 @implementer(IExceptionResponse)
@@ -47,16 +45,20 @@ def __init__(self, detail=None, value=None, **kw):
     def __str__(self, skip_body=False):
         return self.message
 
+    # noinspection PyUnusedLocal
     @staticmethod
     def json_formatter(status, body, title, environ):
-        # Remove new line symbol and multiple spaces from body
-        return {'description': ' '.join(body.replace('\n\n', '. ').split()),
-                'code': status}
+        # type: (AnyStr, AnyStr, AnyStr, Dict[AnyStr, AnyStr]) -> Dict[AnyStr, AnyStr]
+        body_parts = [p.strip() for p in body.split('\n') if p != '']               # remove new line and extra spaces
+        body_parts = [p + '.' if not p.endswith('.') else p for p in body_parts]    # add terminating dot per sentence
+        body_parts = [p[0].upper() + p[1:] for p in body_parts if len(p)]           # capitalize first word
+        body_parts = ' '.join(p for p in body_parts if p)
+        return {'description': body_parts, 'code': status}
 
     def prepare(self, environ):
         if not self.body:
             accept_value = environ.get('HTTP_ACCEPT', '')
-            accept = MIMEAccept(accept_value)
+            accept = create_accept_header(accept_value)
 
             # Attempt to match text/xml or application/json, if those don't
             # match, we will fall through to defaulting to text/xml
@@ -73,6 +75,7 @@ class JsonPageTemplate(object):
                     def __init__(self, excobj):
                         self.excobj = excobj
 
+                    # noinspection PyUnusedLocal
                     def substitute(self, code, locator, message):
                         return json.dumps(self.excobj.json_formatter(
                             status=code, body=message, title=None, environ=environ))