Merge pull request #1184 from ITISFoundation/FREEZE_Ueberwald_early

Freeze ueberwald hotfixes

.env-devel

@@ -13,13 +13,12 @@ POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
 
-RABBITMQ_USER=admin
-RABBITMQ_LOG_CHANNEL=comp.backend.channels.log
-RABBITMQ_PASSWORD=adminadmin
-RABBITMQ_PROGRESS_CHANNEL=comp.backend.channels.progress
-
 RABBIT_HOST=rabbit
+RABBIT_LOG_CHANNEL=comp.backend.channels.log
+RABBIT_PASSWORD=adminadmin
 RABBIT_PORT=5672
+RABBIT_PROGRESS_CHANNEL=comp.backend.channels.progress
+RABBIT_USER=admin
 
 REGISTRY_AUTH=True
 REGISTRY_PW=adminadmin
@@ -43,3 +42,6 @@ WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=1
 # python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())"
 WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32.
 WEBSERVER_STUDIES_ACCESS_ENABLED=0
+WEBSERVER_PROMETHEUS_HOST=http://prometheus
+WEBSERVER_PROMETHEUS_PORT=9090
+WEBSERVER_PROMETHEUS_API_VERSION=v1

api/specs/webserver/v0/components/schemas/activity.yaml

@@ -0,0 +1,38 @@
+ActivityEnveloped:
+  type: object
+  required:
+    - data
+  properties:
+    data:
+      $ref: '#/Activity'
+      additionalProperties: true
+    error:
+      nullable: true
+      default: null
+
+Activity:
+  type: object
+  properties:
+    stats:
+      $ref: '#/Status'
+    limits:
+      $ref: '#/Limits'
+    queued:
+      type: boolean
+
+Status:
+  type: object
+  properties:
+    cpuUsage:
+      type: number
+      minimum: 0
+    memoryUsage:
+      type: number
+
+Limits:
+  type: object
+  properties:
+    cpus:
+      type: number
+    mem:
+      type: number

api/specs/webserver/v0/openapi-activity.yaml

@@ -0,0 +1,21 @@
+openapi: 3.0.0
+info:
+  title: activity management API
+  version: 0.1.0
+  description: API to be consumed by the activity manager to list and run actions on services
+servers:
+  - description: API server
+    url: '/v0'
+paths:
+  /activity/status:
+    get:
+      operationId: get_status      
+      responses:
+        '200':
+          description: Object containing queuing, CPU and Memory usage/limits information of services
+          content:
+            application/json:
+              schema:
+                $ref: './components/schemas/activity.yaml#/ActivityEnveloped'
+        default:
+          $ref: './openapi.yaml#/components/responses/DefaultErrorResponse'

api/specs/webserver/v0/openapi-diagnostics.yaml

@@ -31,6 +31,7 @@ paths:
           schema:
             type: string
         - in: path
+          required: true
           name: action
           schema:
             type: string

api/specs/webserver/v0/openapi.yaml

@@ -129,6 +129,10 @@ paths:
   /nodes/{nodeInstanceUUID}/iframe:
     $ref: './openapi-node-v0.0.1.yaml#/paths/~1nodes~1{nodeInstanceUUID}~1iframe'
 
+  # ACTIVITY -------------------------------------------------------------------------
+  /activity/status:
+    $ref: './openapi-activity.yaml#/paths/~1activity~1status'
+
 components:
   responses:
     DefaultErrorResponse:

ci/travis/integration-testing/simcore-sdk

@@ -27,7 +27,7 @@ install() {
 before_script() {
     if bash ci/travis/helpers/test_for_changes "${FOLDER_CHECKS[@]}";
     then
-        pip freeze
+        pip list -v
         # pull the test images if registry is set up, else build the images
         make pull-version || ((make pull-cache || true) && make build tag-version)
         make info-images

ci/travis/system-testing/swarm-deploy

@@ -26,12 +26,9 @@ install() {
 before_script() {
     pip list -v
     make info-images
-    make up-version
 }
 
 script() {
-    # wait for a minute to let the swarm warm up...
-    make info-swarm
     pytest -v tests/swarm-deploy
 }
 

packages/postgres-database/src/simcore_postgres_database/cli.py

@@ -47,9 +47,13 @@ def safe_func(*args, **kargs):
 #@retry(wait=wait_fixed(0.1), stop=stop_after_delay(60))
 def _ping(url):
     """checks whether database is responsive"""
-    engine = sa.create_engine(str(url))
-    conn = engine.connect()
-    conn.close()
+    try:
+        engine = sa.create_engine(str(url))
+        conn = engine.connect()
+        conn.close()
+    finally:
+        engine.dispose()
+
 
 
 @safe(if_fails_return=None)

packages/service-library/src/servicelib/aiopg_utils.py

@@ -1,68 +1,11 @@
-"""
+""" Helpers for aiopg
 
-TODO: test!
+    - aiopg is used as client sdk to interact with postgres database asynchronously
 
-SEE https://aiopg.readthedocs.io/en/stable/
-SEE asyncpg https://magicstack.github.io/asyncpg/current/index.html
 """
-import aiopg.sa
-import attr
-import psycopg2
-import sqlalchemy as sa
-import logging
-import warnings
-
-logger = logging.getLogger(__name__)
-
-warnings.warn("DO NOT USE IN PRODUCTION, STILL UNDER DEVELOPMENT")
-
-@attr.s(auto_attribs=True)
-class AiopgExecutor:
-    """
-        Executes sa statements using aiopg Engine
-
-    SEE https://github.com/aio-libs/aiopg/issues/321
-    SEE http://docs.sqlalchemy.org/en/latest/faq/metadata_schema.html#how-can-i-get-the-create-table-drop-table-output-as-a-string)
-    """
-    engine: aiopg.sa.engine.Engine
-    statement: str=None
-    dsn: str=None # Data Source Name
-
-    @property
-    def sa_engine(self):
-        return sa.create_engine(
-            self.dsn,
-            strategy="mock",
-            executor=self._compile
-        )
-
-    def _compile(self, sql, *multiparams, **params):
-        # pylint: disable=W0613, unused-argument
-        self.statement = str(sql.compile(dialect=self.sa_engine.dialect))
-
-    async def execute(self):
-        async with self.engine.acquire() as conn:
-            logger.debug(self.statement)
-            resp = await conn.execute(self.statement)
-            return resp
 
+from psycopg2 import Error as DBAPIError
 
-
-
-async def create_all(engine: aiopg.sa.engine.Engine, metadata: sa.MetaData, dsn: str):
-    executor = AiopgExecutor(engine, dsn=dsn)
-    metadata.create_all(executor.sa_engine, checkfirst=True)
-    await executor.execute()
-
-
-async def drop_all(engine: aiopg.sa.engine.Engine, metadata: sa.MetaData):
-    executor = AiopgExecutor(engine)
-    metadata.drop_all(executor.sa_engine, checkfirst=True)
-    await executor.execute()
-
-
-# EXCEPTIONS -------------------------------------
-#
 # aiopg reuses DBAPI exceptions
 #
 # StandardError
@@ -80,11 +23,8 @@ async def drop_all(engine: aiopg.sa.engine.Engine, metadata: sa.MetaData):
 # SEE https://aiopg.readthedocs.io/en/stable/core.html?highlight=Exception#exceptions
 # SEE http://initd.org/psycopg/docs/module.html#dbapi-exceptions
 
-# alias add prefix DBAPI
-DBAPIError = psycopg2.Error
 
 
-__all__ = (
-    'create_all',
-    'drop_all'
-)
+__all__ = [
+    'DBAPIError'
+]

packages/service-library/src/servicelib/application.py

@@ -5,17 +5,26 @@
 from .application_keys import APP_CONFIG_KEY
 from .client_session import persistent_client_session
 
+async def startup_info(app: web.Application):
+    print(f"STARTING UP {app}...", flush=True)
+
+
+async def shutdown_info(app: web.Application):
+    print(f"SHUTDOWN {app} ...", flush=True)
+
 
 def create_safe_application(config: Optional[Dict]=None) -> web.Application:
     app = web.Application()
 
     # Enxures config entry
     app[APP_CONFIG_KEY] = config or {}
 
+    app.on_startup.append(startup_info)
+    app.on_cleanup.append(shutdown_info)
+
     # Ensures persistent client session
     # NOTE: Ensures client session context is run first,
     # then any further get_client_sesions will be correctly closed
     app.cleanup_ctx.append(persistent_client_session)
 
-
     return app

packages/service-library/src/servicelib/application_keys.py

@@ -11,24 +11,21 @@
 
 # REQUIREMENTS:
 # - guarantees all keys are unique
-# TODO: facilitate key generation
-# TODO: hierarchical classification
+# - one place for all common keys
+# - hierarchical classification
 # TODO: should be read-only (frozen?)
 
+#
+# web.Application keys, i.e. app[APP_*_KEY]
+#
+APP_CONFIG_KEY           = f'{__name__ }.config'
+APP_OPENAPI_SPECS_KEY    = f'{__name__ }.openapi_specs'
+APP_JSONSCHEMA_SPECS_KEY = f'{__name__ }.jsonschema_specs'
 
-# APP=application
-APP_CONFIG_KEY         = __name__ + '.config'
-APP_OPENAPI_SPECS_KEY  = __name__ + '.openapi_specs'
-APP_SESSION_SECRET_KEY = __name__ + '.session_secret'
-APP_JSONSCHEMA_SPECS_KEY = __name__ + '.jsonschema_specs'
+APP_DB_ENGINE_KEY        = f'{__name__ }.db_engine'
 
-APP_DB_ENGINE_KEY      = __name__ + '.db_engine'
-APP_DB_SESSION_KEY     = __name__ + '.db_session'
-APP_DB_POOL_KEY        = __name__ + '.db_pool'
+APP_CLIENT_SESSION_KEY   = f'{__name__ }.session'
 
-APP_CLIENT_SESSION_KEY = f"{__name__ }.session"
-
-# RSP=response
-
-
-# TODO: tool to convert dotted __name__ to section in dict
+#
+# web.Response keys, i.e. app[RSP_*_KEY]
+#

packages/service-library/src/servicelib/monitoring.py

@@ -16,12 +16,13 @@
 from aiohttp import web
 from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram
 
+
 log = logging.getLogger(__name__)
 
 
 def middleware_factory(app_name):
     @web.middleware
-    async def middleware_handler(request, handler):
+    async def middleware_handler(request: web.Request, handler):
         # See https://prometheus.io/docs/concepts/metric_types
         try:
             request['start_time'] = time.time()
@@ -35,10 +36,17 @@ async def middleware_handler(request, handler):
             resp = exc
             raise
         except Exception as exc: #pylint: disable=broad-except
-            # Prevents issue #1025. FIXME: why middleware below is not non-http exception safe?
-            log.exception("Unexpected exception. \
-                Error middleware below should only raise web.HTTPExceptions.")
+            # Prevents issue #1025.
             resp = web.HTTPInternalServerError(reason=str(exc))
+            resp_time = time.time() - request['start_time']
+
+            # NOTE: all access to API (i.e. and not other paths as /socket, /x, etc) shall return web.HTTPErrors since processed by error_middleware_factory
+            log.exception('Unexpected server error "%s" from access: %s "%s %s" done in %3.2f secs. Responding with status %s',
+                type(exc),
+                request.remote, request.method, request.path,
+                resp_time,
+                resp.status
+            )
         finally:
             # metrics on the same request
             resp_time = time.time() - request['start_time']

packages/service-library/src/servicelib/rest_middlewares.py

@@ -27,15 +27,16 @@ def is_api_request(request: web.Request, api_version: str) -> bool:
     return request.path.startswith(base_path)
 
 
-def _process_and_raise_unexpected_error(err):
+def _process_and_raise_unexpected_error(request: web.BaseRequest, err: Exception):
     # TODO: send info + trace to client ONLY in debug mode!!!
-    logger.exception("Unexpected exception on server side")
-    exc = create_error_response(
-            [err,],
-            "Unexpected Server error",
-            web.HTTPInternalServerError
-        )
-    raise exc
+    resp = create_error_response( [err,], "Unexpected Server error", web.HTTPInternalServerError)
+
+    logger.exception('Unexpected server error "%s" from access: %s "%s %s". Responding with status %s',
+        type(err),
+        request.remote, request.method, request.path,
+        resp.status
+    )
+    raise resp
 
 
 def error_middleware_factory(api_version: str = DEFAULT_API_VERSION):
@@ -78,15 +79,15 @@ async def _middleware(request: web.Request, handler):
                         payload = wrap_as_envelope(data=payload)
                         ex.text = json.dumps(payload)
                 except Exception as err:  # pylint: disable=W0703
-                    _process_and_raise_unexpected_error(err)
+                    _process_and_raise_unexpected_error(request, err)
             raise ex
 
         except web.HTTPRedirection as ex:
             logger.debug("Redirected to %s", ex)
             raise
 
         except Exception as err:  # pylint: disable=W0703
-            _process_and_raise_unexpected_error(err)
+            _process_and_raise_unexpected_error(request, err)
 
     return _middleware