Merge pull request #114 from vkuznet/main

Add foxden module

CHAP/foxden/__init__.py

@@ -0,0 +1,6 @@
+"""This subpackage contains pieces for communication with FOXDEN services.
+"""
+
+from CHAP.foxden.processor import (
+    FoxdenProvenanceProcessor,
+)

CHAP/foxden/processor.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : processor.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Processor module for FOXDEN services
+"""
+
+# system modules
+from time import time
+
+# local modules
+from CHAP import Processor
+from CHAP.foxden.writer import FoxdenWriter
+
+
+class FoxdenProvenanceProcessor(Processor):
+    """A Processor to communicate with FOXDEN provenance server."""
+#     def __init__(self):
+#         self.writer = FoxdenWriter()
+
+    def process(self, data, url, dryRun=False, verbose=False):
+        """process data API"""
+
+        t0 = time()
+        self.logger.info(f'Executing "process" with url {url} data {data} dryrun {dryRun}')
+        writer = FoxdenWriter()
+
+#         data = self.writer.write(data, url, dryRun)
+        data = writer.write(data, url, dryRun=dryRun)
+
+        self.logger.info(f'Finished "process" in {time()-t0:.3f} seconds\n')
+
+        return data
+
+
+if __name__ == '__main__':
+    # local modules
+    from CHAP.processor import main
+
+    main()

CHAP/foxden/writer.py

@@ -0,0 +1,65 @@
+"""FOXDE command line writer."""
+
+# system modules
+import os
+
+# Local modules
+from CHAP.writer import main
+
+class FoxdenWriter():
+    """FOXDEN writer writes data to specific FOXDEN service
+    """
+
+    def write(self, data, url, method="POST", headers={}, timeout=10, dryRun=False):
+        """Write the input data as text to a file.
+
+        :param data: input data
+        :type data: list[PipelineData]
+        :param url: url of service
+        :type url: str
+        :param method: HTTP method to use, POST for creation and PUT for update
+        :type method: str
+        :param headers: HTTP headers to use
+        :type headers: dictionary
+        :param timeout: timeout of HTTP request
+        :type timeout: str
+        :param dryRun: dryRun option to verify HTTP workflow
+        :type dryRun: boolean
+        :return: contents of the input data
+        :rtype: object
+        """
+        import requests
+        if 'Content-Type' not in headers:
+            headers['Content-type'] = 'application/json'
+        if 'Accept' not in headers:
+            headers['Accept'] = 'application/json'
+        if dryRun:
+            print("### HTTP writer call", url, headers, data)
+            return []
+        token = ""
+        fname = os.getenv("CHESS_WRITE_TOKEN")
+        if not fname:
+            msg = f'CHESS_WRITE_TOKEN env variable is not set'
+            raise Exception(msg)
+        with open(fname, 'r') as istream:
+            token = istream.read()
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        else:
+            msg = f'No valid write token found in CHESS_WRITE_TOKEN env variable'
+            raise Exception(msg)
+
+        # make actual HTTP request to FOXDEN service
+        if method.lower() == 'post':
+            resp = requests.post(url, headers=headers, timeout=timeout, data=data)
+        elif method.lower() == 'put':
+            resp = requests.put(url, headers=headers, timeout=timeout, data=data)
+        else:
+            msg = f"unsupporteed method {method}"
+            raise Exception(msg)
+        data = resp.content
+        return data
+
+
+if __name__ == '__main__':
+    main()

CHAP/test/foxden/test.yaml

@@ -0,0 +1,5 @@
+pipeline:
+  - common.PrintProcessor: {}
+  - foxden.FoxdenProvenanceProcessor:
+      url: localhost
+      dryRun: True

setup.py

@@ -13,6 +13,7 @@
 
 # [set version]
 version = 'PACKAGE_VERSION'
+version = 'v0.0.12'
 # [version set]
 
 def datafiles(idir, pattern=None):
@@ -54,6 +55,7 @@ def datafiles(idir, pattern=None):
         'CHAP.sin2psi',
         'CHAP.tomo',
         'CHAP.utils',
+        'CHAP.foxden',
         'MLaaS'
     ],
     package_dir={
@@ -65,6 +67,7 @@ def datafiles(idir, pattern=None):
         'CHAP.saxswaxs': 'CHAP/saxswaxs',
         'CHAP.sin2psi': 'CHAP/sin2psi',
         'CHAP.tomo': 'CHAP/tomo',
+        'CHAP.foxden': 'CHAP/foxden',
         'CHAP.utils': 'CHAP/utils',
         'MLaaS': 'MLaaS'
     },