Merge pull request #22 from BoostV/feature/version_information

Feature/version information

.gitignore

@@ -1,4 +1,5 @@
 /env
 **.pyc
 .vscode
-/tmp
+/tmp
+version.txt

Dockerfile

@@ -4,10 +4,11 @@ FROM python:3.9.1 AS builder
 RUN python3 -m venv /opt/venv
 ENV PATH="/opt/venv/bin:${PATH}"
 
-RUN pip install --upgrade pip && pip install pip-tools
+RUN pip install --upgrade pip && pip install pip-tools && pip install --upgrade pip
 
-COPY requirements.txt .
-RUN pip install -r requirements.txt
+COPY requirements-freeze.txt .
+RUN cat requirements-freeze.txt | grep --invert-match pkg_resources > requirements-fixed.txt
+RUN pip install -r requirements-fixed.txt
 
 # Second stage
 FROM python:3.9.1-slim
@@ -20,6 +21,8 @@ RUN chown -R user:user /code && chmod -R 755 /code
 
 USER user
 
+COPY --from=builder /requirements-fixed.txt /code/requirements-freeze.txt
+COPY version.txt /code
 COPY optimizerapi/ /code
 
 ENV FLASK_ENV=production

README.md

@@ -12,7 +12,7 @@ Alternatively the project can be build and run with the following commands:
     source env/bin/activate
     pip install --upgrade pip
 
-    pip install -r requirements.txt
+    pip install -r requirements-freeze.txt
     python optimizerapi/server.py
 
 Now open [http://localhost:9090/v1.0/ui/](http://localhost:9090/v1.0/ui/) in a browser to explore the API through Swagger UI
@@ -27,7 +27,7 @@ or use pytest-watch for continuously running tests
     ptw
 # Building docker container
 
-    docker build -t process-optimizer-api .
+    git describe --always > version.txt && docker build -t process-optimizer-api .
 # Obtain encryption key
 
 Run server once and extract a fresh encryption key from the logs.
@@ -46,3 +46,20 @@ Running using python
 or use docker
 
     docker run -d --name process-optimizer-api --env PICKLE_KEY=<key from previous step> -p 9090:9090 process-optimizer-api:latest
+
+# Adding or updating dependencies
+
+When adding a new dependency, you should manually add it to `requirements.txt` and then run the following commands:
+
+    pip install -r requirements.tx
+    pip freeze | grep --invert-match pkg_resources > requirements-freeze.txt
+
+Now you should check if the freeze operation resulted in unwanted upates by running:
+
+    git diff requirements-freeze.txt
+
+After manually fixing any dependencies, you should run:
+
+    pip install -r requirements-freeze.txt
+
+Remember to commit both the changed `requirements.txt` and `requirements-freeze.txt` files.

optimizerapi/optimizer.py

@@ -1,3 +1,6 @@
+import os
+import platform
+from time import strftime
 import json
 import json_tricks
 from ProcessOptimizer import Optimizer, expected_minimum
@@ -29,13 +32,13 @@ def run(body) -> dict:
     dict
         a JSON encodable dictionary representation of the result.
     """
-    print("Receive: " + str(body))
+    # print("Receive: " + str(body))
     data = [(run["xi"], run["yi"]) for run in body["data"]]
     cfg = body["optimizerConfig"]
     extras = {}
     if ("extras" in body):
         extras = body["extras"]
-    print("Received extras " + str(extras))
+    # print("Received extras " + str(extras))
     space = [(convertNumberType(x["from"], x["type"]), convertNumberType(x["to"], x["type"])) if (x["type"] == "discrete" or x["type"] == "continuous") else tuple(x["categories"]) for x in cfg["space"]]
     dimensions = [x["name"] for x in cfg["space"]]
     hyperparams = {
@@ -46,13 +49,24 @@ def run(body) -> dict:
     }
     optimizer = Optimizer(space, **hyperparams)
 
+    Xi = []
+    Yi = []
     if data:
         Xi, Yi = map(list, zip(*data))
         result = optimizer.tell(Xi, Yi)
     else:
         result = {}
 
     response = processResult(result, optimizer, dimensions, cfg, extras, data, space)
+
+    response["result"]["extras"]["parameters"] = {
+        "dimensions": dimensions,
+        "space": space,
+        "hyperparams": hyperparams,
+        "Xi": Xi,
+        "Yi": Yi,
+        "extras": extras
+    }
 
     # It is necesarry to convert response to a json string and then back to 
     # dictionary because NumPy types are not serializable by default
@@ -115,20 +129,10 @@ def processResult(result, optimizer, dimensions, cfg, extras, data, space):
     experimentSuggestionCount = 1
     if ("experimentSuggestionCount" in extras):
         experimentSuggestionCount = extras["experimentSuggestionCount"]
-    print("Exp:" + str(experimentSuggestionCount))
+
+    # print("Exp:" + str(experimentSuggestionCount))
     resultDetails["next"] = optimizer.ask(n_points=experimentSuggestionCount) # TODO Hent n_points fra brugeren
 
-    ##################### Copied and modified from views.py::view_report #####################
-
-    if "expected_minimum" in result:
-        temp_exp_min =[]
-        for entry,value in zip(header_list[:-1], result.expected_minimum[0]):
-            temp_exp_min.append([entry, value])
-        exp_min_out = {'value':temp_exp_min, 'result':result.expected_minimum[1]}
-        resultDetails['expected_minimum'] = exp_min_out
-
-    ##################### END #####################
-
     if len(data) >= cfg["initialPoints"]:
         # Plotting is only possible if the model has 
         # processed more that "initialPoints" data points
@@ -140,6 +144,8 @@ def processResult(result, optimizer, dimensions, cfg, extras, data, space):
 
     resultDetails["pickled"] = securepickle.pickleToString(result, securepickle.get_crypto())
 
+    addVersionInfo(resultDetails["extras"])
+
     # print(str(response))
     return response
 
@@ -179,4 +185,31 @@ def addPlot(result, id="generic", close=True, debug=False):
 
     # print("IMAGE: " + str(pic_hash, "utf-8"))
     if close:
-        plt.clf()
+        plt.clf()
+
+def addVersionInfo(extras):
+    """Add various version information to the dictionary supplied.
+    
+    Parameters
+    ----------
+    extras : dict
+            The dictionary to hold the version information
+    """
+
+    with open("requirements-freeze.txt", "r") as requirementsFile:
+        requirements = requirementsFile.readlines()
+        extras["libraries"] = [x.rstrip() for x in requirements]
+
+    extras["pythonVersion"] = platform.python_version()
+
+    if os.path.isfile("version.txt"):
+        with open("version.txt", "r") as versionFile:
+            extras["apiVersion"] = versionFile.readline().rstrip()
+    else:
+        import subprocess
+        try:
+            extras["apiVersion"] = subprocess.check_output(["git", "describe", "--always"]).strip().decode()
+        except:
+            extras["apiVersion"] = 'Unknown development version'
+
+    extras["timeOfExecution"] = strftime("%Y-%m-%d %H:%M:%S")

requirements-freeze.txt

@@ -0,0 +1,52 @@
+attrs==20.3.0
+bokeh==2.2.3
+certifi==2020.12.5
+cffi==1.14.5
+chardet==4.0.0
+click==7.1.2
+clickclick==20.10.2
+colorama==0.4.4
+connexion==2.7.0
+cryptography==3.4.7
+cycler==0.10.0
+deap==1.3.1
+docopt==0.6.2
+Flask==1.1.2
+idna==2.10
+inflection==0.5.1
+iniconfig==1.1.1
+itsdangerous==1.1.0
+Jinja2==2.11.3
+joblib==1.0.1
+json-tricks==3.15.5
+jsonschema==3.2.0
+kiwisolver==1.3.1
+MarkupSafe==1.1.1
+matplotlib==3.3.4
+numpy==1.20.1
+openapi-spec-validator==0.2.9
+packaging==20.9
+Pillow==8.1.0
+pluggy==0.13.1
+ProcessOptimizer==0.7.1
+py==1.10.0
+pycparser==2.20
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest==6.2.2
+pytest-watch==4.2.0
+python-dateutil==2.8.1
+PyYAML==5.4.1
+requests==2.25.1
+scikit-learn==0.24.2
+scipy==1.6.0
+six==1.15.0
+swagger-ui-bundle==0.0.8
+threadpoolctl==2.1.0
+toml==0.10.2
+tornado==6.1
+typing-extensions==3.7.4.3
+urllib3==1.26.3
+waitress==2.0.0
+watchdog==2.0.2
+Werkzeug==1.0.1

requirements.txt

@@ -1,7 +1,7 @@
 connexion==2.7.0
 connexion[swagger-ui]
 Flask==1.1.2
-ProcessOptimizer==0.6.4
+ProcessOptimizer==0.7.1
 json-tricks==3.15.5
 cryptography==3.4.7
 waitress==2.0.0

tests/test_optimizer.py

@@ -1,6 +1,9 @@
 from .context import optimizer
 
 #  {'data': [{'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, {'xi': [651, 42, 722, 'Ræv'], 'yi': 0.2}], 'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 5, 'kappa': 1.96, 'xi': 0.012, 'space': [{'type': 'numeric', 'name': 'Sukker', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Peber', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Hvedemel', 'from': 0, 'to': 1000}, {'type': 'category', 'name': 'Kunde', 'categories': ['Mus', 'Ræv']}]}}
+#   'data': [{'xi': [0, 5, 'Rød'], 'yi': 10}, {'xi': [5, 8.33, 'Hvid'], 'yi': 3}, {'xi': [10, 1.66, 'Rød'], 'yi': 5}], 
+#   'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 3, 'kappa': 1.96, 'xi': 0.01, 
+#   'space': [{'type': 'discrete', 'name': 'Alkohol', 'from': 0, 'to': 10}, {'type': 'continuous', 'name': 'Vand', 'from': 0, 'to': 10}, {'type': 'category', 'name': 'Farve', 'categories': ['Rød', 'Hvid']}]}}                                                                                                                                                  Received extras {'experimentSuggestionCount': 2}         
 
 sampleData = [
         {'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, 
@@ -15,9 +18,9 @@
         'kappa': 1.96, 
         'xi': 0.012, 
         'space': [
-            {'type': 'numeric', 'name': 'Sukker', 'from': 0, 'to': 1000}, 
-            {'type': 'numeric', 'name': 'Peber', 'from': 0, 'to': 1000}, 
-            {'type': 'numeric', 'name': 'Hvedemel', 'from': 0, 'to': 1000}, 
+            {'type': 'discrete', 'name': 'Sukker', 'from': 0, 'to': 1000}, 
+            {'type': 'continuous', 'name': 'Peber', 'from': 0, 'to': 1000}, 
+            {'type': 'continuous', 'name': 'Hvedemel', 'from': 0, 'to': 1000}, 
             {'type': 'category', 'name': 'Kunde', 'categories': ['Mus', 'Ræv']}
             ]
     }