Merge pull request #1 from telefonicasc/initial-implementation

ADD initial implementation

.github/workflows/unit-tests.yml

@@ -0,0 +1,32 @@
+name: Python Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    name: Run Python Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.11
+
+    - name: Install dependencies
+      run: pip install .
+
+    - name: Install pytest
+      run: pip install pytest
+
+    - name: Run tests
+      run: pytest tests/

.gitignore

@@ -157,4 +157,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/

README.md

@@ -1,2 +1,118 @@
 # tcjexl
-Recubrimiento de la librería de jexl incluyendo un conjunto de transformaciones por defecto
+
+This is a wrapper of the [pyjexl](https://pypi.org/project/pyjexl/) library, including a set of default transformations  (detailed in [this section](#included-transformations))
+
+Example:
+
+```python
+from tcjexl import JEXL
+
+jexl = JEXL()
+
+context = {"a": 5, "b": 7, "c": "a TEXT String"}
+
+print(jexl.evaluate('a+b', context))
+print(jexl.evaluate('c|lowercase', context))
+```
+
+Result:
+
+```
+12
+a text string
+```
+
+## Included transformations
+
+**NOTE:** JEXL pipeline is needed even if the transformation doesn't need an parameter to work (e.g. `currentTime` that provides the current system time and doesn't need and argument). In this case, we use `0|` for these cases (e.g. `0|currentTime`).
+
+### Math related transformations
+
+- `rnd`: returns a random number between two integers. Examples: `0|rnd(10)` returns a random number between 0 (included) and 10 (not included). `12|rnd(99)` returns a random number between 12 (included) and 99 (not included).
+- `rndFloat`: returns a random number between two decimal numbers. Examples: `0.2|rndFloat(12.7)` returns a random number between 0.2 and 12.7.
+- `round`: rounds a number with a given number of precision digits. Example: `0.12312|round(2)` returns `0.12`, rounding 0.12312 to two decimals.
+- `floor`: rounds a number to the lesser integer. Example: `4.9|floor` returns `4`.
+- `parseInt`: converts a string to integer
+- `parseFloat`: coverts a string to float
+
+### String related transformations
+
+- `uppercase`: converts a given string to uppercase.
+- `lowercase`: converts a given string to lowercase.
+- `toString`: returns string representation
+- `substring`: returns a substring. Examaple: `aGivenString|substring(3,6)` returns `ven`.
+- `includes`: returns `True` is string passed as argument is included in the one that comes in the pipe. Example: `"aGivenString"|includes("Given")` return `True` while `"aGivenString"|includes("Text")` returns `False`. 
+- `len`: returns the number of items in an array or the length of a string.
+
+### List related transformations
+
+- `next`: returns the next item in an array. Example: `12|next([1,2,3,12,15,18])` returns 15.
+- `indexOf`: returns the index corresponding to an item in an array. Example: `[1, 2, 3, 12, 15, 18]|indexOf(15)` returns `4`.
+- `rndList`: returns an array of random elements within two limits. Example: `0|rndList(6,8)` is an array with 8 items and each item is a random number between 0 and 6.
+- `rndFloatList`: similar to `rndList`, but with decimal numbers.
+- `zipStringList`: concat two arrays of the same length with a given separator.
+- `concatList`: concat array elements.
+
+### Date related transformations
+
+- `currentTime`: returns current time in UTC format.
+- `currentTimeIso`: returns current time in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Time_zone_designators) format.
+- `toIsoString`: allows to format a date into [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Time_zone_designators) format.
+- `currentTimeFormat`: allows to format current time with a given format. For instance, if current date is 07/07/2023 and we use `0|currentTimeFormat("%Y")` then `2023` will be returned.
+- `timeFormat`: allows to format a given date with a given format. For instance, if current date is 07/07/2023 and we use `0|currentTime|timeFormat("%Y")` then `2023` will be returned.
+- `currentHour24`: returns current time in 24 hours format. Example: `0|currentHour24`.
+- `currentDay`: returns the current day of the month. Example: `0|currentDay`.
+
+### Interpolation transformations
+
+- `interpolate`: returns number interpolation, given an initial value, a final value and a number of steps. Example: `3|interpolate(0,10,9)`
+- `linearInterpolator`: returns linear value interpolation, taking into account an array of values in `[number, value]` format. Example: `number|linearInterpolator([ [0,0], [1,1], [2,1.5], [8,1.8], [10,2.0]])` for number 2 returns `1.5`, for number 5 returns the linear interpolation between 2 and 8, taking into account the associated values 1.5 and 1.8 respectively.
+- `linearSelector`: allows to select a given value taking into account ranges defined between two elements in an array. Example: `0|rndFloat(1)|linearSelector([[0.02,'BLACK'], [0.04,'NO FLAG'], [0.06,'RED'], [0.21,'YELLOW'], [1,'GREEN']])`, if input is `0.02 < (0|rndFloat(1)) ≤ 0,04` returns `NO FLAG`.
+- `randomLinearInterpolator`: returns linear value interpolation with a random factor, taking into account an array of values in `[number, value]` format. Example: `number|randomLinearInterpolator([0,1],[ [0,0], [1,1], [2,1.5], [8,1.8], [10,2.0]])` for number 2 returns a value close to 1.5 (close due to a random factor is applied), for number 5 returns the lineal interpolation between 2 and 8, taking into account the associated values 1.5 and 1.8 respectively and the random factor. The random factor is specified as a `[min, max]` array and the calculated interpolated value is multiplied by a random number between `min` and `max`. For instance, with `[0.85, 0.99]` the result will be closer to the interpolation but with `[0, 1]` the spread will be wider.
+- `alertMaxValue`: returns `True` when input value is greater on equal to a given condition. Example: `0|rnd(5)|alertMaxValue(2)` returns `True` when `0|rnd(5)` is 3, 4 or 5 (for other input values result will be `False`).
+- `valueResolver`: given an array `[str, value]` allows to map string with values. Example: `flag|valueResolver([['BLACK', 'stormy'], ['NO FLAG', 'curly'], ['RED', 'stormy'], ['YELLOW', 'curly'], ['GREEN', 'plain']])` is the evaluation of `flag` field is `GREEN` then the returned value would be `plain`.
+
+# Miscelaneous transformations
+
+- `typeOf`: returns type representation of the data (e.g. `str`, `int`, `float`, etc.)
+- `strToLocation`: given a latitude and a longitude, it returns an array to build a location. Example: `"value1, value2"|strToLocation`. Example: `"value1, value2"|strToLocation` returns `[value1, value2]`, so we can use this:
+
+```json
+{
+  "init": null,
+  "type": "geo:json",
+  "exp": "{coordinates:(point|strToLocation),type: \"Point\"}"
+}
+```
+
+## Packaging
+
+* Check `VERSION` value in `setup.py` file.
+* Run
+
+```bash
+python3 setup.py sdist bdist_wheel
+```
+
+* The file `tcjexl-<version>.tar.gz` is generated in the `dist` directory.
+
+## Uploading package to pypi repository
+
+Once the package has been build as explained in the previous section it can be uploaded to pypi repository.
+
+First, install the twine tool:
+
+```bash
+pip install twine
+```
+
+Next, run:
+
+```bash
+twine upload dist/tcjexl-x.y.z.tar.gz
+```
+
+You need to be registered at https://pypi.org with permissions at https://pypi.org/project/tcjexl/, as during the
+upload process you will be prompted to provide your user and password.
+
+## Changelog

setup.py

@@ -0,0 +1,52 @@
+# Copyright 2024 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U.
+#
+# This file is part of tcjexl
+#
+# tcjexl is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# tcjexl is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/.
+
+import pathlib
+from setuptools import find_packages, setup
+
+HERE = pathlib.Path(__file__).parent
+
+VERSION = '0.0.0.post'
+PACKAGE_NAME = 'tcjexl'
+AUTHOR = ''
+AUTHOR_EMAIL = ''
+URL = ''
+
+LICENSE = '' #License type
+DESCRIPTION = 'Wrapper of pyjexl with additional set of transformations' # Short description
+LONG_DESCRIPTION = (HERE / "README.md").read_text(encoding='utf-8') # Reference to the README.md document with a longer description
+LONG_DESC_TYPE = "text/markdown"
+
+# Required packages. They will be installed if not already installed
+INSTALL_REQUIRES = [
+    'pyjexl==0.3.0'
+]
+
+setup(
+    name=PACKAGE_NAME,
+    version=VERSION,
+    description=DESCRIPTION,
+    long_description=LONG_DESCRIPTION,
+    long_description_content_type=LONG_DESC_TYPE,
+    author=AUTHOR,
+    author_email=AUTHOR_EMAIL,
+    url=URL,
+    install_requires=INSTALL_REQUIRES,
+    license=LICENSE,
+    packages=find_packages(),
+    include_package_data=True
+)

tcjexl/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2024 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U.
+#
+# This file is part of tcjexl
+#
+# tcjexl is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# tcjexl is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/.
+
+from tcjexl.jexl import JEXL

tcjexl/expression_functions.py

@@ -0,0 +1,77 @@
+# Copyright 2024 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U.
+#
+# This file is part of tcjexl
+#
+# tcjexl is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# tcjexl is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/.
+
+import random
+
+def zipStrList(left: list, right: list, sep: str = ""):
+    l_length = len(left)
+    r_length = len(right)
+    if l_length == r_length:
+        return [str(left[i]) + sep + str(right[i]) for i in range(l_length)]
+    else:
+        raise ValueError(f"zipStrList input error, left length: {l_length}, right len length: {r_length}")
+
+
+def linearInterpolator(t, interpolations):
+    def interpolate(start, end, t):
+        return start + (end - start) * t
+
+    for i in range(len(interpolations)):
+        if i < len(interpolations) - 1 and interpolations[i][0] <= t < interpolations[i + 1][0]:
+            start_time, start_value = interpolations[i]
+            end_time, end_value = interpolations[i + 1]
+            time_ratio = (t - start_time) / (end_time - start_time)
+            return interpolate(start_value, end_value, time_ratio)
+
+    raise ValueError("Invalid input or interpolations")
+
+
+def valueResolver(t: str, elements: list):
+    dictionary = {}
+    for k, v in elements:
+        dictionary[k] = v
+    r = dictionary.get(t, None)
+    if r is None:
+        raise ValueError(f"Invalid key {t}")
+    return r
+
+
+def linearSelector(t, interpolations):
+    for i in range(len(interpolations)):
+        if t <= interpolations[i][0]:
+            _, start_value = interpolations[i]
+            return start_value
+
+    raise ValueError("Invalid input or interpolations")
+
+
+def randomLinearInterpolator(t, rndFactor, interpolations):
+    def interpolate(start, end, t):
+        return start + (end - start) * t
+
+    def random_interpolate(start, end, t):
+        random_value = random.uniform(rndFactor[0], rndFactor[1])
+        return interpolate(start, end, t) * random_value
+
+    for i in range(len(interpolations)):
+        if i < len(interpolations) - 1 and interpolations[i][0] <= t < interpolations[i + 1][0]:
+            start_time, start_value = interpolations[i]
+            end_time, end_value = interpolations[i + 1]
+            time_ratio = (t - start_time) / (end_time - start_time)
+            return random_interpolate(start_value, end_value, time_ratio)
+
+    raise ValueError("Invalid input or interpolations")

tcjexl/jexl.py

@@ -0,0 +1,77 @@
+# Copyright 2024 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U.
+#
+# This file is part of tcjexl
+#
+# tcjexl is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# tcjexl is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/.
+
+
+import pyjexl
+import random
+import datetime
+import math
+import functools
+
+from datetime import timezone
+from .expression_functions import linearInterpolator, linearSelector, randomLinearInterpolator, zipStrList, valueResolver
+
+class JEXL(pyjexl.JEXL):
+    def __init__(self, context=None, now=datetime.datetime.now(timezone.utc)):
+        super().__init__(context=context)
+
+        self.now = now
+
+        # Tested by test_transforms_math.py
+        super().add_transform("rnd", lambda ini, end: random.randrange(ini, end))
+        super().add_transform("rndFloat", lambda ini, end: random.uniform(ini, end))
+        super().add_transform("round", lambda x, decimals: round(x, decimals))
+        super().add_transform("floor", lambda x: math.floor(x))
+        super().add_transform("parseInt", lambda x: int(x))
+        super().add_transform("parseFloat", lambda x: float(x))
+
+        # Tested by test_transforms_string.py
+        super().add_transform("uppercase", lambda x: str(x).upper())
+        super().add_transform("lowercase", lambda x: str(x).lower())
+        super().add_transform("toString", lambda x: str(x))
+        super().add_transform("substring", lambda x, ini, fin: x[ini:fin])
+        super().add_transform("includes", lambda x, str: str in x)
+        super().add_transform("len", lambda data: len(data))
+
+        # Tested by test_transforms_list.py
+        super().add_transform("next", lambda x, arr: arr[(arr.index(x) + 1) % len(arr)])
+        super().add_transform("indexOf", lambda x, str: x.index(str))
+        super().add_transform("rndList", lambda init, end, length: [random.randrange(init, end) for _ in range(length)])
+        super().add_transform("rndFloatList", lambda ini, end, length: [random.uniform(ini, end) for _ in range(length)])
+        super().add_transform("zipStringList", zipStrList)
+        super().add_transform("concatList", lambda list_value: functools.reduce(lambda a, b: a + b, list_value))
+
+        # Tested by test_transforms_date.py
+        super().add_transform("currentTime", lambda x: self.now())
+        super().add_transform("currentTimeIso", lambda x: self.now().strftime('%Y-%m-%dT%H:%M:%S') + ".000Z")
+        super().add_transform("toIsoString", lambda date: date.strftime('%Y-%m-%dT%H:%M:%S') + ".000Z")
+        super().add_transform("currentTimeFormat", lambda x, string: self.now().strftime(string))
+        super().add_transform("timeFormat", lambda date, string: date.strftime(string))
+        super().add_transform("currentHour24", lambda x: int(self.now().hour))
+        super().add_transform("currentDay", lambda x: int(self.now().day))
+
+        # Tested by test_transforms_interpolation.py
+        super().add_transform("interpolate", lambda step, ini, end, nSteps: ((end - ini) * (step % nSteps)) / nSteps)
+        super().add_transform("linearInterpolator", lambda t, array: linearInterpolator(t, array))
+        super().add_transform("linearSelector", lambda t, array: linearSelector(t, array))
+        super().add_transform("randomLinearInterpolator", lambda t, rndFactor, array: randomLinearInterpolator(t, rndFactor, array))
+        super().add_transform("alertMaxValue", lambda value, max_value: True if value >= max_value else False)
+        super().add_transform("valueResolver", lambda t, values: valueResolver(t, values))
+
+        # Tested by test_transforms_misc.py
+        super().add_transform("typeOf", lambda x: f'{type(x)}'[8:-2])
+        super().add_transform("strToLocation", lambda str: [float(x) for x in str.split(",")])