refactor(language): better structure (#365)

DEVELOPMENT.md

@@ -2,29 +2,15 @@
 
 ## Prerequisites
 
-Install [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#installing-and-updating).
-
-Install [Node.js](https://nodejs.org/) 20.x with NVM and set it as default:
-
-```shell
-nvm install lts/iron
-```
-
-```shell
-nvm alias default lts/iron
-```
+Install [Bun](https://bun.sh).
 
 Install [Go](https://go.dev) for your platform.
 
 Install [Rust](https://www.rust-lang.org) for your platform.
 
-Install [PDM](https://pdm-project.org).
-
-Install Python 3.12 with PDM:
+Install [Poetry](https://python-poetry.org/docs/#installation).
 
-```shell
-pdm py install [email protected]
-```
+Install [Python](https://www.python.org) 3.12 with [pyenv](https://github.com/pyenv/pyenv).
 
 ## Run Infrastructure Services
 
@@ -171,3 +157,4 @@ Start each microservice separately in a new terminal as described here:
 - [Voltaserve Conversion](conversion/README.md)
 - [Voltaserve Mosaic](mosaic/README.md)
 - [Voltaserve Language](mosaic/README.md)
+- [Voltaserve Console](console/README.md)

console/pyproject.toml

@@ -1,6 +1,7 @@
 [tool.poetry]
 name = "voltaserve-console"
-version = "0.0.1"
+version = "3.0.0"
+license = "BUSL-1.1"
 description = "Asynchronous API for voltaserve-console microservice"
 authors = ["Piotr Łoboda <[email protected]>"]
 maintainers = ["Piotr Łoboda <[email protected]>"]

docker-compose.yml

@@ -87,7 +87,7 @@ services:
       - SMTP_SENDER_ADDRESS=${VOLTASERVE_SMTP_SENDER_ADDRESS}
       - SMTP_SENDER_NAME=${VOLTASERVE_SMTP_SENDER_NAME}
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8080/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8080/v3/health || exit 1
     depends_on:
       - cockroach
       - redis
@@ -113,7 +113,7 @@ services:
       - SMTP_SENDER_ADDRESS=${VOLTASERVE_SMTP_SENDER_ADDRESS}
       - SMTP_SENDER_NAME=${VOLTASERVE_SMTP_SENDER_NAME}
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8081/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8081/v3/health || exit 1
     depends_on:
       - cockroach
       - meilisearch
@@ -147,7 +147,7 @@ services:
       - REDIS_ADDRESS=redis:6379
       - S3_URL=minio:9000
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8082/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8082/v3/health || exit 1
     depends_on:
       - idp
       - api
@@ -166,7 +166,7 @@ services:
       - MOSAIC_URL=http://mosaic:8085
       - S3_URL=minio:9000
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8083/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8083/v3/health || exit 1
     depends_on:
       - api
       - minio
@@ -178,7 +178,7 @@ services:
     ports:
       - ${VOLTASERVE_LANGUAGE_PORT}:8084
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8084/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8084/v3/health || exit 1
     restart: on-failure
   mosaic:
     image: voltaserve/mosaic
@@ -189,7 +189,7 @@ services:
     environment:
       - S3_URL=minio:9000
     healthcheck:
-      test: wget --quiet --spider http://127.0.0.1:8085/v2/health || exit 1
+      test: wget --quiet --spider http://127.0.0.1:8085/v3/health || exit 1
     restart: on-failure
   console:
     image: voltaserve/console

language/.flake8

@@ -1,4 +1,2 @@
 [flake8]
-exclude = .git,.venv,.dockerignore,.pdm-python,Dockerfile
-extend-ignore = E203
-max-line-length = 88
+exclude = .git,.venv,.dockerignore,.pdm-python,Dockerfile

language/.gitignore

@@ -1,3 +1,3 @@
-/__pycache__
+__pycache__
 /.venv
 /.pdm-python

language/Dockerfile

@@ -8,22 +8,20 @@
 # by the GNU Affero General Public License v3.0 only, included in the file
 # licenses/AGPL.txt.
 
-FROM python:3.12-alpine
+FROM python:3.12-slim-bookworm
 
 WORKDIR /app
 
-RUN apk update &&\
-    apk add --no-cache build-base &&\
-    pip3 install pipx
-
 COPY . .
 
-ENV PATH="/root/.local/bin:$PATH"
+RUN apt-get update &&\
+    apt-get install -y curl build-essential curl rust-all
+
+RUN pip install poetry &&\
+    poetry config virtualenvs.create false
 
-RUN pipx install pdm --python $(which python) &&\
-    pdm install --prod --no-editable &&\
-    .venv/bin/python3 -m ensurepip
+RUN poetry install --no-root
 
-ENTRYPOINT ["pdm", "run", "flask", "run", "--host=0.0.0.0", "--port=8084"]
+ENTRYPOINT ["flask", "run", "--host=0.0.0.0", "--port=8084"]
 
 EXPOSE 8084

language/README.md

@@ -2,45 +2,30 @@
 
 ## Getting Started
 
-Use Python 3.12:
-
-```shell
-pdm use [email protected]
-```
-
 Install dependencies:
 
 ```shell
-pdm install
-```
-
-Activate the virtual environment created by PDM:
-
-```shell
-source .venv/bin/activate
+poetry install
 ```
 
-Make sure PIP is available:
-
+Run:
 ```shell
-python3 -m ensurepip
+poetry run flask run --host=0.0.0.0 --port=8084
 ```
 
-Run for development:
+Add `--debug` flag for development:
 
 ```shell
-flask run --host=0.0.0.0 --port=8084 --debug
+poetry run flask run --host=0.0.0.0 --port=8084 --debug
 ```
 
-You can omit the `--debug` flag when not developing.
-
 Lint code:
 
 ```shell
-flake8 .
+poetry run flake8 .
 ```
 
 Format code:
 ```shell
-black .
+poetry run black .
 ```

language/api/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.

language/api/routers/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.

language/api/routers/entities.py

@@ -0,0 +1,34 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.
+
+from flask import Blueprint, request, jsonify
+import spacy.cli
+from ..services.entities import EntityExtractor
+
+bp = Blueprint("entities", __name__)
+
+multi_language_model = "xx_ent_wiki_sm"
+spacy.cli.download(multi_language_model)
+
+nlp = spacy.load(multi_language_model)
+nlp.add_pipe("sentencizer")
+
+
+@bp.route("/v3/entities", methods=["POST"])
+def get_entities():
+    global nlp
+
+    content = request.json
+    text = content["text"]
+
+    entity_extractor = EntityExtractor(nlp)
+    dtos = entity_extractor.run(text)
+
+    return jsonify(dtos)

language/api/routers/health.py

@@ -0,0 +1,18 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.
+
+from flask import Blueprint
+
+bp = Blueprint("health", __name__)
+
+
+@bp.route("/v3/health", methods=["GET"])
+def get_health():
+    return "OK", 200

language/api/routers/version.py

@@ -0,0 +1,18 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.
+
+from flask import Blueprint
+
+bp = Blueprint("version", __name__)
+
+
+@bp.route("/version", methods=["GET"])
+def get_version():
+    return {"version": "3.0.0"}

language/api/services/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2023 Anass Bouassaba.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the GNU Affero General Public License v3.0 only, included in the file
+# licenses/AGPL.txt.