Merge pull request #4 from epoch8/3.6.5-e8-full

3.6.5 e8 full

Makefile

@@ -285,6 +285,14 @@ build-docker-spacy-ru:
 	docker buildx bake -f docker/docker-bake.hcl base-builder && \
 	docker buildx bake -f docker/docker-bake.hcl spacy-ru
 
+build-docker-sbert-l:
+	export IMAGE_NAME=rasa && \
+	docker buildx use default && \
+	docker buildx bake -f docker/docker-bake.hcl base && \
+	docker buildx bake -f docker/docker-bake.hcl base-poetry && \
+	docker buildx bake -f docker/docker-bake.hcl base-builder && \
+	docker buildx bake -f docker/docker-bake.hcl sbert-l
+
 build-docker-spacy-ru-gpu:
 	export IMAGE_NAME=rasa && \
 	export BASE_IMAGE=nvidia/cuda:11.2.2-devel-ubuntu20.04 && \
@@ -309,6 +317,9 @@ stop-integration-containers: ## Stop the integration test containers.
 build-e8: build-docker
 	docker tag rasa:localdev ghcr.io/epoch8/rasa/rasa:$(shell cat version)
 
+build-e8-sbert-l: build-docker-sbert-l
+	docker tag rasa:localdev-sbert-l ghcr.io/epoch8/rasa/rasa-sbert-l:$(shell cat version)
+
 build-e8-spacy-ru: build-docker-spacy-ru
 	docker tag rasa:localdev-spacy-ru ghcr.io/epoch8/rasa/rasa-spacy-ru:$(shell cat version)
 
@@ -322,4 +333,7 @@ upload-spacy-ru:
 	docker push ghcr.io/epoch8/rasa/rasa-spacy-ru:$(shell cat ./version)
 
 upload-spacy-ru-gpu:
-	docker push ghcr.io/epoch8/rasa/rasa-spacy-ru:$(shell cat ./version)-gpu
+	docker push ghcr.io/epoch8/rasa/rasa-spacy-ru:$(shell cat ./version)-gpu
+
+upload-sbert-l:
+	docker push ghcr.io/epoch8/rasa/rasa-sbert-l:$(shell cat ./version)

docker/Dockerfile.pretrained_embeddings_sbert_large

@@ -0,0 +1,56 @@
+# The image tagged with the 'spacy-it' suffix
+ARG IMAGE_BASE_NAME
+ARG BASE_IMAGE_HASH
+ARG BASE_BUILDER_IMAGE_HASH
+
+FROM ${IMAGE_BASE_NAME}:base-builder-${BASE_BUILDER_IMAGE_HASH} as builder
+
+# copy files
+COPY . /build/
+COPY docker/configs/config_pretrained_embeddings_sbert_large.yml /build/config.yml
+
+# change working directory
+WORKDIR /build
+
+# install dependencies
+RUN python -m venv /opt/venv && \
+  . /opt/venv/bin/activate && pip install --no-cache-dir -U "pip==22.*" -U "wheel>0.38.0"
+RUN . /opt/venv/bin/activate && poetry install --extras transformers --no-dev --no-root --no-interaction
+RUN . /opt/venv/bin/activate && poetry build -f wheel -n && \
+  pip install --no-deps dist/*.whl && \
+  rm -rf dist *.egg-info
+
+# make sure we use the virtualenv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# start a new build stage
+FROM ${IMAGE_BASE_NAME}:base-${BASE_IMAGE_HASH} as runner
+
+# copy everything from /opt
+COPY --from=builder /opt/venv /opt/venv
+
+# make sure we use the virtualenv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# set HOME environment variable
+ENV HOME=/app
+
+# update permissions & change user to not run as root
+WORKDIR /app
+
+RUN pip install huggingface_hub
+RUN python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='ai-forever/sbert_large_mt_nlu_ru', ignore_patterns=['*.msgpack', '*.bin'], local_dir='./weighs', local_dir_use_symlinks=True);"
+
+RUN chgrp -R 0 /app && chmod -R g=u /app && chmod o+wr /app
+USER 1001
+
+# Create a volume for temporary data
+VOLUME /tmp
+
+# change shell
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# the entry point
+EXPOSE 5005
+ENTRYPOINT ["rasa"]
+CMD ["--help"]

docker/configs/config_pretrained_embeddings_sbert_large.yml

@@ -0,0 +1,18 @@
+language: "ru"
+
+pipeline:
+  - name: WhitespaceTokenizer
+  - name: RegexFeaturizer
+
+  - name: LanguageModelFeaturizer
+    model_name: "bert"
+    model_weights: "weighs"
+
+  - name: DIETClassifier
+    epochs: 50
+    batch_size: 4
+    constrain_similarities: true
+
+  - name: FallbackClassifier
+    threshold: 0.2
+    ambiguity_threshold: 0.05

docker/docker-bake.hcl

@@ -194,6 +194,25 @@ target "spacy-en" {
   ]
 }
 
+target "sbert-l" {
+  dockerfile = "docker/Dockerfile.pretrained_embeddings_sbert_large"
+  tags       = ["${IMAGE_NAME}:${IMAGE_TAG}-sbert-l"]
+
+  args = {
+    IMAGE_BASE_NAME         = "${IMAGE_NAME}"
+    BASE_IMAGE_HASH         = "${BASE_IMAGE_HASH}"
+    BASE_BUILDER_IMAGE_HASH = "${BASE_BUILDER_IMAGE_HASH}"
+  }
+
+  cache-to = ["type=inline"]
+
+  cache-from = [
+    "type=registry,ref=${IMAGE_NAME}:base-${BASE_IMAGE_HASH}",
+    "type=registry,ref=${IMAGE_NAME}:base-builder-${BASE_BUILDER_IMAGE_HASH}",
+    "type=registry,ref=${IMAGE_NAME}:${IMAGE_TAG}-sbert-l",
+  ]
+}
+
 target "spacy-ru" {
   dockerfile = "docker/Dockerfile.pretrained_embeddings_spacy_ru"
   tags       = ["${IMAGE_NAME}:${IMAGE_TAG}-spacy-ru"]