Merge branch 'master' into fix-conda-not-found

Dockerfile

@@ -130,12 +130,9 @@ ENV NVIDIA_VISIBLE_DEVICES=all
 
 WORKDIR /build
 
-COPY Makefile .
-RUN make get-sources
-COPY go.mod .
-RUN make prepare
 COPY . .
 COPY .git .
+RUN make prepare
 
 # stablediffusion does not tolerate a newer version of abseil, build it first
 RUN GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
@@ -144,7 +141,7 @@ RUN if [ "${BUILD_GRPC}" = "true" ]; then \
     git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
     cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
       -DgRPC_BUILD_TESTS=OFF \
-       ../.. && make -j12 install && rm -rf grpc \
+       ../.. && make -j12 install \
     ; fi
 
 # Rebuild with defaults backends
@@ -186,7 +183,11 @@ WORKDIR /build
 # see https://github.com/go-skynet/LocalAI/pull/658#discussion_r1241971626 and
 # https://github.com/go-skynet/LocalAI/pull/434
 COPY . .
-RUN make prepare-sources
+
+COPY --from=builder /build/sources ./sources/
+COPY --from=builder /build/grpc ./grpc/
+
+RUN make prepare-sources && cd /build/grpc/cmake/build && make install && rm -rf grpc
 
 # Copy the binary
 COPY --from=builder /build/local-ai ./

Entitlements.plist

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>com.apple.security.network.client</key>
+    <true/>
+    <key>com.apple.security.network.server</key>
+    <true/>
+</dict>
+</plist>

Makefile

@@ -8,7 +8,7 @@ GOLLAMA_VERSION?=aeba71ee842819da681ea537e78846dc75949ac0
 
 GOLLAMA_STABLE_VERSION?=50cee7712066d9e38306eccadcfbb44ea87df4b7
 
-CPPLLAMA_VERSION?=8e672efe632bb6a7333964a255c4b96f018b9a65
+CPPLLAMA_VERSION?=e9c13ff78114af6fc6a4f27cc8dcdda0f3d389fb
 
 # gpt4all version
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
@@ -106,6 +106,12 @@ ifeq ($(BUILD_TYPE),clblas)
 	CGO_LDFLAGS+=-lOpenCL -lclblast
 endif
 
+ifeq ($(OS),Darwin)
+	ifeq ($(OSX_SIGNING_IDENTITY),)
+		OSX_SIGNING_IDENTITY := $(shell security find-identity -v -p codesigning | grep '"' | head -n 1 | sed -E 's/.*"(.*)"/\1/')
+	endif
+endif
+
 # glibc-static or glibc-devel-static required
 ifeq ($(STATIC),true)
 	LD_FLAGS=-linkmode external -extldflags -static
@@ -216,7 +222,10 @@ sources/go-llama-ggml/libbinding.a: sources/go-llama-ggml
 sources/go-piper/libpiper_binding.a: sources/go-piper
 	$(MAKE) -C sources/go-piper libpiper_binding.a example/main
 
-get-sources: sources/go-llama sources/go-llama-ggml sources/go-ggml-transformers sources/gpt4all sources/go-piper sources/go-rwkv sources/whisper.cpp sources/go-bert sources/go-stable-diffusion
+backend/cpp/llama/llama.cpp:
+	$(MAKE) -C backend/cpp/llama llama.cpp	
+
+get-sources: backend/cpp/llama/llama.cpp sources/go-llama sources/go-llama-ggml sources/go-ggml-transformers sources/gpt4all sources/go-piper sources/go-rwkv sources/whisper.cpp sources/go-bert sources/go-stable-diffusion
 	touch $@
 
 replace:
@@ -230,6 +239,7 @@ replace:
 
 prepare-sources: get-sources replace
 	$(GOCMD) mod download
+	touch $@
 
 ## GENERIC
 rebuild: ## Rebuilds the project
@@ -273,6 +283,9 @@ dist: build
 	mkdir -p release
 	cp $(BINARY_NAME) release/$(BINARY_NAME)-$(BUILD_ID)-$(OS)-$(ARCH)
 
+osx-signed: build
+	codesign --deep --force --sign "$(OSX_SIGNING_IDENTITY)" --entitlements "./Entitlements.plist" "./$(BINARY_NAME)"
+
 ## Run
 run: prepare ## run local-ai
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) run ./
@@ -417,7 +430,7 @@ ifdef BUILD_GRPC_FOR_BACKEND_LLAMA
 	export _PROTOBUF_PROTOC=${INSTALLED_PACKAGES}/bin/proto && \
 	export _GRPC_CPP_PLUGIN_EXECUTABLE=${INSTALLED_PACKAGES}/bin/grpc_cpp_plugin && \
 	export PATH=${PATH}:${INSTALLED_PACKAGES}/bin && \
-	CMAKE_ARGS="${ADDED_CMAKE_ARGS}" LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server 
+	CMAKE_ARGS="${CMAKE_ARGS} ${ADDED_CMAKE_ARGS}" LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server
 else
 	echo "BUILD_GRPC_FOR_BACKEND_LLAMA is not defined."
 	LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server			

README.md

@@ -22,7 +22,7 @@
 
 > :bulb: Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [:speech_balloon: Discord](https://discord.gg/uJAeKSAGDy) [:book: Documentation website](https://localai.io/)
 > 
-> [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/)
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/) [ 🚀 Roadmap ](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
 
 
 [![tests](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml)[![Build and Release](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml)[![build container images](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml)[![Bump dependencies](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml)[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/localai)](https://artifacthub.io/packages/search?repo=localai)
@@ -79,7 +79,12 @@ LocalAI was created by [Ettore Di Giacinto](https://github.com/mudler/) and is a
 
 Note that this started just as a [fun weekend project](https://localai.io/#backstory) in order to try to create the necessary pieces for a full AI assistant like `ChatGPT`: the community is growing fast and we are working hard to make it better and more stable. If you want to help, please consider contributing (see below)!
 
-## 🔥🔥 [Hot topics / Roadmap](https://localai.io/#-hot-topics--roadmap)
+## 🔥🔥 Hot topics / Roadmap
+
+- [Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
+
+Hot topics:
+- https://github.com/mudler/LocalAI/issues/1126
 
 ## 🚀 [Features](https://localai.io/features/)
 
@@ -91,7 +96,7 @@ Note that this started just as a [fun weekend project](https://localai.io/#backs
 - 🧠 [Embeddings generation for vector databases](https://localai.io/features/embeddings/)
 - ✍️ [Constrained grammars](https://localai.io/features/constrained_grammars/)
 - 🖼️ [Download Models directly from Huggingface ](https://localai.io/models/)
-
+- 🆕 [Vision API](https://localai.io/features/gpt-vision/)
 
 ## :book: 🎥 [Media, Blogs, Social](https://localai.io/basics/news/#media-blogs-social)
 

backend/python/bark/ttsbark.py

@@ -1,8 +1,7 @@
+#!/usr/bin/env python3
 """
-This is the extra gRPC server of LocalAI
+This is an extra gRPC server of LocalAI for Bark TTS
 """
-
-#!/usr/bin/env python3
 from concurrent import futures
 import time
 import argparse

backend/python/sentencetransformers/sentencetransformers.py

@@ -1,7 +1,7 @@
+#!/usr/bin/env python3
 """
 Extra gRPC server for HuggingFace SentenceTransformer models.
 """
-#!/usr/bin/env python3
 from concurrent import futures
 
 import argparse

backend/python/transformers/transformers.py

@@ -1,7 +1,7 @@
+#!/usr/bin/env python3
 """
-Extra gRPC server for HuggingFace SentenceTransformer models.
+Extra gRPC server for HuggingFace AutoModel models.
 """
-#!/usr/bin/env python3
 from concurrent import futures
 
 import argparse

docs/content/_index.en.md

@@ -20,7 +20,7 @@ title = "LocalAI"
 
 > 💡 Get help - [❓FAQ](https://localai.io/faq/) [❓How tos](https://localai.io/howtos/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [💭Discord](https://discord.gg/uJAeKSAGDy)
 >
-> [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/)
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [📣 News](https://localai.io/basics/news/) [ 🛫 Examples ](https://github.com/go-skynet/LocalAI/tree/master/examples/) [ 🖼️ Models ](https://localai.io/models/) [ 🚀 Roadmap ](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
 
 **LocalAI** is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families that are compatible with the ggml format. Does not require GPU. It is maintained by [mudler](https://github.com/mudler).
 
@@ -84,37 +84,15 @@ Note that this started just as a [fun weekend project](https://localai.io/#backs
 - 🧠 [Embeddings generation for vector databases](https://localai.io/features/embeddings/)
 - ✍️ [Constrained grammars](https://localai.io/features/constrained_grammars/)
 - 🖼️ [Download Models directly from Huggingface ](https://localai.io/models/)
+- 🆕 [Vision API](https://localai.io/features/gpt-vision/)
+
 
 ## 🔥🔥 Hot topics / Roadmap
 
-- Backend rework for LocalAI v2 https://github.com/go-skynet/LocalAI/issues/1126
-- Roadmap: https://github.com/go-skynet/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap
-
-Some items:
-
-- [ ] Enable gallery management directly from the webui. https://github.com/go-skynet/LocalAI/issues/918
-- [x] llama.cpp lora adapters: https://github.com/go-skynet/LocalAI/issues/919
-- [ ] image variants and edits: https://github.com/go-skynet/LocalAI/issues/921
-- [ ] esrgan for diffusers: https://github.com/go-skynet/LocalAI/issues/917
-- [ ] ggml-stablediffusion: https://github.com/go-skynet/LocalAI/issues/916
-- [ ] SAM: https://github.com/go-skynet/LocalAI/issues/915
-- [x] diffusers lora adapters: https://github.com/go-skynet/LocalAI/issues/914
-- [ ] resource management and control: https://github.com/go-skynet/LocalAI/issues/912
-- [ ] ChatGLM: https://github.com/go-skynet/LocalAI/issues/754
-- [ ] text-to-video : https://github.com/go-skynet/LocalAI/issues/933
-- [ ] rustformers: https://github.com/go-skynet/LocalAI/issues/939
-- [x] Vall-e: https://github.com/go-skynet/LocalAI/issues/985
-- [ ] Speculative sampling: https://github.com/go-skynet/LocalAI/issues/1013
-- [ ] Falcon/GPTNeoX on llama.cpp: https://github.com/go-skynet/LocalAI/issues/1009
-- [ ] transformers/vllm: https://github.com/go-skynet/LocalAI/issues/1015
-- [ ] TortoiseTTS: https://github.com/go-skynet/LocalAI/issues/1016
-- [ ] Exllama2: https://github.com/go-skynet/LocalAI/issues/1053
-- [ ] ctransformers: https://github.com/go-skynet/LocalAI/issues/1056
-- [ ] GPTQ for LLama: https://github.com/go-skynet/LocalAI/issues/1055
-- [ ] LLaVA and miniGPT-4: https://github.com/go-skynet/LocalAI/issues/1054
-- [ ] Test generation inference: https://github.com/go-skynet/LocalAI/issues/1042
-- [ ] docs - extending LocalAI with external backends: https://github.com/go-skynet/LocalAI/issues/1057
+- [Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
 
+Hot topics:
+- https://github.com/mudler/LocalAI/issues/1126
 
 ## How does it work?
 

docs/content/features/_index.en.md

@@ -13,4 +13,5 @@ This section contains the documentation for the features supported by LocalAI.
 - [🎨 Image generation]({{%relref "features/image-generation" %}})
 - [🧠 Embeddings]({{%relref "features/embeddings" %}})
 - [🔥 OpenAI functions]({{%relref "features/openai-functions" %}})
+- [🆕 GPT Vision API]({{%relref "features/gpt-vision" %}})
 - [✍️ Constrained grammars]({{%relref "features/constrained_grammars" %}})

docs/content/features/embeddings.md

@@ -61,23 +61,23 @@ curl http://localhost:8080/embeddings -X POST -H "Content-Type: application/json
 
 ## Huggingface embeddings
 
-To use `sentence-formers` and models in `huggingface` you can use the `huggingface` embedding backend.
+To use `sentence-transformers` and models in `huggingface` you can use the `sentencetransformers` embedding backend.
 
 ```yaml
 name: text-embedding-ada-002
-backend: huggingface-embeddings
+backend: sentencetransformers
 embeddings: true
 parameters:
   model: all-MiniLM-L6-v2
 ```
 
-The `huggingface` backend uses Python [sentence-transformers](https://github.com/UKPLab/sentence-transformers). For a list of all pre-trained models available see here: https://github.com/UKPLab/sentence-transformers#pre-trained-models
+The `sentencetransformers` backend uses Python [sentence-transformers](https://github.com/UKPLab/sentence-transformers). For a list of all pre-trained models available see here: https://github.com/UKPLab/sentence-transformers#pre-trained-models
 
 {{% notice note %}}
 
-- The `huggingface` backend is an optional backend of LocalAI and uses Python. If you are running `LocalAI` from the containers you are good to go and should be already configured for use. If you are running `LocalAI` manually you must install the python dependencies (`pip install -r /path/to/LocalAI/extra/requirements`) and specify the extra backend in the `EXTERNAL_GRPC_BACKENDS` environment variable ( `EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/path/to/LocalAI/extra/grpc/huggingface/huggingface.py"` ) .
-- The `huggingface` backend does support only embeddings of text, and not of tokens. If you need to embed tokens you can use the `bert` backend or `llama.cpp`.
-- No models are required to be downloaded before using the `huggingface` backend. The models will be downloaded automatically the first time the API is used.
+- The `sentencetransformers` backend is an optional backend of LocalAI and uses Python. If you are running `LocalAI` from the containers you are good to go and should be already configured for use. If you are running `LocalAI` manually you must install the python dependencies (`pip install -r /path/to/LocalAI/extra/requirements`) and specify the extra backend in the `EXTERNAL_GRPC_BACKENDS` environment variable ( `EXTERNAL_GRPC_BACKENDS="sentencetransformers:/path/to/LocalAI/backend/python/sentencetransformers/sentencetransformers.py"` ) .
+- The `sentencetransformers` backend does support only embeddings of text, and not of tokens. If you need to embed tokens you can use the `bert` backend or `llama.cpp`.
+- No models are required to be downloaded before using the `sentencetransformers` backend. The models will be downloaded automatically the first time the API is used.
 
 {{% /notice %}}
 

docs/content/features/gpt-vision.md

@@ -0,0 +1,30 @@
+
++++
+disableToc = false
+title = "🆕 GPT Vision"
+weight = 2
++++
+
+{{% notice note %}}
+Available only on `master` builds
+{{% /notice %}}
+
+LocalAI supports understanding images by using [LLaVA](https://llava.hliu.cc/), and implements the [GPT Vision API](https://platform.openai.com/docs/guides/vision) from OpenAI.
+
+![llava](https://github.com/mudler/LocalAI/assets/2420543/cb0a0897-3b58-4350-af66-e6f4387b58d3)
+
+## Usage
+
+OpenAI docs: https://platform.openai.com/docs/guides/vision
+
+To let LocalAI understand and reply with what sees in the image, use the `/v1/chat/completions` endpoint, for example with curl:
+
+```bash
+curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+     "model": "llava",
+     "messages": [{"role": "user", "content": [{"type":"text", "text": "What is in the image?"}, {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" }}], "temperature": 0.9}]}'
+```
+
+### Setup
+
+To setup the LLaVa models, follow the full example in the [configuration examples](https://github.com/mudler/LocalAI/blob/master/examples/configurations/README.md#llava).

docs/content/model-compatibility/vall-e-x.md

@@ -1,7 +1,7 @@
 
 +++
 disableToc = false
-title = "🆕 Vall-E-X"
+title = "Vall-E-X"
 weight = 4
 +++
 

docs/content/model-compatibility/vllm.md

@@ -1,7 +1,7 @@
 
 +++
 disableToc = false
-title = "🆕 vLLM"
+title = "vLLM"
 weight = 4
 +++