From 4f147052819843d604e9cd07a34ecac252c06b8b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 25 Jun 2024 18:14:15 +0200 Subject: [PATCH 1/4] models(gallery): add deepseek-v2-lite Signed-off-by: Ettore Di Giacinto --- core/config/guesser.go | 15 +++++++++++++++ gallery/deepseek.yaml | 20 ++++++++++++++++++++ gallery/index.yaml | 25 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 gallery/deepseek.yaml diff --git a/core/config/guesser.go b/core/config/guesser.go index 7cc3e99b95fd..6c6ef4302991 100644 --- a/core/config/guesser.go +++ b/core/config/guesser.go @@ -20,6 +20,7 @@ const ( ChatML Mistral03 Gemma + DeepSeek2 ) type settingsConfig struct { @@ -37,6 +38,17 @@ var defaultsSettings map[familyType]settingsConfig = map[familyType]settingsConf Completion: "{{.Input}}", }, }, + DeepSeek2: { + StopWords: []string{"<｜end▁of▁sentence｜>"}, + TemplateConfig: TemplateConfig{ + ChatMessage: `{{if eq .RoleName "user" -}}User: {{.Content }} +{{ end -}} +{{if eq .RoleName "assistant" -}}Assistant: {{.Content}}<｜end▁of▁sentence｜>{{end}} +{{if eq .RoleName "system" -}}{{.Content}} +{{end -}}`, + Chat: "{{.Input -}}\nAssistant: ", + }, + }, LLaMa3: { StopWords: []string{"<|eot_id|>"}, TemplateConfig: TemplateConfig{ @@ -208,8 +220,11 @@ func identifyFamily(f *gguf.GGUFFile) familyType { qwen2 := arch == "qwen2" phi3 := arch == "phi-3" gemma := strings.HasPrefix(f.Model().Name, "gemma") + deepseek2 := arch == "deepseek2" switch { + case deepseek2: + return DeepSeek2 case gemma: return Gemma case llama3: diff --git a/gallery/deepseek.yaml b/gallery/deepseek.yaml new file mode 100644 index 000000000000..6a777585c1ac --- /dev/null +++ b/gallery/deepseek.yaml @@ -0,0 +1,20 @@ +--- +name: "deepseek" + +config_file: | + mmap: true + context_size: 8192 + template: + chat_message: |- + {{if eq .RoleName "user" -}}User: {{.Content }} + {{ end -}} + {{if eq .RoleName "assistant" -}}Assistant: {{.Content}}<｜end▁of▁sentence｜>{{end}} + {{if eq .RoleName "system" -}}{{.Content}} + {{end -}} + chat: | + {{.Input -}} + Assistant: + completion: | + {{.Input}} + stopwords: + - '<｜end▁of▁sentence｜>' diff --git a/gallery/index.yaml b/gallery/index.yaml index f6cc0b9656cb..a5b28a2f4f00 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,29 @@ --- +## Deepseek +- &deepseek + url: "github:mudler/LocalAI/gallery/deepseek.yaml@master" + name: "deepseek-coder-v2-lite-instruct" + icon: "https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/logo.svg?raw=true" + license: deepseek + description: | + DeepSeek-Coder-V2, an open-source Mixture-of-Experts (MoE) code language model that achieves performance comparable to GPT4-Turbo in code-specific tasks. Specifically, DeepSeek-Coder-V2 is further pre-trained from DeepSeek-Coder-V2-Base with 6 trillion tokens sourced from a high-quality and multi-source corpus. Through this continued pre-training, DeepSeek-Coder-V2 substantially enhances the coding and mathematical reasoning capabilities of DeepSeek-Coder-V2-Base, while maintaining comparable performance in general language tasks. Compared to DeepSeek-Coder, DeepSeek-Coder-V2 demonstrates significant advancements in various aspects of code-related tasks, as well as reasoning and general capabilities. Additionally, DeepSeek-Coder-V2 expands its support for programming languages from 86 to 338, while extending the context length from 16K to 128K. + In standard benchmark evaluations, DeepSeek-Coder-V2 achieves superior performance compared to closed-source models such as GPT4-Turbo, Claude 3 Opus, and Gemini 1.5 Pro in coding and math benchmarks. The list of supported programming languages can be found in the paper. + urls: + - https://github.com/deepseek-ai/DeepSeek-Coder-V2/tree/main + - https://huggingface.co/LoneStriker/DeepSeek-Coder-V2-Lite-Instruct-GGUF + tags: + - llm + - gguf + - gpu + - deepseek + - cpu + overrides: + parameters: + model: DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf + files: + - filename: DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf + sha256: 50ec78036433265965ed1afd0667c00c71c12aa70bcf383be462cb8e159db6c0 + uri: huggingface://LoneStriker/DeepSeek-Coder-V2-Lite-Instruct-GGUF/DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf ## Start QWEN2 - &qwen2 url: "github:mudler/LocalAI/gallery/chatml.yaml@master" From 746a7f0099ff831ac8d2ac801f3997ca9fa010a6 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 12 Jul 2024 13:51:32 -0400 Subject: [PATCH 2/4] Update deepseek.yaml The trailing space here is presumably part of the template string - try use a chomp keep to get yaml lint to accept it? Signed-off-by: Dave --- gallery/deepseek.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/deepseek.yaml b/gallery/deepseek.yaml index 6a777585c1ac..4ae5916bd34a 100644 --- a/gallery/deepseek.yaml +++ b/gallery/deepseek.yaml @@ -11,7 +11,7 @@ config_file: | {{if eq .RoleName "assistant" -}}Assistant: {{.Content}}<｜end▁of▁sentence｜>{{end}} {{if eq .RoleName "system" -}}{{.Content}} {{end -}} - chat: | + chat: |+ {{.Input -}} Assistant: completion: | From 0c16d30f40c8a38227a082b92c6cc1c3f1771063 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 12 Jul 2024 13:59:26 -0400 Subject: [PATCH 3/4] Update deepseek.yaml chomp didn't fix, erase the space and see what happens. Signed-off-by: Dave --- gallery/deepseek.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery/deepseek.yaml b/gallery/deepseek.yaml index 4ae5916bd34a..b7f8605f0b52 100644 --- a/gallery/deepseek.yaml +++ b/gallery/deepseek.yaml @@ -11,9 +11,9 @@ config_file: | {{if eq .RoleName "assistant" -}}Assistant: {{.Content}}<｜end▁of▁sentence｜>{{end}} {{if eq .RoleName "system" -}}{{.Content}} {{end -}} - chat: |+ + chat: | {{.Input -}} - Assistant: + Assistant: completion: | {{.Input}} stopwords: From 5d672c6420e54583a1e92649cd0d7c750f56985b Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 12 Jul 2024 14:00:42 -0400 Subject: [PATCH 4/4] Update deepseek.yaml Signed-off-by: Dave --- gallery/deepseek.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/deepseek.yaml b/gallery/deepseek.yaml index b7f8605f0b52..fa8870a1d54d 100644 --- a/gallery/deepseek.yaml +++ b/gallery/deepseek.yaml @@ -13,7 +13,7 @@ config_file: | {{end -}} chat: | {{.Input -}} - Assistant: + Assistant: # Space is preserved for templating reasons, but line does not end with one for the linter. completion: | {{.Input}} stopwords: