diff --git a/app/package.json b/app/package.json index 19d0446..6f24e92 100644 --- a/app/package.json +++ b/app/package.json @@ -1,7 +1,7 @@ { "name": "@av/harbor-app", "private": true, - "version": "0.2.4", + "version": "0.2.5", "type": "module", "scripts": { "dev": "vite", diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 6992ce1..5d24105 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -1474,7 +1474,7 @@ dependencies = [ [[package]] name = "harbor-app" -version = "0.2.3" +version = "0.2.5" dependencies = [ "fix-path-env", "serde", diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index ca54243..497c6f5 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "harbor-app" -version = "0.2.4" +version = "0.2.5" description = "A companion app for Harbor LLM toolkit" authors = ["av"] edition = "2021" diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index 0a260f6..7837120 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2.0.0-rc", "productName": "Harbor", - "version": "0.2.4", + "version": "0.2.5", "identifier": "com.harbor.app", "build": { "beforeDevCommand": "bun run dev", diff --git a/app/src/AppRoutes.tsx b/app/src/AppRoutes.tsx index a34fbee..a19f8d5 100644 --- a/app/src/AppRoutes.tsx +++ b/app/src/AppRoutes.tsx @@ -21,7 +21,7 @@ export const ROUTES: Record = { }, config: { id: 'config', - name: Profiles, + name: Profiles, path: '/config', element: , }, diff --git a/app/src/LinearLoading.tsx b/app/src/LinearLoading.tsx deleted file mode 100644 index 1f3f859..0000000 --- a/app/src/LinearLoading.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { useEffect, useState } from 'react'; - -export const TOGGLE_DELAY = 250; - -export const LinearLoader = ({ loading }: { loading: boolean }) => { - const [showLoader, setShowLoader] = useState(false); - - useEffect(() => { - let timer: number; - - if (loading) { - timer = setTimeout(() => setShowLoader(true), TOGGLE_DELAY); - } else { - setShowLoader(false); - } - return () => clearTimeout(timer); - }, [loading]); - - if (!showLoader) return null; - - return ; -}; diff --git a/app/src/Loading.tsx b/app/src/Loading.tsx new file mode 100644 index 0000000..fb24cec --- /dev/null +++ b/app/src/Loading.tsx @@ -0,0 +1,32 @@ +import { useEffect, useState } from 'react'; + +export const TOGGLE_DELAY = 250; + +export const LoaderElements = { + linear: , + overlay: ( +
+ +
+ ), +} + +export const Loader = ({ loading, loader = "linear" }: { loading: boolean, loader: keyof typeof LoaderElements }) => { + const [showLoader, setShowLoader] = useState(false); + const loaderComponent = LoaderElements[loader]; + + useEffect(() => { + let timer: number; + + if (loading) { + timer = setTimeout(() => setShowLoader(true), TOGGLE_DELAY); + } else { + setShowLoader(false); + } + return () => clearTimeout(timer); + }, [loading]); + + if (!showLoader) return null; + + return loaderComponent; +}; diff --git a/app/src/config/Config.tsx b/app/src/config/Config.tsx index b4c0c3f..e80b2b2 100644 --- a/app/src/config/Config.tsx +++ b/app/src/config/Config.tsx @@ -1,7 +1,7 @@ import { FC } from "react"; import { useHarborConfig } from "./useHarborConfig"; import { ProfileSelector } from "../settings/ProfileSelector"; -import { LinearLoader } from "../LinearLoading"; +import { Loader } from "../Loading"; import { ScrollToTop } from "../ScrollToTop"; export const Config: FC = () => { @@ -9,7 +9,7 @@ export const Config: FC = () => { return ( <> - + diff --git a/app/src/home/Doctor.tsx b/app/src/home/Doctor.tsx index 20d2534..e88b137 100644 --- a/app/src/home/Doctor.tsx +++ b/app/src/home/Doctor.tsx @@ -1,7 +1,7 @@ import { useMemo } from "react"; import { Section } from "../Section"; import { useHarbor } from "../useHarbor"; -import { LinearLoader } from "../LinearLoading"; +import { Loader } from "../Loading"; import { IconButton } from "../IconButton"; import { IconRotateCW } from "../Icons"; @@ -56,7 +56,7 @@ export const Doctor = () => { } children={ <> - + {error && {error.message}} {output} diff --git a/app/src/home/ServiceCard.tsx b/app/src/home/ServiceCard.tsx index ce257e6..137b964 100644 --- a/app/src/home/ServiceCard.tsx +++ b/app/src/home/ServiceCard.tsx @@ -6,7 +6,7 @@ import { IconBandage, IconExternalLink, } from "../Icons"; -import { ACTION_ICONS, HarborService, HST } from "../serviceMetadata"; +import { ACTION_ICONS, HarborService, HST, HSTColorOpts, HSTColors } from "../serviceMetadata"; import { runHarbor } from "../useHarbor"; import { toasted } from "../utils"; @@ -40,7 +40,7 @@ export const ServiceCard = ( const toggleService = () => { const msg = (str: string) => ( - {service.handle} + {service.handle} {str} ); @@ -75,9 +75,12 @@ export const ServiceCard = ( : ACTION_ICONS.up; const canLaunch = !service.tags.includes(HST.cli); + const gradientTag = service.tags.find(t => HSTColorOpts.includes(t as HST)); + + const gradientClass = gradientTag ? `bg-gradient-to-tr from-0% to-50% ${HSTColors[gradientTag]}` : ""; return ( -
+

{service.handle} diff --git a/app/src/home/ServiceList.tsx b/app/src/home/ServiceList.tsx index f1bc4a2..03890c0 100644 --- a/app/src/home/ServiceList.tsx +++ b/app/src/home/ServiceList.tsx @@ -5,7 +5,7 @@ import { Section } from "../Section"; import { ServiceCard } from "./ServiceCard"; import { useServiceList } from "./useServiceList"; import { useArrayState } from "../useArrayState"; -import { LinearLoader } from "../LinearLoading"; +import { Loader } from "../Loading"; import { IconButton } from "../IconButton"; import { ACTION_ICONS, HarborService, HST } from "../serviceMetadata"; import { runHarbor } from "../useHarbor"; @@ -122,8 +122,8 @@ export const ServiceList = () => {

} children={ - <> - +
+ {error &&
{error.message}
} {services && (
    @@ -142,7 +142,7 @@ export const ServiceList = () => { })}
)} - +
} /> ); diff --git a/app/src/home/Version.tsx b/app/src/home/Version.tsx index 3f07bc0..a731e59 100644 --- a/app/src/home/Version.tsx +++ b/app/src/home/Version.tsx @@ -1,4 +1,4 @@ -import { LinearLoader } from "../LinearLoading"; +import { Loader } from "../Loading"; import { Section } from "../Section"; import { useHarbor } from "../useHarbor"; @@ -11,7 +11,7 @@ export const Version = () => { header="Version" children={ <> - + {error && {error.message}} {result?.stdout} diff --git a/app/src/serviceMetadata.tsx b/app/src/serviceMetadata.tsx index d3669f9..e111e24 100644 --- a/app/src/serviceMetadata.tsx +++ b/app/src/serviceMetadata.tsx @@ -1,7 +1,7 @@ import { IconPlaneLanding, IconRocketLaunch } from "./Icons"; export const ACTION_ICONS = { - loading: , + loading: , up: , down: , }; @@ -19,6 +19,14 @@ export enum HST { audio = 'Audio', }; +export const HSTColors: Partial> = { + [HST.backend]: 'from-primary/10', + [HST.frontend]: 'from-secondary/10', + [HST.satellite]: 'from-accent/10', +}; + +export const HSTColorOpts = Object.keys(HSTColors) as HST[]; + export type HarborService = { handle: string; isRunning: boolean; @@ -176,5 +184,8 @@ export const serviceMetadata: Record> = { }, anythingllm: { tags: [HST.frontend, HST.partial] - } + }, + nexa: { + tags: [HST.backend, HST.partial], + }, }; \ No newline at end of file diff --git a/app/src/settings/Settings.tsx b/app/src/settings/Settings.tsx index b65e55a..9f13a70 100644 --- a/app/src/settings/Settings.tsx +++ b/app/src/settings/Settings.tsx @@ -4,7 +4,7 @@ import { THEMES, useTheme } from "../theme"; import { useAutostart } from "../useAutostart"; export const Settings = () => { - const [theme, setTheme] = useTheme(); + const theme = useTheme(); const autostart = useAutostart(); const handleAutostartChange = (e: React.ChangeEvent) => { @@ -16,14 +16,38 @@ export const Settings = () => {
-

Theme

-

Saved automatically.

+
+
+

Auto Start

+

+ Launch Harbor App when your system starts. +

+ +
+ +
+
+ +
+

Theme

+

+ Customize the look and feel of Harbor App. +

+
- {theme} + {theme.theme}
    { > {THEMES.map((t) => { return ( -
  • setTheme(t)} key={t} value={t}> +
  • theme.setTheme(t)} key={t} value={t}> {t}
    • ); @@ -50,24 +74,68 @@ export const Settings = () => {
+ +
-

Auto Start

-

- Launch Harbor App when your system starts. -

- -
- +
+

Hue

+

+ Adjust the hue of the theme color. +

+ + theme.setHue(parseInt(e.target.value))} + /> +
+ +
+

Saturation

+

+ How vibrant the colors are. +

+ + theme.setSaturation(parseInt(e.target.value))} + /> +
+ +
+

Contrast

+

+ The difference between the lightest and darkest colors. +

+ + theme.setContrast(parseInt(e.target.value))} + /> +
+ +
+

Brightness

+

+ The overall lightness or darkness of the theme. +

+ + theme.setBrightness(parseInt(e.target.value))} + /> +
+ +
+

Invert

+

+ Change the colors to their opposites. +

+ + theme.setInvert(parseInt(e.target.value))} + />
} diff --git a/app/src/theme.tsx b/app/src/theme.tsx index e0cc7c4..dac2319 100644 --- a/app/src/theme.tsx +++ b/app/src/theme.tsx @@ -1,39 +1,103 @@ import themes from "daisyui/src/theming/themes"; import * as localStorage from "./localStorage"; -import { useCallback } from "react"; +import { useCallback, useRef } from "react"; import { useStoredState } from "./useStoredState"; export const DEFAULT_THEME = "harborLight"; + +// "dim" theme crashes tauri app host +// due to unexplainable reasons, so removing it +// from the list of available themes permanently +export const DISABLED_THEMES = new Set(['dim']) + export const THEMES = [ - "harborLight", - "harborDark", - ...Object.keys(themes), + "harborLight", + "harborDark", + ...Object.keys(themes).filter((theme) => !DISABLED_THEMES.has(theme)), ]; +export const DEFAULT_THEME_STATE = { + theme: DEFAULT_THEME, + hue: 0, + saturation: 100, + contrast: 100, + brightness: 100, + invert: 0, +}; + export const getTheme = () => { - return localStorage.readLocalStorage('theme', DEFAULT_THEME); + return localStorage.readLocalStorage('themeState', DEFAULT_THEME_STATE); } -export const setTheme = (newTheme: string) => { +export const setTheme = (theme: typeof DEFAULT_THEME_STATE) => { const themeRoot = document.documentElement; + if (themeRoot) { - themeRoot.setAttribute('data-theme', newTheme); + themeRoot.setAttribute('data-theme', theme.theme); + } + + const filterRoot = document.body; + + if (filterRoot) { + const parts = [ + `hue-rotate(${theme.hue}deg)`, + `saturate(${theme.saturation}%)`, + `contrast(${theme.contrast}%)`, + `brightness(${theme.brightness}%)`, + `invert(${theme.invert}%)`, + ] + + document.body.style.filter = parts.join(' '); } } export const init = () => { - const theme = getTheme(); - setTheme(theme); + setTheme(getTheme()); } -export const useTheme = (): [string, (newTheme: string) => void] => { - const [theme, setThemeState] = useStoredState('theme', DEFAULT_THEME); +export const useTheme = () => { + const [theme, setThemeState] = useStoredState('themeState', DEFAULT_THEME_STATE); + const themeRef = useRef(theme); + themeRef.current = theme; - const changeTheme = useCallback((newTheme: string) => { + const updateTheme = useCallback((newTheme: typeof DEFAULT_THEME_STATE) => { setTheme(newTheme); setThemeState(newTheme); }, []); - return [theme, changeTheme]; + const changeTheme = useCallback((newTheme: string) => { + updateTheme({ ...themeRef.current, theme: newTheme }); + }, []); + + const changeHue = useCallback((newHue: number) => { + updateTheme({ ...themeRef.current, hue: newHue }); + }, []); + + const changeSaturation = useCallback((newSaturation: number) => { + updateTheme({ ...themeRef.current, saturation: newSaturation }); + }, []); + + const changeContrast = useCallback((newContrast: number) => { + updateTheme({ ...themeRef.current, contrast: newContrast }); + }, []); + + const changeBrightness = useCallback((newBrightness: number) => { + updateTheme({ ...themeRef.current, brightness: newBrightness }); + }, []); + + const changeInvert = useCallback((newInvert: number) => { + updateTheme({ ...themeRef.current, invert: newInvert }); + }, []); + + return { + ...theme, + reset: () => updateTheme(DEFAULT_THEME_STATE), + setTheme: changeTheme, + setHue: changeHue, + setSaturation: changeSaturation, + setContrast: changeContrast, + setBrightness: changeBrightness, + setInvert: changeInvert, + }; } \ No newline at end of file diff --git a/compose.nexa.yml b/compose.nexa.yml new file mode 100644 index 0000000..03ccc99 --- /dev/null +++ b/compose.nexa.yml @@ -0,0 +1,38 @@ +services: + nexa: + container_name: ${HARBOR_CONTAINER_PREFIX}.nexa + build: + context: ./nexa + dockerfile: Dockerfile + # This can (and will) be overriden by .x.nvidia file + args: + - HARBOR_NEXA_IMAGE=ubuntu:22.04 + networks: + - harbor-network + volumes: + - ${HARBOR_OLLAMA_CACHE}:/root/.ollama + - ${HARBOR_HF_CACHE}:/root/.cache/huggingface + - ${HARBOR_LLAMACPP_CACHE}:/root/.cache/llama.cpp + - ${HARBOR_VLLM_CACHE}:/root/.cache/vllm + - ${HARBOR_NEXA_CACHE}:/root/.cache/nexa + - ./nexa/openai_models.py:/usr/local/lib/python3.9/dist-packages/nexa/openai_models.py + env_file: + - ./.env + - ./nexa/override.env + command: > + server + --host 0.0.0.0 + --port 8000 + ${HARBOR_NEXA_MODEL} + + nexa-proxy: + build: + context: ./nexa + dockerfile: proxy.Dockerfile + container_name: ${HARBOR_CONTAINER_PREFIX}.nexa-proxy + ports: + - ${HARBOR_NEXA_HOST_PORT}:8000 + volumes: + - ./nexa/proxy_server.py:/app/proxy_server.py + networks: + - harbor-network \ No newline at end of file diff --git a/compose.x.nexa.nvidia.yml b/compose.x.nexa.nvidia.yml new file mode 100644 index 0000000..6f67c00 --- /dev/null +++ b/compose.x.nexa.nvidia.yml @@ -0,0 +1,14 @@ +services: + nexa: + build: + # This is a CUDA-enabled override for the base + # image that is CPU-only + args: + - HARBOR_NEXA_IMAGE=nvidia/cuda:12.4.0-base-ubuntu22.04 + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] \ No newline at end of file diff --git a/compose.x.webui.nexa.yml b/compose.x.webui.nexa.yml new file mode 100644 index 0000000..b3005b9 --- /dev/null +++ b/compose.x.webui.nexa.yml @@ -0,0 +1,4 @@ +services: + webui: + volumes: + - ./open-webui/configs/config.nexa.json:/app/configs/config.nexa.json \ No newline at end of file diff --git a/default.env b/default.env deleted file mode 100644 index 79e01f7..0000000 --- a/default.env +++ /dev/null @@ -1,317 +0,0 @@ -# ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ -# ▒█░▒█ ░█▀▀█ ▒█▀▀█ ▒█▀▀█ ▒█▀▀▀█ ▒█▀▀█ -# ▒█▀▀█ ▒█▄▄█ ▒█▄▄▀ ▒█▀▀▄ ▒█░░▒█ ▒█▄▄▀ -# ▒█░▒█ ▒█░▒█ ▒█░▒█ ▒█▄▄█ ▒█▄▄▄█ ▒█░▒█ -# ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ - -# Harbor Configuration. -# --------------------- -# This section contains environment variables that are -# used in Harbor's compose files, configs and can be modified via harbor CLI. -# -# | Using CLI for config management: -# | -# | ```bash -# | harbor config get hf.cache -# | harbor config set hf.cache ~/.cache/huggingface -# | ``` -# | -# | See more at https://github.com/av/harbor/wiki/Harbor-CLI-Reference - -# Abstract/shared -# --------------------- - -HARBOR_HF_CACHE="~/.cache/huggingface" -HARBOR_HF_TOKEN="" - -HARBOR_LLAMACPP_CACHE="~/.cache/llama.cpp" -HARBOR_OLLAMA_CACHE="~/.ollama" -HARBOR_VLLM_CACHE="~/.cache/vllm" -HARBOR_TXTAI_CACHE="~/.cache/txtai" - -# These could be used by specific services, -# in which case they can be set in a centralised -# location like this. -HARBOR_ANYSCALE_KEY="" -HARBOR_APIPIE_KEY="" -HARBOR_COHERE_KEY="" -HARBOR_FIREWORKS_API_KEY="" -HARBOR_GROQ_KEY="" -HARBOR_MISTRAL_KEY="" -HARBOR_OPENROUTER_KEY="" -HARBOR_PERPLEXITY_KEY="" -HARBOR_SHUTTLEAI_KEY="" -HARBOR_TOGETHERAI_KEY="" -HARBOR_ANTHROPIC_KEY="" -HARBOR_BINGAI_TOKEN="" -HARBOR_GOOGLE_KEY="" -HARBOR_ASSISTANTS_KEY="" -HARBOR_CIVITAI_TOKEN="" - -HARBOR_UI_MAIN="webui" -HARBOR_UI_AUTOOPEN=false -HARBOR_SERVICES_DEFAULT="ollama;webui" -HARBOR_SERVICES_TUNNELS="" -HARBOR_CONTAINER_PREFIX="harbor" -HARBOR_CLI_NAME="harbor" -HARBOR_CLI_SHORT="h" -HARBOR_CLI_PATH="~/.local/bin" -HARBOR_LOG_LEVEL="INFO" - -# OpenAI -# --------------------- -# In the Context of Harbor, it means OpenAI API-compatible -# services, such as Ollama, Llama.cpp, LiteLLM, etc. - -HARBOR_OPENAI_URLS="" -HARBOR_OPENAI_KEYS="" - -# This variable is derived as a first item in the list above -HARBOR_OPENAI_KEY="" -HARBOR_OPENAI_URL="" - -# webui -HARBOR_WEBUI_HOST_PORT=33801 -# Persistent secret - user stays logged into -# webui between restarts -HARBOR_WEBUI_SECRET="h@rb0r" -HARBOR_WEBUI_NAME="Harbor" -HARBOR_WEBUI_LOG_LEVEL="INFO" -HARBOR_WEBUI_VERSION="main" - -# llamacpp -HARBOR_LLAMACPP_HOST_PORT=33831 -HARBOR_LLAMACPP_GGUF="" -HARBOR_LLAMACPP_MODEL="https://huggingface.co/microsoft/Phi-3.5-mini-instruct-gguf/blob/main/Phi-3-mini-4k-instruct-q4.gguf" -HARBOR_LLAMACPP_MODEL_SPECIFIER="--hf-repo microsoft/Phi-3.5-mini-instruct-gguf --hf-file Phi-3-mini-4k-instruct-q4.gguf" -HARBOR_LLAMACPP_EXTRA_ARGS="-ngl 32" - -# ollama -HARBOR_OLLAMA_HOST_PORT=33821 -HARBOR_OLLAMA_VERSION="latest" - -# litellm -HARBOR_LITELLM_HOST_PORT=33841 -HARBOR_LITELLM_MASTER_KEY="sk-litellm" -HARBOR_LITELLM_DB_HOST_PORT=33842 -HARBOR_LITELLM_UI_USERNAME="admin" -HARBOR_LITELLM_UI_PASSWORD="admin" - -# lmdeploy -HARBOR_LMDEPLOY_HOST_PORT=33831 - -# searxng -HARBOR_SEARXNG_HOST_PORT=33811 - -# tgi (text-generation-inference) -HARBOR_TGI_HOST_PORT=33851 -HARBOR_TGI_MODEL="google/gemma-2-2b-it" -HARBOR_TGI_QUANT="" -HARBOR_TGI_REVISION="" -HARBOR_TGI_EXTRA_ARGS="--max-concurrent-requests 16" -HARBOR_TGI_MODEL_SPECIFIER="--model-id google/gemma-2-2b-it" - -# tts (openedai-sppech) -HARBOR_TTS_HOST_PORT=33861 -HARBOR_TTS_HOME="voices" -HARBOR_TTS_VOICES_FOLDER="./tts/voices" -HARBOR_TTS_CONFIG_FOLDER="./tts/config" - -# hollama -HARBOR_HOLLAMA_HOST_PORT=33871 - -# LangFuse -HARBOR_LANGFUSE_HOST_PORT=33881 -HARBOR_LANGFUSE_NEXTAUTH_SECRET="langfuse" -HARBOR_LANGFUSE_SALT="salt" -HARBOR_LANGFUSE_DB_HOST_PORT=33882 -# These should be set when configuring -# new project in the service -HARBOR_LANGFUSE_PUBLIC_KEY="" -HARBOR_LANGFUSE_SECRET_KEY="" - -# LibreChat -HARBOR_LIBRECHAT_HOST_PORT=33891 -HARBOR_LIBRECHAT_RAG_HOST_PORT=33892 - -# BionicGPT -HARBOR_BIONICGPT_HOST_PORT=33901 - -# vLLM -HARBOR_VLLM_HOST_PORT=33911 -HARBOR_VLLM_VERSION="v0.6.0" -HARBOR_VLLM_MODEL="microsoft/Phi-3.5-mini-instruct" -HARBOR_VLLM_EXTRA_ARGS="" -HARBOR_VLLM_ATTENTION_BACKEND="FLASH_ATTN" -HARBOR_VLLM_MODEL_SPECIFIER="--model microsoft/Phi-3.5-mini-instruct" - -# Aphrodite -HARBOR_APHRODITE_HOST_PORT=33921 -HARBOR_APHRODITE_VERSION="a03e0e2" -HARBOR_APHRODITE_EXTRA_ARGS="" -HARBOR_APHRODITE_MODEL="neuralmagic/Mistral-7B-Instruct-v0.3-GPTQ-4bit" - -# TabbyAPI -HARBOR_TABBYAPI_HOST_PORT=33931 -HARBOR_TABBYAPI_ADMIN_KEY="adk-tabbyapi" -HARBOR_TABBYAPI_API_KEY="apk-tabbyapi" -HARBOR_TABBYAPI_MODEL="Annuvin/gemma-2-2b-it-abliterated-4.0bpw-exl2" -HARBOR_TABBYAPI_MODEL_SPECIFIER="Annuvin_gemma-2-2b-it-abliterated-4.0bpw-exl2" -HARBOR_TABBYAPI_EXTRA_ARGS="" - -# Parllama -HARBOR_PARLLAMA_CACHE="~/.parllama" - -# Plandex -HARBOR_PLANDEX_HOST_PORT=33941 -HARBOR_PLANDEX_DB_HOST_PORT=33942 -HARBOR_PLANDEX_HOME="~/.plandex-home" - -# Mistral.rs -HARBOR_MISTRALRS_HOST_PORT=33951 -HARBOR_MISTRALRS_VERSION="0.3" -HARBOR_MISTRALRS_MODEL_TYPE="plain" -HARBOR_MISTRALRS_MODEL="microsoft/Phi-3.5-mini-instruct" -HARBOR_MISTRALRS_MODEL_ARCH="phi3" -HARBOR_MISTRALRS_MODEL_ISQ="" -HARBOR_MISTRALRS_MODEL_SPECIFIER="plain -m microsoft/Phi-3.5-mini-instruct -a phi3" -HARBOR_MISTRALRS_EXTRA_ARGS="" - -# Open Interpreter -HARBOR_OPINT_CONFIG_PATH="~/.config/open-interpreter" -HARBOR_OPINT_EXTRA_ARGS="" -HARBOR_OPINT_MODEL="llama3.1" -HARBOR_OPINT_CMD="--model llama3.1" -HARBOR_OPINT_BACKEND="" - -# cmdh -HARBOR_CMDH_MODEL="llama3.1" -HARBOR_CMDH_LLM_HOST="ollama" -HARBOR_CMDH_LLM_KEY="" -HARBOR_CMDH_LLM_URL="" - -# Dify -HARBOR_DIFY_HOST_PORT=33961 -HARBOR_DIFY_DB_HOST_PORT=33962 -HARBOR_DIFY_D2O_HOST_PORT=33963 -HARBOR_DIFY_VERSION="0.6.16" -HARBOR_DIFY_SANDBOX_VERSION="0.2.1" -HARBOR_DIFY_WEAVIATE_VERSION="1.19.0" -HARBOR_DIFY_VOLUMES="./dify/volumes" -HARBOR_DIFY_BOT_TYPE="Chat" -HARBOR_DIFY_OPENAI_WORKFLOW="" - -# Fabric -HARBOR_FABRIC_CONFIG_PATH="~/.config/fabric" -HARBOR_FABRIC_MODEL="llama3.1:8b" - -# Parler -HARBOR_PARLER_HOST_PORT=33971 -HARBOR_PARLER_MODEL="parler-tts/parler-tts-mini-v1" -HARBOR_PARLER_VOICE="Alisa speaks in calm but steady voice. A very clear audio." - -# AirLLM -HARBOR_AIRLLM_HOST_PORT=33981 -HARBOR_AIRLLM_MODEL="meta-llama/Meta-Llama-3.1-8B-Instruct" -HARBOR_AIRLLM_CTX_LEN=128 -HARBOR_AIRLLM_COMPRESSION="4bit" - -# txtai -HARBOR_TXTAI_RAG_HOST_PORT=33991 -HARBOR_TXTAI_RAG_MODEL="llama3.1:8b-instruct-q4_K_M" -HARBOR_TXTAI_RAG_EMBEDDINGS="neuml/txtai-wikipedia-slim" - -# TextGrad -HARBOR_TEXTGRAD_HOST_PORT=34001 - -# Aider -HARBOR_AIDER_HOST_PORT=34011 -HARBOR_AIDER_MODEL="llama3.1:8b-instruct-q6_K" - -# HuggingFace ChatUI -HARBOR_CHATUI_HOST_PORT=34021 -HARBOR_CHATUI_VERSION="latest" -HARBOR_CHATUI_OLLAMA_MODEL="llama3.1:8b" -HARBOR_CHATUI_LITELLM_MODEL="llama3.1:8b" - -# ComfyUI -HARBOR_COMFYUI_HOST_PORT=34031 -HARBOR_COMFYUI_PORTAL_HOST_PORT=34032 -HARBOR_COMFYUI_SYNCTHING_HOST_PORT=34033 -HARBOR_COMFYUI_VERSION="latest-cuda" -HARBOR_COMFYUI_AUTH=true -HARBOR_COMFYUI_USER="harbor" -HARBOR_COMFYUI_PASSWORD="sk-comfyui" -HARBOR_COMFYUI_ARGS="" -HARBOR_COMFYUI_PROVISIONING="https://raw.githubusercontent.com/av/harbor/main/comfyui/provisioning.sh" - -# Perplexica -HARBOR_PERPLEXICA_HOST_PORT=34041 -HARBOR_PERPLEXICA_BACKEND_HOST_PORT=34042 - -# Aichat -HARBOR_AICHAT_HOST_PORT=34051 -HARBOR_AICHAT_MODEL="llama3.1:8b" -HARBOR_AICHAT_CONFIG_PATH="~/.config/aichat" - -# AutoGPT -HARBOR_AUTOGPT_HOST_PORT=34061 -HARBOR_AUTOGPT_MODEL="llama3.1:8b" - -# LobeChat -HARBOR_LOBECHAT_HOST_PORT=34071 -HARBOR_LOBECHAT_VERSION="latest" - -# Omnichain -HARBOR_OMNICHAIN_HOST_PORT=34081 -HARBOR_OMNICHAIN_API_HOST_PORT=34082 -HARBOR_OMNICHAIN_WORKSPACE="./omnichain" - -# Bench -HARBOR_BENCH_PARALLEL=1 -HARBOR_BENCH_DEBUG=false -HARBOR_BENCH_MODEL="llama3.1:8b" -HARBOR_BENCH_API="http://ollama:11434" -HARBOR_BENCH_API_KEY="" -HARBOR_BENCH_VARIANTS="" -HARBOR_BENCH_JUDGE="mistral-nemo:12b-instruct-2407-q8_0" -HARBOR_BENCH_JUDGE_API="http://ollama:11434" -HARBOR_BENCH_JUDGE_API_KEY="" -HARBOR_BENCH_RESULTS="./bench/results" -HARBOR_BENCH_TASKS="./bench/defaultTasks.yml" - -# lm_eval -HARBOR_LMEVAL_TYPE="local-completions" -HARBOR_LMEVAL_RESULTS="./lmeval/results" -HARBOR_LMEVAL_CACHE="./lmeval/cache" -HARBOR_LMEVAL_EXTRA_ARGS="" -HARBOR_LMEVAL_MODEL_SPECIFIER="" -HARBOR_LMEVAL_MODEL_ARGS="" - -# SGLang -HARBOR_SGLANG_HOST_PORT=34091 -HARBOR_SGLANG_VERSION="latest" -HARBOR_SGLANG_MODEL="google/gemma-2-2b-it" -HARBOR_SGLANG_EXTRA_ARGS="" - -# ============================================ -# Service Configuration. -# You can specify any of the service's own environment variables here. -# ============================================ - -# Open WebUI -# See https://docs.openwebui.com/getting-started/env-configuration/ for reference. -# -------------------------------------------- -# WEBUI_NAME=WUI - -# Ollama Configuration. -# Run harbor ollama serve --help for a list of env vars -# -------------------------------------------- -# OLLAMA_DEBUG=1 -# OLLAMA_NUM_PARALLEL=4 - -# vLLM Configuration -# See https://docs.vllm.ai/en/latest/serving/env_vars.html for reference -# -------------------------------------------- -VLLM_NO_USAGE_STATS=true -VLLM_DO_NOT_TRACK=1 \ No newline at end of file diff --git a/harbor.sh b/harbor.sh index 138309d..20e32bf 100755 --- a/harbor.sh +++ b/harbor.sh @@ -251,7 +251,7 @@ compose_with_options() { local all_matched=true for part in "${filename_parts[@]}"; do - if [[ ! " ${options[*]} " =~ " ${part} " ]]; then + if [[ ! " ${options[*]} " =~ " ${part} " ]] && [[ ! " ${options[*]} " =~ " * " ]]; then all_matched=false break fi @@ -1647,6 +1647,7 @@ fix_fs_acl() { docker_fsacl $(eval echo "$(env_manager get opint.config.path)") docker_fsacl $(eval echo "$(env_manager get fabric.config.path)") docker_fsacl $(eval echo "$(env_manager get txtai.cache)") + docker_fsacl $(eval echo "$(env_manager get nexa.cache)") } open_home_code() { @@ -3336,12 +3337,43 @@ run_stt_command() { esac } +run_nexa_command() { + case "$1" in + model) + shift + env_manager_alias nexa.model "$@" + return 0 + ;; + -h | --help) + echo "Please note that this is not Nexa CLI, but a Harbor CLI to manage nexa service." + echo + echo "Usage: harbor [nexa] " + echo + echo "Commands:" + echo " harbor nexa model - Alias for 'harbor lmeval args get|set model'" + echo + echo "Original CLI help:" + ;; + esac + + local services=$(get_active_services) + + $(compose_with_options $services "nexa") run \ + --rm \ + --name $default_container_prefix.nexa-cli \ + --service-ports \ + -e "TERM=xterm-256color" \ + -v "$original_dir:$original_dir" \ + --workdir "$original_dir" \ + nexa "$@" +} + # ======================================================================== # == Main script # ======================================================================== # Globals -version="0.2.4" +version="0.2.5" harbor_repo_url="https://github.com/av/harbor.git" harbor_release_url="https://api.github.com/repos/av/harbor/releases/latest" delimiter="|" @@ -3617,6 +3649,10 @@ main_entrypoint() { shift run_boost_command "$@" ;; + nexa) + shift + run_nexa_command "$@" + ;; tunnel | t) shift establish_tunnel "$@" diff --git a/http-catalog/nexa.http b/http-catalog/nexa.http new file mode 100644 index 0000000..46b904e --- /dev/null +++ b/http-catalog/nexa.http @@ -0,0 +1,27 @@ +@host = http://localhost:34181 + + +### + +GET {{host}}/health + +### + +GET {{host}}/v1/models + +### + +POST {{host}}/v1/chat/completions +Content-Type: application/json +Authorization: sk-fake + +{ + "model": "anything", + "messages": [ + {"role": "user", "content": "How many heads Girrafes have?"} + ], + "options": { + "temperature": 0.2 + }, + "stream": false +} \ No newline at end of file diff --git a/nexa/Dockerfile b/nexa/Dockerfile new file mode 100644 index 0000000..4a559ec --- /dev/null +++ b/nexa/Dockerfile @@ -0,0 +1,15 @@ +ARG HARBOR_NEXA_IMAGE=ubuntu:22.04 + +FROM ${HARBOR_NEXA_IMAGE} +ARG HARBOR_NEXA_IMAGE=ubuntu:22.04 + +# This file will coerce nexa to install CUDA +# version when we're running with CUDA base image +COPY ./nvidia.sh /nvidia.sh +RUN chmod +x /nvidia.sh && /nvidia.sh + +# Install nexa +RUN apt-get update && apt-get install -y curl +RUN curl -fsSL https://public-storage.nexa4ai.com/install.sh | sh + +ENTRYPOINT [ "nexa" ] \ No newline at end of file diff --git a/nexa/nvidia.sh b/nexa/nvidia.sh new file mode 100644 index 0000000..45c137f --- /dev/null +++ b/nexa/nvidia.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This file is to trick Nexa that Nvidia CUDA +# is available during the docker install. + +# Nvidia Container Runtime mounts nvidia-smi and other +# nvidia utils at runtime - they are never available at +# build time even on official CUDA images. + +# Hence, below, aka "trust me bro" + +# Our base iamge +IMAGE=${HARBOR_NEXA_IMAGE} +if [[ $IMAGE == *"nvidia"* ]]; then + echo "Writing fake nvidia-smi file" + echo "echo 'CUDA Version: 12.4.0'" > /usr/bin/nvidia-smi + chmod +x /usr/bin/nvidia-smi + + # Let's test it + nvidia-smi +else + echo "Not an Nvidia image, skipping..." +fi \ No newline at end of file diff --git a/nexa/override.env b/nexa/override.env new file mode 100644 index 0000000..42c5617 --- /dev/null +++ b/nexa/override.env @@ -0,0 +1,2 @@ +# This file can contain additional environment +# variables that'll only be visible to nexa service. \ No newline at end of file diff --git a/nexa/proxy.Dockerfile b/nexa/proxy.Dockerfile new file mode 100644 index 0000000..01271e7 --- /dev/null +++ b/nexa/proxy.Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12 + +WORKDIR /app +RUN pip install fastapi uvicorn httpx +COPY ./proxy_server.py /app/proxy_server.py + +CMD ["uvicorn", "proxy_server:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] \ No newline at end of file diff --git a/nexa/proxy_server.py b/nexa/proxy_server.py new file mode 100644 index 0000000..6834f22 --- /dev/null +++ b/nexa/proxy_server.py @@ -0,0 +1,88 @@ +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import StreamingResponse, JSONResponse +import httpx +import asyncio +from datetime import datetime + +app = FastAPI() + +TARGET_URL = "http://nexa:8000" + +@app.api_route("/health", methods=["GET"]) +def health(): + return JSONResponse(content={"status": "ok"}) + + +@app.api_route("/v1/models", methods=["GET"]) +def get_models(): + return JSONResponse( + content={ + "object": "list", + "data": + [{ + "id": "nexa", + "created": int(datetime.now().timestamp()), + "object": "model", + "owned_by": "nexa" + }] + } + ) + + +@app.api_route( + "/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"] +) +async def proxy(request: Request, path: str): + client = httpx.AsyncClient() + + # Construct the target URL + url = f"{TARGET_URL}/{path}" + + # Forward the request headers + headers = dict(request.headers) + headers.pop("host", None) + + try: + # Stream the request body + async def request_stream(): + async for chunk in request.stream(): + yield chunk + + # Make the request to the target server + response = await client.request( + method=request.method, + url=url, + headers=headers, + params=request.query_params, + content=request_stream(), + timeout=httpx.Timeout(300.0) + ) + + # Stream the response back to the client + async def response_stream(): + try: + async for chunk in response.aiter_bytes(): + yield chunk + except (RequestError, ReadTimeout, ConnectTimeout) as e: + logger.error(f"Error during streaming: {str(e)}") + # Handle the error appropriately + + return StreamingResponse( + response_stream(), + status_code=response.status_code, + headers=dict(response.headers) + ) + + except httpx.RequestError as exc: + raise HTTPException( + status_code=500, + detail=f"Error communicating with target server: {str(exc)}" + ) + + finally: + await client.aclose() + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/open-webui/configs/config.nexa.json b/open-webui/configs/config.nexa.json new file mode 100644 index 0000000..d6ba08f --- /dev/null +++ b/open-webui/configs/config.nexa.json @@ -0,0 +1,11 @@ +{ + "openai": { + "api_base_urls": [ + "http://nexa-proxy:8000/v1" + ], + "api_keys": [ + "sk-nexa" + ], + "enabled": true + } +} \ No newline at end of file diff --git a/package.json b/package.json index ba1f898..d7a7bf1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@av/harbor", - "version": "0.2.4", + "version": "0.2.5", "bin": { "harbor": "./bin/harbor" } diff --git a/profiles/default.env b/profiles/default.env index a614b01..8e2818e 100644 --- a/profiles/default.env +++ b/profiles/default.env @@ -28,6 +28,7 @@ HARBOR_LLAMACPP_CACHE="~/.cache/llama.cpp" HARBOR_OLLAMA_CACHE="~/.ollama" HARBOR_VLLM_CACHE="~/.cache/vllm" HARBOR_TXTAI_CACHE="~/.cache/txtai" +HARBOR_NEXA_CACHE="~/.cache/nexa" # These could be used by specific services, # in which case they can be set in a centralised @@ -377,6 +378,10 @@ HARBOR_ANYTHINGLLM_IMAGE="mintplexlabs/anythingllm" HARBOR_ANYTHINGLLM_VERSION="latest" HARBOR_ANYTHINGLLM_JWT_SECRET="sk-anythingllm-jwt" +# Nexa AI SDK +HARBOR_NEXA_HOST_PORT=34181 +HARBOR_NEXA_MODEL="llama3.2" + # ============================================ # Service Configuration. # You can specify any of the service's own environment variables here.