From c87fca3ec1fbbd8c564860b6fc5162cca77db734 Mon Sep 17 00:00:00 2001 From: Loric <117862619+LoricOSC@users.noreply.github.com> Date: Thu, 11 Jul 2024 14:51:59 -0600 Subject: [PATCH 1/3] Git fetch specific branch instead of full tree during build (#2748) * Git fetch specific branch instead of full tree during build * Recursively create directores for all sources --------- Signed-off-by: Dave Signed-off-by: Dave Lee Co-authored-by: Shane Co-authored-by: Dave --- Makefile | 80 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index db7d7b5e85b3..cdc7b53d3226 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ BINARY_NAME=local-ai DETECT_LIBS?=true # llama.cpp versions -GOLLAMA_STABLE_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be +GOLLAMA_REPO?=https://github.com/go-skynet/go-llama.cpp +GOLLAMA_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be CPPLLAMA_VERSION?=dd07a123b79f9bd9e8a4ba0447427b3083e9347a # gpt4all version @@ -18,18 +19,23 @@ RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6 # whisper.cpp version +WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp WHISPER_CPP_VERSION?=d207c6882247984689091ae9d780d2e51eab1df7 # bert.cpp version +BERT_REPO?=https://github.com/go-skynet/go-bert.cpp BERT_VERSION?=710044b124545415f555e4260d16b146c725a6e4 # go-piper version +PIPER_REPO?=https://github.com/mudler/go-piper PIPER_VERSION?=9d0100873a7dbb0824dfea40e8cec70a1b110759 # stablediffusion version +STABLEDIFFUSION_REPO?=https://github.com/mudler/go-stable-diffusion STABLEDIFFUSION_VERSION?=4a3cd6aeae6f66ee57eae9a0075f8c58c3a6a38f # tinydream version +TINYDREAM_REPO?=https://github.com/M0Rf30/go-tiny-dream TINYDREAM_VERSION?=c04fa463ace9d9a6464313aa5f9cd0f953b6c057 export BUILD_TYPE?= @@ -202,64 +208,104 @@ all: help ## BERT embeddings sources/go-bert.cpp: - git clone --recurse-submodules https://github.com/go-skynet/go-bert.cpp sources/go-bert.cpp - cd sources/go-bert.cpp && git checkout -b build $(BERT_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-bert.cpp + cd sources/go-bert.cpp && \ + git init && \ + git remote add origin $(BERT_REPO) && \ + git fetch origin && \ + git checkout $(BERT_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-bert.cpp/libgobert.a: sources/go-bert.cpp $(MAKE) -C sources/go-bert.cpp libgobert.a ## go-llama.cpp sources/go-llama.cpp: - git clone --recurse-submodules https://github.com/go-skynet/go-llama.cpp sources/go-llama.cpp - cd sources/go-llama.cpp && git checkout -b build $(GOLLAMA_STABLE_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-llama.cpp + cd sources/go-llama.cpp && \ + git init && \ + git remote add origin $(GOLLAMA_REPO) && \ + git fetch origin && \ + git checkout $(GOLLAMA_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-llama.cpp/libbinding.a: sources/go-llama.cpp $(MAKE) -C sources/go-llama.cpp BUILD_TYPE=$(STABLE_BUILD_TYPE) libbinding.a ## go-piper sources/go-piper: - git clone --recurse-submodules https://github.com/mudler/go-piper sources/go-piper - cd sources/go-piper && git checkout -b build $(PIPER_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-piper + cd sources/go-piper && \ + git init && \ + git remote add origin $(PIPER_REPO) && \ + git fetch origin && \ + git checkout $(PIPER_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-piper/libpiper_binding.a: sources/go-piper $(MAKE) -C sources/go-piper libpiper_binding.a example/main piper.o ## GPT4ALL sources/gpt4all: - git clone --recurse-submodules $(GPT4ALL_REPO) sources/gpt4all - cd sources/gpt4all && git checkout -b build $(GPT4ALL_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/gpt4all + cd sources/gpt4all && \ + git init && \ + git remote add origin $(GPT4ALL_REPO) && \ + git fetch origin && \ + git checkout $(GPT4ALL_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/gpt4all/gpt4all-bindings/golang/libgpt4all.a: sources/gpt4all $(MAKE) -C sources/gpt4all/gpt4all-bindings/golang/ libgpt4all.a ## RWKV sources/go-rwkv.cpp: - git clone --recurse-submodules $(RWKV_REPO) sources/go-rwkv.cpp - cd sources/go-rwkv.cpp && git checkout -b build $(RWKV_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-rwkv.cpp + cd sources/go-rwkv.cpp && \ + git init && \ + git remote add origin $(RWKV_REPO) && \ + git fetch origin && \ + git checkout $(RWKV_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-rwkv.cpp/librwkv.a: sources/go-rwkv.cpp cd sources/go-rwkv.cpp && cd rwkv.cpp && cmake . -DRWKV_BUILD_SHARED_LIBRARY=OFF && cmake --build . && cp librwkv.a .. ## stable diffusion sources/go-stable-diffusion: - git clone --recurse-submodules https://github.com/mudler/go-stable-diffusion sources/go-stable-diffusion - cd sources/go-stable-diffusion && git checkout -b build $(STABLEDIFFUSION_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-stable-diffusion + cd sources/go-stable-diffusion && \ + git init && \ + git remote add origin $(STABLEDIFFUSION_REPO) && \ + git fetch origin && \ + git checkout $(STABLEDIFFUSION_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-stable-diffusion/libstablediffusion.a: sources/go-stable-diffusion CPATH="$(CPATH):/usr/include/opencv4" $(MAKE) -C sources/go-stable-diffusion libstablediffusion.a ## tiny-dream sources/go-tiny-dream: - git clone --recurse-submodules https://github.com/M0Rf30/go-tiny-dream sources/go-tiny-dream - cd sources/go-tiny-dream && git checkout -b build $(TINYDREAM_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/go-tiny-dream + cd sources/go-tiny-dream && \ + git init && \ + git remote add origin $(TINYDREAM_REPO) && \ + git fetch origin && \ + git checkout $(TINYDREAM_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/go-tiny-dream/libtinydream.a: sources/go-tiny-dream $(MAKE) -C sources/go-tiny-dream libtinydream.a ## whisper sources/whisper.cpp: - git clone https://github.com/ggerganov/whisper.cpp sources/whisper.cpp - cd sources/whisper.cpp && git checkout -b build $(WHISPER_CPP_VERSION) && git submodule update --init --recursive --depth 1 + mkdir -p sources/whisper.cpp + cd sources/whisper.cpp && \ + git init && \ + git remote add origin $(WHISPER_REPO) && \ + git fetch origin && \ + git checkout $(WHISPER_CPP_VERSION) && \ + git submodule update --init --recursive --depth 1 sources/whisper.cpp/libwhisper.a: sources/whisper.cpp cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a From dcbdc12cc96045c0107d83bc464ef37451f5a08a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 11 Jul 2024 22:59:02 +0200 Subject: [PATCH 2/3] Update bump_deps.yaml Signed-off-by: Ettore Di Giacinto --- .github/workflows/bump_deps.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 2abb2cabc5f7..7dbe22b2e71e 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -9,9 +9,6 @@ jobs: fail-fast: false matrix: include: - - repository: "go-skynet/go-llama.cpp" - variable: "GOLLAMA_VERSION" - branch: "master" - repository: "ggerganov/llama.cpp" variable: "CPPLLAMA_VERSION" branch: "master" From 664b2e352bf962410258d13fa976acd2d97338b2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 11 Jul 2024 22:59:52 +0200 Subject: [PATCH 3/3] fix(ci): small fixups to checksum_checker.sh (#2776) Signed-off-by: Ettore Di Giacinto --- .github/checksum_checker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/checksum_checker.sh b/.github/checksum_checker.sh index 174e6d3f41a2..e301b5ecb597 100644 --- a/.github/checksum_checker.sh +++ b/.github/checksum_checker.sh @@ -17,7 +17,7 @@ function check_and_update_checksum() { new_checksum=$(python3 ./check_and_update.py $uri) result=$? - if [[ result -eq 5]]; then + if [[ $result -eq 5 ]]; then echo "Contaminated entry detected, deleting entry for $model_name..." yq eval -i "del([$idx])" "$input_yaml" return