From 3bcad8bf7469be44797877f59bb4efe70ed2bfc2 Mon Sep 17 00:00:00 2001 From: xhedit <71542719+xhedit@users.noreply.github.com> Date: Wed, 13 Mar 2024 19:05:01 -0400 Subject: [PATCH] Update README.md --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86fccae..518da2e 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,30 @@ pip3 install llm-quantkit # Usage +``` +Usage: quantkit [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + download Download model from huggingface. + safetensor Download and/or convert a pytorch model to safetensor format. + awq Download and/or convert a model to AWQ format. + exl2 Download and/or convert a model to EXL2 format. + gptq Download and/or convert a model to GPTQ format. +``` + Download a model from HF and don't use HF cache: ``` -quantkit teknium/Hermes-Trismegistus-Mistral-7B --no-cache +quantkit download teknium/Hermes-Trismegistus-Mistral-7B --no-cache ```
Only download the safetensors version of a model (useful for models that have both safetensors and pytorch): ``` -quantkit mistralai/Mistral-7B-v0.1 --no-cache --safetensors-only -out mistral7b +quantkit download mistralai/Mistral-7B-v0.1 --no-cache --safetensors-only -out mistral7b ```
@@ -40,7 +54,7 @@ quantkit awq mistralai/Mistral-7B-v0.1 -out Mistral-7B-v0.1-AWQ Convert a model to GPTQ (4 bits / group-size 32): ``` -quantkit awq Mistral-7B-v0.1 -out Mistral-7B-v0.1-AWQ -b 4 --group-size 32 +quantkit gptq mistral7b -out Mistral-7B-v0.1-AWQ -b 4 --group-size 32 ```