use 🧨diffusers model

.github/workflows/test-invoke-conda.yml

@@ -4,8 +4,6 @@ on:
     branches:
       - 'main'
   pull_request:
-    branches:
-      - 'main'
     types:
       - 'ready_for_review'
       - 'opened'
@@ -29,7 +27,7 @@ jobs:
     strategy:
       matrix:
         stable-diffusion-model:
-          - 'stable-diffusion-1.5'
+          - diffusers-1.5
         environment-yaml:
           - environment-lin-amd.yml
           - environment-lin-cuda.yml
@@ -38,33 +36,27 @@ jobs:
         include:
           - environment-yaml: environment-lin-amd.yml
             os: ubuntu-22.04
-            curl-command: curl
             github-env: $GITHUB_ENV
             default-shell: bash -l {0}
           - environment-yaml: environment-lin-cuda.yml
             os: ubuntu-22.04
-            curl-command: curl
             github-env: $GITHUB_ENV
             default-shell: bash -l {0}
           - environment-yaml: environment-mac.yml
             os: macos-12
-            curl-command: curl
             github-env: $GITHUB_ENV
             default-shell: bash -l {0}
           - environment-yaml: environment-win-cuda.yml
             os: windows-2022
-            curl-command: curl.exe
             github-env: $env:GITHUB_ENV
             default-shell: pwsh
-          - stable-diffusion-model: stable-diffusion-1.5
-            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
-            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
     name: ${{ matrix.environment-yaml }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
       CONDA_ENV_NAME: invokeai
       INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+      PYTHONUNBUFFERED: 1
+      HAVE_SECRETS: ${{ secrets.HUGGINGFACE_TOKEN != '' }}
     defaults:
       run:
         shell: ${{ matrix.default-shell }}
@@ -81,6 +73,15 @@ jobs:
       - name: create environment.yml
         run: cp "environments-and-requirements/${{ matrix.environment-yaml }}" environment.yml
 
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: huggingface-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ env.cache-name }}
+
       - name: Use cached conda packages
         id: use-cached-conda-packages
         uses: actions/cache@v3
@@ -108,26 +109,14 @@ jobs:
         if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
         run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
 
-      - name: Use Cached Stable Diffusion Model
-        id: cache-sd-model
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-${{ matrix.stable-diffusion-model }}
-        with:
-          path: ${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}
-          key: ${{ env.cache-name }}
-
-      - name: Download ${{ matrix.stable-diffusion-model }}
-        id: download-stable-diffusion-model
-        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
-        run: |
-          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          ${{ matrix.curl-command }} -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" -L ${{ matrix.stable-diffusion-model-url }}
-
       - name: run configure_invokeai.py
         id: run-preload-models
-        run: |
-          python scripts/configure_invokeai.py --skip-sd-weights --yes
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+        run: >
+          python scripts/configure_invokeai.py
+          --skip-sd-weights --yes
+          --full-precision  # can't use fp16 weights without a GPU
 
       - name: cat invokeai.init
         id: cat-invokeai
@@ -136,14 +125,19 @@ jobs:
       - name: Run the tests
         id: run-tests
         if: matrix.os != 'windows-2022'
-        run: |
-          time python scripts/invoke.py \
-            --no-patchmatch \
-            --no-nsfw_checker \
-            --model ${{ matrix.stable-diffusion-model }} \
-            --from_file ${{ env.TEST_PROMPTS }} \
-            --root="${{ env.INVOKEAI_ROOT }}" \
-            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
+        env:
+          # Set offline mode to make sure configure preloaded successfully.
+          HF_HUB_OFFLINE: 1
+          HF_DATASETS_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
+        run: >
+          python scripts/invoke.py
+          --no-patchmatch
+          --no-nsfw_checker
+          --model ${{ matrix.stable-diffusion-model }}
+          --from_file ${{ env.TEST_PROMPTS }}
+          --root="${{ env.INVOKEAI_ROOT }}"
+          --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
 
       - name: export conda env
         id: export-conda-env

.github/workflows/test-invoke-pip.yml

@@ -4,8 +4,6 @@ on:
     branches:
       - 'main'
   pull_request:
-    branches:
-      - 'main'
     types:
       - 'ready_for_review'
       - 'opened'
@@ -28,7 +26,7 @@ jobs:
     strategy:
       matrix:
         stable-diffusion-model:
-          - stable-diffusion-1.5
+          - diffusers-1.5
         requirements-file:
           - requirements-lin-cuda.txt
           - requirements-lin-amd.txt
@@ -40,26 +38,22 @@ jobs:
         include:
           - requirements-file: requirements-lin-cuda.txt
             os: ubuntu-22.04
-            curl-command: curl
             github-env: $GITHUB_ENV
           - requirements-file: requirements-lin-amd.txt
             os: ubuntu-22.04
-            curl-command: curl
             github-env: $GITHUB_ENV
           - requirements-file: requirements-mac-mps-cpu.txt
             os: macOS-12
-            curl-command: curl
             github-env: $GITHUB_ENV
           - requirements-file: requirements-win-colab-cuda.txt
             os: windows-2022
-            curl-command: curl.exe
             github-env: $env:GITHUB_ENV
-          - stable-diffusion-model: stable-diffusion-1.5
-            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
-            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
     name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
+    env:
+      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+      PYTHONUNBUFFERED: 1
+      HAVE_SECRETS: ${{ secrets.HUGGINGFACE_TOKEN != '' }}
     steps:
       - name: Checkout sources
         id: checkout-sources
@@ -82,6 +76,15 @@ jobs:
           mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
           cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
 
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: huggingface-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ env.cache-name }}
+
       - name: set test prompt to main branch validation
         if: ${{ github.ref == 'refs/heads/main' }}
         run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> ${{ matrix.github-env }}
@@ -110,30 +113,31 @@ jobs:
       - name: install requirements
         run: pip3 install -r '${{ matrix.requirements-file }}'
 
-      - name: Use Cached Stable Diffusion Model
-        id: cache-sd-model
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-${{ matrix.stable-diffusion-model }}
-        with:
-          path: ${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}
-          key: ${{ env.cache-name }}
-
-      - name: Download ${{ matrix.stable-diffusion-model }}
-        id: download-stable-diffusion-model
-        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
-        run: |
-          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          ${{ matrix.curl-command }} -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" -L ${{ matrix.stable-diffusion-model-url }}
-
       - name: run configure_invokeai.py
         id: run-preload-models
-        run: python3 scripts/configure_invokeai.py --skip-sd-weights --yes
+        env:
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+        run: >
+          python3 scripts/configure_invokeai.py
+          --skip-sd-weights --yes
+          --full-precision  # can't use fp16 weights without a GPU
 
       - name: Run the tests
         id: run-tests
         if: matrix.os != 'windows-2022'
-        run: python3 scripts/invoke.py --no-patchmatch --no-nsfw_checker --model ${{ matrix.stable-diffusion-model }} --from_file ${{ env.TEST_PROMPTS }} --root="${{ env.INVOKEAI_ROOT }}" --outdir="${{ env.INVOKEAI_OUTDIR }}"
+        env:
+          # Set offline mode to make sure configure preloaded successfully.
+          HF_HUB_OFFLINE: 1
+          HF_DATASETS_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
+        run: >
+          python3 scripts/invoke.py
+          --no-patchmatch
+          --no-nsfw_checker
+          --model ${{ matrix.stable-diffusion-model }}
+          --from_file ${{ env.TEST_PROMPTS }}
+          --root="${{ env.INVOKEAI_ROOT }}"
+          --outdir="${{ env.INVOKEAI_OUTDIR }}"
 
       - name: Archive results
         id: archive-results

backend/invoke_ai_web_server.py

@@ -1,35 +1,34 @@
-import eventlet
+import base64
 import glob
+import io
+import json
+import math
+import mimetypes
 import os
 import shutil
-import mimetypes
 import traceback
-import math
-import io
-import base64
-import os
-import json
+from threading import Event
+from uuid import uuid4
 
-from werkzeug.utils import secure_filename
+import eventlet
+from PIL import Image
+from PIL.Image import Image as ImageType
 from flask import Flask, redirect, send_from_directory, request, make_response
 from flask_socketio import SocketIO
-from PIL import Image, ImageOps
-from PIL.Image import Image as ImageType
-from uuid import uuid4
-from threading import Event
+from werkzeug.utils import secure_filename
 
+from backend.modules.get_canvas_generation_mode import (
+    get_canvas_generation_mode,
+)
+from backend.modules.parameters import parameters_to_command
 from ldm.generate import Generate
 from ldm.invoke.args import Args, APP_ID, APP_VERSION, calculate_init_img_hash
 from ldm.invoke.conditioning import get_tokens_for_prompt, get_prompt_structure
+from ldm.invoke.generator.diffusers_pipeline import PipelineIntermediateState
+from ldm.invoke.generator.inpaint import infill_methods
 from ldm.invoke.globals import Globals
 from ldm.invoke.pngwriter import PngWriter, retrieve_metadata
 from ldm.invoke.prompt_parser import split_weighted_subprompts, Blend
-from ldm.invoke.generator.inpaint import infill_methods
-
-from backend.modules.parameters import parameters_to_command
-from backend.modules.get_canvas_generation_mode import (
-    get_canvas_generation_mode,
-)
 
 # Loading Arguments
 opt = Args()
@@ -304,7 +303,7 @@ def load_socketio_listeners(self, socketio):
         def handle_request_capabilities():
             print(f">> System config requested")
             config = self.get_system_config()
-            config["model_list"] = self.generate.model_cache.list_models()
+            config["model_list"] = self.generate.model_manager.list_models()
             config["infill_methods"] = infill_methods()
             socketio.emit("systemConfig", config)
 
@@ -317,11 +316,11 @@ def handle_search_models(search_folder: str):
                     {'search_folder': None, 'found_models': None},
                 )
                 else:
-                    search_folder, found_models = self.generate.model_cache.search_models(search_folder)
+                    search_folder, found_models = self.generate.model_manager.search_models(search_folder)
                     socketio.emit(
                         "foundModels",
                         {'search_folder': search_folder, 'found_models': found_models},
-                    )            
+                    )
             except Exception as e:
                 self.socketio.emit("error", {"message": (str(e))})
                 print("\n")
@@ -336,17 +335,17 @@ def handle_add_model(new_model_config: dict):
                 del new_model_config['name']
                 model_attributes = new_model_config
                 update = False
-                current_model_list = self.generate.model_cache.list_models()
+                current_model_list = self.generate.model_manager.list_models()
                 if model_name in current_model_list:
                     update = True
 
                 print(f">> Adding New Model: {model_name}")
 
-                self.generate.model_cache.add_model(
+                self.generate.model_manager.add_model(
                     model_name=model_name, model_attributes=model_attributes, clobber=True)
-                self.generate.model_cache.commit(opt.conf)
+                self.generate.model_manager.commit(opt.conf)
 
-                new_model_list = self.generate.model_cache.list_models()
+                new_model_list = self.generate.model_manager.list_models()
                 socketio.emit(
                     "newModelAdded",
                     {"new_model_name": model_name,
@@ -364,9 +363,9 @@ def handle_add_model(new_model_config: dict):
         def handle_delete_model(model_name: str):
             try:
                 print(f">> Deleting Model: {model_name}")
-                self.generate.model_cache.del_model(model_name)
-                self.generate.model_cache.commit(opt.conf)
-                updated_model_list = self.generate.model_cache.list_models()
+                self.generate.model_manager.del_model(model_name)
+                self.generate.model_manager.commit(opt.conf)
+                updated_model_list = self.generate.model_manager.list_models()
                 socketio.emit(
                     "modelDeleted",
                     {"deleted_model_name": model_name,
@@ -385,7 +384,7 @@ def handle_set_model(model_name: str):
             try:
                 print(f">> Model change requested: {model_name}")
                 model = self.generate.set_model(model_name)
-                model_list = self.generate.model_cache.list_models()
+                model_list = self.generate.model_manager.list_models()
                 if model is None:
                     socketio.emit(
                         "modelChangeFailed",
@@ -797,7 +796,7 @@ def handle_delete_image(url, thumbnail, uuid, category):
 
     # App Functions
     def get_system_config(self):
-        model_list: dict = self.generate.model_cache.list_models()
+        model_list: dict = self.generate.model_manager.list_models()
         active_model_name = None
 
         for model_name, model_dict in model_list.items():
@@ -1205,9 +1204,16 @@ def image_done(image, seed, first_seed, attention_maps_image=None):
 
             print(generation_parameters)
 
+            def diffusers_step_callback_adapter(*cb_args, **kwargs):
+                if isinstance(cb_args[0], PipelineIntermediateState):
+                    progress_state: PipelineIntermediateState = cb_args[0]
+                    return image_progress(progress_state.latents, progress_state.step)
+                else:
+                    return image_progress(*cb_args, **kwargs)
+
             self.generate.prompt2image(
                 **generation_parameters,
-                step_callback=image_progress,
+                step_callback=diffusers_step_callback_adapter,
                 image_callback=image_done
             )