From 4e658bb63a4456a00fd00f73c0c694cbee32ef03 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 30 May 2024 16:14:28 +0200 Subject: [PATCH 01/12] feat: optimize performance lora filtering in metadata (#3048) * feat: add remove_performance_lora method * feat: use class PerformanceLoRA instead of strings in config * refactor: cleanup flags, use __member__ to check if enums contains key * feat: only filter lora of selected performance instead of all performance LoRAs * fix: disable intermediate results for all restricted performances too fast for Gradio, which becomes a bottleneck * refactor: rename parse_json to to_json, rename parse_string to to_string * feat: use speed steps as default instead of hardcoded 30 * feat: add method to_steps to Performance * refactor: remove method ordinal_suffix, not needed anymore * feat: only filter lora of selected performance instead of all performance LoRAs both metadata and history log * feat: do not filter LoRAs in metadata parser but rather in metadata load action --- modules/async_worker.py | 4 ++- modules/config.py | 32 ++++++--------------- modules/flags.py | 23 ++++++++++++++-- modules/meta_parser.py | 43 +++++++++++++++++++---------- modules/private_logger.py | 2 +- modules/util.py | 29 ++++++++++++++++---- tests/test_utils.py | 58 ++++++++++++++++++++++++++++++++++++++- webui.py | 6 ++-- 8 files changed, 144 insertions(+), 53 deletions(-) diff --git a/modules/async_worker.py b/modules/async_worker.py index d7d9b9fd7..9c16d6fcb 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -462,8 +462,10 @@ def handler(async_task): progressbar(async_task, 2, 'Loading models ...') - loras, prompt = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number) + lora_filenames = modules.util.remove_performance_lora(modules.config.lora_filenames, performance_selection) + loras, prompt = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number, lora_filenames=lora_filenames) loras += performance_loras + pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name, loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) diff --git a/modules/config.py b/modules/config.py index cb651c5b6..29a16d6dc 100644 --- a/modules/config.py +++ b/modules/config.py @@ -548,25 +548,9 @@ def add_ratio(x): model_filenames = [] lora_filenames = [] -lora_filenames_no_special = [] vae_filenames = [] wildcard_filenames = [] -sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors' -sdxl_lightning_lora = 'sdxl_lightning_4step_lora.safetensors' -sdxl_hyper_sd_lora = 'sdxl_hyper_sd_4step_lora.safetensors' -loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora, sdxl_hyper_sd_lora] - - -def remove_special_loras(lora_filenames): - global loras_metadata_remove - - loras_no_special = lora_filenames.copy() - for lora_to_remove in loras_metadata_remove: - if lora_to_remove in loras_no_special: - loras_no_special.remove(lora_to_remove) - return loras_no_special - def get_model_filenames(folder_paths, extensions=None, name_filter=None): if extensions is None: @@ -582,10 +566,9 @@ def get_model_filenames(folder_paths, extensions=None, name_filter=None): def update_files(): - global model_filenames, lora_filenames, lora_filenames_no_special, vae_filenames, wildcard_filenames, available_presets + global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets model_filenames = get_model_filenames(paths_checkpoints) lora_filenames = get_model_filenames(paths_loras) - lora_filenames_no_special = remove_special_loras(lora_filenames) vae_filenames = get_model_filenames(path_vae) wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt']) available_presets = get_presets() @@ -634,26 +617,27 @@ def downloading_sdxl_lcm_lora(): load_file_from_url( url='https://huggingface.co/lllyasviel/misc/resolve/main/sdxl_lcm_lora.safetensors', model_dir=paths_loras[0], - file_name=sdxl_lcm_lora + file_name=modules.flags.PerformanceLoRA.EXTREME_SPEED.value ) - return sdxl_lcm_lora + return modules.flags.PerformanceLoRA.EXTREME_SPEED.value + def downloading_sdxl_lightning_lora(): load_file_from_url( url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_lightning_4step_lora.safetensors', model_dir=paths_loras[0], - file_name=sdxl_lightning_lora + file_name=modules.flags.PerformanceLoRA.LIGHTNING.value ) - return sdxl_lightning_lora + return modules.flags.PerformanceLoRA.LIGHTNING.value def downloading_sdxl_hyper_sd_lora(): load_file_from_url( url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_hyper_sd_4step_lora.safetensors', model_dir=paths_loras[0], - file_name=sdxl_hyper_sd_lora + file_name=modules.flags.PerformanceLoRA.HYPER_SD.value ) - return sdxl_hyper_sd_lora + return modules.flags.PerformanceLoRA.HYPER_SD.value def downloading_controlnet_canny(): diff --git a/modules/flags.py b/modules/flags.py index e48052e18..25b0caaec 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -48,7 +48,8 @@ KSAMPLER_NAMES = list(KSAMPLER.keys()) -SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps", "tcd"] +SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", + "align_your_steps", "tcd"] SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys()) sampler_list = SAMPLER_NAMES @@ -91,6 +92,7 @@ '1664*576', '1728*576' ] + class MetadataScheme(Enum): FOOOCUS = 'fooocus' A1111 = 'a1111' @@ -115,6 +117,14 @@ def list(cls) -> list: return list(map(lambda c: c.value, cls)) +class PerformanceLoRA(Enum): + QUALITY = None + SPEED = None + EXTREME_SPEED = 'sdxl_lcm_lora.safetensors' + LIGHTNING = 'sdxl_lightning_4step_lora.safetensors' + HYPER_SD = 'sdxl_hyper_sd_4step_lora.safetensors' + + class Steps(IntEnum): QUALITY = 60 SPEED = 30 @@ -142,6 +152,10 @@ class Performance(Enum): def list(cls) -> list: return list(map(lambda c: c.value, cls)) + @classmethod + def by_steps(cls, steps: int | str): + return cls[Steps(int(steps)).name] + @classmethod def has_restricted_features(cls, x) -> bool: if isinstance(x, Performance): @@ -149,7 +163,10 @@ def has_restricted_features(cls, x) -> bool: return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value, cls.HYPER_SD.value] def steps(self) -> int | None: - return Steps[self.name].value if Steps[self.name] else None + return Steps[self.name].value if self.name in Steps.__members__ else None def steps_uov(self) -> int | None: - return StepsUOV[self.name].value if Steps[self.name] else None + return StepsUOV[self.name].value if self.name in StepsUOV.__members__ else None + + def lora_filename(self) -> str | None: + return PerformanceLoRA[self.name].value if self.name in PerformanceLoRA.__members__ else None diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 586e62da2..0d509a193 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -32,7 +32,7 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_str('prompt', 'Prompt', loaded_parameter_dict, results) get_str('negative_prompt', 'Negative Prompt', loaded_parameter_dict, results) get_list('styles', 'Styles', loaded_parameter_dict, results) - get_str('performance', 'Performance', loaded_parameter_dict, results) + performance = get_str('performance', 'Performance', loaded_parameter_dict, results) get_steps('steps', 'Steps', loaded_parameter_dict, results) get_number('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) get_resolution('resolution', 'Resolution', loaded_parameter_dict, results) @@ -59,19 +59,27 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_freeu('freeu', 'FreeU', loaded_parameter_dict, results) + # prevent performance LoRAs to be added twice, by performance and by lora + performance_filename = None + if performance is not None and performance in Performance.list(): + performance = Performance(performance) + performance_filename = performance.lora_filename() + for i in range(modules.config.default_max_lora_number): - get_lora(f'lora_combined_{i + 1}', f'LoRA {i + 1}', loaded_parameter_dict, results) + get_lora(f'lora_combined_{i + 1}', f'LoRA {i + 1}', loaded_parameter_dict, results, performance_filename) return results -def get_str(key: str, fallback: str | None, source_dict: dict, results: list, default=None): +def get_str(key: str, fallback: str | None, source_dict: dict, results: list, default=None) -> str | None: try: h = source_dict.get(key, source_dict.get(fallback, default)) assert isinstance(h, str) results.append(h) + return h except: results.append(gr.update()) + return None def get_list(key: str, fallback: str | None, source_dict: dict, results: list, default=None): @@ -181,7 +189,7 @@ def get_freeu(key: str, fallback: str | None, source_dict: dict, results: list, results.append(gr.update()) -def get_lora(key: str, fallback: str | None, source_dict: dict, results: list): +def get_lora(key: str, fallback: str | None, source_dict: dict, results: list, performance_filename: str | None): try: split_data = source_dict.get(key, source_dict.get(fallback)).split(' : ') enabled = True @@ -193,6 +201,9 @@ def get_lora(key: str, fallback: str | None, source_dict: dict, results: list): name = split_data[1] weight = split_data[2] + if name == performance_filename: + raise Exception + weight = float(weight) results.append(enabled) results.append(name) @@ -248,7 +259,7 @@ def __init__(self): self.full_prompt: str = '' self.raw_negative_prompt: str = '' self.full_negative_prompt: str = '' - self.steps: int = 30 + self.steps: int = Steps.SPEED.value self.base_model_name: str = '' self.base_model_hash: str = '' self.refiner_model_name: str = '' @@ -261,11 +272,11 @@ def get_scheme(self) -> MetadataScheme: raise NotImplementedError @abstractmethod - def parse_json(self, metadata: dict | str) -> dict: + def to_json(self, metadata: dict | str) -> dict: raise NotImplementedError @abstractmethod - def parse_string(self, metadata: dict) -> str: + def to_string(self, metadata: dict) -> str: raise NotImplementedError def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_prompt, steps, base_model_name, @@ -328,7 +339,7 @@ def get_scheme(self) -> MetadataScheme: 'version': 'Version' } - def parse_json(self, metadata: str) -> dict: + def to_json(self, metadata: str) -> dict: metadata_prompt = '' metadata_negative_prompt = '' @@ -382,9 +393,9 @@ def parse_json(self, metadata: str) -> dict: data['styles'] = str(found_styles) # try to load performance based on steps, fallback for direct A1111 imports - if 'steps' in data and 'performance' not in data: + if 'steps' in data and 'performance' in data is None: try: - data['performance'] = Performance[Steps(int(data['steps'])).name].value + data['performance'] = Performance.by_steps(data['steps']).value except ValueError | KeyError: pass @@ -414,7 +425,7 @@ def parse_json(self, metadata: str) -> dict: lora_split = lora.split(': ') lora_name = lora_split[0] lora_weight = lora_split[2] if len(lora_split) == 3 else lora_split[1] - for filename in modules.config.lora_filenames_no_special: + for filename in modules.config.lora_filenames: path = Path(filename) if lora_name == path.stem: data[f'lora_combined_{li + 1}'] = f'{filename} : {lora_weight}' @@ -422,7 +433,7 @@ def parse_json(self, metadata: str) -> dict: return data - def parse_string(self, metadata: dict) -> str: + def to_string(self, metadata: dict) -> str: data = {k: v for _, k, v in metadata} width, height = eval(data['resolution']) @@ -502,14 +513,14 @@ class FooocusMetadataParser(MetadataParser): def get_scheme(self) -> MetadataScheme: return MetadataScheme.FOOOCUS - def parse_json(self, metadata: dict) -> dict: + def to_json(self, metadata: dict) -> dict: for key, value in metadata.items(): if value in ['', 'None']: continue if key in ['base_model', 'refiner_model']: metadata[key] = self.replace_value_with_filename(key, value, modules.config.model_filenames) elif key.startswith('lora_combined_'): - metadata[key] = self.replace_value_with_filename(key, value, modules.config.lora_filenames_no_special) + metadata[key] = self.replace_value_with_filename(key, value, modules.config.lora_filenames) elif key == 'vae': metadata[key] = self.replace_value_with_filename(key, value, modules.config.vae_filenames) else: @@ -517,7 +528,7 @@ def parse_json(self, metadata: dict) -> dict: return metadata - def parse_string(self, metadata: list) -> str: + def to_string(self, metadata: list) -> str: for li, (label, key, value) in enumerate(metadata): # remove model folder paths from metadata if key.startswith('lora_combined_'): @@ -557,6 +568,8 @@ def replace_value_with_filename(key, value, filenames): elif value == path.stem: return filename + return None + def get_metadata_parser(metadata_scheme: MetadataScheme) -> MetadataParser: match metadata_scheme: diff --git a/modules/private_logger.py b/modules/private_logger.py index eb8f0cc5a..6fdb680c8 100644 --- a/modules/private_logger.py +++ b/modules/private_logger.py @@ -27,7 +27,7 @@ def log(img, metadata, metadata_parser: MetadataParser | None = None, output_for date_string, local_temp_filename, only_name = generate_temp_filename(folder=path_outputs, extension=output_format) os.makedirs(os.path.dirname(local_temp_filename), exist_ok=True) - parsed_parameters = metadata_parser.parse_string(metadata.copy()) if metadata_parser is not None else '' + parsed_parameters = metadata_parser.to_string(metadata.copy()) if metadata_parser is not None else '' image = Image.fromarray(img) if output_format == OutputFormat.PNG.value: diff --git a/modules/util.py b/modules/util.py index 8317dd504..5003f79ab 100644 --- a/modules/util.py +++ b/modules/util.py @@ -16,6 +16,7 @@ import modules.config import modules.sdxl_styles +from modules.flags import Performance LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS) @@ -381,9 +382,6 @@ def get_file_from_folder_list(name, folders): return os.path.abspath(os.path.realpath(os.path.join(folders[0], name))) -def ordinal_suffix(number: int) -> str: - return 'th' if 10 <= number % 100 <= 20 else {1: 'st', 2: 'nd', 3: 'rd'}.get(number % 10, 'th') - def makedirs_with_log(path): try: @@ -397,10 +395,15 @@ def get_enabled_loras(loras: list, remove_none=True) -> list: def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5, - skip_file_check=False, prompt_cleanup=True, deduplicate_loras=True) -> tuple[List[Tuple[AnyStr, float]], str]: + skip_file_check=False, prompt_cleanup=True, deduplicate_loras=True, + lora_filenames=None) -> tuple[List[Tuple[AnyStr, float]], str]: + if lora_filenames is None: + lora_filenames = [] + found_loras = [] prompt_without_loras = '' cleaned_prompt = '' + for token in prompt.split(','): matches = LORAS_PROMPT_PATTERN.findall(token) @@ -410,7 +413,7 @@ def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, flo for match in matches: lora_name = match[1] + '.safetensors' if not skip_file_check: - lora_name = get_filname_by_stem(match[1], modules.config.lora_filenames_no_special) + lora_name = get_filname_by_stem(match[1], lora_filenames) if lora_name is not None: found_loras.append((lora_name, float(match[2]))) token = token.replace(match[0], '') @@ -440,6 +443,22 @@ def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, flo return updated_loras[:loras_limit], cleaned_prompt +def remove_performance_lora(filenames: list, performance: Performance | None): + loras_without_performance = filenames.copy() + + if performance is None: + return loras_without_performance + + performance_lora = performance.lora_filename() + + for filename in filenames: + path = Path(filename) + if performance_lora == path.name: + loras_without_performance.remove(filename) + + return loras_without_performance + + def cleanup_prompt(prompt): prompt = re.sub(' +', ' ', prompt) prompt = re.sub(',+', ',', prompt) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6fd550db3..c1f49c13b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,7 @@ +import os import unittest +import modules.flags from modules import util @@ -77,5 +79,59 @@ def test_can_parse_tokens_with_lora(self): for test in test_cases: prompt, loras, loras_limit, skip_file_check = test["input"] expected = test["output"] - actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit=loras_limit, skip_file_check=skip_file_check) + actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit=loras_limit, + skip_file_check=skip_file_check) + self.assertEqual(expected, actual) + + def test_can_parse_tokens_and_strip_performance_lora(self): + lora_filenames = [ + 'hey-lora.safetensors', + modules.flags.PerformanceLoRA.EXTREME_SPEED.value, + modules.flags.PerformanceLoRA.LIGHTNING.value, + os.path.join('subfolder', modules.flags.PerformanceLoRA.HYPER_SD.value) + ] + + test_cases = [ + { + "input": ("some prompt, ", [], 5, True, modules.flags.Performance.QUALITY), + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt' + ), + }, + { + "input": ("some prompt, ", [], 5, True, modules.flags.Performance.SPEED), + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt' + ), + }, + { + "input": ("some prompt, , ", [], 5, True, modules.flags.Performance.EXTREME_SPEED), + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt' + ), + }, + { + "input": ("some prompt, , ", [], 5, True, modules.flags.Performance.LIGHTNING), + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt' + ), + }, + { + "input": ("some prompt, , ", [], 5, True, modules.flags.Performance.HYPER_SD), + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt' + ), + } + ] + + for test in test_cases: + prompt, loras, loras_limit, skip_file_check, performance = test["input"] + lora_filenames = modules.util.remove_performance_lora(lora_filenames, performance) + expected = test["output"] + actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit=loras_limit, lora_filenames=lora_filenames) self.assertEqual(expected, actual) diff --git a/webui.py b/webui.py index 0dd863506..6f08757d4 100644 --- a/webui.py +++ b/webui.py @@ -461,8 +461,8 @@ def update_history_link(): interactive=not modules.config.default_black_out_nsfw, info='Disable preview during generation.') disable_intermediate_results = gr.Checkbox(label='Disable Intermediate Results', - value=modules.config.default_performance == flags.Performance.EXTREME_SPEED.value, - interactive=modules.config.default_performance != flags.Performance.EXTREME_SPEED.value, + value=flags.Performance.has_restricted_features(modules.config.default_performance), + interactive=not flags.Performance.has_restricted_features(modules.config.default_performance), info='Disable intermediate results during generation, only show final gallery.') disable_seed_increment = gr.Checkbox(label='Disable seed increment', info='Disable automatic seed increment when image number is > 1.', @@ -713,7 +713,7 @@ def trigger_metadata_import(filepath, state_is_generating): parsed_parameters = {} else: metadata_parser = modules.meta_parser.get_metadata_parser(metadata_scheme) - parsed_parameters = metadata_parser.parse_json(parameters) + parsed_parameters = metadata_parser.to_json(parameters) return modules.meta_parser.load_parameter_button_click(parsed_parameters, state_is_generating) From 64c29a8c4340bb11c1104c8d1088d9a91bbd0da9 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 30 May 2024 16:17:36 +0200 Subject: [PATCH 02/12] feat: rework intermediate image display for restricted performances (#3050) disable intermediate results for all performacnes with restricted features make disable_intermediate_results interactive again even if performance has restricted features users who want to disable this option should be able to do so, even if performance will be impacted --- webui.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/webui.py b/webui.py index 6f08757d4..fe33560b0 100644 --- a/webui.py +++ b/webui.py @@ -460,9 +460,8 @@ def update_history_link(): disable_preview = gr.Checkbox(label='Disable Preview', value=modules.config.default_black_out_nsfw, interactive=not modules.config.default_black_out_nsfw, info='Disable preview during generation.') - disable_intermediate_results = gr.Checkbox(label='Disable Intermediate Results', + disable_intermediate_results = gr.Checkbox(label='Disable Intermediate Results', value=flags.Performance.has_restricted_features(modules.config.default_performance), - interactive=not flags.Performance.has_restricted_features(modules.config.default_performance), info='Disable intermediate results during generation, only show final gallery.') disable_seed_increment = gr.Checkbox(label='Disable seed increment', info='Disable automatic seed increment when image number is > 1.', @@ -616,7 +615,7 @@ def preset_selection_change(preset, is_generating): performance_selection.change(lambda x: [gr.update(interactive=not flags.Performance.has_restricted_features(x))] * 11 + [gr.update(visible=not flags.Performance.has_restricted_features(x))] * 1 + - [gr.update(interactive=not flags.Performance.has_restricted_features(x), value=flags.Performance.has_restricted_features(x))] * 1, + [gr.update(value=flags.Performance.has_restricted_features(x))] * 1, inputs=performance_selection, outputs=[ guidance_scale, sharpness, adm_scaler_end, adm_scaler_positive, From 7899261755c0a350987b3e983b56fe609e33684f Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Fri, 31 May 2024 22:24:19 +0200 Subject: [PATCH 03/12] fix: turbo scheduler loading issue (#3065) * fix: correctly load ModelPatcher * feat: do not load model at all, not needed --- ldm_patched/contrib/external_custom_sampler.py | 3 +-- modules/sample_hijack.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ldm_patched/contrib/external_custom_sampler.py b/ldm_patched/contrib/external_custom_sampler.py index 985b03a0a..60d5e3bd2 100644 --- a/ldm_patched/contrib/external_custom_sampler.py +++ b/ldm_patched/contrib/external_custom_sampler.py @@ -107,8 +107,7 @@ def INPUT_TYPES(s): def get_sigmas(self, model, steps, denoise): start_step = 10 - int(10 * denoise) timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps] - ldm_patched.modules.model_management.load_models_gpu([model]) - sigmas = model.model.model_sampling.sigma(timesteps) + sigmas = model.model_sampling.sigma(timesteps) sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) return (sigmas, ) diff --git a/modules/sample_hijack.py b/modules/sample_hijack.py index 4ab3cbbde..84752ede7 100644 --- a/modules/sample_hijack.py +++ b/modules/sample_hijack.py @@ -175,7 +175,7 @@ def calculate_sigmas_scheduler_hacked(model, scheduler_name, steps): elif scheduler_name == "sgm_uniform": sigmas = normal_scheduler(model, steps, sgm=True) elif scheduler_name == "turbo": - sigmas = SDTurboScheduler().get_sigmas(namedtuple('Patcher', ['model'])(model=model), steps=steps, denoise=1.0)[0] + sigmas = SDTurboScheduler().get_sigmas(model=model, steps=steps, denoise=1.0)[0] elif scheduler_name == "align_your_steps": model_type = 'SDXL' if isinstance(model.latent_format, ldm_patched.modules.latent_formats.SDXL) else 'SD1' sigmas = AlignYourStepsScheduler().get_sigmas(model_type=model_type, steps=steps, denoise=1.0)[0] From 07c6c89edf34676da86310dc1749db8ce3d082af Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Fri, 31 May 2024 22:41:36 +0200 Subject: [PATCH 04/12] fix: chown files directly at copy (#3066) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1172c795a..820ae94a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN chown -R user:user /content WORKDIR /content USER user -COPY . /content/app +COPY --chown=user:user . /content/app RUN mv /content/app/models /content/app/models.org CMD [ "sh", "-c", "/content/entrypoint.sh ${CMDARGS}" ] From 3d43976e8e057caf0df8ec9e7846935f80f25859 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 2 Jun 2024 02:13:16 +0200 Subject: [PATCH 05/12] feat: update cmd args (#3075) --- readme.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/readme.md b/readme.md index 0ec06f198..e79b72aa2 100644 --- a/readme.md +++ b/readme.md @@ -370,25 +370,36 @@ entry_with_update.py [-h] [--listen [IP]] [--port PORT] [--web-upload-size WEB_UPLOAD_SIZE] [--hf-mirror HF_MIRROR] [--external-working-path PATH [PATH ...]] - [--output-path OUTPUT_PATH] [--temp-path TEMP_PATH] + [--output-path OUTPUT_PATH] + [--temp-path TEMP_PATH] [--cache-path CACHE_PATH] [--in-browser] - [--disable-in-browser] [--gpu-device-id DEVICE_ID] + [--disable-in-browser] + [--gpu-device-id DEVICE_ID] [--async-cuda-allocation | --disable-async-cuda-allocation] - [--disable-attention-upcast] [--all-in-fp32 | --all-in-fp16] + [--disable-attention-upcast] + [--all-in-fp32 | --all-in-fp16] [--unet-in-bf16 | --unet-in-fp16 | --unet-in-fp8-e4m3fn | --unet-in-fp8-e5m2] - [--vae-in-fp16 | --vae-in-fp32 | --vae-in-bf16] + [--vae-in-fp16 | --vae-in-fp32 | --vae-in-bf16] + [--vae-in-cpu] [--clip-in-fp8-e4m3fn | --clip-in-fp8-e5m2 | --clip-in-fp16 | --clip-in-fp32] - [--directml [DIRECTML_DEVICE]] [--disable-ipex-hijack] + [--directml [DIRECTML_DEVICE]] + [--disable-ipex-hijack] [--preview-option [none,auto,fast,taesd]] [--attention-split | --attention-quad | --attention-pytorch] [--disable-xformers] - [--always-gpu | --always-high-vram | --always-normal-vram | - --always-low-vram | --always-no-vram | --always-cpu [CPU_NUM_THREADS]] - [--always-offload-from-vram] [--disable-server-log] - [--debug-mode] [--is-windows-embedded-python] - [--disable-server-info] [--share] [--preset PRESET] - [--language LANGUAGE] [--disable-offload-from-vram] - [--theme THEME] [--disable-image-log] + [--always-gpu | --always-high-vram | --always-normal-vram | + --always-low-vram | --always-no-vram | --always-cpu [CPU_NUM_THREADS]] + [--always-offload-from-vram] + [--pytorch-deterministic] [--disable-server-log] + [--debug-mode] [--is-windows-embedded-python] + [--disable-server-info] [--multi-user] [--share] + [--preset PRESET] [--disable-preset-selection] + [--language LANGUAGE] + [--disable-offload-from-vram] [--theme THEME] + [--disable-image-log] [--disable-analytics] + [--disable-metadata] [--disable-preset-download] + [--enable-describe-uov-image] + [--always-download-new-model] ``` ## Advanced Features From ab01104d42d8f68abb9f88e79ecc600acdb02f3b Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 2 Jun 2024 13:40:42 +0200 Subject: [PATCH 06/12] feat: make textboxes (incl. positive prompt) resizable (#3074) * feat: make textboxes (incl. positive prompt) resizable again * wip: auto-resize positive prompt on new line dirty approach as container is hidden and 1px padding is applied for border shadow to actually work * feat: set row height to 84, exactly matching 3 lines for positive prompt eliminate need for JS to resize positive prompt onUiLoaded --- css/style.css | 22 +++++++++++++--------- webui.py | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/css/style.css b/css/style.css index 18bacaaf6..6ed0f6285 100644 --- a/css/style.css +++ b/css/style.css @@ -74,31 +74,35 @@ progress::after { text-align: right; width: 215px; } +div:has(> #positive_prompt) { + border: none; +} -.type_row{ - height: 80px !important; +#positive_prompt { + padding: 1px; + background: var(--background-fill-primary); } -.type_row_half{ - height: 32px !important; +.type_row { + height: 84px !important; } -.scroll-hide{ - resize: none !important; +.type_row_half { + height: 34px !important; } -.refresh_button{ +.refresh_button { border: none !important; background: none !important; font-size: none !important; box-shadow: none !important; } -.advanced_check_row{ +.advanced_check_row { width: 250px !important; } -.min_check{ +.min_check { min-width: min(1px, 100%) !important; } diff --git a/webui.py b/webui.py index 0dd863506..63ff6813b 100644 --- a/webui.py +++ b/webui.py @@ -112,10 +112,10 @@ def generate_clicked(task: worker.AsyncTask): gallery = gr.Gallery(label='Gallery', show_label=False, object_fit='contain', visible=True, height=768, elem_classes=['resizable_area', 'main_view', 'final_gallery', 'image_gallery'], elem_id='final_gallery') - with gr.Row(elem_classes='type_row'): + with gr.Row(): with gr.Column(scale=17): prompt = gr.Textbox(show_label=False, placeholder="Type prompt here or paste parameters.", elem_id='positive_prompt', - container=False, autofocus=True, elem_classes='type_row', lines=1024) + autofocus=True, lines=3) default_prompt = modules.config.default_prompt if isinstance(default_prompt, str) and default_prompt != '': From 2d55a5f257b04c783d89c0cd5e4cb9180e9e8c9b Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Tue, 4 Jun 2024 20:15:49 +0200 Subject: [PATCH 07/12] feat: add support for playground v2.5 (#3073) * feat: add support for playground v2.5 * feat: add preset for playground v2.5 * feat: change URL to mashb1t * feat: optimize playground v2.5 preset --- args_manager.py | 3 - .../contrib/external_model_advanced.py | 12 ++- ldm_patched/modules/latent_formats.py | 65 +++++++++++++ ldm_patched/modules/model_sampling.py | 93 ++++++++++++++++--- ldm_patched/modules/samplers.py | 2 +- modules/async_worker.py | 27 +++++- modules/core.py | 4 +- modules/flags.py | 3 +- modules/patch_precision.py | 2 - presets/.gitignore | 1 + presets/playground_v2.5.json | 51 ++++++++++ 11 files changed, 233 insertions(+), 30 deletions(-) create mode 100644 presets/playground_v2.5.json diff --git a/args_manager.py b/args_manager.py index e023da276..5a2b37c97 100644 --- a/args_manager.py +++ b/args_manager.py @@ -1,7 +1,4 @@ import ldm_patched.modules.args_parser as args_parser -import os - -from tempfile import gettempdir args_parser.parser.add_argument("--share", action='store_true', help="Set whether to share on Gradio.") diff --git a/ldm_patched/contrib/external_model_advanced.py b/ldm_patched/contrib/external_model_advanced.py index 9b52c36b5..b9f0ebdca 100644 --- a/ldm_patched/contrib/external_model_advanced.py +++ b/ldm_patched/contrib/external_model_advanced.py @@ -108,7 +108,7 @@ class ModelSamplingContinuousEDM: @classmethod def INPUT_TYPES(s): return {"required": { "model": ("MODEL",), - "sampling": (["v_prediction", "eps"],), + "sampling": (["v_prediction", "edm_playground_v2.5", "eps"],), "sigma_max": ("FLOAT", {"default": 120.0, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}), "sigma_min": ("FLOAT", {"default": 0.002, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}), }} @@ -121,17 +121,25 @@ def INPUT_TYPES(s): def patch(self, model, sampling, sigma_max, sigma_min): m = model.clone() + latent_format = None + sigma_data = 1.0 if sampling == "eps": sampling_type = ldm_patched.modules.model_sampling.EPS elif sampling == "v_prediction": sampling_type = ldm_patched.modules.model_sampling.V_PREDICTION + elif sampling == "edm_playground_v2.5": + sampling_type = ldm_patched.modules.model_sampling.EDM + sigma_data = 0.5 + latent_format = ldm_patched.modules.latent_formats.SDXL_Playground_2_5() class ModelSamplingAdvanced(ldm_patched.modules.model_sampling.ModelSamplingContinuousEDM, sampling_type): pass model_sampling = ModelSamplingAdvanced(model.model.model_config) - model_sampling.set_sigma_range(sigma_min, sigma_max) + model_sampling.set_parameters(sigma_min, sigma_max, sigma_data) m.add_object_patch("model_sampling", model_sampling) + if latent_format is not None: + m.add_object_patch("latent_format", latent_format) return (m, ) class RescaleCFG: diff --git a/ldm_patched/modules/latent_formats.py b/ldm_patched/modules/latent_formats.py index 2252a075e..1606793e0 100644 --- a/ldm_patched/modules/latent_formats.py +++ b/ldm_patched/modules/latent_formats.py @@ -1,3 +1,4 @@ +import torch class LatentFormat: scale_factor = 1.0 @@ -34,6 +35,70 @@ def __init__(self): ] self.taesd_decoder_name = "taesdxl_decoder" +class SDXL_Playground_2_5(LatentFormat): + def __init__(self): + self.scale_factor = 0.5 + self.latents_mean = torch.tensor([-1.6574, 1.886, -1.383, 2.5155]).view(1, 4, 1, 1) + self.latents_std = torch.tensor([8.4927, 5.9022, 6.5498, 5.2299]).view(1, 4, 1, 1) + + self.latent_rgb_factors = [ + # R G B + [ 0.3920, 0.4054, 0.4549], + [-0.2634, -0.0196, 0.0653], + [ 0.0568, 0.1687, -0.0755], + [-0.3112, -0.2359, -0.2076] + ] + self.taesd_decoder_name = "taesdxl_decoder" + + def process_in(self, latent): + latents_mean = self.latents_mean.to(latent.device, latent.dtype) + latents_std = self.latents_std.to(latent.device, latent.dtype) + return (latent - latents_mean) * self.scale_factor / latents_std + + def process_out(self, latent): + latents_mean = self.latents_mean.to(latent.device, latent.dtype) + latents_std = self.latents_std.to(latent.device, latent.dtype) + return latent * latents_std / self.scale_factor + latents_mean + + class SD_X4(LatentFormat): def __init__(self): self.scale_factor = 0.08333 + self.latent_rgb_factors = [ + [-0.2340, -0.3863, -0.3257], + [ 0.0994, 0.0885, -0.0908], + [-0.2833, -0.2349, -0.3741], + [ 0.2523, -0.0055, -0.1651] + ] + +class SC_Prior(LatentFormat): + def __init__(self): + self.scale_factor = 1.0 + self.latent_rgb_factors = [ + [-0.0326, -0.0204, -0.0127], + [-0.1592, -0.0427, 0.0216], + [ 0.0873, 0.0638, -0.0020], + [-0.0602, 0.0442, 0.1304], + [ 0.0800, -0.0313, -0.1796], + [-0.0810, -0.0638, -0.1581], + [ 0.1791, 0.1180, 0.0967], + [ 0.0740, 0.1416, 0.0432], + [-0.1745, -0.1888, -0.1373], + [ 0.2412, 0.1577, 0.0928], + [ 0.1908, 0.0998, 0.0682], + [ 0.0209, 0.0365, -0.0092], + [ 0.0448, -0.0650, -0.1728], + [-0.1658, -0.1045, -0.1308], + [ 0.0542, 0.1545, 0.1325], + [-0.0352, -0.1672, -0.2541] + ] + +class SC_B(LatentFormat): + def __init__(self): + self.scale_factor = 1.0 / 0.43 + self.latent_rgb_factors = [ + [ 0.1121, 0.2006, 0.1023], + [-0.2093, -0.0222, -0.0195], + [-0.3087, -0.1535, 0.0366], + [ 0.0290, -0.1574, -0.4078] + ] \ No newline at end of file diff --git a/ldm_patched/modules/model_sampling.py b/ldm_patched/modules/model_sampling.py index 57f51a000..bd8cb18c2 100644 --- a/ldm_patched/modules/model_sampling.py +++ b/ldm_patched/modules/model_sampling.py @@ -1,5 +1,4 @@ import torch -import numpy as np from ldm_patched.ldm.modules.diffusionmodules.util import make_beta_schedule import math @@ -12,12 +11,28 @@ def calculate_denoised(self, sigma, model_output, model_input): sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1)) return model_input - model_output * sigma + def noise_scaling(self, sigma, noise, latent_image, max_denoise=False): + if max_denoise: + noise = noise * torch.sqrt(1.0 + sigma ** 2.0) + else: + noise = noise * sigma + + noise += latent_image + return noise + + def inverse_noise_scaling(self, sigma, latent): + return latent class V_PREDICTION(EPS): def calculate_denoised(self, sigma, model_output, model_input): sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1)) return model_input * self.sigma_data ** 2 / (sigma ** 2 + self.sigma_data ** 2) - model_output * sigma * self.sigma_data / (sigma ** 2 + self.sigma_data ** 2) ** 0.5 +class EDM(V_PREDICTION): + def calculate_denoised(self, sigma, model_output, model_input): + sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1)) + return model_input * self.sigma_data ** 2 / (sigma ** 2 + self.sigma_data ** 2) + model_output * sigma * self.sigma_data / (sigma ** 2 + self.sigma_data ** 2) ** 0.5 + class ModelSamplingDiscrete(torch.nn.Module): def __init__(self, model_config=None): @@ -42,24 +57,23 @@ def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps else: betas = make_beta_schedule(beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s) alphas = 1. - betas - alphas_cumprod = torch.tensor(np.cumprod(alphas, axis=0), dtype=torch.float32) - # alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1]) + alphas_cumprod = torch.cumprod(alphas, dim=0) timesteps, = betas.shape self.num_timesteps = int(timesteps) self.linear_start = linear_start self.linear_end = linear_end + # self.register_buffer('betas', torch.tensor(betas, dtype=torch.float32)) + # self.register_buffer('alphas_cumprod', torch.tensor(alphas_cumprod, dtype=torch.float32)) + # self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32)) + sigmas = ((1 - alphas_cumprod) / alphas_cumprod) ** 0.5 self.set_sigmas(sigmas) - self.set_alphas_cumprod(alphas_cumprod.float()) def set_sigmas(self, sigmas): - self.register_buffer('sigmas', sigmas) - self.register_buffer('log_sigmas', sigmas.log()) - - def set_alphas_cumprod(self, alphas_cumprod): - self.register_buffer("alphas_cumprod", alphas_cumprod.float()) + self.register_buffer('sigmas', sigmas.float()) + self.register_buffer('log_sigmas', sigmas.log().float()) @property def sigma_min(self): @@ -94,8 +108,6 @@ def percent_to_sigma(self, percent): class ModelSamplingContinuousEDM(torch.nn.Module): def __init__(self, model_config=None): super().__init__() - self.sigma_data = 1.0 - if model_config is not None: sampling_settings = model_config.sampling_settings else: @@ -103,9 +115,11 @@ def __init__(self, model_config=None): sigma_min = sampling_settings.get("sigma_min", 0.002) sigma_max = sampling_settings.get("sigma_max", 120.0) - self.set_sigma_range(sigma_min, sigma_max) + sigma_data = sampling_settings.get("sigma_data", 1.0) + self.set_parameters(sigma_min, sigma_max, sigma_data) - def set_sigma_range(self, sigma_min, sigma_max): + def set_parameters(self, sigma_min, sigma_max, sigma_data): + self.sigma_data = sigma_data sigmas = torch.linspace(math.log(sigma_min), math.log(sigma_max), 1000).exp() self.register_buffer('sigmas', sigmas) #for compatibility with some schedulers @@ -134,3 +148,56 @@ def percent_to_sigma(self, percent): log_sigma_min = math.log(self.sigma_min) return math.exp((math.log(self.sigma_max) - log_sigma_min) * percent + log_sigma_min) + +class StableCascadeSampling(ModelSamplingDiscrete): + def __init__(self, model_config=None): + super().__init__() + + if model_config is not None: + sampling_settings = model_config.sampling_settings + else: + sampling_settings = {} + + self.set_parameters(sampling_settings.get("shift", 1.0)) + + def set_parameters(self, shift=1.0, cosine_s=8e-3): + self.shift = shift + self.cosine_s = torch.tensor(cosine_s) + self._init_alpha_cumprod = torch.cos(self.cosine_s / (1 + self.cosine_s) * torch.pi * 0.5) ** 2 + + #This part is just for compatibility with some schedulers in the codebase + self.num_timesteps = 10000 + sigmas = torch.empty((self.num_timesteps), dtype=torch.float32) + for x in range(self.num_timesteps): + t = (x + 1) / self.num_timesteps + sigmas[x] = self.sigma(t) + + self.set_sigmas(sigmas) + + def sigma(self, timestep): + alpha_cumprod = (torch.cos((timestep + self.cosine_s) / (1 + self.cosine_s) * torch.pi * 0.5) ** 2 / self._init_alpha_cumprod) + + if self.shift != 1.0: + var = alpha_cumprod + logSNR = (var/(1-var)).log() + logSNR += 2 * torch.log(1.0 / torch.tensor(self.shift)) + alpha_cumprod = logSNR.sigmoid() + + alpha_cumprod = alpha_cumprod.clamp(0.0001, 0.9999) + return ((1 - alpha_cumprod) / alpha_cumprod) ** 0.5 + + def timestep(self, sigma): + var = 1 / ((sigma * sigma) + 1) + var = var.clamp(0, 1.0) + s, min_var = self.cosine_s.to(var.device), self._init_alpha_cumprod.to(var.device) + t = (((var * min_var) ** 0.5).acos() / (torch.pi * 0.5)) * (1 + s) - s + return t + + def percent_to_sigma(self, percent): + if percent <= 0.0: + return 999999999.9 + if percent >= 1.0: + return 0.0 + + percent = 1.0 - percent + return self.sigma(torch.tensor(percent)) \ No newline at end of file diff --git a/ldm_patched/modules/samplers.py b/ldm_patched/modules/samplers.py index 35cb3d738..9ed1fcd28 100644 --- a/ldm_patched/modules/samplers.py +++ b/ldm_patched/modules/samplers.py @@ -523,7 +523,7 @@ def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=N KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu", - "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", "tcd"] + "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", "tcd", "edm_playground_v2.5"] class KSAMPLER(Sampler): def __init__(self, sampler_function, extra_options={}, inpaint_options={}): diff --git a/modules/async_worker.py b/modules/async_worker.py index 9c16d6fcb..76e10f924 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -828,16 +828,33 @@ def handler(async_task): if scheduler_name in ['lcm', 'tcd']: final_scheduler_name = 'sgm_uniform' - if pipeline.final_unet is not None: - pipeline.final_unet = core.opModelSamplingDiscrete.patch( + + def patch_discrete(unet): + return core.opModelSamplingDiscrete.patch( pipeline.final_unet, sampling=scheduler_name, zsnr=False)[0] + + if pipeline.final_unet is not None: + pipeline.final_unet = patch_discrete(pipeline.final_unet) if pipeline.final_refiner_unet is not None: - pipeline.final_refiner_unet = core.opModelSamplingDiscrete.patch( - pipeline.final_refiner_unet, + pipeline.final_refiner_unet = patch_discrete(pipeline.final_refiner_unet) + print(f'Using {scheduler_name} scheduler.') + elif scheduler_name == 'edm_playground_v2.5': + final_scheduler_name = 'karras' + + def patch_edm(unet): + return core.opModelSamplingContinuousEDM.patch( + unet, sampling=scheduler_name, - zsnr=False)[0] + sigma_max=120.0, + sigma_min=0.002)[0] + + if pipeline.final_unet is not None: + pipeline.final_unet = patch_edm(pipeline.final_unet) + if pipeline.final_refiner_unet is not None: + pipeline.final_refiner_unet = patch_edm(pipeline.final_refiner_unet) + print(f'Using {scheduler_name} scheduler.') async_task.yields.append(['preview', (flags.preparation_step_count, 'Moving model to GPU ...', None)]) diff --git a/modules/core.py b/modules/core.py index 3ca4cc5b8..78c897592 100644 --- a/modules/core.py +++ b/modules/core.py @@ -21,8 +21,7 @@ from modules.util import get_file_from_folder_list from ldm_patched.modules.lora import model_lora_keys_unet, model_lora_keys_clip from modules.config import path_embeddings -from ldm_patched.contrib.external_model_advanced import ModelSamplingDiscrete - +from ldm_patched.contrib.external_model_advanced import ModelSamplingDiscrete, ModelSamplingContinuousEDM opEmptyLatentImage = EmptyLatentImage() opVAEDecode = VAEDecode() @@ -32,6 +31,7 @@ opControlNetApplyAdvanced = ControlNetApplyAdvanced() opFreeU = FreeU_V2() opModelSamplingDiscrete = ModelSamplingDiscrete() +opModelSamplingContinuousEDM = ModelSamplingContinuousEDM() class StableDiffusionModel: diff --git a/modules/flags.py b/modules/flags.py index 25b0caaec..adaea1d19 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -48,8 +48,7 @@ KSAMPLER_NAMES = list(KSAMPLER.keys()) -SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", - "align_your_steps", "tcd"] +SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps", "tcd", "edm_playground_v2.5"] SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys()) sampler_list = SAMPLER_NAMES diff --git a/modules/patch_precision.py b/modules/patch_precision.py index 22ffda0ad..83569bdd1 100644 --- a/modules/patch_precision.py +++ b/modules/patch_precision.py @@ -51,8 +51,6 @@ def patched_register_schedule(self, given_betas=None, beta_schedule="linear", ti self.linear_end = linear_end sigmas = torch.tensor(((1 - alphas_cumprod) / alphas_cumprod) ** 0.5, dtype=torch.float32) self.set_sigmas(sigmas) - alphas_cumprod = torch.tensor(alphas_cumprod, dtype=torch.float32) - self.set_alphas_cumprod(alphas_cumprod) return diff --git a/presets/.gitignore b/presets/.gitignore index 481930c56..27e74136a 100644 --- a/presets/.gitignore +++ b/presets/.gitignore @@ -2,5 +2,6 @@ !anime.json !default.json !lcm.json +!playground_v2.5.json !realistic.json !sai.json \ No newline at end of file diff --git a/presets/playground_v2.5.json b/presets/playground_v2.5.json new file mode 100644 index 000000000..311bbc1dd --- /dev/null +++ b/presets/playground_v2.5.json @@ -0,0 +1,51 @@ +{ + "default_model": "playground-v2.5-1024px-aesthetic.fp16.safetensors", + "default_refiner": "None", + "default_refiner_switch": 0.5, + "default_loras": [ + [ + true, + "None", + 1.0 + ], + [ + true, + "None", + 1.0 + ], + [ + true, + "None", + 1.0 + ], + [ + true, + "None", + 1.0 + ], + [ + true, + "None", + 1.0 + ] + ], + "default_cfg_scale": 3, + "default_sample_sharpness": 2.0, + "default_sampler": "dpmpp_2m", + "default_scheduler": "edm_playground_v2.5", + "default_performance": "Speed", + "default_prompt": "", + "default_prompt_negative": "", + "default_styles": [ + "Fooocus V2", + "Fooocus Enhance", + "Fooocus Sharp" + ], + "default_aspect_ratio": "1024*1024", + "checkpoint_downloads": { + "playground-v2.5-1024px-aesthetic.fp16.safetensors": "https://huggingface.co/mashb1t/fav_models/resolve/main/fav/playground-v2.5-1024px-aesthetic.fp16.safetensors" + }, + "embeddings_downloads": {}, + "lora_downloads": {}, + "previous_default_models": [] +} \ No newline at end of file From b58bc7774e0d9f858ba52cd06b597f9f31db3793 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Tue, 4 Jun 2024 21:03:37 +0200 Subject: [PATCH 08/12] fix: correct sampling when gamma is 0 (#3093) --- ldm_patched/k_diffusion/sampling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ldm_patched/k_diffusion/sampling.py b/ldm_patched/k_diffusion/sampling.py index d1bc1e4b2..ea5540a42 100644 --- a/ldm_patched/k_diffusion/sampling.py +++ b/ldm_patched/k_diffusion/sampling.py @@ -832,5 +832,7 @@ def sample_tcd(model, x, sigmas, extra_args=None, callback=None, disable=None, n if eta > 0 and sigmas[i + 1] > 0: noise = noise_sampler(sigmas[i], sigmas[i + 1]) x = x / alpha_prod_s[i+1].sqrt() + noise * (sigmas[i+1]**2 + 1 - 1/alpha_prod_s[i+1]).sqrt() + else: + x *= torch.sqrt(1.0 + sigmas[i + 1] ** 2) return x \ No newline at end of file From 85a8deeceea11269d4d1a599847b766bcc8c9616 Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Tue, 4 Jun 2024 23:35:10 +0200 Subject: [PATCH 09/12] release: bump version to 2.4.2, update changelog --- fooocus_version.py | 2 +- presets/playground_v2.5.json | 2 +- update_log.md | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 750114584..4ddefe9ad 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.4.1' +version = '2.4.2' diff --git a/presets/playground_v2.5.json b/presets/playground_v2.5.json index 311bbc1dd..fb9bd7f3a 100644 --- a/presets/playground_v2.5.json +++ b/presets/playground_v2.5.json @@ -29,7 +29,7 @@ 1.0 ] ], - "default_cfg_scale": 3, + "default_cfg_scale": 3.0, "default_sample_sharpness": 2.0, "default_sampler": "dpmpp_2m", "default_scheduler": "edm_playground_v2.5", diff --git a/update_log.md b/update_log.md index 733f077bb..81d0718f1 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,12 @@ +# [2.4.2](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.2) + +* Fix some small bugs (tcd scheduler when gamma is 0, chown in Dockerfile, update cmd args in readme, translation for aspect ratios, vae default after file reload) +* Fix performance LoRA replacement when data is loaded from history log and inline prompt +* Add support and preset for playground v2.5 (only works with performance Quality or Speed, use with scheduler edm_playground_v2) +* Make textboxes (incl. positive prompt) resizable +* Hide intermediate images when performance of Gradio would bottleneck the generation process (Extreme Speed, Lightning, Hyper-SD) + + # [2.4.1](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.1) * Fix some small bugs (e.g. adjust clip skip default value from 1 to 2, add type check to aspect ratios js update function) From 04d764820e5cc526092e1aee36b4e78182c42d54 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:42:26 +0200 Subject: [PATCH 10/12] fix: correctly set alphas_cumprod (#3106) --- ldm_patched/modules/model_sampling.py | 6 ++++++ modules/patch_precision.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/ldm_patched/modules/model_sampling.py b/ldm_patched/modules/model_sampling.py index bd8cb18c2..8971b4e6e 100644 --- a/ldm_patched/modules/model_sampling.py +++ b/ldm_patched/modules/model_sampling.py @@ -1,6 +1,7 @@ import torch from ldm_patched.ldm.modules.diffusionmodules.util import make_beta_schedule import math +import numpy as np class EPS: def calculate_input(self, sigma, noise): @@ -69,12 +70,17 @@ def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps # self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32)) sigmas = ((1 - alphas_cumprod) / alphas_cumprod) ** 0.5 + alphas_cumprod = torch.tensor(np.cumprod(alphas, axis=0), dtype=torch.float32) self.set_sigmas(sigmas) + self.set_alphas_cumprod(alphas_cumprod.float()) def set_sigmas(self, sigmas): self.register_buffer('sigmas', sigmas.float()) self.register_buffer('log_sigmas', sigmas.log().float()) + def set_alphas_cumprod(self, alphas_cumprod): + self.register_buffer("alphas_cumprod", alphas_cumprod.float()) + @property def sigma_min(self): return self.sigmas[0] diff --git a/modules/patch_precision.py b/modules/patch_precision.py index 83569bdd1..22ffda0ad 100644 --- a/modules/patch_precision.py +++ b/modules/patch_precision.py @@ -51,6 +51,8 @@ def patched_register_schedule(self, given_betas=None, beta_schedule="linear", ti self.linear_end = linear_end sigmas = torch.tensor(((1 - alphas_cumprod) / alphas_cumprod) ** 0.5, dtype=torch.float32) self.set_sigmas(sigmas) + alphas_cumprod = torch.tensor(alphas_cumprod, dtype=torch.float32) + self.set_alphas_cumprod(alphas_cumprod) return From 5abae220c5660ba82541f161f6bd9a4367be565a Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 6 Jun 2024 19:29:08 +0200 Subject: [PATCH 11/12] feat: parse env var strings to expected config value types (#3107) * fix: add try_parse_bool for env var strings to enable config overrides of boolean values * fix: fallback to given value if not parseable * feat: extend eval to all valid types * fix: remove return type * fix: prevent strange type conversions by providing expected type * feat: add tests --- modules/config.py | 118 +++++++++++++++++++++++++------------- modules/extra_utils.py | 15 +++++ tests/test_extra_utils.py | 74 ++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 39 deletions(-) create mode 100644 tests/test_extra_utils.py diff --git a/modules/config.py b/modules/config.py index 29a16d6dc..e3c427d2c 100644 --- a/modules/config.py +++ b/modules/config.py @@ -2,13 +2,14 @@ import json import math import numbers + import args_manager import tempfile import modules.flags import modules.sdxl_styles from modules.model_loader import load_file_from_url -from modules.extra_utils import makedirs_with_log, get_files_from_folder +from modules.extra_utils import makedirs_with_log, get_files_from_folder, try_eval_env_var from modules.flags import OutputFormat, Performance, MetadataScheme @@ -200,7 +201,7 @@ def get_dir_or_set_default(key, default_value, as_array=False, make_directory=Fa path_outputs = get_path_output() -def get_config_item_or_set_default(key, default_value, validator, disable_empty_as_none=False): +def get_config_item_or_set_default(key, default_value, validator, disable_empty_as_none=False, expected_type=None): global config_dict, visited_keys if key not in visited_keys: @@ -208,6 +209,7 @@ def get_config_item_or_set_default(key, default_value, validator, disable_empty_ v = os.getenv(key) if v is not None: + v = try_eval_env_var(v, expected_type) print(f"Environment: {key} = {v}") config_dict[key] = v @@ -252,41 +254,49 @@ def init_temp_path(path: str | None, default_path: str) -> str: key='temp_path', default_value=default_temp_path, validator=lambda x: isinstance(x, str), + expected_type=str ), default_temp_path) temp_path_cleanup_on_launch = get_config_item_or_set_default( key='temp_path_cleanup_on_launch', default_value=True, - validator=lambda x: isinstance(x, bool) + validator=lambda x: isinstance(x, bool), + expected_type=bool ) default_base_model_name = default_model = get_config_item_or_set_default( key='default_model', default_value='model.safetensors', - validator=lambda x: isinstance(x, str) + validator=lambda x: isinstance(x, str), + expected_type=str ) previous_default_models = get_config_item_or_set_default( key='previous_default_models', default_value=[], - validator=lambda x: isinstance(x, list) and all(isinstance(k, str) for k in x) + validator=lambda x: isinstance(x, list) and all(isinstance(k, str) for k in x), + expected_type=list ) default_refiner_model_name = default_refiner = get_config_item_or_set_default( key='default_refiner', default_value='None', - validator=lambda x: isinstance(x, str) + validator=lambda x: isinstance(x, str), + expected_type=str ) default_refiner_switch = get_config_item_or_set_default( key='default_refiner_switch', default_value=0.8, - validator=lambda x: isinstance(x, numbers.Number) and 0 <= x <= 1 + validator=lambda x: isinstance(x, numbers.Number) and 0 <= x <= 1, + expected_type=numbers.Number ) default_loras_min_weight = get_config_item_or_set_default( key='default_loras_min_weight', default_value=-2, - validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10 + validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10, + expected_type=numbers.Number ) default_loras_max_weight = get_config_item_or_set_default( key='default_loras_max_weight', default_value=2, - validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10 + validator=lambda x: isinstance(x, numbers.Number) and -10 <= x <= 10, + expected_type=numbers.Number ) default_loras = get_config_item_or_set_default( key='default_loras', @@ -320,38 +330,45 @@ def init_temp_path(path: str | None, default_path: str) -> str: validator=lambda x: isinstance(x, list) and all( len(y) == 3 and isinstance(y[0], bool) and isinstance(y[1], str) and isinstance(y[2], numbers.Number) or len(y) == 2 and isinstance(y[0], str) and isinstance(y[1], numbers.Number) - for y in x) + for y in x), + expected_type=list ) default_loras = [(y[0], y[1], y[2]) if len(y) == 3 else (True, y[0], y[1]) for y in default_loras] default_max_lora_number = get_config_item_or_set_default( key='default_max_lora_number', default_value=len(default_loras) if isinstance(default_loras, list) and len(default_loras) > 0 else 5, - validator=lambda x: isinstance(x, int) and x >= 1 + validator=lambda x: isinstance(x, int) and x >= 1, + expected_type=int ) default_cfg_scale = get_config_item_or_set_default( key='default_cfg_scale', default_value=7.0, - validator=lambda x: isinstance(x, numbers.Number) + validator=lambda x: isinstance(x, numbers.Number), + expected_type=numbers.Number ) default_sample_sharpness = get_config_item_or_set_default( key='default_sample_sharpness', default_value=2.0, - validator=lambda x: isinstance(x, numbers.Number) + validator=lambda x: isinstance(x, numbers.Number), + expected_type=numbers.Number ) default_sampler = get_config_item_or_set_default( key='default_sampler', default_value='dpmpp_2m_sde_gpu', - validator=lambda x: x in modules.flags.sampler_list + validator=lambda x: x in modules.flags.sampler_list, + expected_type=str ) default_scheduler = get_config_item_or_set_default( key='default_scheduler', default_value='karras', - validator=lambda x: x in modules.flags.scheduler_list + validator=lambda x: x in modules.flags.scheduler_list, + expected_type=str ) default_vae = get_config_item_or_set_default( key='default_vae', default_value=modules.flags.default_vae, - validator=lambda x: isinstance(x, str) + validator=lambda x: isinstance(x, str), + expected_type=str ) default_styles = get_config_item_or_set_default( key='default_styles', @@ -360,121 +377,144 @@ def init_temp_path(path: str | None, default_path: str) -> str: "Fooocus Enhance", "Fooocus Sharp" ], - validator=lambda x: isinstance(x, list) and all(y in modules.sdxl_styles.legal_style_names for y in x) + validator=lambda x: isinstance(x, list) and all(y in modules.sdxl_styles.legal_style_names for y in x), + expected_type=list ) default_prompt_negative = get_config_item_or_set_default( key='default_prompt_negative', default_value='', validator=lambda x: isinstance(x, str), - disable_empty_as_none=True + disable_empty_as_none=True, + expected_type=str ) default_prompt = get_config_item_or_set_default( key='default_prompt', default_value='', validator=lambda x: isinstance(x, str), - disable_empty_as_none=True + disable_empty_as_none=True, + expected_type=str ) default_performance = get_config_item_or_set_default( key='default_performance', default_value=Performance.SPEED.value, - validator=lambda x: x in Performance.list() + validator=lambda x: x in Performance.list(), + expected_type=str ) default_advanced_checkbox = get_config_item_or_set_default( key='default_advanced_checkbox', default_value=False, - validator=lambda x: isinstance(x, bool) + validator=lambda x: isinstance(x, bool), + expected_type=bool ) default_max_image_number = get_config_item_or_set_default( key='default_max_image_number', default_value=32, - validator=lambda x: isinstance(x, int) and x >= 1 + validator=lambda x: isinstance(x, int) and x >= 1, + expected_type=int ) default_output_format = get_config_item_or_set_default( key='default_output_format', default_value='png', - validator=lambda x: x in OutputFormat.list() + validator=lambda x: x in OutputFormat.list(), + expected_type=str ) default_image_number = get_config_item_or_set_default( key='default_image_number', default_value=2, - validator=lambda x: isinstance(x, int) and 1 <= x <= default_max_image_number + validator=lambda x: isinstance(x, int) and 1 <= x <= default_max_image_number, + expected_type=int ) checkpoint_downloads = get_config_item_or_set_default( key='checkpoint_downloads', default_value={}, - validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()) + validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()), + expected_type=dict ) lora_downloads = get_config_item_or_set_default( key='lora_downloads', default_value={}, - validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()) + validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()), + expected_type=dict ) embeddings_downloads = get_config_item_or_set_default( key='embeddings_downloads', default_value={}, - validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()) + validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()), + expected_type=dict ) available_aspect_ratios = get_config_item_or_set_default( key='available_aspect_ratios', default_value=modules.flags.sdxl_aspect_ratios, - validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1 + validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1, + expected_type=list ) default_aspect_ratio = get_config_item_or_set_default( key='default_aspect_ratio', default_value='1152*896' if '1152*896' in available_aspect_ratios else available_aspect_ratios[0], - validator=lambda x: x in available_aspect_ratios + validator=lambda x: x in available_aspect_ratios, + expected_type=str ) default_inpaint_engine_version = get_config_item_or_set_default( key='default_inpaint_engine_version', default_value='v2.6', - validator=lambda x: x in modules.flags.inpaint_engine_versions + validator=lambda x: x in modules.flags.inpaint_engine_versions, + expected_type=str ) default_cfg_tsnr = get_config_item_or_set_default( key='default_cfg_tsnr', default_value=7.0, - validator=lambda x: isinstance(x, numbers.Number) + validator=lambda x: isinstance(x, numbers.Number), + expected_type=numbers.Number ) default_clip_skip = get_config_item_or_set_default( key='default_clip_skip', default_value=2, - validator=lambda x: isinstance(x, int) and 1 <= x <= modules.flags.clip_skip_max + validator=lambda x: isinstance(x, int) and 1 <= x <= modules.flags.clip_skip_max, + expected_type=int ) default_overwrite_step = get_config_item_or_set_default( key='default_overwrite_step', default_value=-1, - validator=lambda x: isinstance(x, int) + validator=lambda x: isinstance(x, int), + expected_type=int ) default_overwrite_switch = get_config_item_or_set_default( key='default_overwrite_switch', default_value=-1, - validator=lambda x: isinstance(x, int) + validator=lambda x: isinstance(x, int), + expected_type=int ) example_inpaint_prompts = get_config_item_or_set_default( key='example_inpaint_prompts', default_value=[ 'highly detailed face', 'detailed girl face', 'detailed man face', 'detailed hand', 'beautiful eyes' ], - validator=lambda x: isinstance(x, list) and all(isinstance(v, str) for v in x) + validator=lambda x: isinstance(x, list) and all(isinstance(v, str) for v in x), + expected_type=list ) default_black_out_nsfw = get_config_item_or_set_default( key='default_black_out_nsfw', default_value=False, - validator=lambda x: isinstance(x, bool) + validator=lambda x: isinstance(x, bool), + expected_type=bool ) default_save_metadata_to_images = get_config_item_or_set_default( key='default_save_metadata_to_images', default_value=False, - validator=lambda x: isinstance(x, bool) + validator=lambda x: isinstance(x, bool), + expected_type=bool ) default_metadata_scheme = get_config_item_or_set_default( key='default_metadata_scheme', default_value=MetadataScheme.FOOOCUS.value, - validator=lambda x: x in [y[1] for y in modules.flags.metadata_scheme if y[1] == x] + validator=lambda x: x in [y[1] for y in modules.flags.metadata_scheme if y[1] == x], + expected_type=str ) metadata_created_by = get_config_item_or_set_default( key='metadata_created_by', default_value='', - validator=lambda x: isinstance(x, str) + validator=lambda x: isinstance(x, str), + expected_type=str ) example_inpaint_prompts = [[x] for x in example_inpaint_prompts] diff --git a/modules/extra_utils.py b/modules/extra_utils.py index 9906c8202..c2dfa8104 100644 --- a/modules/extra_utils.py +++ b/modules/extra_utils.py @@ -1,4 +1,6 @@ import os +from ast import literal_eval + def makedirs_with_log(path): try: @@ -24,3 +26,16 @@ def get_files_from_folder(folder_path, extensions=None, name_filter=None): filenames.append(path) return filenames + + +def try_eval_env_var(value: str, expected_type=None): + try: + value_eval = value + if expected_type is bool: + value_eval = value.title() + value_eval = literal_eval(value_eval) + if expected_type is not None and not isinstance(value_eval, expected_type): + return value + return value_eval + except: + return value diff --git a/tests/test_extra_utils.py b/tests/test_extra_utils.py new file mode 100644 index 000000000..a849aa16d --- /dev/null +++ b/tests/test_extra_utils.py @@ -0,0 +1,74 @@ +import numbers +import os +import unittest + +import modules.flags +from modules import extra_utils + + +class TestUtils(unittest.TestCase): + def test_try_eval_env_var(self): + test_cases = [ + { + "input": ("foo", str), + "output": "foo" + }, + { + "input": ("1", int), + "output": 1 + }, + { + "input": ("1.0", float), + "output": 1.0 + }, + { + "input": ("1", numbers.Number), + "output": 1 + }, + { + "input": ("1.0", numbers.Number), + "output": 1.0 + }, + { + "input": ("true", bool), + "output": True + }, + { + "input": ("True", bool), + "output": True + }, + { + "input": ("false", bool), + "output": False + }, + { + "input": ("False", bool), + "output": False + }, + { + "input": ("True", str), + "output": "True" + }, + { + "input": ("False", str), + "output": "False" + }, + { + "input": ("['a', 'b', 'c']", list), + "output": ['a', 'b', 'c'] + }, + { + "input": ("{'a':1}", dict), + "output": {'a': 1} + }, + { + "input": ("('foo', 1)", tuple), + "output": ('foo', 1) + } + ] + + for test in test_cases: + value, expected_type = test["input"] + expected = test["output"] + actual = extra_utils.try_eval_env_var(value, expected_type) + self.assertEqual(expected, actual) From ba77e7f706c07be9d1a9e1de929c5f40a662376f Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 6 Jun 2024 19:34:44 +0200 Subject: [PATCH 12/12] release: bump version to 2.4.3, update changelog (#3109) --- fooocus_version.py | 2 +- update_log.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 4ddefe9ad..84d1586b0 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.4.2' +version = '2.4.3' diff --git a/update_log.md b/update_log.md index 81d0718f1..8aa436479 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,8 @@ +# [2.4.3](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.3) + +* Fix alphas_cumprod setter for TCD sampler +* Add parser for env var strings to expected config value types to allow override of all non-path config keys + # [2.4.2](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.2) * Fix some small bugs (tcd scheduler when gamma is 0, chown in Dockerfile, update cmd args in readme, translation for aspect ratios, vae default after file reload) @@ -6,7 +11,6 @@ * Make textboxes (incl. positive prompt) resizable * Hide intermediate images when performance of Gradio would bottleneck the generation process (Extreme Speed, Lightning, Hyper-SD) - # [2.4.1](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.1) * Fix some small bugs (e.g. adjust clip skip default value from 1 to 2, add type check to aspect ratios js update function)