dev(narugo): narugo's workflow code

cfgs/infer/anime/text2img_anime.yaml

@@ -1,4 +1,4 @@
-_base_: [cfgs/infer/text2img.yaml]
+_base_: [ cfgs/infer/text2img.yaml ]
 
 pretrained_model: 'deepghs/animefull-latest'  # animefull-latest model
 prompt: 'masterpiece, best quality, 1girl, solo, tohsaka rin'  # image of 远坂凛(tohsaka rin)
@@ -13,9 +13,25 @@ infer_args:
   num_inference_steps: 30  # how many steps
 
 new_components:
-  scheduler:
-    _target_: diffusers.EulerAncestralDiscreteScheduler # change Sampler
+#  scheduler: # Euler a
+#    _target_: diffusers.EulerAncestralDiscreteScheduler # change Sampler
+#    beta_start: 0.00085
+#    beta_end: 0.012
+#    beta_schedule: 'scaled_linear'
+  scheduler: # DPM++ 2M Karras
+    _target_: diffusers.DPMSolverMultistepScheduler
     beta_start: 0.00085
     beta_end: 0.012
-    beta_schedule: 'scaled_linear'
+    algorithm_type: dpmsolver++
+    beta_schedule: scaled_linear
+    use_karras_sigmas: true
+  vae: # use NAI's vae
+    _target_: diffusers.AutoencoderKL.from_pretrained
+    pretrained_model_name_or_path: deepghs/animefull-latest  # path to vae model
+    subfolder: vae
 
+output_dir: 'output/'
+interface:
+  - _target_: hcpdiff.vis.DiskInterface
+    show_steps: 0
+    save_root: '${output_dir}'

cfgs/infer/anime/text2img_anime_lora.yaml

@@ -30,6 +30,7 @@ exp_dir: 'exps/2023-07-26-01-05-35'  # experiment directory
 model_steps: 1000  # steps of selected model
 emb_dir: '${exp_dir}/ckpts/'
 output_dir: 'output/'
+encoder_attention_mask: false
 
 merge:
   alpha: 0.85 # lora权重, default: 0.85

cfgs/infer/anime/text2img_sdxl_anime.yaml

@@ -0,0 +1,26 @@
+_base_: [ cfgs/infer/text2img_sdxl.yaml ]
+
+pretrained_model: 'KBlueLeaf/kohaku-xl-beta7.1'  # base model created by kohaku
+prompt: 'masterpiece, best quality, 1girl, solo, tohsaka rin'  # image of 远坂凛(tohsaka rin)
+neg_prompt: 'bad anatomy, disembodied limb, realistic, blurry, watercolor'
+
+clip_skip: 1 #动漫模型通常会跳过一个CLIP层
+
+infer_args:
+  width: 1024
+  height: 1536  # image size
+  guidance_scale: 7.5  # scale, when higher, the images will tend to be more similar
+  num_inference_steps: 30  # how many steps
+
+new_components:
+  scheduler: # DPM++ 2M Karras
+    _target_: diffusers.DPMSolverMultistepScheduler
+    beta_start: 0.00085
+    beta_end: 0.012
+    algorithm_type: dpmsolver++
+    beta_schedule: scaled_linear
+    use_karras_sigmas: true
+  vae: # use sdxl-base-v1.0's vae
+    _target_: diffusers.AutoencoderKL.from_pretrained
+    pretrained_model_name_or_path: stabilityai/stable-diffusion-xl-base-1.0  # path to vae model
+    subfolder: vae

cfgs/infer/text2img_sdxl.yaml

@@ -1,4 +1,4 @@
-_base_: [cfgs/infer/text2img.yaml]
+_base_: [ cfgs/infer/text2img.yaml ]
 
 emb_dir: embs_sdxl
 

cfgs/train/examples/lora_anime_character.yaml

@@ -5,6 +5,13 @@ model:
   pretrained_model_name_or_path: 'deepghs/animefull-latest'
   clip_skip: 1
 
+train:
+  train_steps: 1000
+  save_step: 100
+
+  scheduler:
+    num_training_steps: ${train.train_steps}
+
 character_name: surtr_arknights
 dataset_dir: '/root/autodl-tmp/dataset/surtr_3'
 # if exp_dir is not set, a random time-based directory will be used
@@ -16,7 +23,7 @@ text_encoder_rank: 4
 tokenizer_pt:
   emb_dir: 'embs/' #自定义word目录
   replace: False #训练后是否替换原有word
-  train: 
+  train:
     - name: ${character_name}
       lr: 0.003
 
@@ -49,16 +56,16 @@ data:
           pt1: ${character_name}
 
     # support images with any size, not recommended for anime training
-    # bucket:
-    #   _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
-    #   target_area: ${times:512,512}
-    #   num_bucket: 5
+    bucket:
+      _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
+      target_area: ${hcp.eval:"512*512"}
+      num_bucket: 5
 
     # all images must have the same size, such as 512x704
-    bucket:
-      _target_: hcpdiff.data.bucket.SizeBucket.from_files # aspect ratio bucket
-      target_area: ---
-      num_bucket: 1
+#    bucket:
+#      _target_: hcpdiff.data.bucket.SizeBucket.from_files # aspect ratio bucket
+#      target_area: ---
+#      num_bucket: 1
 
 logger:
   - _target_: hcpdiff.loggers.CLILogger

cfgs/train/examples/lora_anime_character_reg.yaml

@@ -0,0 +1,124 @@
+_base_:
+  - cfgs/train/examples/lora_conventional.yaml
+  - cfgs/train/dataset/regularization_dataset.yaml
+
+model:
+  pretrained_model_name_or_path: 'deepghs/animefull-latest'
+  clip_skip: 1
+
+train:
+  train_steps: 1000
+  save_step: 100
+
+  scheduler:
+    num_training_steps: ${train.train_steps}
+
+character_name: surtr_arknights
+dataset_dir: '/root/autodl-tmp/dataset/surtr_3'
+reg_dataset_dir: '/root/autodl-tmp/reg_dataset'
+# if exp_dir is not set, a random time-based directory will be used
+# exp_dir: 'exps/surtr'
+
+tag_dropout: 0.3
+
+unet_rank: 8
+text_encoder_rank: 4
+
+tokenizer_pt:
+  emb_dir: 'embs/' #自定义word目录
+  replace: False #训练后是否替换原有word
+  train:
+    - name: ${character_name}
+      lr: 0.003
+
+lora_unet:
+  - lr: 1e-4
+    rank: ${unet_rank}
+    layers:
+      - 're:.*\.attn.?$'
+      - 're:.*\.ff$'
+
+lora_text_encoder:
+  - lr: 1e-5
+    rank: ${text_encoder_rank}
+    layers:
+      - 're:.*self_attn$'
+      - 're:.*mlp$'
+
+data:
+  dataset1:
+    batch_size: 4
+    cache_latents: True
+
+    source:
+      data_source1:
+        img_root: ${dataset_dir}
+        prompt_template: 'prompt_tuning_template/object_caption.txt'
+        caption_file: ${dataset_dir}  # path to image captions (file_words)
+
+        word_names:
+          pt1: ${character_name}
+
+        text_transforms:
+          _target_: torchvision.transforms.Compose
+          transforms:
+            - _target_: hcpdiff.utils.caption_tools.TagShuffle
+            - _target_: hcpdiff.utils.caption_tools.TagDropout
+              p: ${tag_dropout}
+            - _target_: hcpdiff.utils.caption_tools.TemplateFill
+              word_names: ${....word_names}
+
+    # support images with any size, not recommended for anime training
+    # bucket:
+    #   _target_: hcpdiff.data.bucket.RatioSizeBucket.from_files # aspect ratio bucket
+    #   target_area: ---
+    #   max_area: ${hcp.eval:"768*768"}
+    #   num_bucket: 10
+    bucket:
+      _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
+      target_area: ${hcp.eval:"512*512"}
+      num_bucket: 5
+
+  dataset_class:
+    batch_size: 1
+    cache_latents: True
+    loss_weight: 1.0
+
+    source:
+      data_source1:
+        img_root: ${reg_dataset_dir}
+        prompt_template: 'prompt_tuning_template/object_caption.txt'
+        caption_file: ${reg_dataset_dir}
+
+        word_names:
+          pt1: ''
+
+        text_transforms:
+          _target_: torchvision.transforms.Compose
+          transforms:
+            - _target_: hcpdiff.utils.caption_tools.TagShuffle
+            - _target_: hcpdiff.utils.caption_tools.TagDropout
+              p: ${tag_dropout}
+            - _target_: hcpdiff.utils.caption_tools.TemplateFill
+              word_names: ${....word_names}
+
+    bucket:
+      _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
+      target_area: ${hcp.eval:"512*512"}
+      num_bucket: 20
+
+logger:
+  - _target_: hcpdiff.loggers.CLILogger
+    _partial_: True
+    out_path: 'train.log'
+    log_step: 20
+  - _target_: hcpdiff.loggers.TBLogger
+    _partial_: True
+    out_path: 'tblog/'
+    log_step: 5
+#  - _target_: hcpdiff.loggers.WanDBLogger
+#    _partial_: True
+#    out_path: null
+#    log_step: 5
+
+

cfgs/train/examples/lora_anime_character_reg_v1.5.yaml

@@ -0,0 +1,122 @@
+_base_:
+  - cfgs/train/examples/lora_conventional.yaml
+  - cfgs/train/dataset/regularization_dataset.yaml
+
+model:
+  pretrained_model_name_or_path: 'deepghs/animefull-latest'
+  clip_skip: 1
+
+train:
+  train_steps: 10000
+  save_step: 200
+  scheduler:
+    num_training_steps: ${train.train_steps}
+
+character_name: tohsaka_rin_fatestaynightufotable
+# if exp_dir is not set, a random time-based directory will be used
+exp_dir: 'exps/tohsaka_rin_t4_r1_w1_p0.3_initc_dunet0.01_note_720br'
+
+dataset:
+  dir: '/nfs3/lora_datasets/tohsaka_rin_fatestaynightufotable'
+  bs: 4
+  resolution: 720
+  num_bucket: 5
+reg_dataset:
+  dir: '/nfs3/lora_datasets/reg_all'
+  cache: '/nfs3/lora_datasets/reg_all.bin'
+  bs: 1
+  loss_weight: 1.0
+  resolution: 720
+  num_bucket: 20
+
+tag_dropout: 0.2
+pt:
+  emb_dir: 'embs/'
+  lr: 0.03
+unet_:
+  lr: 1e-4
+  rank: 0.01
+
+tokenizer_pt:
+  emb_dir: ${pt.emb_dir}
+  replace: False
+  train:
+    - name: ${character_name}
+      lr: ${pt.lr}
+
+lora_unet:
+  - lr: ${unet_.lr}
+    rank: ${unet_.rank}
+    layers:
+      - 're:.*\.attn.?$'
+      - 're:.*\.ff$'
+lora_text_encoder: []
+
+data:
+  dataset1:
+    batch_size: ${dataset.bs}
+    cache_latents: True
+
+    source:
+      data_source1:
+        img_root: ${dataset.dir}
+        prompt_template: 'prompt_tuning_template/object_caption.txt'
+        caption_file: ${dataset.dir}  # path to image captions (file_words)
+
+        word_names:
+          pt1: ${character_name}
+
+        text_transforms:
+          _target_: torchvision.transforms.Compose
+          transforms:
+            - _target_: hcpdiff.utils.caption_tools.TagShuffle
+            - _target_: hcpdiff.utils.caption_tools.TagDropout
+              p: ${tag_dropout}
+            - _target_: hcpdiff.utils.caption_tools.TemplateFill
+              word_names: ${....word_names}
+
+    bucket:
+      _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
+      target_area: ${hcp.eval:"${dataset.resolution}*${dataset.resolution}"}
+      num_bucket: ${dataset.num_bucket}
+
+  dataset_class:
+    batch_size: ${reg_dataset.bs}
+    cache_latents: True
+    loss_weight: ${reg_dataset.loss_weight}
+    cache_path: ${reg_dataset.cache}
+
+    source:
+      data_source1:
+        img_root: ${reg_dataset.dir}
+        prompt_template: 'prompt_tuning_template/object_caption.txt'
+        caption_file: ${reg_dataset.dir}
+
+        word_names:
+          pt1: ''
+
+        text_transforms:
+          _target_: torchvision.transforms.Compose
+          transforms:
+            - _target_: hcpdiff.utils.caption_tools.TagShuffle
+            - _target_: hcpdiff.utils.caption_tools.TagDropout
+              p: ${tag_dropout}
+            - _target_: hcpdiff.utils.caption_tools.TemplateFill
+              word_names: ${....word_names}
+
+    bucket:
+      _target_: hcpdiff.data.bucket.RatioBucket.from_files # aspect ratio bucket
+      target_area: ${hcp.eval:"${reg_dataset.resolution}*${reg_dataset.resolution}"}
+      num_bucket: ${reg_dataset.num_bucket}
+
+logger:
+  - _target_: hcpdiff.loggers.CLILogger
+    _partial_: True
+    out_path: 'train.log'
+    log_step: 20
+  - _target_: hcpdiff.loggers.TBLogger
+    _partial_: True
+    out_path: 'tblog/'
+    log_step: 5
+
+