$ python3 run.py config/foo.yml
Running 1 job
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 1.4.17 (you have 1.4.15). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1.
  check_for_updates()
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/mediapipe_face/mediapipe_face_common.py:7: UserWarning: The module 'mediapipe' is not installed. The package will have limited functionality. Please install it using the command: pip install 'mediapipe'
  warnings.warn(
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_5m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_5m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
  return register_model(fn_wrapper)
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_11m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_11m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
  return register_model(fn_wrapper)
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
  return register_model(fn_wrapper)
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_384 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_384. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
  return register_model(fn_wrapper)
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_512 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_512. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
  return register_model(fn_wrapper)
{
    "type": "sd_trainer",
    "training_folder": "output",
    "performance_log_every": 1000,
    "device": "cuda:0",
    "network": {
        "type": "lora",
        "linear": 16,
        "linear_alpha": 16
    },
    "save": {
        "dtype": "float16",
        "save_every": 250,
        "max_step_saves_to_keep": 20,
        "push_to_hub": false
    },
    "datasets": [
        {
            "folder_path": "/home/ubuntu/ai-toolkit/dataset/v1",
            "caption_ext": "txt",
            "caption_dropout_rate": 0.05,
            "shuffle_tokens": false,
            "cache_latents_to_disk": true,
            "resolution": [
                512,
                768,
                1024
            ]
        }
    ],
    "train": {
        "batch_size": 1,
        "steps": 3000,
        "gradient_accumulation_steps": 1,
        "train_unet": true,
        "train_text_encoder": false,
        "gradient_checkpointing": true,
        "noise_scheduler": "flowmatch",
        "optimizer": "adamw8bit",
        "lr": 0.0001,
        "ema_config": {
            "use_ema": true,
            "ema_decay": 0.99
        },
        "dtype": "bf16"
    },
    "model": {
        "name_or_path": "black-forest-labs/FLUX.1-dev",
        "is_flux": true,
        "quantize": true,
        "low_vram": true
    },
    "sample": {
        "sampler": "flowmatch",
        "sample_every": 250,
        "width": 1024,
        "height": 1024,
        "prompts": [
            "[trigger] holding a sign that says 'I LOVE PROMPTS!'",
            "woman with red hair, playing chess at the park, bomb going off in the background",
            "[trigger] a woman holding a coffee cup, in a beanie, sitting at a cafe",
            "a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini",
            "a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background",
            "a bear building a log cabin in the snow covered mountains",
            "woman playing the guitar, on stage, singing a song, laser lights, punk rocker",
            "hipster man with a beard, building a chair, in a wood shop",
            "photo of a man, white background, medium shot, modeling clothing, studio lighting, white backdrop",
            "a man holding a sign that says, 'this is a sign'",
            "a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle"
        ],
        "seed": 42,
        "walk_seed": true,
        "guidance_scale": 4,
        "sample_steps": 20
    }
}
Using EMA
/home/ubuntu/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py:61: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
  self.scaler = torch.cuda.amp.GradScaler()

#############################################
# Running job: foo
#############################################


Running  1 process
Loading Flux model
Loading transformer
Quantizing transformer
/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:1965: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. 
If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
  warnings.warn(
Loading vae
Loading t5
You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
Downloading shards: 100%|███████████████████████████████████████████| 2/2 [00:00<00:00, 1085.48it/s]
Loading checkpoint shards: 100%|██████████████████████████████████████| 2/2 [00:00<00:00,  2.95it/s]
Quantizing T5
Loading clip
making pipe
preparing
create LoRA network. base dim (rank): 16, alpha: 16
neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
create LoRA for Text Encoder: 0 modules.
create LoRA for U-Net: 494 modules.
enable LoRA for U-Net
Dataset: /home/ubuntu/ai-toolkit/dataset/v1
  -  Preprocessing image dimensions
100%|████████████████████████████████████████████████████████████| 23/23 [00:00<00:00, 55698.03it/s]
  -  Found 23 images
Bucket sizes for /home/ubuntu/ai-toolkit/dataset/v1:
448x576: 3 files
192x256: 1 files
576x448: 1 files
384x640: 3 files
448x512: 3 files
128x192: 1 files
512x512: 1 files
320x512: 1 files
640x384: 1 files
384x576: 4 files
320x448: 1 files
192x320: 1 files
256x384: 1 files
64x192: 1 files
14 buckets made
Caching latents for /home/ubuntu/ai-toolkit/dataset/v1
 - Saving latents to disk
Caching latents to disk: 100%|████████████████████████████████████| 23/23 [00:00<00:00, 8785.08it/s]
Dataset: /home/ubuntu/ai-toolkit/dataset/v1
  -  Preprocessing image dimensions
100%|████████████████████████████████████████████████████████████| 23/23 [00:00<00:00, 61367.04it/s]
  -  Found 23 images
Bucket sizes for /home/ubuntu/ai-toolkit/dataset/v1:
576x768: 2 files
192x256: 1 files
704x512: 1 files
384x704: 2 files
576x640: 1 files
128x192: 1 files
640x832: 1 files
576x960: 1 files
704x768: 2 files
320x512: 1 files
960x512: 1 files
512x576: 1 files
576x704: 1 files
576x896: 3 files
320x448: 1 files
192x320: 1 files
256x384: 1 files
64x192: 1 files
18 buckets made
Caching latents for /home/ubuntu/ai-toolkit/dataset/v1
 - Saving latents to disk
Caching latents to disk: 100%|███████████████████████████████████| 23/23 [00:00<00:00, 32657.07it/s]
Dataset: /home/ubuntu/ai-toolkit/dataset/v1
  -  Preprocessing image dimensions
100%|████████████████████████████████████████████████████████████| 23/23 [00:00<00:00, 67935.91it/s]
  -  Found 23 images
Bucket sizes for /home/ubuntu/ai-toolkit/dataset/v1:
576x768: 2 files
192x256: 1 files
704x512: 1 files
384x704: 2 files
576x640: 1 files
128x192: 1 files
896x1152: 1 files
576x1024: 1 files
960x1024: 1 files
320x512: 1 files
1344x768: 1 files
704x768: 1 files
512x576: 1 files
576x704: 1 files
768x1280: 1 files
704x1024: 1 files
640x1024: 1 files
320x448: 1 files
192x320: 1 files
256x384: 1 files
64x192: 1 files
21 buckets made
Caching latents for /home/ubuntu/ai-toolkit/dataset/v1
 - Saving latents to disk
Caching latents to disk: 100%|███████████████████████████████████| 23/23 [00:00<00:00, 34257.45it/s]
Generating baseline samples before training
Generating Images:   0%|                                                     | 0/11 [00:00<?, ?it/s]flux prompt ["[trigger] holding a sign that says 'I LOVE PROMPTS!'"]
flux prompt [False]
Error running job: text input must be of type `str` (single example), `List[str]` (batch or single pretokenized example) or `List[List[str]]` (batch of pretokenized examples).

========================================
Result:
 - 0 completed jobs
 - 1 failure
========================================
Traceback (most recent call last):
  File "/home/ubuntu/ai-toolkit/run.py", line 90, in <module>
    main()
  File "/home/ubuntu/ai-toolkit/run.py", line 86, in main
    raise e
  File "/home/ubuntu/ai-toolkit/run.py", line 78, in main
    job.run()
  File "/home/ubuntu/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
    process.run()
  File "/home/ubuntu/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1593, in run
    self.sample(self.step_num)
  File "/home/ubuntu/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 275, in sample
    self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler)
  File "/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/ai-toolkit/toolkit/stable_diffusion_model.py", line 1112, in generate_images
    unconditional_embeds = self.encode_prompt(
                           ^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/ai-toolkit/toolkit/stable_diffusion_model.py", line 2006, in encode_prompt
    prompt_embeds, pooled_prompt_embeds = train_tools.encode_prompts_flux(
                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/ai-toolkit/toolkit/train_tools.py", line 537, in encode_prompts_flux
    text_inputs = tokenizer[0](
                  ^^^^^^^^^^^^^
  File "/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 3024, in __call__
    encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ubuntu/ai-toolkit/venv/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 3084, in _call_one
    raise ValueError(
ValueError: text input must be of type `str` (single example), `List[str]` (batch or single pretokenized example) or `List[List[str]]` (batch of pretokenized examples).