diff --git "a/logs/0_log.txt" "b/logs/0_log.txt" new file mode 100644--- /dev/null +++ "b/logs/0_log.txt" @@ -0,0 +1,2359 @@ +Running 1 job +{ + "type": "ui_trainer", + "training_folder": "/home/featurize/ai-toolkit/output", + "sqlite_db_path": "/home/featurize/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 250, + "max_step_saves_to_keep": 4000, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/home/featurize/ai-toolkit/datasets/Xiang_Handsome_Flux_SRPO_Keye_EN_Captioned", + "control_path": null, + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "do_i2v": true, + "flip_x": false, + "flip_y": false + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 30000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "sigmoid", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1 + }, + "model": { + "name_or_path": "black-forest-labs/FLUX.1-dev", + "quantize": true, + "qtype": "qfloat8", + "quantize_te": true, + "qtype_te": "qfloat8", + "arch": "flux", + "low_vram": false, + "model_kwargs": {} + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "width": 1024, + "height": 1024, + "samples": [ + { + "prompt": "Wang Xiang , Photograph of a young Asian male with fair skin and black, slightly tousled hair. He has a slender build and is wearing a dark blue hooded sweatshirt with large white letters \"KE\" on the chest. His hands are crossed and resting on a white horizontal bar in front of him. He has a soft, neutral expression with slightly parted lips and dark brown eyes. The background is a bright, overexposed window, creating a stark contrast with his dark clothing. The image has a soft, natural light and a slightly grainy texture." + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 25, + "num_frames": 1, + "fps": 1 + } +} +Using SQLite database at /home/featurize/ai-toolkit/aitk_db.db +Job ID: "7df47ed8-0cea-4baf-80d0-f5dcea074624" + +############################################# +# Running job: Flux_Xiang_lora +############################################# + + +Running 1 process +Loading Flux model +Loading transformer + config.json: 0%| | 0.00/378 [00:00 + File "/home/featurize/ai-toolkit/run.py", line 120, in + main()main() + + File "/home/featurize/ai-toolkit/run.py", line 108, in main + File "/home/featurize/ai-toolkit/run.py", line 108, in main + raise eraise e + + File "/home/featurize/ai-toolkit/run.py", line 96, in main + File "/home/featurize/ai-toolkit/run.py", line 96, in main + job.run()job.run() + + File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + File "/home/featurize/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run()process.run() + + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1990, in run + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1990, in run + self.sample(self.step_num)self.sample(self.step_num) + + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 270, in sample + File "/home/featurize/ai-toolkit/extensions_built_in/sd_trainer/UITrainer.py", line 270, in sample + super().sample(step, is_first)super().sample(step, is_first) + + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 363, in sample + File "/home/featurize/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 363, in sample + self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler)self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler) + + File "/environment/miniconda3/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + File "/environment/miniconda3/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + return func(*args, **kwargs)return func(*args, **kwargs) + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + File "/home/featurize/ai-toolkit/toolkit/stable_diffusion_model.py", line 1156, in generate_images + File "/home/featurize/ai-toolkit/toolkit/stable_diffusion_model.py", line 1156, in generate_images + self.save_device_state()self.save_device_state() + + File "/home/featurize/ai-toolkit/toolkit/stable_diffusion_model.py", line 2906, in save_device_state + File "/home/featurize/ai-toolkit/toolkit/stable_diffusion_model.py", line 2906, in save_device_state + te_has_grad = encoder.encoder.block[0].layer[0].SelfAttention.q.weight.requires_gradte_has_grad = encoder.encoder.block[0].layer[0].SelfAttention.q.weight.requires_grad + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + File "/environment/miniconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1940, in __getattr__ + File "/environment/miniconda3/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1940, in __getattr__ + raise AttributeError(raise AttributeError( + +AttributeErrorAttributeError: : 'FakeTextEncoder' object has no attribute 'encoder''FakeTextEncoder' object has no attribute 'encoder' +