LiquidAI

/

LFM2-VL-450M

like 147

Follow

Liquid AI 3.32k

I have been trying to replicate the SFT notebook on a dataset of mine.

The dataset has very large panoramic images.

I keep getting the error above, regardless of the sizing of sequence lengths, maximum image tokens.

import asyncio
import os

import torch
import transformers
import trl
from peft import LoraConfig
from transformers import (
    AutoModelForImageTextToText, 
    AutoProcessor, 
    EarlyStoppingCallback, 
    IntervalStrategy
)
from trl import SFTConfig, SFTTrainer

from common.dataset import load_isiqa_2019_dataset


os.environ["WANDB_DISABLED"] = "true"


if __name__ == "__main__":
    processor = AutoProcessor.from_pretrained(
        "LiquidAI/LFM2-VL-450M",
        trust_remote_code=True,
        do_image_splitting=True,
        min_image_tokens=64,
        max_image_tokens=256,
    )

    model = AutoModelForImageTextToText.from_pretrained(
        "LiquidAI/LFM2-VL-450M",
        trust_remote_code=True,
        device_map="cuda:0",
        torch_dtype="bfloat16",
    )

    dataset = asyncio.run(load_isiqa_2019_dataset())

    def get_collator(processor):
        def collator(x):
            batch = processor.apply_chat_template(
                x, tokenize=True, padding=True, return_dict=True, return_tensors="pt"
            )

            y = batch["input_ids"].clone()
            y[y == processor.tokenizer.pad_token_id] = -100

            batch["labels"] = y
            return batch
        return collator

    config = SFTConfig(
        output_dir=os.path.join("experiments", "lfm2-piqa"),
        num_train_epochs=4,
        per_device_train_batch_size=1,
        gradient_accumulation_steps=32,
        learning_rate=5e-4,
        warmup_ratio=0.1,
        weight_decay=0.01,
        logging_steps=10,
        optim="adamw_torch_8bit",
        gradient_checkpointing=True,
        max_length=1024,
        eval_strategy=IntervalStrategy.STEPS,
        eval_steps=10,
        dataset_kwargs={"skip_prepare_dataset": True},
        report_to=["trackio"],
        metric_for_best_model="eval_loss",
        load_best_model_at_end=True,
    )

    trainer = SFTTrainer(
        model=model,
        args=config,
        train_dataset=dataset["training"],
        eval_dataset=dataset["validation"],
        data_collator=get_collator(processor),
        processing_class=processor.tokenizer,
        # compute_metrics=compute_metrics,  # TODO
        peft_config=LoraConfig(
            lora_alpha=64,
            lora_dropout=0.05,
            r=64,
            bias="none",
            target_modules=[
                "q_proj", 
                "v_proj", 
                "fc1", 
                "fc2", 
                "linear",
                "gate_proj", 
                "up_proj", 
                "down_proj",
            ],
            task_type="CAUSAL_LM",
        ),
        callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
    )

    trainer.train()

    model = model.merge_and_unload()

    model.save_pretrained(config.output_dir)  
    processor.save_pretrained(config.output_dir)