t2v_lora_low_vram.yaml

# =============================================================================
# LTX-2 Text-to-Video LoRA Training Configuration (Low VRAM)
# =============================================================================
#
# This is a memory-optimized variant of the standard text-to-video LoRA config.
# It uses 8-bit optimizer, int8 quantization, and reduced LoRA rank to minimize
# GPU memory usage while maintaining good training quality.
#
# Memory optimizations applied:
#   - 8-bit AdamW optimizer (reduces optimizer state memory by ~75%)
#   - INT8 model quantization (reduces model memory by ~50%)
#   - Lower LoRA rank (16 vs 32, reduces trainable parameters)
#   - Gradient checkpointing enabled
#
# Recommended for GPUs with 32GB VRAM (e.g., RTX 5090).
#
# Use this configuration when you want to:
#   - Fine-tune LTX-2 on your own video dataset with limited GPU memory
#   - Train joint audio-video generation from text prompts
#   - Create custom video generation styles or audiovisual concepts
#
# Dataset structure:
#   preprocessed_data_root/
#   ├── latents/        # Video latents (VAE-encoded videos)
#   ├── conditions/     # Text embeddings for each video
#   └── audio_latents/  # Audio latents (VAE-encoded audio)
#
# =============================================================================

# -----------------------------------------------------------------------------
# Model Configuration
# -----------------------------------------------------------------------------
# Specifies the base model to fine-tune and the training mode.
model:
  # The LTX model checkpoint. A unified checkpoint is one .safetensors holding every
  # component; a split pack ships one file per component, and this names its transformer.
  model_path: "path/to/ltx-2-model.safetensors"

  # The Gemma text encoder: a model directory for a unified checkpoint, or the packed
  # text-encoder .safetensors for a split pack.
  text_encoder_path: "path/to/gemma-text-encoder"

  # VAE components. Leave these out for a unified checkpoint — they default to model_path.
  # A split pack's transformer carries no VAE weights, so there they are required:
  # video_vae_path: "path/to/vae/ltx-2.5-video-vae-bf16.safetensors"
  # audio_vae_path: "path/to/vae/ltx-2.5-audio-vae-bf16.safetensors"

  # Training mode: "lora" for efficient adapter training, "full" for full fine-tuning
  # LoRA is recommended for most use cases (faster, less memory, prevents overfitting)
  training_mode: "lora"

  # Optional: Path to resume training from a checkpoint
  # Can be a checkpoint file (.safetensors) or directory (uses latest checkpoint)
  load_checkpoint: null

# -----------------------------------------------------------------------------
# LoRA Configuration
# -----------------------------------------------------------------------------
# Controls the Low-Rank Adaptation parameters for efficient fine-tuning.
# Using a lower rank (16) to reduce trainable parameters and memory usage.
# This still provides good capacity for many fine-tuning tasks.
lora:
  # Rank of the LoRA matrices (higher = more capacity but more parameters)
  # Typical values: 8, 16, 32, 64. Using 16 for low VRAM configuration.
  rank: 16

  # Alpha scaling factor (usually set equal to rank)
  # The effective scaling is alpha/rank, so alpha=rank means scaling of 1.0
  alpha: 16

  # Dropout probability for LoRA layers (0.0 = no dropout)
  # Can help with regularization if overfitting occurs
  dropout: 0.0

  # Which transformer modules to apply LoRA to
  # The LTX-2 transformer has separate attention and FFN blocks for video and audio:
  #
  # VIDEO MODULES:
  #   - attn1.to_k, attn1.to_q, attn1.to_v, attn1.to_out.0  (video self-attention)
  #   - attn2.to_k, attn2.to_q, attn2.to_v, attn2.to_out.0  (video cross-attention to text)
  #   - ff.net.0.proj, ff.net.2                             (video feed-forward)
  #
  # AUDIO MODULES:
  #   - audio_attn1.to_k, audio_attn1.to_q, audio_attn1.to_v, audio_attn1.to_out.0  (audio self-attention)
  #   - audio_attn2.to_k, audio_attn2.to_q, audio_attn2.to_v, audio_attn2.to_out.0  (audio cross-attention to text)
  #   - audio_ff.net.0.proj, audio_ff.net.2                                         (audio feed-forward)
  #
  # AUDIO-VIDEO CROSS-ATTENTION MODULES (for cross-modal interaction):
  #   - audio_to_video_attn.to_k, audio_to_video_attn.to_q, audio_to_video_attn.to_v, audio_to_video_attn.to_out.0
  #       (Q from video, K/V from audio - allows video to attend to audio features)
  #   - video_to_audio_attn.to_k, video_to_audio_attn.to_q, video_to_audio_attn.to_v, video_to_audio_attn.to_out.0
  #       (Q from audio, K/V from video - allows audio to attend to video features)
  #
  # Using short patterns like "to_k" matches ALL attention modules (video, audio, and cross-modal).
  # For audio-video training, this is the recommended approach.
  target_modules:
    # Attention layers (matches both video and audio branches)
    - "to_k"
    - "to_q"
    - "to_v"
    - "to_out.0"
    # Uncomment below to also train feed-forward layers (can increase the LoRA's capacity):
    # - "ff.net.0.proj"
    # - "ff.net.2"
    # - "audio_ff.net.0.proj"
    # - "audio_ff.net.2"

# -----------------------------------------------------------------------------
# Training Strategy Configuration
# -----------------------------------------------------------------------------
# Defines the training approach using the unified flexible strategy.
# This configuration trains both video and audio generation from text prompts.
training_strategy:
  # Strategy name: "flexible" for the unified conditioning framework
  # Supports all training modes (T2V, I2V, V2V, A2V, V2A, etc.) through
  # modality-specific configuration blocks.
  name: "flexible"

  # Video modality configuration
  # When is_generated is true, the model learns to generate (denoise) video
  video:
    # Whether the model generates video (true) or uses it as frozen conditioning (false)
    is_generated: true
    # Directory name (within preprocessed_data_root) containing video latents
    latents_dir: "latents"

  # Audio modality configuration
  # When is_generated is true, the model learns to generate (denoise) audio
  audio:
    # Whether the model generates audio (true) or uses it as frozen conditioning (false)
    is_generated: true
    # Directory name (within preprocessed_data_root) containing audio latents
    latents_dir: "audio_latents"

# -----------------------------------------------------------------------------
# Optimization Configuration
# -----------------------------------------------------------------------------
# Controls the training optimization parameters.
optimization:
  # Learning rate for the optimizer
  # Typical range for LoRA: 1e-5 to 1e-4
  learning_rate: 1e-4

  # Total number of training steps
  steps: 2000

  # Batch size per GPU
  # Reduce if running out of memory
  batch_size: 1

  # Number of gradient accumulation steps
  # Effective batch size = batch_size * gradient_accumulation_steps * num_gpus
  gradient_accumulation_steps: 1

  # Maximum gradient norm for clipping (helps training stability)
  max_grad_norm: 1.0

  # Optimizer type: "adamw" (standard) or "adamw8bit" (memory-efficient)
  # Using 8-bit AdamW to reduce optimizer state memory by ~75%
  optimizer_type: "adamw8bit"

  # Learning rate scheduler type
  # Options: "constant", "linear", "cosine", "cosine_with_restarts", "polynomial"
  scheduler_type: "linear"

  # Additional scheduler parameters (depends on scheduler_type)
  scheduler_params: { }

  # Enable gradient checkpointing to reduce memory usage
  # Recommended for training with limited GPU memory
  enable_gradient_checkpointing: true

# -----------------------------------------------------------------------------
# Acceleration Configuration
# -----------------------------------------------------------------------------
# Hardware acceleration and memory optimization settings.
acceleration:
  # Mixed precision training mode
  # Options: "no" (fp32), "fp16" (half precision), "bf16" (bfloat16, recommended)
  mixed_precision_mode: "bf16"

  # Model quantization for reduced memory usage
  # Options: null (none), "int8-quanto", "int4-quanto", "int2-quanto", "fp8-quanto", "fp8uz-quanto"
  # Using INT8 quantization to reduce base model memory consumption by ~50%
  quantization: "int8-quanto"

  # Load text encoder in 8-bit precision to save memory
  # Useful when GPU memory is limited
  load_text_encoder_in_8bit: true

  # Offload optimizer state to CPU during validation video sampling and restore it after.
  # Frees VRAM for the VAE decoder when optimizer state is large (full fine-tune, high-rank
  # LoRA). No effect under FSDP (sharded state).
  offload_optimizer_during_validation: true

# -----------------------------------------------------------------------------
# Data Configuration
# -----------------------------------------------------------------------------
# Specifies the training data location and loading parameters.
data:
  # Root directory containing preprocessed training data
  # Should contain: latents/, conditions/, and audio_latents/
  preprocessed_data_root: "/path/to/preprocessed/data"

  # Number of worker processes for data loading
  # Used for parallel data loading to speed up data loading
  num_dataloader_workers: 2

# -----------------------------------------------------------------------------
# Validation Configuration
# -----------------------------------------------------------------------------
# Controls validation sampling during training.
# NOTE: Validation sampling use simplified inference pipelines and prioritizes speed over
# maximum quality. For production-quality inference, use `packages/ltx-pipelines`.
validation:
  # Validation samples — each sample describes a self-contained generation request.
  # Use 'conditions' to add conditioning (first_frame, prefix, suffix, reference, video_to_audio, etc.)
  # See docs/configuration-reference.md#validation-condition-types for the full list of condition types.
  samples:
    - prompt: >-
        A woman with long brown hair sits at a wooden desk in a cozy home office, typing on a
        laptop while occasionally glancing at notes beside her. Soft natural light streams through
        a large window, casting warm shadows across the room. She pauses to take a sip from a
        ceramic mug, then continues working with focused concentration. The audio captures the
        gentle clicking of keyboard keys, the soft rustle of papers, and ambient room tone with
        occasional distant bird chirps from outside.
    - prompt: >-
        A chef in a white uniform stands in a professional kitchen, carefully plating a gourmet
        dish with precise movements. Steam rises from freshly cooked vegetables as he arranges
        them with tweezers. The stainless steel surfaces gleam under bright overhead lights, and
        various pots simmer on the stove behind him. The audio features the sizzling of pans,
        the clinking of utensils against plates, and the ambient hum of kitchen ventilation.

  # Negative prompt to avoid unwanted artifacts
  negative_prompt: >-
    blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise,
    grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones,
    deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands,
    wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of
    field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent
    lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny
    valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction,
    mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise,
    off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward
    pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting,
    inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts.

  # Output video dimensions [width, height, frames]
  # Width and height must be divisible by the VAE spatial factor (32 for the default VAE).
  # Frames must satisfy frames % T == 1, where T is the VAE temporal factor
  # (8 for the default VAE, i.e. 1, 9, 17, 25, ...).
  video_dims: [ 576, 576, 49 ]

  # Frame rate for generated videos
  frame_rate: 24.0

  # Random seed for reproducible validation outputs
  seed: 42

  # Number of denoising steps for validation inference
  # Higher values = better quality but slower generation
  inference_steps: 30

  # Generate validation videos every N training steps
  # Set to null to disable validation during training
  interval: 100

  # Video CFG scale for validation sampling
  video_cfg_scale: 3.0

  # Audio CFG scale for validation sampling
  audio_cfg_scale: 7.0

  # Video STG scale; set to 0.0 to disable video STG
  video_stg_scale: 1.0

  # Audio STG scale; set to 0.0 to disable audio STG
  audio_stg_scale: 1.0

  # Transformer blocks perturbed for STG
  stg_blocks: [28]

  # Guidance variance rescale; set to 0.0 to disable
  guidance_rescale: 0.7

  # Video modality guidance scale for audio-to-video isolation guidance
  video_modality_guidance_scale: 3.0

  # Audio modality guidance scale for video-to-audio isolation guidance
  audio_modality_guidance_scale: 3.0

  # Whether to generate audio in validation samples
  # Independent of training_strategy audio modality settings — you can generate audio
  # in validation even when not training the audio branch
  generate_audio: true

  # Skip validation at the beginning of training (step 0)
  skip_initial_validation: false

# -----------------------------------------------------------------------------
# Checkpoint Configuration
# -----------------------------------------------------------------------------
# Controls model checkpoint saving during training.
checkpoints:
  # Save a checkpoint every N steps
  # Set to null to disable intermediate checkpoints
  interval: 250

  # Number of most recent checkpoints to keep
  # Set to -1 to keep all checkpoints
  keep_last_n: -1

  # Precision to use when saving checkpoint weights
  # Options: "bfloat16" (default, smaller files) or "float32" (full precision)
  precision: "bfloat16"

# -----------------------------------------------------------------------------
# Flow Matching Configuration
# -----------------------------------------------------------------------------
# Parameters for the flow matching training objective.
flow_matching:
  # Timestep sampling mode
  # "shifted_logit_normal" is recommended for LTX-2 models
  timestep_sampling_mode: "shifted_logit_normal"

  # Additional parameters for timestep sampling
  timestep_sampling_params: { }

# -----------------------------------------------------------------------------
# Hugging Face Hub Configuration
# -----------------------------------------------------------------------------
# Settings for uploading trained models to the Hugging Face Hub.
hub:
  # Whether to push the trained model to the Hub
  push_to_hub: false

  # Repository ID on Hugging Face Hub (e.g., "username/my-lora-model")
  # Required if push_to_hub is true
  hub_model_id: null

# -----------------------------------------------------------------------------
# Weights & Biases Configuration
# -----------------------------------------------------------------------------
# Settings for experiment tracking with W&B.
wandb:
  # Enable W&B logging
  enabled: false

  # W&B project name
  project: "ltx-2-trainer"

  # W&B username or team (null uses default account)
  entity: null

  # Tags to help organize runs
  tags: [ "ltx2", "lora", "t2v", "low-vram" ]

  # Log validation media (video/audio) to W&B
  log_validation_videos: true

# -----------------------------------------------------------------------------
# General Configuration
# -----------------------------------------------------------------------------
# Global settings for the training run.

# Random seed for reproducibility
seed: 42

# Directory to save outputs (checkpoints, validation videos, logs)
output_dir: "outputs/t2v_lora_low_vram"