WIP for david

examples/sft/alpaca-llama-fix.yaml

@@ -0,0 +1,55 @@
+# Model configuration
+model:
+  activation_function: silu
+  gradient_checkpointing: true
+  hidden_dim: 4096
+  initializer_range: 0.02
+  intermediate_dim: 11008
+  layer_norm_epsilon: 1.0e-05
+  num_heads: 32
+  num_kv_heads: 32
+  num_layers: 32
+  reference_checkpoint: meta-llama/Llama-2-7b-hf
+  seq_len: 4096
+  type: llama
+  use_bias: false
+  use_layer_norm_weight: false
+
+# Training configuration
+trainer:
+  mp: p=f32,c=bfloat16
+  tracker:
+    type: wandb
+    project: "levanter-sft"
+    tags: ["llama", "sft"]
+  num_train_steps: 1218
+  train_batch_size: 64
+  tensor_parallel_axes: ["mlp", "heads"]
+  fsdp_axis: "embed"
+  batch_axis: "batch"
+  steps_per_eval: 1000
+
+# Optimizer settings
+optimizer:
+  learning_rate: 2e-5
+  weight_decay: 0.0
+  min_lr_ratio: 0.1
+  warmup: 100
+
+# Supervised data configuration
+supervised_data:
+  cache_dir: "gs://levanter-checkpoints/marin/sft_cache/alpaca-olmo"
+  input_field: "instruction"
+  output_field: "output"
+  hf_dataset_name: "tatsu-lab/alpaca"  # Changed from id
+  hf_dataset_split: "train"
+  name: "alpaca"  # Optional metadata
+  tags: ["instruction-tuning"]  # Optional metadata
+  validation_urls: []  # Empty list for no validation files
+
+# Additional settings
+tokenizer: "allenai/OLMo-1B"
+max_tune_length: 2048
+epoch: 3
+
+initialize_from_hf: false

examples/sft/alpaca-llama-sft.yaml

@@ -19,7 +19,7 @@ trainer:
     type: wandb
     project: "levanter-sft"
     tags: ["llama", "sft"]
-  num_train_steps: 1218
+  num_train_steps: 750000
   train_batch_size: 64
   tensor_parallel_axes: ["mlp", "heads"]
   fsdp_axis: "embed"

examples/sft/sft.py

@@ -61,7 +61,8 @@ def train(config: SFTConfig):
             converter = converter.replaced(tokenizer=tokenizer)
 
         model_config = converter.default_config
-
+    elif config.trainer.initialize_from is None:
+        raise ValueError("Must specify either --initialize_from_hf or --initialize_from")
     else:
         converter = None