diff --git a/candle-examples/examples/flux/t5_tokenizer.py b/candle-examples/examples/flux/t5_tokenizer.py new file mode 100644 index 0000000000..cebd0624bf --- /dev/null +++ b/candle-examples/examples/flux/t5_tokenizer.py @@ -0,0 +1,6 @@ +from transformers import AutoModelForCausalLM, AutoTokenizer + +BASE_MODEL = "google/t5-v1_1-xxl" +tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) +# The tokenizer will be saved in /tmp/tokenizer/tokenizer.json +tokenizer.save_pretrained("/tmp/tokenizer/")