Merge pull request #16 from winzig/openai-new-embeddings

Adding support for new OpenAI embeddings
aiqinxuancai · Feb 8, 2024 · e0b73d4 · e0b73d4
2 parents 414ece2 + c434f1e
commit e0b73d4
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/TiktokenSharp/Services/EncodingManager.cs b/TiktokenSharp/Services/EncodingManager.cs
@@ -61,6 +61,8 @@ public static EncodingManager Instance
                                                             { "code-davinci-edit-001", "p50k_edit" },
                                                             // embeddings
                                                             { "text-embedding-ada-002", "cl100k_base" },
+                                                            { "text-embedding-3-large", "cl100k_base" },
+                                                            { "text-embedding-3-small", "cl100k_base" },
                                                             // old embeddings
                                                             { "text-similarity-davinci-001", "r50k_base" },
                                                             { "text-similarity-curie-001", "r50k_base" },