Merge pull request #43 from matthiasthomas/matthias/add-support-for-l…

…atest-embeddings-models feat: add support for latest embedding models
pkoukk · May 21, 2024 · 5fef437 · 5fef437
2 parents 475cdcd + 14e35cc
commit 5fef437
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -176,7 +176,7 @@ func NumTokensFromMessages(messages []openai.ChatCompletionMessage, model string
 # Available Encodings
  | Encoding name           | OpenAI models                                        |
  | ----------------------- | ---------------------------------------------------- |
- | `cl100k_base`           | `gpt-4`, `gpt-3.5-turbo`, `text-embedding-ada-002`   |
+ | `cl100k_base`           | `gpt-4`, `gpt-3.5-turbo`, `text-embedding-ada-002`, `text-embedding-3-small`, `text-embedding-3-large`   |
  | `p50k_base`             | Codex models, `text-davinci-002`, `text-davinci-003` |
  | `r50k_base` (or `gpt2`) | GPT-3 models like `davinci`                          |
 
@@ -208,6 +208,8 @@ func NumTokensFromMessages(messages []openai.ChatCompletionMessage, model string
 | text-davinci-edit-001        | p50k_edit     |
 | code-davinci-edit-001        | p50k_edit     |
 | text-embedding-ada-002       | cl100k_base   |
+| text-embedding-3-small       | cl100k_base   |
+| text-embedding-3-large       | cl100k_base   |
 | text-similarity-davinci-001  | r50k_base     |
 | text-similarity-curie-001    | r50k_base     |
 | text-similarity-babbage-001  | r50k_base     |

diff --git a/README_zh-hans.md b/README_zh-hans.md
@@ -169,7 +169,7 @@ func NumTokensFromMessages(messages []openai.ChatCompletionMessage, model string
 # available encodings
  | Encoding name           | OpenAI models                                        |
  | ----------------------- | ---------------------------------------------------- |
- | `cl100k_base`           | `gpt-4`, `gpt-3.5-turbo`, `text-embedding-ada-002`   |
+ | `cl100k_base`           | `gpt-4`, `gpt-3.5-turbo`, `text-embedding-ada-002`, 	`text-embedding-3-small`, `text-embedding-3-large`   |
  | `p50k_base`             | Codex models, `text-davinci-002`, `text-davinci-003` |
  | `r50k_base` (or `gpt2`) | GPT-3 models like `davinci`                          |
 
@@ -200,6 +200,8 @@ func NumTokensFromMessages(messages []openai.ChatCompletionMessage, model string
 | text-davinci-edit-001        | p50k_edit     |
 | code-davinci-edit-001        | p50k_edit     |
 | text-embedding-ada-002       | cl100k_base   |
+| text-embedding-3-small       | cl100k_base   |
+| text-embedding-3-large       | cl100k_base   |
 | text-similarity-davinci-001  | r50k_base     |
 | text-similarity-curie-001    | r50k_base     |
 | text-similarity-babbage-001  | r50k_base     |

diff --git a/encoding.go b/encoding.go
@@ -45,6 +45,8 @@ var MODEL_TO_ENCODING = map[string]string{
 	"code-davinci-edit-001": MODEL_P50K_EDIT,
 	// embeddings
 	"text-embedding-ada-002": MODEL_CL100K_BASE,
+	"text-embedding-3-large": MODEL_CL100K_BASE,
+	"text-embedding-3-small": MODEL_CL100K_BASE,
 	// old embeddings
 	"text-similarity-davinci-001":  MODEL_R50K_BASE,
 	"text-similarity-curie-001":    MODEL_R50K_BASE,