ENH: Add support of sglang for llama 3 qwen 2 (xorbitsai#1947)

Valdanitooooo · Jul 27, 2024 · 202e6cb · 202e6cb
1 parent aa51ff2
commit 202e6cb
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/xinference/model/llm/sglang/core.py b/xinference/model/llm/sglang/core.py
@@ -63,11 +63,21 @@ class SGLANGGenerateConfig(TypedDict, total=False):
 except ImportError:
     SGLANG_INSTALLED = False
 
-SGLANG_SUPPORTED_MODELS = ["llama-2", "mistral-v0.1", "mixtral-v0.1"]
+SGLANG_SUPPORTED_MODELS = [
+    "llama-2",
+    "llama-3",
+    "llama-3.1",
+    "mistral-v0.1",
+    "mixtral-v0.1",
+]
 SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
+    "llama-3-instruct",
+    "llama-3.1-instruct",
     "qwen-chat",
     "qwen1.5-chat",
+    "qwen2-instruct",
+    "qwen2-moe-instruct",
     "mistral-instruct-v0.1",
     "mistral-instruct-v0.2",
     "mixtral-instruct-v0.1",