Skip to content

Commit

Permalink
Use llama2-instruct style prompting
Browse files Browse the repository at this point in the history
this also works well with mistral-7b-instruct

See https://github.com/facebookresearch/llama/blob/v2/llama/generation.py
  • Loading branch information
jepler committed Sep 29, 2023
1 parent ddc6d92 commit 00aa37e
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions src/chap/backends/llama_cpp.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ class Parameters:
url: str = "http://localhost:8080/completion"
"""The URL of a llama.cpp server's completion endpoint."""

start_prompt: str = """<s>[INST] <<SYS>>\n"""
after_system: str = "\n<</SYS>>\n\n"
after_user: str = """ [/INST] """
after_assistant: str = """ </s><s>[INST] """

def __init__(self):
self.parameters = self.Parameters()

Expand All @@ -26,19 +31,19 @@ def __init__(self):

def make_full_query(self, messages, max_query_size):
del messages[1:-max_query_size]
rows = []
result = [self.parameters.start_prompt]
for m in messages:
content = (m.content or "").strip()
if not content:
continue
result.append(content)
if m.role == "system":
rows.append(f"ASSISTANT'S RULE: {content}\n")
result.append(self.parameters.after_system)
elif m.role == "assistant":
rows.append(f"ASSISTANT: {content}\n")
result.append(self.parameters.after_assistant)
elif m.role == "user":
rows.append(f"USER: {content}")
rows.append("ASSISTANT: ")
full_query = ("\n".join(rows)).rstrip()
result.append(self.parameters.after_user)
full_query = "".join(result)
return full_query

async def aask(
Expand Down

0 comments on commit 00aa37e

Please sign in to comment.