Merge pull request #38 from jepler/misc-updates

Update anthropic back-end & other small changes

src/chap/backends/anthropic.py

@@ -17,7 +17,7 @@ class Anthropic(AutoAskMixin):
  @dataclass
  class Parameters:
  url: str = "https://api.anthropic.com"
- model: str = "claude-3-sonnet-20240229"
+ model: str = "claude-3-5-sonnet-20240620"
  max_new_tokens: int = 1000
 
  def __init__(self) -> None:
@@ -94,5 +94,5 @@ def get_key(cls) -> str:
 
 
 def factory() -> Backend:
- """Uses the huggingface text-generation-interface web API"""
+ """Uses the anthropic text-generation-interface web API"""
  return Anthropic()

src/chap/backends/llama_cpp.py

@@ -18,10 +18,16 @@ class Parameters:
  url: str = "http://localhost:8080/completion"
  """The URL of a llama.cpp server's completion endpoint."""
 
- start_prompt: str = "<s>"
- system_format: str = "<<SYS>>{}<</SYS>>"
- user_format: str = " [INST] {} [/INST]"
- assistant_format: str = " {}</s>"
+ start_prompt: str = "<|begin_of_text|>"
+ system_format: str = (
+ "<|start_header_id|>system<|end_header_id|>\n\n{}<|eot_id|>"
+ )
+ user_format: str = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>"
+ assistant_format: str = (
+ "<|start_header_id|>assistant<|end_header_id|>\n\n{}<|eot_id|>"
+ )
+ end_prompt: str = "<|start_header_id|>assistant<|end_header_id|>\n\n"
+ stop: str | None = None
 
  def __init__(self) -> None:
  super().__init__()
@@ -59,7 +65,7 @@ async def aask(
  params = {
  "prompt": self.make_full_query(session + [User(query)], max_query_size),
  "stream": True,
- "stop": ["</s>", "<s>", "[INST]"],
+ "stop": ["</s>", "<s>", "[INST]", "<|eot_id|>"],
  }
  new_content: list[str] = []
  try:

src/chap/core.py

@@ -129,7 +129,11 @@ def configure_api_from_environment(
  setattr(api.parameters, field.name, tv)
 
 
-def get_api(ctx: click.Context, name: str = "openai_chatgpt") -> Backend:
+def get_api(ctx: click.Context | None = None, name: str | None = None) -> Backend:
+ if ctx is None:
+ ctx = click.Context(click.Command("chap"))
+ if name is None:
+ name = os.environ.get("CHAP_BACKEND", "openai_chatgpt")
  name = name.replace("-", "_")
  backend = cast(
  Backend, importlib.import_module(f"{__package__}.backends.{name}").factory()