diff --git a/flake.nix b/flake.nix index 0017472..8ff4af2 100644 --- a/flake.nix +++ b/flake.nix @@ -17,9 +17,11 @@ inherit (pkgs) callPackage mkShell writeShellScriptBin; - models = callPackage ./models.nix { }; + models = import ./models.nix; - llama-cpp = callPackage ./llama-cpp.nix { inherit models; }; + model-pkgs = callPackage ./model-pkgs.nix { inherit models; }; + + llama-cpp = callPackage ./llama-cpp.nix { inherit model-pkgs; }; in rec { diff --git a/llama-cpp.nix b/llama-cpp.nix index 08e6fef..68eca6a 100644 --- a/llama-cpp.nix +++ b/llama-cpp.nix @@ -1,6 +1,6 @@ { writeShellScriptBin , llama-cpp -, models +, model-pkgs }: let @@ -17,7 +17,7 @@ let # For now all models in this repo are for llama-cpp so grab all of them (filter (s: s != "override" && s != "overrideDerivation") - (attrNames models) + (attrNames model-pkgs) ) ++ [ null ] # this generates a package without an embedded model that can be passed models at runtime ; @@ -32,7 +32,7 @@ let (acc: model: acc // ( let overrides = if mode == "cuda" then { cudaSupport = true; } else { }; - args = if model != null then "--model ${models.${model}}" else ""; + args = if model != null then "--model ${model-pkgs.${model}}" else ""; bin = "${llama-cpp.override overrides}/bin/llama-cpp-${cmd}"; name = if model != null then diff --git a/model-pkgs.nix b/model-pkgs.nix new file mode 100644 index 0000000..b1e9d72 --- /dev/null +++ b/model-pkgs.nix @@ -0,0 +1,63 @@ +{ fetchurl +, models +}: + +let + + inherit (builtins) attrNames foldl' replaceStrings; + + fetchHuggingFaceModel = { owner, repo, commit, model, hash }: + fetchurl { + url = "https://huggingface.co/${owner}/${repo}/resolve/${commit}/${model}?download=true"; + inherit hash; + }; + + # HuggingFace model list (objects with parameters for fetchHuggingFaceModel function) + hf-model-list = + foldl' + (acc: owner: acc ++ + (foldl' + (acc: repo: acc ++ + (foldl' + (acc: model: acc ++ [ + { + inherit owner repo model; + commit = models.HF.${owner}."${repo}".commit; + hash = models.HF.${owner}."${repo}".models."${model}"; + } + ]) + [ ] + (attrNames models.HF.${owner}.${repo}.models) + ) + ) + [ ] + (attrNames models.HF.${owner}) + ) + ) + [ ] + (attrNames models.HF); + + # HuggingFace model packages + hf-model-packages = + foldl' + (acc: pkg: acc // (with pkg; + let + repoStr = replaceStrings [ "." ] [ "_" ] repo; + modelStr = replaceStrings [ "." ] [ "_" ] model; + in + { + "HF__${owner}__${repoStr}__${modelStr}" = fetchHuggingFaceModel { + inherit owner repo model; + commit = models.HF.${owner}.${repo}.commit; + hash = models.HF.${owner}.${repo}.models."${model}"; + }; + } + )) + { } + hf-model-list; + + packages = hf-model-packages; + +in + +packages diff --git a/models.nix b/models.nix index 149112c..18826bf 100644 --- a/models.nix +++ b/models.nix @@ -1,90 +1,27 @@ -{ fetchurl -}: - -let - - inherit (builtins) attrNames foldl' replaceStrings; - - fetchHuggingFaceModel = { owner, repo, commit, model, hash }: - fetchurl { - url = "https://huggingface.co/${owner}/${repo}/resolve/${commit}/${model}?download=true"; - inherit hash; - }; - - models = { - # HuggingFace - HF = { - TheBloke = { - "CapybaraHermes-2.5-Mistral-7B-GGUF" = { - commit = "8bea614edd9a2d5d9985a6e6c1ecc166261cacb8"; - models = { - "capybarahermes-2.5-mistral-7b.Q4_K_M.gguf" = "sha256-68zLv7A+UIiM14B0XdYcHz2gLTheeNvCxQYtzcWlycg="; - }; +{ + # HuggingFace + HF = { + TheBloke = { + "CapybaraHermes-2.5-Mistral-7B-GGUF" = { + commit = "8bea614edd9a2d5d9985a6e6c1ecc166261cacb8"; + models = { + "capybarahermes-2.5-mistral-7b.Q4_K_M.gguf" = "sha256-68zLv7A+UIiM14B0XdYcHz2gLTheeNvCxQYtzcWlycg="; }; - "Llama-2-7B-Chat-GGUF" = { - commit = "191239b3e26b2882fb562ffccdd1cf0f65402adb"; - models = { - "llama-2-7b-chat.Q2_K.gguf" = "sha256-wN0wTXYejgXQgswpAtdiSn+HhY/fqk7wmDMP/nZ/8NM="; - "llama-2-7b-chat.Q4_K_M.gguf" = "sha256-CKVWbWHXy2tCDD5Dh6OeAHjh8v5fBV86A4hzhTBNS/o="; - "llama-2-7b-chat.Q5_K_M.gguf" = "sha256-4LmZIM9HuUx40vsGoezrnteVF236P3/qxkYp8bUrmX8="; - }; + }; + "Llama-2-7B-Chat-GGUF" = { + commit = "191239b3e26b2882fb562ffccdd1cf0f65402adb"; + models = { + "llama-2-7b-chat.Q2_K.gguf" = "sha256-wN0wTXYejgXQgswpAtdiSn+HhY/fqk7wmDMP/nZ/8NM="; + "llama-2-7b-chat.Q4_K_M.gguf" = "sha256-CKVWbWHXy2tCDD5Dh6OeAHjh8v5fBV86A4hzhTBNS/o="; + "llama-2-7b-chat.Q5_K_M.gguf" = "sha256-4LmZIM9HuUx40vsGoezrnteVF236P3/qxkYp8bUrmX8="; }; - "Llama-2-13B-chat-GGUF" = { - commit = "4458acc949de0a9914c3eab623904d4fe999050a"; - models = { - "llama-2-13b-chat.Q5_K_M.gguf" = "sha256-7zbgkCQAQPlzJXWMGtjiPzgBRmqO7OOp6sLSLZQvVIo="; - }; + }; + "Llama-2-13B-chat-GGUF" = { + commit = "4458acc949de0a9914c3eab623904d4fe999050a"; + models = { + "llama-2-13b-chat.Q5_K_M.gguf" = "sha256-7zbgkCQAQPlzJXWMGtjiPzgBRmqO7OOp6sLSLZQvVIo="; }; }; }; }; - - # # HuggingFace model list (objects with parameters for fetchHuggingFaceModel function) - hfModelList = - foldl' - (acc: owner: acc ++ - (foldl' - (acc: repo: acc ++ - (foldl' - (acc: model: acc ++ [ - { - inherit owner repo model; - commit = models.HF.${owner}."${repo}".commit; - hash = models.HF.${owner}."${repo}".models."${model}"; - } - ]) - [ ] - (attrNames models.HF.${owner}.${repo}.models) - ) - ) - [ ] - (attrNames models.HF.${owner}) - ) - ) - [ ] - (attrNames models.HF); - - # HuggingFace model packages - hfModelPackages = - foldl' - (acc: pkg: acc // (with pkg; - let - repoStr = replaceStrings [ "." ] [ "_" ] repo; - modelStr = replaceStrings [ "." ] [ "_" ] model; - in - { - "HF__${owner}__${repoStr}__${modelStr}" = fetchHuggingFaceModel { - inherit owner repo model; - commit = models.HF.${owner}.${repo}.commit; - hash = models.HF.${owner}.${repo}.models."${model}"; - }; - } - )) - { } - hfModelList; - - packages = hfModelPackages; - -in - -packages +}