From 1a2474558c88e6d31f5c9a2b1b95664d6b515dae Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Fri, 4 Aug 2023 17:02:19 -0700 Subject: [PATCH 1/3] DEVOPS-471 add paths to allow for using new model directories --- lib/model/fasttext.py | 4 +++- lib/model/generic_transformer.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/model/fasttext.py b/lib/model/fasttext.py index 86a5e97..70b6a0f 100644 --- a/lib/model/fasttext.py +++ b/lib/model/fasttext.py @@ -1,3 +1,4 @@ +import os from typing import Union, Dict, List import fasttext @@ -11,7 +12,8 @@ def __init__(self): """ Load fasttext model (https://huggingface.co/facebook/fasttext-language-identification) """ - model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification", filename="model.bin") + model_directory = os.getenv("MODEL_DIR", "./models") + model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification", filename=f"{model_directory}/model.bin") self.model = fasttext.load_model(model_path) diff --git a/lib/model/generic_transformer.py b/lib/model/generic_transformer.py index 2e8bb56..5fb88b6 100644 --- a/lib/model/generic_transformer.py +++ b/lib/model/generic_transformer.py @@ -1,3 +1,4 @@ +import os from typing import Union, Dict, List from sentence_transformers import SentenceTransformer @@ -11,7 +12,7 @@ def __init__(self, model_name: str): """ self.model = None if model_name: - self.model = SentenceTransformer(model_name) + self.model = SentenceTransformer(model_name, cache_folder=os.getenv("MODEL_DIR", "./models")) def respond(self, docs: Union[List[schemas.Message], schemas.Message]) -> List[schemas.TextOutput]: """ From c19b0595a27595765a17da17766e6d5ac87c5059 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Fri, 4 Aug 2023 17:27:35 -0700 Subject: [PATCH 2/3] updates --- lib/model/fasttext.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/model/fasttext.py b/lib/model/fasttext.py index 70b6a0f..e180f72 100644 --- a/lib/model/fasttext.py +++ b/lib/model/fasttext.py @@ -12,8 +12,7 @@ def __init__(self): """ Load fasttext model (https://huggingface.co/facebook/fasttext-language-identification) """ - model_directory = os.getenv("MODEL_DIR", "./models") - model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification", filename=f"{model_directory}/model.bin") + model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification") self.model = fasttext.load_model(model_path) From 33452aebf070f72d06ad30f34d5a12eb1b5e49ef Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Fri, 4 Aug 2023 18:05:32 -0700 Subject: [PATCH 3/3] just give up on playing with fasttest relocation for the moment, get the main thing fixed... --- lib/model/fasttext.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/model/fasttext.py b/lib/model/fasttext.py index e180f72..86a5e97 100644 --- a/lib/model/fasttext.py +++ b/lib/model/fasttext.py @@ -1,4 +1,3 @@ -import os from typing import Union, Dict, List import fasttext @@ -12,7 +11,7 @@ def __init__(self): """ Load fasttext model (https://huggingface.co/facebook/fasttext-language-identification) """ - model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification") + model_path = hf_hub_download(repo_id="facebook/fasttext-language-identification", filename="model.bin") self.model = fasttext.load_model(model_path)