Add load-lm action to lm-util tool

rwth-i6 · Nov 19, 2021 · 473d202 · 473d202
1 parent 44bacb9
commit 473d202
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/src/Tools/Lm/LmUtilityTool.cc b/src/Tools/Lm/LmUtilityTool.cc
@@ -64,6 +64,7 @@ class LmUtilityTool : public Core::Application {
 public:
     enum Action {
         actionNotGiven,
+        actionLoadLm,
         actionComputePerplexityFromTextFile
     };
 
@@ -81,14 +82,15 @@ class LmUtilityTool : public Core::Application {
     int main(std::vector<std::string> const& arguments);
 
 private:
+    void loadLm();
     void computePerplexityFromTextFile();
 };
 
 APPLICATION(LmUtilityTool)
 
 // ---------- Implementations ----------
 
-const Core::Choice          LmUtilityTool::choiceAction("compute-perplexity-from-text-file", actionComputePerplexityFromTextFile, Core::Choice::endMark());
+const Core::Choice          LmUtilityTool::choiceAction("load-lm", actionLoadLm, "compute-perplexity-from-text-file", actionComputePerplexityFromTextFile, Core::Choice::endMark());
 const Core::ParameterChoice LmUtilityTool::paramAction("action", &choiceAction, "action to perform", actionNotGiven);
 const Core::ParameterString LmUtilityTool::paramFile("file", "input file");
 const Core::ParameterString LmUtilityTool::paramEncoding("encoding", "the encoding of the input file", "utf8");
@@ -117,13 +119,19 @@ LmUtilityTool::LmUtilityTool()
 
 int LmUtilityTool::main(std::vector<std::string> const& arguments) {
     switch (paramAction(config)) {
+        case actionLoadLm: loadLm(); break;
         case actionComputePerplexityFromTextFile: computePerplexityFromTextFile(); break;
         default:
         case actionNotGiven: error("no action given");
     }
     return EXIT_SUCCESS;
 }
 
+void LmUtilityTool::loadLm() {
+    Bliss::LexiconRef      lexicon(Bliss::Lexicon::create(select("lexicon")));
+    LanguageModelRef       lm(Lm::Module::instance().createLanguageModel(select("lm"), lexicon));
+}
+
 void LmUtilityTool::computePerplexityFromTextFile() {
     bool                   renormalize = paramRenormalize(config);
     size_t                 batch_size  = paramBatchSize(config);