Update test dtype as bfloat16 and minor fixes

tests/lora/test_llama_hpu.py

@@ -41,21 +41,22 @@ def _test_llama_lora(sql_lora_files, tp_size):
                    enable_lora=True,
                    max_num_seqs=16,
                    max_loras=4,
+                   dtype='bfloat16',
                    tensor_parallel_size=tp_size)
 
     expected_no_lora_output = [
         "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]",  # noqa: E501
         " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ",  # noqa: E501
-        "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m",  # noqa: E501
-        " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ",  # noqa: E501
+        "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kamu/ [kamɯ́]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kamu/ [kamɯ́]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What",  # noqa: E501
+        " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio.\n\n answer:\n\n SELECT people.sex, AVG(unsure_rate) AS avg_unsure_rate\n FROM candidate\n JOIN people ON candidate.people_id = people.people_id\n GROUP BY people.sex\n ORDER BY avg_unsure_rate DESC\n\n ",  # noqa: E501
         " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ",  # noqa: E501
         "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE",  # noqa: E501
     ]
     expected_lora_output = [
         "  SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ",  # noqa: E501
         "  SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",  # noqa: E501
         "  SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ",  # noqa: E501
-        "  SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ",  # noqa: E501
+        "  SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(*) DESC LIMIT 1) ",  # noqa: E501
         "  SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",  # noqa: E501
         "  SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' "  # noqa: E501
     ]
@@ -80,20 +81,20 @@ def test_llama_lora_1x(sql_lora_files):
     p = Process(target=_test_llama_lora, args=(sql_lora_files, 1))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0
 
 
 def test_llama_lora_2x(sql_lora_files):
     # Work-around to resolve stalling issue in multi-card scenario
     p = Process(target=_test_llama_lora, args=(sql_lora_files, 2))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0
 
 
 def test_llama_lora_4x(sql_lora_files):
     # Work-around to resolve stalling issue in multi-card scenario
     p = Process(target=_test_llama_lora, args=(sql_lora_files, 4))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0

tests/lora/test_multilora_hpu.py

@@ -112,20 +112,20 @@ def test_llama_multilora_1x(sql_lora_files):
     p = Process(target=_test_llama_multilora, args=(sql_lora_files, 1))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0
 
 
 def test_llama_multilora_2x(sql_lora_files):
     # Work-around to resolve stalling issue in multi-card scenario
     p = Process(target=_test_llama_multilora, args=(sql_lora_files, 2))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0
 
 
 def test_llama_multilora_4x(sql_lora_files):
     # Work-around to resolve stalling issue in multi-card scenario
     p = Process(target=_test_llama_multilora, args=(sql_lora_files, 4))
     p.start()
     p.join()
-    assert p.exitcode == 0, "Results don't match with the reference"
+    assert p.exitcode == 0

vllm/worker/habana_model_runner.py

@@ -438,8 +438,8 @@ def load_model(self) -> None:
                 assert hasattr(
                     self.model, "embedding_padding_modules"
                 ), "Model does not have embedding_padding_modules"
-                if self.block_size > self.scheduler_config.max_num_batched_tokens // self.scheduler_config.max_num_seqs:
-                    self.block_size = self.scheduler_config.max_num_batched_tokens // self.scheduler_config.max_num_seqs
+                if self.block_size > self.scheduler_config.max_num_batched_tokens // self.scheduler_config.max_num_seqs:  # noqa: E501
+                    self.block_size = self.scheduler_config.max_num_batched_tokens // self.scheduler_config.max_num_seqs  # noqa: E501
                 self.lora_manager = LRUCacheWorkerLoRAManager(
                     self.scheduler_config.max_num_seqs,
                     self.scheduler_config.max_num_batched_tokens,