Create release 1.20.1

cicd/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.20.0
+current_version = 1.20.1
 commit = False
 tag = False
 

lakehouse_engine/core/gab_sql_generator.py

@@ -6,7 +6,7 @@
 from typing import Any, Callable, Optional
 
 from pyspark.sql import DataFrame
-from pyspark.sql.functions import col, lit
+from pyspark.sql.functions import col, lit, struct, to_json
 
 from lakehouse_engine.core.exec_env import ExecEnv
 from lakehouse_engine.utils.gab_utils import GABUtils
@@ -475,8 +475,10 @@ def _delete_statement_generator(self) -> str:
             col("query_id") == lit(self.query_id)
         )
 
-        df_map = df_filtered.select(col("mappings")).toJSON()
-        view_df = df_map.collect()[0]
+        df_map = df_filtered.select(col("mappings"))
+        view_df = df_map.select(
+            to_json(struct([df_map[x] for x in df_map.columns]))
+        ).collect()[0][0]
         line = json.loads(view_df)
 
         for line_v in line.values():

lakehouse_engine/utils/gab_utils.py

@@ -8,7 +8,7 @@
 
 import pendulum
 from pyspark.sql import DataFrame
-from pyspark.sql.functions import col, lit
+from pyspark.sql.functions import col, lit, struct, to_json
 
 from lakehouse_engine.core.definitions import GABCadence, GABDefaults
 from lakehouse_engine.core.exec_env import ExecEnv
@@ -106,13 +106,15 @@ def get_json_column_as_dict(
             query_id: gab configuration table use case identifier.
             query_column: column to get as json.
         """
-        column_df = (
-            lookup_query_builder.filter(col("query_id") == lit(query_id))
-            .select(col(query_column))
-            .toJSON()
-            .collect()[0]
-        )
-        json_column = json.loads(column_df)
+        column_df = lookup_query_builder.filter(
+            col("query_id") == lit(query_id)
+        ).select(col(query_column))
+
+        column_df_json = column_df.select(
+            to_json(struct([column_df[x] for x in column_df.columns]))
+        ).collect()[0][0]
+
+        json_column = json.loads(column_df_json)
 
         for mapping in json_column.values():
             column_as_json = ast.literal_eval(mapping)

setup.py

@@ -14,7 +14,7 @@
 
 setuptools.setup(
     name="lakehouse-engine",
-    version="1.20.0",
+    version="1.20.1",
     author="Adidas Lakehouse Foundations Team",
     author_email="[email protected]",
     description="A Spark framework serving as the engine for several lakehouse "