Merge pull request #834 from hkad98/jkd/pandas-perf

Improve GoodPandas performance
gooddata · Oct 7, 2024 · ebe3139 · ebe3139
2 parents 2edc83c + 3bee712
commit ebe3139
Show file tree

Hide file tree

Showing 41 changed files with 9,515 additions and 35,421 deletions.
diff --git a/gooddata-pandas/gooddata_pandas/data_access.py b/gooddata-pandas/gooddata_pandas/data_access.py
@@ -6,7 +6,7 @@
 from gooddata_sdk import (
     Attribute,
     AttributeFilter,
-    CatalogWorkspaceContent,
+    CatalogAttribute,
     ExecutionDefinition,
     ExecutionResponse,
     Filter,
@@ -16,6 +16,7 @@
     ObjId,
     TableDimension,
 )
+from gooddata_sdk.utils import IdObjType
 
 from gooddata_pandas.utils import (
     ColumnsDef,
@@ -319,27 +320,34 @@ def _extract_for_metrics_only(response: ExecutionResponse, cols: list, col_to_me
     return data
 
 
-def _typed_result(catalog: CatalogWorkspaceContent, attribute: Attribute, result_values: list[Any]) -> list[Any]:
+def _find_attribute(attributes: list[CatalogAttribute], id_obj: IdObjType) -> Union[CatalogAttribute, None]:
+    for attribute in attributes:
+        if attribute.find_label(id_obj) is not None:
+            return attribute
+    return None
+
+
+def _typed_result(attributes: list[CatalogAttribute], attribute: Attribute, result_values: list[Any]) -> list[Any]:
     """
     Internal function to convert result_values to proper data types.
 
     Args:
-        catalog (CatalogWorkspaceContent): The catalog workspace content.
+        attributes (list[CatalogAttribute]): The catalog of attributes.
         attribute (Attribute): The attribute for which the typed result will be computed.
         result_values (list[Any]): A list of raw values.
 
     Returns:
         list[Any]: A list of converted values with proper data types.
     """
-    catalog_attribute = catalog.find_label_attribute(attribute.label)
+    catalog_attribute = _find_attribute(attributes, attribute.label)
     if catalog_attribute is None:
         raise ValueError(f"Unable to find attribute {attribute.label} in catalog")
     return [_typed_attribute_value(catalog_attribute, value) for value in result_values]
 
 
 def _extract_from_attributes_and_maybe_metrics(
     response: ExecutionResponse,
-    catalog: CatalogWorkspaceContent,
+    attributes: list[CatalogAttribute],
     cols: list[str],
     col_to_attr_idx: dict[str, int],
     col_to_metric_idx: dict[str, int],
@@ -382,12 +390,12 @@ def _extract_from_attributes_and_maybe_metrics(
         for idx_name in index:
             rs = result.get_all_header_values(attribute_dim, safe_index_to_attr_idx[idx_name])
             attribute = index_to_attribute[idx_name]
-            index[idx_name] += _typed_result(catalog, attribute, rs)
+            index[idx_name] += _typed_result(attributes, attribute, rs)
         for col in cols:
             if col in col_to_attr_idx:
                 rs = result.get_all_header_values(attribute_dim, col_to_attr_idx[col])
                 attribute = col_to_attribute[col]
-                data[col] += _typed_result(catalog, attribute, rs)
+                data[col] += _typed_result(attributes, attribute, rs)
             elif col_to_metric_idx[col] < len(result.data):
                 data[col] += result.data[col_to_metric_idx[col]]
         if result.is_complete(attribute_dim):
@@ -440,10 +448,10 @@ def compute_and_extract(
     if not exec_def.has_attributes():
         return _extract_for_metrics_only(response, cols, col_to_metric_idx), dict()
     else:
-        catalog = sdk.catalog_workspace_content.get_full_catalog(workspace_id)
+        attributes = sdk.catalog_workspace_content.get_attributes_catalog(workspace_id, include=["labels", "datasets"])
         return _extract_from_attributes_and_maybe_metrics(
             response,
-            catalog,
+            attributes,
             cols,
             col_to_attr_idx,
             col_to_metric_idx,

diff --git a/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml b/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml
@@ -121,7 +121,7 @@ interactions:
         content-length:
           - '1219'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -182,10 +182,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
+              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
       body: null
       headers:
         Accept:
@@ -252,7 +252,7 @@ interactions:
         content-length:
           - '2168'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -355,7 +355,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
+              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
           resultSize: 4237
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2?offset=0%2C0&limit=100%2C100
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
       body: null
       headers:
         Accept:
@@ -439,7 +439,7 @@ interactions:
         content-length:
           - '11295'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:43 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:

diff --git a/...ata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml b/...ata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml
@@ -121,7 +121,7 @@ interactions:
         content-length:
           - '1219'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -182,10 +182,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
+              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
       body: null
       headers:
         Accept:
@@ -252,7 +252,7 @@ interactions:
         content-length:
           - '2168'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -355,7 +355,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 4e372376a5bef9f8d64b3eec9022efea2e41c7d5:05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2
+              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
           resultSize: 4237
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/4e372376a5bef9f8d64b3eec9022efea2e41c7d5%3A05407e0d701640fb410ed25e354a7e702b55be7bcf8bf949eb4b5fa4776019e2?offset=0%2C0&limit=100%2C100
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
       body: null
       headers:
         Accept:
@@ -439,7 +439,7 @@ interactions:
         content-length:
           - '11295'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:42 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:

diff --git a/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml b/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml
@@ -119,7 +119,7 @@ interactions:
         content-length:
           - '1179'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -178,10 +178,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
+              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
       body: null
       headers:
         Accept:
@@ -248,7 +248,7 @@ interactions:
         content-length:
           - '2067'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -349,7 +349,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
+              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -363,7 +363,7 @@ interactions:
           resultSize: 2913
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90?offset=0&limit=500
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
       body: null
       headers:
         Accept:
@@ -430,7 +430,7 @@ interactions:
         content-length:
           - '98918'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -4819,7 +4819,7 @@ interactions:
               - 364
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
       body: null
       headers:
         Accept:
@@ -4886,7 +4886,7 @@ interactions:
         content-length:
           - '2067'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -4987,7 +4987,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: 6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8:5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90
+              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -5001,7 +5001,7 @@ interactions:
           resultSize: 2913
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/6d6c03d47ad8bedf8a436f87a7f2638bfe2e0fc8%3A5e583be017c9937b7fe0f13ba25eb2c731d1b4f9bf129b1384ed5b1b9db86b90?offset=0&limit=500
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
       body: null
       headers:
         Accept:
@@ -5068,7 +5068,7 @@ interactions:
         content-length:
           - '98918'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 09:16:48 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string: