Skip to content

Commit d484e7a

Browse files
hkad98hkad98
committed
perf: rewrite data access in gooddata-pandas
data_access.py had poor performance because it loaded all attributes. The new implementation fetches only the necessary ones which should perform better on bigger workspaces. JIRA: STL-1036 risk: low
1 parent ff4c2de commit d484e7a

43 files changed

Lines changed: 892 additions & 30807 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎gooddata-pandas/gooddata_pandas/data_access.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
ObjId,
1717
TableDimension,
1818
)
19-
from gooddata_sdk.utils import IdObjType
19+
from gooddata_sdk.utils import IdObjType, filter_for_attributes_labels
2020

2121
from gooddata_pandas.utils import (
2222
ColumnsDef,
@@ -446,7 +446,12 @@ def compute_and_extract(
446446
if not exec_def.has_attributes():
447447
return _extract_for_metrics_only(response, cols, col_to_metric_idx), dict()
448448
else:
449-
attributes = sdk.catalog_workspace_content.get_attributes_catalog(workspace_id, include=["labels", "datasets"])
449+
filter_query = filter_for_attributes_labels(exec_def.attributes)
450+
# if there is to many labels then all attributes are fetched and no rsql filter is used
451+
# it prevention again 414 Request-URI Too Long
452+
attributes = sdk.catalog_workspace_content.get_attributes_catalog(
453+
workspace_id, include=["labels", "datasets"], rsql_filter=filter_query
454+
)
450455
return _extract_from_attributes_and_maybe_metrics(
451456
response,
452457
attributes,

‎gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) 2024 GoodData Corporation
1+
# (C) 2025 GoodData Corporation
22
version: 1
33
interactions:
44
- request:
@@ -121,7 +121,7 @@ interactions:
121121
content-length:
122122
- '1219'
123123
set-cookie:
124-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
124+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
125125
Path=/; HTTPOnly; SameSite=Lax
126126
body:
127127
string:
@@ -182,10 +182,10 @@ interactions:
182182
name: Order Amount
183183
localIdentifier: dim_1
184184
links:
185-
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
185+
executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
186186
- request:
187187
method: GET
188-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
188+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b/metadata
189189
body: null
190190
headers:
191191
Accept:
@@ -252,7 +252,7 @@ interactions:
252252
content-length:
253253
- '2168'
254254
set-cookie:
255-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
255+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
256256
Path=/; HTTPOnly; SameSite=Lax
257257
body:
258258
string:
@@ -355,7 +355,7 @@ interactions:
355355
name: Order Amount
356356
localIdentifier: dim_1
357357
links:
358-
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
358+
executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
359359
resultSpec:
360360
dimensions:
361361
- localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
372372
resultSize: 4237
373373
- request:
374374
method: GET
375-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
375+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b?offset=0%2C0&limit=100%2C100
376376
body: null
377377
headers:
378378
Accept:
@@ -439,7 +439,7 @@ interactions:
439439
content-length:
440440
- '11295'
441441
set-cookie:
442-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
442+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
443443
Path=/; HTTPOnly; SameSite=Lax
444444
body:
445445
string:

‎gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) 2024 GoodData Corporation
1+
# (C) 2025 GoodData Corporation
22
version: 1
33
interactions:
44
- request:
@@ -121,7 +121,7 @@ interactions:
121121
content-length:
122122
- '1219'
123123
set-cookie:
124-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
124+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
125125
Path=/; HTTPOnly; SameSite=Lax
126126
body:
127127
string:
@@ -182,10 +182,10 @@ interactions:
182182
name: Order Amount
183183
localIdentifier: dim_1
184184
links:
185-
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
185+
executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
186186
- request:
187187
method: GET
188-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
188+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b/metadata
189189
body: null
190190
headers:
191191
Accept:
@@ -252,7 +252,7 @@ interactions:
252252
content-length:
253253
- '2168'
254254
set-cookie:
255-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
255+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
256256
Path=/; HTTPOnly; SameSite=Lax
257257
body:
258258
string:
@@ -355,7 +355,7 @@ interactions:
355355
name: Order Amount
356356
localIdentifier: dim_1
357357
links:
358-
executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
358+
executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
359359
resultSpec:
360360
dimensions:
361361
- localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
372372
resultSize: 4237
373373
- request:
374374
method: GET
375-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
375+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b?offset=0%2C0&limit=100%2C100
376376
body: null
377377
headers:
378378
Accept:
@@ -439,7 +439,7 @@ interactions:
439439
content-length:
440440
- '11295'
441441
set-cookie:
442-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
442+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
443443
Path=/; HTTPOnly; SameSite=Lax
444444
body:
445445
string:

‎gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# (C) 2024 GoodData Corporation
1+
# (C) 2025 GoodData Corporation
22
version: 1
33
interactions:
44
- request:
@@ -119,7 +119,7 @@ interactions:
119119
content-length:
120120
- '1179'
121121
set-cookie:
122-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
122+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:55 GMT;
123123
Path=/; HTTPOnly; SameSite=Lax
124124
body:
125125
string:
@@ -178,10 +178,10 @@ interactions:
178178
name: Order Amount
179179
localIdentifier: dim_0
180180
links:
181-
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
181+
executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
182182
- request:
183183
method: GET
184-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
184+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d/metadata
185185
body: null
186186
headers:
187187
Accept:
@@ -248,7 +248,7 @@ interactions:
248248
content-length:
249249
- '2067'
250250
set-cookie:
251-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
251+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:55 GMT;
252252
Path=/; HTTPOnly; SameSite=Lax
253253
body:
254254
string:
@@ -349,7 +349,7 @@ interactions:
349349
name: Order Amount
350350
localIdentifier: dim_0
351351
links:
352-
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
352+
executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
353353
resultSpec:
354354
dimensions:
355355
- localIdentifier: dim_0
@@ -363,7 +363,7 @@ interactions:
363363
resultSize: 2913
364364
- request:
365365
method: GET
366-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
366+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d?offset=0&limit=500
367367
body: null
368368
headers:
369369
Accept:
@@ -430,7 +430,7 @@ interactions:
430430
content-length:
431431
- '98918'
432432
set-cookie:
433-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
433+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
434434
Path=/; HTTPOnly; SameSite=Lax
435435
body:
436436
string:
@@ -4819,7 +4819,7 @@ interactions:
48194819
- 364
48204820
- request:
48214821
method: GET
4822-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
4822+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d/metadata
48234823
body: null
48244824
headers:
48254825
Accept:
@@ -4886,7 +4886,7 @@ interactions:
48864886
content-length:
48874887
- '2067'
48884888
set-cookie:
4889-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
4889+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
48904890
Path=/; HTTPOnly; SameSite=Lax
48914891
body:
48924892
string:
@@ -4987,7 +4987,7 @@ interactions:
49874987
name: Order Amount
49884988
localIdentifier: dim_0
49894989
links:
4990-
executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
4990+
executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
49914991
resultSpec:
49924992
dimensions:
49934993
- localIdentifier: dim_0
@@ -5001,7 +5001,7 @@ interactions:
50015001
resultSize: 2913
50025002
- request:
50035003
method: GET
5004-
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
5004+
uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d?offset=0&limit=500
50055005
body: null
50065006
headers:
50075007
Accept:
@@ -5068,7 +5068,7 @@ interactions:
50685068
content-length:
50695069
- '98918'
50705070
set-cookie:
5071-
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
5071+
- SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
50725072
Path=/; HTTPOnly; SameSite=Lax
50735073
body:
50745074
string:

0 commit comments

Comments
 (0)