diff --git a/app/src/dataSource/index.ts b/app/src/dataSource/index.tsx similarity index 68% rename from app/src/dataSource/index.ts rename to app/src/dataSource/index.tsx index 3d684cf..f796019 100644 --- a/app/src/dataSource/index.ts +++ b/app/src/dataSource/index.tsx @@ -122,20 +122,47 @@ function initBatchGetDatas(action: string) { const batchGetDatasBySql = initBatchGetDatas("batch_get_datas_by_sql"); const batchGetDatasByPayload = initBatchGetDatas("batch_get_datas_by_payload"); +const DEFAULT_LIMIT = 50_000; + +function notifyDataLimit() { + commonStore.setNotification({ + type: "warning", + title: "Data Limit Reached", + message: (<> + The current computation has returned more than 50,000 rows of data. + To ensure optimal performance, we are currently rendering only the first 50,000 rows. + If you need to render the entire all datas, please use the 'limit' tool + ( + + ) to manually set the maximum number of rows to be returned. + ) + }, 60_000); +} export function getDatasFromKernelBySql(fieldMetas: any) { return async (payload: IDataQueryPayload) => { const sql = parser_dsl_with_meta( "pygwalker_mid_table", - JSON.stringify(payload), + JSON.stringify({...payload, limit: payload.limit ?? DEFAULT_LIMIT}), JSON.stringify({"pygwalker_mid_table": fieldMetas}) ); - const result = await batchGetDatasBySql.getDatas(sql); - return (result ?? []) as IRow[]; + const result = await batchGetDatasBySql.getDatas(sql) ?? []; + if (!payload.limit && result.length === DEFAULT_LIMIT) { + notifyDataLimit(); + } + return result as IRow[]; } } export async function getDatasFromKernelByPayload(payload: IDataQueryPayload) { - const result = await batchGetDatasByPayload.getDatas(payload); - return (result ?? []) as IRow[]; + const result = await batchGetDatasByPayload.getDatas({...payload, limit: payload.limit ?? DEFAULT_LIMIT}) ?? []; + if (!payload.limit && result.length === DEFAULT_LIMIT) { + notifyDataLimit(); + } + return result as IRow[]; } diff --git a/pygwalker/__init__.py b/pygwalker/__init__.py index e4254a9..0a2b272 100644 --- a/pygwalker/__init__.py +++ b/pygwalker/__init__.py @@ -10,7 +10,7 @@ from pygwalker.services.global_var import GlobalVarManager from pygwalker.services.kaggle import show_tips_user_kaggle as __show_tips_user_kaggle -__version__ = "0.4.9.2" +__version__ = "0.4.9.3" __hash__ = __rand_str() from pygwalker.api.jupyter import walk, render, table diff --git a/pygwalker/api/pygwalker.py b/pygwalker/api/pygwalker.py index 9bcd4fa..1d04e70 100644 --- a/pygwalker/api/pygwalker.py +++ b/pygwalker/api/pygwalker.py @@ -38,7 +38,6 @@ from pygwalker.utils.randoms import generate_hash_code from pygwalker.communications.hacker_comm import HackerCommunication, BaseCommunication from pygwalker._constants import JUPYTER_BYTE_LIMIT, JUPYTER_WIDGETS_BYTE_LIMIT -from pygwalker.errors import DataCountLimitError from pygwalker import __version__ @@ -404,34 +403,24 @@ def upload_spec_to_cloud(data: Dict[str, Any]): def _get_datas(data: Dict[str, Any]): sql = data["sql"] datas = self.data_parser.get_datas_by_sql(sql) - if len(datas) > GlobalVarManager.max_data_length: - raise DataCountLimitError() return { "datas": datas } def _get_datas_by_payload(data: Dict[str, Any]): datas = self.data_parser.get_datas_by_payload(data["payload"]) - if len(datas) > GlobalVarManager.max_data_length: - raise DataCountLimitError() return { "datas": datas } def _batch_get_datas_by_sql(data: Dict[str, Any]): result = self.data_parser.batch_get_datas_by_sql(data["queryList"]) - for datas in result: - if len(datas) > GlobalVarManager.max_data_length: - raise DataCountLimitError() return { "datas": result } def _batch_get_datas_by_payload(data: Dict[str, Any]): result = self.data_parser.batch_get_datas_by_payload(data["queryList"]) - for datas in result: - if len(datas) > GlobalVarManager.max_data_length: - raise DataCountLimitError() return { "datas": result } diff --git a/pygwalker/errors.py b/pygwalker/errors.py index 5c79bf5..6238082 100644 --- a/pygwalker/errors.py +++ b/pygwalker/errors.py @@ -43,15 +43,6 @@ class ViewSqlSameColumnError(BaseError): pass -class DataCountLimitError(BaseError): - """Raised when the data count is too large.""" - def __init__(self) -> None: - super().__init__( - "The query returned too many data entries, making it difficult for the frontend to render. Please adjust your chart configuration and try again.", - code=ErrorCode.UNKNOWN_ERROR - ) - - class StreamlitPygwalkerApiError(BaseError): """Raised when the config is invalid.""" def __init__(self) -> None: diff --git a/pygwalker/services/global_var.py b/pygwalker/services/global_var.py index 7822a00..c9aff4c 100644 --- a/pygwalker/services/global_var.py +++ b/pygwalker/services/global_var.py @@ -1,7 +1,7 @@ import os from pandas import DataFrame -from typing_extensions import Literal +from typing_extensions import Literal, deprecated from .config import get_config @@ -46,6 +46,7 @@ def set_last_exported_dataframe(cls, df: DataFrame): cls.last_exported_dataframe = df @classmethod + @deprecated("use ui config instead.") def set_max_data_length(cls, length: int): cls.max_data_length = length