Skip to content

Commit d80a07a

Browse files
shuoweilshuoweil
committed
now large dataset can get the correct row counts
1 parent 189d8bc commit d80a07a

File tree

4 files changed

+79
-25
lines changed

4 files changed

+79
-25
lines changed

‎bigframes/display/anywidget.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ class TableWidget(WIDGET_BASE):
5757
page_size = traitlets.Int(0).tag(sync=True)
5858
row_count = traitlets.Int(0).tag(sync=True)
5959
table_html = traitlets.Unicode().tag(sync=True)
60+
_initial_load_complete = traitlets.Bool(False).tag(sync=True)
6061

6162
def __init__(self, dataframe: bigframes.dataframe.DataFrame):
6263
"""Initialize the TableWidget.
@@ -106,6 +107,7 @@ def __init__(self, dataframe: bigframes.dataframe.DataFrame):
106107
)
107108

108109
self._set_table_html()
110+
self._initial_load_complete = True
109111
self._initializing = False
110112

111113
@functools.cached_property

‎bigframes/display/table_widget.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,12 @@ function render({ model, el }) {
137137
}
138138
});
139139
model.on(Event.CHANGE_TABLE_HTML, handleTableHTMLChange);
140+
model.on(`change:${ModelProperty.ROW_COUNT}`, updateButtonStates);
141+
model.on(`change:_initial_load_complete`, (val) => {
142+
if (val) {
143+
updateButtonStates();
144+
}
145+
});
140146

141147
// Assemble the DOM
142148
paginationContainer.appendChild(prevPage);

‎bigframes/session/bq_caching_executor.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,10 @@ def execute(
188188
execution_spec: ex_spec.ExecutionSpec,
189189
) -> executor.ExecuteResult:
190190
# TODO: Support export jobs in combination with semi executors
191-
if execution_spec.destination_spec is None:
191+
if (
192+
execution_spec.destination_spec is None
193+
and execution_spec.promise_under_10gb
194+
):
192195
plan = self.prepare_plan(array_value.node, target="simplify")
193196
for exec in self._semi_executors:
194197
maybe_result = exec.execute(

‎notebooks/dataframes/anywidget_mode.ipynb‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -128,22 +128,34 @@
128128
"metadata": {},
129129
"outputs": [
130130
{
131-
"ename": "TypeError",
132-
"evalue": "BigQueryCachingExecutor.execute() got an unexpected keyword argument 'ordered'",
133-
"output_type": "error",
134-
"traceback": [
135-
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
136-
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
137-
"\u001b[0;32m~/.pyenv/versions/3.10.18/lib/python3.10/site-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_real_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_method\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 346\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
138-
"\u001b[0;32m~/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 193\u001b[0m \u001b[0mtask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mPANDAS_PARAM_TRACKING_TASK\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m )\n\u001b[0;32m--> 195\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 196\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0m_call_stack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
139-
"\u001b[0;32m~/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/log_adapter.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 181\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mNotImplementedError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 182\u001b[0m \u001b[0;31m# Log method parameters that are implemented in pandas but either missing (TypeError)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
140-
"\u001b[0;32m~/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dataframe.py\u001b[0m in \u001b[0;36m_repr_html_\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 857\u001b[0m \u001b[0;31m# This ensures that each cell gets its own widget and prevents\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 858\u001b[0m \u001b[0;31m# unintended sharing between cells\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 859\u001b[0;31m \u001b[0mwidget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTableWidget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 860\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0mipython_display\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwidget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
141-
"\u001b[0;32m~/src/github.com/googleapis/python-bigquery-dataframes/bigframes/display/anywidget.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dataframe)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# Force execution with explicit destination to get total_rows metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m execute_result = dataframe._block.session._executor.execute( \n\u001b[0m\u001b[1;32m 93\u001b[0m \u001b[0mbigframes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mArrayValue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_block\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# Wrap in ArrayValue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0mordered\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
142-
"\u001b[0;31mTypeError\u001b[0m: BigQueryCachingExecutor.execute() got an unexpected keyword argument 'ordered'"
143-
]
131+
"data": {
132+
"text/html": [
133+
"Query job 6d85c081-49c7-408a-ab96-e0e9e5102419 is DONE. 171.4 MB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:6d85c081-49c7-408a-ab96-e0e9e5102419&page=queryresults\">Open Job</a>"
134+
],
135+
"text/plain": [
136+
"<IPython.core.display.HTML object>"
137+
]
138+
},
139+
"metadata": {},
140+
"output_type": "display_data"
141+
},
142+
{
143+
"data": {
144+
"application/vnd.jupyter.widget-view+json": {
145+
"model_id": "31ba8e41e4ca4579b85409237cb7a566",
146+
"version_major": 2,
147+
"version_minor": 0
148+
},
149+
"text/plain": [
150+
"TableWidget(page_size=10, row_count=5552452, table_html='<table border=\"1\" class=\"dataframe table table-stripe…"
151+
]
152+
},
153+
"metadata": {},
154+
"output_type": "display_data"
144155
},
145156
{
146157
"data": {
158+
"text/html": [],
147159
"text/plain": [
148160
"Computation deferred. Computation will process 171.4 MB"
149161
]
@@ -172,16 +184,37 @@
172184
"metadata": {},
173185
"outputs": [
174186
{
175-
"ename": "TypeError",
176-
"evalue": "BigQueryCachingExecutor.execute() got an unexpected keyword argument 'ordered'",
177-
"output_type": "error",
178-
"traceback": [
179-
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
180-
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
181-
"\u001b[0;32m<ipython-input-7-49e9807f0359>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# Create widget programmatically\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mwidget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTableWidget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Total pages: {math.ceil(widget.row_count / widget.page_size)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
182-
"\u001b[0;32m~/src/github.com/googleapis/python-bigquery-dataframes/bigframes/display/anywidget.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dataframe)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# Force execution with explicit destination to get total_rows metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m execute_result = dataframe._block.session._executor.execute( \n\u001b[0m\u001b[1;32m 93\u001b[0m \u001b[0mbigframes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mArrayValue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_block\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# Wrap in ArrayValue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0mordered\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
183-
"\u001b[0;31mTypeError\u001b[0m: BigQueryCachingExecutor.execute() got an unexpected keyword argument 'ordered'"
187+
"data": {
188+
"text/html": [
189+
"Query job 48cb4908-a59a-420f-8fcb-200d0d9187ef is DONE. 171.4 MB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:48cb4908-a59a-420f-8fcb-200d0d9187ef&page=queryresults\">Open Job</a>"
190+
],
191+
"text/plain": [
192+
"<IPython.core.display.HTML object>"
193+
]
194+
},
195+
"metadata": {},
196+
"output_type": "display_data"
197+
},
198+
{
199+
"name": "stdout",
200+
"output_type": "stream",
201+
"text": [
202+
"Total pages: 555246\n"
184203
]
204+
},
205+
{
206+
"data": {
207+
"application/vnd.jupyter.widget-view+json": {
208+
"model_id": "5d22f3f19e4140b0ba51869e97c3f690",
209+
"version_major": 2,
210+
"version_minor": 0
211+
},
212+
"text/plain": [
213+
"TableWidget(page_size=10, row_count=5552452, table_html='<table border=\"1\" class=\"dataframe table table-stripe…"
214+
]
215+
},
216+
"metadata": {},
217+
"output_type": "display_data"
185218
}
186219
],
187220
"source": [
@@ -206,10 +239,20 @@
206239
},
207240
{
208241
"cell_type": "code",
209-
"execution_count": null,
242+
"execution_count": 8,
210243
"id": "12b68f15",
211244
"metadata": {},
212-
"outputs": [],
245+
"outputs": [
246+
{
247+
"name": "stdout",
248+
"output_type": "stream",
249+
"text": [
250+
"Current page: 0\n",
251+
"After next: 1\n",
252+
"After prev: 0\n"
253+
]
254+
}
255+
],
213256
"source": [
214257
"# Simulate button clicks programmatically\n",
215258
"print(\"Current page:\", widget.page)\n",

0 commit comments

Comments
 (0)