diff --git a/testing.ipynb b/testing.ipynb index 916c9ee..ae3106c 100644 --- a/testing.ipynb +++ b/testing.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "id": "initial_id", "metadata": { "ExecuteTime": { @@ -14,13 +14,16 @@ "source": [ "import snowflake.connector\n", "\n", - "con = snowflake.connector.connect(account='lt51601.europe-west2.gcp', database='MY_CUSTOM_APP', connection_name='default', host='localhostcomputing.com', port='8084', role='ACCOUNTADMIN')\n", + "con = snowflake.connector.connect(account='lt51601.europe-west2.gcp', \n", + " database='my_iceberg_jinjat', connection_name='default',\n", + " host='localhostcomputing.com', \n", + " port='8084', role='ACCOUNTADMIN')\n", "cur = con.cursor()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "7165f75bf836dbe3", "metadata": { "ExecuteTime": { @@ -32,9 +35,223 @@ "outputs_hidden": false } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
created_onnamedatabase_nameschema_namekindcommentcluster_byrowsbytesowner...change_trackingis_externalenable_schema_evolutionowner_role_typeis_eventbudgetis_hybridis_icebergis_dynamicis_immutable
0292278994-08-16 23:12:55.807000-08:00CUSTOMERMY_ICEBERG_JINJATTPCH_SF1TABLE60000043210240ACCOUNTADMIN...ONNNROLENNoneNNNN
1292278994-08-16 23:12:55.807000-08:00DYNAMIC_ICEBERG_TABLE_CUSTOMERMY_ICEBERG_JINJATTPCH_SF1TABLE60000042527232ACCOUNTADMIN...ONNNROLENNoneNYYN
2292278994-08-16 23:12:55.807000-08:00DYNAMIC_TABLE_CUSTOMERMY_ICEBERG_JINJATTPCH_SF1TABLE60000058829312ACCOUNTADMIN...ONNNROLENNoneNNYN
3292278994-08-16 23:12:55.807000-08:00ORDERS_ICEBERG_TPCH_SF1000MY_ICEBERG_JINJATTPCH_SF1TABLE150000000042525678592ACCOUNTADMIN...OFFNNROLENNoneNYNN
4292278994-08-16 23:12:55.807000-08:00TEST_TABLE_MANAGEDMY_ICEBERG_JINJATTPCH_SF1TABLE00ACCOUNTADMIN...OFFNNROLENNoneNYNN
\n", + "

5 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " created_on name \\\n", + "0 292278994-08-16 23:12:55.807000-08:00 CUSTOMER \n", + "1 292278994-08-16 23:12:55.807000-08:00 DYNAMIC_ICEBERG_TABLE_CUSTOMER \n", + "2 292278994-08-16 23:12:55.807000-08:00 DYNAMIC_TABLE_CUSTOMER \n", + "3 292278994-08-16 23:12:55.807000-08:00 ORDERS_ICEBERG_TPCH_SF1000 \n", + "4 292278994-08-16 23:12:55.807000-08:00 TEST_TABLE_MANAGED \n", + "\n", + " database_name schema_name kind comment cluster_by rows \\\n", + "0 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n", + "1 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n", + "2 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n", + "3 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 1500000000 \n", + "4 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 0 \n", + "\n", + " bytes owner ... change_tracking is_external \\\n", + "0 43210240 ACCOUNTADMIN ... ON N \n", + "1 42527232 ACCOUNTADMIN ... ON N \n", + "2 58829312 ACCOUNTADMIN ... ON N \n", + "3 42525678592 ACCOUNTADMIN ... OFF N \n", + "4 0 ACCOUNTADMIN ... OFF N \n", + "\n", + " enable_schema_evolution owner_role_type is_event budget is_hybrid \\\n", + "0 N ROLE N None N \n", + "1 N ROLE N None N \n", + "2 N ROLE N None N \n", + "3 N ROLE N None N \n", + "4 N ROLE N None N \n", + "\n", + " is_iceberg is_dynamic is_immutable \n", + "0 N N N \n", + "1 Y Y N \n", + "2 N Y N \n", + "3 Y N N \n", + "4 Y N N \n", + "\n", + "[5 rows x 22 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "con.cursor().execute(\"show tables\").fetch_pandas_all()" + "con.cursor().execute(\"show tables in schema tpch_sf1\").fetch_pandas_all()" ] }, { diff --git a/universql/warehouse/duckdb/duckdb.py b/universql/warehouse/duckdb/duckdb.py index 2a0a908..afb2d5c 100644 --- a/universql/warehouse/duckdb/duckdb.py +++ b/universql/warehouse/duckdb/duckdb.py @@ -63,7 +63,8 @@ def sync_duckdb_catalog(self, locations: typing.Dict[ sqlglot.exp.Table, sqlglot.exp.Expression], ast: sqlglot.exp.Expression) -> Optional[ sqlglot.exp.Expression]: - views = [f"CREATE OR REPLACE VIEW main.\"{table.sql()}\" AS SELECT * FROM {expression.sql()};" for + + views = [f"CREATE OR REPLACE VIEW main.{sqlglot.exp.parse_identifier(table.sql())} AS SELECT * FROM {expression.sql()};" for table, expression in locations.items()] views_sql = "\n".join(views) if views: @@ -122,8 +123,13 @@ def _do_query(self, raw_query: str) -> (str, List, pyarrow.Table): local_error_message = (f"Unable to find location of Iceberg tables. " f"See: https://github.com/buremba/universql#cant-query-native-snowflake-tables. Cause: {e.msg}") - transformed_ast = self.sync_duckdb_catalog(locations, - simplify(ast)) if locations is not None else None + try: + transformed_ast = self.sync_duckdb_catalog(locations, + simplify(ast)) if locations is not None else None + except Exception as e: + transformed_ast = e + local_error_message = f"Unable to sync DuckDB catalog. {str(e)}" + if transformed_ast is None: last_compute = None break