Draft: Make SQL query backend configurable (potential Apache Flight)

docs/README.md

@@ -17,6 +17,7 @@ A no-code GUI for exploring multi-modal AI/ML datasets — tabular data with ima
 | [patterns/hooks.md](patterns/hooks.md) | `shouldFetch` pattern and hook return conventions — extracted from rules.md |
 | [media-urls.md](media-urls.md) | How media asset URLs are resolved in the Query tab; points to `SqlQueryPanel.tsx` |
 | [product-define.md](product-define.md) | Product positioning and comparison with Tableau, Voxel51, W&B, ChatGPT, etc. |
+| [query-backends.md](query-backends.md) | Pluggable SQL backend (DuckDB or any Arrow Flight SQL service) — env vars and constructor wiring |
 
 ## User-facing docs
 

docs/query-backends.md

@@ -0,0 +1,86 @@
+# Query backends
+
+SmooSense ships with an in-process **DuckDB** SQL engine by default. The same
+read-only request handlers (`/api/query`, `/api/umap`) can route through any
+**Arrow Flight SQL** service (DataFusion, Ballista, Dremio, Spice, etc.) by
+swapping the backend at app construction time.
+
+## Architecture
+
+The Flask app stores a single callable under
+`app.config["QUERY_CONNECTION_MAKER"]`. Each request invokes it to obtain a
+fresh DB-API 2.0 cursor (PEP 249) and uses only the portable surface:
+
+```python
+cur = connection_maker()
+cur.execute(query)
+column_names = [d[0] for d in cur.description]
+rows = cur.fetchall()
+```
+
+That contract is satisfied by the DuckDB connection (default), the ADBC Flight
+SQL driver, and any other DB-API-compliant Python client.
+
+## Configuration via environment variables
+
+Set on the server process before constructing `SmooSenseApp`. No code changes
+required.
+
+| Variable | Purpose |
+| --- | --- |
+| `SMOOSENSE_QUERY_BACKEND` | `duckdb` (default) or `flightsql`. Inferred as `flightsql` if `SMOOSENSE_FLIGHTSQL_URI` is set. |
+| `SMOOSENSE_FLIGHTSQL_URI` | gRPC endpoint, e.g. `grpc://datafusion-svc:50051` or `grpc+tls://host:443`. Required when backend is `flightsql`. |
+| `SMOOSENSE_FLIGHTSQL_USERNAME` | Optional Basic-auth username. |
+| `SMOOSENSE_FLIGHTSQL_PASSWORD` | Optional Basic-auth password. |
+| `SMOOSENSE_FLIGHTSQL_TOKEN` | Optional Bearer token (sent as `Authorization: Bearer <token>`). Mutually exclusive with username/password in practice. |
+| `SMOOSENSE_FLIGHTSQL_HEADERS` | Optional JSON object of extra RPC headers, e.g. `{"x-tenant":"abc"}`. |
+| `SMOOSENSE_FLIGHTSQL_TLS_INSECURE` | `1` / `true` to disable TLS certificate verification. Test/staging only. |
+
+Install the driver alongside SmooSense:
+
+```bash
+pip install "smoosense[flightsql]"
+```
+
+## Configuration via the Flask app
+
+For programmatic setups (custom auth, dynamic options, embedded usage),
+construct any DB-API maker and pass it to `SmooSenseApp`:
+
+```python
+from smoosense.app import SmooSenseApp
+from smoosense.utils.query_backends import flightsql_connection_maker
+
+maker = flightsql_connection_maker(
+    "grpc+tls://datafusion.internal:443",
+    token="…",
+    headers={"x-tenant": "research"},
+)
+
+SmooSenseApp(query_connection_maker=maker).run()
+```
+
+A custom maker just has to be `Callable[[], DBAPICursor]`. Any DB-API client
+works — psycopg, pyarrow ADBC for Postgres, etc. — provided the server speaks
+SQL the SmooSense handlers can use.
+
+## What the server side has to support
+
+The handlers run plain SQL strings:
+
+- `/api/query` runs whatever the user types after the engine-agnostic
+  `check_permissions` filter (rejects `COPY`, `EXPORT`, `DELETE`, `ATTACH`,
+  `UPDATE`).
+- `/api/umap` issues `SELECT <cols> FROM '<table_path>' [WHERE <cond>]` against
+  a file path. DuckDB and DataFusion both accept `FROM '<path>'` directly;
+  most other engines need an `ObjectStore` (or equivalent) registered on the
+  session, or a `CREATE EXTERNAL TABLE` per dataset. This is server-side
+  configuration — SmooSense does not push file paths through the protocol.
+
+## Migration notes
+
+- The Flask config key was renamed from `DUCKDB_CONNECTION_MAKER` to
+  `QUERY_CONNECTION_MAKER`. Update any extension that read the old key.
+- DuckDB-specific S3 wiring (creds, region, endpoint) only runs when the
+  DuckDB backend is selected. Flight SQL services manage their own object
+  store credentials server-side.

smoosense-py/pyproject.toml

@@ -55,6 +55,11 @@ daft = [
     "daft>=0.6.2",
 ]
 
+flightsql = [
+    "adbc-driver-flightsql>=1.0.0",
+    "adbc-driver-manager>=1.0.0",
+]
+
 emb = [
     "torch>=2.5.0",
     "pillow>=10.4.0",
@@ -142,5 +147,5 @@ exclude = [
 ]
 
 [[tool.mypy.overrides]]
-module = ["authlib.*", "boto3.*", "botocore.*", "cv2.*", "pandas.*", "IPython.*", "daft.*", "lancedb.*", "pyarrow.*", "torch.*", "PIL.*", "transformers.*", "tqdm.*", "umap.*"]
+module = ["adbc_driver_flightsql.*", "adbc_driver_manager.*", "authlib.*", "boto3.*", "botocore.*", "cv2.*", "pandas.*", "IPython.*", "daft.*", "lancedb.*", "pyarrow.*", "torch.*", "PIL.*", "transformers.*", "tqdm.*", "umap.*"]
 ignore_missing_imports = true

smoosense-py/smoosense/app.py

@@ -17,6 +17,10 @@
 from smoosense.handlers.shell import shell_bp
 from smoosense.handlers.umap import umap_bp
 from smoosense.utils.duckdb_connections import duckdb_connection_default, duckdb_connection_using_s3
+from smoosense.utils.query_backends import (
+    QueryConnectionMaker,
+    build_query_connection_maker_from_env,
+)
 
 PWD = os.path.dirname(os.path.abspath(__file__))
 
@@ -32,26 +36,33 @@ def __init__(
         s3_client: BaseClient | None = None,
         s3_prefix_to_save_shareable_link: str = "",
         folder_shortcuts: dict[str, str] | None = None,
+        query_connection_maker: QueryConnectionMaker | None = None,
     ):
         self.s3_client = s3_client if s3_client is not None else boto3.client("s3")
 
-        # Check if S3/AWS configuration is available
-        # This includes explicit s3_client, environment variables, or AWS config files
-        has_s3_config = any(
-            [
-                s3_client is not None,
-                os.getenv("S3_PROFILE") is not None,
-                os.getenv("AWS_ENDPOINT_URL") is not None,
-                os.getenv("AWS_ACCESS_KEY_ID") is not None,
-                os.getenv("AWS_SECRET_ACCESS_KEY") is not None,
-                os.path.exists(os.path.expanduser("~/.aws/credentials")),
-            ]
-        )
-
-        if has_s3_config:
-            self.duckdb_connection_maker = duckdb_connection_using_s3(s3_client=self.s3_client)
-        else:
-            self.duckdb_connection_maker = duckdb_connection_default()
+        # Resolve the SQL query backend.
+        # Priority: explicit query_connection_maker > env-configured backend > DuckDB default.
+        if query_connection_maker is None:
+            query_connection_maker = build_query_connection_maker_from_env()
+
+        if query_connection_maker is None:
+            # Default DuckDB engine — pick the variant that matches available AWS config.
+            has_s3_config = any(
+                [
+                    s3_client is not None,
+                    os.getenv("S3_PROFILE") is not None,
+                    os.getenv("AWS_ENDPOINT_URL") is not None,
+                    os.getenv("AWS_ACCESS_KEY_ID") is not None,
+                    os.getenv("AWS_SECRET_ACCESS_KEY") is not None,
+                    os.path.exists(os.path.expanduser("~/.aws/credentials")),
+                ]
+            )
+            if has_s3_config:
+                query_connection_maker = duckdb_connection_using_s3(s3_client=self.s3_client)
+            else:
+                query_connection_maker = duckdb_connection_default()
+
+        self.query_connection_maker: QueryConnectionMaker = query_connection_maker
 
         self.local_folder_prefix: str | None = os.environ.get("SMOOSENSE_LOCAL_FOLDER_PREFIX")
 
@@ -111,7 +122,7 @@ def create_app(self) -> Flask:
 
         # Store the s3_client in app config so blueprints can access it
         app.config["S3_CLIENT"] = self.s3_client
-        app.config["DUCKDB_CONNECTION_MAKER"] = self.duckdb_connection_maker
+        app.config["QUERY_CONNECTION_MAKER"] = self.query_connection_maker
         app.config["PASSOVER_CONFIG"] = self.passover_config
 
         # Initialize Auth0 if configured

smoosense-py/smoosense/handlers/query.py

@@ -6,7 +6,7 @@
 
 from smoosense.handlers.auth import requires_auth_api
 from smoosense.utils.api import handle_api_errors
-from smoosense.utils.duckdb_connections import check_permissions
+from smoosense.utils.query_backends import check_permissions
 from smoosense.utils.serialization import serialize
 
 logger = logging.getLogger(__name__)
@@ -33,12 +33,13 @@ def run_query() -> Response:
     error = None
 
     try:
-        # DuckDB query engine (default)
-        connection_maker = current_app.config["DUCKDB_CONNECTION_MAKER"]
-        con = connection_maker()
-        result = con.execute(query)
-        column_names = [desc[0] for desc in result.description] if result.description else []
-        rows = result.fetchall()
+        # DB-API 2.0 cursor from the configured backend (DuckDB by default,
+        # or any Arrow Flight SQL service when configured via env vars).
+        connection_maker = current_app.config["QUERY_CONNECTION_MAKER"]
+        cur = connection_maker()
+        cur.execute(query)
+        column_names = [desc[0] for desc in cur.description] if cur.description else []
+        rows = cur.fetchall()
 
     except Exception as e:
         error = str(e)

smoosense-py/smoosense/handlers/umap.py

@@ -82,11 +82,11 @@ def compute_umap() -> Response:
 
     try:
         query = f"SELECT {select_clause} FROM '{table_path}' {where_clause}"
-        connection_maker = current_app.config["DUCKDB_CONNECTION_MAKER"]
-        con = connection_maker()
-        result = con.execute(query)
-        column_names = [desc[0] for desc in result.description] if result.description else []
-        rows = result.fetchall()
+        connection_maker = current_app.config["QUERY_CONNECTION_MAKER"]
+        cur = connection_maker()
+        cur.execute(query)
+        column_names = [desc[0] for desc in cur.description] if cur.description else []
+        rows = cur.fetchall()
 
         # Find column indices
         emb_idx = column_names.index(emb_column)

smoosense-py/smoosense/utils/duckdb_connections.py

@@ -8,6 +8,8 @@
 from duckdb import DuckDBPyConnection
 from pydantic import ConfigDict, validate_call
 
+from smoosense.utils.query_backends import check_permissions as check_permissions  # re-export
+
 logger = logging.getLogger(__name__)
 
 
@@ -106,14 +108,6 @@ def maker() -> DuckDBPyConnection:
     return maker
 
 
-def check_permissions(query: str) -> None:
-    tokens = [w.lower() for w in query.split() if w]
-    forbidden = ["copy", "export", "delete", "attach", "update"]
-    if any(w in tokens for w in forbidden):
-        logger.warning(f"Forbidden query: {query}")
-        raise PermissionError("You are only allowed to run readonly queries")
-
-
 if __name__ == "__main__":
     cm = duckdb_connection_using_one_zone_s3()
     con = cm()