feat: Better unsupported file error (#63)

Author: AntoinePrv

.github/workflows/build.yml

@@ -18,10 +18,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Setup Pixi
-        uses: prefix-dev/setup-pixi@v0.9.3
+        uses: prefix-dev/setup-pixi@v0.9.6
         with:
-          pixi-version: v0.62.2
+          pixi-version: v0.70.2
           cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
           manifest-path: pyproject.toml
       - name: Run checks and formatters through pre-commit
         run: pixi run pre-commit run --all-files
@@ -33,10 +34,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Setup Pixi
-        uses: prefix-dev/setup-pixi@v0.9.3
+        uses: prefix-dev/setup-pixi@v0.9.6
         with:
-          pixi-version: v0.62.2
+          pixi-version: v0.70.2
           cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
           manifest-path: pyproject.toml
       - name: Test server extension
         run: pixi run test-pytest
@@ -88,6 +90,7 @@ jobs:
     name: Integration tests
     needs: test-unit
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     env:
       PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
     steps:
@@ -118,13 +121,13 @@ jobs:
       - name: Install browser
         run: |
           set -eux
-          jlpm playwright install-deps
-          jlpm playwright install chromium
+          npx playwright install-deps
+          npx playwright install chromium
         working-directory: ui-tests
       - name: Execute integration tests
         working-directory: ui-tests
         run: |
-          jlpm playwright test
+          npx playwright test
       - name: Upload Playwright Test report
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/enforce-label.yml

@@ -20,10 +20,11 @@ jobs:
           # pixi/commitlint from target branch.
           ref: ${{ github.event.pull_request.base.sha }}
       - name: Setup Pixi
-        uses: prefix-dev/setup-pixi@v0.9.3
+        uses: prefix-dev/setup-pixi@v0.9.6
         with:
-          pixi-version: v0.62.2
+          pixi-version: v0.70.2
           cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
           manifest-path: pyproject.toml
       - name: Verify conventional commit conventions on PR title
         id: commitlint

.pre-commit-config.yaml

@@ -44,13 +44,15 @@ repos:
         entry: pixi run fmt-biome""
         language: system
         types_or: [javascript, jsx, ts, tsx, json, css]
+        exclude: "(^|/)package\\.json$"
 
       # Linters
       - id: pixi-biome
         name: Run biome check through pixi
         entry: pixi run check-biome
         language: system
         types_or: [javascript, jsx, ts, tsx, json]
+        exclude: "(^|/)package\\.json$"
       - id: pixi-typescript
         name: Run typescript checks through pixi
         entry: pixi run check-typescript

arbalister/arrow.py

@@ -1,4 +1,5 @@
 import codecs
+import importlib.util
 import pathlib
 from typing import Any, Callable
 
@@ -9,6 +10,19 @@
 
 ReadCallable = Callable[..., dn.DataFrame]
 
+# Optional third-party modules required to read each format. Formats not listed here
+# rely only on the core dependencies and are always readable.
+_READER_REQUIREMENTS: dict[ff.FileFormat, tuple[str, ...]] = {
+    ff.FileFormat.Sqlite: ("adbc_driver_manager", "adbc_driver_sqlite"),
+}
+
+
+def missing_requirements(format: ff.FileFormat) -> list[str]:
+    """Return the modules required to read the format that are not installed."""
+    return [
+        module for module in _READER_REQUIREMENTS.get(format, ()) if importlib.util.find_spec(module) is None
+    ]
+
 
 def _read_csv(
     ctx: dn.SessionContext, path: str | pathlib.Path, delimiter: str, **kwargs: dict[str, Any]
@@ -22,7 +36,7 @@ def _read_ipc(ctx: dn.SessionContext, path: str | pathlib.Path, **kwargs: dict[s
     import pyarrow.feather
 
     #  table = pyarrow.feather.read_table(path, {**{"memory_map": True}, **kwargs})
-    table = pyarrow.feather.read_table(path, **kwargs)
+    table = pyarrow.feather.read_table(path, **kwargs)  # type: ignore[no-untyped-call]
     return ctx.from_arrow(table)
 
 
@@ -31,7 +45,7 @@ def _read_orc(ctx: dn.SessionContext, path: str | pathlib.Path, **kwargs: dict[s
     # Evolution for native datafusion reader
     import pyarrow.orc
 
-    table = pyarrow.orc.read_table(path, **kwargs)
+    table = pyarrow.orc.read_table(path, **kwargs)  # type: ignore[no-untyped-call]
     return ctx.from_arrow(table)
 
 
@@ -128,11 +142,11 @@ def write_csv(
                 memory_pool: pa.MemoryPool | None = None,
                 **kwargs: dict[str, Any],
             ) -> None:
-                pyarrow.csv.write_csv(
+                pyarrow.csv.write_csv(  # type: ignore[attr-defined]
                     data=data,
                     output_file=str(output_file),
                     memory_pool=memory_pool,
-                    write_options=pyarrow.csv.WriteOptions(**kwargs),
+                    write_options=pyarrow.csv.WriteOptions(**kwargs),  # type: ignore[attr-defined]
                 )
 
             out = write_csv

arbalister/routes.py

@@ -242,6 +242,42 @@ async def get(self, path: str) -> None:
                 await self.finish(dataclasses.asdict(no_response))
 
 
+@dataclasses.dataclass(frozen=True, slots=True)
+class FileSupportResponse:
+    """Whether a file can be read, and why not when it cannot."""
+
+    supported: bool
+    reason: str | None = None
+
+
+class FileSupportRouteHandler(BaseRouteHandler):
+    """A handler to check whether a file path is supported before opening it."""
+
+    @tornado.web.authenticated
+    async def get(self, path: str) -> None:
+        """HTTP GET return whether the file is supported."""
+        file = self.data_file(path)
+
+        try:
+            file_format = ff.FileFormat.from_filename(file)
+        except ValueError as error:
+            response = FileSupportResponse(supported=False, reason=str(error))
+            await self.finish(dataclasses.asdict(response))
+            return
+
+        missing = abw.missing_requirements(file_format)
+        if missing:
+            reason = (
+                f"Reading {file_format} files requires additional package(s) that are not "
+                f"installed: {', '.join(missing)}."
+            )
+            response = FileSupportResponse(supported=False, reason=reason)
+        else:
+            response = FileSupportResponse(supported=True)
+
+        await self.finish(dataclasses.asdict(response))
+
+
 def make_datafusion_config() -> dn.SessionConfig:
     """Return the datafusion config."""
     config = (
@@ -270,6 +306,11 @@ def setup_route_handlers(web_app: jupyter_server.serverapp.ServerWebApplication)
             FileInfoRouteHandler,
             {"context": context},
         ),
+        (
+            url_path_join(base_url, r"arbalister/file/supported/([^?]*)"),
+            FileSupportRouteHandler,
+            {"context": context},
+        ),
     ]
 
     web_app.add_handlers(host_pattern, handlers)  # type: ignore[no-untyped-call]

arbalister/tests/test_routes.py

@@ -229,6 +229,37 @@ async def test_stats_route(
     assert table.schema.names == full_table.schema.names
 
 
+async def test_file_supported_route(
+    jp_fetch: JpFetch,
+    table_file: pathlib.Path,
+) -> None:
+    """Test that a readable file is reported as supported."""
+    response = await jp_fetch("arbalister/file/supported/", str(table_file))
+
+    assert response.code == 200
+    assert response.headers["Content-Type"] == "application/json; charset=UTF-8"
+
+    payload = json.loads(response.body)
+    assert payload["supported"] is True
+    assert payload["reason"] is None
+
+
+async def test_file_supported_route_unknown_type(
+    jp_fetch: JpFetch,
+    jp_root_dir: pathlib.Path,
+) -> None:
+    """Test that an unknown file type is reported as unsupported with a reason."""
+    unknown = jp_root_dir / "data.unknown_ext"
+    unknown.write_text("nothing")
+
+    response = await jp_fetch("arbalister/file/supported/", "data.unknown_ext")
+
+    assert response.code == 200
+    payload = json.loads(response.body)
+    assert payload["supported"] is False
+    assert payload["reason"]
+
+
 async def test_file_info_route_sqlite(
     jp_fetch: JpFetch,
     table_file: pathlib.Path,

data/generate.py

@@ -51,9 +51,9 @@ def _generate_coordinate_table_slice(
         .with_columns(
             *[
                 dnf.concat(
-                    dn.lit("("),  # type: ignore[no-untyped-call]
+                    dn.lit("("),
                     dnf.col("row"),
-                    dn.lit(f", {j})"),  # type: ignore[no-untyped-call]
+                    dn.lit(f", {j})"),
                 ).alias(f"col_{j}")
                 for j in range(num_cols)
             ]
@@ -75,7 +75,7 @@ def _sink_coordinate_table(
         print(f"Generating {row_start}", flush=True)
         row_end = min(row_start + chunk_size, num_rows)
         table = _generate_coordinate_table_slice(row_start, row_end, num_cols=num_cols, ctx=ctx)
-        writer.write(table)
+        writer.write(table)  # type: ignore[no-untyped-call]
 
 
 def sink_coordinate_table(
@@ -87,9 +87,9 @@ def sink_coordinate_table(
     ctx = dn.SessionContext()
     print("Generating schema", flush=True)
     schema = _generate_coordinate_table_slice(0, 1, num_cols=num_cols, ctx=ctx).schema
-    writer = paq.ParquetWriter(path, schema)
+    writer = paq.ParquetWriter(path, schema)  # type: ignore[no-untyped-call]
     _sink_coordinate_table(num_rows=num_rows, num_cols=num_cols, writer=writer, ctx=ctx)
-    writer.close()
+    writer.close()  # type: ignore[no-untyped-call]
 
 
 def configure_command_single(cmd: argparse.ArgumentParser) -> argparse.ArgumentParser: