Add duckdb implementation

.github/workflows/build.yml

@@ -21,6 +21,13 @@ jobs:
         uses: actions/setup-python@v6.2.0
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Install DuckDB CLI
+        run: |
+          wget https://github.com/duckdb/duckdb/releases/download/v1.5.1/duckdb_cli-linux-amd64.zip
+          unzip duckdb_cli-linux-amd64.zip
+          chmod +x duckdb
+          sudo mv duckdb /usr/local/bin/duckdb
+          duckdb --version
       - name: Install application
         env:
           pythonversion: ${{ matrix.python-version }}

README.md

@@ -97,6 +97,7 @@ The following **command line clients** are used to access the various databases:
 | Big Query | `bq` | See the [Google Cloud SDK](https://cloud.google.com/sdk/docs/quickstarts) page for details. |
 | Snowflake | `snowsql` | See [SnowSQL (CLI Client)](https://docs.snowflake.com/en/user-guide/snowsql.html) |
 | Databricks | `dbsqlcli` | Included when using package extra `databricks` via package [databricks-sql-cli](https://pypi.org/project/databricks-sql-cli/). See [Databricks SQL CLI](https://docs.databricks.com/dev-tools/databricks-sql-cli.html#) |
+| DuckDB | `duckdb` | See [DuckDB Installation](https://duckdb.org/install/?environment=cli) |
 
  
 

docs/databases-overview.md

@@ -16,6 +16,7 @@ The following database engines are supported:
 | [Oracle Database]         | OracleDB            | -
 | [Snowflake]               | SnowflakeDB         | snowflake
 | [SQLite]                  | SQLiteDB            | sqlite
+| [DuckDB]                  | DuckDB              | duckdb
 
 
 [PostgreSQL]: https://www.postgresql.org/
@@ -27,6 +28,7 @@ The following database engines are supported:
 [Oracle Database]: https://www.oracle.com/database/
 [Snowflake]: https://www.snowflake.com/
 [SQLite]: https://www.sqlite.org/
+[DuckDB]: https://duckdb.org/
 [Microsoft SQL Server]: https://www.microsoft.com/en-us/sql-server
 [Azure Synapse Analytics]: https://azure.microsoft.com/en-us/services/synapse-analytics/
 
@@ -47,6 +49,7 @@ Shows which functions are supported with which database engine:
 | OracleDB            | Yes      | Yes          | -          | -          |
 | SnowflakeDB         | Yes      | Yes          | -          | -          |
 | SQLiteDB            | Yes      | Yes          | -          | Yes        |
+| DuckDB              | Yes      | Yes          | Yes        | Yes        |
 
 *Write STDOUT* gives the possibility to write a query to STDOUT
 
@@ -66,6 +69,7 @@ Shows the formats supported per database engine
 | RedshiftDB          | Yes | Yes   | -    | -       | -   |
 | BigQueryDB          | Yes | Yes   | Yes  | Yes     | Yes |
 | SQLServerDB         | Yes | -     | -    | -       | -   |
+| DuckDB              | Yes | Yes   | -    | -       | -   |
 
 
 ### Write STDOUT
@@ -81,6 +85,7 @@ Shows the formats supported per database engine
 | OracleDB            | Yes | -     | -    | -       | -   |
 | SnowflakeDB         | Yes | -     | -    | -       | -   |
 | SQLiteDB            | Yes | -     | -    | -       | -   |
+| DuckDB              | Yes | Yes   | -    | Yes     | Yes |
 
 
 Copy matrix
@@ -99,3 +104,4 @@ Shows which copy operations are implemented by default.
 | OracleDB     | Yes          | Yes        | Yes        | -            | -       | -           | -        | -           | -        |
 | SnowflakeDB  | -            | -          | -          | -            | -       | -           | -        | -           | -        |
 | SQLiteDB     | Yes          | Yes        | Yes        | -            | -       | -           | -        | -           | -        |
+| DuckDb       | -            | -          | -          | -            | -       | -           | -        | -           | -        |

docs/dbs/DuckDB.rst

@@ -0,0 +1,53 @@
+SQLite
+======
+
+
+
+Installation
+------------
+
+Use extras `duckdb` to install all required packages.
+
+.. code-block:: shell
+
+    $ pip install mara-db[duckdb]
+
+The shell command `duckdb` is required. You can find installation instructions at [DuckDB Install]
+
+[DuckDB Install]: https://duckdb.org/install/?environment=cli
+
+Configuration examples
+----------------------
+
+.. tabs::
+
+    .. group-tab:: Local file
+
+        .. code-block:: python
+
+            import mara_db.dbs
+            mara_db.config.databases = lambda: {
+                'dwh': mara_db.dbs.DuckDB(
+                    file_name='database.duckdb'),
+            }
+
+|
+
+|
+
+API reference
+-------------
+
+This section contains database specific API in the module.
+
+
+Configuration
+~~~~~~~~~~~~~
+
+.. module:: mara_db.dbs
+    :noindex:
+
+.. autoclass:: DuckDB
+    :special-members: __init__
+    :inherited-members:
+    :members:

docs/index.rst

@@ -57,6 +57,7 @@ This section focuses on the supported database engines.
    dbs/Mysql
    dbs/Snowflake
    dbs/SQLite
+   dbs/DuckDB
 
 
 CLI commands

mara_db/dbs.py

@@ -276,6 +276,21 @@ def sqlalchemy_url(self):
         return f"databricks+connector://token:{self.access_token}@{self.host}:443/"
 
 
+class DuckDB(DB):
+    def __init__(self, file_name: pathlib.Path) -> None:
+        """
+        Connection information for a DuckDB database
+
+        Args:
+            file_name: The name of the database file
+        """
+        self.file_name = file_name
+
+    @property
+    def sqlalchemy_url(self):
+        return f'duckdb:///{self.file_name}'
+
+
 
 @functools.singledispatch
 def connect(db: object, **kargs) -> object:
@@ -347,6 +362,12 @@ def __(db, **kargs) -> object:
         driver_path=db.odbc_driver_path)
 
 
+@connect.register(DuckDB)
+def __(db, **kargs) -> object:
+    import duckdb
+    return duckdb.connect(database=db.file_name)
+
+
 
 @contextlib.contextmanager
 def cursor_context(db: Union[str, DB]) -> object:

mara_db/shell.py

@@ -222,6 +222,16 @@ def __(db: dbs.DatabricksDB, timezone: str = None, echo_queries: bool = None):
             + ' -e /dev/stdin')
 
 
+@query_command.register(dbs.DuckDB)
+def __(db: dbs.DuckDB, timezone: str = None, echo_queries: bool = None):
+    assert timezone is None, "unimplemented parameter for DuckDB"
+    assert not echo_queries, "unimplemented parameter for DuckDB"
+    return ('duckdb'
+            + ' -bail'
+            + (f' {db.file_name}' if db.file_name else ''))
+
+
+
 # -------------------------------
 
 
@@ -526,6 +536,45 @@ def __(db: dbs.DatabricksDB, header: bool = None, footer: bool = None, delimiter
             + ('\n  | sed 1d' if remove_header else ''))
 
 
+@copy_to_stdout_command.register(dbs.DuckDB)
+def __(db: dbs.DuckDB, header: bool = None, footer: bool = None, delimiter_char: str = None, csv_format: bool = None,
+       pipe_format: formats.Format = None):
+    _check_format_with_args_used(pipe_format, header=header, footer=footer, delimiter_char=delimiter_char, csv_format=csv_format)
+    if not pipe_format:
+        pipe_format = _get_format_from_args(header=header, footer=footer, delimiter_char=delimiter_char, csv_format=csv_format)
+
+    extra_options = ''
+    copy_to_format_name = None
+
+    if isinstance(pipe_format, formats.ParquetFormat):
+        copy_to_format_name = 'parquet'
+    elif isinstance(pipe_format, formats.OrcFormat):
+        copy_to_format_name = 'orc'
+    elif isinstance(pipe_format, formats.CsvFormat):
+        if pipe_format.footer:
+            raise ValueError('pipe_format.footer is not supported for DuckDB')
+
+        if pipe_format.delimiter_char == '\t':
+            extra_options = ' -cmd ".mode tabs"'
+        else:
+            extra_options = (' -csv'
+                       + (f" -separator {pipe_format.delimiter_char}" if pipe_format.delimiter_char else ''))
+
+        extra_options += ((' -noheader' if not pipe_format.header else '')
+                     + (f" -nullvalue \"{pipe_format.null_value_string}\"" if pipe_format.null_value_string else ''))
+    elif isinstance(pipe_format, formats.NativeFormat):
+        extra_options = ' -csv'
+    elif isinstance(pipe_format, formats.JsonlFormat):
+        extra_options = ' -jsonlines'
+    else:
+        raise ValueError(f'Unsupported pipe_format for DuckDB: {pipe_format}')
+
+    return ((f'''| (echo "COPY (" && cat && echo ") TO STDOUT (FORMAT {copy_to_format_name}) ") \\\n  | ''' if copy_to_format_name else '')
+             + (query_command(db, echo_queries=False)
+                + ' -readonly'
+                + extra_options))
+
+
 # -------------------------------
 
 
@@ -800,6 +849,36 @@ def __(db: dbs.SqlcmdSQLServerDB, target_table: str, csv_format: bool = None, sk
             + '}')
 
 
+@copy_from_stdin_command.register(dbs.DuckDB)
+def __(db: dbs.DuckDB, target_table: str, csv_format: bool = None, skip_header: bool = None,
+       delimiter_char: str = None, quote_char: str = None, null_value_string: str = None, timezone: str = None,
+       pipe_format: formats.Format = None):
+    _check_format_with_args_used(pipe_format, header=skip_header, delimiter_char=delimiter_char, csv_format=csv_format,
+                                 quote_char=quote_char, null_value_string=null_value_string)
+    if not pipe_format:
+        pipe_format = _get_format_from_args(header=skip_header, delimiter_char=delimiter_char, csv_format=csv_format,
+                                            quote_char=quote_char, null_value_string=null_value_string)
+
+    read_command = None
+
+    if isinstance(pipe_format, formats.CsvFormat):
+        read_command = f"read_csv('/dev/stdin', delim = '{pipe_format.delimiter_char or ','}', header = {'true' if pipe_format.header else 'false'})"
+    elif isinstance(pipe_format, formats.JsonlFormat):
+        read_command = "read_json_objects_auto('/dev/stdin', format='newline_delimited')"
+    elif isinstance(pipe_format, formats.NativeFormat):
+        read_command = "read_csv_auto('/dev/stdin')"
+    else:
+        raise ValueError(f'Unsupported pipe_format for DuckDB: {pipe_format}')
+
+    return (query_command(db)
+            + f' -c "INSERT INTO {target_table}'
+            + (' ( data )' if isinstance(pipe_format, formats.JsonlFormat) else '')
+            + ' '
+            + 'SELECT *'
+            + (' AS data' if isinstance(pipe_format, formats.JsonlFormat) else '')
+            + f' FROM {read_command};"')
+
+
 # -------------------------------
 
 

setup.cfg

@@ -30,11 +30,16 @@ test =
     pytest-docker
     pytest-dependency
     SQLAlchemy>=1.2.0
+    duckdb
+    duckdb-engine
 bigquery =
     google-cloud-bigquery
     google-cloud-bigquery-storage
     pyarrow
     sqlalchemy-bigquery
+duckdb =
+    duckdb
+    duckdb-engine
 mssql = pyodbc
 mysql = mysqlclient
 postgres = psycopg2-binary>=2.7.3