diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index a5218df..b4d26c7 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.12' - name: Install dependencies run: python -m pip install .[dev] - name: Build diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 95b72fc..1e86004 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5.6.0 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: python -m pip install -r requirements.txt .[dev] - name: Build package diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 24655f1..86a41e2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.12" - name: Install dependencies run: python -m pip install .[qa] - name: Linting by ruff @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9, '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d67c351 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,73 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +locopy is a Python library for ETL processing with Amazon Redshift (`COPY`/`UNLOAD`) and Snowflake (`COPY INTO`). It wraps boto3 for S3 operations and is DB-API 2.0 adapter agnostic (tested with psycopg2, pg8000, snowflake-connector-python). Supports Python 3.10-3.14. + +## Common Commands + +### Install for development +```bash +pip install .[dev,psycopg2,pg8000,snowflake] +# Test data setup (needed for tests): +cp tests/data/.locopyrc ~/.locopyrc +cp tests/data/.locopy-sfrc ~/.locopy-sfrc +``` + +### Run tests +```bash +make not_integration # Unit tests only (default CI target) +make coverage # All tests with coverage +pytest tests/test_utility.py # Single test file +pytest tests/test_utility.py::test_find_column_type -v # Single test +pytest -m 'not integration' # Skip integration tests (same as make not_integration) +``` + +### Lint and format +```bash +ruff check # Lint +ruff check --fix # Lint with auto-fix +ruff format --check # Check formatting +ruff format # Auto-format +``` + +### Build docs +```bash +make sphinx +``` + +### Dependency version bumps (edgetest) +Edgetest config is in `pyproject.toml` under `[edgetest.envs.core]`. It bumps upper bounds for boto3, PyYAML, pandas, numpy. The CI workflow runs weekly and creates PRs with updated `pyproject.toml` and `requirements.txt`. The lockfile is generated via `uv pip compile --output-file=requirements.txt pyproject.toml`. + +## Architecture + +``` +locopy/ +├── database.py # Database - base class for DB connections (connect, execute, to_dataframe) +├── s3.py # S3 - boto3 wrapper for upload/download/delete on S3 buckets +├── redshift.py # Redshift(S3, Database) - multiple inheritance, adds COPY/UNLOAD + load_and_copy/unload_and_copy +├── snowflake.py # Snowflake(S3, Database) - multiple inheritance, adds COPY INTO + internal stage support +├── utility.py # Helpers: file splitting, compression, YAML config reading, column type detection +├── errors.py # Custom exception hierarchy: LocopyError, DBError, S3Error (each with sub-exceptions) +├── logger.py # Logging setup +└── _version.py # Single source of version (__version__) +``` + +**Key inheritance pattern:** Both `Redshift` and `Snowflake` use multiple inheritance from `S3` and `Database`. The `S3` class handles AWS session/credentials and file transfer. `Database` handles DB connection lifecycle and query execution. The subclasses override `connect()` to set up both S3 and DB connections. + +**Column type detection:** `utility.py` has `find_column_type` as a `@singledispatch` function with separate implementations for pandas (`find_column_type_pandas`) and polars (`find_column_type_polars`) DataFrames. When bumping pandas/polars versions, watch for dtype representation changes (e.g., pandas 3.0 changed string dtype from `object` to `StringDtype` and datetime resolution from `ns` to `us`). + +**Version** is defined in `locopy/_version.py` and read dynamically by setuptools via `pyproject.toml` (`[tool.setuptools.dynamic]`). + +## Code Style + +- Linter/formatter: **ruff** (config in `pyproject.toml`). Pre-commit hooks enforce ruff + trailing whitespace + debug statements. +- Docstring convention: **numpy style** (`[tool.ruff.lint.pydocstyle] convention = "numpy"`) +- Relative imports are banned (`ban-relative-imports = "all"`) +- Target Python version: 3.12 (ruff target) + +## Test Markers + +- `@pytest.mark.integration` - Integration tests requiring real DB/S3 connections (skipped in CI unit test runs) diff --git a/locopy/_version.py b/locopy/_version.py index 0e65f4b..3d63ec6 100644 --- a/locopy/_version.py +++ b/locopy/_version.py @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.6.8" +__version__ = "0.7.0" diff --git a/locopy/database.py b/locopy/database.py index de0dc6b..0c24c71 100644 --- a/locopy/database.py +++ b/locopy/database.py @@ -17,7 +17,7 @@ """Database Module.""" import time -from typing import Dict, Generator, List, Optional, Union +from typing import Dict, Generator, List import pandas import polars @@ -73,8 +73,8 @@ class Database: def __init__( self, dbapi: object, - config_yaml: Optional[str] = None, - **kwargs: Union[str, int], + config_yaml: str | None = None, + **kwargs: str | int, ) -> None: self.dbapi = dbapi self.connection = kwargs or {} @@ -205,8 +205,8 @@ def column_names(self) -> List[str]: return [column[0].lower() for column in self.cursor.description] def to_dataframe( - self, df_type: str = "pandas", size: Optional[int] = None - ) -> Optional[Union[pandas.DataFrame, polars.DataFrame]]: + self, df_type: str = "pandas", size: int | None = None + ) -> pandas.DataFrame | polars.DataFrame | None: """Return a dataframe of the last query results. Parameters @@ -244,7 +244,7 @@ def to_dataframe( elif df_type == "polars": return polars.DataFrame(fetched, schema=columns, orient="row") - def to_dict(self) -> Generator[Dict[str, Union[str, int, float]], None, None]: + def to_dict(self) -> Generator[Dict[str, str | int | float], None, None]: """Generate dictionaries of rows. Yields @@ -254,7 +254,7 @@ def to_dict(self) -> Generator[Dict[str, Union[str, int, float]], None, None]: """ columns = self.column_names() for row in self.cursor: - yield dict(zip(columns, row)) + yield dict(zip(columns, row, strict=False)) def _is_connected(self) -> bool: """Check the connection and cursor class arrtributes are initalized. diff --git a/locopy/s3.py b/locopy/s3.py index 65836d8..c1a066c 100644 --- a/locopy/s3.py +++ b/locopy/s3.py @@ -21,7 +21,7 @@ """ import os -from typing import List, Optional, Tuple +from typing import List, Tuple from boto3 import Session from boto3.s3.transfer import TransferConfig @@ -88,7 +88,7 @@ class S3: """ def __init__( - self, profile: Optional[str] = None, kms_key: Optional[str] = None, **kwargs + self, profile: str | None = None, kms_key: str | None = None, **kwargs ) -> None: self.profile = profile self.kms_key = kms_key @@ -153,7 +153,7 @@ def _generate_s3_path(self, bucket: str, key: str) -> str: """ return f"s3://{bucket}/{key}" - def _generate_unload_path(self, bucket: str, folder: Optional[str]) -> str: + def _generate_unload_path(self, bucket: str, folder: str | None) -> str: """Return the S3 file URL. If a valid (not None) folder is provided, returns in the format s3://bucket/folder. @@ -226,7 +226,7 @@ def upload_to_s3(self, local: str, bucket: str, key: str) -> None: raise S3UploadError("Error uploading to S3.") from e def upload_list_to_s3( - self, local_list: List[str], bucket: str, folder: Optional[str] = None + self, local_list: List[str], bucket: str, folder: str | None = None ) -> List[str]: """ Upload a list of files to a S3 bucket. @@ -300,7 +300,7 @@ def download_from_s3(self, bucket: str, key: str, local: str) -> None: raise S3DownloadError("Error downloading from S3.") from e def download_list_from_s3( - self, s3_list: List[str], local_path: Optional[str] = None + self, s3_list: List[str], local_path: str | None = None ) -> List[str]: """ Download a list of files from s3. diff --git a/locopy/utility.py b/locopy/utility.py index 1435d72..3a97edd 100644 --- a/locopy/utility.py +++ b/locopy/utility.py @@ -27,7 +27,7 @@ from collections import OrderedDict from functools import singledispatch from itertools import cycle -from typing import Dict, List, Union +from typing import Dict, List import pandas as pd import polars as pl @@ -47,7 +47,7 @@ def write_file( - data: List[List[Union[str, int, float]]], + data: List[List[str | int | float]], delimiter: str, filepath: str, mode: str = "w", @@ -225,7 +225,7 @@ def concatenate_files( raise LocopyConcatError("Error concateneating files.") from e -def read_config_yaml(config_yaml: Union[str, object]) -> Dict[str, Union[str, int]]: +def read_config_yaml(config_yaml: str | object) -> Dict[str, str | int]: """Read a configuration YAML file. Populate the database connection attributes, and validate required ones. @@ -357,13 +357,13 @@ def check_column_type_pyarrow(pa_dtype): datatype = check_column_type_pyarrow(data.dtype.pyarrow_dtype) column_type.append(datatype) else: - if (data.dtype in ["datetime64[ns]", "M8[ns]"]) or ( - re.match(r"(datetime64\[ns\,\W)([a-zA-Z]+)(\])", str(data.dtype)) - ): + if pd.api.types.is_datetime64_any_dtype(data.dtype): column_type.append("timestamp") elif str(data.dtype).lower().startswith("bool"): column_type.append("boolean") - elif str(data.dtype).startswith("object"): + elif str(data.dtype).startswith("object") or isinstance( + data.dtype, pd.StringDtype + ): data_type = validate_float_object(data) or validate_date_object(data) if not data_type: column_type.append("varchar") @@ -376,7 +376,7 @@ def check_column_type_pyarrow(pa_dtype): else: column_type.append("varchar") logger.info("Parsing column %s to %s", column, column_type[-1]) - return OrderedDict(zip(list(dataframe.columns), column_type)) + return OrderedDict(zip(list(dataframe.columns), column_type, strict=False)) @find_column_type.register(pl.DataFrame) @@ -467,7 +467,7 @@ def validate_float_object(column): else: column_type.append(data_type) logger.info("Parsing column %s to %s", column, column_type[-1]) - return OrderedDict(zip(list(dataframe.columns), column_type)) + return OrderedDict(zip(list(dataframe.columns), column_type, strict=False)) class ProgressPercentage: diff --git a/pyproject.toml b/pyproject.toml index 11daa3f..ed34e1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,125 +2,110 @@ name = "locopy" description = "Loading/Unloading to Amazon Redshift using Python" readme = "README.rst" +license = { text = "Apache Software License" } authors = [ - { name="Faisal Dosani", email="faisal.dosani@capitalone.com" }, + { name = "Faisal Dosani", email = "faisal.dosani@capitalone.com" }, ] -license = {text = "Apache Software License"} -dependencies = ["boto3<=1.42.87,>=1.9.92", "PyYAML<=6.0.2,>=5.1", "pandas<=2.3.3,>=1.5.0", "numpy<=2.2.6,>=1.22.0", "polars>=0.20.0", "pyarrow>=10.0.1"] - -requires-python = ">=3.9.0" +requires-python = ">=3.10.0" classifiers = [ - "Intended Audience :: Developers", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] - -dynamic = ["version"] - -[project.urls] -Homepage = "https://github.com/capitalone/locopy" -Documentation = "https://capitalone.github.io/locopy/" +dynamic = [ "version" ] +dependencies = [ + "boto3<=1.42.88,>=1.9.92", + "numpy<=2.4.4,>=1.22", + "pandas<=3.0.2,>=1.5", + "polars>=0.20", + "pyarrow>=10.0.1", + "pyyaml<=6.0.2,>=5.1", +] +optional-dependencies.build = [ "build", "twine", "wheel" ] +optional-dependencies.dev = [ "locopy[build]", "locopy[docs]", "locopy[qa]", "locopy[tests]" ] +optional-dependencies.docs = [ "sphinx", "sphinx-rtd-theme" ] +optional-dependencies.edgetest = [ "edgetest", "edgetest-conda" ] +optional-dependencies.pg8000 = [ "pg8000>=1.13.1" ] +optional-dependencies.psycopg2 = [ "psycopg2-binary>=2.7.7" ] +optional-dependencies.qa = [ "pre-commit", "ruff==0.11.12" ] +optional-dependencies.snowflake = [ "snowflake-connector-python[pandas]>=2.1.2" ] +optional-dependencies.tests = [ "hypothesis", "pytest", "pytest-cov" ] +urls.Documentation = "https://capitalone.github.io/locopy/" +urls.Homepage = "https://github.com/capitalone/locopy" [tool.setuptools] -packages = ["locopy"] +packages = [ "locopy" ] zip-safe = false - -[tool.setuptools.dynamic] -version = {attr = "locopy._version.__version__"} - -[project.optional-dependencies] -psycopg2 = ["psycopg2-binary>=2.7.7"] -pg8000 = ["pg8000>=1.13.1"] -snowflake = ["snowflake-connector-python[pandas]>=2.1.2"] -docs = ["sphinx", "sphinx_rtd_theme"] -tests = ["hypothesis", "pytest", "pytest-cov"] -qa = ["pre-commit", "ruff==0.11.12"] -build = ["build", "twine", "wheel"] -edgetest = ["edgetest", "edgetest-conda"] -dev = ["locopy[tests]", "locopy[docs]", "locopy[qa]", "locopy[build]"] +dynamic.version = { attr = "locopy._version.__version__" } # Linters, formatters and type checkers [tool.ruff] -extend-include = ["*.ipynb"] -target-version = "py39" -src = ["src"] - - -[tool.ruff.lint] -preview = true -select = [ - "E", # pycodestyle errors - "W", # pycodestyle warnings - "F", # pyflakes - "D", # pydocstyle - "I", # isort - "UP", # pyupgrade - "B", # flake8-bugbear - # "A", # flake8-builtins - "C4", # flake8-comprehensions - #"C901", # mccabe complexity - # "G", # flake8-logging-format - "T20", # flake8-print - "TID252", # flake8-tidy-imports ban relative imports - # "ARG", # flake8-unused-arguments - "SIM", # flake8-simplify - "NPY", # numpy rules - "LOG", # flake8-logging - "RUF", # Ruff errors +target-version = "py312" +src = [ "src" ] +extend-include = [ "*.ipynb" ] +lint.select = [ + "B", # flake8-bugbear + # "A", # flake8-builtins + "C4", # flake8-comprehensions + "D", # pydocstyle + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "LOG", # flake8-logging + "NPY", # numpy rules + "RUF", # Ruff errors + # "ARG", # flake8-unused-arguments + "SIM", # flake8-simplify + # "C901", # mccabe complexity + # "G", # flake8-logging-format + "T20", # flake8-print + "TID252", # flake8-tidy-imports ban relative imports + "UP", # pyupgrade + "W", # pycodestyle warnings ] - - -ignore = [ - "E111", # Check indentation level. Using formatter instead. - "E114", # Check indentation level. Using formatter instead. - "E117", # Check indentation level. Using formatter instead. - "E203", # Check whitespace. Using formatter instead. - "E501", # Line too long. Using formatter instead. - "D206", # Docstring indentation. Using formatter instead. - "D300", # Use triple single quotes. Using formatter instead. - "SIM108", # Use ternary operator instead of if-else blocks. - "SIM105", # Use `contextlib.suppress(FileNotFoundError)` instead of `try`-`except`-`pass` - "UP035", # `typing.x` is deprecated, use `x` instead - "UP006", # `typing.x` is deprecated, use `x` instead +lint.ignore = [ + "D206", # Docstring indentation. Using formatter instead. + "D300", # Use triple single quotes. Using formatter instead. + "E111", # Check indentation level. Using formatter instead. + "E114", # Check indentation level. Using formatter instead. + "E117", # Check indentation level. Using formatter instead. + "E203", # Check whitespace. Using formatter instead. + "E501", # Line too long. Using formatter instead. + "SIM105", # Use `contextlib.suppress(FileNotFoundError)` instead of `try`-`except`-`pass` + "SIM108", # Use ternary operator instead of if-else blocks. + "UP006", # `typing.x` is deprecated, use `x` instead + "UP035", # `typing.x` is deprecated, use `x` instead +] +lint.per-file-ignores."**/{tests,docs}/*" = [ "ARG", "D", "E402", "F841" ] +lint.per-file-ignores."__init__.py" = [ "E402" ] +lint.flake8-tidy-imports.ban-relative-imports = "all" +lint.pydocstyle.convention = "numpy" +lint.preview = true + +[tool.pytest] +ini_options.markers = [ + "integration", ] - - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402"] -"**/{tests,docs}/*" = ["E402", "D", "F841", "ARG"] - - -[tool.ruff.lint.flake8-tidy-imports] -ban-relative-imports = "all" - - -[tool.ruff.lint.pydocstyle] -convention = "numpy" [edgetest.envs.core] -python_version = "3.10" +python_version = "3.12" extras = [ - "tests", - "psycopg2", - "pg8000", - "snowflake", + "tests", + "psycopg2", + "pg8000", + "snowflake", ] command = "pytest tests -m 'not integration'" upgrade = [ - "boto3", - "PyYAML", - "pandas", - "numpy", -] - -[tool.pytest.ini_options] -markers = [ - "integration", + "boto3", + "PyYAML", + "pandas", + "numpy", ] diff --git a/tests/test_utility.py b/tests/test_utility.py index 11a6bf0..bda1a05 100644 --- a/tests/test_utility.py +++ b/tests/test_utility.py @@ -368,7 +368,7 @@ def test_find_column_type_new(): input_text = input_text.astype( dtype={ "a": pd.Int64Dtype(), - "b": pd.DatetimeTZDtype(tz=datetime.timezone.utc), + "b": pd.DatetimeTZDtype(tz=datetime.timezone.utc), # noqa: UP017 "c": pd.Float64Dtype(), "d": pd.StringDtype(), "e": pd.BooleanDtype(),