diff --git a/providers/sftp/pyproject.toml b/providers/sftp/pyproject.toml index 29ed8a12d56f3..8a8ea707df388 100644 --- a/providers/sftp/pyproject.toml +++ b/providers/sftp/pyproject.toml @@ -66,6 +66,7 @@ dependencies = [ # TODO: Bump to >= 4.0.0 once https://github.com/apache/airflow/issues/54079 is handled "paramiko>=3.5.1,<4.0.0", "asyncssh>=2.12.0; python_version < '3.14'", + "asgiref>=3.5.2", "asyncssh>=2.22.0; python_version >= '3.14'", ] diff --git a/providers/sftp/src/airflow/providers/sftp/constants.py b/providers/sftp/src/airflow/providers/sftp/constants.py new file mode 100644 index 0000000000000..eeef10cc02aeb --- /dev/null +++ b/providers/sftp/src/airflow/providers/sftp/constants.py @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Constants for the SFTP provider.""" + +from __future__ import annotations + + +class SFTPOperation: + """Operation that can be used with SFTP.""" + + PUT = "put" + GET = "get" + DELETE = "delete" diff --git a/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py b/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py index 2f31c66d3b24e..25ddf2367ab83 100644 --- a/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py +++ b/providers/sftp/src/airflow/providers/sftp/hooks/sftp.py @@ -706,6 +706,75 @@ def get_files_by_pattern(self, path, fnmatch_pattern) -> list[str]: return matched_files + def transfer( + self, + operation: str, + local_filepath: str | list[str] | None, + remote_filepath: str | list[str], + confirm: bool = True, + create_intermediate_dirs: bool = False, + concurrency: int = 1, + prefetch: bool = True, + ) -> None: + """ + Perform a synchronous SFTP transfer operation (GET, PUT, or DELETE). + + Centralises transfer logic so both the operator and the trigger + can delegate to the hook, in line with the DRY principle. + + :param operation: The SFTP operation - put, get, or delete. + :param local_filepath: Local file path(s). + :param remote_filepath: Remote file path(s). + :param confirm: Whether to confirm file size after PUT (default: True). + :param create_intermediate_dirs: Create missing intermediate directories (default: False). + :param concurrency: Number of threads for directory transfers (default: 1). + :param prefetch: Whether to prefetch during GET (default: True). + """ + from airflow.providers.sftp.constants import SFTPOperation + + if isinstance(local_filepath, str): + local_filepath_array = [local_filepath] if local_filepath else [] + else: + local_filepath_array = local_filepath or [] + + if isinstance(remote_filepath, str): + remote_filepath_array = [remote_filepath] + else: + remote_filepath_array = list(remote_filepath) + + if operation.lower() == SFTPOperation.GET: + for local, remote in zip(local_filepath_array, remote_filepath_array): + if create_intermediate_dirs: + Path(os.path.dirname(local)).mkdir(parents=True, exist_ok=True) + if self.isdir(remote): + if concurrency > 1: + self.retrieve_directory_concurrently( + remote, local, workers=concurrency, prefetch=prefetch + ) + else: + self.retrieve_directory(remote, local) + else: + self.retrieve_file(remote, local, prefetch=prefetch) + elif operation.lower() == SFTPOperation.PUT: + for local, remote in zip(local_filepath_array, remote_filepath_array): + if create_intermediate_dirs: + self.create_directory(os.path.dirname(remote)) + if os.path.isdir(local): + if concurrency > 1: + self.store_directory_concurrently( + remote, local, confirm=confirm, workers=concurrency + ) + else: + self.store_directory(remote, local, confirm=confirm) + else: + self.store_file(remote, local, confirm=confirm) + elif operation.lower() == SFTPOperation.DELETE: + for remote in remote_filepath_array: + if self.isdir(remote): + self.delete_directory(remote, include_files=True) + else: + self.delete_file(remote) + class SFTPHookAsync(BaseHook): """ @@ -1040,3 +1109,27 @@ async def get_mod_time(self, path: str) -> str: # type: ignore[return] return mod_time except asyncssh.SFTPNoSuchFile: raise AirflowException("No files matching") + + async def transfer( + self, + operation: str, + local_filepath: str | list[str] | None, + remote_filepath: str | list[str], + confirm: bool = True, + create_intermediate_dirs: bool = False, + concurrency: int = 1, + prefetch: bool = True, + ) -> None: + """Perform an SFTP transfer operation (GET, PUT, or DELETE) via a thread executor.""" + from asgiref.sync import sync_to_async + + sync_hook = SFTPHook(ssh_conn_id=self.sftp_conn_id) + await sync_to_async(sync_hook.transfer)( + operation=operation, + local_filepath=local_filepath, + remote_filepath=remote_filepath, + confirm=confirm, + create_intermediate_dirs=create_intermediate_dirs, + concurrency=concurrency, + prefetch=prefetch, + ) \ No newline at end of file diff --git a/providers/sftp/src/airflow/providers/sftp/operators/sftp.py b/providers/sftp/src/airflow/providers/sftp/operators/sftp.py index 0b47b5b7d5ddb..7c83532b5ca10 100644 --- a/providers/sftp/src/airflow/providers/sftp/operators/sftp.py +++ b/providers/sftp/src/airflow/providers/sftp/operators/sftp.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,29 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This module contains SFTP operator.""" from __future__ import annotations import errno -import os import socket from collections.abc import Sequence -from pathlib import Path from typing import Any import paramiko -from airflow.providers.common.compat.sdk import AirflowException, BaseOperator +from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, conf +from airflow.providers.sftp.constants import SFTPOperation from airflow.providers.sftp.hooks.sftp import SFTPHook - - -class SFTPOperation: - """Operation that can be used with SFTP.""" - - PUT = "put" - GET = "get" - DELETE = "delete" +from airflow.providers.sftp.triggers.sftp import SFTPTrigger class SFTPOperator(BaseOperator): @@ -95,6 +85,7 @@ def __init__( create_intermediate_dirs: bool = False, concurrency: int = 1, prefetch: bool = True, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), **kwargs, ) -> None: super().__init__(**kwargs) @@ -108,8 +99,25 @@ def __init__( self.remote_filepath = remote_filepath self.concurrency = concurrency self.prefetch = prefetch + self.deferrable = deferrable def execute(self, context: Any) -> str | list[str] | None: + if self.deferrable: + self.defer( + trigger=SFTPTrigger( + ssh_conn_id=self.ssh_conn_id, + local_filepath=self.local_filepath, + remote_filepath=self.remote_filepath, + operation=self.operation, + confirm=self.confirm, + create_intermediate_dirs=self.create_intermediate_dirs, + remote_host=self.remote_host, + concurrency=self.concurrency, + prefetch=self.prefetch, + ), + method_name="execute_complete", + ) + if self.local_filepath is None: local_filepath_array = [] elif isinstance(self.local_filepath, str): @@ -163,62 +171,17 @@ def execute(self, context: Any) -> str | list[str] | None: if not self.sftp_hook: raise AirflowException("Cannot operate without sftp_hook or ssh_conn_id.") - if self.operation.lower() in (SFTPOperation.GET, SFTPOperation.PUT): - for _local_filepath, _remote_filepath in zip(local_filepath_array, remote_filepath_array): - if self.operation.lower() == SFTPOperation.GET: - local_folder = os.path.dirname(_local_filepath) - if self.create_intermediate_dirs: - Path(local_folder).mkdir(parents=True, exist_ok=True) - file_msg = f"from {_remote_filepath} to {_local_filepath}" - self.log.info("Starting to transfer %s", file_msg) - if self.sftp_hook.isdir(_remote_filepath): - if self.concurrency > 1: - self.sftp_hook.retrieve_directory_concurrently( - _remote_filepath, - _local_filepath, - workers=self.concurrency, - prefetch=self.prefetch, - ) - elif self.concurrency == 1: - self.sftp_hook.retrieve_directory(_remote_filepath, _local_filepath) - else: - self.sftp_hook.retrieve_file(_remote_filepath, _local_filepath) - elif self.operation.lower() == SFTPOperation.PUT: - remote_folder = os.path.dirname(_remote_filepath) - if self.create_intermediate_dirs: - self.sftp_hook.create_directory(remote_folder) - file_msg = f"from {_local_filepath} to {_remote_filepath}" - self.log.info("Starting to transfer file %s", file_msg) - if os.path.isdir(_local_filepath): - if self.concurrency > 1: - self.sftp_hook.store_directory_concurrently( - _remote_filepath, - _local_filepath, - confirm=self.confirm, - workers=self.concurrency, - ) - elif self.concurrency == 1: - self.sftp_hook.store_directory( - _remote_filepath, _local_filepath, confirm=self.confirm - ) - else: - self.sftp_hook.store_file(_remote_filepath, _local_filepath, confirm=self.confirm) - elif self.operation.lower() == SFTPOperation.DELETE: - for _remote_filepath in remote_filepath_array: - file_msg = f"{_remote_filepath}" - self.log.info("Starting to delete %s", file_msg) - try: - if self.sftp_hook.isdir(_remote_filepath): - self.sftp_hook.delete_directory(_remote_filepath, include_files=True) - else: - self.sftp_hook.delete_file(_remote_filepath) - except OSError as exc: - if self._is_missing_path_error(exc): - self.log.warning( - "Remote path %s does not exist. Skipping delete.", _remote_filepath - ) - continue - raise + file_msg = f"{self.operation.upper()} {self.local_filepath} <-> {self.remote_filepath}" + self.log.info("Starting to transfer %s", file_msg) + self.sftp_hook.transfer( + operation=self.operation, + local_filepath=self.local_filepath, + remote_filepath=self.remote_filepath, + confirm=self.confirm, + create_intermediate_dirs=self.create_intermediate_dirs, + concurrency=self.concurrency, + prefetch=self.prefetch, + ) except Exception as e: raise AirflowException( @@ -227,6 +190,21 @@ def execute(self, context: Any) -> str | list[str] | None: return self.local_filepath + def execute_complete(self, context: Any, event: dict) -> str | list[str] | None: + """ + Execute when the trigger fires in deferrable mode. + + :param context: The task context. + :param event: The event yielded by SFTPTrigger. + :return: The local filepath(s). + """ + if event.get("status") == "error": + raise AirflowException( + f"Error during deferrable SFTP {self.operation.upper()} operation: {event.get('message')}" + ) + self.log.info("File transfer completed successfully via deferrable mode.") + return event.get("local_filepath") + @staticmethod def _is_missing_path_error(exc: Exception) -> bool: if isinstance(exc, FileNotFoundError): @@ -316,4 +294,4 @@ def get_openlineage_facets_on_start(self): def _get_namespace(self, scheme, host, port, path) -> str: port = port or paramiko.config.SSH_PORT authority = f"{host}:{port}" - return f"{scheme}://{authority}" + return f"{scheme}://{authority}" \ No newline at end of file diff --git a/providers/sftp/src/airflow/providers/sftp/triggers/sftp.py b/providers/sftp/src/airflow/providers/sftp/triggers/sftp.py index 6ecdbaa158940..35fb2082e3f6e 100644 --- a/providers/sftp/src/airflow/providers/sftp/triggers/sftp.py +++ b/providers/sftp/src/airflow/providers/sftp/triggers/sftp.py @@ -139,3 +139,135 @@ async def run(self) -> AsyncIterator[TriggerEvent]: def _get_async_hook(self) -> SFTPHookAsync: return SFTPHookAsync(sftp_conn_id=self.sftp_conn_id) + + +class SFTPTrigger(BaseTrigger): + """ + Trigger for SFTPOperator deferrable mode. + + Fires when a file transfer (PUT, GET, or DELETE) completes + on the SFTP server, freeing the worker slot during the transfer. + + :param ssh_conn_id: The SSH connection ID to use. + :param local_filepath: Local file path(s) to transfer. + :param remote_filepath: Remote file path(s) on the SFTP server. + :param operation: The SFTP operation - put, get, or delete. + :param confirm: Whether to confirm the file transfer. + :param create_intermediate_dirs: Whether to create intermediate dirs. + :param remote_host: Remote host to connect to (overrides connection). + :param concurrency: Number of threads for directory transfers. + :param prefetch: Whether to prefetch during file retrieval. + """ + + def __init__( + self, + ssh_conn_id: str | None = None, + local_filepath: str | list[str] | None = None, + remote_filepath: str | list[str] = "", + operation: str = "put", + confirm: bool = True, + create_intermediate_dirs: bool = False, + remote_host: str | None = None, + concurrency: int = 1, + prefetch: bool = True, + ) -> None: + super().__init__() + self.ssh_conn_id = ssh_conn_id + self.local_filepath = local_filepath + self.remote_filepath = remote_filepath + self.operation = operation + self.confirm = confirm + self.create_intermediate_dirs = create_intermediate_dirs + self.remote_host = remote_host + self.concurrency = concurrency + self.prefetch = prefetch + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serialize the trigger for storage in the database.""" + return ( + f"{self.__class__.__module__}.{self.__class__.__name__}", + { + "ssh_conn_id": self.ssh_conn_id, + "local_filepath": self.local_filepath, + "remote_filepath": self.remote_filepath, + "operation": self.operation, + "confirm": self.confirm, + "create_intermediate_dirs": self.create_intermediate_dirs, + "remote_host": self.remote_host, + "concurrency": self.concurrency, + "prefetch": self.prefetch, + }, + ) + + async def run(self) -> AsyncIterator[TriggerEvent]: + """Run the file transfer asynchronously and yield a TriggerEvent when done.""" + try: + loop = asyncio.get_running_loop() + await loop.run_in_executor( + None, + self._do_transfer, + ) + yield TriggerEvent( + { + "status": "success", + "local_filepath": self.local_filepath, + } + ) + except Exception as e: + yield TriggerEvent({"status": "error", "message": str(e)}) + + def _do_transfer(self) -> None: + """Run the actual synchronous SFTP transfer in a thread executor.""" + import os + from pathlib import Path + + from airflow.providers.sftp.constants import SFTPOperation + from airflow.providers.sftp.hooks.sftp import SFTPHook + + sftp_hook = SFTPHook( + ssh_conn_id=self.ssh_conn_id, + remote_host=self.remote_host or "", + ) + + if isinstance(self.local_filepath, str): + local_filepath_array = [self.local_filepath] if self.local_filepath else [] + else: + local_filepath_array = self.local_filepath or [] + + if isinstance(self.remote_filepath, str): + remote_filepath_array = [self.remote_filepath] + else: + remote_filepath_array = list(self.remote_filepath) + + if self.operation.lower() == SFTPOperation.GET: + for local, remote in zip(local_filepath_array, remote_filepath_array): + if self.create_intermediate_dirs: + Path(os.path.dirname(local)).mkdir(parents=True, exist_ok=True) + if sftp_hook.isdir(remote): + if self.concurrency > 1: + sftp_hook.retrieve_directory_concurrently( + remote, local, workers=self.concurrency, prefetch=self.prefetch + ) + else: + sftp_hook.retrieve_directory(remote, local) + else: + sftp_hook.retrieve_file(remote, local, prefetch=self.prefetch) + elif self.operation.lower() == SFTPOperation.PUT: + for local, remote in zip(local_filepath_array, remote_filepath_array): + if self.create_intermediate_dirs: + sftp_hook.create_directory(os.path.dirname(remote)) + if os.path.isdir(local): + if self.concurrency > 1: + sftp_hook.store_directory_concurrently( + remote, local, confirm=self.confirm, workers=self.concurrency + ) + else: + sftp_hook.store_directory(remote, local, confirm=self.confirm) + else: + sftp_hook.store_file(remote, local, confirm=self.confirm) + elif self.operation.lower() == SFTPOperation.DELETE: + for remote in remote_filepath_array: + if sftp_hook.isdir(remote): + sftp_hook.delete_directory(remote, include_files=True) + else: + sftp_hook.delete_file(remote) diff --git a/providers/sftp/tests/unit/sftp/operators/test_sftp.py b/providers/sftp/tests/unit/sftp/operators/test_sftp.py index 815d981320107..09ab48f74270c 100644 --- a/providers/sftp/tests/unit/sftp/operators/test_sftp.py +++ b/providers/sftp/tests/unit/sftp/operators/test_sftp.py @@ -28,11 +28,13 @@ import paramiko import pytest +from airflow.exceptions import TaskDeferred from airflow.models import DAG, Connection from airflow.providers.common.compat.openlineage.facet import Dataset from airflow.providers.common.compat.sdk import AirflowException from airflow.providers.sftp.hooks.sftp import SFTPHook from airflow.providers.sftp.operators.sftp import SFTPOperation, SFTPOperator +from airflow.providers.sftp.triggers.sftp import SFTPTrigger from airflow.providers.ssh.hooks.ssh import SSHHook from airflow.providers.ssh.operators.ssh import SSHOperator from airflow.utils import timezone @@ -675,3 +677,123 @@ def test_extract_sftp_hook(self, get_connection, get_conn, operation, expected): assert lineage.inputs == expected[0] assert lineage.outputs == expected[1] + + +class TestSFTPOperatorDeferrable: + """Tests for SFTPOperator deferrable mode.""" + + def test_sftp_operator_defers_when_deferrable_true(self): + """Test that SFTPOperator defers when deferrable=True.""" + operator = SFTPOperator( + task_id="test_sftp_defer", + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation=SFTPOperation.PUT, + deferrable=True, + ) + with pytest.raises(TaskDeferred) as exc: + operator.execute(context={}) + assert isinstance(exc.value.trigger, SFTPTrigger) + assert exc.value.method_name == "execute_complete" + + def test_sftp_operator_execute_complete_success(self): + """Test execute_complete returns local_filepath on success.""" + operator = SFTPOperator( + task_id="test_sftp_complete", + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation=SFTPOperation.PUT, + deferrable=True, + ) + event = {"status": "success", "local_filepath": "/tmp/test.txt"} + result = operator.execute_complete(context={}, event=event) + assert result == "/tmp/test.txt" + + def test_sftp_operator_execute_complete_raises_on_error(self): + """Test execute_complete raises AirflowException on error.""" + operator = SFTPOperator( + task_id="test_sftp_error", + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation=SFTPOperation.PUT, + deferrable=True, + ) + event = {"status": "error", "message": "Connection refused"} + with pytest.raises(AirflowException, match="Connection refused"): + operator.execute_complete(context={}, event=event) + + +class TestSFTPTrigger: + """Tests for SFTPTrigger.""" + + def test_serialize_roundtrip(self): + """Test that serialize() produces correct output for reconstruction.""" + trigger = SFTPTrigger( + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation="put", + confirm=True, + create_intermediate_dirs=False, + remote_host=None, + concurrency=1, + prefetch=True, + ) + classpath, kwargs = trigger.serialize() + assert classpath == "airflow.providers.sftp.triggers.sftp.SFTPTrigger" + assert kwargs["ssh_conn_id"] == "ssh_default" + assert kwargs["local_filepath"] == "/tmp/test.txt" + assert kwargs["remote_filepath"] == "/remote/test.txt" + assert kwargs["operation"] == "put" + assert kwargs["confirm"] is True + assert kwargs["remote_host"] is None + assert kwargs["concurrency"] == 1 + assert kwargs["prefetch"] is True + + def test_run_success(self): + """Test run() yields TriggerEvent with status success.""" + import asyncio + from unittest.mock import patch + + trigger = SFTPTrigger( + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation="put", + ) + with patch.object(trigger, "_do_transfer", return_value=None): + events = [] + + async def collect(): + async for event in trigger.run(): + events.append(event) + + asyncio.run(collect()) + assert len(events) == 1 + assert events[0].payload["status"] == "success" + + def test_run_error(self): + """Test run() yields TriggerEvent with status error on exception.""" + import asyncio + from unittest.mock import patch + + trigger = SFTPTrigger( + ssh_conn_id="ssh_default", + local_filepath="/tmp/test.txt", + remote_filepath="/remote/test.txt", + operation="put", + ) + with patch.object(trigger, "_do_transfer", side_effect=Exception("Connection failed")): + events = [] + + async def collect(): + async for event in trigger.run(): + events.append(event) + + asyncio.run(collect()) + assert len(events) == 1 + assert events[0].payload["status"] == "error" + assert "Connection failed" in events[0].payload["message"] diff --git a/providers/sftp/tests/unit/sftp/test_constants.py b/providers/sftp/tests/unit/sftp/test_constants.py new file mode 100644 index 0000000000000..b349ae0ac648d --- /dev/null +++ b/providers/sftp/tests/unit/sftp/test_constants.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.sftp.constants import SFTPOperation + + +def test_sftp_operation_values(): + assert SFTPOperation.PUT == "put" + assert SFTPOperation.GET == "get" diff --git a/scripts/ci/prek/known_airflow_exceptions.txt b/scripts/ci/prek/known_airflow_exceptions.txt index f1ddfbd1efc27..186d6b6899146 100644 --- a/scripts/ci/prek/known_airflow_exceptions.txt +++ b/scripts/ci/prek/known_airflow_exceptions.txt @@ -231,7 +231,6 @@ providers/google/src/airflow/providers/google/cloud/hooks/gdm.py::1 providers/google/src/airflow/providers/google/cloud/hooks/kubernetes_engine.py::2 providers/google/src/airflow/providers/google/cloud/hooks/looker.py::8 providers/google/src/airflow/providers/google/cloud/hooks/managed_kafka.py::1 -providers/google/src/airflow/providers/google/cloud/hooks/mlengine.py::2 providers/google/src/airflow/providers/google/cloud/hooks/spanner.py::5 providers/google/src/airflow/providers/google/cloud/hooks/stackdriver.py::2 providers/google/src/airflow/providers/google/cloud/hooks/tasks.py::3 @@ -381,7 +380,7 @@ providers/presto/src/airflow/providers/presto/hooks/presto.py::1 providers/samba/src/airflow/providers/samba/transfers/gcs_to_samba.py::1 providers/segment/src/airflow/providers/segment/hooks/segment.py::2 providers/sftp/src/airflow/providers/sftp/hooks/sftp.py::6 -providers/sftp/src/airflow/providers/sftp/operators/sftp.py::2 +providers/sftp/src/airflow/providers/sftp/operators/sftp.py::3 providers/sftp/src/airflow/providers/sftp/sensors/sftp.py::4 providers/singularity/src/airflow/providers/singularity/operators/singularity.py::2 providers/slack/src/airflow/providers/slack/hooks/slack.py::1