Support wildcards in instance-to-host transfers

docs/reference/command-line-interface/transfer.md

@@ -43,6 +43,16 @@ multipass transfer --recursive ample-pigeon:dir .
 
 ```{caution}Symbolic links are not followed during recursive transfer.```
 
+When copying from an instance to the host, source paths can contain `*`, `?`, and character-range
+wildcards. Quote the source so that your host shell passes the pattern to Multipass unchanged:
+
+```{code-block} text
+multipass transfer 'ample-pigeon:logs/*.txt' .
+```
+
+If the pattern matches multiple files, the destination must be a directory. Wildcards in local
+source paths are expanded by the host shell as usual.
+
 ---
 
 The full `multipass help transfer` output explains the available options:

include/multipass/ssh/sftp_client.h

@@ -24,6 +24,7 @@
 #include <filesystem>
 #include <functional>
 #include <iostream>
+#include <vector>
 
 #include <QFlags>
 
@@ -54,6 +55,7 @@ class SFTPClient
     SFTPClient(SSHSessionUPtr ssh_session);
 
     virtual bool is_remote_dir(const fs::path& path);
+    virtual std::vector<fs::path> expand_remote_path(const fs::path& path);
     virtual bool push(const fs::path& source_path, const fs::path& target_path, Flags flags = {});
     virtual bool pull(const fs::path& source_path, const fs::path& target_path, Flags flags = {});
     virtual void from_cin(std::istream& cin, const fs::path& target_path, bool make_parent);

src/client/cli/cmd/transfer.cpp

@@ -58,15 +58,27 @@ mp::ReturnCodeVariant cmd::Transfer::run(mp::ArgParser* parser)
                 if (const auto args = std::get_if<InstanceSourcesLocalTarget>(&arguments); args)
                 {
                     auto& [sources, target] = *args;
+                    std::vector<fs::path> expanded_sources;
+                    for (auto [source, source_end] = sources.equal_range(instance_name);
+                         source != source_end;
+                         ++source)
+                    {
+                        auto matches = sftp_client->expand_remote_path(source->second);
+                        std::move(matches.begin(),
+                                  matches.end(),
+                                  std::back_inserter(expanded_sources));
+                    }
+
                     std::error_code err;
-                    if (sources.size() > 1 && !MP_FILEOPS.is_directory(target, err) && !err)
+                    if ((sources.size() > 1 || expanded_sources.size() > 1) &&
+                        !MP_FILEOPS.is_directory(target, err) && !err)
                         throw std::runtime_error{
                             fmt::format("Target {:?} is not a directory", target)};
                     else if (err)
                         throw std::runtime_error{
                             fmt::format("Cannot access {:?}: {}", target, err.message())};
-                    for (auto [s, s_end] = sources.equal_range(instance_name); s != s_end; ++s)
-                        success &= sftp_client->pull(s->second, target, flags);
+                    for (const auto& source : expanded_sources)
+                        success &= sftp_client->pull(source, target, flags);
                 }
 
                 if (const auto args = std::get_if<LocalSourcesInstanceTarget>(&arguments); args)

src/ssh/sftp_client.cpp

@@ -27,15 +27,63 @@
 #include <multipass/ssh/throw_on_error.h>
 #include <multipass/utils.h>
 
+#include <algorithm>
 #include <array>
 #include <fcntl.h>
 #include <fmt/std.h>
 #include <functional>
+#include <iterator>
+#include <string_view>
+
+#include <QRegularExpression>
 
 constexpr int file_mode = 0664;
 const std::string stream_file_name{"stream_output.dat"};
 const char* log_category = "sftp";
 
+namespace
+{
+bool has_wildcards(const std::string_view value)
+{
+    return value.find_first_of("*?[") != std::string_view::npos;
+}
+
+std::vector<std::string> split_remote_path(const std::string_view path)
+{
+    std::vector<std::string> components;
+    for (std::size_t begin = 0; begin < path.size();)
+    {
+        begin = path.find_first_not_of('/', begin);
+        if (begin == std::string_view::npos)
+            break;
+
+        const auto end = path.find('/', begin);
+        components.emplace_back(path.substr(begin, end - begin));
+        begin = end == std::string_view::npos ? path.size() : end + 1;
+    }
+    return components;
+}
+
+std::string append_remote_component(const std::string_view path, const std::string_view component)
+{
+    if (path.empty())
+        return std::string{component};
+    if (path == "/")
+        return fmt::format("/{}", component);
+    return fmt::format("{}/{}", path, component);
+}
+
+bool wildcard_matches(const std::string_view pattern,
+                      const std::string_view name,
+                      const QRegularExpression& expression)
+{
+    if (name.starts_with('.') && !pattern.starts_with('.'))
+        return false;
+
+    return expression.match(QString::fromStdString(std::string{name})).hasMatch();
+}
+} // namespace
+
 namespace multipass
 {
 namespace mpl = logging;
@@ -76,6 +124,84 @@ bool SFTPClient::is_remote_dir(const fs::path& path)
     return attr && attr->type == SSH_FILEXFER_TYPE_DIRECTORY;
 }
 
+std::vector<fs::path> SFTPClient::expand_remote_path(const fs::path& path)
+{
+    const auto remote_path = path.generic_string();
+    if (!has_wildcards(remote_path))
+        return {path};
+
+    auto candidates = std::vector<std::string>{remote_path.starts_with('/') ? "/" : ""};
+    auto expanded_pattern = false;
+
+    for (const auto& component : split_remote_path(remote_path))
+    {
+        if (!has_wildcards(component))
+        {
+            std::transform(candidates.begin(),
+                           candidates.end(),
+                           candidates.begin(),
+                           [&component](const auto& candidate) {
+                               return append_remote_component(candidate, component);
+                           });
+
+            if (expanded_pattern)
+            {
+                std::erase_if(candidates, [this](const auto& candidate) {
+                    return !mp_sftp_stat(sftp.get(), candidate.c_str());
+                });
+                if (candidates.empty())
+                    throw SFTPError{"no matches found: {}", path};
+            }
+            continue;
+        }
+
+        std::vector<std::string> matches;
+        const QRegularExpression expression{
+            QRegularExpression::wildcardToRegularExpression(QString::fromStdString(component))};
+        for (const auto& candidate : candidates)
+        {
+            const auto directory_path = candidate.empty() ? "." : candidate;
+            auto directory = mp_sftp_opendir(sftp.get(), directory_path.c_str());
+            if (!directory)
+            {
+                if (expanded_pattern)
+                    continue;
+                throw SFTPError{"cannot open remote directory '{}': {}",
+                                directory_path,
+                                MP_LIBSSH.ssh_get_error(sftp->session)};
+            }
+
+            while (auto entry = mp_sftp_readdir(sftp.get(), directory.get()))
+            {
+                const std::string_view name{entry->name};
+                if (name != "." && name != ".." && wildcard_matches(component, name, expression))
+                    matches.emplace_back(append_remote_component(candidate, name));
+            }
+
+            if (!MP_LIBSSH.sftp_dir_eof(directory.get()))
+                throw SFTPError{"cannot read remote directory '{}': {}",
+                                directory_path,
+                                MP_LIBSSH.ssh_get_error(sftp->session)};
+        }
+
+        if (matches.empty())
+            throw SFTPError{"no matches found: {}", path};
+
+        std::sort(matches.begin(), matches.end());
+        matches.erase(std::unique(matches.begin(), matches.end()), matches.end());
+        candidates = std::move(matches);
+        expanded_pattern = true;
+    }
+
+    std::vector<fs::path> paths;
+    paths.reserve(candidates.size());
+    std::transform(candidates.begin(),
+                   candidates.end(),
+                   std::back_inserter(paths),
+                   [](auto& candidate) { return fs::path{std::move(candidate)}; });
+    return paths;
+}
+
 bool SFTPClient::push(const fs::path& source_path, const fs::path& target_path, const Flags flags)
 try
 {

tests/cli/cli_transfer_test.py

@@ -47,6 +47,26 @@ def test_transfer_single_file(self, instance):
             assert pull_file.exists()
             assert pull_file.read_text() == "hello from the other side"
 
+    def test_transfer_remote_wildcard(self, instance):
+        """Transfer files matching a wildcard from the guest to the host."""
+        with TempDirectory() as tmp:
+            source = tmp / "wildcard-source"
+            source.mkdir()
+            (source / "first.txt").write_text("first")
+            (source / "second.txt").write_text("second")
+            (source / "ignored.log").write_text("ignored")
+
+            assert multipass("transfer", "--recursive", str(source), f"{instance}:")
+
+            target = tmp / "wildcard-target"
+            target.mkdir()
+            assert multipass(
+                "transfer", f"{instance}:wildcard-source/*.txt", str(target)
+            )
+            assert (target / "first.txt").read_text() == "first"
+            assert (target / "second.txt").read_text() == "second"
+            assert not (target / "ignored.log").exists()
+
     def test_transfer_single_file_create_parents(self, instance):
         """Transfer a single file from the host to guest where the target is a
         nested folder structure."""

tests/unit/c_mock_defines.cmake

@@ -79,6 +79,7 @@ add_c_mocks(
   sftp_stat
   sftp_lstat
   sftp_opendir
+  sftp_closedir
   sftp_readdir
   sftp_readlink
   sftp_mkdir

tests/unit/mock_sftp.cpp

@@ -29,6 +29,7 @@ IMPL_MOCK_DEFAULT(1, sftp_close);
 IMPL_MOCK_DEFAULT(2, sftp_stat);
 IMPL_MOCK_DEFAULT(2, sftp_lstat);
 IMPL_MOCK_DEFAULT(2, sftp_opendir);
+IMPL_MOCK_DEFAULT(1, sftp_closedir);
 IMPL_MOCK_DEFAULT(2, sftp_readdir);
 IMPL_MOCK_DEFAULT(2, sftp_readlink);
 IMPL_MOCK_DEFAULT(3, sftp_mkdir);

tests/unit/mock_sftp.h

@@ -32,6 +32,7 @@ DECL_MOCK(sftp_close);
 DECL_MOCK(sftp_stat);
 DECL_MOCK(sftp_lstat);
 DECL_MOCK(sftp_opendir);
+DECL_MOCK(sftp_closedir);
 DECL_MOCK(sftp_readdir);
 DECL_MOCK(sftp_readlink);
 DECL_MOCK(sftp_mkdir);

tests/unit/mock_sftp_client.h

@@ -25,6 +25,7 @@ namespace multipass::test
 struct MockSFTPClient : public SFTPClient
 {
     MOCK_METHOD(bool, is_remote_dir, (const fs::path& path), (override));
+    MOCK_METHOD(std::vector<fs::path>, expand_remote_path, (const fs::path& path), (override));
     MOCK_METHOD(bool,
                 push,
                 (const fs::path& source_path, const fs::path& target_path, Flags flags),

tests/unit/test_cli_client.cpp

@@ -62,6 +62,7 @@ namespace mp = multipass;
 namespace mcp = multipass::cli::platform;
 namespace mpt = multipass::test;
 namespace mpu = multipass::utils;
+namespace fs = mp::fs;
 using namespace testing;
 
 namespace
@@ -643,6 +644,8 @@ TEST_F(Client, transferCmdInstanceSourceLocalTarget)
 
     EXPECT_CALL(*mocked_sftp_utils, make_SFTPClient)
         .WillOnce(Return(std::move(mocked_sftp_client)));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"foo"}))
+        .WillOnce(Return(std::vector<fs::path>{"foo"}));
     EXPECT_CALL(*mocked_sftp_client_p, pull).WillOnce(Return(true));
     EXPECT_CALL(mock_daemon, ssh_info)
         .WillOnce([](auto, grpc::ServerReaderWriter<mp::SSHInfoReply, mp::SSHInfoRequest>* server) {
@@ -654,13 +657,72 @@ TEST_F(Client, transferCmdInstanceSourceLocalTarget)
     EXPECT_EQ(send_command({"transfer", "test-vm:foo", "bar"}), mp::ReturnCode::Ok);
 }
 
+TEST_F(Client, transferCmdExpandsRemoteWildcard)
+{
+    auto [mocked_file_ops, mocked_file_ops_guard] = mpt::MockFileOps::inject();
+    auto [mocked_sftp_utils, mocked_sftp_utils_guard] = mpt::MockSFTPUtils::inject();
+    auto mocked_sftp_client = std::make_unique<mpt::MockSFTPClient>();
+    auto mocked_sftp_client_p = mocked_sftp_client.get();
+
+    EXPECT_CALL(*mocked_sftp_utils, make_SFTPClient)
+        .WillOnce(Return(std::move(mocked_sftp_client)));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"dir/*.txt"}))
+        .WillOnce(Return(std::vector<fs::path>{"dir/first.txt", "dir/second.txt"}));
+    EXPECT_CALL(*mocked_file_ops, is_directory(fs::path{"target"}, _)).WillOnce(Return(true));
+    EXPECT_CALL(*mocked_sftp_client_p, pull(fs::path{"dir/first.txt"}, fs::path{"target"}, _))
+        .WillOnce(Return(true));
+    EXPECT_CALL(*mocked_sftp_client_p, pull(fs::path{"dir/second.txt"}, fs::path{"target"}, _))
+        .WillOnce(Return(true));
+    EXPECT_CALL(mock_daemon, ssh_info)
+        .WillOnce([](auto, grpc::ServerReaderWriter<mp::SSHInfoReply, mp::SSHInfoRequest>* server) {
+            mp::SSHInfoReply reply;
+            reply.mutable_ssh_info()->insert({"test-vm", mp::SSHInfo{}});
+            server->Write(reply);
+            return grpc::Status{};
+        });
+
+    EXPECT_EQ(send_command({"transfer", "test-vm:dir/*.txt", "target"}), mp::ReturnCode::Ok);
+}
+
+TEST_F(Client, transferCmdRemoteWildcardMatchesRequireDirectoryTarget)
+{
+    auto [mocked_file_ops, mocked_file_ops_guard] = mpt::MockFileOps::inject();
+    auto [mocked_sftp_utils, mocked_sftp_utils_guard] = mpt::MockSFTPUtils::inject();
+    auto mocked_sftp_client = std::make_unique<mpt::MockSFTPClient>();
+    auto mocked_sftp_client_p = mocked_sftp_client.get();
+
+    EXPECT_CALL(*mocked_sftp_utils, make_SFTPClient)
+        .WillOnce(Return(std::move(mocked_sftp_client)));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"dir/*.txt"}))
+        .WillOnce(Return(std::vector<fs::path>{"dir/first.txt", "dir/second.txt"}));
+    EXPECT_CALL(*mocked_file_ops, is_directory(fs::path{"target"}, _)).WillOnce(Return(false));
+    EXPECT_CALL(mock_daemon, ssh_info)
+        .WillOnce([](auto, grpc::ServerReaderWriter<mp::SSHInfoReply, mp::SSHInfoRequest>* server) {
+            mp::SSHInfoReply reply;
+            reply.mutable_ssh_info()->insert({"test-vm", mp::SSHInfo{}});
+            server->Write(reply);
+            return grpc::Status{};
+        });
+
+    std::stringstream err;
+    EXPECT_EQ(send_command({"transfer", "test-vm:dir/*.txt", "target"}, trash_stream, err),
+              mp::ReturnCode::CommandFail);
+    EXPECT_THAT(err.str(), HasSubstr("Target \"target\" is not a directory"));
+}
+
 TEST_F(Client, transferCmdInstanceSourcesLocalTargetNotDir)
 {
     auto [mocked_file_ops, mocked_file_ops_guard] = mpt::MockFileOps::inject();
     auto [mocked_sftp_utils, mocked_sftp_utils_guard] = mpt::MockSFTPUtils::inject();
+    auto mocked_sftp_client = std::make_unique<mpt::MockSFTPClient>();
+    auto mocked_sftp_client_p = mocked_sftp_client.get();
 
     EXPECT_CALL(*mocked_sftp_utils, make_SFTPClient)
-        .WillOnce(Return(std::make_unique<mpt::MockSFTPClient>()));
+        .WillOnce(Return(std::move(mocked_sftp_client)));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"foo"}))
+        .WillOnce(Return(std::vector<fs::path>{"foo"}));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"baz"}))
+        .WillOnce(Return(std::vector<fs::path>{"baz"}));
     EXPECT_CALL(*mocked_file_ops, is_directory).WillOnce(Return(false));
     EXPECT_CALL(mock_daemon, ssh_info)
         .WillOnce([](auto, grpc::ServerReaderWriter<mp::SSHInfoReply, mp::SSHInfoRequest>* server) {
@@ -680,9 +742,15 @@ TEST_F(Client, transferCmdInstanceSourcesLocalTargetCannotAccess)
 {
     auto [mocked_file_ops, mocked_file_ops_guard] = mpt::MockFileOps::inject();
     auto [mocked_sftp_utils, mocked_sftp_utils_guard] = mpt::MockSFTPUtils::inject();
+    auto mocked_sftp_client = std::make_unique<mpt::MockSFTPClient>();
+    auto mocked_sftp_client_p = mocked_sftp_client.get();
 
     EXPECT_CALL(*mocked_sftp_utils, make_SFTPClient)
-        .WillOnce(Return(std::make_unique<mpt::MockSFTPClient>()));
+        .WillOnce(Return(std::move(mocked_sftp_client)));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"foo"}))
+        .WillOnce(Return(std::vector<fs::path>{"foo"}));
+    EXPECT_CALL(*mocked_sftp_client_p, expand_remote_path(fs::path{"baz"}))
+        .WillOnce(Return(std::vector<fs::path>{"baz"}));
     auto err = std::make_error_code(std::errc::permission_denied);
     EXPECT_CALL(*mocked_file_ops, is_directory).WillOnce([&](auto, std::error_code& e) {
         e = err;